diff --git a/CHANGELOG.md b/CHANGELOG.md index 1608a336..0a4dafce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.112.0 — 2026-10-25 + +### New + +- Player: a Related tab lists videos related to the one playing, leaving out what you have already watched (your uploaded YouTube history included). Channels you follow are marked, and you can switch between On topic and Trending. +- Play a related video in place; Back returns to the one you opened. Admins set a daily per-user limit in Config › Quota (default 200, 0 turns it off). +- Side panel: the list controls and the watched toggle are glyphs on one line, and Auto and Loop show when they are on, also under the pointer. + +### Fixed + +- Live search: a search that never reached YouTube no longer uses up a daily slot, a busy moment says so after a few seconds instead of waiting on, and a first page that failed offers Retry. +- Menus: Play from the beginning wears the same ↺ glyph everywhere, and Reset its own eraser glyph. +- On the day of this update, API searches made earlier that day stop counting toward the daily search limit, until Pacific midnight. + ## 0.111.0 — 2026-10-24 ### New diff --git a/VERSION b/VERSION index 5af665a1..e23e3fd2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.111.0 +0.112.0 diff --git a/backend/app/catalog/queries.py b/backend/app/catalog/queries.py index 6d98ba61..63b2caee 100644 --- a/backend/app/catalog/queries.py +++ b/backend/app/catalog/queries.py @@ -158,6 +158,19 @@ def blocked_ids(db: Session, user: User) -> set[str]: ) +def followed_among(db: Session, user: User, channel_ids: set[str]) -> set[str]: + """Which of these channels the user subscribes to.""" + if not channel_ids: + return set() + return set( + db.scalars( + select(Subscription.channel_id).where( + owned_by(Subscription.user_id, user), Subscription.channel_id.in_(channel_ids) + ) + ) + ) + + def blocked_channels(db: Session, user: User, lang: str) -> list[Channel]: return list( db.execute( diff --git a/backend/app/catalog/search.py b/backend/app/catalog/search.py index 95ca4bc7..97a400f4 100644 --- a/backend/app/catalog/search.py +++ b/backend/app/catalog/search.py @@ -75,7 +75,8 @@ def ingest_candidates( ) -> list[str]: """Materialise one page of search candidates into the catalog (channel stubs + enrich, video stubs flagged via_search, enrich, Shorts classification) and return the KEEPABLE - video ids — non-Short, non-live — in the input relevance order.""" + video ids — non-Short, non-live — in the input relevance order. A videos.list that fails + raises its YouTubeError: with nothing enriched, nothing could be kept.""" ids = [it["id"] for it in candidates] if not ids: return [] @@ -132,13 +133,16 @@ def ingest_candidates( with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH): detail_by_id = {it["id"]: it for it in yt.get_videos(ids)} except YouTubeError: - # Every candidate then stays unenriched, and an unenriched video cannot be judged live or - # Short — so it is dropped. A working search returns nothing and reads as "no results". + # An unenriched video cannot be judged live or Short, so none of this page could be kept: + # swallowed, the failure answered as a page with nothing on it — "no results" for a search + # that never ran (the shared budget spent, no API key). The stubs stay (`insert_stubs` + # committed them), as the channels' do above; the caller answers the failure (its refusal + # says which). log.warning( "the videos found by a search could not be enriched", extra=logs.event("search.video_enrich_failed", candidates=len(ids)), ) - detail_by_id = {} + raise enriched_at = datetime.now( timezone.utc ) # when THIS enrich ran (distinct from the stub placeholder) @@ -212,17 +216,25 @@ def clear(db: Session, user: User, video_ids: list[str]) -> int: """Discard these results "as if never added": drop this user's finds of them, then delete the ones nobody kept — a `via_search` video no reader holds (`reader.queries.kept_by_a_reader`), whose channel nobody subscribes to, and that no other user still has as a find. Returns how - many videos went.""" - db.execute( - delete(SearchFind).where( - owned_by(SearchFind.user_id, user), SearchFind.video_id.in_(video_ids) + many videos went. + + Only the ones this user found: a video in the catalog with no find at all (a related list's, + which records none) is in someone's list right now, and an id any user could name would let + them delete it from under that list.""" + mine = list( + db.scalars( + delete(SearchFind) + .where(owned_by(SearchFind.user_id, user), SearchFind.video_id.in_(video_ids)) + .returning(SearchFind.video_id) ) ) + if not mine: + return 0 subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists() other_find = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists() result = db.execute( delete(Video).where( - Video.id.in_(video_ids), + Video.id.in_(mine), Video.via_search.is_(True), ~reader_queries.kept_by_a_reader(Video.id), ~subscribed, diff --git a/backend/app/config.py b/backend/app/config.py index b1279f0c..eb1ab096 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -113,6 +113,11 @@ class Settings(BaseSettings): # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable. search_daily_limit_per_user: int = 70 + # Per-user cap on related-video requests per day — a list opened and each "Show more" (the + # InnerTube watch page, no quota; its enrich is up to ~8 units: up to 4 pages, each a videos.list + # and a channels.list). A failed one gives its slot back. 0 turns related videos off. + # Admin-tunable. + related_daily_limit_per_user: int = 200 # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list, # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks. search_source: SearchSource = "scrape" diff --git a/backend/app/quota.py b/backend/app/quota.py index 552babb8..13ec2e66 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -44,6 +44,7 @@ class QuotaAction: VIDEOS_ENRICH = "videos_enrich" VIDEOS_LOOKUP = "videos_lookup" VIDEOS_SEARCH = "videos_search" + VIDEOS_RELATED = "videos_related" CHANNELS_DISCOVER = "channels_discover" CHANNELS_EXPLORE = "channels_explore" CHANNELS_SUBSCRIBE = "channels_subscribe" @@ -142,7 +143,7 @@ def backfill_reserve_reached() -> bool: condition for every backfill/explore path (recent, deep, immediate full-history, explore, and rolling revalidation). - Reads in its OWN short-lived session (like record_usage / log_action), for two reasons: it + Reads in its OWN short-lived session (like record_usage / claim_action), for two reasons: it replaces the remaining/reserve comparison that was open-coded at every call site, and — critically for the per-page guards in paging loops — a guard must not hold the caller's transaction open across the network fetch that follows it. Quota is committed independently of any caller, so a @@ -151,24 +152,40 @@ def backfill_reserve_reached() -> bool: return remaining_today(qs) <= sysconfig.get_int(qs, "backfill_quota_reserve") -def log_action(user_id: int, action: str) -> None: - """Record a zero-cost action event so per-user daily caps still count it. +def claim_action(user_id: int, action: str, limit: int) -> int | None: + """Take one of this user's `limit` daily `action` slots, or None when they are all taken — the + check and the take in one transaction, under a lock per user and action. Checked first and + logged after the work, N requests sent together at the last free slot all passed the check + (the work in between is seconds of scraping). Returns the event's id, for `refund_action` when + the work then produced nothing. - `record_usage` only logs when units are actually charged; a scrape-based search spends no - quota yet must still be rate-limited per user, so it logs its event here directly. - - Writes in its OWN session (like record_usage) — quota accounting is durable independent of the - caller's transaction and must never commit the caller's session (that would flush a job's - partial state mid-run, defeating its own rollback).""" + The ONE way to count a capped action: a claim is a zero-unit event, and `actions_today` counts + those alone, so a call the action then pays for (search.list's own `record_usage` event, under + the same action) is accounting, never a second slot. Own session, committed at once — quota + accounting is durable whatever the caller's transaction does next, and must never commit it.""" with SessionLocal() as qs: - qs.add(QuotaEvent(user_id=user_id, action=action, units=0)) + qs.execute(select(func.pg_advisory_xact_lock(func.hashtext(action), user_id))) + if actions_today(qs, user_id, action) >= limit: + return None + event = QuotaEvent(user_id=user_id, action=action, units=0) + qs.add(event) + qs.commit() + return event.id + + +def refund_action(event_id: int) -> None: + """Give a slot `claim_action` took back: the work it was taken for failed outright, and a + request that showed the user nothing must not count against them.""" + with SessionLocal() as qs: + qs.execute(delete(QuotaEvent).where(QuotaEvent.id == event_id)) qs.commit() def actions_today(db: Session, user_id: int, action: str) -> int: - """How many quota events of `action` this user has logged so far in the current Pacific - day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events, - not units, so it only works for actions charged exactly once per user action.""" + """How many slots of `action` this user has claimed so far in the current Pacific day — for + per-user, per-action daily caps (the live search, the related lists). Counts the zero-unit + claim events (`claim_action`) only: `record_usage`'s events for units the action spent are not + slots (it logs only when units were charged).""" return ( db.scalar( select(func.count()) @@ -176,6 +193,7 @@ def actions_today(db: Session, user_id: int, action: str) -> int: .where( owned_by(QuotaEvent.user_id, user_id), QuotaEvent.action == action, + QuotaEvent.units == 0, QuotaEvent.created_at >= pacific_day_start_utc(), ) ) diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index ff090100..ad3c5353 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -319,6 +319,36 @@ def held_by(db: Session, user: User, video_ids: list[str]) -> set[str]: ) +def seen_by(db: Session, user: User, video_ids: list[str]) -> set[str]: + """Which of these videos the user has already seen or set aside: watched or hidden here, or in + their uploaded Takeout history unless they took it back to "new" here since. The ids need not + be in the catalog — a Takeout row names a video the catalog may never have had — so a list + of what YouTube suggests can drop them before anything is ingested.""" + if not video_ids: + return set() + marked = select(VideoState.video_id).where( + owned_by(VideoState.user_id, user), + VideoState.video_id.in_(video_ids), + VideoState.status.in_(("watched", "hidden")), + ) + unmarked_since = ( + select(VideoUnmark.id) + .where( + owned_by(VideoUnmark.user_id, user), + VideoUnmark.video_id == TakeoutItem.video_id, + VideoUnmark.at > TakeoutItem.at, + ) + .exists() + ) + in_history = select(TakeoutItem.video_id).where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.video_id.in_(video_ids), + ~unmarked_since, + ) + return set(db.scalars(marked)) | set(db.scalars(in_history)) + + def feed_rows_by_ids(db: Session, user: User, video_ids: list[str]) -> list[dict]: """These videos serialized with the feed's projection (their channel, this user's watch state, saved), in the order given.""" diff --git a/backend/app/routes/me.py b/backend/app/routes/me.py index 0c5101cd..66d9cf02 100644 --- a/backend/app/routes/me.py +++ b/backend/app/routes/me.py @@ -79,6 +79,10 @@ def get_me( # Which live-search source runs, so the search button can say what a search costs: "scrape" # spends no API quota, "api" spends 100 units a page. "search_source": sysconfig.get_str(db, "search_source"), + # Whether live search and related videos are on: an admin turns either off with a daily + # limit of 0, and the UI then leaves out what would only be refused. + "search_enabled": sysconfig.get_int(db, "search_daily_limit_per_user") > 0, + "related_enabled": sysconfig.get_int(db, "related_daily_limit_per_user") > 0, "can_read": has_read_scope(user), "can_write": has_write_scope(user), "pending_invites": pending_invites, diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index 9271ea65..bc1bf444 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -13,10 +13,18 @@ this search-discovered "noise" by default. Shorts and currently-live/upcoming vi ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are dropped from the search response up front). +A video's related videos (`/related/{video_id}`, the list YouTube shows beside it) come in the same +way: read from the website (`related_scrape`), ingested as search finds are, under their own daily +cap — with what this user has already seen left out. + This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the catalog is `catalog/search`.""" -from fastapi import APIRouter, Depends +import threading +from collections.abc import Callable +from typing import Literal + +from fastapi import APIRouter, Depends, Path from pydantic import BaseModel from sqlalchemy.orm import Session @@ -26,8 +34,10 @@ from app.catalog import queries from app.catalog import search as catalog_search from app.db import get_db from app.models import User +from app.ratelimit import RateLimiter from app.reader import queries as reader_queries -from app.youtube import search_scrape +from app.youtube import related_scrape, search_scrape +from app.youtube.innertube import ScrapeError from app.youtube.client import YouTubeClient, YouTubeError from app.refusal import refused from app.routes._common import youtube_refusal @@ -37,6 +47,115 @@ router = APIRouter(prefix="/api/search", tags=["search"]) # search.list costs this many quota units per page. SEARCH_COST = 100 +# The scrape source reads at most this many pages for one search. +SEARCH_MAX_PAGES = 12 + +# A related list is filled to this many unseen videos, reading at most this many pages (about 20 +# videos each, measured) — the ones the user has seen are left out, so a page alone can fall short. +RELATED_TARGET = 20 +RELATED_MAX_PAGES = 4 +# The locale YouTube fills a related list in: on the video's topic, or with what trends in Hungary. +RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")} + +# A request that fails outright gives its slot back, but its shape is the user's (a cursor is a +# free string): one built to fail would read YouTube for nothing, without end. So only this many +# refunds an hour per user and list; past them a failure keeps its slot, as a success does. +REFUNDS_PER_HOUR = 5 +_refunds = RateLimiter(max_events=REFUNDS_PER_HOUR, window_seconds=3600) +# The API source's budget check and the search.list call it pays for run one at a time: checked +# apart, searches sent together at the last 100 units all passed, and all spent. One process +# serves the API (a single uvicorn worker, as `ratelimit` says), so a process lock holds them all. +# A search waits this long for it, then is refused as busy: each waiter holds a request worker, and +# a slow YouTube (30 s a call at worst) would otherwise park them all behind one another. +_api_search = threading.Lock() +API_SEARCH_WAIT_SECONDS = 10.0 + + +def _gather( + db: Session, + yt: YouTubeClient, + user: User, + *, + cursor: str | None, + max_pages: int, + fetch: Callable[[str | None], dict], + candidates_of: Callable[[list[dict], list[str]], list[dict]], + enough: Callable[[list[str]], bool], + scrape_failed: Callable[[Exception], None], + ingest_failed: Callable[[int, int], None], +) -> tuple[list[str], str | None]: + """Page through a YouTube list into the catalog: fetch a page, keep its `candidates_of`, ingest + them, and go on from its continuation until `enough` or `max_pages`. Returns the kept ids and + the cursor to read further from. A page that fails after earlier ones gathered something ends + the list with what it has; a first page that fails raises the 422 refusal. + + 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway dropout + (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY), which hides the real + message and repeats the work. A Data API failure is logged where it happened; the scraper's is + not, and its text does not ride the answer — so `scrape_failed` says it. An ingest (DB writes + + enrich) fails the same way, guarded for any exception so a write or shape failure is a 422 with + a message too, not a bare 500; `ingest_failed` logs it inside the `except`, with the trace.""" + collected: list[str] = [] + next_cursor = cursor + for _ in range(max_pages): + try: + page = fetch(next_cursor) + except (YouTubeError, ScrapeError) as exc: + if collected: + break # keep what earlier pages already gathered + if isinstance(exc, ScrapeError): + scrape_failed(exc) + raise youtube_refusal(exc) from exc + candidates = candidates_of(page["items"], collected) + try: + collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates)) + except Exception as exc: # any ingest failure, not just YouTubeError + db.rollback() + ingest_failed(len(candidates), len(collected)) + if collected: + break + raise youtube_refusal(exc) from exc + next_cursor = page["next_page_token"] + if enough(collected) or not next_cursor: + break + return collected, next_cursor + + +def _give_back(claimed: int, feature: str, user_id: int) -> None: + """Refund the slot of a request that failed outright, while this user has refunds left.""" + if _refunds.allow(f"{feature}:{user_id}"): + quota.refund_action(claimed) + return + log.info( + "a failed request kept its slot: this hour's refunds are spent", + extra=logs.event(f"{feature}.refund_refused", per_hour=REFUNDS_PER_HOUR), + ) + + +def _budget_spent() -> None: + """Say that a live search was refused because the shared YouTube budget cannot pay for it.""" + log.info( + "a live search was refused: the shared YouTube budget is spent", + extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST), + ) + + +def _busy() -> None: + """Say that a live search was refused because other API searches held the line too long.""" + log.info( + "a live search was refused: other searches held the line", + extra=logs.event("search.refused", reason="busy", waited=API_SEARCH_WAIT_SECONDS), + ) + + +def _cap_spent(feature: str, limit: int) -> None: + """Say that a request was refused for the per-user daily cap. The 429 is in the request line + either way; which of the caps produced it is not, and they are answered by different things — + this one by tomorrow, the shared budget by the admin.""" + log.info( + "a request was refused: the per-user daily cap is spent", + extra=logs.event(f"{feature}.refused", reason="daily_cap", limit=limit), + ) @router.get("/youtube") @@ -59,109 +178,122 @@ def search_youtube( # Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page). source = sysconfig.get_str(db, "search_source") - # Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard - # since the search itself spends no quota. + # Every request takes a slot of the per-user daily cap, a cursor's too (anti-abuse); with + # scrape it's the only guard since the search itself spends no quota. A limit of 0 turns live + # search off on this instance — a refusal tomorrow will not change. daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user") - used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) - if used >= daily_limit: - # The 429 is in the request line either way; which of the two caps produced it is not, and - # they are answered by different things — this one by tomorrow, the other by the admin. - log.info( - "a live search was refused: the per-user daily cap is spent", - extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit), - ) - raise refused(429, "search_daily_limit", limit=daily_limit) - # The 100-unit budget pre-check only applies to the API source. + if daily_limit <= 0: + raise refused(409, "search_disabled") + # The 100-unit budget pre-check only applies to the API source. It is asked again under the + # lock, right before the call (`fetch`); this one refuses before a slot is taken. if source == "api" and not quota.can_spend(db, SEARCH_COST): - log.info( - "a live search was refused: the shared YouTube budget is spent", - extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST), - ) + _budget_spent() raise refused(429, "quota_spent") + # The check and the take are one step (`quota.claim_action`), whichever the source: checked + # first and logged after, requests sent together at the last free slot all passed — and with + # the API source each of them spent 100 units. A search that fails outright gives it back below. + claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_SEARCH, daily_limit) + if claimed is None: + _cap_spent("search", daily_limit) + raise refused(429, "search_daily_limit", limit=daily_limit) # Channels this user has blocked are dropped before ingestion — never shown, never stored. blocked = queries.blocked_ids(db, user) - collected: list[str] = [] - next_cursor = cursor - MAX_PAGES = 12 # scrape safety bound - with YouTubeClient(db, user) as yt: - for _ in range(MAX_PAGES): - try: - if source == "api": - with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): - page = yt.search_videos(term, page_token=next_cursor) - else: - page = search_scrape.search(term, continuation=next_cursor) - except (YouTubeError, search_scrape.ScrapeError) as exc: - if collected: - break # keep what we already gathered - # 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway - # dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY), - # which hides the real message and burns a second quota-costed search. Upstream failure - # on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1) - # A Data API failure is logged where it happened; the scraper's is not, and its text - # no longer rides the answer — so it is said here. - if isinstance(exc, search_scrape.ScrapeError): - log.warning( - "the YouTube search scrape failed", - extra=logs.event("search.scrape_failed", error=str(exc)), - ) - raise youtube_refusal(exc) from exc + # Whether search.list was paid for: YouTube charges a call it refuses too, so once the client + # recorded its units — or the request went out and no answer came back — the search has spent + # the shared budget. One that never reached YouTube (no connection, no token) has not. + charged = False - # Drop currently-live/upcoming, channel-less, and blocked-channel results. - candidates = [ - it - for it in page["items"] - if it["live_broadcast"] == "none" - and it["channel_id"] - and it["channel_id"] not in blocked - ] - # Ingest (DB writes + enrich) runs outside the page-fetch try above; guard it too so a - # write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a - # bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1). - try: - collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates)) - except Exception as exc: # any ingest failure, not just YouTubeError - db.rollback() - # The term is NOT in this line. A search string is exactly what the privacy - # policy keeps out of the log unless the user turned Teljes/TRACE on for - # themselves, and `redact_text` does not cover it — it scrubs addresses and - # credential params, not what someone typed into the search box. What the - # ERROR line carries instead is the shape of the failure, which is what tells - # an ingest bug from an upstream one; the term follows at TRACE, and only - # under this user's own consent — not whenever the process runs at TRACE. - log.exception( - "live search ingest failed", - extra=logs.event( - "search.live.ingest_failed", - source=source, - candidates=len(candidates), - collected=len(collected), - ), - ) - if logs.consented_trace(): - log.log( - logs.TRACE, - "the term the ingest failed on", - # log-pii-ok: only under this user's own Teljes consent - extra=logs.event("search.live.term", term=term), - ) - if collected: - break # keep what earlier pages already gathered - raise youtube_refusal(exc) from exc - next_cursor = page["next_page_token"] - # The API source pages cost 100 units each — never auto-page; one page per request. - if source == "api" or len(collected) >= limit or not next_cursor: - break + def fetch(token: str | None) -> dict: + if source != "api": + return search_scrape.search(term, continuation=token) + if not _api_search.acquire(timeout=API_SEARCH_WAIT_SECONDS): + _busy() + raise refused(429, "search_busy") + try: + return paid_search(token) + finally: + _api_search.release() - # Count this as ONE per-user search against the daily cap. The scrape source spends no - # quota, so it never logs an event on its own; the API source already logged exactly one - # (via record_usage — it never auto-pages). Logging once here, after a search that produced - # at least one page (a total failure raises 502 above and never reaches this), keeps the cap - # counting user searches instead of internal continuation pages. - if source != "api": - quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH) + def paid_search(token: str | None) -> dict: + nonlocal charged + with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): + if not quota.can_spend(db, SEARCH_COST): + _budget_spent() + raise refused(429, "quota_spent") + spent = yt.spent + try: + page = yt.search_videos(term, page_token=token) + except YouTubeError as exc: + charged = yt.spent > spent or exc.sent + raise + charged = True + return page + + def candidates_of(items: list[dict], _collected: list[str]) -> list[dict]: + # Drop currently-live/upcoming, channel-less, and blocked-channel results. + return [ + it + for it in items + if it["live_broadcast"] == "none" + and it["channel_id"] + and it["channel_id"] not in blocked + ] + + def scrape_failed(exc: Exception) -> None: + log.warning( + "the YouTube search scrape failed", + extra=logs.event("search.scrape_failed", error=str(exc)), + ) + + def ingest_failed(candidates: int, collected: int) -> None: + # The term is NOT in this line. A search string is exactly what the privacy policy keeps + # out of the log unless the user turned Teljes/TRACE on for themselves, and `redact_text` + # does not cover it — it scrubs addresses and credential params, not what someone typed + # into the search box. What the ERROR line carries instead is the shape of the failure, + # which is what tells an ingest bug from an upstream one; the term follows at TRACE, and + # only under this user's own consent — not whenever the process runs at TRACE. + log.exception( + "live search ingest failed", + extra=logs.event( + "search.live.ingest_failed", + source=source, + candidates=candidates, + collected=collected, + ), + ) + if logs.consented_trace(): + log.log( + logs.TRACE, + "the term the ingest failed on", + # log-pii-ok: only under this user's own Teljes consent + extra=logs.event("search.live.term", term=term), + ) + + try: + with YouTubeClient(db, user) as yt: + collected, next_cursor = _gather( + db, + yt, + user, + cursor=cursor, + # The API source pages cost 100 units each — never auto-page; one page per request. + max_pages=1 if source == "api" else SEARCH_MAX_PAGES, + fetch=fetch, + candidates_of=candidates_of, + enough=lambda got: len(got) >= limit, + scrape_failed=scrape_failed, + ingest_failed=ingest_failed, + ) + except Exception: + # A search that showed the user nothing does not count against them, unless search.list + # was already paid for: then the slot is what stops a failing search (a bogus cursor gets + # a 400 every time) from spending the shared budget without end. A busy refusal paid + # nothing and gives it back. + if not charged: + _give_back(claimed, "search", user.id) + raise ordered = collected[:limit] @@ -201,3 +333,78 @@ def clear_search( removed = catalog_search.clear(db, user, ids) db.commit() return {"ok": True, "removed": removed} + + +@router.get("/related/{video_id}") +def related_videos( + video_id: str = Path(max_length=16), + cursor: str | None = None, + region: Literal["topical", "hu"] = "topical", + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + """The videos YouTube shows beside this one, materialised into the catalog like search results, + without the ones this user has watched or hidden here or in their Takeout history, and without + blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads + further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`).""" + # Every request takes a slot, a cursor's too: each one scrapes pages and enriches what they + # hold, and a cursor is a free string — counting only first pages let a replayed cursor read + # without end. The check and the take are one step (`quota.claim_action`); a request that fails + # outright gives its slot back, a few an hour (`_give_back`). A limit of 0 turns related videos + # off on this instance. + daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user") + if daily_limit <= 0: + raise refused(409, "related_disabled") + claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_RELATED, daily_limit) + if claimed is None: + _cap_spent("related", daily_limit) + raise refused(429, "related_daily_limit", limit=daily_limit) + + hl, gl = RELATED_REGIONS[region] + blocked = queries.blocked_ids(db, user) + + def candidates_of(items: list[dict], collected: list[str]) -> list[dict]: + ids = [it["id"] for it in items] + drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id} + return [it for it in items if it["id"] not in drop and it["channel_id"] not in blocked] + + def scrape_failed(exc: Exception) -> None: + log.warning( + "the related-videos scrape failed", + extra=logs.event("related.scrape_failed", error=str(exc)), + ) + + def ingest_failed(candidates: int, collected: int) -> None: + log.exception( + "a related list's ingest failed", + extra=logs.event("related.ingest_failed", candidates=candidates, collected=collected), + ) + + try: + with YouTubeClient(db, user) as yt: + collected, next_cursor = _gather( + db, + yt, + user, + cursor=cursor, + max_pages=RELATED_MAX_PAGES, + fetch=lambda token: related_scrape.related( + video_id, continuation=token, hl=hl, gl=gl + ), + candidates_of=candidates_of, + enough=lambda got: len(got) >= RELATED_TARGET, + scrape_failed=scrape_failed, + ingest_failed=ingest_failed, + ) + except Exception: + _give_back(claimed, "related", user.id) + raise + + rows = reader_queries.feed_rows_by_ids(db, user, collected) + # A related list mixes channels the user follows with ones they never met: each row says which. + followed = queries.followed_among(db, user, {r["channel_id"] for r in rows}) + return { + "items": [{**r, "followed": r["channel_id"] in followed} for r in rows], + "next_cursor": next_cursor, + "has_more": bool(next_cursor), + } diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 6ad61490..d5c1adb5 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -140,6 +140,15 @@ SPECS: tuple[ConfigSpec, ...] = ( max=100_000, applies=LIVE, ), + ConfigSpec( + "related_daily_limit_per_user", + "int", + "quota", + "related_daily_limit_per_user", + min=0, + max=100_000, + applies=LIVE, + ), # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE), ConfigSpec( diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index b7c549e0..8cb188ec 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -30,12 +30,26 @@ class YouTubeError(Exception): message is prose, it changes under us, and it is deliberately the poorer of the two here. `path` is the API endpoint that ANSWERED with the failure (`playlistItems`); empty when no - endpoint did — the request never got through (`transport`) or the token step refused first.""" + endpoint did — the request never got through (`transport`) or the token step refused first. - def __init__(self, message: str, *, reason: str = "", path: str = ""): + `sent` is a `transport` failure that came after the request may have gone out (a read timeout, + a dropped answer): nothing answered, but YouTube may have counted the call.""" + + def __init__(self, message: str, *, reason: str = "", path: str = "", sent: bool = False): super().__init__(message) self.reason = reason self.path = path + self.sent = sent + + +# The transport failures that stop a request before it reaches YouTube: nothing went out. +_UNSENT = ( + httpx.ConnectError, + httpx.ConnectTimeout, + httpx.PoolTimeout, + httpx.ProxyError, + httpx.UnsupportedProtocol, +) def _error_reason(resp: httpx.Response) -> str: @@ -162,6 +176,9 @@ def best_thumbnail(thumbnails: dict | None) -> str | None: class YouTubeClient: + # The quota units this client has recorded: what its calls have spent of the shared budget. + spent: int = 0 + def __init__(self, db, user: User): self.db = db self.user = user @@ -189,9 +206,16 @@ class YouTubeClient: return self._http.request(method, url, **kwargs) except httpx.HTTPError as exc: raise YouTubeError( - f"YouTube API unreachable ({exc.__class__.__name__}: {exc})", reason="transport" + f"YouTube API unreachable ({exc.__class__.__name__}: {exc})", + reason="transport", + sent=not isinstance(exc, _UNSENT), ) from exc + def _spend(self, units: int) -> None: + """Record what a call YouTube answered cost — a refused one too: it is charged all the same.""" + quota.record_usage(units) + self.spent += units + # --- auth --- def _access_token(self) -> str: tok = self.user.token @@ -269,7 +293,7 @@ class YouTubeClient: else: headers["Authorization"] = f"Bearer {self._access_token()}" resp = self._send("GET", f"{API_BASE}/{path}", params=p, headers=headers) - quota.record_usage(cost) + self._spend(cost) if resp.status_code != 200: raise _upstream_failure("GET", path, resp, expected) return resp.json() @@ -401,7 +425,7 @@ class YouTubeClient: params={"id": subscription_id}, headers={"Authorization": f"Bearer {self._access_token()}"}, ) - quota.record_usage(50) + self._spend(50) if resp.status_code not in (200, 204): raise _upstream_failure("DELETE", "subscriptions", resp) @@ -431,7 +455,7 @@ class YouTubeClient: def _write(self, method: str, path: str, *, params: dict, json: dict | None = None) -> dict: headers = {"Authorization": f"Bearer {self._access_token()}"} resp = self._send(method, f"{API_BASE}/{path}", params=params, json=json, headers=headers) - quota.record_usage(50) + self._spend(50) if resp.status_code not in (200, 204): raise _upstream_failure(method, path, resp) return resp.json() if resp.content else {} diff --git a/backend/app/youtube/innertube.py b/backend/app/youtube/innertube.py new file mode 100644 index 00000000..ec6316d8 --- /dev/null +++ b/backend/app/youtube/innertube.py @@ -0,0 +1,160 @@ +"""The website's own InnerTube API (`youtubei/v1/*`), read the way the website reads it — no quota. + +The zero-quota search (`search_scrape`) and the related videos (`related_scrape`) both POST here with +the public WEB key and a client version, read out of a live results page. This deliberately calls an +undocumented endpoint: if YouTube changes the JSON shape or blocks the host, each caller says so as a +`ScrapeError`, never as an empty answer. + +The SOCS=CAI consent-skip cookie is sent like the Shorts probe's, so a server-side request isn't +bounced to consent.youtube.com.""" + +import re + +import httpx + +from app import logs + +log = logs.get_logger(__name__) + +_API_URL = "https://www.youtube.com/youtubei/v1/" +_RESULTS_URL = "https://www.youtube.com/results" +# Public WEB InnerTube key + a recent client version. These are also embedded in every results +# page; we fetch the live pair once and cache it, falling back to these constants. +_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" +_FALLBACK_CLIENT_VERSION = "2.20240101.00.00" + +_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is + # redirected to consent.youtube.com). + "Cookie": "SOCS=CAI", + "Accept-Language": "en-US,en;q=0.9", +} + +_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') +_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') + +# Cached (api_key, client_version) extracted from a results page; refreshed on demand. +_config: tuple[str, str] | None = None + + +class ScrapeError(Exception): + """An InnerTube read could not be completed (network, blocked, or unexpected shape).""" + + +def runs_text(node: dict | None) -> str | None: + """The text of an InnerTube text node: a `simpleText`, or its `runs` joined.""" + if not node: + return None + if node.get("simpleText"): + return node["simpleText"] + runs = node.get("runs") or [] + if runs: + return "".join(r.get("text", "") for r in runs) + return None + + +def scraped_video( + vid: str, channel_id: object, channel_title: object, title: object, thumbnails: object +) -> dict: + """One scraped video in the shape `YouTubeClient.search_videos` answers, so a page of either + source is ingested the same way. `published_at` is None (the website shows only a relative + "3 years ago"; the enrich fills the real date) and `live_broadcast` "none" (live/upcoming are + judged after the enrich, by `live_status`). The largest thumbnail is the last.""" + thumbs = thumbnails if isinstance(thumbnails, list) else [] + last = thumbs[-1] if thumbs else None + return { + "id": vid, + "channel_id": channel_id if isinstance(channel_id, str) and channel_id else None, + "channel_title": channel_title if isinstance(channel_title, str) else None, + "title": title if isinstance(title, str) else None, + "published_at": None, + "thumbnail_url": last.get("url") if isinstance(last, dict) else None, + "live_broadcast": "none", + } + + +def video_renderer(node: dict) -> dict | None: + """A `videoRenderer` (search) or `compactVideoRenderer` (the older related column) flattened — + the two draw a video the same way. None without a video id; the channel is None when the byline + names none.""" + vid = node.get("videoId") + if not isinstance(vid, str) or not vid: + return None + owner = node.get("ownerText") or node.get("longBylineText") or {} + runs = owner.get("runs") if isinstance(owner, dict) else None + first = runs[0] if isinstance(runs, list) and runs and isinstance(runs[0], dict) else {} + endpoint = (first.get("navigationEndpoint") or {}).get("browseEndpoint") or {} + return scraped_video( + vid, + endpoint.get("browseId"), + first.get("text"), + runs_text(node.get("title")), + (node.get("thumbnail") or {}).get("thumbnails"), + ) + + +def continuation_token(node: object) -> str | None: + """The "load more" cursor of a `continuationItemRenderer`, or None when `node` is not one.""" + if not isinstance(node, dict): + return None + command = (node.get("continuationEndpoint") or {}).get("continuationCommand") or {} + token = command.get("token") + return token if isinstance(token, str) and token else None + + +def _get_config(client: httpx.Client) -> tuple[str, str]: + """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them. + + The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no + keys on it) at the wrong moment pinned every later search to a hard-coded key and version for + the life of the process — and when those went stale, search returned nothing, with no error + anywhere. Now the next call simply tries the live pair again.""" + global _config + if _config is not None: + return _config + try: + html = client.get(_RESULTS_URL, params={"search_query": "music"}).text + except httpx.HTTPError as exc: + log.warning( + "the InnerTube config could not be fetched; falling back to the pinned key", + extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__), + ) + return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) + key = _KEY_RE.search(html) + ver = _VER_RE.search(html) + if key and ver: + _config = (key.group(1), ver.group(1)) + return _config + log.warning( + "the results page carried no InnerTube config; falling back to the pinned key", + extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"), + ) + return ( + key.group(1) if key else _FALLBACK_KEY, + ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, + ) + + +def post(endpoint: str, body: dict, *, hl: str = "en", gl: str = "US") -> dict: + """POST `body` to `youtubei/v1/` as the WEB client in the `hl`/`gl` locale and return + the decoded JSON object. A network failure, an HTTP error or a body that is not a JSON object (a + block, consent or captcha page) raises `ScrapeError`; the caller judges the shape.""" + with httpx.Client(timeout=20.0, headers=_HEADERS) as client: + api_key, client_version = _get_config(client) + context = { + "client": {"clientName": "WEB", "clientVersion": client_version, "hl": hl, "gl": gl} + } + try: + resp = client.post( + _API_URL + endpoint, params={"key": api_key}, json={"context": context, **body} + ) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as exc: + raise ScrapeError(f"InnerTube request failed: {exc}") from exc + except ValueError as exc: # non-JSON body (blocked / consent / captcha) + raise ScrapeError("InnerTube returned a non-JSON response") from exc + if not isinstance(data, dict): + raise ScrapeError("InnerTube returned an unexpected shape") + return data diff --git a/backend/app/youtube/related_scrape.py b/backend/app/youtube/related_scrape.py new file mode 100644 index 00000000..33f4124e --- /dev/null +++ b/backend/app/youtube/related_scrape.py @@ -0,0 +1,141 @@ +"""A video's related videos, read from the watch page's InnerTube `next` endpoint — no quota. + +The Data API has no related-videos call any more, so this is the only source: the same list the +website shows beside a playing video. A page holds about 20 videos and a continuation token for the +next 20. The returned page has the search scrape's shape (``{"items": [...], "next_page_token"}``) +so the catalog ingests it the same way (`catalog.search.ingest_candidates`). + +YouTube draws each item as a `lockupViewModel` now; the older `compactVideoRenderer` is read too, so +a rollback on its side does not empty the list. An item that names no single channel — a video by +several channels draws an avatar stack, not one avatar — is left out: the catalog row needs its +channel up front, and the stack does not say which one uploaded it. + +`hl`/`gl` pick the locale YouTube fills the list in: ``en``/``US`` keeps it on the video's topic, +``hu``/``HU`` mixes in what is trending in Hungary (measured).""" + +from app.youtube.innertube import ( + ScrapeError, + continuation_token, + post, + scraped_video, + video_renderer, +) + +__all__ = ["ScrapeError", "related"] + +# Where a first page and a continuation page put their item list. +_FIRST = ( + "contents", + "twoColumnWatchNextResults", + "secondaryResults", + "secondaryResults", + "results", +) +_APPENDS = ("appendContinuationItemsAction", "reloadContinuationItemsCommand") + + +def _at(node: object, *path: str | int) -> object: + """The value under `path` in nested dicts/lists, or None where the path stops.""" + for key in path: + if isinstance(node, dict) and isinstance(key, str): + node = node.get(key) + elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node): + node = node[key] + else: + return None + return node + + +def _lockup(node: dict) -> dict | None: + if node.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO" or not node.get("contentId"): + return None + meta = _at(node, "metadata", "lockupMetadataViewModel") + channel_id = _at( + meta, + "image", + "decoratedAvatarViewModel", + "rendererContext", + "commandContext", + "onTap", + "innertubeCommand", + "browseEndpoint", + "browseId", + ) + channel_title = _at( + meta, "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0 + ) + return scraped_video( + node["contentId"], + channel_id, + _at(channel_title, "text", "content"), + _at(meta, "title", "content"), + _at(node, "contentImage", "thumbnailViewModel", "image", "sources"), + ) + + +def _unavailable(data: dict) -> bool: + """Whether the watch page says the video is not available: its main column draws a + `backgroundPromoRenderer` ("This video isn't available anymore") and there is no related column. + YouTube's own word for "nothing related" — the one empty page that is not a shape we lost.""" + sections = _at(data, "contents", "twoColumnWatchNextResults", "results", "results", "contents") + for section in sections if isinstance(sections, list) else []: + inner = _at(section, "itemSectionRenderer", "contents") + for node in inner if isinstance(inner, list) else []: + if isinstance(node, dict) and "backgroundPromoRenderer" in node: + return True + return False + + +def _items(data: dict) -> list | None: + """The page's item list (first page or continuation), or None when it has neither frame.""" + first = _at(data, *_FIRST) + if isinstance(first, list): + return first + if _unavailable(data): + return [] + for endpoint in data.get("onResponseReceivedEndpoints") or []: + for key in _APPENDS: + items = _at(endpoint, key, "continuationItems") + if isinstance(items, list): + return items + return None + + +def related( + video_id: str, continuation: str | None = None, *, hl: str = "en", gl: str = "US" +) -> dict: + """One page of `video_id`'s related videos (`continuation` = the token a previous page gave). + + A video YouTube says it no longer has is an empty page. An answer of a shape this does not know + — no related column, or one whose entries are all renderers it cannot read — is a `ScrapeError`: + not an empty page, which would read as "nothing related".""" + body = {"continuation": continuation} if continuation else {"videoId": video_id} + data = post("next", body, hl=hl, gl=gl) + items = _items(data) + if items is None: + raise ScrapeError("InnerTube returned no related list") + out: list[dict] = [] + seen: set[str] = set() + token = None + known = 0 # entries drawn by a renderer read here (a video's, or the cursor) + for entry in items: + if not isinstance(entry, dict): + continue + if "lockupViewModel" in entry: + item = _lockup(entry["lockupViewModel"]) + elif "compactVideoRenderer" in entry: + item = video_renderer(entry["compactVideoRenderer"]) + else: + found = continuation_token(entry.get("continuationItemRenderer")) + known += found is not None + token = token or found + continue + known += 1 + # A video by several channels draws an avatar stack, not one: no channel, left out. + if item and item["channel_id"] and item["id"] not in seen: + seen.add(item["id"]) + out.append(item) + # A list with entries but none of them one we read: YouTube draws its items some new way. + if items and not known: + raise ScrapeError("InnerTube's related list holds no renderer this reads") + return {"items": out, "next_page_token": token} diff --git a/backend/app/youtube/search_scrape.py b/backend/app/youtube/search_scrape.py index 95dc1b89..22adeac0 100644 --- a/backend/app/youtube/search_scrape.py +++ b/backend/app/youtube/search_scrape.py @@ -6,9 +6,9 @@ calls — and read the result list straight out of the JSON. The returned video through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does, so the search itself is effectively free and only the (cheap, shared) enrichment is charged. -The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't -bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube -changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``. +The request itself (the live key, the consent cookie) is `innertube`'s. This deliberately scrapes an +undocumented endpoint: if YouTube changes the JSON shape or blocks the host, an admin can flip +``search_source`` back to ``api``. The returned page mirrors :meth:`YouTubeClient.search_videos` exactly (``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the @@ -17,110 +17,16 @@ only difference is the cursor is an InnerTube *continuation token*, not a Data A the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"`` and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path).""" -import re +from app.youtube.innertube import ScrapeError, continuation_token, post, video_renderer -import httpx +__all__ = ["ScrapeError", "search"] -from app import logs - -log = logs.get_logger(__name__) - -_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search" -_RESULTS_URL = "https://www.youtube.com/results" -# Public WEB InnerTube key + a recent client version. These are also embedded in every results -# page; we fetch the live pair once and cache it, falling back to these constants. -_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" -_FALLBACK_CLIENT_VERSION = "2.20240101.00.00" # sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==). _TYPE_VIDEO = "EgIQAQ==" -_HEADERS = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", - # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is - # redirected to consent.youtube.com). - "Cookie": "SOCS=CAI", - "Accept-Language": "en-US,en;q=0.9", -} - # The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands) # put their result list under. _RESULT_FRAMES = {"contents", "onResponseReceivedCommands"} -_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') -_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') - -# Cached (api_key, client_version) extracted from a results page; refreshed on demand. -_config: tuple[str, str] | None = None - - -class ScrapeError(Exception): - """A scrape-based search could not be completed (network, blocked, or unexpected shape).""" - - -def _runs_text(node: dict | None) -> str | None: - if not node: - return None - if node.get("simpleText"): - return node["simpleText"] - runs = node.get("runs") or [] - if runs: - return "".join(r.get("text", "") for r in runs) - return None - - -def _get_config(client: httpx.Client) -> tuple[str, str]: - """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them. - - The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no - keys on it) at the wrong moment pinned every later search to a hard-coded key and version for - the life of the process — and when those went stale, search returned nothing, with no error - anywhere. Now the next search simply tries the live pair again.""" - global _config - if _config is not None: - return _config - try: - html = client.get(_RESULTS_URL, params={"search_query": "music"}).text - except httpx.HTTPError as exc: - log.warning( - "the InnerTube config could not be fetched; falling back to the pinned key", - extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__), - ) - return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) - key = _KEY_RE.search(html) - ver = _VER_RE.search(html) - if key and ver: - _config = (key.group(1), ver.group(1)) - return _config - log.warning( - "the results page carried no InnerTube config; falling back to the pinned key", - extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"), - ) - return ( - key.group(1) if key else _FALLBACK_KEY, - ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, - ) - - -def _video_item(vr: dict) -> dict | None: - """Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos.""" - vid = vr.get("videoId") - if not vid: - return None - owner = vr.get("ownerText") or vr.get("longBylineText") or {} - runs = owner.get("runs") or [] - channel_id = channel_title = None - if runs: - channel_title = runs[0].get("text") - channel_id = runs[0].get("navigationEndpoint", {}).get("browseEndpoint", {}).get("browseId") - thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or [] - return { - "id": vid, - "channel_id": channel_id, - "channel_title": channel_title, - "title": _runs_text(vr.get("title")), - "published_at": None, # scrape exposes only a relative time; enrich fills the real date - "thumbnail_url": thumbs[-1]["url"] if thumbs else None, - "live_broadcast": "none", # live/upcoming filtered post-enrich via live_status - } def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: @@ -130,7 +36,7 @@ def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: if isinstance(obj, dict): vr = obj.get("videoRenderer") if isinstance(vr, dict): - item = _video_item(vr) + item = video_renderer(vr) if item and item["id"] not in seen: seen.add(item["id"]) out.append(item) @@ -144,11 +50,9 @@ def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: def _find_continuation(obj) -> str | None: """First continuationItemRenderer token (the 'load more' cursor), or None at the end.""" if isinstance(obj, dict): - cir = obj.get("continuationItemRenderer") - if isinstance(cir, dict): - tok = cir.get("continuationEndpoint", {}).get("continuationCommand", {}).get("token") - if tok: - return tok + tok = continuation_token(obj.get("continuationItemRenderer")) + if tok: + return tok for v in obj.values(): tok = _find_continuation(v) if tok: @@ -167,34 +71,14 @@ def search(q: str, continuation: str | None = None) -> dict: `continuation` is the InnerTube continuation token from a previous page (None for the first page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as YouTubeClient.search_videos so the route handles both sources identically.""" - with httpx.Client(timeout=20.0, headers=_HEADERS) as client: - api_key, client_version = _get_config(client) - context = { - "client": { - "clientName": "WEB", - "clientVersion": client_version, - "hl": "en", - "gl": "US", - } - } - body: dict = {"context": context} - if continuation: - body["continuation"] = continuation - else: - body["query"] = q - body["params"] = _TYPE_VIDEO - try: - resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body) - resp.raise_for_status() - data = resp.json() - except httpx.HTTPError as exc: - raise ScrapeError(f"InnerTube request failed: {exc}") from exc - except ValueError as exc: # non-JSON body (blocked / consent / captcha) - raise ScrapeError("InnerTube returned a non-JSON response") from exc + body: dict = ( + {"continuation": continuation} if continuation else {"query": q, "params": _TYPE_VIDEO} + ) + data = post("search", body) # Even a search with no results carries its result list's frame. Without one this is not "no # results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so. - if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys(): + if not _RESULT_FRAMES & data.keys(): raise ScrapeError("InnerTube returned an unexpected shape") items: list[dict] = [] _collect_videos(data, items, set()) diff --git a/backend/tests/fixtures/innertube_next_continuation.json b/backend/tests/fixtures/innertube_next_continuation.json new file mode 100644 index 00000000..cd78025d --- /dev/null +++ b/backend/tests/fixtures/innertube_next_continuation.json @@ -0,0 +1,527 @@ +{ + "onResponseReceivedEndpoints": [ + { + "appendContinuationItemsAction": { + "continuationItems": [ + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/_HfdncCbMOE/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLARvB39pkdsiuUUQTsZjkhvbKrfcQ", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/_HfdncCbMOE/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLB15aZU5-tzwYCEbNTnZK3UX6ZCzQ", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "1:08:18", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "_HfdncCbMOE", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2025): Large Language Models (Liquid AI)" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "62K" + }, + "accessibilityLabel": "62 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "1y ago" + }, + "accessibilityLabel": "1 year ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "_HfdncCbMOE", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4294964720, + "raisedBackground": 4294964720, + "additiveBackground": 566850118, + "textPrimary": 4281800969, + "textSecondary": 4285876787, + "outline": 860759827, + "keyLight": 4292983720, + "collectionStack": 4291337092, + "invertedBackground": 4287578649, + "textPrimaryInverse": 4294964720, + "textDisabled": 2135828243, + "dropShadow": 432632390, + "cardOutline": 180974150, + "overlayBackground": 3426162953, + "overlayTextPrimary": 4294964720, + "solidWash": 4283311891, + "solidRim": 4281800969, + "overlaySolidWash": 4294699488, + "overlaySolidRim": 4294964720, + "tonalWash": 2583688688, + "tonalRim": 202310912, + "overlayTonalWash": 436205040, + "overlayTonalRim": 436205040, + "outlineRim": 1280190227, + "overlayOutlineRim": 871618504, + "solidWashInverse": 4294170568, + "solidRimInverse": 2147215840, + "tonalWashInverse": 654308848, + "tonalRimInverse": 654043616, + "outlineRimInverse": 654308848, + "solidBackground": 4283311891, + "solidBackgroundInverse": 4291337092, + "tonalBackground": 868785028, + "tonalBackgroundInverse": 435410888, + "outlineInverse": 872147424, + "directionalLightStopPrimary": 4294699488, + "directionalLightStopSecondary": 4294964720, + "primary": 4279174400, + "filledStateLayer": 4281800969, + "tonalTouchLight": 4294964720, + "tonalHoverLight": 3439061472, + "tonalStateLayer": 420414720, + "overlayDisabledContainer": 871618504, + "overlayTonalTouchLight": 1291577824, + "overlayTonalHoverLight": 1291577824, + "overlayTonalStateLayer": 436205040, + "disabledContainer": 420414720, + "filledTouchLight": 4285876787, + "filledHoverLight": 4284627235, + "onOverlayDisabledContainer": 2583688688, + "overlayTonalContainer": 2583423456, + "overlayOutline": 872412656, + "onFilledContainer": 4294964720, + "filledContainer": 4280027906, + "tonalContainer": 641145097, + "overlayPrimary": 4294964720, + "onOverlayFilledContainer": 4279174400, + "overlayFilledTouchLight": 4294964720, + "overlayFilledHoverLight": 4294699488, + "overlayFilledStateLayer": 4292983720, + "overlayFilledContainer": 4294170568, + "onDisabledContainer": 1712260352, + "temporaryOutline": 860759827 + }, + "darkThemePalette": { + "baseBackground": 4280027906, + "raisedBackground": 4281800969, + "additiveBackground": 734622278, + "textPrimary": 4294699488, + "textSecondary": 4292983720, + "outline": 1072945096, + "keyLight": 4284627235, + "collectionStack": 4285876787, + "invertedBackground": 4293107837, + "textPrimaryInverse": 4280027906, + "textDisabled": 2146686920, + "dropShadow": 432632390, + "cardOutline": 180974150, + "overlayBackground": 3426162953, + "overlayTextPrimary": 4294964720, + "solidWash": 4294699488, + "solidRim": 4294964720, + "overlaySolidWash": 4294699488, + "overlaySolidRim": 4294964720, + "tonalWash": 217307080, + "tonalRim": 218101232, + "overlayTonalWash": 436205040, + "overlayTonalRim": 436205040, + "outlineRim": 871618504, + "overlayOutlineRim": 871618504, + "solidWashInverse": 4285876787, + "solidRimInverse": 871618504, + "tonalWashInverse": 872412656, + "tonalRimInverse": 870431656, + "outlineRimInverse": 1291843056, + "solidBackground": 4291337092, + "solidBackgroundInverse": 4283311891, + "tonalBackground": 435410888, + "tonalBackgroundInverse": 868785028, + "outlineInverse": 860759827, + "directionalLightStopPrimary": 4281800969, + "directionalLightStopSecondary": 4280027906, + "primary": 4294964720, + "filledStateLayer": 4292983720, + "tonalTouchLight": 1073474016, + "tonalHoverLight": 654043616, + "tonalStateLayer": 436205040, + "overlayDisabledContainer": 871618504, + "overlayTonalTouchLight": 1291577824, + "overlayTonalHoverLight": 1291577824, + "overlayTonalStateLayer": 436205040, + "disabledContainer": 436205040, + "filledTouchLight": 4294964720, + "filledHoverLight": 4294699488, + "onOverlayDisabledContainer": 2583688688, + "overlayTonalContainer": 2583423456, + "overlayOutline": 872412656, + "onFilledContainer": 4279174400, + "filledContainer": 4294170568, + "tonalContainer": 435410888, + "overlayPrimary": 4294964720, + "onOverlayFilledContainer": 4279174400, + "overlayFilledTouchLight": 4294964720, + "overlayFilledHoverLight": 4294699488, + "overlayFilledStateLayer": 4292983720, + "overlayFilledContainer": 4294170568, + "onDisabledContainer": 1728050672, + "temporaryOutline": 871618504 + } + } + } + }, + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/P7Hkh2zOGQ0/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLAHInNtMierSAgtrGl5rLSKc-vdzQ", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/P7Hkh2zOGQ0/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDXrm5g-rdjRIqqK309ImTdqpw4CA", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "1:08:19", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "P7Hkh2zOGQ0", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2024): Google - Generative AI for Media" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "49K" + }, + "accessibilityLabel": "49 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "2y ago" + }, + "accessibilityLabel": "2 years ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "P7Hkh2zOGQ0", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4293981951, + "raisedBackground": 4293981951, + "additiveBackground": 558201801, + "textPrimary": 4278655801, + "textSecondary": 4280831104, + "outline": 856631890, + "keyLight": 4288855783, + "collectionStack": 4286094803, + "invertedBackground": 4279847311, + "textPrimaryInverse": 4293981951, + "textDisabled": 2131700306, + "dropShadow": 423984073, + "cardOutline": 172325833, + "overlayBackground": 3423017785, + "overlayTextPrimary": 4293981951, + "solidWash": 4279183954, + "solidRim": 4278655801, + "overlaySolidWash": 4292864764, + "overlaySolidRim": 4293981951, + "tonalWash": 2582705919, + "tonalRim": 201328143, + "overlayTonalWash": 435222271, + "overlayTonalRim": 435222271, + "outlineRim": 1276062290, + "overlayOutlineRim": 868669942, + "solidWashInverse": 4291222006, + "solidRimInverse": 2145381116, + "tonalWashInverse": 653326079, + "tonalRimInverse": 652208892, + "outlineRimInverse": 653326079, + "solidBackground": 4279183954, + "solidBackgroundInverse": 4286094803, + "tonalBackground": 863542739, + "tonalBackgroundInverse": 432462326, + "outlineInverse": 870312700, + "directionalLightStopPrimary": 4292864764, + "directionalLightStopSecondary": 4293981951, + "primary": 4278191631, + "filledStateLayer": 4278655801, + "tonalTouchLight": 4293981951, + "tonalHoverLight": 3437226748, + "tonalStateLayer": 419431951, + "overlayDisabledContainer": 868669942, + "overlayTonalTouchLight": 1289743100, + "overlayTonalHoverLight": 1289743100, + "overlayTonalStateLayer": 435222271, + "disabledContainer": 419431951, + "filledTouchLight": 4280831104, + "filledHoverLight": 4280040297, + "onOverlayDisabledContainer": 2582705919, + "overlayTonalContainer": 2581588732, + "overlayOutline": 871429887, + "onFilledContainer": 4293981951, + "filledContainer": 4278324509, + "tonalContainer": 637999929, + "overlayPrimary": 4293981951, + "onOverlayFilledContainer": 4278191631, + "overlayFilledTouchLight": 4293981951, + "overlayFilledHoverLight": 4292864764, + "overlayFilledStateLayer": 4288855783, + "overlayFilledContainer": 4291222006, + "onDisabledContainer": 1711277583, + "temporaryOutline": 856631890 + }, + "darkThemePalette": { + "baseBackground": 4278324509, + "raisedBackground": 4278655801, + "additiveBackground": 725973961, + "textPrimary": 4292864764, + "textSecondary": 4288855783, + "outline": 1069996534, + "keyLight": 4280040297, + "collectionStack": 4280831104, + "invertedBackground": 4286359267, + "textPrimaryInverse": 4278324509, + "textDisabled": 2143738358, + "dropShadow": 423984073, + "cardOutline": 172325833, + "overlayBackground": 3423017785, + "overlayTextPrimary": 4293981951, + "solidWash": 4292864764, + "solidRim": 4293981951, + "overlaySolidWash": 4292864764, + "overlaySolidRim": 4293981951, + "tonalWash": 214358518, + "tonalRim": 217118463, + "overlayTonalWash": 435222271, + "overlayTonalRim": 435222271, + "outlineRim": 868669942, + "overlayOutlineRim": 868669942, + "solidWashInverse": 4280831104, + "solidRimInverse": 868669942, + "tonalWashInverse": 871429887, + "tonalRimInverse": 866303719, + "outlineRimInverse": 1290860287, + "solidBackground": 4286094803, + "solidBackgroundInverse": 4279183954, + "tonalBackground": 432462326, + "tonalBackgroundInverse": 863542739, + "outlineInverse": 856631890, + "directionalLightStopPrimary": 4278655801, + "directionalLightStopSecondary": 4278324509, + "primary": 4293981951, + "filledStateLayer": 4288855783, + "tonalTouchLight": 1071639292, + "tonalHoverLight": 652208892, + "tonalStateLayer": 435222271, + "overlayDisabledContainer": 868669942, + "overlayTonalTouchLight": 1289743100, + "overlayTonalHoverLight": 1289743100, + "overlayTonalStateLayer": 435222271, + "disabledContainer": 435222271, + "filledTouchLight": 4293981951, + "filledHoverLight": 4292864764, + "onOverlayDisabledContainer": 2582705919, + "overlayTonalContainer": 2581588732, + "overlayOutline": 871429887, + "onFilledContainer": 4278191631, + "filledContainer": 4291222006, + "tonalContainer": 432462326, + "overlayPrimary": 4293981951, + "onOverlayFilledContainer": 4278191631, + "overlayFilledTouchLight": 4293981951, + "overlayFilledHoverLight": 4292864764, + "overlayFilledStateLayer": 4288855783, + "overlayFilledContainer": 4291222006, + "onDisabledContainer": 1727067903, + "temporaryOutline": 868669942 + } + } + } + }, + { + "continuationItemRenderer": { + "trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN", + "continuationEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "sendPost": true, + "apiUrl": "/youtubei/v1/next" + } + }, + "continuationCommand": { + "token": "CCgSDRILU2RUWkFNREtyTlkYACqPDE1nb0lodVQ4eG9EdXVLSUYyczZMNnd6NENBcjFDQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhQ2dQeVBnQUtEc0ktQ3dqaDRleUUzTFAzdV93QkNnUHlQZ0FLRGNJLUNnaU5zcmptOXBENTJEOEtBX0ktQUFvTndqNEtDTExudjR5ZXg3U1dTUW9EOGo0QUNnM0NQZ29JNnZHODhaR1JpTWNnQ2dQeVBnQUtEc0ktQ3dpRF85S3Z5WS1hbThJQkNnUHlQZ0FLRHNJLUN3alppNmJYbWR6YWxZb0JDZ1B5UGdBS0RzSS1Dd2lkemJfS2dPdWN0YUFCQ2dQeVBnQUtEY0ktQ2dqMXktMjYyb2JQeFdNS0FfSS1BQW9Pd2o0TENPLUE4SWVaMUlESXJRRUtBX0ktQUFvTndqNEtDUHlZNDVhUzhkeWpJZ29EOGo0QUNnM0NQZ29Jb191aWtfSE05ZHdTQ2dQeVBnQUtEc0ktQ3dqaTBiemlrdnpBbkxvQkNnUHlQZ0FLRHNJLUN3aW05cXI1eXNtTWlzOEJDZ1B5UGdBS0RjSS1DZ2k3M2V5NXlzQ1pzMG9LQV9JLUFBb093ajRMQ0xmT281WFU1X2JPbFFFS0FfSS1BQW9Od2o0S0NLTEVwYXlXOTliWkdnb0Q4ajRBQ2c3Q1Bnc0k2dWZSMnRPM3hLem9BUW9EOGo0QUNnN0NQZ3NJN2ZlZXZQRHV2WlRXQVFvRDhqNEFDZzNDUGdvSXFibS0zYXVBdDVWcUNnUHlQZ0FLRGNJLUNnaUc1UHpHZ082NG9nVVNLQUFDQkFZSUNnd09FQklVRmhnYUhCNGdJaVFtS0Nvc0xqQXlORFk0T2p3LVFFSkVSa2hLVEU0YUJBZ0FFQUVhQkFnQ0VBTWFCQWdFRUFVYUJBZ0dFQWNhQkFnSUVBa2FCQWdLRUFzYUJBZ01FQTBhQkFnT0VBOGFCQWdRRUJFYUJBZ1NFQk1hQkFnVUVCVWFCQWdXRUJjYUJBZ1lFQmthQkFnYUVCc2FCQWdjRUIwYUJBZ2VFQjhhQkFnZ0VDRWFCQWdpRUNNYUJBZ2tFQ1VhQkFnbUVDY2FCQWdvRUNrYUJBZ3FFQ3NhQkFnc0VDMGFCQWd1RUM4YUJBZ3dFREVhQkFneUVETWFCQWcwRURVYUJBZzJFRGNhQkFnNEVEa2FCQWc2RURzYUJBZzhFRDBhQkFnLUVEOGFCQWhBRUVFYUJBaENFRU1hQkFoRUVFVWFCQWhHRUVjYUJBaElFRWthQkFoS0VFc2FCQWhNRUUwYUJBaE9FRThxS0FBQ0JBWUlDZ3dPRUJJVUZoZ2FIQjRnSWlRbUtDb3NMakF5TkRZNE9qdy1RRUpFUmtoS1RFNGoPd2F0Y2gtbmV4dC1mZWVk", + "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" + } + } + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/backend/tests/fixtures/innertube_next_first.json b/backend/tests/fixtures/innertube_next_first.json new file mode 100644 index 00000000..dd31a462 --- /dev/null +++ b/backend/tests/fixtures/innertube_next_first.json @@ -0,0 +1,805 @@ +{ + "contents": { + "twoColumnWatchNextResults": { + "secondaryResults": { + "secondaryResults": { + "results": [ + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/to-lHJfK4pw/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLBKP_oFChvoqvwVrKAy_PG9bn2_AA", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/to-lHJfK4pw/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDyH3Z8DJhXMmFJ5eKEWMufk7ql9g", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "1:02:00", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "to-lHJfK4pw", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2025): Reinforcement Learning" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "128K" + }, + "accessibilityLabel": "128 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "1y ago" + }, + "accessibilityLabel": "1 year ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "to-lHJfK4pw", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4293981439, + "raisedBackground": 4293981439, + "additiveBackground": 558196425, + "textPrimary": 4278850870, + "textSecondary": 4282009454, + "outline": 857088843, + "keyLight": 4289509597, + "collectionStack": 4287338944, + "invertedBackground": 4279842703, + "textPrimaryInverse": 4293981439, + "textDisabled": 2132157259, + "dropShadow": 423978697, + "cardOutline": 172320457, + "overlayBackground": 3423212854, + "overlayTextPrimary": 4293981439, + "solidWash": 4279640907, + "solidRim": 4278850870, + "overlaySolidWash": 4292929530, + "overlaySolidRim": 4293981439, + "tonalWash": 2582705407, + "tonalRim": 201327631, + "overlayTonalWash": 435221759, + "overlayTonalRim": 435221759, + "outlineRim": 1276519243, + "overlayOutlineRim": 868930802, + "solidWashInverse": 4291482866, + "solidRimInverse": 2145445882, + "tonalWashInverse": 653325567, + "tonalRimInverse": 652273658, + "outlineRimInverse": 653325567, + "solidBackground": 4279640907, + "solidBackgroundInverse": 4287338944, + "tonalBackground": 864786880, + "tonalBackgroundInverse": 432723186, + "outlineInverse": 870377466, + "directionalLightStopPrimary": 4292929530, + "directionalLightStopSecondary": 4293981439, + "primary": 4278191119, + "filledStateLayer": 4278850870, + "tonalTouchLight": 4293981439, + "tonalHoverLight": 3437291514, + "tonalStateLayer": 419431439, + "overlayDisabledContainer": 868930802, + "overlayTonalTouchLight": 1289807866, + "overlayTonalHoverLight": 1289807866, + "overlayTonalStateLayer": 435221759, + "disabledContainer": 419431439, + "filledTouchLight": 4282009454, + "filledHoverLight": 4280825181, + "onOverlayDisabledContainer": 2582705407, + "overlayTonalContainer": 2581653498, + "overlayOutline": 871429375, + "onFilledContainer": 4293981439, + "filledContainer": 4278323484, + "tonalContainer": 638194998, + "overlayPrimary": 4293981439, + "onOverlayFilledContainer": 4278191119, + "overlayFilledTouchLight": 4293981439, + "overlayFilledHoverLight": 4292929530, + "overlayFilledStateLayer": 4289509597, + "overlayFilledContainer": 4291482866, + "onDisabledContainer": 1711277071, + "temporaryOutline": 857088843 + }, + "darkThemePalette": { + "baseBackground": 4278323484, + "raisedBackground": 4278850870, + "additiveBackground": 725968585, + "textPrimary": 4292929530, + "textSecondary": 4289509597, + "outline": 1070257394, + "keyLight": 4280825181, + "collectionStack": 4282009454, + "invertedBackground": 4286355427, + "textPrimaryInverse": 4278323484, + "textDisabled": 2143999218, + "dropShadow": 423978697, + "cardOutline": 172320457, + "overlayBackground": 3423212854, + "overlayTextPrimary": 4293981439, + "solidWash": 4292929530, + "solidRim": 4293981439, + "overlaySolidWash": 4292929530, + "overlaySolidRim": 4293981439, + "tonalWash": 214619378, + "tonalRim": 217117951, + "overlayTonalWash": 435221759, + "overlayTonalRim": 435221759, + "outlineRim": 868930802, + "overlayOutlineRim": 868930802, + "solidWashInverse": 4282009454, + "solidRimInverse": 868930802, + "tonalWashInverse": 871429375, + "tonalRimInverse": 866957533, + "outlineRimInverse": 1290859775, + "solidBackground": 4287338944, + "solidBackgroundInverse": 4279640907, + "tonalBackground": 432723186, + "tonalBackgroundInverse": 864786880, + "outlineInverse": 857088843, + "directionalLightStopPrimary": 4278850870, + "directionalLightStopSecondary": 4278323484, + "primary": 4293981439, + "filledStateLayer": 4289509597, + "tonalTouchLight": 1071704058, + "tonalHoverLight": 652273658, + "tonalStateLayer": 435221759, + "overlayDisabledContainer": 868930802, + "overlayTonalTouchLight": 1289807866, + "overlayTonalHoverLight": 1289807866, + "overlayTonalStateLayer": 435221759, + "disabledContainer": 435221759, + "filledTouchLight": 4293981439, + "filledHoverLight": 4292929530, + "onOverlayDisabledContainer": 2582705407, + "overlayTonalContainer": 2581653498, + "overlayOutline": 871429375, + "onFilledContainer": 4278191119, + "filledContainer": 4291482866, + "tonalContainer": 432723186, + "overlayPrimary": 4293981439, + "onOverlayFilledContainer": 4278191119, + "overlayFilledTouchLight": 4293981439, + "overlayFilledHoverLight": 4292929530, + "overlayFilledStateLayer": 4289509597, + "overlayFilledContainer": 4291482866, + "onDisabledContainer": 1727067391, + "temporaryOutline": 868930802 + } + } + } + }, + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/R8V8CbuxryI/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLDqXMjrpgDyQKHL2lWrxczwxKoefw", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/R8V8CbuxryI/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLB7JX9Z238s6DGqIS5-fhcEIkUxKw", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "49:36", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "R8V8CbuxryI", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191: Deep Generative Modeling" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "38K" + }, + "accessibilityLabel": "38 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "6mo ago" + }, + "accessibilityLabel": "6 months ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "R8V8CbuxryI", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4294965488, + "raisedBackground": 4294965488, + "additiveBackground": 566856261, + "textPrimary": 4281737226, + "textSecondary": 4285420858, + "outline": 860565782, + "keyLight": 4292724140, + "collectionStack": 4290815883, + "invertedBackground": 4287583769, + "textPrimaryInverse": 4294965488, + "textDisabled": 2135634198, + "dropShadow": 432638533, + "cardOutline": 180980293, + "overlayBackground": 3426099210, + "overlayTextPrimary": 4294965488, + "solidWash": 4283117846, + "solidRim": 4281737226, + "overlaySolidWash": 4294635232, + "overlaySolidRim": 4294965488, + "tonalWash": 2583689456, + "tonalRim": 202311680, + "overlayTonalWash": 436205808, + "overlayTonalRim": 436205808, + "outlineRim": 1279996182, + "overlayOutlineRim": 871554762, + "solidWashInverse": 4294106826, + "solidRimInverse": 2147151584, + "tonalWashInverse": 654309616, + "tonalRimInverse": 653979360, + "outlineRimInverse": 654309616, + "solidBackground": 4283117846, + "solidBackgroundInverse": 4290815883, + "tonalBackground": 868263819, + "tonalBackgroundInverse": 435347146, + "outlineInverse": 872083168, + "directionalLightStopPrimary": 4294635232, + "directionalLightStopSecondary": 4294965488, + "primary": 4279175168, + "filledStateLayer": 4281737226, + "tonalTouchLight": 4294965488, + "tonalHoverLight": 3438997216, + "tonalStateLayer": 420415488, + "overlayDisabledContainer": 871554762, + "overlayTonalTouchLight": 1291513568, + "overlayTonalHoverLight": 1291513568, + "overlayTonalStateLayer": 436205808, + "disabledContainer": 420415488, + "filledTouchLight": 4285420858, + "filledHoverLight": 4284302120, + "onOverlayDisabledContainer": 2583689456, + "overlayTonalContainer": 2583359200, + "overlayOutline": 872413424, + "onFilledContainer": 4294965488, + "filledContainer": 4280029186, + "tonalContainer": 641081354, + "overlayPrimary": 4294965488, + "onOverlayFilledContainer": 4279175168, + "overlayFilledTouchLight": 4294965488, + "overlayFilledHoverLight": 4294635232, + "overlayFilledStateLayer": 4292724140, + "overlayFilledContainer": 4294106826, + "onDisabledContainer": 1712261120, + "temporaryOutline": 860565782 + }, + "darkThemePalette": { + "baseBackground": 4280029186, + "raisedBackground": 4281737226, + "additiveBackground": 734628421, + "textPrimary": 4294635232, + "textSecondary": 4292724140, + "outline": 1072881354, + "keyLight": 4284302120, + "collectionStack": 4285420858, + "invertedBackground": 4293112700, + "textPrimaryInverse": 4280029186, + "textDisabled": 2146623178, + "dropShadow": 432638533, + "cardOutline": 180980293, + "overlayBackground": 3426099210, + "overlayTextPrimary": 4294965488, + "solidWash": 4294635232, + "solidRim": 4294965488, + "overlaySolidWash": 4294635232, + "overlaySolidRim": 4294965488, + "tonalWash": 217243338, + "tonalRim": 218102000, + "overlayTonalWash": 436205808, + "overlayTonalRim": 436205808, + "outlineRim": 871554762, + "overlayOutlineRim": 871554762, + "solidWashInverse": 4285420858, + "solidRimInverse": 871554762, + "tonalWashInverse": 872413424, + "tonalRimInverse": 870172076, + "outlineRimInverse": 1291843824, + "solidBackground": 4290815883, + "solidBackgroundInverse": 4283117846, + "tonalBackground": 435347146, + "tonalBackgroundInverse": 868263819, + "outlineInverse": 860565782, + "directionalLightStopPrimary": 4281737226, + "directionalLightStopSecondary": 4280029186, + "primary": 4294965488, + "filledStateLayer": 4292724140, + "tonalTouchLight": 1073409760, + "tonalHoverLight": 653979360, + "tonalStateLayer": 436205808, + "overlayDisabledContainer": 871554762, + "overlayTonalTouchLight": 1291513568, + "overlayTonalHoverLight": 1291513568, + "overlayTonalStateLayer": 436205808, + "disabledContainer": 436205808, + "filledTouchLight": 4294965488, + "filledHoverLight": 4294635232, + "onOverlayDisabledContainer": 2583689456, + "overlayTonalContainer": 2583359200, + "overlayOutline": 872413424, + "onFilledContainer": 4279175168, + "filledContainer": 4294106826, + "tonalContainer": 435347146, + "overlayPrimary": 4294965488, + "onOverlayFilledContainer": 4279175168, + "overlayFilledTouchLight": 4294965488, + "overlayFilledHoverLight": 4294635232, + "overlayFilledStateLayer": 4292724140, + "overlayFilledContainer": 4294106826, + "onDisabledContainer": 1728051440, + "temporaryOutline": 871554762 + } + } + } + }, + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/HLKo4fJx_7k/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLAQcvMQEX1ny9Z8qu2mIMogpEMaVA", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/HLKo4fJx_7k/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLCkoPSkHifs5ifpWXXcb96XG_Ly1w", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "47:36", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "HLKo4fJx_7k", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2025): Language Models and New Frontiers" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "63K" + }, + "accessibilityLabel": "63 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "1y ago" + }, + "accessibilityLabel": "1 year ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "HLKo4fJx_7k", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4294964208, + "raisedBackground": 4294964208, + "additiveBackground": 566847045, + "textPrimary": 4281734410, + "textSecondary": 4285417018, + "outline": 860561942, + "keyLight": 4292720812, + "collectionStack": 4290812043, + "invertedBackground": 4287575065, + "textPrimaryInverse": 4294964208, + "textDisabled": 2135630358, + "dropShadow": 432629317, + "cardOutline": 180971077, + "overlayBackground": 3426096394, + "overlayTextPrimary": 4294964208, + "solidWash": 4283114006, + "solidRim": 4281734410, + "overlaySolidWash": 4294633184, + "overlaySolidRim": 4294964208, + "tonalWash": 2583688176, + "tonalRim": 202310656, + "overlayTonalWash": 436204528, + "overlayTonalRim": 436204528, + "outlineRim": 1279992342, + "overlayOutlineRim": 871551946, + "solidWashInverse": 4294104010, + "solidRimInverse": 2147149536, + "tonalWashInverse": 654308336, + "tonalRimInverse": 653977312, + "outlineRimInverse": 654308336, + "solidBackground": 4283114006, + "solidBackgroundInverse": 4290812043, + "tonalBackground": 868259979, + "tonalBackgroundInverse": 435344330, + "outlineInverse": 872081120, + "directionalLightStopPrimary": 4294633184, + "directionalLightStopSecondary": 4294964208, + "primary": 4279174144, + "filledStateLayer": 4281734410, + "tonalTouchLight": 4294964208, + "tonalHoverLight": 3438995168, + "tonalStateLayer": 420414464, + "overlayDisabledContainer": 871551946, + "overlayTonalTouchLight": 1291511520, + "overlayTonalHoverLight": 1291511520, + "overlayTonalStateLayer": 436204528, + "disabledContainer": 420414464, + "filledTouchLight": 4285417018, + "filledHoverLight": 4284298280, + "onOverlayDisabledContainer": 2583688176, + "overlayTonalContainer": 2583357152, + "overlayOutline": 872412144, + "onFilledContainer": 4294964208, + "filledContainer": 4280027394, + "tonalContainer": 641078538, + "overlayPrimary": 4294964208, + "onOverlayFilledContainer": 4279174144, + "overlayFilledTouchLight": 4294964208, + "overlayFilledHoverLight": 4294633184, + "overlayFilledStateLayer": 4292720812, + "overlayFilledContainer": 4294104010, + "onDisabledContainer": 1712260096, + "temporaryOutline": 860561942 + }, + "darkThemePalette": { + "baseBackground": 4280027394, + "raisedBackground": 4281734410, + "additiveBackground": 734619205, + "textPrimary": 4294633184, + "textSecondary": 4292720812, + "outline": 1072878538, + "keyLight": 4284298280, + "collectionStack": 4285417018, + "invertedBackground": 4293105020, + "textPrimaryInverse": 4280027394, + "textDisabled": 2146620362, + "dropShadow": 432629317, + "cardOutline": 180971077, + "overlayBackground": 3426096394, + "overlayTextPrimary": 4294964208, + "solidWash": 4294633184, + "solidRim": 4294964208, + "overlaySolidWash": 4294633184, + "overlaySolidRim": 4294964208, + "tonalWash": 217240522, + "tonalRim": 218100720, + "overlayTonalWash": 436204528, + "overlayTonalRim": 436204528, + "outlineRim": 871551946, + "overlayOutlineRim": 871551946, + "solidWashInverse": 4285417018, + "solidRimInverse": 871551946, + "tonalWashInverse": 872412144, + "tonalRimInverse": 870168748, + "outlineRimInverse": 1291842544, + "solidBackground": 4290812043, + "solidBackgroundInverse": 4283114006, + "tonalBackground": 435344330, + "tonalBackgroundInverse": 868259979, + "outlineInverse": 860561942, + "directionalLightStopPrimary": 4281734410, + "directionalLightStopSecondary": 4280027394, + "primary": 4294964208, + "filledStateLayer": 4292720812, + "tonalTouchLight": 1073407712, + "tonalHoverLight": 653977312, + "tonalStateLayer": 436204528, + "overlayDisabledContainer": 871551946, + "overlayTonalTouchLight": 1291511520, + "overlayTonalHoverLight": 1291511520, + "overlayTonalStateLayer": 436204528, + "disabledContainer": 436204528, + "filledTouchLight": 4294964208, + "filledHoverLight": 4294633184, + "onOverlayDisabledContainer": 2583688176, + "overlayTonalContainer": 2583357152, + "overlayOutline": 872412144, + "onFilledContainer": 4279174144, + "filledContainer": 4294104010, + "tonalContainer": 435344330, + "overlayPrimary": 4294964208, + "onOverlayFilledContainer": 4279174144, + "overlayFilledTouchLight": 4294964208, + "overlayFilledHoverLight": 4294633184, + "overlayFilledStateLayer": 4292720812, + "overlayFilledContainer": 4294104010, + "onDisabledContainer": 1728050160, + "temporaryOutline": 871551946 + } + } + } + }, + { + "continuationItemRenderer": { + "trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN", + "continuationEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "sendPost": true, + "apiUrl": "/youtubei/v1/next" + } + }, + "continuationCommand": { + "token": "CBQSDRILU2RUWkFNREtyTlkYACqXBk1nb0l5ZGl1dXRUNDdQc2EyczZMNnd5LUJBcTdCQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D", + "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" + } + }, + "button": { + "buttonRenderer": { + "style": "STYLE_SUGGESTIVE", + "size": "SIZE_DEFAULT", + "isDisabled": false, + "text": { + "runs": [ + { + "text": "Show more" + } + ] + }, + "command": { + "commandMetadata": { + "webCommandMetadata": { + "sendPost": true, + "apiUrl": "/youtubei/v1/next" + } + }, + "continuationCommand": { + "token": "CBQSDRILU2RUWkFNREtyTlkYACqXBk1nb0l5ZGl1dXRUNDdQc2EyczZMNnd5LUJBcTdCQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D", + "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" + } + } + } + } + } + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/backend/tests/test_discover.py b/backend/tests/test_discover.py index 31566b26..37c9e5d4 100644 --- a/backend/tests/test_discover.py +++ b/backend/tests/test_discover.py @@ -1105,10 +1105,16 @@ def test_two_overlapping_reorders_never_share_a_slot(db, as_user): worker = threading.Thread(target=second_tab) worker.start() - # Commit A only once B is blocked on a row lock, so B's read is the stale snapshot. + # Commit A only once B is blocked on a row lock, so B's read is the stale snapshot. pg_locks is + # the whole server's: only a wait in this database is B's (a parallel worker's is not). waiting = 0 for _ in range(200): - waiting = probe.scalar(text("SELECT count(*) FROM pg_locks WHERE NOT granted")) + waiting = probe.scalar( + text( + "SELECT count(*) FROM pg_locks WHERE NOT granted AND pid IN" + " (SELECT pid FROM pg_stat_activity WHERE datname = current_database())" + ) + ) probe.rollback() if waiting: break diff --git a/backend/tests/test_me_search_source.py b/backend/tests/test_me_search_source.py index 7f5bb2f3..b3c110a4 100644 --- a/backend/tests/test_me_search_source.py +++ b/backend/tests/test_me_search_source.py @@ -1,5 +1,6 @@ """`/api/me` names the live-search source, so the search button's tooltip can say what a search costs -under the source that actually runs (the default website source spends no API quota).""" +under the source that actually runs (the default website source spends no API quota), and says +whether live search and related videos are on at all.""" from app import sysconfig @@ -24,3 +25,24 @@ def test_me_names_the_running_search_source(client, db, make_user): sysconfig.set_value(db, "search_source", "api") db.commit() assert _me(client)["search_source"] == "api" + + +def test_me_says_whether_live_search_and_related_videos_are_on(client, db, make_user): + # A daily limit of 0 turns the feature off; the UI leaves it out instead of offering a refusal. + user = make_user(password=_PW) + login = client.post("/auth/password-login", json={"email": user.email, "password": _PW}) + assert login.status_code == 200 + + me = _me(client) + assert (me["search_enabled"], me["related_enabled"]) == (True, True) + + sysconfig.set_value(db, "search_daily_limit_per_user", 0) + db.commit() + me = _me(client) + assert (me["search_enabled"], me["related_enabled"]) == (False, True) + + sysconfig.set_value(db, "search_daily_limit_per_user", 5) + sysconfig.set_value(db, "related_daily_limit_per_user", 0) + db.commit() + me = _me(client) + assert (me["search_enabled"], me["related_enabled"]) == (True, False) diff --git a/backend/tests/test_owner_scoping.py b/backend/tests/test_owner_scoping.py index 8ab99547..c85a69de 100644 --- a/backend/tests/test_owner_scoping.py +++ b/backend/tests/test_owner_scoping.py @@ -75,9 +75,9 @@ def test_trimming_read_notifications_trims_only_mine(db, make_user): def test_todays_actions_count_only_mine(db, make_user): me, other = make_user(), make_user() - quota.log_action(me.id, quota.QuotaAction.VIDEOS_SEARCH) - quota.log_action(other.id, quota.QuotaAction.VIDEOS_SEARCH) - quota.log_action(other.id, quota.QuotaAction.VIDEOS_SEARCH) + quota.claim_action(me.id, quota.QuotaAction.VIDEOS_SEARCH, 5) + quota.claim_action(other.id, quota.QuotaAction.VIDEOS_SEARCH, 5) + quota.claim_action(other.id, quota.QuotaAction.VIDEOS_SEARCH, 5) assert quota.actions_today(db, me.id, quota.QuotaAction.VIDEOS_SEARCH) == 1 diff --git a/backend/tests/test_playlist_sync_claim.py b/backend/tests/test_playlist_sync_claim.py index fb133640..afce741e 100644 --- a/backend/tests/test_playlist_sync_claim.py +++ b/backend/tests/test_playlist_sync_claim.py @@ -89,9 +89,13 @@ def test_a_claim_is_released_when_its_block_raises(db, make_user): except RuntimeError: pass # Asked of pg_locks, not by claiming again: a lock leaked on a pooled connection would be granted - # again to that same connection (advisory locks are re-entrant per session). + # again to that same connection (advisory locks are re-entrant per session). pg_locks is the whole + # server's, so only this database's: a parallel worker's own claim on the same id counted here. held = db.scalar( - text("SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND objid = :id"), + text( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND objid = :id" + " AND database = (SELECT oid FROM pg_database WHERE datname = current_database())" + ), {"id": pl.id}, ) assert held == 0 diff --git a/backend/tests/test_quota_ownership.py b/backend/tests/test_quota_ownership.py index a951ef98..36a269ee 100644 --- a/backend/tests/test_quota_ownership.py +++ b/backend/tests/test_quota_ownership.py @@ -44,15 +44,15 @@ def test_reads_see_separate_session_spend_mid_transaction(db): assert after - before == 7 -def test_log_action_survives_caller_rollback(db): +def test_a_claimed_slot_survives_caller_rollback(db): user = User(email="searcher@example.com") db.add(user) db.commit() # a committed user to attribute the (free) action to db.add(User(email="pending@example.com")) # uncommitted caller work - quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH) + assert quota.claim_action(user.id, quota.QuotaAction.VIDEOS_SEARCH, 5) is not None db.rollback() - # The zero-cost action event survived (per-user daily caps must still count it) ... + # The zero-cost claim event survived (per-user daily caps must still count it) ... events = db.execute(select(QuotaEvent)).scalars().all() assert len(events) == 1 assert events[0].user_id == user.id diff --git a/backend/tests/test_related_route.py b/backend/tests/test_related_route.py new file mode 100644 index 00000000..3b279b7a --- /dev/null +++ b/backend/tests/test_related_route.py @@ -0,0 +1,428 @@ +"""`GET /api/search/related/{video_id}` — a video's related videos, without what the user has seen. + +The scrape is replaced by pages the test hands out, and YouTube's videos.list by an offline enrich +(30-minute videos, so no Shorts probe goes out). What is held: the list leaves out what this user +watched or hid — here or in their Takeout history, unless taken back to "new" since — blocked +channels and the video itself; it pages on until it has enough; it lands in the catalog the way +search finds do; and a failure, or the day's cap, says so instead of answering an empty list.""" + +import threading +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import func, select, text + +from app import quota, sysconfig +from app.catalog.models import BlockedChannel, Channel, Subscription, Video +from app.models import QuotaEvent +from app.reader.models import TakeoutItem, TakeoutKind, VideoState, VideoUnmark +from app.routes import search as search_routes +from app.youtube.client import YouTubeError +from app.youtube.innertube import ScrapeError + +T0 = datetime(2026, 10, 1, tzinfo=timezone.utc) +SOURCE = "srcvideo001" + + +def _vid(n: int) -> str: + return f"rel{n:08d}" + + +def _item(n: int, channel: str = "UCrelated01") -> dict: + return { + "id": _vid(n), + "channel_id": channel, + "channel_title": f"Chan {channel}", + "title": f"Related {n}", + "published_at": None, + "thumbnail_url": None, + "live_broadcast": "none", + } + + +class _FakeYT: + def __init__(self, *_a, **_kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_channels(self, ids): + return [] + + def get_videos(self, ids): + return [ + { + "id": i, + "snippet": {"title": f"T {i}", "publishedAt": "2025-01-01T00:00:00Z"}, + "contentDetails": {"duration": "PT30M"}, + "statistics": {}, + "status": {}, + } + for i in ids + ] + + +@pytest.fixture +def pages(monkeypatch, db): + """The scrape answers with the pages in `state["pages"]`, one per call, recording each call.""" + sysconfig.set_value(db, "shorts_probe_max_seconds", 60) + state: dict = {"pages": [], "calls": []} + + def related(video_id, continuation=None, *, hl="en", gl="US"): + state["calls"].append((video_id, continuation, hl, gl)) + page = state["pages"].pop(0) + if isinstance(page, Exception): + raise page + return page + + monkeypatch.setattr(search_routes.related_scrape, "related", related) + monkeypatch.setattr(search_routes, "YouTubeClient", _FakeYT) + return state + + +def _page(items: list[dict], token: str | None = "NEXT") -> dict: + return {"items": items, "next_page_token": token} + + +def _ids(resp) -> list[str]: + assert resp.status_code == 200, resp.text + return [it["id"] for it in resp.json()["items"]] + + +def _known(db, *ns: int, channel: str = "UCrelated01") -> None: + """These related videos already in the catalog (a state needs the row).""" + if db.get(Channel, channel) is None: + db.add(Channel(id=channel, title="Chan")) + for n in ns: + db.add(Video(id=_vid(n), channel_id=channel, title=f"Related {n}", published_at=T0)) + db.commit() + + +def test_a_related_list_comes_into_the_catalog_as_search_finds_do(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(n) for n in range(1, 21)])] + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert _ids(resp) == [_vid(n) for n in range(1, 21)] + assert resp.json()["next_cursor"] == "NEXT" and resp.json()["has_more"] is True + video = db.get(Video, _vid(1)) + # Flagged like a search find: hidden from the shared Library, reclaimed when nobody keeps it. + assert video.via_search is True and video.enriched_at is not None + assert pages["calls"] == [(SOURCE, None, "en", "US")] + + +def test_what_the_user_watched_or_hid_here_is_left_out(db, make_user, as_user, pages): + user = make_user() + other = make_user() + _known(db, 1, 2, 3, 4) + db.add_all( + [ + VideoState(user_id=user.id, video_id=_vid(1), status="watched", watched_at=T0), + VideoState(user_id=user.id, video_id=_vid(2), status="hidden"), + # In progress is not "seen": it stays, to be finished. + VideoState(user_id=user.id, video_id=_vid(3), status="new", position_seconds=120), + # Someone else's watch is theirs. + VideoState(user_id=other.id, video_id=_vid(4), status="watched", watched_at=T0), + ] + ) + db.commit() + pages["pages"] = [_page([_item(n) for n in range(1, 6)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(3), _vid(4), _vid(5)] + + +def test_what_the_takeout_history_shows_watched_is_left_out_unless_taken_back( + db, make_user, as_user, pages +): + user = make_user() + other = make_user() + _known(db, 2) + db.add_all( + [ + # Not in the catalog at all: the history alone names it. + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0), + # Watched on YouTube, then taken back to "new" here: it may come again. + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(2), at=T0), + VideoUnmark(user_id=user.id, video_id=_vid(2), at=T0 + timedelta(days=1)), + # Only in Watch later: not watched. + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCH_LATER, video_id=_vid(3), at=T0), + # Another user's history. + TakeoutItem(user_id=other.id, kind=TakeoutKind.WATCHED, video_id=_vid(4), at=T0), + ] + ) + db.commit() + pages["pages"] = [_page([_item(n) for n in range(1, 5)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2), _vid(3), _vid(4)] + # Left out before the ingest: what the user has seen is not stored for a list that hides it. + assert db.get(Video, _vid(1)) is None + + +def test_an_unmark_older_than_the_history_row_does_not_bring_it_back(db, make_user, as_user, pages): + user = make_user() + _known(db, 1) + db.add_all( + [ + VideoUnmark(user_id=user.id, video_id=_vid(1), at=T0 - timedelta(days=1)), + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0), + ] + ) + db.commit() + pages["pages"] = [_page([_item(1), _item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)] + + +def test_another_users_unmark_does_not_bring_it_back(db, make_user, as_user, pages): + user = make_user() + other = make_user() + _known(db, 1) + db.add_all( + [ + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0), + VideoUnmark(user_id=other.id, video_id=_vid(1), at=T0 + timedelta(days=1)), + ] + ) + db.commit() + pages["pages"] = [_page([_item(1), _item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)] + + +def test_blocked_channels_and_the_video_itself_are_left_out_and_never_stored( + db, make_user, as_user, pages +): + user = make_user() + db.add(Channel(id="UCblocked01", title="Blocked")) + db.commit() + db.add(BlockedChannel(user_id=user.id, channel_id="UCblocked01")) + db.commit() + own = {**_item(9), "id": SOURCE} + pages["pages"] = [_page([_item(1), _item(2, "UCblocked01"), own], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + assert db.get(Video, _vid(2)) is None + + +def test_it_pages_on_until_enough_unseen_videos_are_gathered(db, make_user, as_user, pages): + user = make_user() + _known(db, *range(1, 16)) + for n in range(1, 16): + db.add(VideoState(user_id=user.id, video_id=_vid(n), status="watched", watched_at=T0)) + db.commit() + pages["pages"] = [ + _page([_item(n) for n in range(1, 21)], token="page2"), # 15 seen, 5 new + _page([_item(n) for n in range(21, 41)], token="page3"), # 20 new: 25 now + _page([_item(n) for n in range(41, 61)], token="page4"), # never asked + ] + ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}")) + assert ids == [_vid(n) for n in range(16, 41)] + assert [c[1] for c in pages["calls"]] == [None, "page2"] + assert resp.json()["next_cursor"] == "page3" + + +def test_a_repeat_on_a_later_page_counts_once(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [ + _page([_item(n) for n in range(1, 11)], token="page2"), + _page([_item(n) for n in range(5, 25)], token=None), + ] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [ + _vid(n) for n in range(1, 25) + ] + + +def test_it_reads_at_most_four_pages(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(n)], token=f"page{n + 1}") for n in range(1, 7)] + ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}")) + assert ids == [_vid(n) for n in range(1, 5)] + assert len(pages["calls"]) == 4 and resp.json()["next_cursor"] == "page5" + + +def test_the_cursor_and_the_hungarian_locale_reach_the_scrape(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(1)], token=None)] + resp = as_user(user).get(f"/api/search/related/{SOURCE}?cursor=C1®ion=hu") + assert _ids(resp) == [_vid(1)] + assert pages["calls"] == [(SOURCE, "C1", "hu", "HU")] + assert resp.json()["has_more"] is False + + +def test_an_unknown_region_is_refused(db, make_user, as_user, pages): + user = make_user() + assert as_user(user).get(f"/api/search/related/{SOURCE}?region=xx").status_code == 422 + assert pages["calls"] == [] + + +def test_a_failed_scrape_says_youtube_is_unavailable(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [ScrapeError("blocked")] + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"} + + +def test_a_failure_on_a_later_page_keeps_what_came_before(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(1), _item(2)], token="page2"), ScrapeError("blocked")] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1), _vid(2)] + + +def test_one_list_is_one_event_against_the_days_cap(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 1) + pages["pages"] = [ + _page([_item(1)], token="page2"), + _page([_item(2)], token=None), + ] + client = as_user(user) + assert client.get(f"/api/search/related/{SOURCE}").status_code == 200 + events = db.scalars( + select(QuotaEvent).where( + QuotaEvent.user_id == user.id, QuotaEvent.action == quota.QuotaAction.VIDEOS_RELATED + ) + ).all() + assert len(events) == 1 + refused = client.get(f"/api/search/related/{SOURCE}") + assert refused.status_code == 429 + assert refused.json()["detail"] == { + "code": "refused", + "reason": "related_daily_limit", + "limit": 1, + } + assert len(pages["calls"]) == 2 # the refused one asked YouTube nothing + + +def test_reading_further_takes_a_slot_too(db, make_user, as_user, pages): + # Each request scrapes and enriches; a cursor is a free string, so one that cost nothing let a + # replayed cursor read without end. + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 1) + pages["pages"] = [_page([_item(1)], token=None), _page([_item(2)], token=None)] + client = as_user(user) + assert client.get(f"/api/search/related/{SOURCE}").status_code == 200 + refused = client.get(f"/api/search/related/{SOURCE}?cursor=more") + assert refused.status_code == 429 + assert refused.json()["detail"]["reason"] == "related_daily_limit" + assert len(pages["calls"]) == 1 + + +def test_a_list_that_failed_gives_its_slot_back(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 1) + pages["pages"] = [ScrapeError("blocked"), _page([_item(1)], token=None)] + client = as_user(user) + assert client.get(f"/api/search/related/{SOURCE}").status_code == 422 + assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 0 + assert _ids(client.get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + + +def test_failures_past_the_hours_refunds_keep_their_slot(db, make_user, as_user, pages): + # A cursor is a free string: a request built to fail would otherwise read YouTube without end. + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 50) + n = search_routes.REFUNDS_PER_HOUR + pages["pages"] = [ScrapeError("bad cursor") for _ in range(n + 1)] + client = as_user(user) + for _ in range(n + 1): + assert client.get(f"/api/search/related/{SOURCE}?cursor=bogus").status_code == 422 + assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 1 + + +def test_a_zero_limit_turns_related_videos_off(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 0) + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert resp.status_code == 409 + assert resp.json()["detail"] == {"code": "refused", "reason": "related_disabled"} + assert pages["calls"] == [] + + +def test_the_last_slot_is_taken_under_a_lock(db, make_user): + # Checked first and logged after seconds of scraping, requests sent together at the last free + # slot all passed. The claim waits for one already taking a slot for this user and action. + user = make_user() + action = quota.QuotaAction.VIDEOS_RELATED + db.execute(select(func.pg_advisory_xact_lock(func.hashtext(action), user.id))) + got: list[int | None] = [] + t = threading.Thread(target=lambda: got.append(quota.claim_action(user.id, action, 1))) + t.start() + t.join(timeout=1) + assert t.is_alive() # it waits for the one holding the lock + # ...and counts only once it has the lock: the one holding it takes the last slot meanwhile. + db.add(QuotaEvent(user_id=user.id, action=action, units=0)) + db.commit() + t.join(timeout=10) + assert got == [None] + + +def _videos_list_fails(monkeypatch, *, on_call: int = 1, reason: str = "quotaExceeded") -> None: + """YouTube's videos.list refuses from its `on_call`-th call on.""" + calls = {"n": 0} + real = _FakeYT.get_videos + + def get_videos(self, ids): + calls["n"] += 1 + if calls["n"] >= on_call: + raise YouTubeError("videos -> 403", reason=reason, path="videos") + return real(self, ids) + + monkeypatch.setattr(_FakeYT, "get_videos", get_videos) + + +def test_a_lookup_that_fails_says_why_not_no_videos(db, make_user, as_user, pages, monkeypatch): + # Nothing it found can be kept unenriched; an empty list would read as "nothing related". + user = make_user() + _videos_list_fails(monkeypatch) + pages["pages"] = [_page([_item(1)], token=None)] + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "youtube", "reason": "quota"} + assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 0 + + +def test_a_lookup_failing_on_a_later_page_keeps_what_came_before( + db, make_user, as_user, pages, monkeypatch +): + user = make_user() + _videos_list_fails(monkeypatch, on_call=2) + pages["pages"] = [_page([_item(1)], token="page2"), _page([_item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + + +def test_an_ingest_that_breaks_the_session_is_rolled_back( + db, make_user, as_user, pages, monkeypatch +): + # A database error leaves the session unusable until it is rolled back: the rows of the pages + # before could not be read for the answer. + user = make_user() + real = search_routes.catalog_search.ingest_candidates + calls = {"n": 0} + + def ingest(db_, yt, user_, candidates): + calls["n"] += 1 + if calls["n"] == 2: + db_.execute(text("SELECT 1/0")) + return real(db_, yt, user_, candidates) + + monkeypatch.setattr(search_routes.catalog_search, "ingest_candidates", ingest) + pages["pages"] = [_page([_item(1)], token="page2"), _page([_item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + + +def test_each_row_says_whether_the_user_follows_its_channel(db, make_user, as_user, pages): + user = make_user() + stranger = make_user() + _known(db, 1, channel="UCfollowed1") + _known(db, 2, channel="UCstranger1") + db.add(Subscription(user_id=user.id, channel_id="UCfollowed1")) + db.add(Subscription(user_id=stranger.id, channel_id="UCstranger1")) + db.commit() + pages["pages"] = [ + _page([_item(1, "UCfollowed1"), _item(2, "UCstranger1"), _item(3, "UCnobody01")], None) + ] + rows = as_user(user).get(f"/api/search/related/{SOURCE}").json()["items"] + assert {r["id"]: r["followed"] for r in rows} == { + _vid(1): True, + _vid(2): False, # another user's follow is not this one's + _vid(3): False, + } diff --git a/backend/tests/test_related_scrape.py b/backend/tests/test_related_scrape.py new file mode 100644 index 00000000..b0efb93c --- /dev/null +++ b/backend/tests/test_related_scrape.py @@ -0,0 +1,232 @@ +"""youtube/related_scrape — a video's related videos from the InnerTube `next` endpoint. + +The two pages under `fixtures/` are real answers (2026-10-24), trimmed to three items and the +continuation: they hold the `lockupViewModel` shape YouTube draws today. Everything here breaks +silently when YouTube moves — a parser that stops finding videos, a cursor that stops paging — so a +shape it does not know must say so, never read as "nothing related". The network is an httpx +MockTransport.""" + +import copy +import json +from pathlib import Path + +import httpx +import pytest + +from app.youtube import innertube, related_scrape + +FIXTURES = Path(__file__).parent / "fixtures" +FIRST = json.loads((FIXTURES / "innertube_next_first.json").read_text(encoding="utf-8")) +CONTINUATION = json.loads( + (FIXTURES / "innertube_next_continuation.json").read_text(encoding="utf-8") +) +RESULTS_HTML = ( + '' +) + + +def _results(page: dict) -> list: + return page["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"][ + "results" + ] + + +@pytest.fixture +def net(monkeypatch): + """Route every httpx.Client the InnerTube module opens through a handler the test sets.""" + calls: list[httpx.Request] = [] + state: dict = {"handler": None, "calls": calls} + + def transport(request: httpx.Request) -> httpx.Response: + calls.append(request) + if request.url.path == "/results": + return httpx.Response(200, text=RESULTS_HTML) + return state["handler"](request) + + real = httpx.Client + monkeypatch.setattr( + innertube.httpx, + "Client", + lambda **kw: real(transport=httpx.MockTransport(transport), **kw), + ) + monkeypatch.setattr(innertube, "_config", None) + return state + + +def _answer(net, page, status=200): + net["handler"] = lambda request: ( + httpx.Response(status, text=page) + if isinstance(page, str) + else httpx.Response(status, json=page) + ) + + +def test_a_first_page_gives_its_videos_with_their_channel_and_the_next_cursor(net): + _answer(net, FIRST) + page = related_scrape.related("SdTZAMDKrNY") + assert [it["id"] for it in page["items"]] == ["to-lHJfK4pw", "R8V8CbuxryI", "HLKo4fJx_7k"] + first = page["items"][0] + assert first["channel_id"] == "UCtslD4DGH6PKyG_1gFAX7sg" + assert first["channel_title"] == "Alexander Amini" + assert first["title"] == "MIT 6.S191 (2025): Reinforcement Learning" + assert first["thumbnail_url"].startswith("https://i.ytimg.com/vi/to-lHJfK4pw/") + assert first["published_at"] is None and first["live_broadcast"] == "none" + assert page["next_page_token"] + body = json.loads(net["calls"][-1].content) + assert body["videoId"] == "SdTZAMDKrNY" and "continuation" not in body + assert net["calls"][-1].url.path == "/youtubei/v1/next" + + +def test_the_largest_thumbnail_is_taken(net): + _answer(net, FIRST) + item = related_scrape.related("SdTZAMDKrNY")["items"][0] + sources = _results(FIRST)[0]["lockupViewModel"]["contentImage"]["thumbnailViewModel"]["image"][ + "sources" + ] + assert item["thumbnail_url"] == sources[-1]["url"] + + +def test_a_continuation_page_reads_its_videos_and_sends_only_the_cursor(net): + _answer(net, CONTINUATION) + page = related_scrape.related("SdTZAMDKrNY", continuation="TOKEN", hl="hu", gl="HU") + assert len(page["items"]) == 2 + assert all(it["channel_id"].startswith("UC") for it in page["items"]) + assert page["next_page_token"] + body = json.loads(net["calls"][-1].content) + assert body["continuation"] == "TOKEN" and "videoId" not in body + assert body["context"]["client"]["hl"] == "hu" and body["context"]["client"]["gl"] == "HU" + + +def test_the_locale_defaults_to_the_topical_one(net): + _answer(net, FIRST) + related_scrape.related("SdTZAMDKrNY") + client = json.loads(net["calls"][-1].content)["context"]["client"] + assert (client["hl"], client["gl"]) == ("en", "US") + + +def test_a_video_by_several_channels_is_left_out(net): + page = copy.deepcopy(FIRST) + meta = _results(page)[1]["lockupViewModel"]["metadata"]["lockupMetadataViewModel"] + meta["image"] = {"avatarStackViewModel": {"avatars": []}} + _answer(net, page) + ids = [it["id"] for it in related_scrape.related("SdTZAMDKrNY")["items"]] + assert ids == ["to-lHJfK4pw", "HLKo4fJx_7k"] + + +def test_what_is_not_a_video_is_left_out_and_a_repeat_counts_once(net): + page = copy.deepcopy(FIRST) + results = _results(page) + playlist = copy.deepcopy(results[0]) + playlist["lockupViewModel"]["contentType"] = "LOCKUP_CONTENT_TYPE_PLAYLIST" + playlist["lockupViewModel"]["contentId"] = "PLxyz" + results.insert(0, playlist) + results.insert(2, copy.deepcopy(results[1])) + results.insert(0, {"adSlotRenderer": {}}) + _answer(net, page) + ids = [it["id"] for it in related_scrape.related("SdTZAMDKrNY")["items"]] + assert ids == ["to-lHJfK4pw", "R8V8CbuxryI", "HLKo4fJx_7k"] + + +def test_the_previous_compact_renderer_is_still_read(net): + page = copy.deepcopy(FIRST) + _results(page).insert( + 0, + { + "compactVideoRenderer": { + "videoId": "oldshape001", + "title": {"simpleText": "Old shape"}, + "longBylineText": { + "runs": [ + { + "text": "Old Chan", + "navigationEndpoint": {"browseEndpoint": {"browseId": "UColdchan"}}, + } + ] + }, + "thumbnail": {"thumbnails": [{"url": "s.jpg"}, {"url": "b.jpg"}]}, + } + }, + ) + _answer(net, page) + first = related_scrape.related("SdTZAMDKrNY")["items"][0] + assert first == { + "id": "oldshape001", + "channel_id": "UColdchan", + "channel_title": "Old Chan", + "title": "Old shape", + "published_at": None, + "thumbnail_url": "b.jpg", + "live_broadcast": "none", + } + + +def test_the_last_page_has_no_cursor(net): + page = copy.deepcopy(FIRST) + _results(page).pop() # the continuationItemRenderer + _answer(net, page) + assert related_scrape.related("SdTZAMDKrNY")["next_page_token"] is None + + +def test_a_video_youtube_no_longer_has_is_an_empty_page(net): + _answer( + net, + { + "contents": { + "twoColumnWatchNextResults": { + "results": { + "results": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [{"backgroundPromoRenderer": {}}] + } + } + ] + } + } + } + } + }, + ) + assert related_scrape.related("aaaaaaaaaaa") == {"items": [], "next_page_token": None} + + +def test_a_watch_frame_with_no_related_column_and_no_word_why_is_an_error(net): + # The frame alone, its related column moved somewhere this does not look: a lost shape, not + # "nothing related". + page = copy.deepcopy(FIRST) + watch = page["contents"]["twoColumnWatchNextResults"] + watch["sideColumn"] = watch.pop("secondaryResults") + _answer(net, page) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") + + +def test_a_related_list_drawn_by_renderers_this_does_not_read_is_an_error(net): + page = copy.deepcopy(FIRST) + results = _results(page) + results[:] = [{"videoTileViewModel": entry} for entry in results] + _answer(net, page) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") + + +@pytest.mark.parametrize( + "answer", + [ + {"responseContext": {}}, # no watch page frame at all + {"onResponseReceivedEndpoints": [{"somethingElse": {}}]}, # a continuation we do not know + "consent", # not JSON + [1, 2], # JSON, not an object + ], +) +def test_an_answer_of_an_unknown_shape_is_an_error_not_an_empty_list(net, answer): + _answer(net, answer) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") + + +def test_a_refused_request_is_an_error_whatever_its_body_says(net): + _answer(net, FIRST, status=429) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") diff --git a/backend/tests/test_route_commits.py b/backend/tests/test_route_commits.py index 5a6fc0bb..5dea6860 100644 --- a/backend/tests/test_route_commits.py +++ b/backend/tests/test_route_commits.py @@ -291,6 +291,23 @@ def test_search_results_cleared_through_the_route_are_gone(db, make_user, as_use assert db.execute(select(SearchFind)).first() is None +def test_clearing_leaves_a_video_the_caller_never_found(db, make_user, as_user): + # A related list ingests its rows with no find: they are in someone's list right now, and an id + # any user could name must not delete them from under it. + user = make_user() + _video(db, "s1", via_search=True) + _video(db, "s2", via_search=True) + db.add(SearchFind(user_id=user.id, video_id="s1")) + db.commit() + r = as_user(user).post("/api/search/clear", json={"video_ids": ["s1", "s2"]}) + assert r.json() == {"ok": True, "removed": 1} + db.expire_all() + assert db.get(Video, "s1") is None and db.get(Video, "s2") is not None + # Naming only ids it never found removes nothing. + r = as_user(user).post("/api/search/clear", json={"video_ids": ["s2"]}) + assert r.json() == {"ok": True, "removed": 0} + + # --- Plex show and season marks --------------------------------------------------------------- diff --git a/backend/tests/test_search_cap.py b/backend/tests/test_search_cap.py new file mode 100644 index 00000000..e79df0ba --- /dev/null +++ b/backend/tests/test_search_cap.py @@ -0,0 +1,361 @@ +"""`GET /api/search/youtube` — the per-user daily cap of the live search, both sources. + +The scrape and search.list are replaced by pages the test hands out, and YouTube's videos.list by an +offline enrich. What is held: every request takes a slot, a cursor's too; a search that failed +outright gives its slot back; a zero limit turns live search off, which says so instead of "try +again tomorrow"; a lookup that fails says why instead of answering "no results"; only a few +failures an hour get their slot back; and the API source — 100 units a page — reads one page per +request, is refused before the call when the shared budget cannot pay for it (checked again under +one lock with the call, whose wait is bounded), counts one slot per search though search.list logs +its own event, and keeps the slot of a search search.list was already paid for — but not of one +that never reached YouTube.""" + +import threading + +import pytest + +from app import quota, sysconfig +from app.routes import search as search_routes +from app.youtube.client import YouTubeError +from app.youtube.innertube import ScrapeError + +ACTION = quota.QuotaAction.VIDEOS_SEARCH + + +def _item(n: int) -> dict: + return { + "id": f"srch{n:07d}", + "channel_id": "UCsearch001", + "channel_title": "Chan", + "title": f"Found {n}", + "published_at": None, + "thumbnail_url": None, + "live_broadcast": "none", + } + + +class _FakeYT: + fail_videos = False + + def __init__(self, *_a, **_kw): + self.spent = 0 + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_channels(self, ids): + return [] + + # search.list: the pages the test hands out, each logged as the real client does (100 units, + # under the action the route attributes it to) — a refusal YouTube answered too, but not a + # failure no endpoint answered (no connection, no token). + api_pages: list = [] + api_calls = 0 + + def search_videos(self, q, page_token=None, order="relevance"): + _FakeYT.api_calls += 1 + page = _FakeYT.api_pages.pop(0) + if not (isinstance(page, YouTubeError) and not page.path): + quota.record_usage(100) + self.spent += 100 + if isinstance(page, Exception): + raise page + return page + + def get_videos(self, ids): + if _FakeYT.fail_videos: + raise YouTubeError("videos -> 403", reason="quotaExceeded", path="videos") + return [ + { + "id": i, + "snippet": {"title": f"T {i}", "publishedAt": "2025-01-01T00:00:00Z"}, + "contentDetails": {"duration": "PT30M"}, + "statistics": {}, + "status": {}, + } + for i in ids + ] + + +@pytest.fixture +def pages(monkeypatch, db): + """The scrape answers with the pages in `state["pages"]`, one per call.""" + sysconfig.set_value(db, "shorts_probe_max_seconds", 60) + sysconfig.set_value(db, "search_source", "scrape") + state: dict = {"pages": [], "calls": 0} + + def search(term, continuation=None): + state["calls"] += 1 + page = state["pages"].pop(0) + if isinstance(page, Exception): + raise page + return page + + monkeypatch.setattr(search_routes.search_scrape, "search", search) + monkeypatch.setattr(search_routes, "YouTubeClient", _FakeYT) + monkeypatch.setattr(_FakeYT, "fail_videos", False) + monkeypatch.setattr(_FakeYT, "api_pages", []) + monkeypatch.setattr(_FakeYT, "api_calls", 0) + return state + + +def _page(n: int) -> dict: + return {"items": [_item(n)], "next_page_token": None} + + +def test_every_request_takes_a_slot_a_cursors_too(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + pages["pages"] = [_page(1), _page(2)] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 200 + refused = client.get("/api/search/youtube?q=cats&cursor=more") + assert refused.status_code == 429 + assert refused.json()["detail"]["reason"] == "search_daily_limit" + assert pages["calls"] == 1 + + +def test_a_search_that_failed_gives_its_slot_back(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + pages["pages"] = [ScrapeError("blocked"), _page(1)] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 422 + assert quota.actions_today(db, user.id, ACTION) == 0 + assert client.get("/api/search/youtube?q=cats").status_code == 200 + assert quota.actions_today(db, user.id, ACTION) == 1 + + +def test_a_zero_limit_turns_live_search_off(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 0) + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 409 + assert resp.json()["detail"] == {"code": "refused", "reason": "search_disabled"} + assert pages["calls"] == 0 + + +def test_a_zero_limit_turns_the_api_source_off_too(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_source", "api") + sysconfig.set_value(db, "search_daily_limit_per_user", 0) + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 409 + assert resp.json()["detail"]["reason"] == "search_disabled" + + +def test_a_lookup_that_fails_says_why_not_no_results(db, make_user, as_user, pages, monkeypatch): + user = make_user() + monkeypatch.setattr(_FakeYT, "fail_videos", True) + pages["pages"] = [_page(1)] + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "youtube", "reason": "quota"} + assert quota.actions_today(db, user.id, ACTION) == 0 + + +def _api(db) -> None: + sysconfig.set_value(db, "search_source", "api") + sysconfig.set_value(db, "quota_daily_budget", 10000) + + +def test_the_api_source_reads_one_page_per_request(db, make_user, as_user, pages): + # Each page is 100 units: a short page is answered as it is, with its cursor, never paged on. + _api(db) + _FakeYT.api_pages = [{"items": [_item(1)], "next_page_token": "nextpage"}, _page(2)] + resp = as_user(make_user()).get("/api/search/youtube?q=cats&limit=20") + assert resp.status_code == 200 + assert _FakeYT.api_calls == 1 + assert resp.json()["next_cursor"] == "nextpage" + + +def test_the_api_source_counts_one_slot_per_search(db, make_user, as_user, pages): + # search.list logs its own event under the action (record_usage): accounting, not a second slot. + user = make_user() + _api(db) + sysconfig.set_value(db, "search_daily_limit_per_user", 2) + _FakeYT.api_pages = [_page(1), _page(2), _page(3)] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 200 + assert quota.actions_today(db, user.id, ACTION) == 1 + assert client.get("/api/search/youtube?q=dogs").status_code == 200 + refused = client.get("/api/search/youtube?q=birds") + assert refused.status_code == 429 + assert refused.json()["detail"]["reason"] == "search_daily_limit" + assert _FakeYT.api_calls == 2 + + +def test_the_api_source_takes_its_slot_under_the_lock(db, make_user, as_user, pages, monkeypatch): + user = make_user() + _api(db) + claimed: list = [] + real = quota.claim_action + + def claim(uid, action, limit): + claimed.append(action) + return real(uid, action, limit) + + monkeypatch.setattr(search_routes.quota, "claim_action", claim) + _FakeYT.api_pages = [_page(1)] + assert as_user(user).get("/api/search/youtube?q=cats").status_code == 200 + assert claimed == [ACTION] + + +def test_the_api_source_is_refused_before_the_call_when_the_budget_cannot_pay( + db, make_user, as_user, pages +): + user = make_user() + _api(db) + sysconfig.set_value(db, "quota_daily_budget", 50) + _FakeYT.api_pages = [_page(1)] + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "quota_spent" + assert _FakeYT.api_calls == 0 + assert quota.actions_today(db, user.id, ACTION) == 0 + + +def test_an_api_search_that_search_list_refused_keeps_its_slot(db, make_user, as_user, pages): + # YouTube charges a refused call too (the client records the units first), so a search that + # reached search.list has spent the shared budget: refunding it let a bogus cursor spend it all. + user = make_user() + _api(db) + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + _FakeYT.api_pages = [ + YouTubeError("search -> 400", reason="invalidPageToken", path="search"), + _page(1), + ] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats&cursor=bogus").status_code == 422 + assert quota.actions_today(db, user.id, ACTION) == 1 + refused = client.get("/api/search/youtube?q=cats") + assert refused.status_code == 429 + assert refused.json()["detail"]["reason"] == "search_daily_limit" + assert _FakeYT.api_calls == 1 + + +def test_an_api_search_the_budget_refused_at_the_call_gives_its_slot_back( + db, make_user, as_user, pages, monkeypatch +): + user = make_user() + _api(db) + answers = iter([True, False]) # the pre-check passes; the budget is gone by the call + monkeypatch.setattr(search_routes.quota, "can_spend", lambda _db, _u: next(answers)) + _FakeYT.api_pages = [_page(1)] + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "quota_spent" + assert _FakeYT.api_calls == 0 + assert quota.actions_today(db, user.id, ACTION) == 0 + + +def test_the_budget_check_and_the_call_it_pays_for_hold_one_lock( + db, make_user, as_user, pages, monkeypatch +): + # Checked apart, searches sent together at the last 100 units all passed, and all spent: the + # check and the call sit inside ONE hold of the lock, not two. + user = make_user() + _api(db) + events: list[str] = [] + real_check = quota.can_spend + real_call = _FakeYT.search_videos + + class _Recorded: + def __init__(self): + self._lock = threading.Lock() + + def acquire(self, timeout=-1): + events.append("acquire") + return self._lock.acquire(timeout=timeout) + + def release(self): + events.append("release") + self._lock.release() + + def can_spend(db_, units): + events.append("check") + return real_check(db_, units) + + def search_videos(self, q, page_token=None, order="relevance"): + events.append("call") + return real_call(self, q, page_token=page_token, order=order) + + monkeypatch.setattr(search_routes, "_api_search", _Recorded()) + monkeypatch.setattr(search_routes.quota, "can_spend", can_spend) + monkeypatch.setattr(_FakeYT, "search_videos", search_videos) + _FakeYT.api_pages = [_page(1)] + assert as_user(user).get("/api/search/youtube?q=cats").status_code == 200 + assert events == ["check", "acquire", "check", "call", "release"] + + +def test_a_search_kept_waiting_for_the_lock_is_refused_as_busy_and_keeps_nothing( + db, make_user, as_user, pages, monkeypatch +): + # Each waiter holds a request worker: a slow YouTube must not park them all behind one another. + user = make_user() + _api(db) + monkeypatch.setattr(search_routes, "API_SEARCH_WAIT_SECONDS", 0.05) + _FakeYT.api_pages = [_page(1)] + # Held by another search for a while: a wait without its bound would get it, and be answered. + assert search_routes._api_search.acquire(timeout=1) + later = threading.Timer(2.0, search_routes._api_search.release) + later.start() + try: + resp = as_user(user).get("/api/search/youtube?q=cats") + finally: + later.join() + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "search_busy" + assert _FakeYT.api_calls == 0 + assert quota.actions_today(db, user.id, ACTION) == 0 + + +@pytest.mark.parametrize( + "failure", + [ + YouTubeError("YouTube API unreachable (ConnectError)", reason="transport"), + YouTubeError("User has no stored OAuth token"), + ], + ids=["no-connection", "no-token"], +) +def test_an_api_search_that_never_reached_youtube_gives_its_slot_back( + db, make_user, as_user, pages, failure +): + user = make_user() + _api(db) + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + _FakeYT.api_pages = [failure, _page(1)] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 422 + assert quota.actions_today(db, user.id, ACTION) == 0 + assert client.get("/api/search/youtube?q=cats").status_code == 200 + + +def test_an_api_search_whose_answer_never_came_keeps_its_slot(db, make_user, as_user, pages): + # The request went out (a read timeout): YouTube may have counted it. + user = make_user() + _api(db) + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + _FakeYT.api_pages = [ + YouTubeError("YouTube API unreachable (ReadTimeout)", reason="transport", sent=True), + _page(1), + ] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 422 + assert quota.actions_today(db, user.id, ACTION) == 1 + assert client.get("/api/search/youtube?q=cats").status_code == 429 + + +def test_failures_past_the_hours_refunds_keep_their_slot(db, make_user, as_user, pages): + # A cursor is a free string: a request built to fail would otherwise read YouTube without end. + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 50) + n = search_routes.REFUNDS_PER_HOUR + pages["pages"] = [ScrapeError("bad cursor") for _ in range(n + 1)] + client = as_user(user) + for _ in range(n + 1): + assert client.get("/api/search/youtube?q=cats&cursor=bogus").status_code == 422 + assert quota.actions_today(db, user.id, ACTION) == 1 diff --git a/backend/tests/test_search_scrape.py b/backend/tests/test_search_scrape.py index 79f071ae..705a91b8 100644 --- a/backend/tests/test_search_scrape.py +++ b/backend/tests/test_search_scrape.py @@ -9,7 +9,7 @@ import json import httpx import pytest -from app.youtube import search_scrape +from app.youtube import innertube, search_scrape RESULTS_HTML = ( '' @@ -79,11 +79,11 @@ def net(monkeypatch): real = httpx.Client monkeypatch.setattr( - search_scrape.httpx, + innertube.httpx, "Client", lambda **kw: real(transport=httpx.MockTransport(transport), **kw), ) - monkeypatch.setattr(search_scrape, "_config", None) + monkeypatch.setattr(innertube, "_config", None) state["calls"] = calls return state @@ -171,7 +171,7 @@ def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, re with no error anywhere.""" net["handler"] = _serve(results_html=results_html) search_scrape.search("a") - assert net["calls"][-1].url.params["key"] == search_scrape._FALLBACK_KEY + assert net["calls"][-1].url.params["key"] == innertube._FALLBACK_KEY net["handler"] = _serve() search_scrape.search("b") diff --git a/backend/tests/test_search_scrape_ingest.py b/backend/tests/test_search_scrape_ingest.py index 8d817880..6c38c3cf 100644 --- a/backend/tests/test_search_scrape_ingest.py +++ b/backend/tests/test_search_scrape_ingest.py @@ -13,10 +13,13 @@ a placeholder never becomes the video's permanent date. from datetime import datetime, timezone +import pytest + from app import sysconfig from app.models import User from app.catalog.models import Video from app.catalog.search import ingest_candidates +from app.youtube.client import YouTubeError class _FakeYT: @@ -97,6 +100,25 @@ def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db): assert c.enriched_at is None +class _FailingYT(_FakeYT): + def get_videos(self, ids): + raise YouTubeError("videos -> 403", reason="quotaExceeded", path="videos") + + +def test_a_videos_list_failure_raises_and_keeps_the_stubs(db): + # Swallowed, it answered as a page with nothing on it — "no results" for a search whose lookup + # never ran. The stubs stay: the FK targets, backfilled later. + user = User(email="searcher3@example.com") + db.add(user) + db.commit() + + with pytest.raises(YouTubeError): + ingest_candidates(db, _FailingYT({}), user, [_candidate("vid_ddddddd", "UC_chan_d")]) + + db.rollback() # what the caller does: the stubs were committed before the lookup + assert db.get(Video, "vid_ddddddd") is not None + + def test_search_youtube_route_returns_422_when_ingest_fails(db, monkeypatch, override_dep): # Route-level guard (B-core-1): an ingest failure with nothing yet collected must surface as a 422 # + message on this JSON endpoint (the FE hides 5xx behind a generic "connection lost" + auto-retry, diff --git a/backend/tests/test_youtube_refusals.py b/backend/tests/test_youtube_refusals.py index 94d51dbd..2400578a 100644 --- a/backend/tests/test_youtube_refusals.py +++ b/backend/tests/test_youtube_refusals.py @@ -193,3 +193,45 @@ def test_any_other_token_refusal_keeps_the_grant(monkeypatch): c._access_token() assert tok.scopes == "s" assert refusal_reason(caught.value) == "unavailable" + + +@pytest.mark.parametrize( + ("failure", "sent"), + [ + (httpx.ConnectError("refused"), False), + (httpx.ConnectTimeout("slow connect"), False), + (httpx.ProxyError("proxy down"), False), + (httpx.ReadTimeout("no answer"), True), + (httpx.RemoteProtocolError("dropped"), True), + ], +) +def test_a_transport_failure_says_whether_the_request_went_out(failure, sent): + """A search that never reached YouTube spent nothing and gets its slot back; one whose answer + never came may have been counted, and keeps it (`routes/search`).""" + + def request(*_a, **_k): + raise failure + + c = yt.YouTubeClient.__new__(yt.YouTubeClient) + c._http = SimpleNamespace(request=request) + with pytest.raises(YouTubeError) as caught: + c._send("GET", "https://example.invalid") + assert caught.value.reason == "transport" + assert caught.value.sent is sent + + +def test_a_call_youtube_answered_is_spent_a_refused_one_too(monkeypatch): + resp = httpx.Response( + 400, + json={"error": {"errors": [{"reason": "invalidPageToken"}]}}, + request=httpx.Request("GET", f"{yt.API_BASE}/search"), + ) + recorded: list[int] = [] + c = yt.YouTubeClient.__new__(yt.YouTubeClient) + c.db, c.spent = None, 0 + monkeypatch.setattr(yt.sysconfig, "get_str", lambda d, key: "a-key") + monkeypatch.setattr(yt.quota, "record_usage", recorded.append) + monkeypatch.setattr(c, "_send", lambda *a, **k: resp) + with pytest.raises(YouTubeError): + c._get("search", {}, cost=100) + assert (recorded, c.spent) == ([100], 100) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 05543621..e81a0579 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -225,7 +225,10 @@ Also in force: tag chip is `Chip`; a sortable row's grip is `DragHandle`. On the always-dark player chrome both `Button` and `IconButton` use the `overlay` variant (white, a white layer), whatever the theme; the Plex control row is IconButton's `player` rung (36, a 20px glyph, round, 44 on touch). A pressed toggle is - `pressed`: the solid accent on a `Button`, the accent glyph on an `IconButton`. A raw `, + ); + const btn = screen.getByRole("button", { name: "Loop: All" }); + expect(btn.hasAttribute("aria-pressed")).toBe(false); + expect(btn.hasAttribute("data-lit")).toBe(true); + expect(btn.className).toContain("data-lit:bg-accent"); + }); + it("a toggle announces aria-pressed and paints the pressed state", () => { const { rerender } = render( )} - {wide && ( + {wide && !relatedOn && (
{t("player.description")} @@ -1466,6 +1621,43 @@ export default function PlayerModal({ />
)} + {wide && relatedOn && ( +
+ {/* The class replaces Tabs' own, so it keeps the row's layout and gap with + its tighter margin. */} + setPanelTab(id === "related" ? "related" : "description")} + idBase={panelTabs.idBase} + label={t("player.panelTabs")} + className="flex flex-wrap gap-1 mb-3" + /> +
+ {panelTab === "description" ? ( +
+ +
+ ) : ( + // Out into the panel's padding: in 20rem, every rem is the title's. +
+ +
+ )} +
+
+ )} )} diff --git a/frontend/src/features/player/RelatedVideos.dom.test.tsx b/frontend/src/features/player/RelatedVideos.dom.test.tsx new file mode 100644 index 00000000..482562de --- /dev/null +++ b/frontend/src/features/player/RelatedVideos.dom.test.tsx @@ -0,0 +1,410 @@ +// @vitest-environment jsdom +// +// A video's related videos, as the player shows them. What is held: a row says when the user follows +// its channel; the region the user picks is asked for and kept on the account; a video marked seen +// here leaves the list (and comes back when the server refuses the mark); reading further sends the +// cursor; and a failure says so rather than drawing an empty list. +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { FloatingTree } from "@floating-ui/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +vi.mock("react-i18next", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), + }; +}); +vi.mock("../navigation/navigationContext", () => ({ + useNavigationActions: () => ({ openChannel: vi.fn() }), +})); +vi.mock("../video/AddToPlaylist", () => ({ default: () => null })); + +import { RelatedVideos } from "./RelatedVideos"; +import { api, HttpError, type RelatedVideo, type Video } from "../../lib/api"; +import { clearAll, getNotifications } from "../../lib/notifications"; +import { qk } from "../../lib/queryKeys"; +import { setHintsEnabled } from "../../lib/hints"; +import { writeVideoState } from "../video/videoWrites"; + +beforeAll(() => { + class Stub { + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } + globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver; + globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver; +}); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + clearAll(); +}); + +const row = (id: string, extra: Partial = {}): RelatedVideo => + ({ + id, + title: `Video ${id}`, + channel_id: `UC${id}`, + channel_title: `Channel ${id}`, + watch_url: `https://www.youtube.com/watch?v=${id}`, + status: "new", + saved: false, + position_seconds: 0, + followed: false, + ...extra, + }) as RelatedVideo; + +const page = (items: RelatedVideo[], next: string | null = null) => ({ + items, + has_more: next != null, + next_cursor: next, +}); + +function setup( + onPlay: (v: Video, startAt: number | null) => void = () => {}, + onWrite?: (write: Promise) => void, + qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }), +) { + vi.spyOn(api, "comfortPins").mockResolvedValue({ items: [] } as never); + vi.spyOn(api, "downloadIndex").mockResolvedValue({}); + const saved = vi.spyOn(api, "savePrefs").mockResolvedValue({} as never); + render( + + + + + , + ); + return { saved, qc }; +} + +// A row's title (its picture is a link of the same name). +const title = async (name: string) => + (await screen.findAllByRole("link", { name })).find((a) => a.hasAttribute("data-nav-open"))!; +const titles = () => + screen + .queryAllByRole("link") + .filter((a) => a.hasAttribute("data-nav-open")) + .map((a) => a.textContent); +// The menu's surface is visibility:hidden in jsdom: an item is matched on its text. +const menuItem = (label: string) => + screen.queryAllByRole("menuitem", { hidden: true }).find((b) => b.textContent === label)!; + +describe("RelatedVideos", () => { + it("marks the rows whose channel the user follows, and only those", async () => { + vi.spyOn(api, "related").mockResolvedValue( + page([row("A", { followed: true }), row("B"), row("C", { followed: true })]), + ); + setup(); + await title("Video A"); + const marks = screen.getAllByRole("img", { name: "player.related.followed" }); + const owners = marks.map((m) => m.closest("[data-video-id]")?.getAttribute("data-video-id")); + expect(owners).toEqual(["A", "C"]); + }); + + it("asks for the region picked, and keeps the pick on the account", async () => { + const asked = vi.spyOn(api, "related").mockResolvedValue(page([row("A")])); + const { saved } = setup(); + await title("Video A"); + expect(asked.mock.calls).toEqual([["SRC", "topical", null]]); + fireEvent.click(screen.getByRole("button", { name: "player.related.hu" })); + await waitFor(() => expect(asked.mock.calls).toContainEqual(["SRC", "hu", null])); + expect(saved).toHaveBeenCalledWith({ relatedRegion: "hu" }); + }); + + it("sets the region switch at the toolbar rung, not the 10px caption one", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A")])); + setup(); + await title("Video A"); + const track = screen.getByRole("group", { name: "player.related.region" }); + expect(track.className).toMatch(/\btext-xs\b/); + }); + + // A plain pick resumes (null: the player knows the freshest position); the row's own position + // travels on the video it hands over. + it("plays a row from where the user left it", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A", { position_seconds: 95 })])); + const onPlay = vi.fn(); + setup(onPlay); + fireEvent.click(await title("Video A")); + expect(onPlay).toHaveBeenCalledWith( + expect.objectContaining({ id: "A", position_seconds: 95 }), + null, + ); + }); + + it("drops a video hidden here at once, and brings it back when the server refuses", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A"), row("B")])); + let refuse: (e: Error) => void = () => {}; + const write = vi + .spyOn(api, "setState") + .mockReturnValue(new Promise((_, reject) => (refuse = reject)) as never); + setup(); + fireEvent.contextMenu((await title("Video A")).closest("[data-video-id]")!); + fireEvent.click(menuItem("card.hide")); + await waitFor(() => expect(write).toHaveBeenCalledWith("A", "hidden")); + expect(titles()).toEqual(["Video B"]); + await act(async () => refuse(new Error("offline"))); + expect(titles()).toEqual(["Video A", "Video B"]); + }); + + it("keeps a video marked watched out of the list once the server has it", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A"), row("B")])); + vi.spyOn(api, "setState").mockResolvedValue(undefined as never); + setup(); + fireEvent.contextMenu((await title("Video B")).closest("[data-video-id]")!); + fireEvent.click(menuItem("card.markWatched")); + await waitFor(() => expect(titles()).toEqual(["Video A"])); + }); + + it("reads further with the cursor the last page gave", async () => { + const asked = vi + .spyOn(api, "related") + .mockResolvedValueOnce(page([row("A")], "CUR1")) + .mockResolvedValueOnce(page([row("B")])); + setup(); + await title("Video A"); + fireEvent.click(screen.getByRole("button", { name: "player.related.more" })); + await title("Video B"); + expect(asked.mock.calls[1]).toEqual(["SRC", "topical", "CUR1"]); + expect(screen.queryByRole("button", { name: "player.related.more" })).toBeNull(); + }); + + it("says so when YouTube refuses, rather than drawing an empty list", async () => { + vi.spyOn(api, "related").mockRejectedValue( + new HttpError(422, "YouTube isn't reachable", undefined), + ); + setup(); + expect((await screen.findByRole("alert")).textContent).toContain("YouTube isn't reachable"); + expect(screen.queryByText("player.related.empty")).toBeNull(); + }); + + // A deferred answer, settled when the test says. + const deferred = () => { + let resolve: () => void = () => {}; + let reject: (e: Error) => void = () => {}; + const promise = new Promise((res, rej) => { + resolve = res as () => void; + reject = rej; + }); + return { promise, resolve, reject }; + }; + + it("keeps a row hidden while its hide is on its way, though an earlier save landed", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A"), row("B")])); + const save = deferred(); + const saving = vi.spyOn(api, "watchLaterAdd").mockReturnValue(save.promise as never); + const hide = deferred(); + const hiding = vi.spyOn(api, "setState").mockReturnValue(hide.promise as never); + setup(); + const card = (await title("Video A")).closest("[data-video-id]")!; + fireEvent.contextMenu(card); + fireEvent.click(menuItem("card.saveForLater")); + await waitFor(() => expect(saving).toHaveBeenCalledWith("A")); + fireEvent.contextMenu(card); + fireEvent.click(menuItem("card.hide")); + await waitFor(() => expect(hiding).toHaveBeenCalledWith("A", "hidden")); + expect(titles()).toEqual(["Video B"]); + await act(async () => save.resolve()); + expect(titles()).toEqual(["Video B"]); + }); + + it("announces a hide with the video it names, for the notification center to find", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A")])); + vi.spyOn(api, "setState").mockResolvedValue(undefined as never); + const writes: Promise[] = []; + setup(undefined, (w) => void writes.push(w)); + fireEvent.contextMenu((await title("Video A")).closest("[data-video-id]")!); + fireEvent.click(menuItem("card.hide")); + await waitFor(() => expect(getNotifications().length).toBeGreaterThan(0)); + expect(getNotifications().map((n) => n.meta)).toContainEqual({ + kind: "video-hidden", + videoId: "A", + title: "Video A", + channelId: "UCA", + channelName: "Channel A", + }); + // And the write reached the host's session. + expect(writes).toHaveLength(1); + }); + + it("flips a row's bookmark at once, and back when the server refuses", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A")])); + const save = deferred(); + vi.spyOn(api, "watchLaterAdd").mockReturnValue(save.promise as never); + setup(); + const card = (await title("Video A")).closest("[data-video-id]")!; + // What the menu offers is the row's state: un-save once saved. + const offers = (label: string) => { + fireEvent.contextMenu(card); + return !!menuItem(label); + }; + expect(offers("card.saveForLater")).toBe(true); + fireEvent.click(menuItem("card.saveForLater")); + await waitFor(() => expect(offers("card.menu.unsave")).toBe(true)); + fireEvent.keyDown(document.activeElement ?? document.body, { key: "Escape" }); + await act(async () => save.reject(new Error("offline"))); + await waitFor(() => expect(offers("card.saveForLater")).toBe(true)); + }); + + it("is kept for the session, not collected when the list closes", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A")])); + const { qc } = setup(); + await title("Video A"); + const [cached] = qc.getQueryCache().findAll({ queryKey: qk.related() }); + expect(cached!.options.gcTime).toBe(Infinity); + }); + + it("says the next page failed under the rows it has, and retries that page", async () => { + const asked = vi + .spyOn(api, "related") + .mockResolvedValueOnce(page([row("A")], "CUR1")) + .mockRejectedValueOnce(new HttpError(422, "YouTube isn't reachable", undefined)) + .mockResolvedValueOnce(page([row("B")])); + setup(); + await title("Video A"); + fireEvent.click(screen.getByRole("button", { name: "player.related.more" })); + expect((await screen.findByRole("alert")).textContent).toContain("YouTube isn't reachable"); + expect(titles()).toEqual(["Video A"]); + fireEvent.click(screen.getByRole("button", { name: "common.retry" })); + await title("Video B"); + expect(asked.mock.calls.map((c) => c[2])).toEqual([null, "CUR1", "CUR1"]); + }); + + it("with nothing unseen read so far, says so above Show more, and retries the page it failed", async () => { + const asked = vi + .spyOn(api, "related") + .mockResolvedValueOnce(page([], "CUR1")) + .mockRejectedValueOnce(new HttpError(422, "YouTube isn't reachable", undefined)) + .mockResolvedValueOnce(page([row("B")])); + setup(); + expect(await screen.findByText("player.related.emptySoFar")).toBeTruthy(); + expect(screen.queryByText("player.related.empty")).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "player.related.more" })); + await screen.findByRole("alert"); + fireEvent.click(screen.getByRole("button", { name: "common.retry" })); + await title("Video B"); + expect(asked.mock.calls.map((c) => c[2])).toEqual([null, "CUR1", "CUR1"]); + }); + + it.each([ + ["the day's cap", { code: "refused", reason: "related_daily_limit", limit: 5 }], + ["turned off", { code: "refused", reason: "related_disabled" }], + ["the shared budget", { code: "youtube", reason: "quota" }], + ])("offers no retry a retry cannot answer: %s", async (_name, detail) => { + vi.spyOn(api, "related").mockRejectedValue(new HttpError(429, "Refused", detail)); + setup(); + expect((await screen.findByRole("alert")).textContent).toContain("Refused"); + expect(screen.queryByRole("button", { name: "common.retry" })).toBeNull(); + }); + + it("offers a retry where trying again can help", async () => { + vi.spyOn(api, "related").mockRejectedValue( + new HttpError(422, "YouTube isn't reachable", { code: "youtube", reason: "unavailable" }), + ); + setup(); + await screen.findByRole("alert"); + expect(screen.getByRole("button", { name: "common.retry" })).toBeTruthy(); + }); + + it("names the followed mark on hover", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A", { followed: true })])); + setup(); + await title("Video A"); + const mark = screen.getByRole("img", { name: "player.related.followed" }); + fireEvent.mouseEnter(mark.parentElement!); + fireEvent.pointerEnter(mark.parentElement!, { pointerType: "mouse" }); + expect(await screen.findByText("player.related.followed")).toBeTruthy(); + }); + + // An infinite query builds its next page from the pages it read when that fetch started: a mark + // that landed in between was dropped with them, for the session. + it("keeps a hide that landed while Show more was on its way", async () => { + let land: (p: ReturnType) => void = () => {}; + vi.spyOn(api, "related") + .mockResolvedValueOnce(page([row("A"), row("C")], "CUR1")) + .mockReturnValueOnce(new Promise((r) => (land = r)) as never); + const write = vi.spyOn(api, "setState").mockResolvedValue({ ok: true } as never); + setup(); + await title("Video A"); + fireEvent.click(screen.getByRole("button", { name: "player.related.more" })); + fireEvent.contextMenu((await title("Video A")).closest("[data-video-id]")!); + fireEvent.click(menuItem("card.hide")); + await waitFor(() => expect(write).toHaveBeenCalledWith("A", "hidden")); + await act(async () => {}); + await act(async () => land(page([row("B")]))); + await title("Video B"); + expect(titles()).toEqual(["Video C", "Video B"]); + }); + + // Hidden in this session, then un-hidden on another device: a list read after the mark carries + // the server's word, and the mark does not paint over it. + it("shows a list read after a mark as the server has it", async () => { + vi.spyOn(api, "setState").mockResolvedValue({ ok: true } as never); + vi.spyOn(api, "related").mockResolvedValue(page([row("A"), row("B")])); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + await writeVideoState(qc, "A", "hidden"); + await new Promise((r) => setTimeout(r, 5)); // the list is read a moment after the mark + setup(undefined, undefined, qc); + await title("Video B"); + expect(titles()).toEqual(["Video A", "Video B"]); + }); + + // The Notification Center's undo, the feed's reset, a card elsewhere: every write reaches it. + it("follows a video's state written anywhere else", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([row("A"), row("B")])); + vi.spyOn(api, "setState").mockResolvedValue({ ok: true } as never); + const { qc } = setup(); + await title("Video A"); + await act(async () => void (await writeVideoState(qc, "A", "hidden"))); + await waitFor(() => expect(titles()).toEqual(["Video B"])); + await act(async () => void (await writeVideoState(qc, "A", "new"))); + await waitFor(() => expect(titles()).toEqual(["Video A", "Video B"])); + }); + + // Show more is the request Retry would make: refused for good, it is not offered either. + it("offers no Show more once the next page was refused for good", async () => { + vi.spyOn(api, "related") + .mockResolvedValueOnce(page([row("A")], "CUR1")) + .mockRejectedValueOnce( + new HttpError(429, "Limit", { code: "refused", reason: "related_daily_limit", limit: 3 }), + ); + setup(); + await title("Video A"); + fireEvent.click(screen.getByRole("button", { name: "player.related.more" })); + expect((await screen.findByRole("alert")).textContent).toContain("Limit"); + expect(screen.queryByRole("button", { name: "player.related.more" })).toBeNull(); + expect(screen.queryByRole("button", { name: "common.retry" })).toBeNull(); + expect(titles()).toEqual(["Video A"]); + }); + + it("names the followed mark on hover with hints turned off too", async () => { + setHintsEnabled(false); + try { + vi.spyOn(api, "related").mockResolvedValue(page([row("A", { followed: true })])); + setup(); + await title("Video A"); + const mark = screen.getByRole("img", { name: "player.related.followed" }); + fireEvent.mouseEnter(mark.parentElement!); + fireEvent.pointerEnter(mark.parentElement!, { pointerType: "mouse" }); + expect(await screen.findByText("player.related.followed")).toBeTruthy(); + } catch (e) { + setHintsEnabled(true); + throw e; + } + setHintsEnabled(true); + }); + + it("says when nothing is left that the user has not seen", async () => { + vi.spyOn(api, "related").mockResolvedValue(page([])); + setup(); + expect(await screen.findByText("player.related.empty")).toBeTruthy(); + expect(screen.queryByRole("alert")).toBeNull(); + }); +}); diff --git a/frontend/src/features/player/RelatedVideos.tsx b/frontend/src/features/player/RelatedVideos.tsx new file mode 100644 index 00000000..19873bdf --- /dev/null +++ b/frontend/src/features/player/RelatedVideos.tsx @@ -0,0 +1,236 @@ +import { useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useInfiniteQuery, useQueryClient } from "@tanstack/react-query"; +import { CircleCheck } from "lucide-react"; + +import { + api, + HttpError, + refusalReason, + type RelatedRegion, + type Video, + type VideoStatus, +} from "../../lib/api"; +import { qk } from "../../lib/queryKeys"; +import { nextPageOnClick } from "../../lib/nextPage"; +import { usePref, type UsePrefOptions } from "../../lib/usePref"; +import { markedRows, stampFetch, useVideoMarks } from "../../lib/progressCache"; +import { notifyActionError } from "../../lib/actionErrors"; +import { Button } from "../../components/ui/Button"; +import SegmentedToggle from "../../components/ui/SegmentedToggle"; +import { LoadingState, StateMessage } from "../../components/ui/QueryState"; +import { useContextMenu } from "../../components/ui/useContextMenu"; +import NavGrid from "../../components/ui/NavGrid"; +import Tooltip from "../../components/ui/Tooltip"; +import VideoCard from "../video/VideoCard"; +import { VideoMenu } from "../video/VideoMenu"; +import { announceVideoState } from "../video/stateNotice"; +import { writeVideoState, writeWatchLater } from "../video/videoWrites"; + +/** Which list YouTube fills in — on the video's topic or with what trends in Hungary — is a mood, + * decided at the list, and the account keeps the last one (the owner's pick). */ +const RELATED_REGION_PREF = "relatedRegion"; +const RELATED_REGION_OPTIONS: UsePrefOptions = { + parse: (raw) => (raw === "hu" ? "hu" : "topical"), +}; + +// The list leaves out what the user has seen (the server does, as it fetches); a video marked here +// leaves it the same way, at once. +const isSeen = (v: Video) => v.status === "watched" || v.status === "hidden"; + +/** The refusals a retry cannot answer: the day's cap (tomorrow does), the feature turned off on + * this instance and the shared budget spent (the admin, or tomorrow). */ +const FINAL_REASONS = new Set(["related_daily_limit", "related_disabled", "quota"]); +const isFinal = (e: unknown) => FINAL_REASONS.has(refusalReason(e) ?? ""); + +// A write not yet confirmed: its change shows at once, and leaves when the server answers. +type Pending = { seq: number; id: string; change: Partial