From 8b37c470ba5d11c45757f0b2e62d660b782e816d Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 24 Oct 2026 16:00:37 +0200 Subject: [PATCH 01/20] feat(catalog): a video's related videos, without what the user has seen GET /api/search/related/{video_id} reads the list YouTube shows beside a video from the website's InnerTube "next" endpoint (no quota), and brings it into the catalog the way live-search results come in. It leaves out what the user watched or hid, here or in their Takeout history (unless taken back to "new" since), blocked channels and the video itself, and pages on until 20 are gathered (at most 4 pages). region=topical|hu picks the locale the list is filled in. Its own per-user daily cap (related_daily_limit_per_user, one event per opened list); a failed scrape answers 422 youtube/unavailable. The InnerTube request (live key, consent cookie, errors) moves into youtube/innertube.py, shared by the search scrape and the related one. --- backend/app/config.py | 3 + backend/app/quota.py | 1 + backend/app/reader/queries.py | 30 + backend/app/routes/search.py | 92 +- backend/app/sysconfig.py | 9 + backend/app/youtube/innertube.py | 111 +++ backend/app/youtube/related_scrape.py | 150 ++++ backend/app/youtube/search_scrape.py | 113 +-- .../fixtures/innertube_next_continuation.json | 527 ++++++++++++ .../tests/fixtures/innertube_next_first.json | 805 ++++++++++++++++++ backend/tests/test_related_route.py | 298 +++++++ backend/tests/test_related_scrape.py | 212 +++++ backend/tests/test_search_scrape.py | 8 +- frontend/src/i18n/locales/en/config.json | 4 + frontend/src/i18n/locales/en/errors.json | 1 + .../src/i18n/locales/en/quotaActions.json | 1 + frontend/src/i18n/locales/hu/config.json | 4 + frontend/src/i18n/locales/hu/errors.json | 1 + .../src/i18n/locales/hu/quotaActions.json | 1 + 19 files changed, 2263 insertions(+), 108 deletions(-) create mode 100644 backend/app/youtube/innertube.py create mode 100644 backend/app/youtube/related_scrape.py create mode 100644 backend/tests/fixtures/innertube_next_continuation.json create mode 100644 backend/tests/fixtures/innertube_next_first.json create mode 100644 backend/tests/test_related_route.py create mode 100644 backend/tests/test_related_scrape.py diff --git a/backend/app/config.py b/backend/app/config.py index b1279f0c..ffc420b8 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -113,6 +113,9 @@ class Settings(BaseSettings): # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable. search_daily_limit_per_user: int = 70 + # Per-user cap on related-video lists opened per day (the InnerTube watch page, no quota; each + # list's enrich is 1 unit per 50 videos). A pure anti-abuse rate limit. Admin-tunable. + related_daily_limit_per_user: int = 200 # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list, # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks. search_source: SearchSource = "scrape" diff --git a/backend/app/quota.py b/backend/app/quota.py index 552babb8..108e45c4 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -44,6 +44,7 @@ class QuotaAction: VIDEOS_ENRICH = "videos_enrich" VIDEOS_LOOKUP = "videos_lookup" VIDEOS_SEARCH = "videos_search" + VIDEOS_RELATED = "videos_related" CHANNELS_DISCOVER = "channels_discover" CHANNELS_EXPLORE = "channels_explore" CHANNELS_SUBSCRIBE = "channels_subscribe" diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index ff090100..ad3c5353 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -319,6 +319,36 @@ def held_by(db: Session, user: User, video_ids: list[str]) -> set[str]: ) +def seen_by(db: Session, user: User, video_ids: list[str]) -> set[str]: + """Which of these videos the user has already seen or set aside: watched or hidden here, or in + their uploaded Takeout history unless they took it back to "new" here since. The ids need not + be in the catalog — a Takeout row names a video the catalog may never have had — so a list + of what YouTube suggests can drop them before anything is ingested.""" + if not video_ids: + return set() + marked = select(VideoState.video_id).where( + owned_by(VideoState.user_id, user), + VideoState.video_id.in_(video_ids), + VideoState.status.in_(("watched", "hidden")), + ) + unmarked_since = ( + select(VideoUnmark.id) + .where( + owned_by(VideoUnmark.user_id, user), + VideoUnmark.video_id == TakeoutItem.video_id, + VideoUnmark.at > TakeoutItem.at, + ) + .exists() + ) + in_history = select(TakeoutItem.video_id).where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.video_id.in_(video_ids), + ~unmarked_since, + ) + return set(db.scalars(marked)) | set(db.scalars(in_history)) + + def feed_rows_by_ids(db: Session, user: User, video_ids: list[str]) -> list[dict]: """These videos serialized with the feed's projection (their channel, this user's watch state, saved), in the order given.""" diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index 9271ea65..23c2f5bd 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -13,10 +13,16 @@ this search-discovered "noise" by default. Shorts and currently-live/upcoming vi ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are dropped from the search response up front). +A video's related videos (`/related/{video_id}`, the list YouTube shows beside it) come in the same +way: read from the website (`related_scrape`), ingested as search finds are, under their own daily +cap — with what this user has already seen left out. + This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the catalog is `catalog/search`.""" -from fastapi import APIRouter, Depends +from typing import Literal + +from fastapi import APIRouter, Depends, Path from pydantic import BaseModel from sqlalchemy.orm import Session @@ -27,7 +33,7 @@ from app.catalog import search as catalog_search from app.db import get_db from app.models import User from app.reader import queries as reader_queries -from app.youtube import search_scrape +from app.youtube import related_scrape, search_scrape from app.youtube.client import YouTubeClient, YouTubeError from app.refusal import refused from app.routes._common import youtube_refusal @@ -38,6 +44,13 @@ router = APIRouter(prefix="/api/search", tags=["search"]) # search.list costs this many quota units per page. SEARCH_COST = 100 +# A related list is filled to this many unseen videos, reading at most this many pages (about 20 +# videos each, measured) — the ones the user has seen are left out, so a page alone can fall short. +RELATED_TARGET = 20 +RELATED_MAX_PAGES = 4 +# The locale YouTube fills a related list in: on the video's topic, or with what trends in Hungary. +RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")} + @router.get("/youtube") def search_youtube( @@ -201,3 +214,78 @@ def clear_search( removed = catalog_search.clear(db, user, ids) db.commit() return {"ok": True, "removed": removed} + + +@router.get("/related/{video_id}") +def related_videos( + video_id: str = Path(max_length=16), + cursor: str | None = None, + region: Literal["topical", "hu"] = "topical", + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + """The videos YouTube shows beside this one, materialised into the catalog like search results, + without the ones this user has watched or hidden here or in their Takeout history, and without + blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads + further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`).""" + daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user") + used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) + if used >= daily_limit: + log.info( + "a related list was refused: the per-user daily cap is spent", + extra=logs.event("related.refused", reason="daily_cap", limit=daily_limit), + ) + raise refused(429, "related_daily_limit", limit=daily_limit) + + hl, gl = RELATED_REGIONS[region] + blocked = queries.blocked_ids(db, user) + collected: list[str] = [] + next_cursor = cursor + with YouTubeClient(db, user) as yt: + for _ in range(RELATED_MAX_PAGES): + try: + page = related_scrape.related(video_id, continuation=next_cursor, hl=hl, gl=gl) + except related_scrape.ScrapeError as exc: + if collected: + break # keep what earlier pages already gathered + log.warning( + "the related-videos scrape failed", + extra=logs.event("related.scrape_failed", error=str(exc)), + ) + raise youtube_refusal(exc) from exc + ids = [it["id"] for it in page["items"]] + drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id} + candidates = [ + it + for it in page["items"] + if it["id"] not in drop and it["channel_id"] not in blocked + ] + # As in the search above: an ingest failure answers 422, not a bare 500. + try: + collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates)) + except Exception as exc: # any ingest failure, not just YouTubeError + db.rollback() + log.exception( + "a related list's ingest failed", + extra=logs.event( + "related.ingest_failed", + candidates=len(candidates), + collected=len(collected), + ), + ) + if collected: + break + raise youtube_refusal(exc) from exc + next_cursor = page["next_page_token"] + if len(collected) >= RELATED_TARGET or not next_cursor: + break + + # One opened list is one event against the cap, however many pages it read; reading further + # (a cursor) is the same list. + if cursor is None: + quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED) + return { + "items": reader_queries.feed_rows_by_ids(db, user, collected), + "next_cursor": next_cursor, + "has_more": bool(next_cursor), + } diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 6ad61490..d5c1adb5 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -140,6 +140,15 @@ SPECS: tuple[ConfigSpec, ...] = ( max=100_000, applies=LIVE, ), + ConfigSpec( + "related_daily_limit_per_user", + "int", + "quota", + "related_daily_limit_per_user", + min=0, + max=100_000, + applies=LIVE, + ), # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE), ConfigSpec( diff --git a/backend/app/youtube/innertube.py b/backend/app/youtube/innertube.py new file mode 100644 index 00000000..54a8083e --- /dev/null +++ b/backend/app/youtube/innertube.py @@ -0,0 +1,111 @@ +"""The website's own InnerTube API (`youtubei/v1/*`), read the way the website reads it — no quota. + +The zero-quota search (`search_scrape`) and the related videos (`related_scrape`) both POST here with +the public WEB key and a client version, read out of a live results page. This deliberately calls an +undocumented endpoint: if YouTube changes the JSON shape or blocks the host, each caller says so as a +`ScrapeError`, never as an empty answer. + +The SOCS=CAI consent-skip cookie is sent like the Shorts probe's, so a server-side request isn't +bounced to consent.youtube.com.""" + +import re + +import httpx + +from app import logs + +log = logs.get_logger(__name__) + +_API_URL = "https://www.youtube.com/youtubei/v1/" +_RESULTS_URL = "https://www.youtube.com/results" +# Public WEB InnerTube key + a recent client version. These are also embedded in every results +# page; we fetch the live pair once and cache it, falling back to these constants. +_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" +_FALLBACK_CLIENT_VERSION = "2.20240101.00.00" + +_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is + # redirected to consent.youtube.com). + "Cookie": "SOCS=CAI", + "Accept-Language": "en-US,en;q=0.9", +} + +_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') +_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') + +# Cached (api_key, client_version) extracted from a results page; refreshed on demand. +_config: tuple[str, str] | None = None + + +class ScrapeError(Exception): + """An InnerTube read could not be completed (network, blocked, or unexpected shape).""" + + +def runs_text(node: dict | None) -> str | None: + """The text of an InnerTube text node: a `simpleText`, or its `runs` joined.""" + if not node: + return None + if node.get("simpleText"): + return node["simpleText"] + runs = node.get("runs") or [] + if runs: + return "".join(r.get("text", "") for r in runs) + return None + + +def _get_config(client: httpx.Client) -> tuple[str, str]: + """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them. + + The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no + keys on it) at the wrong moment pinned every later search to a hard-coded key and version for + the life of the process — and when those went stale, search returned nothing, with no error + anywhere. Now the next call simply tries the live pair again.""" + global _config + if _config is not None: + return _config + try: + html = client.get(_RESULTS_URL, params={"search_query": "music"}).text + except httpx.HTTPError as exc: + log.warning( + "the InnerTube config could not be fetched; falling back to the pinned key", + extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__), + ) + return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) + key = _KEY_RE.search(html) + ver = _VER_RE.search(html) + if key and ver: + _config = (key.group(1), ver.group(1)) + return _config + log.warning( + "the results page carried no InnerTube config; falling back to the pinned key", + extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"), + ) + return ( + key.group(1) if key else _FALLBACK_KEY, + ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, + ) + + +def post(endpoint: str, body: dict, *, hl: str = "en", gl: str = "US") -> dict: + """POST `body` to `youtubei/v1/` as the WEB client in the `hl`/`gl` locale and return + the decoded JSON object. A network failure, an HTTP error or a body that is not a JSON object (a + block, consent or captcha page) raises `ScrapeError`; the caller judges the shape.""" + with httpx.Client(timeout=20.0, headers=_HEADERS) as client: + api_key, client_version = _get_config(client) + context = { + "client": {"clientName": "WEB", "clientVersion": client_version, "hl": hl, "gl": gl} + } + try: + resp = client.post( + _API_URL + endpoint, params={"key": api_key}, json={"context": context, **body} + ) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as exc: + raise ScrapeError(f"InnerTube request failed: {exc}") from exc + except ValueError as exc: # non-JSON body (blocked / consent / captcha) + raise ScrapeError("InnerTube returned a non-JSON response") from exc + if not isinstance(data, dict): + raise ScrapeError("InnerTube returned an unexpected shape") + return data diff --git a/backend/app/youtube/related_scrape.py b/backend/app/youtube/related_scrape.py new file mode 100644 index 00000000..e66a725c --- /dev/null +++ b/backend/app/youtube/related_scrape.py @@ -0,0 +1,150 @@ +"""A video's related videos, read from the watch page's InnerTube `next` endpoint — no quota. + +The Data API has no related-videos call any more, so this is the only source: the same list the +website shows beside a playing video. A page holds about 20 videos and a continuation token for the +next 20. The returned page has the search scrape's shape (``{"items": [...], "next_page_token"}``) +so the catalog ingests it the same way (`catalog.search.ingest_candidates`). + +YouTube draws each item as a `lockupViewModel` now; the older `compactVideoRenderer` is read too, so +a rollback on its side does not empty the list. An item that names no single channel — a video by +several channels draws an avatar stack, not one avatar — is left out: the catalog row needs its +channel up front, and the stack does not say which one uploaded it. + +`hl`/`gl` pick the locale YouTube fills the list in: ``en``/``US`` keeps it on the video's topic, +``hu``/``HU`` mixes in what is trending in Hungary (measured).""" + +from app.youtube.innertube import ScrapeError, post, runs_text + +__all__ = ["ScrapeError", "related"] + +# Where a first page and a continuation page put their item list. +_FIRST = ( + "contents", + "twoColumnWatchNextResults", + "secondaryResults", + "secondaryResults", + "results", +) +_APPENDS = ("appendContinuationItemsAction", "reloadContinuationItemsCommand") + + +def _at(node: object, *path: str | int) -> object: + """The value under `path` in nested dicts/lists, or None where the path stops.""" + for key in path: + if isinstance(node, dict) and isinstance(key, str): + node = node.get(key) + elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node): + node = node[key] + else: + return None + return node + + +def _item(vid: str, channel_id: object, channel_title: object, title: object, sources: object): + if not isinstance(channel_id, str) or not channel_id: + return None + thumbs = sources if isinstance(sources, list) else [] + return { + "id": vid, + "channel_id": channel_id, + "channel_title": channel_title if isinstance(channel_title, str) else None, + "title": title if isinstance(title, str) else None, + "published_at": None, # only a relative "1y ago" here; the enrich fills the real date + "thumbnail_url": _at(thumbs, -1, "url"), + "live_broadcast": "none", # live/upcoming are judged after the enrich, by live_status + } + + +def _lockup(node: dict) -> dict | None: + if node.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO" or not node.get("contentId"): + return None + meta = _at(node, "metadata", "lockupMetadataViewModel") + channel_id = _at( + meta, + "image", + "decoratedAvatarViewModel", + "rendererContext", + "commandContext", + "onTap", + "innertubeCommand", + "browseEndpoint", + "browseId", + ) + channel_title = _at( + meta, "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0 + ) + return _item( + node["contentId"], + channel_id, + _at(channel_title, "text", "content"), + _at(meta, "title", "content"), + _at(node, "contentImage", "thumbnailViewModel", "image", "sources"), + ) + + +def _compact(node: dict) -> dict | None: + vid = node.get("videoId") + if not vid: + return None + byline = _at(node, "longBylineText", "runs", 0) + return _item( + vid, + _at(byline, "navigationEndpoint", "browseEndpoint", "browseId"), + _at(byline, "text"), + runs_text(node.get("title")), + _at(node, "thumbnail", "thumbnails"), + ) + + +def _items(data: dict) -> list | None: + """The page's item list (first page or continuation), or None when it has neither frame.""" + first = _at(data, *_FIRST) + if isinstance(first, list): + return first + # A video YouTube no longer has keeps the watch page's frame but draws an "isn't available" + # promo in it and no related column: nothing related, said by YouTube itself. + if isinstance(_at(data, *_FIRST[:2]), dict): + return [] + for endpoint in data.get("onResponseReceivedEndpoints") or []: + for key in _APPENDS: + items = _at(endpoint, key, "continuationItems") + if isinstance(items, list): + return items + return None + + +def related( + video_id: str, continuation: str | None = None, *, hl: str = "en", gl: str = "US" +) -> dict: + """One page of `video_id`'s related videos (`continuation` = the token a previous page gave). + + A video YouTube no longer has is an empty page. An answer without the watch page's frame is a + `ScrapeError` — not an empty page, which would read as "nothing related".""" + body = {"continuation": continuation} if continuation else {"videoId": video_id} + data = post("next", body, hl=hl, gl=gl) + items = _items(data) + if items is None: + raise ScrapeError("InnerTube returned no related list") + out: list[dict] = [] + seen: set[str] = set() + token = None + for entry in items: + if not isinstance(entry, dict): + continue + if "lockupViewModel" in entry: + item = _lockup(entry["lockupViewModel"]) + elif "compactVideoRenderer" in entry: + item = _compact(entry["compactVideoRenderer"]) + else: + token = token or _at( + entry, + "continuationItemRenderer", + "continuationEndpoint", + "continuationCommand", + "token", + ) + continue + if item and item["id"] not in seen: + seen.add(item["id"]) + out.append(item) + return {"items": out, "next_page_token": token if isinstance(token, str) else None} diff --git a/backend/app/youtube/search_scrape.py b/backend/app/youtube/search_scrape.py index 95dc1b89..e32583e2 100644 --- a/backend/app/youtube/search_scrape.py +++ b/backend/app/youtube/search_scrape.py @@ -6,9 +6,9 @@ calls — and read the result list straight out of the JSON. The returned video through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does, so the search itself is effectively free and only the (cheap, shared) enrichment is charged. -The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't -bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube -changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``. +The request itself (the live key, the consent cookie) is `innertube`'s. This deliberately scrapes an +undocumented endpoint: if YouTube changes the JSON shape or blocks the host, an admin can flip +``search_source`` back to ``api``. The returned page mirrors :meth:`YouTubeClient.search_videos` exactly (``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the @@ -17,87 +17,16 @@ only difference is the cursor is an InnerTube *continuation token*, not a Data A the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"`` and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path).""" -import re +from app.youtube.innertube import ScrapeError, post, runs_text -import httpx +__all__ = ["ScrapeError", "search"] -from app import logs - -log = logs.get_logger(__name__) - -_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search" -_RESULTS_URL = "https://www.youtube.com/results" -# Public WEB InnerTube key + a recent client version. These are also embedded in every results -# page; we fetch the live pair once and cache it, falling back to these constants. -_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" -_FALLBACK_CLIENT_VERSION = "2.20240101.00.00" # sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==). _TYPE_VIDEO = "EgIQAQ==" -_HEADERS = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", - # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is - # redirected to consent.youtube.com). - "Cookie": "SOCS=CAI", - "Accept-Language": "en-US,en;q=0.9", -} - # The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands) # put their result list under. _RESULT_FRAMES = {"contents", "onResponseReceivedCommands"} -_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') -_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') - -# Cached (api_key, client_version) extracted from a results page; refreshed on demand. -_config: tuple[str, str] | None = None - - -class ScrapeError(Exception): - """A scrape-based search could not be completed (network, blocked, or unexpected shape).""" - - -def _runs_text(node: dict | None) -> str | None: - if not node: - return None - if node.get("simpleText"): - return node["simpleText"] - runs = node.get("runs") or [] - if runs: - return "".join(r.get("text", "") for r in runs) - return None - - -def _get_config(client: httpx.Client) -> tuple[str, str]: - """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them. - - The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no - keys on it) at the wrong moment pinned every later search to a hard-coded key and version for - the life of the process — and when those went stale, search returned nothing, with no error - anywhere. Now the next search simply tries the live pair again.""" - global _config - if _config is not None: - return _config - try: - html = client.get(_RESULTS_URL, params={"search_query": "music"}).text - except httpx.HTTPError as exc: - log.warning( - "the InnerTube config could not be fetched; falling back to the pinned key", - extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__), - ) - return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) - key = _KEY_RE.search(html) - ver = _VER_RE.search(html) - if key and ver: - _config = (key.group(1), ver.group(1)) - return _config - log.warning( - "the results page carried no InnerTube config; falling back to the pinned key", - extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"), - ) - return ( - key.group(1) if key else _FALLBACK_KEY, - ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, - ) def _video_item(vr: dict) -> dict | None: @@ -116,7 +45,7 @@ def _video_item(vr: dict) -> dict | None: "id": vid, "channel_id": channel_id, "channel_title": channel_title, - "title": _runs_text(vr.get("title")), + "title": runs_text(vr.get("title")), "published_at": None, # scrape exposes only a relative time; enrich fills the real date "thumbnail_url": thumbs[-1]["url"] if thumbs else None, "live_broadcast": "none", # live/upcoming filtered post-enrich via live_status @@ -167,34 +96,14 @@ def search(q: str, continuation: str | None = None) -> dict: `continuation` is the InnerTube continuation token from a previous page (None for the first page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as YouTubeClient.search_videos so the route handles both sources identically.""" - with httpx.Client(timeout=20.0, headers=_HEADERS) as client: - api_key, client_version = _get_config(client) - context = { - "client": { - "clientName": "WEB", - "clientVersion": client_version, - "hl": "en", - "gl": "US", - } - } - body: dict = {"context": context} - if continuation: - body["continuation"] = continuation - else: - body["query"] = q - body["params"] = _TYPE_VIDEO - try: - resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body) - resp.raise_for_status() - data = resp.json() - except httpx.HTTPError as exc: - raise ScrapeError(f"InnerTube request failed: {exc}") from exc - except ValueError as exc: # non-JSON body (blocked / consent / captcha) - raise ScrapeError("InnerTube returned a non-JSON response") from exc + body: dict = ( + {"continuation": continuation} if continuation else {"query": q, "params": _TYPE_VIDEO} + ) + data = post("search", body) # Even a search with no results carries its result list's frame. Without one this is not "no # results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so. - if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys(): + if not _RESULT_FRAMES & data.keys(): raise ScrapeError("InnerTube returned an unexpected shape") items: list[dict] = [] _collect_videos(data, items, set()) diff --git a/backend/tests/fixtures/innertube_next_continuation.json b/backend/tests/fixtures/innertube_next_continuation.json new file mode 100644 index 00000000..cd78025d --- /dev/null +++ b/backend/tests/fixtures/innertube_next_continuation.json @@ -0,0 +1,527 @@ +{ + "onResponseReceivedEndpoints": [ + { + "appendContinuationItemsAction": { + "continuationItems": [ + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/_HfdncCbMOE/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLARvB39pkdsiuUUQTsZjkhvbKrfcQ", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/_HfdncCbMOE/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLB15aZU5-tzwYCEbNTnZK3UX6ZCzQ", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "1:08:18", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "_HfdncCbMOE", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2025): Large Language Models (Liquid AI)" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "62K" + }, + "accessibilityLabel": "62 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "1y ago" + }, + "accessibilityLabel": "1 year ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "_HfdncCbMOE", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4294964720, + "raisedBackground": 4294964720, + "additiveBackground": 566850118, + "textPrimary": 4281800969, + "textSecondary": 4285876787, + "outline": 860759827, + "keyLight": 4292983720, + "collectionStack": 4291337092, + "invertedBackground": 4287578649, + "textPrimaryInverse": 4294964720, + "textDisabled": 2135828243, + "dropShadow": 432632390, + "cardOutline": 180974150, + "overlayBackground": 3426162953, + "overlayTextPrimary": 4294964720, + "solidWash": 4283311891, + "solidRim": 4281800969, + "overlaySolidWash": 4294699488, + "overlaySolidRim": 4294964720, + "tonalWash": 2583688688, + "tonalRim": 202310912, + "overlayTonalWash": 436205040, + "overlayTonalRim": 436205040, + "outlineRim": 1280190227, + "overlayOutlineRim": 871618504, + "solidWashInverse": 4294170568, + "solidRimInverse": 2147215840, + "tonalWashInverse": 654308848, + "tonalRimInverse": 654043616, + "outlineRimInverse": 654308848, + "solidBackground": 4283311891, + "solidBackgroundInverse": 4291337092, + "tonalBackground": 868785028, + "tonalBackgroundInverse": 435410888, + "outlineInverse": 872147424, + "directionalLightStopPrimary": 4294699488, + "directionalLightStopSecondary": 4294964720, + "primary": 4279174400, + "filledStateLayer": 4281800969, + "tonalTouchLight": 4294964720, + "tonalHoverLight": 3439061472, + "tonalStateLayer": 420414720, + "overlayDisabledContainer": 871618504, + "overlayTonalTouchLight": 1291577824, + "overlayTonalHoverLight": 1291577824, + "overlayTonalStateLayer": 436205040, + "disabledContainer": 420414720, + "filledTouchLight": 4285876787, + "filledHoverLight": 4284627235, + "onOverlayDisabledContainer": 2583688688, + "overlayTonalContainer": 2583423456, + "overlayOutline": 872412656, + "onFilledContainer": 4294964720, + "filledContainer": 4280027906, + "tonalContainer": 641145097, + "overlayPrimary": 4294964720, + "onOverlayFilledContainer": 4279174400, + "overlayFilledTouchLight": 4294964720, + "overlayFilledHoverLight": 4294699488, + "overlayFilledStateLayer": 4292983720, + "overlayFilledContainer": 4294170568, + "onDisabledContainer": 1712260352, + "temporaryOutline": 860759827 + }, + "darkThemePalette": { + "baseBackground": 4280027906, + "raisedBackground": 4281800969, + "additiveBackground": 734622278, + "textPrimary": 4294699488, + "textSecondary": 4292983720, + "outline": 1072945096, + "keyLight": 4284627235, + "collectionStack": 4285876787, + "invertedBackground": 4293107837, + "textPrimaryInverse": 4280027906, + "textDisabled": 2146686920, + "dropShadow": 432632390, + "cardOutline": 180974150, + "overlayBackground": 3426162953, + "overlayTextPrimary": 4294964720, + "solidWash": 4294699488, + "solidRim": 4294964720, + "overlaySolidWash": 4294699488, + "overlaySolidRim": 4294964720, + "tonalWash": 217307080, + "tonalRim": 218101232, + "overlayTonalWash": 436205040, + "overlayTonalRim": 436205040, + "outlineRim": 871618504, + "overlayOutlineRim": 871618504, + "solidWashInverse": 4285876787, + "solidRimInverse": 871618504, + "tonalWashInverse": 872412656, + "tonalRimInverse": 870431656, + "outlineRimInverse": 1291843056, + "solidBackground": 4291337092, + "solidBackgroundInverse": 4283311891, + "tonalBackground": 435410888, + "tonalBackgroundInverse": 868785028, + "outlineInverse": 860759827, + "directionalLightStopPrimary": 4281800969, + "directionalLightStopSecondary": 4280027906, + "primary": 4294964720, + "filledStateLayer": 4292983720, + "tonalTouchLight": 1073474016, + "tonalHoverLight": 654043616, + "tonalStateLayer": 436205040, + "overlayDisabledContainer": 871618504, + "overlayTonalTouchLight": 1291577824, + "overlayTonalHoverLight": 1291577824, + "overlayTonalStateLayer": 436205040, + "disabledContainer": 436205040, + "filledTouchLight": 4294964720, + "filledHoverLight": 4294699488, + "onOverlayDisabledContainer": 2583688688, + "overlayTonalContainer": 2583423456, + "overlayOutline": 872412656, + "onFilledContainer": 4279174400, + "filledContainer": 4294170568, + "tonalContainer": 435410888, + "overlayPrimary": 4294964720, + "onOverlayFilledContainer": 4279174400, + "overlayFilledTouchLight": 4294964720, + "overlayFilledHoverLight": 4294699488, + "overlayFilledStateLayer": 4292983720, + "overlayFilledContainer": 4294170568, + "onDisabledContainer": 1728050672, + "temporaryOutline": 871618504 + } + } + } + }, + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/P7Hkh2zOGQ0/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLAHInNtMierSAgtrGl5rLSKc-vdzQ", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/P7Hkh2zOGQ0/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDXrm5g-rdjRIqqK309ImTdqpw4CA", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "1:08:19", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "P7Hkh2zOGQ0", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2024): Google - Generative AI for Media" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "49K" + }, + "accessibilityLabel": "49 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "2y ago" + }, + "accessibilityLabel": "2 years ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "P7Hkh2zOGQ0", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4293981951, + "raisedBackground": 4293981951, + "additiveBackground": 558201801, + "textPrimary": 4278655801, + "textSecondary": 4280831104, + "outline": 856631890, + "keyLight": 4288855783, + "collectionStack": 4286094803, + "invertedBackground": 4279847311, + "textPrimaryInverse": 4293981951, + "textDisabled": 2131700306, + "dropShadow": 423984073, + "cardOutline": 172325833, + "overlayBackground": 3423017785, + "overlayTextPrimary": 4293981951, + "solidWash": 4279183954, + "solidRim": 4278655801, + "overlaySolidWash": 4292864764, + "overlaySolidRim": 4293981951, + "tonalWash": 2582705919, + "tonalRim": 201328143, + "overlayTonalWash": 435222271, + "overlayTonalRim": 435222271, + "outlineRim": 1276062290, + "overlayOutlineRim": 868669942, + "solidWashInverse": 4291222006, + "solidRimInverse": 2145381116, + "tonalWashInverse": 653326079, + "tonalRimInverse": 652208892, + "outlineRimInverse": 653326079, + "solidBackground": 4279183954, + "solidBackgroundInverse": 4286094803, + "tonalBackground": 863542739, + "tonalBackgroundInverse": 432462326, + "outlineInverse": 870312700, + "directionalLightStopPrimary": 4292864764, + "directionalLightStopSecondary": 4293981951, + "primary": 4278191631, + "filledStateLayer": 4278655801, + "tonalTouchLight": 4293981951, + "tonalHoverLight": 3437226748, + "tonalStateLayer": 419431951, + "overlayDisabledContainer": 868669942, + "overlayTonalTouchLight": 1289743100, + "overlayTonalHoverLight": 1289743100, + "overlayTonalStateLayer": 435222271, + "disabledContainer": 419431951, + "filledTouchLight": 4280831104, + "filledHoverLight": 4280040297, + "onOverlayDisabledContainer": 2582705919, + "overlayTonalContainer": 2581588732, + "overlayOutline": 871429887, + "onFilledContainer": 4293981951, + "filledContainer": 4278324509, + "tonalContainer": 637999929, + "overlayPrimary": 4293981951, + "onOverlayFilledContainer": 4278191631, + "overlayFilledTouchLight": 4293981951, + "overlayFilledHoverLight": 4292864764, + "overlayFilledStateLayer": 4288855783, + "overlayFilledContainer": 4291222006, + "onDisabledContainer": 1711277583, + "temporaryOutline": 856631890 + }, + "darkThemePalette": { + "baseBackground": 4278324509, + "raisedBackground": 4278655801, + "additiveBackground": 725973961, + "textPrimary": 4292864764, + "textSecondary": 4288855783, + "outline": 1069996534, + "keyLight": 4280040297, + "collectionStack": 4280831104, + "invertedBackground": 4286359267, + "textPrimaryInverse": 4278324509, + "textDisabled": 2143738358, + "dropShadow": 423984073, + "cardOutline": 172325833, + "overlayBackground": 3423017785, + "overlayTextPrimary": 4293981951, + "solidWash": 4292864764, + "solidRim": 4293981951, + "overlaySolidWash": 4292864764, + "overlaySolidRim": 4293981951, + "tonalWash": 214358518, + "tonalRim": 217118463, + "overlayTonalWash": 435222271, + "overlayTonalRim": 435222271, + "outlineRim": 868669942, + "overlayOutlineRim": 868669942, + "solidWashInverse": 4280831104, + "solidRimInverse": 868669942, + "tonalWashInverse": 871429887, + "tonalRimInverse": 866303719, + "outlineRimInverse": 1290860287, + "solidBackground": 4286094803, + "solidBackgroundInverse": 4279183954, + "tonalBackground": 432462326, + "tonalBackgroundInverse": 863542739, + "outlineInverse": 856631890, + "directionalLightStopPrimary": 4278655801, + "directionalLightStopSecondary": 4278324509, + "primary": 4293981951, + "filledStateLayer": 4288855783, + "tonalTouchLight": 1071639292, + "tonalHoverLight": 652208892, + "tonalStateLayer": 435222271, + "overlayDisabledContainer": 868669942, + "overlayTonalTouchLight": 1289743100, + "overlayTonalHoverLight": 1289743100, + "overlayTonalStateLayer": 435222271, + "disabledContainer": 435222271, + "filledTouchLight": 4293981951, + "filledHoverLight": 4292864764, + "onOverlayDisabledContainer": 2582705919, + "overlayTonalContainer": 2581588732, + "overlayOutline": 871429887, + "onFilledContainer": 4278191631, + "filledContainer": 4291222006, + "tonalContainer": 432462326, + "overlayPrimary": 4293981951, + "onOverlayFilledContainer": 4278191631, + "overlayFilledTouchLight": 4293981951, + "overlayFilledHoverLight": 4292864764, + "overlayFilledStateLayer": 4288855783, + "overlayFilledContainer": 4291222006, + "onDisabledContainer": 1727067903, + "temporaryOutline": 868669942 + } + } + } + }, + { + "continuationItemRenderer": { + "trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN", + "continuationEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "sendPost": true, + "apiUrl": "/youtubei/v1/next" + } + }, + "continuationCommand": { + "token": "CCgSDRILU2RUWkFNREtyTlkYACqPDE1nb0lodVQ4eG9EdXVLSUYyczZMNnd6NENBcjFDQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhQ2dQeVBnQUtEc0ktQ3dqaDRleUUzTFAzdV93QkNnUHlQZ0FLRGNJLUNnaU5zcmptOXBENTJEOEtBX0ktQUFvTndqNEtDTExudjR5ZXg3U1dTUW9EOGo0QUNnM0NQZ29JNnZHODhaR1JpTWNnQ2dQeVBnQUtEc0ktQ3dpRF85S3Z5WS1hbThJQkNnUHlQZ0FLRHNJLUN3alppNmJYbWR6YWxZb0JDZ1B5UGdBS0RzSS1Dd2lkemJfS2dPdWN0YUFCQ2dQeVBnQUtEY0ktQ2dqMXktMjYyb2JQeFdNS0FfSS1BQW9Pd2o0TENPLUE4SWVaMUlESXJRRUtBX0ktQUFvTndqNEtDUHlZNDVhUzhkeWpJZ29EOGo0QUNnM0NQZ29Jb191aWtfSE05ZHdTQ2dQeVBnQUtEc0ktQ3dqaTBiemlrdnpBbkxvQkNnUHlQZ0FLRHNJLUN3aW05cXI1eXNtTWlzOEJDZ1B5UGdBS0RjSS1DZ2k3M2V5NXlzQ1pzMG9LQV9JLUFBb093ajRMQ0xmT281WFU1X2JPbFFFS0FfSS1BQW9Od2o0S0NLTEVwYXlXOTliWkdnb0Q4ajRBQ2c3Q1Bnc0k2dWZSMnRPM3hLem9BUW9EOGo0QUNnN0NQZ3NJN2ZlZXZQRHV2WlRXQVFvRDhqNEFDZzNDUGdvSXFibS0zYXVBdDVWcUNnUHlQZ0FLRGNJLUNnaUc1UHpHZ082NG9nVVNLQUFDQkFZSUNnd09FQklVRmhnYUhCNGdJaVFtS0Nvc0xqQXlORFk0T2p3LVFFSkVSa2hLVEU0YUJBZ0FFQUVhQkFnQ0VBTWFCQWdFRUFVYUJBZ0dFQWNhQkFnSUVBa2FCQWdLRUFzYUJBZ01FQTBhQkFnT0VBOGFCQWdRRUJFYUJBZ1NFQk1hQkFnVUVCVWFCQWdXRUJjYUJBZ1lFQmthQkFnYUVCc2FCQWdjRUIwYUJBZ2VFQjhhQkFnZ0VDRWFCQWdpRUNNYUJBZ2tFQ1VhQkFnbUVDY2FCQWdvRUNrYUJBZ3FFQ3NhQkFnc0VDMGFCQWd1RUM4YUJBZ3dFREVhQkFneUVETWFCQWcwRURVYUJBZzJFRGNhQkFnNEVEa2FCQWc2RURzYUJBZzhFRDBhQkFnLUVEOGFCQWhBRUVFYUJBaENFRU1hQkFoRUVFVWFCQWhHRUVjYUJBaElFRWthQkFoS0VFc2FCQWhNRUUwYUJBaE9FRThxS0FBQ0JBWUlDZ3dPRUJJVUZoZ2FIQjRnSWlRbUtDb3NMakF5TkRZNE9qdy1RRUpFUmtoS1RFNGoPd2F0Y2gtbmV4dC1mZWVk", + "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" + } + } + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/backend/tests/fixtures/innertube_next_first.json b/backend/tests/fixtures/innertube_next_first.json new file mode 100644 index 00000000..dd31a462 --- /dev/null +++ b/backend/tests/fixtures/innertube_next_first.json @@ -0,0 +1,805 @@ +{ + "contents": { + "twoColumnWatchNextResults": { + "secondaryResults": { + "secondaryResults": { + "results": [ + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/to-lHJfK4pw/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLBKP_oFChvoqvwVrKAy_PG9bn2_AA", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/to-lHJfK4pw/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDyH3Z8DJhXMmFJ5eKEWMufk7ql9g", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "1:02:00", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "to-lHJfK4pw", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2025): Reinforcement Learning" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "128K" + }, + "accessibilityLabel": "128 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "1y ago" + }, + "accessibilityLabel": "1 year ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "to-lHJfK4pw", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4293981439, + "raisedBackground": 4293981439, + "additiveBackground": 558196425, + "textPrimary": 4278850870, + "textSecondary": 4282009454, + "outline": 857088843, + "keyLight": 4289509597, + "collectionStack": 4287338944, + "invertedBackground": 4279842703, + "textPrimaryInverse": 4293981439, + "textDisabled": 2132157259, + "dropShadow": 423978697, + "cardOutline": 172320457, + "overlayBackground": 3423212854, + "overlayTextPrimary": 4293981439, + "solidWash": 4279640907, + "solidRim": 4278850870, + "overlaySolidWash": 4292929530, + "overlaySolidRim": 4293981439, + "tonalWash": 2582705407, + "tonalRim": 201327631, + "overlayTonalWash": 435221759, + "overlayTonalRim": 435221759, + "outlineRim": 1276519243, + "overlayOutlineRim": 868930802, + "solidWashInverse": 4291482866, + "solidRimInverse": 2145445882, + "tonalWashInverse": 653325567, + "tonalRimInverse": 652273658, + "outlineRimInverse": 653325567, + "solidBackground": 4279640907, + "solidBackgroundInverse": 4287338944, + "tonalBackground": 864786880, + "tonalBackgroundInverse": 432723186, + "outlineInverse": 870377466, + "directionalLightStopPrimary": 4292929530, + "directionalLightStopSecondary": 4293981439, + "primary": 4278191119, + "filledStateLayer": 4278850870, + "tonalTouchLight": 4293981439, + "tonalHoverLight": 3437291514, + "tonalStateLayer": 419431439, + "overlayDisabledContainer": 868930802, + "overlayTonalTouchLight": 1289807866, + "overlayTonalHoverLight": 1289807866, + "overlayTonalStateLayer": 435221759, + "disabledContainer": 419431439, + "filledTouchLight": 4282009454, + "filledHoverLight": 4280825181, + "onOverlayDisabledContainer": 2582705407, + "overlayTonalContainer": 2581653498, + "overlayOutline": 871429375, + "onFilledContainer": 4293981439, + "filledContainer": 4278323484, + "tonalContainer": 638194998, + "overlayPrimary": 4293981439, + "onOverlayFilledContainer": 4278191119, + "overlayFilledTouchLight": 4293981439, + "overlayFilledHoverLight": 4292929530, + "overlayFilledStateLayer": 4289509597, + "overlayFilledContainer": 4291482866, + "onDisabledContainer": 1711277071, + "temporaryOutline": 857088843 + }, + "darkThemePalette": { + "baseBackground": 4278323484, + "raisedBackground": 4278850870, + "additiveBackground": 725968585, + "textPrimary": 4292929530, + "textSecondary": 4289509597, + "outline": 1070257394, + "keyLight": 4280825181, + "collectionStack": 4282009454, + "invertedBackground": 4286355427, + "textPrimaryInverse": 4278323484, + "textDisabled": 2143999218, + "dropShadow": 423978697, + "cardOutline": 172320457, + "overlayBackground": 3423212854, + "overlayTextPrimary": 4293981439, + "solidWash": 4292929530, + "solidRim": 4293981439, + "overlaySolidWash": 4292929530, + "overlaySolidRim": 4293981439, + "tonalWash": 214619378, + "tonalRim": 217117951, + "overlayTonalWash": 435221759, + "overlayTonalRim": 435221759, + "outlineRim": 868930802, + "overlayOutlineRim": 868930802, + "solidWashInverse": 4282009454, + "solidRimInverse": 868930802, + "tonalWashInverse": 871429375, + "tonalRimInverse": 866957533, + "outlineRimInverse": 1290859775, + "solidBackground": 4287338944, + "solidBackgroundInverse": 4279640907, + "tonalBackground": 432723186, + "tonalBackgroundInverse": 864786880, + "outlineInverse": 857088843, + "directionalLightStopPrimary": 4278850870, + "directionalLightStopSecondary": 4278323484, + "primary": 4293981439, + "filledStateLayer": 4289509597, + "tonalTouchLight": 1071704058, + "tonalHoverLight": 652273658, + "tonalStateLayer": 435221759, + "overlayDisabledContainer": 868930802, + "overlayTonalTouchLight": 1289807866, + "overlayTonalHoverLight": 1289807866, + "overlayTonalStateLayer": 435221759, + "disabledContainer": 435221759, + "filledTouchLight": 4293981439, + "filledHoverLight": 4292929530, + "onOverlayDisabledContainer": 2582705407, + "overlayTonalContainer": 2581653498, + "overlayOutline": 871429375, + "onFilledContainer": 4278191119, + "filledContainer": 4291482866, + "tonalContainer": 432723186, + "overlayPrimary": 4293981439, + "onOverlayFilledContainer": 4278191119, + "overlayFilledTouchLight": 4293981439, + "overlayFilledHoverLight": 4292929530, + "overlayFilledStateLayer": 4289509597, + "overlayFilledContainer": 4291482866, + "onDisabledContainer": 1727067391, + "temporaryOutline": 868930802 + } + } + } + }, + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/R8V8CbuxryI/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLDqXMjrpgDyQKHL2lWrxczwxKoefw", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/R8V8CbuxryI/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLB7JX9Z238s6DGqIS5-fhcEIkUxKw", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "49:36", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "R8V8CbuxryI", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191: Deep Generative Modeling" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "38K" + }, + "accessibilityLabel": "38 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "6mo ago" + }, + "accessibilityLabel": "6 months ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "R8V8CbuxryI", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4294965488, + "raisedBackground": 4294965488, + "additiveBackground": 566856261, + "textPrimary": 4281737226, + "textSecondary": 4285420858, + "outline": 860565782, + "keyLight": 4292724140, + "collectionStack": 4290815883, + "invertedBackground": 4287583769, + "textPrimaryInverse": 4294965488, + "textDisabled": 2135634198, + "dropShadow": 432638533, + "cardOutline": 180980293, + "overlayBackground": 3426099210, + "overlayTextPrimary": 4294965488, + "solidWash": 4283117846, + "solidRim": 4281737226, + "overlaySolidWash": 4294635232, + "overlaySolidRim": 4294965488, + "tonalWash": 2583689456, + "tonalRim": 202311680, + "overlayTonalWash": 436205808, + "overlayTonalRim": 436205808, + "outlineRim": 1279996182, + "overlayOutlineRim": 871554762, + "solidWashInverse": 4294106826, + "solidRimInverse": 2147151584, + "tonalWashInverse": 654309616, + "tonalRimInverse": 653979360, + "outlineRimInverse": 654309616, + "solidBackground": 4283117846, + "solidBackgroundInverse": 4290815883, + "tonalBackground": 868263819, + "tonalBackgroundInverse": 435347146, + "outlineInverse": 872083168, + "directionalLightStopPrimary": 4294635232, + "directionalLightStopSecondary": 4294965488, + "primary": 4279175168, + "filledStateLayer": 4281737226, + "tonalTouchLight": 4294965488, + "tonalHoverLight": 3438997216, + "tonalStateLayer": 420415488, + "overlayDisabledContainer": 871554762, + "overlayTonalTouchLight": 1291513568, + "overlayTonalHoverLight": 1291513568, + "overlayTonalStateLayer": 436205808, + "disabledContainer": 420415488, + "filledTouchLight": 4285420858, + "filledHoverLight": 4284302120, + "onOverlayDisabledContainer": 2583689456, + "overlayTonalContainer": 2583359200, + "overlayOutline": 872413424, + "onFilledContainer": 4294965488, + "filledContainer": 4280029186, + "tonalContainer": 641081354, + "overlayPrimary": 4294965488, + "onOverlayFilledContainer": 4279175168, + "overlayFilledTouchLight": 4294965488, + "overlayFilledHoverLight": 4294635232, + "overlayFilledStateLayer": 4292724140, + "overlayFilledContainer": 4294106826, + "onDisabledContainer": 1712261120, + "temporaryOutline": 860565782 + }, + "darkThemePalette": { + "baseBackground": 4280029186, + "raisedBackground": 4281737226, + "additiveBackground": 734628421, + "textPrimary": 4294635232, + "textSecondary": 4292724140, + "outline": 1072881354, + "keyLight": 4284302120, + "collectionStack": 4285420858, + "invertedBackground": 4293112700, + "textPrimaryInverse": 4280029186, + "textDisabled": 2146623178, + "dropShadow": 432638533, + "cardOutline": 180980293, + "overlayBackground": 3426099210, + "overlayTextPrimary": 4294965488, + "solidWash": 4294635232, + "solidRim": 4294965488, + "overlaySolidWash": 4294635232, + "overlaySolidRim": 4294965488, + "tonalWash": 217243338, + "tonalRim": 218102000, + "overlayTonalWash": 436205808, + "overlayTonalRim": 436205808, + "outlineRim": 871554762, + "overlayOutlineRim": 871554762, + "solidWashInverse": 4285420858, + "solidRimInverse": 871554762, + "tonalWashInverse": 872413424, + "tonalRimInverse": 870172076, + "outlineRimInverse": 1291843824, + "solidBackground": 4290815883, + "solidBackgroundInverse": 4283117846, + "tonalBackground": 435347146, + "tonalBackgroundInverse": 868263819, + "outlineInverse": 860565782, + "directionalLightStopPrimary": 4281737226, + "directionalLightStopSecondary": 4280029186, + "primary": 4294965488, + "filledStateLayer": 4292724140, + "tonalTouchLight": 1073409760, + "tonalHoverLight": 653979360, + "tonalStateLayer": 436205808, + "overlayDisabledContainer": 871554762, + "overlayTonalTouchLight": 1291513568, + "overlayTonalHoverLight": 1291513568, + "overlayTonalStateLayer": 436205808, + "disabledContainer": 436205808, + "filledTouchLight": 4294965488, + "filledHoverLight": 4294635232, + "onOverlayDisabledContainer": 2583689456, + "overlayTonalContainer": 2583359200, + "overlayOutline": 872413424, + "onFilledContainer": 4279175168, + "filledContainer": 4294106826, + "tonalContainer": 435347146, + "overlayPrimary": 4294965488, + "onOverlayFilledContainer": 4279175168, + "overlayFilledTouchLight": 4294965488, + "overlayFilledHoverLight": 4294635232, + "overlayFilledStateLayer": 4292724140, + "overlayFilledContainer": 4294106826, + "onDisabledContainer": 1728051440, + "temporaryOutline": 871554762 + } + } + } + }, + { + "lockupViewModel": { + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/HLKo4fJx_7k/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLAQcvMQEX1ny9Z8qu2mIMogpEMaVA", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/HLKo4fJx_7k/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLCkoPSkHifs5ifpWXXcb96XG_Ly1w", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "47:36", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + "animationActivationTargetId": "HLKo4fJx_7k", + "animatedText": "Now playing", + "animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE" + } + } + ] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "MIT 6.S191 (2025): Language Models and New Frontiers" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel Alexander Amini", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "browseEndpoint": { + "browseId": "UCtslD4DGH6PKyG_1gFAX7sg" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "Alexander Amini" + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "63K" + }, + "accessibilityLabel": "63 thousand views", + "leadingIcon": { + "name": "PLAY_ARROW_OUTLINED", + "height": 12, + "width": 12, + "marginEndToken": { + "spacing": 2 + } + } + }, + { + "text": { + "content": "1y ago" + }, + "accessibilityLabel": "1 year ago" + } + ] + } + ], + "delimiter": " " + } + } + } + }, + "contentId": "HLKo4fJx_7k", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "themedPalette": { + "lightThemePalette": { + "baseBackground": 4294964208, + "raisedBackground": 4294964208, + "additiveBackground": 566847045, + "textPrimary": 4281734410, + "textSecondary": 4285417018, + "outline": 860561942, + "keyLight": 4292720812, + "collectionStack": 4290812043, + "invertedBackground": 4287575065, + "textPrimaryInverse": 4294964208, + "textDisabled": 2135630358, + "dropShadow": 432629317, + "cardOutline": 180971077, + "overlayBackground": 3426096394, + "overlayTextPrimary": 4294964208, + "solidWash": 4283114006, + "solidRim": 4281734410, + "overlaySolidWash": 4294633184, + "overlaySolidRim": 4294964208, + "tonalWash": 2583688176, + "tonalRim": 202310656, + "overlayTonalWash": 436204528, + "overlayTonalRim": 436204528, + "outlineRim": 1279992342, + "overlayOutlineRim": 871551946, + "solidWashInverse": 4294104010, + "solidRimInverse": 2147149536, + "tonalWashInverse": 654308336, + "tonalRimInverse": 653977312, + "outlineRimInverse": 654308336, + "solidBackground": 4283114006, + "solidBackgroundInverse": 4290812043, + "tonalBackground": 868259979, + "tonalBackgroundInverse": 435344330, + "outlineInverse": 872081120, + "directionalLightStopPrimary": 4294633184, + "directionalLightStopSecondary": 4294964208, + "primary": 4279174144, + "filledStateLayer": 4281734410, + "tonalTouchLight": 4294964208, + "tonalHoverLight": 3438995168, + "tonalStateLayer": 420414464, + "overlayDisabledContainer": 871551946, + "overlayTonalTouchLight": 1291511520, + "overlayTonalHoverLight": 1291511520, + "overlayTonalStateLayer": 436204528, + "disabledContainer": 420414464, + "filledTouchLight": 4285417018, + "filledHoverLight": 4284298280, + "onOverlayDisabledContainer": 2583688176, + "overlayTonalContainer": 2583357152, + "overlayOutline": 872412144, + "onFilledContainer": 4294964208, + "filledContainer": 4280027394, + "tonalContainer": 641078538, + "overlayPrimary": 4294964208, + "onOverlayFilledContainer": 4279174144, + "overlayFilledTouchLight": 4294964208, + "overlayFilledHoverLight": 4294633184, + "overlayFilledStateLayer": 4292720812, + "overlayFilledContainer": 4294104010, + "onDisabledContainer": 1712260096, + "temporaryOutline": 860561942 + }, + "darkThemePalette": { + "baseBackground": 4280027394, + "raisedBackground": 4281734410, + "additiveBackground": 734619205, + "textPrimary": 4294633184, + "textSecondary": 4292720812, + "outline": 1072878538, + "keyLight": 4284298280, + "collectionStack": 4285417018, + "invertedBackground": 4293105020, + "textPrimaryInverse": 4280027394, + "textDisabled": 2146620362, + "dropShadow": 432629317, + "cardOutline": 180971077, + "overlayBackground": 3426096394, + "overlayTextPrimary": 4294964208, + "solidWash": 4294633184, + "solidRim": 4294964208, + "overlaySolidWash": 4294633184, + "overlaySolidRim": 4294964208, + "tonalWash": 217240522, + "tonalRim": 218100720, + "overlayTonalWash": 436204528, + "overlayTonalRim": 436204528, + "outlineRim": 871551946, + "overlayOutlineRim": 871551946, + "solidWashInverse": 4285417018, + "solidRimInverse": 871551946, + "tonalWashInverse": 872412144, + "tonalRimInverse": 870168748, + "outlineRimInverse": 1291842544, + "solidBackground": 4290812043, + "solidBackgroundInverse": 4283114006, + "tonalBackground": 435344330, + "tonalBackgroundInverse": 868259979, + "outlineInverse": 860561942, + "directionalLightStopPrimary": 4281734410, + "directionalLightStopSecondary": 4280027394, + "primary": 4294964208, + "filledStateLayer": 4292720812, + "tonalTouchLight": 1073407712, + "tonalHoverLight": 653977312, + "tonalStateLayer": 436204528, + "overlayDisabledContainer": 871551946, + "overlayTonalTouchLight": 1291511520, + "overlayTonalHoverLight": 1291511520, + "overlayTonalStateLayer": 436204528, + "disabledContainer": 436204528, + "filledTouchLight": 4294964208, + "filledHoverLight": 4294633184, + "onOverlayDisabledContainer": 2583688176, + "overlayTonalContainer": 2583357152, + "overlayOutline": 872412144, + "onFilledContainer": 4279174144, + "filledContainer": 4294104010, + "tonalContainer": 435344330, + "overlayPrimary": 4294964208, + "onOverlayFilledContainer": 4279174144, + "overlayFilledTouchLight": 4294964208, + "overlayFilledHoverLight": 4294633184, + "overlayFilledStateLayer": 4292720812, + "overlayFilledContainer": 4294104010, + "onDisabledContainer": 1728050160, + "temporaryOutline": 871551946 + } + } + } + }, + { + "continuationItemRenderer": { + "trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN", + "continuationEndpoint": { + "commandMetadata": { + "webCommandMetadata": { + "sendPost": true, + "apiUrl": "/youtubei/v1/next" + } + }, + "continuationCommand": { + "token": "CBQSDRILU2RUWkFNREtyTlkYACqXBk1nb0l5ZGl1dXRUNDdQc2EyczZMNnd5LUJBcTdCQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D", + "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" + } + }, + "button": { + "buttonRenderer": { + "style": "STYLE_SUGGESTIVE", + "size": "SIZE_DEFAULT", + "isDisabled": false, + "text": { + "runs": [ + { + "text": "Show more" + } + ] + }, + "command": { + "commandMetadata": { + "webCommandMetadata": { + "sendPost": true, + "apiUrl": "/youtubei/v1/next" + } + }, + "continuationCommand": { + "token": "CBQSDRILU2RUWkFNREtyTlkYACqXBk1nb0l5ZGl1dXRUNDdQc2EyczZMNnd5LUJBcTdCQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D", + "request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT" + } + } + } + } + } + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/backend/tests/test_related_route.py b/backend/tests/test_related_route.py new file mode 100644 index 00000000..b5a45398 --- /dev/null +++ b/backend/tests/test_related_route.py @@ -0,0 +1,298 @@ +"""`GET /api/search/related/{video_id}` — a video's related videos, without what the user has seen. + +The scrape is replaced by pages the test hands out, and YouTube's videos.list by an offline enrich +(30-minute videos, so no Shorts probe goes out). What is held: the list leaves out what this user +watched or hid — here or in their Takeout history, unless taken back to "new" since — blocked +channels and the video itself; it pages on until it has enough; it lands in the catalog the way +search finds do; and a failure, or the day's cap, says so instead of answering an empty list.""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from app import quota, sysconfig +from app.catalog.models import BlockedChannel, Channel, Video +from app.models import QuotaEvent +from app.reader.models import TakeoutItem, TakeoutKind, VideoState, VideoUnmark +from app.routes import search as search_routes +from app.youtube.innertube import ScrapeError + +T0 = datetime(2026, 10, 1, tzinfo=timezone.utc) +SOURCE = "srcvideo001" + + +def _vid(n: int) -> str: + return f"rel{n:08d}" + + +def _item(n: int, channel: str = "UCrelated01") -> dict: + return { + "id": _vid(n), + "channel_id": channel, + "channel_title": f"Chan {channel}", + "title": f"Related {n}", + "published_at": None, + "thumbnail_url": None, + "live_broadcast": "none", + } + + +class _FakeYT: + def __init__(self, *_a, **_kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_channels(self, ids): + return [] + + def get_videos(self, ids): + return [ + { + "id": i, + "snippet": {"title": f"T {i}", "publishedAt": "2025-01-01T00:00:00Z"}, + "contentDetails": {"duration": "PT30M"}, + "statistics": {}, + "status": {}, + } + for i in ids + ] + + +@pytest.fixture +def pages(monkeypatch, db): + """The scrape answers with the pages in `state["pages"]`, one per call, recording each call.""" + sysconfig.set_value(db, "shorts_probe_max_seconds", 60) + state: dict = {"pages": [], "calls": []} + + def related(video_id, continuation=None, *, hl="en", gl="US"): + state["calls"].append((video_id, continuation, hl, gl)) + page = state["pages"].pop(0) + if isinstance(page, Exception): + raise page + return page + + monkeypatch.setattr(search_routes.related_scrape, "related", related) + monkeypatch.setattr(search_routes, "YouTubeClient", _FakeYT) + return state + + +def _page(items: list[dict], token: str | None = "NEXT") -> dict: + return {"items": items, "next_page_token": token} + + +def _ids(resp) -> list[str]: + assert resp.status_code == 200, resp.text + return [it["id"] for it in resp.json()["items"]] + + +def _known(db, *ns: int, channel: str = "UCrelated01") -> None: + """These related videos already in the catalog (a state needs the row).""" + if db.get(Channel, channel) is None: + db.add(Channel(id=channel, title="Chan")) + for n in ns: + db.add(Video(id=_vid(n), channel_id=channel, title=f"Related {n}", published_at=T0)) + db.commit() + + +def test_a_related_list_comes_into_the_catalog_as_search_finds_do(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(n) for n in range(1, 21)])] + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert _ids(resp) == [_vid(n) for n in range(1, 21)] + assert resp.json()["next_cursor"] == "NEXT" and resp.json()["has_more"] is True + video = db.get(Video, _vid(1)) + # Flagged like a search find: hidden from the shared Library, reclaimed when nobody keeps it. + assert video.via_search is True and video.enriched_at is not None + assert pages["calls"] == [(SOURCE, None, "en", "US")] + + +def test_what_the_user_watched_or_hid_here_is_left_out(db, make_user, as_user, pages): + user = make_user() + other = make_user() + _known(db, 1, 2, 3, 4) + db.add_all( + [ + VideoState(user_id=user.id, video_id=_vid(1), status="watched", watched_at=T0), + VideoState(user_id=user.id, video_id=_vid(2), status="hidden"), + # In progress is not "seen": it stays, to be finished. + VideoState(user_id=user.id, video_id=_vid(3), status="new", position_seconds=120), + # Someone else's watch is theirs. + VideoState(user_id=other.id, video_id=_vid(4), status="watched", watched_at=T0), + ] + ) + db.commit() + pages["pages"] = [_page([_item(n) for n in range(1, 6)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(3), _vid(4), _vid(5)] + + +def test_what_the_takeout_history_shows_watched_is_left_out_unless_taken_back( + db, make_user, as_user, pages +): + user = make_user() + other = make_user() + _known(db, 2) + db.add_all( + [ + # Not in the catalog at all: the history alone names it. + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0), + # Watched on YouTube, then taken back to "new" here: it may come again. + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(2), at=T0), + VideoUnmark(user_id=user.id, video_id=_vid(2), at=T0 + timedelta(days=1)), + # Only in Watch later: not watched. + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCH_LATER, video_id=_vid(3), at=T0), + # Another user's history. + TakeoutItem(user_id=other.id, kind=TakeoutKind.WATCHED, video_id=_vid(4), at=T0), + ] + ) + db.commit() + pages["pages"] = [_page([_item(n) for n in range(1, 5)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2), _vid(3), _vid(4)] + + +def test_an_unmark_older_than_the_history_row_does_not_bring_it_back(db, make_user, as_user, pages): + user = make_user() + _known(db, 1) + db.add_all( + [ + VideoUnmark(user_id=user.id, video_id=_vid(1), at=T0 - timedelta(days=1)), + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0), + ] + ) + db.commit() + pages["pages"] = [_page([_item(1), _item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)] + + +def test_another_users_unmark_does_not_bring_it_back(db, make_user, as_user, pages): + user = make_user() + other = make_user() + _known(db, 1) + db.add_all( + [ + TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0), + VideoUnmark(user_id=other.id, video_id=_vid(1), at=T0 + timedelta(days=1)), + ] + ) + db.commit() + pages["pages"] = [_page([_item(1), _item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)] + + +def test_blocked_channels_and_the_video_itself_are_left_out_and_never_stored( + db, make_user, as_user, pages +): + user = make_user() + db.add(Channel(id="UCblocked01", title="Blocked")) + db.commit() + db.add(BlockedChannel(user_id=user.id, channel_id="UCblocked01")) + db.commit() + own = {**_item(9), "id": SOURCE} + pages["pages"] = [_page([_item(1), _item(2, "UCblocked01"), own], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + assert db.get(Video, _vid(2)) is None + + +def test_it_pages_on_until_enough_unseen_videos_are_gathered(db, make_user, as_user, pages): + user = make_user() + _known(db, *range(1, 16)) + for n in range(1, 16): + db.add(VideoState(user_id=user.id, video_id=_vid(n), status="watched", watched_at=T0)) + db.commit() + pages["pages"] = [ + _page([_item(n) for n in range(1, 21)], token="page2"), # 15 seen, 5 new + _page([_item(n) for n in range(21, 41)], token="page3"), # 20 new: 25 now + _page([_item(n) for n in range(41, 61)], token="page4"), # never asked + ] + ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}")) + assert ids == [_vid(n) for n in range(16, 41)] + assert [c[1] for c in pages["calls"]] == [None, "page2"] + assert resp.json()["next_cursor"] == "page3" + + +def test_a_repeat_on_a_later_page_counts_once(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [ + _page([_item(n) for n in range(1, 11)], token="page2"), + _page([_item(n) for n in range(5, 25)], token=None), + ] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [ + _vid(n) for n in range(1, 25) + ] + + +def test_it_reads_at_most_four_pages(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(n)], token=f"page{n + 1}") for n in range(1, 7)] + ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}")) + assert ids == [_vid(n) for n in range(1, 5)] + assert len(pages["calls"]) == 4 and resp.json()["next_cursor"] == "page5" + + +def test_the_cursor_and_the_hungarian_locale_reach_the_scrape(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(1)], token=None)] + resp = as_user(user).get(f"/api/search/related/{SOURCE}?cursor=C1®ion=hu") + assert _ids(resp) == [_vid(1)] + assert pages["calls"] == [(SOURCE, "C1", "hu", "HU")] + assert resp.json()["has_more"] is False + + +def test_an_unknown_region_is_refused(db, make_user, as_user, pages): + user = make_user() + assert as_user(user).get(f"/api/search/related/{SOURCE}?region=xx").status_code == 422 + assert pages["calls"] == [] + + +def test_a_failed_scrape_says_youtube_is_unavailable(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [ScrapeError("blocked")] + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"} + + +def test_a_failure_on_a_later_page_keeps_what_came_before(db, make_user, as_user, pages): + user = make_user() + pages["pages"] = [_page([_item(1), _item(2)], token="page2"), ScrapeError("blocked")] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1), _vid(2)] + + +def test_one_list_is_one_event_against_the_days_cap(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 1) + pages["pages"] = [ + _page([_item(1)], token="page2"), + _page([_item(2)], token=None), + ] + client = as_user(user) + assert client.get(f"/api/search/related/{SOURCE}").status_code == 200 + events = db.scalars( + select(QuotaEvent).where( + QuotaEvent.user_id == user.id, QuotaEvent.action == quota.QuotaAction.VIDEOS_RELATED + ) + ).all() + assert len(events) == 1 + refused = client.get(f"/api/search/related/{SOURCE}") + assert refused.status_code == 429 + assert refused.json()["detail"] == { + "code": "refused", + "reason": "related_daily_limit", + "limit": 1, + } + assert len(pages["calls"]) == 2 # the refused one asked YouTube nothing + + +def test_reading_further_is_the_same_list_not_another_one(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 2) + pages["pages"] = [_page([_item(1)], token=None), _page([_item(2)], token=None)] + client = as_user(user) + assert client.get(f"/api/search/related/{SOURCE}").status_code == 200 + assert client.get(f"/api/search/related/{SOURCE}?cursor=more").status_code == 200 + assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 1 diff --git a/backend/tests/test_related_scrape.py b/backend/tests/test_related_scrape.py new file mode 100644 index 00000000..2897ba6d --- /dev/null +++ b/backend/tests/test_related_scrape.py @@ -0,0 +1,212 @@ +"""youtube/related_scrape — a video's related videos from the InnerTube `next` endpoint. + +The two pages under `fixtures/` are real answers (2026-10-24), trimmed to three items and the +continuation: they hold the `lockupViewModel` shape YouTube draws today. Everything here breaks +silently when YouTube moves — a parser that stops finding videos, a cursor that stops paging — so a +shape it does not know must say so, never read as "nothing related". The network is an httpx +MockTransport.""" + +import copy +import json +from pathlib import Path + +import httpx +import pytest + +from app.youtube import innertube, related_scrape + +FIXTURES = Path(__file__).parent / "fixtures" +FIRST = json.loads((FIXTURES / "innertube_next_first.json").read_text(encoding="utf-8")) +CONTINUATION = json.loads( + (FIXTURES / "innertube_next_continuation.json").read_text(encoding="utf-8") +) +RESULTS_HTML = ( + '' +) + + +def _results(page: dict) -> list: + return page["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"][ + "results" + ] + + +@pytest.fixture +def net(monkeypatch): + """Route every httpx.Client the InnerTube module opens through a handler the test sets.""" + calls: list[httpx.Request] = [] + state: dict = {"handler": None, "calls": calls} + + def transport(request: httpx.Request) -> httpx.Response: + calls.append(request) + if request.url.path == "/results": + return httpx.Response(200, text=RESULTS_HTML) + return state["handler"](request) + + real = httpx.Client + monkeypatch.setattr( + innertube.httpx, + "Client", + lambda **kw: real(transport=httpx.MockTransport(transport), **kw), + ) + monkeypatch.setattr(innertube, "_config", None) + return state + + +def _answer(net, page, status=200): + net["handler"] = lambda request: ( + httpx.Response(status, text=page) + if isinstance(page, str) + else httpx.Response(status, json=page) + ) + + +def test_a_first_page_gives_its_videos_with_their_channel_and_the_next_cursor(net): + _answer(net, FIRST) + page = related_scrape.related("SdTZAMDKrNY") + assert [it["id"] for it in page["items"]] == ["to-lHJfK4pw", "R8V8CbuxryI", "HLKo4fJx_7k"] + first = page["items"][0] + assert first["channel_id"] == "UCtslD4DGH6PKyG_1gFAX7sg" + assert first["channel_title"] == "Alexander Amini" + assert first["title"] == "MIT 6.S191 (2025): Reinforcement Learning" + assert first["thumbnail_url"].startswith("https://i.ytimg.com/vi/to-lHJfK4pw/") + assert first["published_at"] is None and first["live_broadcast"] == "none" + assert page["next_page_token"] + body = json.loads(net["calls"][-1].content) + assert body["videoId"] == "SdTZAMDKrNY" and "continuation" not in body + assert net["calls"][-1].url.path == "/youtubei/v1/next" + + +def test_the_largest_thumbnail_is_taken(net): + _answer(net, FIRST) + item = related_scrape.related("SdTZAMDKrNY")["items"][0] + sources = _results(FIRST)[0]["lockupViewModel"]["contentImage"]["thumbnailViewModel"]["image"][ + "sources" + ] + assert item["thumbnail_url"] == sources[-1]["url"] + + +def test_a_continuation_page_reads_its_videos_and_sends_only_the_cursor(net): + _answer(net, CONTINUATION) + page = related_scrape.related("SdTZAMDKrNY", continuation="TOKEN", hl="hu", gl="HU") + assert len(page["items"]) == 2 + assert all(it["channel_id"].startswith("UC") for it in page["items"]) + assert page["next_page_token"] + body = json.loads(net["calls"][-1].content) + assert body["continuation"] == "TOKEN" and "videoId" not in body + assert body["context"]["client"]["hl"] == "hu" and body["context"]["client"]["gl"] == "HU" + + +def test_the_locale_defaults_to_the_topical_one(net): + _answer(net, FIRST) + related_scrape.related("SdTZAMDKrNY") + client = json.loads(net["calls"][-1].content)["context"]["client"] + assert (client["hl"], client["gl"]) == ("en", "US") + + +def test_a_video_by_several_channels_is_left_out(net): + page = copy.deepcopy(FIRST) + meta = _results(page)[1]["lockupViewModel"]["metadata"]["lockupMetadataViewModel"] + meta["image"] = {"avatarStackViewModel": {"avatars": []}} + _answer(net, page) + ids = [it["id"] for it in related_scrape.related("SdTZAMDKrNY")["items"]] + assert ids == ["to-lHJfK4pw", "HLKo4fJx_7k"] + + +def test_what_is_not_a_video_is_left_out_and_a_repeat_counts_once(net): + page = copy.deepcopy(FIRST) + results = _results(page) + playlist = copy.deepcopy(results[0]) + playlist["lockupViewModel"]["contentType"] = "LOCKUP_CONTENT_TYPE_PLAYLIST" + playlist["lockupViewModel"]["contentId"] = "PLxyz" + results.insert(0, playlist) + results.insert(2, copy.deepcopy(results[1])) + results.insert(0, {"adSlotRenderer": {}}) + _answer(net, page) + ids = [it["id"] for it in related_scrape.related("SdTZAMDKrNY")["items"]] + assert ids == ["to-lHJfK4pw", "R8V8CbuxryI", "HLKo4fJx_7k"] + + +def test_the_previous_compact_renderer_is_still_read(net): + page = copy.deepcopy(FIRST) + _results(page).insert( + 0, + { + "compactVideoRenderer": { + "videoId": "oldshape001", + "title": {"simpleText": "Old shape"}, + "longBylineText": { + "runs": [ + { + "text": "Old Chan", + "navigationEndpoint": {"browseEndpoint": {"browseId": "UColdchan"}}, + } + ] + }, + "thumbnail": {"thumbnails": [{"url": "s.jpg"}, {"url": "b.jpg"}]}, + } + }, + ) + _answer(net, page) + first = related_scrape.related("SdTZAMDKrNY")["items"][0] + assert first == { + "id": "oldshape001", + "channel_id": "UColdchan", + "channel_title": "Old Chan", + "title": "Old shape", + "published_at": None, + "thumbnail_url": "b.jpg", + "live_broadcast": "none", + } + + +def test_the_last_page_has_no_cursor(net): + page = copy.deepcopy(FIRST) + _results(page).pop() # the continuationItemRenderer + _answer(net, page) + assert related_scrape.related("SdTZAMDKrNY")["next_page_token"] is None + + +def test_a_video_youtube_no_longer_has_is_an_empty_page(net): + _answer( + net, + { + "contents": { + "twoColumnWatchNextResults": { + "results": { + "results": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [{"backgroundPromoRenderer": {}}] + } + } + ] + } + } + } + } + }, + ) + assert related_scrape.related("aaaaaaaaaaa") == {"items": [], "next_page_token": None} + + +@pytest.mark.parametrize( + "answer", + [ + {"responseContext": {}}, # no watch page frame at all + {"onResponseReceivedEndpoints": [{"somethingElse": {}}]}, # a continuation we do not know + "consent", # not JSON + [1, 2], # JSON, not an object + ], +) +def test_an_answer_of_an_unknown_shape_is_an_error_not_an_empty_list(net, answer): + _answer(net, answer) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") + + +def test_a_refused_request_is_an_error_whatever_its_body_says(net): + _answer(net, FIRST, status=429) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") diff --git a/backend/tests/test_search_scrape.py b/backend/tests/test_search_scrape.py index 79f071ae..705a91b8 100644 --- a/backend/tests/test_search_scrape.py +++ b/backend/tests/test_search_scrape.py @@ -9,7 +9,7 @@ import json import httpx import pytest -from app.youtube import search_scrape +from app.youtube import innertube, search_scrape RESULTS_HTML = ( '' @@ -79,11 +79,11 @@ def net(monkeypatch): real = httpx.Client monkeypatch.setattr( - search_scrape.httpx, + innertube.httpx, "Client", lambda **kw: real(transport=httpx.MockTransport(transport), **kw), ) - monkeypatch.setattr(search_scrape, "_config", None) + monkeypatch.setattr(innertube, "_config", None) state["calls"] = calls return state @@ -171,7 +171,7 @@ def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, re with no error anywhere.""" net["handler"] = _serve(results_html=results_html) search_scrape.search("a") - assert net["calls"][-1].url.params["key"] == search_scrape._FALLBACK_KEY + assert net["calls"][-1].url.params["key"] == innertube._FALLBACK_KEY net["handler"] = _serve() search_scrape.search("b") diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index 15510873..976c1d44 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -54,6 +54,10 @@ "label": "Backfill quota reserve", "hint": "Units kept in reserve so scheduled backfill never starves interactive syncs / enrichment." }, + "related_daily_limit_per_user": { + "label": "Related videos — daily limit per user", + "hint": "Max related-video lists a user can open per day. They come from YouTube's website and spend no search quota; the videos in a list are looked up for 1 unit per 50. A pure anti-abuse limit. Set 0 to turn related videos off." + }, "search_daily_limit_per_user": { "label": "Live search — daily limit per user", "hint": "Max live YouTube searches per user per day. With the API source each search costs 100 units (so the shared budget affords only ~80-90/day total); with the scrape source it's a pure anti-abuse limit. Set 0 to disable live search." diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index 0b1ae16c..0db7a8df 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -188,6 +188,7 @@ "quota_spent": "Today's shared YouTube budget is spent. Try again tomorrow.", "recipient_required": "A recipient email is required", "registration_closed": "Registration is currently closed", + "related_daily_limit": "You've opened today's limit of related lists ({{limit}}). Try again tomorrow.", "search_daily_limit": "You've reached today's YouTube search limit ({{limit}}). Try again tomorrow.", "search_term_required": "Enter something to search for", "session_gone": "The playback session ended", diff --git a/frontend/src/i18n/locales/en/quotaActions.json b/frontend/src/i18n/locales/en/quotaActions.json index 73aa3b7e..b740fe9f 100644 --- a/frontend/src/i18n/locales/en/quotaActions.json +++ b/frontend/src/i18n/locales/en/quotaActions.json @@ -10,6 +10,7 @@ "videos_enrich": "Video enrichment", "videos_lookup": "Video lookup", "videos_search": "Live search", + "videos_related": "Related videos", "channels_discover": "Channel discovery", "channels_subscribe": "Subscribe", "channels_unsubscribe": "Unsubscribe", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index fe41b1ef..aabc465a 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -54,6 +54,10 @@ "label": "Behúzási kvótatartalék", "hint": "Tartalékban tartott egységek, hogy az ütemezett behúzás ne éheztesse ki az interaktív szinkront / dúsítást." }, + "related_daily_limit_per_user": { + "label": "Kapcsolódó videók — napi limit / felhasználó", + "hint": "Hány kapcsolódóvideó-listát nyithat meg egy felhasználó naponta. A listák a YouTube weboldaláról jönnek, keresési kvótát nem fogyasztanak; a bennük lévő videók lekérése 50-enként 1 egység. Tiszta visszaélés-elleni limit. 0 = a kapcsolódó videók kikapcsolva." + }, "search_daily_limit_per_user": { "label": "Élő keresés — napi limit / felhasználó", "hint": "Maximális élő YouTube-keresés felhasználónként naponta. Az API-forrásnál egy keresés 100 egységbe kerül (így a közös büdzséből összesen csak ~80-90 fér bele naponta); a scrape-forrásnál ez tiszta visszaélés-elleni limit. 0 = élő keresés kikapcsolva." diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index b902ced6..6ee470d8 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -188,6 +188,7 @@ "quota_spent": "A mai közös YouTube-keret elfogyott. Próbáld holnap.", "recipient_required": "Adj meg egy címzett e-mail címet", "registration_closed": "A regisztráció jelenleg zárva van", + "related_daily_limit": "Elérted a mai kapcsolódó listák korlátját ({{limit}}). Próbáld holnap.", "search_daily_limit": "Elérted a mai YouTube-keresési korlátot ({{limit}}). Próbáld holnap.", "search_term_required": "Írj be egy keresőkifejezést", "session_gone": "A lejátszási munkamenet véget ért", diff --git a/frontend/src/i18n/locales/hu/quotaActions.json b/frontend/src/i18n/locales/hu/quotaActions.json index 2321cb7c..5b02c615 100644 --- a/frontend/src/i18n/locales/hu/quotaActions.json +++ b/frontend/src/i18n/locales/hu/quotaActions.json @@ -10,6 +10,7 @@ "videos_enrich": "Videóadatok dúsítása", "videos_lookup": "Videó-lekérdezés", "videos_search": "Élő keresés", + "videos_related": "Kapcsolódó videók", "channels_discover": "Csatorna-felfedezés a listákból", "channels_subscribe": "Feliratkozás", "channels_unsubscribe": "Leiratkozás", From 686c673fb2c1431eeb4636f14fa6ad466b75c605 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 24 Oct 2026 16:57:52 +0200 Subject: [PATCH 02/20] feat(player): a video's related videos, on demand beside it The player shows what YouTube lists beside the video, without what the user has watched or hidden. Wide, a Related tab beside the description; stacked, a button beside the info opens them in a drawer. A row plays in the same player (Back returns) and the list stays the opened video's. A switch picks on-topic or Hungarian-trending lists, kept on the account. Each related row says whether the user follows its channel; the row is the video card's narrow form with its whole menu. VideoMenu moves to features/video, where the player can reach it. --- backend/app/catalog/queries.py | 13 + backend/app/routes/search.py | 5 +- backend/tests/test_related_route.py | 21 +- docs/DESIGN.md | 8 + .../features/player/PlayerModal.dom.test.tsx | 64 +++++ frontend/src/features/player/PlayerModal.tsx | 76 +++++- .../player/RelatedVideos.dom.test.tsx | 175 +++++++++++++ .../src/features/player/RelatedVideos.tsx | 230 ++++++++++++++++++ frontend/src/features/video/VideoCard.tsx | 43 +++- .../video}/VideoMenu.dom.test.tsx | 0 .../feed => features/video}/VideoMenu.tsx | 2 +- frontend/src/i18n/locales/en/player.json | 12 + frontend/src/i18n/locales/hu/player.json | 12 + frontend/src/lib/api/feed.ts | 30 +++ .../playerModalHooks.progress.dom.test.tsx | 11 + frontend/src/lib/progressCache.ts | 10 +- frontend/src/lib/queryKeys.ts | 2 + frontend/src/modules/feed/VirtualFeed.tsx | 2 +- 18 files changed, 697 insertions(+), 19 deletions(-) create mode 100644 frontend/src/features/player/RelatedVideos.dom.test.tsx create mode 100644 frontend/src/features/player/RelatedVideos.tsx rename frontend/src/{modules/feed => features/video}/VideoMenu.dom.test.tsx (100%) rename frontend/src/{modules/feed => features/video}/VideoMenu.tsx (90%) diff --git a/backend/app/catalog/queries.py b/backend/app/catalog/queries.py index 6d98ba61..63b2caee 100644 --- a/backend/app/catalog/queries.py +++ b/backend/app/catalog/queries.py @@ -158,6 +158,19 @@ def blocked_ids(db: Session, user: User) -> set[str]: ) +def followed_among(db: Session, user: User, channel_ids: set[str]) -> set[str]: + """Which of these channels the user subscribes to.""" + if not channel_ids: + return set() + return set( + db.scalars( + select(Subscription.channel_id).where( + owned_by(Subscription.user_id, user), Subscription.channel_id.in_(channel_ids) + ) + ) + ) + + def blocked_channels(db: Session, user: User, lang: str) -> list[Channel]: return list( db.execute( diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index 23c2f5bd..c5386899 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -284,8 +284,11 @@ def related_videos( # (a cursor) is the same list. if cursor is None: quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED) + rows = reader_queries.feed_rows_by_ids(db, user, collected) + # A related list mixes channels the user follows with ones they never met: each row says which. + followed = queries.followed_among(db, user, {r["channel_id"] for r in rows}) return { - "items": reader_queries.feed_rows_by_ids(db, user, collected), + "items": [{**r, "followed": r["channel_id"] in followed} for r in rows], "next_cursor": next_cursor, "has_more": bool(next_cursor), } diff --git a/backend/tests/test_related_route.py b/backend/tests/test_related_route.py index b5a45398..705ffffa 100644 --- a/backend/tests/test_related_route.py +++ b/backend/tests/test_related_route.py @@ -12,7 +12,7 @@ import pytest from sqlalchemy import select from app import quota, sysconfig -from app.catalog.models import BlockedChannel, Channel, Video +from app.catalog.models import BlockedChannel, Channel, Subscription, Video from app.models import QuotaEvent from app.reader.models import TakeoutItem, TakeoutKind, VideoState, VideoUnmark from app.routes import search as search_routes @@ -296,3 +296,22 @@ def test_reading_further_is_the_same_list_not_another_one(db, make_user, as_user assert client.get(f"/api/search/related/{SOURCE}").status_code == 200 assert client.get(f"/api/search/related/{SOURCE}?cursor=more").status_code == 200 assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 1 + + +def test_each_row_says_whether_the_user_follows_its_channel(db, make_user, as_user, pages): + user = make_user() + stranger = make_user() + _known(db, 1, channel="UCfollowed1") + _known(db, 2, channel="UCstranger1") + db.add(Subscription(user_id=user.id, channel_id="UCfollowed1")) + db.add(Subscription(user_id=stranger.id, channel_id="UCstranger1")) + db.commit() + pages["pages"] = [ + _page([_item(1, "UCfollowed1"), _item(2, "UCstranger1"), _item(3, "UCnobody01")], None) + ] + rows = as_user(user).get(f"/api/search/related/{SOURCE}").json()["items"] + assert {r["id"]: r["followed"] for r in rows} == { + _vid(1): True, + _vid(2): False, # another user's follow is not this one's + _vid(3): False, + } diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 05543621..10420072 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -564,6 +564,13 @@ of a full-width panel and nothing on the right. open. Stacked, the description opens from an ⓘ, because a hover popover never opens on a tap (measured live). The wheel: over the video it sets the volume; over a panel with something to scroll it scrolls only, never the volume, even at its end; over one with nothing to scroll it sets the volume. [WCAG 1.4.13] +- **Related videos (owner's picks A, A, a, a):** on demand, never open by itself — the modal has no room to + spare. Wide, a "Related" tab beside the description in the side panel; stacked, a button beside the ⓘ + opens them in a drawer over the player. A row is the thumbnailed video row in its narrow form (a 6rem + picture, the relative time only, the "…" its one control — every toggle is a row of that menu), with + a check before the channel the user follows. A click plays it in the same player, as a description + link does (Back returns), and the list stays the opened video's. What the user watched or hid is + never in it. Over it, a two-way switch: on topic or Hungarian trending, kept on the account. - **Clip mode (both players, owner's picks A + P1 + a):** a scissors in the control row (Plex: the C5 row, before the ⋯; YouTube: the actions beside Download, and the docked bar) or `C` opens a clip panel IN the player; `[` / `]` mark the start / end at the playhead and open it when shut. The video being watched is the @@ -795,3 +802,4 @@ of a full-width panel and nothing on the right. | 2026-10-17 | Text on glass (owner ★) | Light `--muted` 15% toward `--fg` in every scheme (≥ 6:1 on the bare page); the item count, the badges and the current Files row on opaque tints mixed into the surface | Only the two failing components; the item count's number in `--fg` | Pixel-measured over every scheme × mode × backdrop, the count fell to 3.8:1 (dark too: thumbnails scroll under the toolbar's glass) and Channels' tabs to 4.1:1; an `--fg` count outshouts the filters beside it | | 2026-10-18 | Scheduler run history (owner D4, E1, E2 ★) | A row's last three runs as dots, newest first (interrupted: a hollow warn ring, set apart from skipped and failed), and a history button that opens the job's runs on the Activity page; a user's ad-hoc runs in their own block under the jobs, with no interval and no run button | An inline expandable history per row; a dark-grey interrupted dot; ad-hoc runs only on the Activity page | Three dots answer "did it run, did it work" at a glance; the whole history needs filters, which the Activity page has | | 2026-10-18 | Activity page (owner D1, D2, G1–G3 ★) | One page, "Napló" / "Activity": four tabs (Runs, Admin actions, Server log, Quota) under one filter bar (text, a period of 1 h · 24 h · 7 d · 30 d · 90 d defaulting to 7 days, an account), the view in the URL; runs and admin actions as dense rows that open in place as a log line does, filtered and paged on the server; a run's or a request's log lines one pivot away, as a history entry | One merged timeline; a page per source; a default period per tab; card rows; the audit log's client-side table | Each source keeps its own table and retention, read in one place; one bar keeps what you narrowed across tabs; about 1.8k runs a day need a log's density and a server-side filter | +| 2026-10-24 | Related videos (owner U1–U3 ★, D1 A, D3) | On demand: a tab beside the description (wide), a drawer (stacked); the narrow thumbnailed row, its menu alone, a followed check; plays in place and the list stays the opened video's; an on-topic / Hungarian-trending switch kept on the account | Always shown; the end-of-video grid alone; a dialog; a big card or a text row; a list that follows the playing video | The modal is full on a small screen; a picture is what a choice among strangers rests on; a list that stays put is tried one by one, and spends no more of the day's cap | diff --git a/frontend/src/features/player/PlayerModal.dom.test.tsx b/frontend/src/features/player/PlayerModal.dom.test.tsx index 701cecc0..3fee4b11 100644 --- a/frontend/src/features/player/PlayerModal.dom.test.tsx +++ b/frontend/src/features/player/PlayerModal.dom.test.tsx @@ -52,12 +52,15 @@ beforeAll(() => { // --- the fake IFrame API ------------------------------------------------------------------------ const clock = { current: 0, duration: 3000 }; const started: { videoId: string; start: number | undefined }[] = []; +// What the player was asked to switch to in place (a description link, a related video). +const loaded: { videoId: string; startSeconds: number }[] = []; // The last player's event handlers, so a test can say "the player is ready" when it wants to. let playerEvents: { onReady?: () => void; onStateChange?: (e: { data: number }) => void } = {}; const base = { getCurrentTime: () => clock.current, getDuration: () => clock.duration, getVideoData: () => null, + loadVideoById: (o: { videoId: string; startSeconds: number }) => void loaded.push(o), }; // Anything else the modal calls on the player (volume, rate, destroy…) is a no-op. Not `then`: // a thenable would be swallowed by the first `await` that touched it. @@ -96,6 +99,7 @@ afterEach(() => { cleanup(); vi.restoreAllMocks(); started.length = 0; + loaded.length = 0; }); const lastStart = () => started[started.length - 1]; @@ -777,6 +781,66 @@ describe("PlayerModal — the side panel on a wide window", () => { expect(spin(text, 100)).toBe(true); }); + // A row is reached by its title (its picture is a link of the same name). + const titleOf = (links: HTMLElement[]) => links.find((a) => a.hasAttribute("data-nav-open"))!; + const related = (id: string, position_seconds = 0) => ({ + ...vid(id, position_seconds), + followed: false, + }); + + it("wide: the related videos are a tab beside the description, and one plays in place", async () => { + wideWindow(true); + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + const asked = vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1"), related("rel2", 120)], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 0); + renderModal( + {}} onState={() => {}} />, + ); + act(() => playerEvents.onReady?.()); + // Closed until asked: nothing is fetched for a player nobody opened the list in. + expect(screen.getByTestId("player-description")).toBeTruthy(); + expect(asked).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("tab", { name: "player.related.title" })); + fireEvent.click(titleOf(await screen.findAllByRole("link", { name: "Video rel2" }))); + // In the same player, from where the user left it. + expect(loaded).toEqual([{ videoId: "rel2", startSeconds: 120 }]); + // The list stays the opened video's, to try the next one; it is not asked for again. + expect(screen.getAllByRole("link", { name: "Video rel1" }).length).toBeGreaterThan(0); + expect(asked.mock.calls).toEqual([["A", "topical", null]]); + expect(screen.getByRole("button", { name: "player.back" })).toBeTruthy(); + }); + + it("stacked: a button opens the related videos in a drawer, and playing one closes it", async () => { + wideWindow(false); + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1")], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 0); + const onClose = vi.fn(); + renderModal( + {}} />, + ); + act(() => playerEvents.onReady?.()); + expect(screen.queryByRole("tab", { name: "player.related.title" })).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "player.related.title" })); + const drawer = await screen.findByRole("dialog", { name: "player.related.title" }); + fireEvent.click(titleOf(await within(drawer).findAllByRole("link", { name: "Video rel1" }))); + expect(loaded).toEqual([{ videoId: "rel1", startSeconds: 0 }]); + await waitFor(() => + expect(screen.queryByRole("dialog", { name: "player.related.title" })).toBeNull(), + ); + // Only the drawer: the click on its row must not reach the player's backdrop (measured live — + // rendered outside the card, the whole player closed). + expect(onClose).not.toHaveBeenCalled(); + }); + it("stacked: the card scrolls under the wheel, the video in it is still the volume", () => { wideWindow(false); vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); diff --git a/frontend/src/features/player/PlayerModal.tsx b/frontend/src/features/player/PlayerModal.tsx index b0afdd7c..3eeb531d 100644 --- a/frontend/src/features/player/PlayerModal.tsx +++ b/frontend/src/features/player/PlayerModal.tsx @@ -23,6 +23,7 @@ import { CheckCheck, ExternalLink, Info, + ListVideo, Maximize, Minimize, Minimize2, @@ -83,6 +84,10 @@ import { SURFACE } from "../../components/ui/surfaces"; import { IconLink, LinkButton } from "../../components/ui/LinkButton"; import { SectionLabel } from "../../components/ui/Headings"; import { useMediaQuery } from "../../lib/useMediaQuery"; +import Modal from "../../components/ui/Modal"; +import Tabs from "../../components/ui/Tabs"; +import { useTabPanel } from "../../components/ui/tabPanel"; +import { RelatedVideos } from "./RelatedVideos"; // Experiment (branch experiment/inline-player): play the video in-app via the // YouTube IFrame Player API (not a bare embed) so we can read playback position @@ -669,6 +674,17 @@ export default function PlayerModal({ // a tablet could not read a description at all. Anchored ABOVE, so it grows upward over the player // rather than off-screen; `flip` still rescues it when there is no room up there. const [showDesc, setShowDesc] = useState(false); + // The related videos, on demand (the owner's pick: the modal has little room to spare). Wide, they + // take the side panel's place of the description, a tab beside it; stacked, a drawer over the video + // opened from a button beside the ⓘ. The list is the OPENED video's: one played from it leaves the + // list in place, to try the next one (the owner's pick). + const [panelTab, setPanelTab] = useState<"description" | "related">("description"); + const [relatedOpen, setRelatedOpen] = useState(false); + const panelTabs = useTabPanel(); + const playRelated = (v: Video, startAt: number) => { + setRelatedOpen(false); + loadVideo(v.id, startAt); + }; const descPop = useAnchoredPopover({ open: showDesc, onOpenChange: setShowDesc, @@ -1245,6 +1261,29 @@ export default function PlayerModal({ )} + {!wide && ( + setRelatedOpen(true)} + aria-expanded={relatedOpen} + label={t("player.related.title")} + > + + + )} + {/* Inside the card, whose click stops at its edge: a portal's events still bubble + along the React tree, and from the backdrop a click on a row closed the player. */} + {relatedOpen && !wide && ( + setRelatedOpen(false)} + placement="drawer" + maxWidth="max-w-sm" + initialFocus="close" + > + + + )} {navigated && ( + + )} + {error && rows.length > 0 && ( + void query.fetchNextPage(nextPageOnClick(query.isFetchingNextPage))} + > + {error} + + )} + + )} + + + ); +} diff --git a/frontend/src/features/video/VideoCard.tsx b/frontend/src/features/video/VideoCard.tsx index 5d235783..6a892e30 100644 --- a/frontend/src/features/video/VideoCard.tsx +++ b/frontend/src/features/video/VideoCard.tsx @@ -374,6 +374,8 @@ function VideoCard({ menuOpen = false, navIndex, shelf = false, + narrow = false, + channelMark, }: { video: Video; view: FeedView; @@ -397,6 +399,12 @@ function VideoCard({ navIndex?: number; /** On a shelf (a horizontal strip): a one-line title over the channel alone (ui/mediaCardStyles). */ shelf?: boolean; + /** The thumbnailed row in a side panel's ~20rem column: a smaller picture, and the "…" alone at the + * row's end — the hover toggles leave no room for the title there, and every one of them is a row + * of the menu. */ + narrow?: boolean; + /** Something the list says about the channel (a "followed" mark), set before its name. */ + channelMark?: ReactNode; }) { const { t, i18n } = useTranslation(); const lang = i18n.language; @@ -610,15 +618,46 @@ function VideoCard({ const textBlock = ( <> {title} -
{channelLink}
+
+ {channelMark ? ( +
+ {channelMark} +
{channelLink}
+
+ ) : ( + channelLink + )} +
{views} {views && " · "} - {published} + {/* The narrow row keeps the relative time and moves the date into its tooltip, as the small + card does: in a 20rem panel the full pair wrapped to a line of its own. */} + {narrow ? : published}
); + if (spec.family === "row" && narrow) { + return ( +
+ +
{textBlock}
+ {more && ( +
+ +
+ )} +
+ ); + } + if (spec.family === "row") { return (
): boo return v.status !== "new" || v.position_seconds > 0; } +/** A video's related videos, in the locale YouTube fills the list in: on the video's topic, or with + * what trends in Hungary (the server's `RELATED_REGIONS`). */ +export type RelatedRegion = "topical" | "hu"; + +/** A row of a related list: a feed row that also says whether the user follows its channel. */ +export interface RelatedVideo extends Video { + followed: boolean; +} + +export interface RelatedResponse { + items: RelatedVideo[]; + has_more: boolean; + next_cursor: string | null; +} + export interface VideoDetail { id: string; description: string | null; @@ -235,6 +250,21 @@ export const feedApi = { if (limit) p.set("limit", String(limit)); return req(`/api/search/youtube?${p.toString()}`, {}, { quiet: true }); }, + // The videos YouTube shows beside this one, without what the user has watched or hidden. `quiet` + // for the same reason as the search: the list renders its refusal (the day's cap, YouTube) inline. + related: ( + videoId: string, + region: RelatedRegion, + cursor: string | null, + ): Promise => { + const p = new URLSearchParams({ region }); + if (cursor) p.set("cursor", cursor); + return req( + `/api/search/related/${encodeURIComponent(videoId)}?${p.toString()}`, + {}, + { quiet: true }, + ); + }, // Discard a set of live-search results "as if never added" (drops your search-finds + deletes // the now-orphaned, un-kept videos). Returns how many videos were removed. clearSearch: (videoIds: string[]): Promise<{ removed: number }> => diff --git a/frontend/src/lib/playerModalHooks.progress.dom.test.tsx b/frontend/src/lib/playerModalHooks.progress.dom.test.tsx index dbc10ceb..d4043ad0 100644 --- a/frontend/src/lib/playerModalHooks.progress.dom.test.tsx +++ b/frontend/src/lib/playerModalHooks.progress.dom.test.tsx @@ -186,6 +186,17 @@ describe("patchCachedProgress", () => { expect(after.pages[0]?.items[0]?.position_seconds).toBe(830); }); + it("patches a related list too — a row played from it resumes where it was left", () => { + const qc = new QueryClient(); + const key = qk.related("SRC", "topical"); + qc.setQueryData(key, { pages: [{ items: [video()], next_cursor: null }], pageParams: [null] }); + + patchCachedProgress(qc, new Map([[VID, 830]])); + + const after = qc.getQueryData(key) as { pages: { items: Video[] }[] }; + expect(after.pages[0]?.items[0]?.position_seconds).toBe(830); + }); + it("is a no-op when nothing was stored", () => { const qc = new QueryClient(); seedFeed(qc, video()); diff --git a/frontend/src/lib/progressCache.ts b/frontend/src/lib/progressCache.ts index abbfe323..673bbf99 100644 --- a/frontend/src/lib/progressCache.ts +++ b/frontend/src/lib/progressCache.ts @@ -74,11 +74,13 @@ export function patchCachedProgress(qc: QueryClient, positions: Map({ queryKey: qk.feed() }, patch); qc.setQueriesData({ queryKey: qk.playlist() }, patch); qc.setQueriesData({ queryKey: qk.ytSearch() }, patch); + qc.setQueriesData({ queryKey: qk.related() }, patch); } diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index 7a138cf6..47dae62f 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -32,6 +32,8 @@ export const qk = { ytSearch: (q?: string | null, count?: number) => q !== undefined ? (["yt-search", q, count] as const) : (["yt-search"] as const), videoDetail: (id: string) => ["video-detail", id] as const, + related: (id?: string, region?: string) => + id !== undefined ? (["related", id, region] as const) : (["related"] as const), // --- channels --- channels: () => ["channels"] as const, diff --git a/frontend/src/modules/feed/VirtualFeed.tsx b/frontend/src/modules/feed/VirtualFeed.tsx index 1c5fe0c3..42a4ae38 100644 --- a/frontend/src/modules/feed/VirtualFeed.tsx +++ b/frontend/src/modules/feed/VirtualFeed.tsx @@ -2,7 +2,7 @@ import type { Video, VideoStatus } from "../../lib/api"; import { FEED_VIEW_SPEC, type FeedView } from "../../lib/feedView"; import type { NEXT_PAGE } from "../../lib/nextPage"; import VideoCard from "../../features/video/VideoCard"; -import { VideoMenu } from "./VideoMenu"; +import { VideoMenu } from "../../features/video/VideoMenu"; import type { VideoActionHandlers } from "../../features/video/useVideoActions"; import { useContextMenu } from "../../components/ui/useContextMenu"; import VirtualGrid from "../../components/ui/VirtualGrid"; From 520509111689bb94b3ea95f7d1e797788b12787c Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 24 Oct 2026 17:03:01 +0200 Subject: [PATCH 03/20] refactor(player): the related-region pref key stays private to its list --- frontend/src/features/player/RelatedVideos.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/features/player/RelatedVideos.tsx b/frontend/src/features/player/RelatedVideos.tsx index 7e77fbbc..62545ecf 100644 --- a/frontend/src/features/player/RelatedVideos.tsx +++ b/frontend/src/features/player/RelatedVideos.tsx @@ -26,7 +26,7 @@ import { VideoMenu } from "../video/VideoMenu"; /** Which list YouTube fills in — on the video's topic or with what trends in Hungary — is a mood, * decided at the list, and the account keeps the last one (the owner's pick). */ -export const RELATED_REGION_PREF = "relatedRegion"; +const RELATED_REGION_PREF = "relatedRegion"; const RELATED_REGION_OPTIONS: UsePrefOptions = { parse: (raw) => (raw === "hu" ? "hu" : "topical"), }; From 7d45c4f59f9945412427dc68e3a1903e304bec06 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 24 Oct 2026 22:45:26 +0200 Subject: [PATCH 04/20] fix(keys): a list inside the topmost layer walks by keyboard The list keys are the page's and an open layer silences them, so the player's related videos could only be reached by Tab. A layer now names its root element; with the focus on a row inside the topmost one, j/k, the arrows and Enter walk that list, the walk stays inside the layer, and the player's seek and volume arrows leave a focused row alone. --- docs/DESIGN.md | 5 +- frontend/src/components/ui/Modal.tsx | 2 +- .../features/player/PlayerModal.dom.test.tsx | 70 ++++++++++++++ frontend/src/features/player/PlayerModal.tsx | 9 +- frontend/src/lib/keymap.ts | 9 +- frontend/src/lib/layerStack.ts | 20 +++- frontend/src/lib/listNav.dom.test.tsx | 29 ++++++ frontend/src/lib/listNav.ts | 92 ++++++++++++------- 8 files changed, 194 insertions(+), 42 deletions(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 10420072..041f49d5 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -498,7 +498,10 @@ of a full-width panel and nothing on the right. ## Interaction rules — _Decided_ - **Keyboard:** everything works by keyboard, in visual order, with no traps. A shortcut is a row in - `lib/keymap.ts`; list and grid navigation is `NavGrid`/`keyboardNav`. [WCAG 2.1.1, 2.4.3, 2.1.2] + `lib/keymap.ts`; list and grid navigation is `NavGrid`/`keyboardNav`. An open dialog or player + silences the page's list keys, but a list inside the topmost one walks while the focus is in it + (the player's related videos); the walk stays inside that layer, and its arrows there move the list, + not the video. [WCAG 2.1.1, 2.4.3, 2.1.2] - **Focus:** every control has a visible focus ring (the base ring in `index.css`; don't add a second). A sticky or fixed layer never covers the focused element. A list item the keyboard lands on rings 2 px outside itself where it has a gap round it (a card); a table row, a log line or a row flush in a diff --git a/frontend/src/components/ui/Modal.tsx b/frontend/src/components/ui/Modal.tsx index d3babaf4..4d3d5a94 100644 --- a/frontend/src/components/ui/Modal.tsx +++ b/frontend/src/components/ui/Modal.tsx @@ -147,7 +147,7 @@ export default function Modal({ // An effect event, so a new handler each render never re-registers: the stack id is assigned once per mount. const close = useEffectEvent(onClose); useEffect(() => { - const id = pushLayer(); + const id = pushLayer({ root: () => rootRef.current }); const onKey = (e: KeyboardEvent) => { if (takeEscape(e, id)) { e.stopPropagation(); diff --git a/frontend/src/features/player/PlayerModal.dom.test.tsx b/frontend/src/features/player/PlayerModal.dom.test.tsx index 3fee4b11..748e1b80 100644 --- a/frontend/src/features/player/PlayerModal.dom.test.tsx +++ b/frontend/src/features/player/PlayerModal.dom.test.tsx @@ -61,7 +61,11 @@ const base = { getDuration: () => clock.duration, getVideoData: () => null, loadVideoById: (o: { videoId: string; startSeconds: number }) => void loaded.push(o), + seekTo: (s: number) => void seeks.push(s), + setVolume: (v: number) => void volumes.push(v), }; +const seeks: number[] = []; +const volumes: number[] = []; // Anything else the modal calls on the player (volume, rate, destroy…) is a no-op. Not `then`: // a thenable would be swallowed by the first `await` that touched it. const fakePlayer = new Proxy(base, { @@ -100,6 +104,8 @@ afterEach(() => { vi.restoreAllMocks(); started.length = 0; loaded.length = 0; + seeks.length = 0; + volumes.length = 0; }); const lastStart = () => started[started.length - 1]; @@ -814,6 +820,70 @@ describe("PlayerModal — the side panel on a wide window", () => { expect(screen.getByRole("button", { name: "player.back" })).toBeTruthy(); }); + // The list keys are the page's, silenced while a layer is up — but this list is IN the layer. + const press = (key: string) => + act(() => void fireEvent.keyDown(document.activeElement ?? document.body, { key })); + const rowOf = (el: Element | null) => el?.closest("[data-nav-index]"); + + it("wide: the keyboard walks the related list, and its arrows are not the video's", async () => { + wideWindow(true); + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1"), related("rel2")], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 0); + renderModal( + {}} onState={() => {}} />, + ); + act(() => playerEvents.onReady?.()); + fireEvent.click(screen.getByRole("tab", { name: "player.related.title" })); + const first = titleOf(await screen.findAllByRole("link", { name: "Video rel1" })); + const second = titleOf(screen.getAllByRole("link", { name: "Video rel2" })); + act(() => first.focus()); + // (jsdom lays nothing out, so the rows read as one row: ←→ step them, ↑↓ have no row to go to.) + press("ArrowRight"); + expect(document.activeElement).toBe(rowOf(second)); + press("ArrowLeft"); + expect(document.activeElement).toBe(rowOf(first)); + press("j"); + expect(document.activeElement).toBe(rowOf(second)); + press("k"); + press("j"); + // None of the arrows reached the video: no seek, no volume step. + press("ArrowUp"); + press("ArrowDown"); + expect(seeks).toEqual([]); + expect(volumes).toEqual([]); + // Enter on the row plays it, like its title's click. + press("Enter"); + expect(loaded).toEqual([{ videoId: "rel2", startSeconds: 0 }]); + }); + + it("stacked: the keyboard walks the list in the drawer too", async () => { + wideWindow(false); + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1"), related("rel2")], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 0); + renderModal( + {}} onState={() => {}} />, + ); + act(() => playerEvents.onReady?.()); + fireEvent.click(screen.getByRole("button", { name: "player.related.title" })); + const drawer = await screen.findByRole("dialog", { name: "player.related.title" }); + const first = titleOf(await within(drawer).findAllByRole("link", { name: "Video rel1" })); + act(() => first.focus()); + press("j"); + expect(document.activeElement).toBe( + rowOf(titleOf(within(drawer).getAllByRole("link", { name: "Video rel2" }))), + ); + }); + it("stacked: a button opens the related videos in a drawer, and playing one closes it", async () => { wideWindow(false); vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); diff --git a/frontend/src/features/player/PlayerModal.tsx b/frontend/src/features/player/PlayerModal.tsx index 3eeb531d..51bb6d24 100644 --- a/frontend/src/features/player/PlayerModal.tsx +++ b/frontend/src/features/player/PlayerModal.tsx @@ -50,6 +50,7 @@ import { anyLayerAbove, escapeHandled, } from "../../lib/layerStack"; +import { focusInNavList } from "../../lib/listNav"; import { trapTab, useDialogChrome } from "../../components/ui/dialogFocus"; import DownloadButton from "../downloads/DownloadButton"; import ClipPanel, { ClipCropFrame, type ClipCrop } from "./ClipPanel"; @@ -709,10 +710,11 @@ export default function PlayerModal({ // over the player by doing so. const overlayOrder = useOverlayOrder(); // This player is a dismissable layer: register it so a Modal or popover opened ABOVE it (e.g. the - // download dialog, or an anchored menu) takes Escape first and this player defers. + // download dialog, or an anchored menu) takes Escape first and this player defers. Its root lets + // the keyboard walk a list inside it (the related videos). const layerIdRef = useRef(null); useEffect(() => { - const id = pushLayer(); + const id = pushLayer({ root: () => dialogRef.current }); layerIdRef.current = id; return () => { removeLayer(id); @@ -740,7 +742,9 @@ export default function PlayerModal({ // there: deferring would scroll something invisible and leave the viewer with no volume control at // all. Both flags, not just fullscreen: since F stopped taking the browser's lock, maximise is the // common case and checking only `isFullscreen` made the keys dead exactly where they matter most. + // On a row of a list in the player (the related videos) the arrows walk the list instead. const volume = (e: KeyboardEvent, up: boolean) => { + if (focusInNavList()) return; const covered = isFullscreenRef.current || maximized; if (!covered && canScrollBy(document.activeElement as HTMLElement | null, up ? -1 : 1)) return; @@ -748,6 +752,7 @@ export default function PlayerModal({ nudgeVolume(up ? 5 : -5); }; const seek = (e: KeyboardEvent, secs: number) => { + if (focusInNavList()) return; e.preventDefault(); nudgeSeek(secs); }; diff --git a/frontend/src/lib/keymap.ts b/frontend/src/lib/keymap.ts index ca7ad049..26878ffc 100644 --- a/frontend/src/lib/keymap.ts +++ b/frontend/src/lib/keymap.ts @@ -25,7 +25,8 @@ import type { Page } from "./urlState"; * `page`, `search` and `list` are the PAGE-LEVEL scopes: they act on the page underneath * every overlay, so `dispatch` keeps them quiet while a layer that owns the keyboard is open. That is * also what keeps them clear of the players' bare letters — the Plex player's `s` and `k` only ever - * run with the player's own layer up. */ + * run with the player's own layer up. The one exception is a list INSIDE the topmost layer, with the + * focus in it (`dispatch`'s `inTopLayer`, decided by lib/listNav). */ export type KeyScope = | "global" | "page" @@ -358,11 +359,13 @@ const stepsOf = (chord: string): string[] => chord.split(" "); * implementation cannot drift without tsc saying so. Skipped while typing unless the row opts in * with `allowInInput` (Escape, and only Escape), on a key a focused widget already handled, and — for a page-level scope — while a layer that * owns the keyboard is open. The rule lives here rather than in each listener so a new page-level - * surface cannot forget it. */ + * surface cannot forget it. `inTopLayer` is the one way past it: the caller has checked that the + * key acts inside that topmost layer, not on the page behind (a list in the player). */ export function dispatch( e: KeyboardEvent, scope: S, handlers: { readonly [K in ShortcutId]: (e: KeyboardEvent) => void }, + { inTopLayer = false }: { inTopLayer?: boolean } = {}, ): boolean { if (sequenceEvents.has(e)) return false; // A key a focused widget already handled is not a shortcut: every listener here is on `window`, @@ -374,7 +377,7 @@ export function dispatch( if (e.key === "Escape" && escapeHandled(e)) return false; const live = liveSequence(e); if (live && live.scope !== scope) return false; - const quiet = isPageLevelScope(scope) && anyBlockingLayer(); + const quiet = isPageLevelScope(scope) && anyBlockingLayer() && !inTopLayer; const typing = isTypingTarget(e); const run = (id: string) => (handlers as Record void>)[id]?.(e); diff --git a/frontend/src/lib/layerStack.ts b/frontend/src/lib/layerStack.ts index 439c9aaa..f86e27f8 100644 --- a/frontend/src/lib/layerStack.ts +++ b/frontend/src/lib/layerStack.ts @@ -26,12 +26,21 @@ const passive = new Set(); // none, and Tab from anything outside the prompt walked into the page behind the scrim. The two // questions are separate; a layer answers each for itself. const defers = new Set(); +// The element a layer draws into, for the layers that hold content of their own to navigate (a +// dialog, the player): a list inside the TOPMOST such layer is walked by the keyboard although a +// layer is open — it is the one the user is looking at, not the page behind. +const roots = new Map HTMLElement | null>(); -export function pushLayer(opts?: { passive?: boolean; trapsTab?: boolean }): number { +export function pushLayer(opts?: { + passive?: boolean; + trapsTab?: boolean; + root?: () => HTMLElement | null; +}): number { const id = nextId++; stack.push(id); if (opts?.passive) passive.add(id); if (opts?.trapsTab === false) defers.add(id); + if (opts?.root) roots.set(id, opts.root); return id; } @@ -39,6 +48,15 @@ export function removeLayer(id: number): void { stack = stack.filter((x) => x !== id); passive.delete(id); defers.delete(id); + roots.delete(id); +} + +/** The element of the topmost layer that owns the keyboard — null while none is open, or while that + * layer gave none (a menu: its own keys move inside it). */ +export function topBlockingLayerRoot(): HTMLElement | null { + const blocking = stack.filter((id) => !passive.has(id)); + const top = blocking[blocking.length - 1]; + return top === undefined ? null : (roots.get(top)?.() ?? null); } /** True while a layer that owns the keyboard is open — a modal, a player, a click-opened menu. The diff --git a/frontend/src/lib/listNav.dom.test.tsx b/frontend/src/lib/listNav.dom.test.tsx index 9df0b364..1256aa5f 100644 --- a/frontend/src/lib/listNav.dom.test.tsx +++ b/frontend/src/lib/listNav.dom.test.tsx @@ -202,6 +202,35 @@ describe("useListNav", () => { removeLayer(tooltip); }); + // A list INSIDE the topmost layer (the player's related videos) is the one the user is in. + it("walks a list inside the topmost layer, and only inside it", () => { + render( + <> + +
+ +
+ , + ); + const dialog = pushLayer({ root: () => screen.getByTestId("layer") }); + act(() => screen.getByTestId("b0-action").focus()); + press("j"); + expect(focused()).toBe("b1"); + press("k"); + press("k"); // its first row: the walk stops there, not on the page's list behind + expect(focused()).toBe("b0"); + // A menu opened over it moves with its own keys: the list stays still. + const menu = pushLayer(); + press("j"); + expect(focused()).toBe("b0"); + removeLayer(menu); + // Focus left behind on the page's list does not walk it while the layer is up. + act(() => screen.getByTestId("a0-action").focus()); + press("j"); + expect(focused()).toBe("a0-action"); + removeLayer(dialog); + }); + // A virtualized grid only mounts a screen of rows: the next item may not exist yet. it("asks the list to reveal an unmounted item, then focuses it once it renders", async () => { const reveal = vi.fn(); diff --git a/frontend/src/lib/listNav.ts b/frontend/src/lib/listNav.ts index daf261f1..a0db0539 100644 --- a/frontend/src/lib/listNav.ts +++ b/frontend/src/lib/listNav.ts @@ -1,5 +1,6 @@ import { useEffect, useEffectEvent, useState, type RefObject } from "react"; import { dispatch, useKeyScope } from "./keymap"; +import { anyBlockingLayer, topBlockingLayerRoot } from "./layerStack"; import { scrollParent } from "./listPaging"; import { focusIsUnclaimed } from "./useFocusKeeper"; @@ -73,11 +74,15 @@ function itemAt(list: NavList, index: number): HTMLElement | null { } function ordered(): NavList[] { + // Inside an open layer (the player) the walk stays in it: `k` on its first row must not land on + // the feed behind the scrim. + const layer = anyBlockingLayer() ? topBlockingLayerRoot() : null; return ( [...lists.values()] // Only lists on screen: a list can be mounted yet hidden (display:none in a collapsed panel or // an inactive tab), and `j` must not walk into rows nobody can see. .filter((l) => l.el.isConnected && l.el.checkVisibility?.() !== false) + .filter((l) => !layer || layer.contains(l.el)) .sort((a, b) => a.el.compareDocumentPosition(b.el) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1, ) @@ -92,6 +97,10 @@ function current(): { list: NavList; index: number; item: HTMLElement } | null { return item && list ? { list, index: indexOf(item), item } : null; } +/** True while the focus is on an item of a keyboard-walked list (or a control inside one): its arrows + * move between the items, so a surface that also binds them (the player's seek) leaves them alone. */ +export const focusInNavList = (): boolean => current() !== null; + /** The mounted items that belong to `el` itself (not to a list nested inside it), in index order. */ function ownItems(el: HTMLElement): HTMLElement[] { return [...el.querySelectorAll(`[${INDEX}]`)] @@ -325,42 +334,57 @@ function onKey(e: KeyboardEvent): void { const arrow = (dx: -1 | 0 | 1, dy: -1 | 0 | 1) => (ev: KeyboardEvent) => { if (move(dx, dy)) ev.preventDefault(); }; - dispatch(e, "list", { - next: (ev) => { - ev.preventDefault(); - step(1); + // The list keys act on the page, so an open layer silences them — unless the focus is in a list + // INSIDE the topmost layer: that list is what the user is in (the player's related videos). + const layer = topBlockingLayerRoot(); + const cur = current(); + const inTopLayer = !!layer && !!cur && layer.contains(cur.item); + dispatch( + e, + "list", + { + next: (ev) => { + ev.preventDefault(); + step(1); + }, + prev: (ev) => { + ev.preventDefault(); + step(-1); + }, + left: arrow(-1, 0), + right: arrow(1, 0), + up: arrow(0, -1), + down: arrow(0, 1), + // Only on the item ROOT: Enter on a button inside the card must stay that button's Enter. + open: (ev) => { + const cur = current(); + if (!cur || document.activeElement !== cur.item) return; + const target = cur.item.querySelector("[data-nav-open]"); + if (!target) return; + ev.preventDefault(); + // A real click on the item's own "open" element, so Enter runs exactly the click path — not a + // second copy of what opening means that could drift from it. + target.click(); + }, }, - prev: (ev) => { - ev.preventDefault(); - step(-1); + { inTopLayer }, + ); + dispatch( + e, + "listDelete", + { + // On the item root, like Enter — and only in a list that registered a remover. + remove: (ev) => { + const cur = current(); + const remove = cur?.list.remove(); + if (!cur || !remove || document.activeElement !== cur.item) return; + ev.preventDefault(); + remove(cur.index); + refocusSlot(cur.list, cur.index, cur.item); + }, }, - left: arrow(-1, 0), - right: arrow(1, 0), - up: arrow(0, -1), - down: arrow(0, 1), - // Only on the item ROOT: Enter on a button inside the card must stay that button's Enter. - open: (ev) => { - const cur = current(); - if (!cur || document.activeElement !== cur.item) return; - const target = cur.item.querySelector("[data-nav-open]"); - if (!target) return; - ev.preventDefault(); - // A real click on the item's own "open" element, so Enter runs exactly the click path — not a - // second copy of what opening means that could drift from it. - target.click(); - }, - }); - dispatch(e, "listDelete", { - // On the item root, like Enter — and only in a list that registered a remover. - remove: (ev) => { - const cur = current(); - const remove = cur?.list.remove(); - if (!cur || !remove || document.activeElement !== cur.item) return; - ev.preventDefault(); - remove(cur.index); - refocusSlot(cur.list, cur.index, cur.item); - }, - }); + { inTopLayer }, + ); } // How long a removal gets to take its row out of the DOM before the focus gives up following it. From b04b51c90931dbe0003c26e74d682722d8e6020c Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 25 Oct 2026 00:32:26 +0200 Subject: [PATCH 05/20] fix(search): every related and live-search request takes a slot, and a failed one gives it back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A related list's cursor cost nothing, and a cursor is a free string: a replayed one read and enriched without end. Every request now takes a slot of the user's daily cap — taken under a lock before the work (`quota.claim_action`), so requests sent together cannot all pass the last free one, and given back when the request fails outright (`quota.refund_action`). The live search's scrape source takes its slots the same way. A limit of 0 turns the feature off: a distinct 409 `related_disabled` / `search_disabled`, and `/api/me` says `related_enabled` / `search_enabled` so the UI can leave it out. A videos.list that fails no longer reads as an empty page: `ingest_candidates` raises, and the search and the related list answer the refusal (the shared budget spent says "quota") instead of "no results". The two routes' page loops are one (`_gather`). `clear` deletes only what the caller found: a related list's rows carry no find, and any user could name them. --- backend/app/catalog/search.py | 30 +- backend/app/config.py | 6 +- backend/app/quota.py | 24 ++ backend/app/routes/me.py | 4 + backend/app/routes/search.py | 346 ++++++++++++--------- backend/tests/test_me_search_source.py | 24 +- backend/tests/test_related_route.py | 108 ++++++- backend/tests/test_route_commits.py | 17 + backend/tests/test_search_cap.py | 132 ++++++++ backend/tests/test_search_scrape_ingest.py | 22 ++ frontend/src/i18n/locales/en/config.json | 2 +- frontend/src/i18n/locales/en/errors.json | 2 + frontend/src/i18n/locales/hu/config.json | 2 +- frontend/src/i18n/locales/hu/errors.json | 2 + 14 files changed, 556 insertions(+), 165 deletions(-) create mode 100644 backend/tests/test_search_cap.py diff --git a/backend/app/catalog/search.py b/backend/app/catalog/search.py index 95ca4bc7..97a400f4 100644 --- a/backend/app/catalog/search.py +++ b/backend/app/catalog/search.py @@ -75,7 +75,8 @@ def ingest_candidates( ) -> list[str]: """Materialise one page of search candidates into the catalog (channel stubs + enrich, video stubs flagged via_search, enrich, Shorts classification) and return the KEEPABLE - video ids — non-Short, non-live — in the input relevance order.""" + video ids — non-Short, non-live — in the input relevance order. A videos.list that fails + raises its YouTubeError: with nothing enriched, nothing could be kept.""" ids = [it["id"] for it in candidates] if not ids: return [] @@ -132,13 +133,16 @@ def ingest_candidates( with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH): detail_by_id = {it["id"]: it for it in yt.get_videos(ids)} except YouTubeError: - # Every candidate then stays unenriched, and an unenriched video cannot be judged live or - # Short — so it is dropped. A working search returns nothing and reads as "no results". + # An unenriched video cannot be judged live or Short, so none of this page could be kept: + # swallowed, the failure answered as a page with nothing on it — "no results" for a search + # that never ran (the shared budget spent, no API key). The stubs stay (`insert_stubs` + # committed them), as the channels' do above; the caller answers the failure (its refusal + # says which). log.warning( "the videos found by a search could not be enriched", extra=logs.event("search.video_enrich_failed", candidates=len(ids)), ) - detail_by_id = {} + raise enriched_at = datetime.now( timezone.utc ) # when THIS enrich ran (distinct from the stub placeholder) @@ -212,17 +216,25 @@ def clear(db: Session, user: User, video_ids: list[str]) -> int: """Discard these results "as if never added": drop this user's finds of them, then delete the ones nobody kept — a `via_search` video no reader holds (`reader.queries.kept_by_a_reader`), whose channel nobody subscribes to, and that no other user still has as a find. Returns how - many videos went.""" - db.execute( - delete(SearchFind).where( - owned_by(SearchFind.user_id, user), SearchFind.video_id.in_(video_ids) + many videos went. + + Only the ones this user found: a video in the catalog with no find at all (a related list's, + which records none) is in someone's list right now, and an id any user could name would let + them delete it from under that list.""" + mine = list( + db.scalars( + delete(SearchFind) + .where(owned_by(SearchFind.user_id, user), SearchFind.video_id.in_(video_ids)) + .returning(SearchFind.video_id) ) ) + if not mine: + return 0 subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists() other_find = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists() result = db.execute( delete(Video).where( - Video.id.in_(video_ids), + Video.id.in_(mine), Video.via_search.is_(True), ~reader_queries.kept_by_a_reader(Video.id), ~subscribed, diff --git a/backend/app/config.py b/backend/app/config.py index ffc420b8..eb1ab096 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -113,8 +113,10 @@ class Settings(BaseSettings): # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable. search_daily_limit_per_user: int = 70 - # Per-user cap on related-video lists opened per day (the InnerTube watch page, no quota; each - # list's enrich is 1 unit per 50 videos). A pure anti-abuse rate limit. Admin-tunable. + # Per-user cap on related-video requests per day — a list opened and each "Show more" (the + # InnerTube watch page, no quota; its enrich is up to ~8 units: up to 4 pages, each a videos.list + # and a channels.list). A failed one gives its slot back. 0 turns related videos off. + # Admin-tunable. related_daily_limit_per_user: int = 200 # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list, # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks. diff --git a/backend/app/quota.py b/backend/app/quota.py index 108e45c4..66d4fa4d 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -166,6 +166,30 @@ def log_action(user_id: int, action: str) -> None: qs.commit() +def claim_action(user_id: int, action: str, limit: int) -> int | None: + """Take one of this user's `limit` daily `action` slots, or None when they are all taken — the + check and the take in one transaction, under a lock per user and action. Checked first and + logged after the work, N requests sent together at the last free slot all passed the check + (the work in between is seconds of scraping). Returns the event's id, for `refund_action` when + the work then produced nothing. Own session, as `log_action`.""" + with SessionLocal() as qs: + qs.execute(select(func.pg_advisory_xact_lock(func.hashtext(action), user_id))) + if actions_today(qs, user_id, action) >= limit: + return None + event = QuotaEvent(user_id=user_id, action=action, units=0) + qs.add(event) + qs.commit() + return event.id + + +def refund_action(event_id: int) -> None: + """Give a slot `claim_action` took back: the work it was taken for failed outright, and a + request that showed the user nothing must not count against them.""" + with SessionLocal() as qs: + qs.execute(delete(QuotaEvent).where(QuotaEvent.id == event_id)) + qs.commit() + + def actions_today(db: Session, user_id: int, action: str) -> int: """How many quota events of `action` this user has logged so far in the current Pacific day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events, diff --git a/backend/app/routes/me.py b/backend/app/routes/me.py index 0c5101cd..66d9cf02 100644 --- a/backend/app/routes/me.py +++ b/backend/app/routes/me.py @@ -79,6 +79,10 @@ def get_me( # Which live-search source runs, so the search button can say what a search costs: "scrape" # spends no API quota, "api" spends 100 units a page. "search_source": sysconfig.get_str(db, "search_source"), + # Whether live search and related videos are on: an admin turns either off with a daily + # limit of 0, and the UI then leaves out what would only be refused. + "search_enabled": sysconfig.get_int(db, "search_daily_limit_per_user") > 0, + "related_enabled": sysconfig.get_int(db, "related_daily_limit_per_user") > 0, "can_read": has_read_scope(user), "can_write": has_write_scope(user), "pending_invites": pending_invites, diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index c5386899..06741528 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -20,6 +20,7 @@ cap — with what this user has already seen left out. This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the catalog is `catalog/search`.""" +from collections.abc import Callable from typing import Literal from fastapi import APIRouter, Depends, Path @@ -34,6 +35,7 @@ from app.db import get_db from app.models import User from app.reader import queries as reader_queries from app.youtube import related_scrape, search_scrape +from app.youtube.innertube import ScrapeError from app.youtube.client import YouTubeClient, YouTubeError from app.refusal import refused from app.routes._common import youtube_refusal @@ -43,6 +45,8 @@ router = APIRouter(prefix="/api/search", tags=["search"]) # search.list costs this many quota units per page. SEARCH_COST = 100 +# The scrape source reads at most this many pages for one search. +SEARCH_MAX_PAGES = 12 # A related list is filled to this many unseen videos, reading at most this many pages (about 20 # videos each, measured) — the ones the user has seen are left out, so a page alone can fall short. @@ -52,6 +56,66 @@ RELATED_MAX_PAGES = 4 RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")} +def _gather( + db: Session, + yt: YouTubeClient, + user: User, + *, + cursor: str | None, + max_pages: int, + fetch: Callable[[str | None], dict], + candidates_of: Callable[[list[dict], list[str]], list[dict]], + enough: Callable[[list[str]], bool], + scrape_failed: Callable[[Exception], None], + ingest_failed: Callable[[int, int], None], +) -> tuple[list[str], str | None]: + """Page through a YouTube list into the catalog: fetch a page, keep its `candidates_of`, ingest + them, and go on from its continuation until `enough` or `max_pages`. Returns the kept ids and + the cursor to read further from. A page that fails after earlier ones gathered something ends + the list with what it has; a first page that fails raises the 422 refusal. + + 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway dropout + (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY), which hides the real + message and repeats the work. A Data API failure is logged where it happened; the scraper's is + not, and its text does not ride the answer — so `scrape_failed` says it. An ingest (DB writes + + enrich) fails the same way, guarded for any exception so a write or shape failure is a 422 with + a message too, not a bare 500; `ingest_failed` logs it inside the `except`, with the trace.""" + collected: list[str] = [] + next_cursor = cursor + for _ in range(max_pages): + try: + page = fetch(next_cursor) + except (YouTubeError, ScrapeError) as exc: + if collected: + break # keep what earlier pages already gathered + if isinstance(exc, ScrapeError): + scrape_failed(exc) + raise youtube_refusal(exc) from exc + candidates = candidates_of(page["items"], collected) + try: + collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates)) + except Exception as exc: # any ingest failure, not just YouTubeError + db.rollback() + ingest_failed(len(candidates), len(collected)) + if collected: + break + raise youtube_refusal(exc) from exc + next_cursor = page["next_page_token"] + if enough(collected) or not next_cursor: + break + return collected, next_cursor + + +def _cap_spent(feature: str, limit: int) -> None: + """Say that a request was refused for the per-user daily cap. The 429 is in the request line + either way; which of the caps produced it is not, and they are answered by different things — + this one by tomorrow, the shared budget by the admin.""" + log.info( + "a request was refused: the per-user daily cap is spent", + extra=logs.event(f"{feature}.refused", reason="daily_cap", limit=limit), + ) + + @router.get("/youtube") def search_youtube( q: str | None = None, @@ -72,109 +136,103 @@ def search_youtube( # Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page). source = sysconfig.get_str(db, "search_source") - # Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard - # since the search itself spends no quota. + # Every request takes a slot of the per-user daily cap, a cursor's too (anti-abuse); with + # scrape it's the only guard since the search itself spends no quota. A limit of 0 turns live + # search off on this instance — a refusal tomorrow will not change. daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user") - used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) - if used >= daily_limit: - # The 429 is in the request line either way; which of the two caps produced it is not, and - # they are answered by different things — this one by tomorrow, the other by the admin. - log.info( - "a live search was refused: the per-user daily cap is spent", - extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit), - ) - raise refused(429, "search_daily_limit", limit=daily_limit) - # The 100-unit budget pre-check only applies to the API source. - if source == "api" and not quota.can_spend(db, SEARCH_COST): - log.info( - "a live search was refused: the shared YouTube budget is spent", - extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST), - ) - raise refused(429, "quota_spent") + if daily_limit <= 0: + raise refused(409, "search_disabled") + claimed: int | None = None + if source == "api": + # The one search.list call logs this action's event itself (record_usage, attributed + # below), so here the cap is only checked: a claimed slot would count it twice. + if quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) >= daily_limit: + _cap_spent("search", daily_limit) + raise refused(429, "search_daily_limit", limit=daily_limit) + # The 100-unit budget pre-check only applies to the API source. + if not quota.can_spend(db, SEARCH_COST): + log.info( + "a live search was refused: the shared YouTube budget is spent", + extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST), + ) + raise refused(429, "quota_spent") + else: + # The check and the take are one step (`quota.claim_action`); a search that fails + # outright gives the slot back below. + claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_SEARCH, daily_limit) + if claimed is None: + _cap_spent("search", daily_limit) + raise refused(429, "search_daily_limit", limit=daily_limit) # Channels this user has blocked are dropped before ingestion — never shown, never stored. blocked = queries.blocked_ids(db, user) - collected: list[str] = [] - next_cursor = cursor - MAX_PAGES = 12 # scrape safety bound - with YouTubeClient(db, user) as yt: - for _ in range(MAX_PAGES): - try: - if source == "api": - with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): - page = yt.search_videos(term, page_token=next_cursor) - else: - page = search_scrape.search(term, continuation=next_cursor) - except (YouTubeError, search_scrape.ScrapeError) as exc: - if collected: - break # keep what we already gathered - # 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway - # dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY), - # which hides the real message and burns a second quota-costed search. Upstream failure - # on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1) - # A Data API failure is logged where it happened; the scraper's is not, and its text - # no longer rides the answer — so it is said here. - if isinstance(exc, search_scrape.ScrapeError): - log.warning( - "the YouTube search scrape failed", - extra=logs.event("search.scrape_failed", error=str(exc)), - ) - raise youtube_refusal(exc) from exc + def fetch(token: str | None) -> dict: + if source == "api": + with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): + return yt.search_videos(term, page_token=token) + return search_scrape.search(term, continuation=token) - # Drop currently-live/upcoming, channel-less, and blocked-channel results. - candidates = [ - it - for it in page["items"] - if it["live_broadcast"] == "none" - and it["channel_id"] - and it["channel_id"] not in blocked - ] - # Ingest (DB writes + enrich) runs outside the page-fetch try above; guard it too so a - # write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a - # bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1). - try: - collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates)) - except Exception as exc: # any ingest failure, not just YouTubeError - db.rollback() - # The term is NOT in this line. A search string is exactly what the privacy - # policy keeps out of the log unless the user turned Teljes/TRACE on for - # themselves, and `redact_text` does not cover it — it scrubs addresses and - # credential params, not what someone typed into the search box. What the - # ERROR line carries instead is the shape of the failure, which is what tells - # an ingest bug from an upstream one; the term follows at TRACE, and only - # under this user's own consent — not whenever the process runs at TRACE. - log.exception( - "live search ingest failed", - extra=logs.event( - "search.live.ingest_failed", - source=source, - candidates=len(candidates), - collected=len(collected), - ), - ) - if logs.consented_trace(): - log.log( - logs.TRACE, - "the term the ingest failed on", - # log-pii-ok: only under this user's own Teljes consent - extra=logs.event("search.live.term", term=term), - ) - if collected: - break # keep what earlier pages already gathered - raise youtube_refusal(exc) from exc - next_cursor = page["next_page_token"] - # The API source pages cost 100 units each — never auto-page; one page per request. - if source == "api" or len(collected) >= limit or not next_cursor: - break + def candidates_of(items: list[dict], _collected: list[str]) -> list[dict]: + # Drop currently-live/upcoming, channel-less, and blocked-channel results. + return [ + it + for it in items + if it["live_broadcast"] == "none" + and it["channel_id"] + and it["channel_id"] not in blocked + ] - # Count this as ONE per-user search against the daily cap. The scrape source spends no - # quota, so it never logs an event on its own; the API source already logged exactly one - # (via record_usage — it never auto-pages). Logging once here, after a search that produced - # at least one page (a total failure raises 502 above and never reaches this), keeps the cap - # counting user searches instead of internal continuation pages. - if source != "api": - quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH) + def scrape_failed(exc: Exception) -> None: + log.warning( + "the YouTube search scrape failed", + extra=logs.event("search.scrape_failed", error=str(exc)), + ) + + def ingest_failed(candidates: int, collected: int) -> None: + # The term is NOT in this line. A search string is exactly what the privacy policy keeps + # out of the log unless the user turned Teljes/TRACE on for themselves, and `redact_text` + # does not cover it — it scrubs addresses and credential params, not what someone typed + # into the search box. What the ERROR line carries instead is the shape of the failure, + # which is what tells an ingest bug from an upstream one; the term follows at TRACE, and + # only under this user's own consent — not whenever the process runs at TRACE. + log.exception( + "live search ingest failed", + extra=logs.event( + "search.live.ingest_failed", + source=source, + candidates=candidates, + collected=collected, + ), + ) + if logs.consented_trace(): + log.log( + logs.TRACE, + "the term the ingest failed on", + # log-pii-ok: only under this user's own Teljes consent + extra=logs.event("search.live.term", term=term), + ) + + try: + with YouTubeClient(db, user) as yt: + collected, next_cursor = _gather( + db, + yt, + user, + cursor=cursor, + # The API source pages cost 100 units each — never auto-page; one page per request. + max_pages=1 if source == "api" else SEARCH_MAX_PAGES, + fetch=fetch, + candidates_of=candidates_of, + enough=lambda got: len(got) >= limit, + scrape_failed=scrape_failed, + ingest_failed=ingest_failed, + ) + except Exception: + # A search that showed the user nothing does not count against them. + if claimed is not None: + quota.refund_action(claimed) + raise ordered = collected[:limit] @@ -228,62 +286,58 @@ def related_videos( without the ones this user has watched or hidden here or in their Takeout history, and without blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`).""" + # Every request takes a slot, a cursor's too: each one scrapes pages and enriches what they + # hold, and a cursor is a free string — counting only first pages let a replayed cursor read + # without end. The check and the take are one step (`quota.claim_action`); a request that fails + # outright gives its slot back. A limit of 0 turns related videos off on this instance. daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user") - used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) - if used >= daily_limit: - log.info( - "a related list was refused: the per-user daily cap is spent", - extra=logs.event("related.refused", reason="daily_cap", limit=daily_limit), - ) + if daily_limit <= 0: + raise refused(409, "related_disabled") + claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_RELATED, daily_limit) + if claimed is None: + _cap_spent("related", daily_limit) raise refused(429, "related_daily_limit", limit=daily_limit) hl, gl = RELATED_REGIONS[region] blocked = queries.blocked_ids(db, user) - collected: list[str] = [] - next_cursor = cursor - with YouTubeClient(db, user) as yt: - for _ in range(RELATED_MAX_PAGES): - try: - page = related_scrape.related(video_id, continuation=next_cursor, hl=hl, gl=gl) - except related_scrape.ScrapeError as exc: - if collected: - break # keep what earlier pages already gathered - log.warning( - "the related-videos scrape failed", - extra=logs.event("related.scrape_failed", error=str(exc)), - ) - raise youtube_refusal(exc) from exc - ids = [it["id"] for it in page["items"]] - drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id} - candidates = [ - it - for it in page["items"] - if it["id"] not in drop and it["channel_id"] not in blocked - ] - # As in the search above: an ingest failure answers 422, not a bare 500. - try: - collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates)) - except Exception as exc: # any ingest failure, not just YouTubeError - db.rollback() - log.exception( - "a related list's ingest failed", - extra=logs.event( - "related.ingest_failed", - candidates=len(candidates), - collected=len(collected), - ), - ) - if collected: - break - raise youtube_refusal(exc) from exc - next_cursor = page["next_page_token"] - if len(collected) >= RELATED_TARGET or not next_cursor: - break - # One opened list is one event against the cap, however many pages it read; reading further - # (a cursor) is the same list. - if cursor is None: - quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED) + def candidates_of(items: list[dict], collected: list[str]) -> list[dict]: + ids = [it["id"] for it in items] + drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id} + return [it for it in items if it["id"] not in drop and it["channel_id"] not in blocked] + + def scrape_failed(exc: Exception) -> None: + log.warning( + "the related-videos scrape failed", + extra=logs.event("related.scrape_failed", error=str(exc)), + ) + + def ingest_failed(candidates: int, collected: int) -> None: + log.exception( + "a related list's ingest failed", + extra=logs.event("related.ingest_failed", candidates=candidates, collected=collected), + ) + + try: + with YouTubeClient(db, user) as yt: + collected, next_cursor = _gather( + db, + yt, + user, + cursor=cursor, + max_pages=RELATED_MAX_PAGES, + fetch=lambda token: related_scrape.related( + video_id, continuation=token, hl=hl, gl=gl + ), + candidates_of=candidates_of, + enough=lambda got: len(got) >= RELATED_TARGET, + scrape_failed=scrape_failed, + ingest_failed=ingest_failed, + ) + except Exception: + quota.refund_action(claimed) + raise + rows = reader_queries.feed_rows_by_ids(db, user, collected) # A related list mixes channels the user follows with ones they never met: each row says which. followed = queries.followed_among(db, user, {r["channel_id"] for r in rows}) diff --git a/backend/tests/test_me_search_source.py b/backend/tests/test_me_search_source.py index 7f5bb2f3..b3c110a4 100644 --- a/backend/tests/test_me_search_source.py +++ b/backend/tests/test_me_search_source.py @@ -1,5 +1,6 @@ """`/api/me` names the live-search source, so the search button's tooltip can say what a search costs -under the source that actually runs (the default website source spends no API quota).""" +under the source that actually runs (the default website source spends no API quota), and says +whether live search and related videos are on at all.""" from app import sysconfig @@ -24,3 +25,24 @@ def test_me_names_the_running_search_source(client, db, make_user): sysconfig.set_value(db, "search_source", "api") db.commit() assert _me(client)["search_source"] == "api" + + +def test_me_says_whether_live_search_and_related_videos_are_on(client, db, make_user): + # A daily limit of 0 turns the feature off; the UI leaves it out instead of offering a refusal. + user = make_user(password=_PW) + login = client.post("/auth/password-login", json={"email": user.email, "password": _PW}) + assert login.status_code == 200 + + me = _me(client) + assert (me["search_enabled"], me["related_enabled"]) == (True, True) + + sysconfig.set_value(db, "search_daily_limit_per_user", 0) + db.commit() + me = _me(client) + assert (me["search_enabled"], me["related_enabled"]) == (False, True) + + sysconfig.set_value(db, "search_daily_limit_per_user", 5) + sysconfig.set_value(db, "related_daily_limit_per_user", 0) + db.commit() + me = _me(client) + assert (me["search_enabled"], me["related_enabled"]) == (True, False) diff --git a/backend/tests/test_related_route.py b/backend/tests/test_related_route.py index 705ffffa..a0f8aa29 100644 --- a/backend/tests/test_related_route.py +++ b/backend/tests/test_related_route.py @@ -6,16 +6,18 @@ watched or hid — here or in their Takeout history, unless taken back to "new" channels and the video itself; it pages on until it has enough; it lands in the catalog the way search finds do; and a failure, or the day's cap, says so instead of answering an empty list.""" +import threading from datetime import datetime, timedelta, timezone import pytest -from sqlalchemy import select +from sqlalchemy import func, select, text from app import quota, sysconfig from app.catalog.models import BlockedChannel, Channel, Subscription, Video from app.models import QuotaEvent from app.reader.models import TakeoutItem, TakeoutKind, VideoState, VideoUnmark from app.routes import search as search_routes +from app.youtube.client import YouTubeError from app.youtube.innertube import ScrapeError T0 = datetime(2026, 10, 1, tzinfo=timezone.utc) @@ -153,6 +155,8 @@ def test_what_the_takeout_history_shows_watched_is_left_out_unless_taken_back( db.commit() pages["pages"] = [_page([_item(n) for n in range(1, 5)], token=None)] assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2), _vid(3), _vid(4)] + # Left out before the ingest: what the user has seen is not stored for a list that hides it. + assert db.get(Video, _vid(1)) is None def test_an_unmark_older_than_the_history_row_does_not_bring_it_back(db, make_user, as_user, pages): @@ -288,14 +292,108 @@ def test_one_list_is_one_event_against_the_days_cap(db, make_user, as_user, page assert len(pages["calls"]) == 2 # the refused one asked YouTube nothing -def test_reading_further_is_the_same_list_not_another_one(db, make_user, as_user, pages): +def test_reading_further_takes_a_slot_too(db, make_user, as_user, pages): + # Each request scrapes and enriches; a cursor is a free string, so one that cost nothing let a + # replayed cursor read without end. user = make_user() - sysconfig.set_value(db, "related_daily_limit_per_user", 2) + sysconfig.set_value(db, "related_daily_limit_per_user", 1) pages["pages"] = [_page([_item(1)], token=None), _page([_item(2)], token=None)] client = as_user(user) assert client.get(f"/api/search/related/{SOURCE}").status_code == 200 - assert client.get(f"/api/search/related/{SOURCE}?cursor=more").status_code == 200 - assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 1 + refused = client.get(f"/api/search/related/{SOURCE}?cursor=more") + assert refused.status_code == 429 + assert refused.json()["detail"]["reason"] == "related_daily_limit" + assert len(pages["calls"]) == 1 + + +def test_a_list_that_failed_gives_its_slot_back(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 1) + pages["pages"] = [ScrapeError("blocked"), _page([_item(1)], token=None)] + client = as_user(user) + assert client.get(f"/api/search/related/{SOURCE}").status_code == 422 + assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 0 + assert _ids(client.get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + + +def test_a_zero_limit_turns_related_videos_off(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "related_daily_limit_per_user", 0) + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert resp.status_code == 409 + assert resp.json()["detail"] == {"code": "refused", "reason": "related_disabled"} + assert pages["calls"] == [] + + +def test_the_last_slot_is_taken_under_a_lock(db, make_user): + # Checked first and logged after seconds of scraping, requests sent together at the last free + # slot all passed. The claim waits for one already taking a slot for this user and action. + user = make_user() + action = quota.QuotaAction.VIDEOS_RELATED + db.execute(select(func.pg_advisory_xact_lock(func.hashtext(action), user.id))) + got: list[int | None] = [] + t = threading.Thread(target=lambda: got.append(quota.claim_action(user.id, action, 1))) + t.start() + t.join(timeout=1) + assert t.is_alive() # it waits for the one holding the lock + db.commit() + t.join(timeout=10) + assert got and got[0] is not None + assert quota.claim_action(user.id, action, 1) is None # the one slot is taken + + +def _videos_list_fails(monkeypatch, *, on_call: int = 1, reason: str = "quotaExceeded") -> None: + """YouTube's videos.list refuses from its `on_call`-th call on.""" + calls = {"n": 0} + real = _FakeYT.get_videos + + def get_videos(self, ids): + calls["n"] += 1 + if calls["n"] >= on_call: + raise YouTubeError("videos -> 403", reason=reason, path="videos") + return real(self, ids) + + monkeypatch.setattr(_FakeYT, "get_videos", get_videos) + + +def test_a_lookup_that_fails_says_why_not_no_videos(db, make_user, as_user, pages, monkeypatch): + # Nothing it found can be kept unenriched; an empty list would read as "nothing related". + user = make_user() + _videos_list_fails(monkeypatch) + pages["pages"] = [_page([_item(1)], token=None)] + resp = as_user(user).get(f"/api/search/related/{SOURCE}") + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "youtube", "reason": "quota"} + assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 0 + + +def test_a_lookup_failing_on_a_later_page_keeps_what_came_before( + db, make_user, as_user, pages, monkeypatch +): + user = make_user() + _videos_list_fails(monkeypatch, on_call=2) + pages["pages"] = [_page([_item(1)], token="page2"), _page([_item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)] + + +def test_an_ingest_that_breaks_the_session_is_rolled_back( + db, make_user, as_user, pages, monkeypatch +): + # A database error leaves the session unusable until it is rolled back: the rows of the pages + # before could not be read for the answer. + user = make_user() + real = search_routes.catalog_search.ingest_candidates + calls = {"n": 0} + + def ingest(db_, yt, user_, candidates): + calls["n"] += 1 + if calls["n"] == 2: + db_.execute(text("SELECT 1/0")) + return real(db_, yt, user_, candidates) + + monkeypatch.setattr(search_routes.catalog_search, "ingest_candidates", ingest) + pages["pages"] = [_page([_item(1)], token="page2"), _page([_item(2)], token=None)] + assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)] def test_each_row_says_whether_the_user_follows_its_channel(db, make_user, as_user, pages): diff --git a/backend/tests/test_route_commits.py b/backend/tests/test_route_commits.py index 5a6fc0bb..5dea6860 100644 --- a/backend/tests/test_route_commits.py +++ b/backend/tests/test_route_commits.py @@ -291,6 +291,23 @@ def test_search_results_cleared_through_the_route_are_gone(db, make_user, as_use assert db.execute(select(SearchFind)).first() is None +def test_clearing_leaves_a_video_the_caller_never_found(db, make_user, as_user): + # A related list ingests its rows with no find: they are in someone's list right now, and an id + # any user could name must not delete them from under it. + user = make_user() + _video(db, "s1", via_search=True) + _video(db, "s2", via_search=True) + db.add(SearchFind(user_id=user.id, video_id="s1")) + db.commit() + r = as_user(user).post("/api/search/clear", json={"video_ids": ["s1", "s2"]}) + assert r.json() == {"ok": True, "removed": 1} + db.expire_all() + assert db.get(Video, "s1") is None and db.get(Video, "s2") is not None + # Naming only ids it never found removes nothing. + r = as_user(user).post("/api/search/clear", json={"video_ids": ["s2"]}) + assert r.json() == {"ok": True, "removed": 0} + + # --- Plex show and season marks --------------------------------------------------------------- diff --git a/backend/tests/test_search_cap.py b/backend/tests/test_search_cap.py new file mode 100644 index 00000000..447db142 --- /dev/null +++ b/backend/tests/test_search_cap.py @@ -0,0 +1,132 @@ +"""`GET /api/search/youtube` — the per-user daily cap of the live search (scrape source). + +The scrape is replaced by pages the test hands out, and YouTube's videos.list by an offline enrich. +What is held: every request takes a slot, a cursor's too; a search that failed outright gives its +slot back; a zero limit turns live search off, which says so instead of "try again tomorrow"; and a +lookup that fails says why instead of answering "no results".""" + +import pytest + +from app import quota, sysconfig +from app.routes import search as search_routes +from app.youtube.client import YouTubeError +from app.youtube.innertube import ScrapeError + +ACTION = quota.QuotaAction.VIDEOS_SEARCH + + +def _item(n: int) -> dict: + return { + "id": f"srch{n:07d}", + "channel_id": "UCsearch001", + "channel_title": "Chan", + "title": f"Found {n}", + "published_at": None, + "thumbnail_url": None, + "live_broadcast": "none", + } + + +class _FakeYT: + fail_videos = False + + def __init__(self, *_a, **_kw): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_channels(self, ids): + return [] + + def get_videos(self, ids): + if _FakeYT.fail_videos: + raise YouTubeError("videos -> 403", reason="quotaExceeded", path="videos") + return [ + { + "id": i, + "snippet": {"title": f"T {i}", "publishedAt": "2025-01-01T00:00:00Z"}, + "contentDetails": {"duration": "PT30M"}, + "statistics": {}, + "status": {}, + } + for i in ids + ] + + +@pytest.fixture +def pages(monkeypatch, db): + """The scrape answers with the pages in `state["pages"]`, one per call.""" + sysconfig.set_value(db, "shorts_probe_max_seconds", 60) + sysconfig.set_value(db, "search_source", "scrape") + state: dict = {"pages": [], "calls": 0} + + def search(term, continuation=None): + state["calls"] += 1 + page = state["pages"].pop(0) + if isinstance(page, Exception): + raise page + return page + + monkeypatch.setattr(search_routes.search_scrape, "search", search) + monkeypatch.setattr(search_routes, "YouTubeClient", _FakeYT) + monkeypatch.setattr(_FakeYT, "fail_videos", False) + return state + + +def _page(n: int) -> dict: + return {"items": [_item(n)], "next_page_token": None} + + +def test_every_request_takes_a_slot_a_cursors_too(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + pages["pages"] = [_page(1), _page(2)] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 200 + refused = client.get("/api/search/youtube?q=cats&cursor=more") + assert refused.status_code == 429 + assert refused.json()["detail"]["reason"] == "search_daily_limit" + assert pages["calls"] == 1 + + +def test_a_search_that_failed_gives_its_slot_back(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 1) + pages["pages"] = [ScrapeError("blocked"), _page(1)] + client = as_user(user) + assert client.get("/api/search/youtube?q=cats").status_code == 422 + assert quota.actions_today(db, user.id, ACTION) == 0 + assert client.get("/api/search/youtube?q=cats").status_code == 200 + assert quota.actions_today(db, user.id, ACTION) == 1 + + +def test_a_zero_limit_turns_live_search_off(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_daily_limit_per_user", 0) + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 409 + assert resp.json()["detail"] == {"code": "refused", "reason": "search_disabled"} + assert pages["calls"] == 0 + + +def test_a_zero_limit_turns_the_api_source_off_too(db, make_user, as_user, pages): + user = make_user() + sysconfig.set_value(db, "search_source", "api") + sysconfig.set_value(db, "search_daily_limit_per_user", 0) + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 409 + assert resp.json()["detail"]["reason"] == "search_disabled" + + +def test_a_lookup_that_fails_says_why_not_no_results(db, make_user, as_user, pages, monkeypatch): + user = make_user() + monkeypatch.setattr(_FakeYT, "fail_videos", True) + pages["pages"] = [_page(1)] + resp = as_user(user).get("/api/search/youtube?q=cats") + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "youtube", "reason": "quota"} + assert quota.actions_today(db, user.id, ACTION) == 0 diff --git a/backend/tests/test_search_scrape_ingest.py b/backend/tests/test_search_scrape_ingest.py index 8d817880..6c38c3cf 100644 --- a/backend/tests/test_search_scrape_ingest.py +++ b/backend/tests/test_search_scrape_ingest.py @@ -13,10 +13,13 @@ a placeholder never becomes the video's permanent date. from datetime import datetime, timezone +import pytest + from app import sysconfig from app.models import User from app.catalog.models import Video from app.catalog.search import ingest_candidates +from app.youtube.client import YouTubeError class _FakeYT: @@ -97,6 +100,25 @@ def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db): assert c.enriched_at is None +class _FailingYT(_FakeYT): + def get_videos(self, ids): + raise YouTubeError("videos -> 403", reason="quotaExceeded", path="videos") + + +def test_a_videos_list_failure_raises_and_keeps_the_stubs(db): + # Swallowed, it answered as a page with nothing on it — "no results" for a search whose lookup + # never ran. The stubs stay: the FK targets, backfilled later. + user = User(email="searcher3@example.com") + db.add(user) + db.commit() + + with pytest.raises(YouTubeError): + ingest_candidates(db, _FailingYT({}), user, [_candidate("vid_ddddddd", "UC_chan_d")]) + + db.rollback() # what the caller does: the stubs were committed before the lookup + assert db.get(Video, "vid_ddddddd") is not None + + def test_search_youtube_route_returns_422_when_ingest_fails(db, monkeypatch, override_dep): # Route-level guard (B-core-1): an ingest failure with nothing yet collected must surface as a 422 # + message on this JSON endpoint (the FE hides 5xx behind a generic "connection lost" + auto-retry, diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index 976c1d44..71fc61e2 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -56,7 +56,7 @@ }, "related_daily_limit_per_user": { "label": "Related videos — daily limit per user", - "hint": "Max related-video lists a user can open per day. They come from YouTube's website and spend no search quota; the videos in a list are looked up for 1 unit per 50. A pure anti-abuse limit. Set 0 to turn related videos off." + "hint": "Max related-video requests a user can make per day — opening a list and each Show more count one. They come from YouTube's website and spend no search quota; looking up the videos in one costs up to about 8 units (up to 4 pages, each a video and a channel lookup). A failed one doesn't count. Set 0 to turn related videos off." }, "search_daily_limit_per_user": { "label": "Live search — daily limit per user", diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index 0db7a8df..e601b553 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -188,7 +188,9 @@ "quota_spent": "Today's shared YouTube budget is spent. Try again tomorrow.", "recipient_required": "A recipient email is required", "registration_closed": "Registration is currently closed", + "related_disabled": "Related videos are turned off on this instance.", "related_daily_limit": "You've opened today's limit of related lists ({{limit}}). Try again tomorrow.", + "search_disabled": "Live YouTube search is turned off on this instance.", "search_daily_limit": "You've reached today's YouTube search limit ({{limit}}). Try again tomorrow.", "search_term_required": "Enter something to search for", "session_gone": "The playback session ended", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index aabc465a..b1bc3eb1 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -56,7 +56,7 @@ }, "related_daily_limit_per_user": { "label": "Kapcsolódó videók — napi limit / felhasználó", - "hint": "Hány kapcsolódóvideó-listát nyithat meg egy felhasználó naponta. A listák a YouTube weboldaláról jönnek, keresési kvótát nem fogyasztanak; a bennük lévő videók lekérése 50-enként 1 egység. Tiszta visszaélés-elleni limit. 0 = a kapcsolódó videók kikapcsolva." + "hint": "Hányszor kérhet kapcsolódó videókat egy felhasználó naponta — egy lista megnyitása és minden „Mutass többet” egynek számít. A YouTube weboldaláról jönnek, keresési kvótát nem fogyasztanak; a bennük lévő videók lekérése kérésenként legfeljebb kb. 8 egység (legfeljebb 4 oldal, mindegyik egy videó- és egy csatornalekérés). A sikertelen nem számít. 0 = a kapcsolódó videók kikapcsolva." }, "search_daily_limit_per_user": { "label": "Élő keresés — napi limit / felhasználó", diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index 6ee470d8..4ec16375 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -188,7 +188,9 @@ "quota_spent": "A mai közös YouTube-keret elfogyott. Próbáld holnap.", "recipient_required": "Adj meg egy címzett e-mail címet", "registration_closed": "A regisztráció jelenleg zárva van", + "related_disabled": "A kapcsolódó videók ki vannak kapcsolva ezen a példányon.", "related_daily_limit": "Elérted a mai kapcsolódó listák korlátját ({{limit}}). Próbáld holnap.", + "search_disabled": "Az élő YouTube-keresés ki van kapcsolva ezen a példányon.", "search_daily_limit": "Elérted a mai YouTube-keresési korlátot ({{limit}}). Próbáld holnap.", "search_term_required": "Írj be egy keresőkifejezést", "session_gone": "A lejátszási munkamenet véget ért", From d53729a63dd9857220201c7d8bdb6dcb4f08256c Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 25 Oct 2026 00:32:27 +0200 Subject: [PATCH 06/20] fix(youtube): a related page is empty only on YouTube's own word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watch frame alone read as "nothing related" — so a related column YouTube moved, or a list drawn by renderers this does not read, answered an empty list instead of an error. Now only the "isn't available" promo YouTube draws for a video it no longer has is an empty page; anything else it cannot read is a ScrapeError. The flattening of a video renderer and the read of the continuation token are one each, in innertube, for the search and the related scrape. --- backend/app/youtube/innertube.py | 49 +++++++++++++++++ backend/app/youtube/related_scrape.py | 79 ++++++++++++--------------- backend/app/youtube/search_scrape.py | 35 ++---------- backend/tests/test_related_scrape.py | 20 +++++++ 4 files changed, 109 insertions(+), 74 deletions(-) diff --git a/backend/app/youtube/innertube.py b/backend/app/youtube/innertube.py index 54a8083e..ec6316d8 100644 --- a/backend/app/youtube/innertube.py +++ b/backend/app/youtube/innertube.py @@ -54,6 +54,55 @@ def runs_text(node: dict | None) -> str | None: return None +def scraped_video( + vid: str, channel_id: object, channel_title: object, title: object, thumbnails: object +) -> dict: + """One scraped video in the shape `YouTubeClient.search_videos` answers, so a page of either + source is ingested the same way. `published_at` is None (the website shows only a relative + "3 years ago"; the enrich fills the real date) and `live_broadcast` "none" (live/upcoming are + judged after the enrich, by `live_status`). The largest thumbnail is the last.""" + thumbs = thumbnails if isinstance(thumbnails, list) else [] + last = thumbs[-1] if thumbs else None + return { + "id": vid, + "channel_id": channel_id if isinstance(channel_id, str) and channel_id else None, + "channel_title": channel_title if isinstance(channel_title, str) else None, + "title": title if isinstance(title, str) else None, + "published_at": None, + "thumbnail_url": last.get("url") if isinstance(last, dict) else None, + "live_broadcast": "none", + } + + +def video_renderer(node: dict) -> dict | None: + """A `videoRenderer` (search) or `compactVideoRenderer` (the older related column) flattened — + the two draw a video the same way. None without a video id; the channel is None when the byline + names none.""" + vid = node.get("videoId") + if not isinstance(vid, str) or not vid: + return None + owner = node.get("ownerText") or node.get("longBylineText") or {} + runs = owner.get("runs") if isinstance(owner, dict) else None + first = runs[0] if isinstance(runs, list) and runs and isinstance(runs[0], dict) else {} + endpoint = (first.get("navigationEndpoint") or {}).get("browseEndpoint") or {} + return scraped_video( + vid, + endpoint.get("browseId"), + first.get("text"), + runs_text(node.get("title")), + (node.get("thumbnail") or {}).get("thumbnails"), + ) + + +def continuation_token(node: object) -> str | None: + """The "load more" cursor of a `continuationItemRenderer`, or None when `node` is not one.""" + if not isinstance(node, dict): + return None + command = (node.get("continuationEndpoint") or {}).get("continuationCommand") or {} + token = command.get("token") + return token if isinstance(token, str) and token else None + + def _get_config(client: httpx.Client) -> tuple[str, str]: """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them. diff --git a/backend/app/youtube/related_scrape.py b/backend/app/youtube/related_scrape.py index e66a725c..33f4124e 100644 --- a/backend/app/youtube/related_scrape.py +++ b/backend/app/youtube/related_scrape.py @@ -13,7 +13,13 @@ channel up front, and the stack does not say which one uploaded it. `hl`/`gl` pick the locale YouTube fills the list in: ``en``/``US`` keeps it on the video's topic, ``hu``/``HU`` mixes in what is trending in Hungary (measured).""" -from app.youtube.innertube import ScrapeError, post, runs_text +from app.youtube.innertube import ( + ScrapeError, + continuation_token, + post, + scraped_video, + video_renderer, +) __all__ = ["ScrapeError", "related"] @@ -40,21 +46,6 @@ def _at(node: object, *path: str | int) -> object: return node -def _item(vid: str, channel_id: object, channel_title: object, title: object, sources: object): - if not isinstance(channel_id, str) or not channel_id: - return None - thumbs = sources if isinstance(sources, list) else [] - return { - "id": vid, - "channel_id": channel_id, - "channel_title": channel_title if isinstance(channel_title, str) else None, - "title": title if isinstance(title, str) else None, - "published_at": None, # only a relative "1y ago" here; the enrich fills the real date - "thumbnail_url": _at(thumbs, -1, "url"), - "live_broadcast": "none", # live/upcoming are judged after the enrich, by live_status - } - - def _lockup(node: dict) -> dict | None: if node.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO" or not node.get("contentId"): return None @@ -73,7 +64,7 @@ def _lockup(node: dict) -> dict | None: channel_title = _at( meta, "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0 ) - return _item( + return scraped_video( node["contentId"], channel_id, _at(channel_title, "text", "content"), @@ -82,18 +73,17 @@ def _lockup(node: dict) -> dict | None: ) -def _compact(node: dict) -> dict | None: - vid = node.get("videoId") - if not vid: - return None - byline = _at(node, "longBylineText", "runs", 0) - return _item( - vid, - _at(byline, "navigationEndpoint", "browseEndpoint", "browseId"), - _at(byline, "text"), - runs_text(node.get("title")), - _at(node, "thumbnail", "thumbnails"), - ) +def _unavailable(data: dict) -> bool: + """Whether the watch page says the video is not available: its main column draws a + `backgroundPromoRenderer` ("This video isn't available anymore") and there is no related column. + YouTube's own word for "nothing related" — the one empty page that is not a shape we lost.""" + sections = _at(data, "contents", "twoColumnWatchNextResults", "results", "results", "contents") + for section in sections if isinstance(sections, list) else []: + inner = _at(section, "itemSectionRenderer", "contents") + for node in inner if isinstance(inner, list) else []: + if isinstance(node, dict) and "backgroundPromoRenderer" in node: + return True + return False def _items(data: dict) -> list | None: @@ -101,9 +91,7 @@ def _items(data: dict) -> list | None: first = _at(data, *_FIRST) if isinstance(first, list): return first - # A video YouTube no longer has keeps the watch page's frame but draws an "isn't available" - # promo in it and no related column: nothing related, said by YouTube itself. - if isinstance(_at(data, *_FIRST[:2]), dict): + if _unavailable(data): return [] for endpoint in data.get("onResponseReceivedEndpoints") or []: for key in _APPENDS: @@ -118,8 +106,9 @@ def related( ) -> dict: """One page of `video_id`'s related videos (`continuation` = the token a previous page gave). - A video YouTube no longer has is an empty page. An answer without the watch page's frame is a - `ScrapeError` — not an empty page, which would read as "nothing related".""" + A video YouTube says it no longer has is an empty page. An answer of a shape this does not know + — no related column, or one whose entries are all renderers it cannot read — is a `ScrapeError`: + not an empty page, which would read as "nothing related".""" body = {"continuation": continuation} if continuation else {"videoId": video_id} data = post("next", body, hl=hl, gl=gl) items = _items(data) @@ -128,23 +117,25 @@ def related( out: list[dict] = [] seen: set[str] = set() token = None + known = 0 # entries drawn by a renderer read here (a video's, or the cursor) for entry in items: if not isinstance(entry, dict): continue if "lockupViewModel" in entry: item = _lockup(entry["lockupViewModel"]) elif "compactVideoRenderer" in entry: - item = _compact(entry["compactVideoRenderer"]) + item = video_renderer(entry["compactVideoRenderer"]) else: - token = token or _at( - entry, - "continuationItemRenderer", - "continuationEndpoint", - "continuationCommand", - "token", - ) + found = continuation_token(entry.get("continuationItemRenderer")) + known += found is not None + token = token or found continue - if item and item["id"] not in seen: + known += 1 + # A video by several channels draws an avatar stack, not one: no channel, left out. + if item and item["channel_id"] and item["id"] not in seen: seen.add(item["id"]) out.append(item) - return {"items": out, "next_page_token": token if isinstance(token, str) else None} + # A list with entries but none of them one we read: YouTube draws its items some new way. + if items and not known: + raise ScrapeError("InnerTube's related list holds no renderer this reads") + return {"items": out, "next_page_token": token} diff --git a/backend/app/youtube/search_scrape.py b/backend/app/youtube/search_scrape.py index e32583e2..22adeac0 100644 --- a/backend/app/youtube/search_scrape.py +++ b/backend/app/youtube/search_scrape.py @@ -17,7 +17,7 @@ only difference is the cursor is an InnerTube *continuation token*, not a Data A the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"`` and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path).""" -from app.youtube.innertube import ScrapeError, post, runs_text +from app.youtube.innertube import ScrapeError, continuation_token, post, video_renderer __all__ = ["ScrapeError", "search"] @@ -29,29 +29,6 @@ _TYPE_VIDEO = "EgIQAQ==" _RESULT_FRAMES = {"contents", "onResponseReceivedCommands"} -def _video_item(vr: dict) -> dict | None: - """Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos.""" - vid = vr.get("videoId") - if not vid: - return None - owner = vr.get("ownerText") or vr.get("longBylineText") or {} - runs = owner.get("runs") or [] - channel_id = channel_title = None - if runs: - channel_title = runs[0].get("text") - channel_id = runs[0].get("navigationEndpoint", {}).get("browseEndpoint", {}).get("browseId") - thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or [] - return { - "id": vid, - "channel_id": channel_id, - "channel_title": channel_title, - "title": runs_text(vr.get("title")), - "published_at": None, # scrape exposes only a relative time; enrich fills the real date - "thumbnail_url": thumbs[-1]["url"] if thumbs else None, - "live_broadcast": "none", # live/upcoming filtered post-enrich via live_status - } - - def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: """Walk the response collecting videoRenderer items in document (relevance) order. Only `videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are @@ -59,7 +36,7 @@ def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: if isinstance(obj, dict): vr = obj.get("videoRenderer") if isinstance(vr, dict): - item = _video_item(vr) + item = video_renderer(vr) if item and item["id"] not in seen: seen.add(item["id"]) out.append(item) @@ -73,11 +50,9 @@ def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: def _find_continuation(obj) -> str | None: """First continuationItemRenderer token (the 'load more' cursor), or None at the end.""" if isinstance(obj, dict): - cir = obj.get("continuationItemRenderer") - if isinstance(cir, dict): - tok = cir.get("continuationEndpoint", {}).get("continuationCommand", {}).get("token") - if tok: - return tok + tok = continuation_token(obj.get("continuationItemRenderer")) + if tok: + return tok for v in obj.values(): tok = _find_continuation(v) if tok: diff --git a/backend/tests/test_related_scrape.py b/backend/tests/test_related_scrape.py index 2897ba6d..b0efb93c 100644 --- a/backend/tests/test_related_scrape.py +++ b/backend/tests/test_related_scrape.py @@ -191,6 +191,26 @@ def test_a_video_youtube_no_longer_has_is_an_empty_page(net): assert related_scrape.related("aaaaaaaaaaa") == {"items": [], "next_page_token": None} +def test_a_watch_frame_with_no_related_column_and_no_word_why_is_an_error(net): + # The frame alone, its related column moved somewhere this does not look: a lost shape, not + # "nothing related". + page = copy.deepcopy(FIRST) + watch = page["contents"]["twoColumnWatchNextResults"] + watch["sideColumn"] = watch.pop("secondaryResults") + _answer(net, page) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") + + +def test_a_related_list_drawn_by_renderers_this_does_not_read_is_an_error(net): + page = copy.deepcopy(FIRST) + results = _results(page) + results[:] = [{"videoTileViewModel": entry} for entry in results] + _answer(net, page) + with pytest.raises(related_scrape.ScrapeError): + related_scrape.related("SdTZAMDKrNY") + + @pytest.mark.parametrize( "answer", [ From b5045165c0f2f619b1606a40f9093e9399420369 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 25 Oct 2026 00:32:28 +0200 Subject: [PATCH 07/20] fix(player): a related video played is tracked, and its list is honest about its state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A related video played in place is the player's item now: its title and its own watched / playlist / download controls, its progress checkpointed and resumed, marked watched at its end — and its end does not step the queue. Back returns to the opened video, where it was left. The related list: - its marks announce through the shared notice (the Notification Center finds them), join the host's player session (the feed refetches on close), and stay per write, so an earlier answer no longer undoes a later mark still on its way; - every host's state write patches the cached related lists, which are never refetched; - kept for the session (gcTime), a retry asks for the page that failed, a list read so far with nothing unseen says so, a refusal a retry cannot answer offers none, and the followed mark names itself on hover. Turned off on the instance (a limit of 0), the tab, the drawer button and the live-search buttons are left out. Keys: the player's keys and its focus reclaim defer to a layer above it (the drawer); going wide moves the drawer's list into the panel's tab; `j` goes on from the slot of a row removed inside the top layer; the help sheet lists the list keys over a layer that holds a list. --- docs/DESIGN.md | 12 +- .../components/ui/KeyboardHelp.dom.test.tsx | 37 +++- frontend/src/components/ui/KeyboardHelp.tsx | 12 +- .../features/player/PlayerModal.dom.test.tsx | 193 +++++++++++++++++- frontend/src/features/player/PlayerModal.tsx | 162 ++++++++++----- .../player/RelatedVideos.dom.test.tsx | 150 +++++++++++++- .../src/features/player/RelatedVideos.tsx | 136 ++++++------ frontend/src/features/video/stateNotice.ts | 38 ++++ frontend/src/i18n/locales/en/player.json | 1 + frontend/src/i18n/locales/hu/player.json | 1 + frontend/src/lib/api/account.ts | 4 + frontend/src/lib/api/feed.test.ts | 13 ++ frontend/src/lib/keymap.ts | 27 ++- frontend/src/lib/layerStack.ts | 3 +- frontend/src/lib/listNav.dom.test.tsx | 45 ++++ frontend/src/lib/listNav.ts | 18 +- .../playerModalHooks.progress.dom.test.tsx | 2 +- frontend/src/lib/progressCache.ts | 18 ++ .../channels/WatchedChannels.dom.test.tsx | 37 +++- .../modules/channels/WatchedVideosDialog.tsx | 3 + .../feed/Feed.playerSession.dom.test.tsx | 67 +++++- .../modules/feed/Feed.searchOff.dom.test.tsx | 85 ++++++++ frontend/src/modules/feed/Feed.tsx | 77 +++---- .../feed/FeedHeader.searchTip.dom.test.tsx | 21 +- frontend/src/modules/feed/FeedHeader.tsx | 3 +- frontend/src/modules/home/Home.dom.test.tsx | 25 +++ frontend/src/modules/home/Home.tsx | 7 +- .../modules/playlists/Playlists.dom.test.tsx | 19 ++ frontend/src/modules/playlists/Playlists.tsx | 13 +- .../src/shell/AppCommandPalette.dom.test.tsx | 2 + 30 files changed, 1024 insertions(+), 207 deletions(-) create mode 100644 frontend/src/features/video/stateNotice.ts create mode 100644 frontend/src/modules/feed/Feed.searchOff.dom.test.tsx diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 041f49d5..3cf78d66 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -571,9 +571,14 @@ of a full-width panel and nothing on the right. spare. Wide, a "Related" tab beside the description in the side panel; stacked, a button beside the ⓘ opens them in a drawer over the player. A row is the thumbnailed video row in its narrow form (a 6rem picture, the relative time only, the "…" its one control — every toggle is a row of that menu), with - a check before the channel the user follows. A click plays it in the same player, as a description - link does (Back returns), and the list stays the opened video's. What the user watched or hid is - never in it. Over it, a two-way switch: on topic or Hungarian trending, kept on the account. + a check before the channel the user follows (named on hover). A click plays it in the same player + (Back returns to the opened video), and the list stays the opened video's. Played, it is the item it + now is (owner's pick D1 a): its title, its own watched / playlist / download controls, its progress + kept, marked watched at its end — and its end does not step the queue. What the user watched or hid + is never in it; a list read so far with nothing unseen says so above Show more. Over it, a two-way + switch: on topic or Hungarian trending, kept on the account. Turned off on the instance (a daily + limit of 0, owner's pick D2 a), there is no tab and no button; a refusal a retry cannot answer (the + day's cap, turned off, the shared budget) offers no Retry. - **Clip mode (both players, owner's picks A + P1 + a):** a scissors in the control row (Plex: the C5 row, before the ⋯; YouTube: the actions beside Download, and the docked bar) or `C` opens a clip panel IN the player; `[` / `]` mark the start / end at the playhead and open it when shut. The video being watched is the @@ -806,3 +811,4 @@ of a full-width panel and nothing on the right. | 2026-10-18 | Scheduler run history (owner D4, E1, E2 ★) | A row's last three runs as dots, newest first (interrupted: a hollow warn ring, set apart from skipped and failed), and a history button that opens the job's runs on the Activity page; a user's ad-hoc runs in their own block under the jobs, with no interval and no run button | An inline expandable history per row; a dark-grey interrupted dot; ad-hoc runs only on the Activity page | Three dots answer "did it run, did it work" at a glance; the whole history needs filters, which the Activity page has | | 2026-10-18 | Activity page (owner D1, D2, G1–G3 ★) | One page, "Napló" / "Activity": four tabs (Runs, Admin actions, Server log, Quota) under one filter bar (text, a period of 1 h · 24 h · 7 d · 30 d · 90 d defaulting to 7 days, an account), the view in the URL; runs and admin actions as dense rows that open in place as a log line does, filtered and paged on the server; a run's or a request's log lines one pivot away, as a history entry | One merged timeline; a page per source; a default period per tab; card rows; the audit log's client-side table | Each source keeps its own table and retention, read in one place; one bar keeps what you narrowed across tabs; about 1.8k runs a day need a log's density and a server-side filter | | 2026-10-24 | Related videos (owner U1–U3 ★, D1 A, D3) | On demand: a tab beside the description (wide), a drawer (stacked); the narrow thumbnailed row, its menu alone, a followed check; plays in place and the list stays the opened video's; an on-topic / Hungarian-trending switch kept on the account | Always shown; the end-of-video grid alone; a dialog; a big card or a text row; a list that follows the playing video | The modal is full on a small screen; a picture is what a choice among strangers rests on; a list that stays put is tried one by one, and spends no more of the day's cap | +| 2026-10-25 | Related: one played; off (owner D1 a, D2 a) | Played from the list, it is the player's item: its title, controls and progress, marked watched at its end, the queue left where it was; a daily limit of 0 takes the tab and the button away | A linked video's in-place play (no controls, nothing kept); a tab that only shows a refusal | What is watched should count as watched, wherever it was started from; a control that can only be refused is not offered | diff --git a/frontend/src/components/ui/KeyboardHelp.dom.test.tsx b/frontend/src/components/ui/KeyboardHelp.dom.test.tsx index 242a5f90..c7b614e4 100644 --- a/frontend/src/components/ui/KeyboardHelp.dom.test.tsx +++ b/frontend/src/components/ui/KeyboardHelp.dom.test.tsx @@ -16,7 +16,8 @@ import { } from "../../lib/keymap"; import { pushLayer, removeLayer } from "../../lib/layerStack"; import { useMediaKeys } from "../../lib/mediaKeys"; -import { useState } from "react"; +import { useRef, useState } from "react"; +import { navItemAttrs, useListNav } from "../../lib/listNav"; vi.mock("react-i18next", async (importOriginal) => { const actual = await importOriginal(); @@ -144,6 +145,40 @@ describe("KeyboardHelp", () => { expect(descriptions()).toContain("keys.help"); }); + // The player's related videos are a list inside the layer: their keys work there (`inTopLayer`), so + // the sheet opened over the player lists them — and over a layer with no list, still does not. + it("lists the list and menu keys over a layer that holds a list of its own", () => { + function LayerList() { + const ref = useRef(null); + useListNav(ref, { enabled: true }); + useContextMenu(); + return ( +
+
+
row
+
+
+ ); + } + render( + + + , + ); + const withList = pushLayer({ root: () => screen.getByTestId("layer") }); + pressH(); + expect(descriptions()).toContain("keys.listStep"); + expect(descriptions()).toContain("keys.contextMenu"); + act(() => closeKeyboardHelp()); + removeLayer(withList); + + const bare = pushLayer(); + pressH(); + removeLayer(bare); + expect(descriptions()).not.toContain("keys.listStep"); + expect(descriptions()).not.toContain("keys.contextMenu"); + }); + // Two live context menus (the Playlists page mounts one, its rail another) must not double the row. it("lists the context-menu chords once however many surfaces register", () => { function WithMenu() { diff --git a/frontend/src/components/ui/KeyboardHelp.tsx b/frontend/src/components/ui/KeyboardHelp.tsx index f62bd219..6b509ed7 100644 --- a/frontend/src/components/ui/KeyboardHelp.tsx +++ b/frontend/src/components/ui/KeyboardHelp.tsx @@ -41,11 +41,13 @@ export default function KeyboardHelp() { // Opened over a player or a dialog, the page-level keys (`s`, `g …`, `j`/`k`) are quiet — listing // them there would promise keys that do nothing until the overlay closes. And a `g` chord is listed // only for a module this account can actually reach. - // The context-menu key goes too: every `useContextMenu` is a page list, and with an overlay up the - // focus Shift+F10 needs is inside the overlay (found in UAT). - const live = overLayer - ? scopes.filter((s) => !isPageLevelScope(s) && s !== "contextMenu") - : scopes; + // The context-menu key goes too, with the page's lists. Over a layer that holds a list of its own + // (the player's related videos) the list keys and the menu key work on THAT list, so they stay — + // but not Delete: no list inside a layer takes it, and a page's remover behind does not reach. + const inLayer = (s: (typeof scopes)[number]) => + (!isPageLevelScope(s) && s !== "contextMenu") || + (overLayer === "layerWithList" && (s === "list" || s === "contextMenu")); + const live = overLayer === "page" ? scopes : scopes.filter(inLayer); const lines = helpLines(live, pages).filter((l) => !l.when || !!conditions[l.when]); const words = { Space: t("keys.cap.space"), ContextMenu: t("keys.cap.menu") }; return ( diff --git a/frontend/src/features/player/PlayerModal.dom.test.tsx b/frontend/src/features/player/PlayerModal.dom.test.tsx index 748e1b80..08bbb306 100644 --- a/frontend/src/features/player/PlayerModal.dom.test.tsx +++ b/frontend/src/features/player/PlayerModal.dom.test.tsx @@ -27,14 +27,19 @@ vi.mock("../navigation/navigationContext", () => ({ useNavigationActions: () => ({ openChannel: vi.fn() }), })); // Their own queries and popovers are not what is under test here. -vi.mock("../video/AddToPlaylist", () => ({ default: () => null })); -vi.mock("../downloads/DownloadButton", () => ({ default: () => null })); +vi.mock("../video/AddToPlaylist", () => ({ + default: ({ videoId }: { videoId: string }) => , +})); +vi.mock("../downloads/DownloadButton", () => ({ + default: ({ videoId }: { videoId: string }) => , +})); import PlayerModal from "./PlayerModal"; import { ConfirmProvider } from "../../components/ui/ConfirmProvider"; import { useConfirm } from "../../components/ui/confirmContext"; -import { api, type Video } from "../../lib/api"; +import { api, type Me, type Video } from "../../lib/api"; import { clearAll, notify } from "../../lib/notifications"; +import { qk } from "../../lib/queryKeys"; beforeAll(() => { class Stub { @@ -96,6 +101,7 @@ const vid = (id: string, position_seconds: number): Video => function renderModal(ui: ReactNode) { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + qc.setQueryData(qk.me(), me); return render({ui}); } @@ -108,6 +114,12 @@ afterEach(() => { volumes.length = 0; }); +// The signed-in account the player reads; related videos are on unless a test turns them off. +let me = { related_enabled: true } as Me; +beforeEach(() => { + me = { related_enabled: true } as Me; +}); + const lastStart = () => started[started.length - 1]; describe("PlayerModal — where each item starts", () => { @@ -434,7 +446,7 @@ describe("PlayerModal — the watch toggle after a queue advance", () => { await settle(); // A really is marked watched — that part is correct and must keep working. - expect(onState.mock.calls).toContainEqual(["A", "watched"]); + expect(onState.mock.calls.map((c) => c.slice(0, 2))).toContainEqual(["A", "watched"]); // B is not, and its toggle says so. Before the fix both toggles read "watchedUnmark" here. expect(screen.queryAllByTitle("player.watchedUnmark")).toHaveLength(0); expect(screen.queryAllByTitle("player.markWatched").length).toBeGreaterThan(0); @@ -449,7 +461,7 @@ describe("PlayerModal — the watch toggle after a queue advance", () => { await waitFor(() => expect(lastStart()?.videoId).toBe("A")); fireEvent.click(screen.getAllByTitle("player.markWatched")[0]!); - expect(onState.mock.calls).toContainEqual(["A", "watched"]); + expect(onState.mock.calls.map((c) => c.slice(0, 2))).toContainEqual(["A", "watched"]); expect(screen.queryAllByTitle("player.watchedUnmark").length).toBeGreaterThan(0); }); }); @@ -813,13 +825,95 @@ describe("PlayerModal — the side panel on a wide window", () => { fireEvent.click(screen.getByRole("tab", { name: "player.related.title" })); fireEvent.click(titleOf(await screen.findAllByRole("link", { name: "Video rel2" }))); // In the same player, from where the user left it. - expect(loaded).toEqual([{ videoId: "rel2", startSeconds: 120 }]); + await waitFor(() => expect(lastStart()).toEqual({ videoId: "rel2", start: 120 })); // The list stays the opened video's, to try the next one; it is not asked for again. expect(screen.getAllByRole("link", { name: "Video rel1" }).length).toBeGreaterThan(0); expect(asked.mock.calls).toEqual([["A", "topical", null]]); expect(screen.getByRole("button", { name: "player.back" })).toBeTruthy(); }); + // A related video played in place is the item it now is: tracked like a queue item, while the + // queue stays where it was and Back returns to the opened one. + const playRel2 = async ( + onState: (id: string, status: string, v?: Video) => void, + queue?: Video[], + ) => { + wideWindow(true); + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1"), related("rel2", 120)], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 40); + renderModal( + {}} + onState={onState} + />, + ); + act(() => playerEvents.onReady?.()); + fireEvent.click(screen.getByRole("tab", { name: "player.related.title" })); + fireEvent.click(titleOf(await screen.findAllByRole("link", { name: "Video rel2" }))); + await waitFor(() => expect(lastStart()?.videoId).toBe("rel2")); + }; + + it("wide: a related video played in place shows its own title and controls", async () => { + await playRel2(() => {}); + expect(screen.getByRole("heading", { name: "Video rel2" })).toBeTruthy(); + expect(screen.getAllByTestId("add-rel2").length).toBeGreaterThan(0); + expect(screen.getAllByTestId("download-rel2").length).toBeGreaterThan(0); + expect(screen.getAllByTitle("player.markWatched").length).toBeGreaterThan(0); + }); + + it("wide: a related video's progress is checkpointed, and Back resumes the opened one", async () => { + const saved = vi.spyOn(api, "saveProgress").mockImplementation(async (id: string) => ({ + video_id: id, + position_seconds: id === "rel2" ? 500 : 77, + })); + clock.current = 77; + await playRel2(() => {}); + clock.current = 500; + fireEvent.click(screen.getByRole("button", { name: "player.back" })); + // Where leaving it for the related one checkpointed it, not the queue's snapshot (40). + await waitFor(() => expect(lastStart()).toEqual({ videoId: "A", start: 77 })); + expect(saved.mock.calls.map((c) => c[0])).toContain("rel2"); + expect(screen.queryByRole("button", { name: "player.back" })).toBeNull(); + }); + + it("wide: a related video watched to its end is marked watched, and the queue stays put", async () => { + const onState = vi.fn(); + const B = vid("B", 0); + await playRel2(onState, [vid("A", 40), B]); + act(() => playerEvents.onStateChange?.({ data: 0 })); + expect(onState.mock.calls.map((c) => c.slice(0, 2))).toContainEqual(["rel2", "watched"]); + // Named for the host's notice: it is none of the host's own rows. + expect(onState.mock.calls[0]![2]).toMatchObject({ id: "rel2", title: "Video rel2" }); + await settle(); + expect(lastStart()?.videoId).toBe("rel2"); + }); + + it("leaves the related videos out when the instance turned them off", () => { + me = { related_enabled: false } as Me; + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + const A = vid("A", 0); + wideWindow(true); + const wideView = renderModal( + {}} onState={() => {}} />, + ); + expect(screen.queryByRole("tab", { name: "player.related.title" })).toBeNull(); + expect(screen.getByTestId("player-description")).toBeTruthy(); + wideView.unmount(); + wideWindow(false); + renderModal( + {}} onState={() => {}} />, + ); + expect(screen.queryByRole("button", { name: "player.related.title" })).toBeNull(); + }); + // The list keys are the page's, silenced while a layer is up — but this list is IN the layer. const press = (key: string) => act(() => void fireEvent.keyDown(document.activeElement ?? document.body, { key })); @@ -858,7 +952,7 @@ describe("PlayerModal — the side panel on a wide window", () => { expect(volumes).toEqual([]); // Enter on the row plays it, like its title's click. press("Enter"); - expect(loaded).toEqual([{ videoId: "rel2", startSeconds: 0 }]); + await waitFor(() => expect(lastStart()?.videoId).toBe("rel2")); }); it("stacked: the keyboard walks the list in the drawer too", async () => { @@ -902,7 +996,7 @@ describe("PlayerModal — the side panel on a wide window", () => { fireEvent.click(screen.getByRole("button", { name: "player.related.title" })); const drawer = await screen.findByRole("dialog", { name: "player.related.title" }); fireEvent.click(titleOf(await within(drawer).findAllByRole("link", { name: "Video rel1" }))); - expect(loaded).toEqual([{ videoId: "rel1", startSeconds: 0 }]); + await waitFor(() => expect(lastStart()?.videoId).toBe("rel1")); await waitFor(() => expect(screen.queryByRole("dialog", { name: "player.related.title" })).toBeNull(), ); @@ -911,6 +1005,89 @@ describe("PlayerModal — the side panel on a wide window", () => { expect(onClose).not.toHaveBeenCalled(); }); + const openDrawer = async () => { + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1")], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 0); + renderModal( + {}} onState={() => {}} />, + ); + act(() => playerEvents.onReady?.()); + fireEvent.click(screen.getByRole("button", { name: "player.related.title" })); + const drawer = await screen.findByRole("dialog", { name: "player.related.title" }); + await within(drawer).findAllByRole("link", { name: "Video rel1" }); + return { drawer }; + }; + + it("stacked: the video's keys are the drawer's while it is open", async () => { + wideWindow(false); + const { drawer } = await openDrawer(); + act(() => within(drawer).getByRole("button", { name: "common.close" }).focus()); + press("ArrowUp"); + press("ArrowDown"); + press("ArrowLeft"); + press("ArrowRight"); + expect(volumes).toEqual([]); + expect(seeks).toEqual([]); + }); + + it("stacked: the window regaining focus leaves it in the drawer", async () => { + wideWindow(false); + const { drawer } = await openDrawer(); + const row = titleOf(within(drawer).getAllByRole("link", { name: "Video rel1" })); + act(() => row.focus()); + act(() => void window.dispatchEvent(new Event("focus"))); + expect(document.activeElement).toBe(row); + }); + + it("going wide moves the drawer's list into the panel's tab, and narrow again reopens nothing", async () => { + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + vi.spyOn(api, "related").mockResolvedValue({ + items: [related("rel1")], + has_more: false, + next_cursor: null, + }); + const A = vid("A", 0); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + qc.setQueryData(qk.me(), me); + const ui = () => ( + + {}} onState={() => {}} /> + + ); + wideWindow(false); + const view = render(ui()); + fireEvent.click(screen.getByRole("button", { name: "player.related.title" })); + await screen.findByRole("dialog", { name: "player.related.title" }); + wideWindow(true); + view.rerender(ui()); + await waitFor(() => + expect(screen.queryByRole("dialog", { name: "player.related.title" })).toBeNull(), + ); + // The list the user was in is the panel's open tab now. + expect( + screen.getByRole("tab", { name: "player.related.title" }).getAttribute("aria-selected"), + ).toBe("true"); + wideWindow(false); + view.rerender(ui()); + expect(screen.queryByRole("dialog", { name: "player.related.title" })).toBeNull(); + }); + + it("wide: the panel's tabs keep the row's gap", () => { + wideWindow(true); + vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); + const A = vid("A", 0); + renderModal( + {}} onState={() => {}} />, + ); + const row = screen.getByRole("tablist").className.split(" "); + expect(row).toEqual(expect.arrayContaining(["flex", "flex-wrap", "gap-1"])); + }); + it("stacked: the card scrolls under the wheel, the video in it is still the volume", () => { wideWindow(false); vi.spyOn(api, "videoDetail").mockReturnValue(new Promise(() => {})); diff --git a/frontend/src/features/player/PlayerModal.tsx b/frontend/src/features/player/PlayerModal.tsx index 51bb6d24..9d2c0429 100644 --- a/frontend/src/features/player/PlayerModal.tsx +++ b/frontend/src/features/player/PlayerModal.tsx @@ -85,6 +85,7 @@ import { SURFACE } from "../../components/ui/surfaces"; import { IconLink, LinkButton } from "../../components/ui/LinkButton"; import { SectionLabel } from "../../components/ui/Headings"; import { useMediaQuery } from "../../lib/useMediaQuery"; +import { useMe } from "../../lib/useMe"; import Modal from "../../components/ui/Modal"; import Tabs from "../../components/ui/Tabs"; import { useTabPanel } from "../../components/ui/tabPanel"; @@ -168,6 +169,7 @@ export default function PlayerModal({ startIndex, onClose, onState, + onWrite, onRemoveFromList, }: { video: Video; @@ -180,7 +182,12 @@ export default function PlayerModal({ queue?: Video[]; startIndex?: number; onClose: () => void; - onState: (id: string, status: VideoStatus) => void; + /** The player's own marks (its watched toggle, the auto-mark at the end), for the item it is on — + * named by `video`, which may be none of the host's (a related video played in place). */ + onState: (id: string, status: VideoStatus, video?: Video) => void; + /** A write the related list made itself, for a host that refetches once the player closes and its + * writes have landed (the feed's player session). */ + onWrite?: (write: Promise) => void; /** Remove the playing video from the list it was opened from (the Playlists module passes its own * `removeItem`; the feed passes nothing — there is no list to remove it from). Returns whether it * actually happened, because the CONSEQUENCE is the player's: its launch queue is frozen, so it @@ -236,6 +243,19 @@ export default function PlayerModal({ // startAt only applies to the item we opened with; advanced items resume their own pos. const resumeAt = startAt != null && active.id === video.id ? startAt : active.position_seconds || 0; + // A related video played in place (the owner's pick) is the item it now is: its progress is + // checkpointed, it is marked watched at its end, and its own watched / playlist / download + // controls show — while the queue stays where it was, and Back returns to the opened item. It is + // the SUBJECT of the player from here down; `active` stays the queue's item, which the related + // list, the counter and prev/next belong to. A queue step leaves it. + const [related, setRelated] = useState<{ video: Video; startAt: number } | null>(null); + const [relatedOf, setRelatedOf] = useState(active.id); + if (relatedOf !== active.id) { + setRelatedOf(active.id); + setRelated(null); + } + const subject = related?.video ?? active; + const subjectResume = related ? related.startAt : resumeAt; const mountRef = useRef(null); const playerRef = useRef(null); // Keyboard/wheel controls. We keep focus on the modal card (not the cross-origin player @@ -251,11 +271,16 @@ export default function PlayerModal({ // useYtNativeControls, useYtBigView) where a fresh identity each render would spread into their // dep lists too. const focusModal = useCallback(() => cardRef.current?.focus(), []); + // The player's place on the layer stack (registered below), for what must defer to a layer above. + const layerIdRef = useRef(null); // What the player does on its own (turning ready, the pointer leaving the video, the window // regaining focus, a wheel over it, leaving fullscreen) takes the focus back only when it may have // slipped into the iframe or off the card. Taking it off a control the viewer tabbed to restarted - // their Tab from the first button. + // their Tab from the first button. Nor while a layer that owns the keyboard is open above the + // player (the related drawer, a dialog): its focus is outside the card by design, and the window + // regaining focus pulled it onto the card behind the scrim. const reclaimFocus = useCallback(() => { + if (layerIdRef.current != null && anyBlockingLayerAbove(layerIdRef.current)) return; const card = cardRef.current; const active = document.activeElement; if ( @@ -351,7 +376,7 @@ export default function PlayerModal({ // currently-playing id may differ from the active item. const [currentVideoId, setCurrentVideoId] = useState(active.id); const currentIdRef = useRef(active.id); - const navigated = currentVideoId !== active.id; + const navigated = currentVideoId !== subject.id; // Clip mode: the range being marked of the video that is playing, or null. Its panel sits under // the video, which is its preview; another video starts without one. const [clip, setClip] = useState(null); @@ -371,7 +396,7 @@ export default function PlayerModal({ // The picture the crop frame is drawn on. The iframe never tells its video's size: a short is // 9:16, anything else is taken as the player's 16:9, and the cut fits the frame to the real picture // (downloads/clip) — so a 4:3 video's frame drawn over its bars loses only what lies on the bars. - const picture = !navigated && active.is_short ? 9 / 16 : 16 / 9; + const picture = !navigated && subject.is_short ? 9 / 16 : 16 / 9; // The docked clip panel's height, measured (it wraps with the window and the text size): the video // above it ends there. const clipDockRef = useRef(null); @@ -488,7 +513,7 @@ export default function PlayerModal({ // Title/author of a navigated-to video, read from the player (free, no API call). const [liveData, setLiveData] = useState<{ title?: string; author?: string } | null>(null); // What is playing, by name: the panel's title, and the dialog's name (`titleId`) in either layout. - const heading = navigated ? (liveData?.title ?? t("player.loading")) : active.title; + const heading = navigated ? (liveData?.title ?? t("player.loading")) : subject.title; // IFrame Player API error code, when the current video can't play in the embed (e.g. an // auto-generated Topic "Art Track" with embedding disabled → 101/150, or removed/private → // 100). We overlay a friendly message + "Open on YouTube" instead of YouTube's bare error. @@ -499,9 +524,13 @@ export default function PlayerModal({ const loadVideo = useStableCallback((id: string, start: number | null) => { const p = playerRef.current; if (!p || typeof p.loadVideoById !== "function") return; - // Explicit start wins; otherwise resume the active item, start others at 0. + // Explicit start wins; otherwise resume the item the player is on, start others at 0. const startSeconds = - start != null ? start : id === active.id ? (sessionPosRef.current.get(id) ?? resumeAt) : 0; + start != null + ? start + : id === subject.id + ? (sessionPosRef.current.get(id) ?? subjectResume) + : 0; setPlayerError(null); p.loadVideoById({ videoId: id, startSeconds: startSeconds || 0 }); currentIdRef.current = id; @@ -515,18 +544,18 @@ export default function PlayerModal({ // one writer arrives late by construction: useYtProgress's cleanup for the OUTGOING item runs // after the commit that already advanced `active`, and near the end of a clip it auto-marks that // item watched (playerModalHooks.ts `maybeAutoWatch`). Keyed, that stray write is simply not this - // item's, so the toggle falls back to `active.status` and no ordering has to be argued about. + // item's, so the toggle falls back to `subject.status` and no ordering has to be argued about. // Unkeyed it landed last and painted the NEXT video as watched; a corrective click then sent // onState(next, "new") for an item that was already new. const [status, setStatus] = useState<{ id: string; value: VideoStatus }>({ id: active.id, value: active.status, }); - const watched = (status.id === active.id ? status.value : active.status) === "watched"; + const watched = (status.id === subject.id ? status.value : subject.status) === "watched"; const setWatched = (on: boolean) => { const next = on ? "watched" : "new"; - setStatus({ id: active.id, value: next }); - onState(active.id, next); + setStatus({ id: subject.id, value: next }); + onState(subject.id, next, subject); }; // On queue advance (active changes), point `currentVideoId` at the new item — DURING that render, @@ -537,9 +566,9 @@ export default function PlayerModal({ // The watch status is NOT synced here: it is keyed by id above, which is both stale-proof and // immune to the late writer this block cannot see (see that comment). const [syncedTo, setSyncedTo] = useState(active.id); - if (syncedTo !== active.id) { - setSyncedTo(active.id); - setCurrentVideoId(active.id); + if (syncedTo !== subject.id) { + setSyncedTo(subject.id); + setCurrentVideoId(subject.id); } // `currentIdRef` must NOT move into the block above, and the reason is EFFECT ORDERING, which the @@ -553,8 +582,8 @@ export default function PlayerModal({ // frozen launch queue's stale position — which is the exact bug the session-position map exists // to prevent (748 instead of 830, in the test's numbers). useEffect(() => { - currentIdRef.current = active.id; - }, [active.id]); + currentIdRef.current = subject.id; + }, [subject.id]); const seekTo = useStableCallback((seconds: number) => { const p = playerRef.current; @@ -646,6 +675,8 @@ export default function PlayerModal({ if (clipOpenRef.current) return; const { autoMode: a, loopMode: l } = modeRef.current; if (l === "one") return replayCurrent(); + // A related video's end is not the queue's: the list beside it is where the next one is picked. + if (related) return; const { queue: q, index: i } = navRef.current; if (!q || q.length === 0) return; const wrap = l === "all"; @@ -681,10 +712,19 @@ export default function PlayerModal({ // list in place, to try the next one (the owner's pick). const [panelTab, setPanelTab] = useState<"description" | "related">("description"); const [relatedOpen, setRelatedOpen] = useState(false); + // The drawer is the stacked layout's: going wide while it is open, the list moves into the panel's + // tab instead. Left open, it reopened unasked when the window narrowed again. + if (wide && relatedOpen) { + setRelatedOpen(false); + setPanelTab("related"); + } const panelTabs = useTabPanel(); + // Turned off on this instance (a daily limit of 0): no tab, no button — nothing that could only + // be refused. + const relatedOn = useMe().related_enabled; const playRelated = (v: Video, startAt: number) => { setRelatedOpen(false); - loadVideo(v.id, startAt); + setRelated({ video: v, startAt }); }; const descPop = useAnchoredPopover({ open: showDesc, @@ -712,7 +752,6 @@ export default function PlayerModal({ // This player is a dismissable layer: register it so a Modal or popover opened ABOVE it (e.g. the // download dialog, or an anchored menu) takes Escape first and this player defers. Its root lets // the keyboard walk a list inside it (the related videos). - const layerIdRef = useRef(null); useEffect(() => { const id = pushLayer({ root: () => dialogRef.current }); layerIdRef.current = id; @@ -764,6 +803,15 @@ export default function PlayerModal({ if (dialogRef.current) trapTab(e, dialogRef.current, cardRef.current); return; } + // A layer that owns the keyboard above the player (the related drawer, a menu, a dialog) has the + // keys: on its ✕ the arrows changed the volume and seeked the video behind it. Escape is the + // exception, and `close` decides for itself which layer it closes. + if ( + e.key !== "Escape" && + layerIdRef.current != null && + anyBlockingLayerAbove(layerIdRef.current) + ) + return; dispatch(e, "ytPlayer", { clip: () => { if (clip) closeClip(); @@ -838,8 +886,8 @@ export default function PlayerModal({ // the end — recreated per active item. The once-bound event closures read fresh queue/mode/volume via // the refs passed in. See lib/playerModalHooks. useYtProgress({ - activeId: active.id, - resumeAt, + activeId: subject.id, + resumeAt: subjectResume, mountRef, playerRef, currentIdRef, @@ -1051,9 +1099,9 @@ export default function PlayerModal({ {/* Both work in fullscreen now: add-to-playlist's popover and download's Modal both portal into the fullscreen element rather than (a body-portaled node is painted underneath the element holding the lock, i.e. invisible). */} - {!navigated && } + {!navigated && } {!navigated && ( - + )} {/* Clip mode here: the clip panel takes this bar's place. */} {playerError == null && ( @@ -1266,7 +1314,7 @@ export default function PlayerModal({ )} - {!wide && ( + {!wide && relatedOn && ( setRelatedOpen(true)} @@ -1286,14 +1334,16 @@ export default function PlayerModal({ maxWidth="max-w-sm" initialFocus="close" > - + )} - {navigated && ( + {(navigated || related) && ( +
+ ))} + + + ); + } + withLayout( + (_, n) => ({ left: 0, top: n * 100 }), + () => { + render(); + const layer = pushLayer({ root: () => screen.getByTestId("layer") }); + try { + act(() => screen.getByTestId("r1-hide").focus()); + press("j"); + expect(focused()).toBe("r2"); + act(() => screen.getByTestId("r2-hide").focus()); + fireEvent.click(screen.getByTestId("r2-hide")); + expect(document.activeElement).toBe(document.body); + press("j"); + expect(focused()).toBe("r3"); + } finally { + removeLayer(layer); + } + }, + ); + }); }); describe("useListNav · leaving the list, and naming the item", () => { diff --git a/frontend/src/lib/listNav.ts b/frontend/src/lib/listNav.ts index a0db0539..6d688fea 100644 --- a/frontend/src/lib/listNav.ts +++ b/frontend/src/lib/listNav.ts @@ -1,5 +1,5 @@ import { useEffect, useEffectEvent, useState, type RefObject } from "react"; -import { dispatch, useKeyScope } from "./keymap"; +import { dispatch, setListInTopLayerProbe, useKeyScope } from "./keymap"; import { anyBlockingLayer, topBlockingLayerRoot } from "./layerStack"; import { scrollParent } from "./listPaging"; import { focusIsUnclaimed } from "./useFocusKeeper"; @@ -101,6 +101,12 @@ function current(): { list: NavList; index: number; item: HTMLElement } | null { * move between the items, so a surface that also binds them (the player's seek) leaves them alone. */ export const focusInNavList = (): boolean => current() !== null; +// The help sheet lists the list keys over a layer only when this says there is a list to walk in it. +setListInTopLayerProbe(() => { + const layer = topBlockingLayerRoot(); + return !!layer && [...lists.keys()].some((el) => layer.contains(el)); +}); + /** The mounted items that belong to `el` itself (not to a list nested inside it), in index order. */ function ownItems(el: HTMLElement): HTMLElement[] { return [...el.querySelectorAll(`[${INDEX}]`)] @@ -335,10 +341,16 @@ function onKey(e: KeyboardEvent): void { if (move(dx, dy)) ev.preventDefault(); }; // The list keys act on the page, so an open layer silences them — unless the focus is in a list - // INSIDE the topmost layer: that list is what the user is in (the player's related videos). + // INSIDE the topmost layer: that list is what the user is in (the player's related videos). Or was, + // a moment ago: a row marked from its menu leaves the list and the focus falls to , and `j` + // goes on from its slot (`slotOfRemoved`) as it does on the page. const layer = topBlockingLayerRoot(); const cur = current(); - const inTopLayer = !!layer && !!cur && layer.contains(cur.item); + const inTopLayer = + !!layer && + (cur + ? layer.contains(cur.item) + : focusIsUnclaimed(document.activeElement) && !!last && layer.contains(last.list.el)); dispatch( e, "list", diff --git a/frontend/src/lib/playerModalHooks.progress.dom.test.tsx b/frontend/src/lib/playerModalHooks.progress.dom.test.tsx index d4043ad0..6fcc56e1 100644 --- a/frontend/src/lib/playerModalHooks.progress.dom.test.tsx +++ b/frontend/src/lib/playerModalHooks.progress.dom.test.tsx @@ -186,7 +186,7 @@ describe("patchCachedProgress", () => { expect(after.pages[0]?.items[0]?.position_seconds).toBe(830); }); - it("patches a related list too — a row played from it resumes where it was left", () => { + it("patches a related list too — its rows open the same player", () => { const qc = new QueryClient(); const key = qk.related("SRC", "topical"); qc.setQueryData(key, { pages: [{ items: [video()], next_cursor: null }], pageParams: [null] }); diff --git a/frontend/src/lib/progressCache.ts b/frontend/src/lib/progressCache.ts index 673bbf99..46bc1351 100644 --- a/frontend/src/lib/progressCache.ts +++ b/frontend/src/lib/progressCache.ts @@ -84,3 +84,21 @@ export function patchCachedProgress(qc: QueryClient, positions: Map({ queryKey: qk.ytSearch() }, patch); qc.setQueriesData({ queryKey: qk.related() }, patch); } + +/** Write one video's change (its status, its Watch-later mark) into every cached related list — + * both regions', every opened video's. Those lists are never refetched (another scrape and a slot + * of the day's cap), and they leave out what the user has seen: a video marked anywhere else — a + * feed card, a playlist, the player itself — still showed in one opened again later. So every + * state write calls it once the server has the change. */ +export function patchRelated(qc: QueryClient, id: string, change: Partial