peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(catalog): a video's related videos, without what the user has seen

Browse Source 
GET /api/search/related/{video_id} reads the list YouTube shows beside a
video from the website's InnerTube "next" endpoint (no quota), and brings
it into the catalog the way live-search results come in. It leaves out
what the user watched or hid, here or in their Takeout history (unless
taken back to "new" since), blocked channels and the video itself, and
pages on until 20 are gathered (at most 4 pages). region=topical|hu picks
the locale the list is filled in. Its own per-user daily cap
(related_daily_limit_per_user, one event per opened list); a failed
scrape answers 422 youtube/unavailable.

The InnerTube request (live key, consent cookie, errors) moves into
youtube/innertube.py, shared by the search scrape and the related one.
This commit is contained in:
peter
2026-10-24 16:00:37 +02:00
parent 4695e1a62f
commit 8b37c470ba
19 changed files with 2263 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/config.py
+3
View File
@@ -113,6 +113,9 @@ class Settings(BaseSettings):
# each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
# the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
search_daily_limit_per_user: int = 70
# Per-user cap on related-video lists opened per day (the InnerTube watch page, no quota; each
# list's enrich is 1 unit per 50 videos). A pure anti-abuse rate limit. Admin-tunable.
related_daily_limit_per_user: int = 200
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
search_source: SearchSource = "scrape"
backend/app/quota.py
+1
View File
@@ -44,6 +44,7 @@ class QuotaAction:
VIDEOS_ENRICH = "videos_enrich"
VIDEOS_LOOKUP = "videos_lookup"
VIDEOS_SEARCH = "videos_search"
VIDEOS_RELATED = "videos_related"
CHANNELS_DISCOVER = "channels_discover"
CHANNELS_EXPLORE = "channels_explore"
CHANNELS_SUBSCRIBE = "channels_subscribe"
backend/app/reader/queries.py
+30
View File
@@ -319,6 +319,36 @@ def held_by(db: Session, user: User, video_ids: list[str]) -> set[str]:
)
def seen_by(db: Session, user: User, video_ids: list[str]) -> set[str]:
"""Which of these videos the user has already seen or set aside: watched or hidden here, or in
their uploaded Takeout history unless they took it back to "new" here since. The ids need not
be in the catalog — a Takeout row names a video the catalog may never have had — so a list
of what YouTube suggests can drop them before anything is ingested."""
if not video_ids:
return set()
marked = select(VideoState.video_id).where(
owned_by(VideoState.user_id, user),
VideoState.video_id.in_(video_ids),
VideoState.status.in_(("watched", "hidden")),
)
unmarked_since = (
select(VideoUnmark.id)
.where(
owned_by(VideoUnmark.user_id, user),
VideoUnmark.video_id == TakeoutItem.video_id,
VideoUnmark.at > TakeoutItem.at,
)
.exists()
)
in_history = select(TakeoutItem.video_id).where(
owned_by(TakeoutItem.user_id, user),
TakeoutItem.kind == TakeoutKind.WATCHED,
TakeoutItem.video_id.in_(video_ids),
~unmarked_since,
)
return set(db.scalars(marked)) | set(db.scalars(in_history))
def feed_rows_by_ids(db: Session, user: User, video_ids: list[str]) -> list[dict]:
"""These videos serialized with the feed's projection (their channel, this user's watch state,
saved), in the order given."""
backend/app/routes/search.py
+90 -2
View File
@@ -13,10 +13,16 @@ this search-discovered "noise" by default. Shorts and currently-live/upcoming vi
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
dropped from the search response up front).
A video's related videos (`/related/{video_id}`, the list YouTube shows beside it) come in the same
way: read from the website (`related_scrape`), ingested as search finds are, under their own daily
cap — with what this user has already seen left out.
This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the
catalog is `catalog/search`."""
from fastapi import APIRouter, Depends
from typing import Literal
from fastapi import APIRouter, Depends, Path
from pydantic import BaseModel
from sqlalchemy.orm import Session
@@ -27,7 +33,7 @@ from app.catalog import search as catalog_search
from app.db import get_db
from app.models import User
from app.reader import queries as reader_queries
from app.youtube import search_scrape
from app.youtube import related_scrape, search_scrape
from app.youtube.client import YouTubeClient, YouTubeError
from app.refusal import refused
from app.routes._common import youtube_refusal
@@ -38,6 +44,13 @@ router = APIRouter(prefix="/api/search", tags=["search"])
# search.list costs this many quota units per page.
SEARCH_COST = 100
# A related list is filled to this many unseen videos, reading at most this many pages (about 20
# videos each, measured) — the ones the user has seen are left out, so a page alone can fall short.
RELATED_TARGET = 20
RELATED_MAX_PAGES = 4
# The locale YouTube fills a related list in: on the video's topic, or with what trends in Hungary.
RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")}
@router.get("/youtube")
def search_youtube(
@@ -201,3 +214,78 @@ def clear_search(
removed = catalog_search.clear(db, user, ids)
db.commit()
return {"ok": True, "removed": removed}
@router.get("/related/{video_id}")
def related_videos(
video_id: str = Path(max_length=16),
cursor: str | None = None,
region: Literal["topical", "hu"] = "topical",
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
"""The videos YouTube shows beside this one, materialised into the catalog like search results,
without the ones this user has watched or hidden here or in their Takeout history, and without
blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads
further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`)."""
daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user")
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED)
if used >= daily_limit:
log.info(
"a related list was refused: the per-user daily cap is spent",
extra=logs.event("related.refused", reason="daily_cap", limit=daily_limit),
)
raise refused(429, "related_daily_limit", limit=daily_limit)
hl, gl = RELATED_REGIONS[region]
blocked = queries.blocked_ids(db, user)
collected: list[str] = []
next_cursor = cursor
with YouTubeClient(db, user) as yt:
for _ in range(RELATED_MAX_PAGES):
try:
page = related_scrape.related(video_id, continuation=next_cursor, hl=hl, gl=gl)
except related_scrape.ScrapeError as exc:
if collected:
break # keep what earlier pages already gathered
log.warning(
"the related-videos scrape failed",
extra=logs.event("related.scrape_failed", error=str(exc)),
)
raise youtube_refusal(exc) from exc
ids = [it["id"] for it in page["items"]]
drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id}
candidates = [
it
for it in page["items"]
if it["id"] not in drop and it["channel_id"] not in blocked
]
# As in the search above: an ingest failure answers 422, not a bare 500.
try:
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
except Exception as exc: # any ingest failure, not just YouTubeError
db.rollback()
log.exception(
"a related list's ingest failed",
extra=logs.event(
"related.ingest_failed",
candidates=len(candidates),
collected=len(collected),
),
)
if collected:
break
raise youtube_refusal(exc) from exc
next_cursor = page["next_page_token"]
if len(collected) >= RELATED_TARGET or not next_cursor:
break
# One opened list is one event against the cap, however many pages it read; reading further
# (a cursor) is the same list.
if cursor is None:
quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED)
return {
"items": reader_queries.feed_rows_by_ids(db, user, collected),
"next_cursor": next_cursor,
"has_more": bool(next_cursor),
}
backend/app/sysconfig.py
+9
View File
@@ -140,6 +140,15 @@ SPECS: tuple[ConfigSpec, ...] = (
max=100_000,
applies=LIVE,
),
ConfigSpec(
"related_daily_limit_per_user",
"int",
"quota",
"related_daily_limit_per_user",
min=0,
max=100_000,
applies=LIVE,
),
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE),
ConfigSpec(
backend/app/youtube/innertube.py
+111
View File
@@ -0,0 +1,111 @@
"""The website's own InnerTube API (`youtubei/v1/*`), read the way the website reads it — no quota.
The zero-quota search (`search_scrape`) and the related videos (`related_scrape`) both POST here with
the public WEB key and a client version, read out of a live results page. This deliberately calls an
undocumented endpoint: if YouTube changes the JSON shape or blocks the host, each caller says so as a
`ScrapeError`, never as an empty answer.
The SOCS=CAI consent-skip cookie is sent like the Shorts probe's, so a server-side request isn't
bounced to consent.youtube.com."""
import re
import httpx
from app import logs
log = logs.get_logger(__name__)
_API_URL = "https://www.youtube.com/youtubei/v1/"
_RESULTS_URL = "https://www.youtube.com/results"
# Public WEB InnerTube key + a recent client version. These are also embedded in every results
# page; we fetch the live pair once and cache it, falling back to these constants.
_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"
_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
# SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
# redirected to consent.youtube.com).
"Cookie": "SOCS=CAI",
"Accept-Language": "en-US,en;q=0.9",
}
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
_config: tuple[str, str] | None = None
class ScrapeError(Exception):
"""An InnerTube read could not be completed (network, blocked, or unexpected shape)."""
def runs_text(node: dict | None) -> str | None:
"""The text of an InnerTube text node: a `simpleText`, or its `runs` joined."""
if not node:
return None
if node.get("simpleText"):
return node["simpleText"]
runs = node.get("runs") or []
if runs:
return "".join(r.get("text", "") for r in runs)
return None
def _get_config(client: httpx.Client) -> tuple[str, str]:
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
the life of the process — and when those went stale, search returned nothing, with no error
anywhere. Now the next call simply tries the live pair again."""
global _config
if _config is not None:
return _config
try:
html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
except httpx.HTTPError as exc:
log.warning(
"the InnerTube config could not be fetched; falling back to the pinned key",
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
)
return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
key = _KEY_RE.search(html)
ver = _VER_RE.search(html)
if key and ver:
_config = (key.group(1), ver.group(1))
return _config
log.warning(
"the results page carried no InnerTube config; falling back to the pinned key",
extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
)
return (
key.group(1) if key else _FALLBACK_KEY,
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
)
def post(endpoint: str, body: dict, *, hl: str = "en", gl: str = "US") -> dict:
"""POST `body` to `youtubei/v1/<endpoint>` as the WEB client in the `hl`/`gl` locale and return
the decoded JSON object. A network failure, an HTTP error or a body that is not a JSON object (a
block, consent or captcha page) raises `ScrapeError`; the caller judges the shape."""
with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
api_key, client_version = _get_config(client)
context = {
"client": {"clientName": "WEB", "clientVersion": client_version, "hl": hl, "gl": gl}
}
try:
resp = client.post(
_API_URL + endpoint, params={"key": api_key}, json={"context": context, **body}
)
resp.raise_for_status()
data = resp.json()
except httpx.HTTPError as exc:
raise ScrapeError(f"InnerTube request failed: {exc}") from exc
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
raise ScrapeError("InnerTube returned a non-JSON response") from exc
if not isinstance(data, dict):
raise ScrapeError("InnerTube returned an unexpected shape")
return data
backend/app/youtube/related_scrape.py
+150
View File
@@ -0,0 +1,150 @@
"""A video's related videos, read from the watch page's InnerTube `next` endpoint — no quota.
The Data API has no related-videos call any more, so this is the only source: the same list the
website shows beside a playing video. A page holds about 20 videos and a continuation token for the
next 20. The returned page has the search scrape's shape (``{"items": [...], "next_page_token"}``)
so the catalog ingests it the same way (`catalog.search.ingest_candidates`).
YouTube draws each item as a `lockupViewModel` now; the older `compactVideoRenderer` is read too, so
a rollback on its side does not empty the list. An item that names no single channel — a video by
several channels draws an avatar stack, not one avatar — is left out: the catalog row needs its
channel up front, and the stack does not say which one uploaded it.
`hl`/`gl` pick the locale YouTube fills the list in: ``en``/``US`` keeps it on the video's topic,
``hu``/``HU`` mixes in what is trending in Hungary (measured)."""
from app.youtube.innertube import ScrapeError, post, runs_text
__all__ = ["ScrapeError", "related"]
# Where a first page and a continuation page put their item list.
_FIRST = (
"contents",
"twoColumnWatchNextResults",
"secondaryResults",
"secondaryResults",
"results",
)
_APPENDS = ("appendContinuationItemsAction", "reloadContinuationItemsCommand")
def _at(node: object, *path: str | int) -> object:
"""The value under `path` in nested dicts/lists, or None where the path stops."""
for key in path:
if isinstance(node, dict) and isinstance(key, str):
node = node.get(key)
elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node):
node = node[key]
else:
return None
return node
def _item(vid: str, channel_id: object, channel_title: object, title: object, sources: object):
if not isinstance(channel_id, str) or not channel_id:
return None
thumbs = sources if isinstance(sources, list) else []
return {
"id": vid,
"channel_id": channel_id,
"channel_title": channel_title if isinstance(channel_title, str) else None,
"title": title if isinstance(title, str) else None,
"published_at": None, # only a relative "1y ago" here; the enrich fills the real date
"thumbnail_url": _at(thumbs, -1, "url"),
"live_broadcast": "none", # live/upcoming are judged after the enrich, by live_status
}
def _lockup(node: dict) -> dict | None:
if node.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO" or not node.get("contentId"):
return None
meta = _at(node, "metadata", "lockupMetadataViewModel")
channel_id = _at(
meta,
"image",
"decoratedAvatarViewModel",
"rendererContext",
"commandContext",
"onTap",
"innertubeCommand",
"browseEndpoint",
"browseId",
)
channel_title = _at(
meta, "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0
)
return _item(
node["contentId"],
channel_id,
_at(channel_title, "text", "content"),
_at(meta, "title", "content"),
_at(node, "contentImage", "thumbnailViewModel", "image", "sources"),
)
def _compact(node: dict) -> dict | None:
vid = node.get("videoId")
if not vid:
return None
byline = _at(node, "longBylineText", "runs", 0)
return _item(
vid,
_at(byline, "navigationEndpoint", "browseEndpoint", "browseId"),
_at(byline, "text"),
runs_text(node.get("title")),
_at(node, "thumbnail", "thumbnails"),
)
def _items(data: dict) -> list | None:
"""The page's item list (first page or continuation), or None when it has neither frame."""
first = _at(data, *_FIRST)
if isinstance(first, list):
return first
# A video YouTube no longer has keeps the watch page's frame but draws an "isn't available"
# promo in it and no related column: nothing related, said by YouTube itself.
if isinstance(_at(data, *_FIRST[:2]), dict):
return []
for endpoint in data.get("onResponseReceivedEndpoints") or []:
for key in _APPENDS:
items = _at(endpoint, key, "continuationItems")
if isinstance(items, list):
return items
return None
def related(
video_id: str, continuation: str | None = None, *, hl: str = "en", gl: str = "US"
) -> dict:
"""One page of `video_id`'s related videos (`continuation` = the token a previous page gave).
A video YouTube no longer has is an empty page. An answer without the watch page's frame is a
`ScrapeError` — not an empty page, which would read as "nothing related"."""
body = {"continuation": continuation} if continuation else {"videoId": video_id}
data = post("next", body, hl=hl, gl=gl)
items = _items(data)
if items is None:
raise ScrapeError("InnerTube returned no related list")
out: list[dict] = []
seen: set[str] = set()
token = None
for entry in items:
if not isinstance(entry, dict):
continue
if "lockupViewModel" in entry:
item = _lockup(entry["lockupViewModel"])
elif "compactVideoRenderer" in entry:
item = _compact(entry["compactVideoRenderer"])
else:
token = token or _at(
entry,
"continuationItemRenderer",
"continuationEndpoint",
"continuationCommand",
"token",
)
continue
if item and item["id"] not in seen:
seen.add(item["id"])
out.append(item)
return {"items": out, "next_page_token": token if isinstance(token, str) else None}
backend/app/youtube/search_scrape.py
+11 -102
View File
@@ -6,9 +6,9 @@ calls — and read the result list straight out of the JSON. The returned video
through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does,
so the search itself is effectively free and only the (cheap, shared) enrichment is charged.
The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't
bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube
changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``.
The request itself (the live key, the consent cookie) is `innertube`'s. This deliberately scrapes an
undocumented endpoint: if YouTube changes the JSON shape or blocks the host, an admin can flip
``search_source`` back to ``api``.
The returned page mirrors :meth:`YouTubeClient.search_videos` exactly
(``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the
@@ -17,87 +17,16 @@ only difference is the cursor is an InnerTube *continuation token*, not a Data A
the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"``
and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path)."""
import re
from app.youtube.innertube import ScrapeError, post, runs_text
import httpx
__all__ = ["ScrapeError", "search"]
from app import logs
log = logs.get_logger(__name__)
_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search"
_RESULTS_URL = "https://www.youtube.com/results"
# Public WEB InnerTube key + a recent client version. These are also embedded in every results
# page; we fetch the live pair once and cache it, falling back to these constants.
_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"
# sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==).
_TYPE_VIDEO = "EgIQAQ=="
_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
# SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
# redirected to consent.youtube.com).
"Cookie": "SOCS=CAI",
"Accept-Language": "en-US,en;q=0.9",
}
# The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands)
# put their result list under.
_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"}
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
_config: tuple[str, str] | None = None
class ScrapeError(Exception):
"""A scrape-based search could not be completed (network, blocked, or unexpected shape)."""
def _runs_text(node: dict | None) -> str | None:
if not node:
return None
if node.get("simpleText"):
return node["simpleText"]
runs = node.get("runs") or []
if runs:
return "".join(r.get("text", "") for r in runs)
return None
def _get_config(client: httpx.Client) -> tuple[str, str]:
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
the life of the process — and when those went stale, search returned nothing, with no error
anywhere. Now the next search simply tries the live pair again."""
global _config
if _config is not None:
return _config
try:
html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
except httpx.HTTPError as exc:
log.warning(
"the InnerTube config could not be fetched; falling back to the pinned key",
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
)
return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
key = _KEY_RE.search(html)
ver = _VER_RE.search(html)
if key and ver:
_config = (key.group(1), ver.group(1))
return _config
log.warning(
"the results page carried no InnerTube config; falling back to the pinned key",
extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
)
return (
key.group(1) if key else _FALLBACK_KEY,
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
)
def _video_item(vr: dict) -> dict | None:
@@ -116,7 +45,7 @@ def _video_item(vr: dict) -> dict | None:
"id": vid,
"channel_id": channel_id,
"channel_title": channel_title,
"title": _runs_text(vr.get("title")),
"title": runs_text(vr.get("title")),
"published_at": None, # scrape exposes only a relative time; enrich fills the real date
"thumbnail_url": thumbs[-1]["url"] if thumbs else None,
"live_broadcast": "none", # live/upcoming filtered post-enrich via live_status
@@ -167,34 +96,14 @@ def search(q: str, continuation: str | None = None) -> dict:
`continuation` is the InnerTube continuation token from a previous page (None for the first
page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as
YouTubeClient.search_videos so the route handles both sources identically."""
with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
api_key, client_version = _get_config(client)
context = {
"client": {
"clientName": "WEB",
"clientVersion": client_version,
"hl": "en",
"gl": "US",
}
}
body: dict = {"context": context}
if continuation:
body["continuation"] = continuation
else:
body["query"] = q
body["params"] = _TYPE_VIDEO
try:
resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body)
resp.raise_for_status()
data = resp.json()
except httpx.HTTPError as exc:
raise ScrapeError(f"InnerTube request failed: {exc}") from exc
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
raise ScrapeError("InnerTube returned a non-JSON response") from exc
body: dict = (
{"continuation": continuation} if continuation else {"query": q, "params": _TYPE_VIDEO}
)
data = post("search", body)
# Even a search with no results carries its result list's frame. Without one this is not "no
# results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so.
if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys():
if not _RESULT_FRAMES & data.keys():
raise ScrapeError("InnerTube returned an unexpected shape")
items: list[dict] = []
_collect_videos(data, items, set())
backend/tests/fixtures/innertube_next_continuation.json
Vendored
+527
View File
@@ -0,0 +1,527 @@
{
"onResponseReceivedEndpoints": [
{
"appendContinuationItemsAction": {
"continuationItems": [
{
"lockupViewModel": {
"contentImage": {
"thumbnailViewModel": {
"image": {
"sources": [
{
"url": "https://i.ytimg.com/vi/_HfdncCbMOE/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLARvB39pkdsiuUUQTsZjkhvbKrfcQ",
"width": 168,
"height": 94
},
{
"url": "https://i.ytimg.com/vi/_HfdncCbMOE/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLB15aZU5-tzwYCEbNTnZK3UX6ZCzQ",
"width": 336,
"height": 188
}
]
},
"overlays": [
{
"thumbnailBottomOverlayViewModel": {
"badges": [
{
"thumbnailBadgeViewModel": {
"text": "1:08:18",
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT",
"animationActivationTargetId": "_HfdncCbMOE",
"animatedText": "Now playing",
"animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE"
}
}
]
}
}
]
}
},
"metadata": {
"lockupMetadataViewModel": {
"title": {
"content": "MIT 6.S191 (2025): Large Language Models (Liquid AI)"
},
"image": {
"decoratedAvatarViewModel": {
"avatar": {
"avatarViewModel": {
"image": {
"sources": [
{
"url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj",
"width": 68,
"height": 68
}
]
},
"avatarImageSize": "AVATAR_SIZE_M"
}
},
"a11yLabel": "Go to channel Alexander Amini",
"rendererContext": {
"commandContext": {
"onTap": {
"innertubeCommand": {
"browseEndpoint": {
"browseId": "UCtslD4DGH6PKyG_1gFAX7sg"
}
}
}
}
}
}
},
"metadata": {
"contentMetadataViewModel": {
"metadataRows": [
{
"metadataParts": [
{
"text": {
"content": "Alexander Amini"
}
}
]
},
{
"metadataParts": [
{
"text": {
"content": "62K"
},
"accessibilityLabel": "62 thousand views",
"leadingIcon": {
"name": "PLAY_ARROW_OUTLINED",
"height": 12,
"width": 12,
"marginEndToken": {
"spacing": 2
}
}
},
{
"text": {
"content": "1y ago"
},
"accessibilityLabel": "1 year ago"
}
]
}
],
"delimiter": " "
}
}
}
},
"contentId": "_HfdncCbMOE",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"themedPalette": {
"lightThemePalette": {
"baseBackground": 4294964720,
"raisedBackground": 4294964720,
"additiveBackground": 566850118,
"textPrimary": 4281800969,
"textSecondary": 4285876787,
"outline": 860759827,
"keyLight": 4292983720,
"collectionStack": 4291337092,
"invertedBackground": 4287578649,
"textPrimaryInverse": 4294964720,
"textDisabled": 2135828243,
"dropShadow": 432632390,
"cardOutline": 180974150,
"overlayBackground": 3426162953,
"overlayTextPrimary": 4294964720,
"solidWash": 4283311891,
"solidRim": 4281800969,
"overlaySolidWash": 4294699488,
"overlaySolidRim": 4294964720,
"tonalWash": 2583688688,
"tonalRim": 202310912,
"overlayTonalWash": 436205040,
"overlayTonalRim": 436205040,
"outlineRim": 1280190227,
"overlayOutlineRim": 871618504,
"solidWashInverse": 4294170568,
"solidRimInverse": 2147215840,
"tonalWashInverse": 654308848,
"tonalRimInverse": 654043616,
"outlineRimInverse": 654308848,
"solidBackground": 4283311891,
"solidBackgroundInverse": 4291337092,
"tonalBackground": 868785028,
"tonalBackgroundInverse": 435410888,
"outlineInverse": 872147424,
"directionalLightStopPrimary": 4294699488,
"directionalLightStopSecondary": 4294964720,
"primary": 4279174400,
"filledStateLayer": 4281800969,
"tonalTouchLight": 4294964720,
"tonalHoverLight": 3439061472,
"tonalStateLayer": 420414720,
"overlayDisabledContainer": 871618504,
"overlayTonalTouchLight": 1291577824,
"overlayTonalHoverLight": 1291577824,
"overlayTonalStateLayer": 436205040,
"disabledContainer": 420414720,
"filledTouchLight": 4285876787,
"filledHoverLight": 4284627235,
"onOverlayDisabledContainer": 2583688688,
"overlayTonalContainer": 2583423456,
"overlayOutline": 872412656,
"onFilledContainer": 4294964720,
"filledContainer": 4280027906,
"tonalContainer": 641145097,
"overlayPrimary": 4294964720,
"onOverlayFilledContainer": 4279174400,
"overlayFilledTouchLight": 4294964720,
"overlayFilledHoverLight": 4294699488,
"overlayFilledStateLayer": 4292983720,
"overlayFilledContainer": 4294170568,
"onDisabledContainer": 1712260352,
"temporaryOutline": 860759827
},
"darkThemePalette": {
"baseBackground": 4280027906,
"raisedBackground": 4281800969,
"additiveBackground": 734622278,
"textPrimary": 4294699488,
"textSecondary": 4292983720,
"outline": 1072945096,
"keyLight": 4284627235,
"collectionStack": 4285876787,
"invertedBackground": 4293107837,
"textPrimaryInverse": 4280027906,
"textDisabled": 2146686920,
"dropShadow": 432632390,
"cardOutline": 180974150,
"overlayBackground": 3426162953,
"overlayTextPrimary": 4294964720,
"solidWash": 4294699488,
"solidRim": 4294964720,
"overlaySolidWash": 4294699488,
"overlaySolidRim": 4294964720,
"tonalWash": 217307080,
"tonalRim": 218101232,
"overlayTonalWash": 436205040,
"overlayTonalRim": 436205040,
"outlineRim": 871618504,
"overlayOutlineRim": 871618504,
"solidWashInverse": 4285876787,
"solidRimInverse": 871618504,
"tonalWashInverse": 872412656,
"tonalRimInverse": 870431656,
"outlineRimInverse": 1291843056,
"solidBackground": 4291337092,
"solidBackgroundInverse": 4283311891,
"tonalBackground": 435410888,
"tonalBackgroundInverse": 868785028,
"outlineInverse": 860759827,
"directionalLightStopPrimary": 4281800969,
"directionalLightStopSecondary": 4280027906,
"primary": 4294964720,
"filledStateLayer": 4292983720,
"tonalTouchLight": 1073474016,
"tonalHoverLight": 654043616,
"tonalStateLayer": 436205040,
"overlayDisabledContainer": 871618504,
"overlayTonalTouchLight": 1291577824,
"overlayTonalHoverLight": 1291577824,
"overlayTonalStateLayer": 436205040,
"disabledContainer": 436205040,
"filledTouchLight": 4294964720,
"filledHoverLight": 4294699488,
"onOverlayDisabledContainer": 2583688688,
"overlayTonalContainer": 2583423456,
"overlayOutline": 872412656,
"onFilledContainer": 4279174400,
"filledContainer": 4294170568,
"tonalContainer": 435410888,
"overlayPrimary": 4294964720,
"onOverlayFilledContainer": 4279174400,
"overlayFilledTouchLight": 4294964720,
"overlayFilledHoverLight": 4294699488,
"overlayFilledStateLayer": 4292983720,
"overlayFilledContainer": 4294170568,
"onDisabledContainer": 1728050672,
"temporaryOutline": 871618504
}
}
}
},
{
"lockupViewModel": {
"contentImage": {
"thumbnailViewModel": {
"image": {
"sources": [
{
"url": "https://i.ytimg.com/vi/P7Hkh2zOGQ0/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLAHInNtMierSAgtrGl5rLSKc-vdzQ",
"width": 168,
"height": 94
},
{
"url": "https://i.ytimg.com/vi/P7Hkh2zOGQ0/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDXrm5g-rdjRIqqK309ImTdqpw4CA",
"width": 336,
"height": 188
}
]
},
"overlays": [
{
"thumbnailBottomOverlayViewModel": {
"badges": [
{
"thumbnailBadgeViewModel": {
"text": "1:08:19",
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT",
"animationActivationTargetId": "P7Hkh2zOGQ0",
"animatedText": "Now playing",
"animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE"
}
}
]
}
}
]
}
},
"metadata": {
"lockupMetadataViewModel": {
"title": {
"content": "MIT 6.S191 (2024): Google - Generative AI for Media"
},
"image": {
"decoratedAvatarViewModel": {
"avatar": {
"avatarViewModel": {
"image": {
"sources": [
{
"url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj",
"width": 68,
"height": 68
}
]
},
"avatarImageSize": "AVATAR_SIZE_M"
}
},
"a11yLabel": "Go to channel Alexander Amini",
"rendererContext": {
"commandContext": {
"onTap": {
"innertubeCommand": {
"browseEndpoint": {
"browseId": "UCtslD4DGH6PKyG_1gFAX7sg"
}
}
}
}
}
}
},
"metadata": {
"contentMetadataViewModel": {
"metadataRows": [
{
"metadataParts": [
{
"text": {
"content": "Alexander Amini"
}
}
]
},
{
"metadataParts": [
{
"text": {
"content": "49K"
},
"accessibilityLabel": "49 thousand views",
"leadingIcon": {
"name": "PLAY_ARROW_OUTLINED",
"height": 12,
"width": 12,
"marginEndToken": {
"spacing": 2
}
}
},
{
"text": {
"content": "2y ago"
},
"accessibilityLabel": "2 years ago"
}
]
}
],
"delimiter": " "
}
}
}
},
"contentId": "P7Hkh2zOGQ0",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"themedPalette": {
"lightThemePalette": {
"baseBackground": 4293981951,
"raisedBackground": 4293981951,
"additiveBackground": 558201801,
"textPrimary": 4278655801,
"textSecondary": 4280831104,
"outline": 856631890,
"keyLight": 4288855783,
"collectionStack": 4286094803,
"invertedBackground": 4279847311,
"textPrimaryInverse": 4293981951,
"textDisabled": 2131700306,
"dropShadow": 423984073,
"cardOutline": 172325833,
"overlayBackground": 3423017785,
"overlayTextPrimary": 4293981951,
"solidWash": 4279183954,
"solidRim": 4278655801,
"overlaySolidWash": 4292864764,
"overlaySolidRim": 4293981951,
"tonalWash": 2582705919,
"tonalRim": 201328143,
"overlayTonalWash": 435222271,
"overlayTonalRim": 435222271,
"outlineRim": 1276062290,
"overlayOutlineRim": 868669942,
"solidWashInverse": 4291222006,
"solidRimInverse": 2145381116,
"tonalWashInverse": 653326079,
"tonalRimInverse": 652208892,
"outlineRimInverse": 653326079,
"solidBackground": 4279183954,
"solidBackgroundInverse": 4286094803,
"tonalBackground": 863542739,
"tonalBackgroundInverse": 432462326,
"outlineInverse": 870312700,
"directionalLightStopPrimary": 4292864764,
"directionalLightStopSecondary": 4293981951,
"primary": 4278191631,
"filledStateLayer": 4278655801,
"tonalTouchLight": 4293981951,
"tonalHoverLight": 3437226748,
"tonalStateLayer": 419431951,
"overlayDisabledContainer": 868669942,
"overlayTonalTouchLight": 1289743100,
"overlayTonalHoverLight": 1289743100,
"overlayTonalStateLayer": 435222271,
"disabledContainer": 419431951,
"filledTouchLight": 4280831104,
"filledHoverLight": 4280040297,
"onOverlayDisabledContainer": 2582705919,
"overlayTonalContainer": 2581588732,
"overlayOutline": 871429887,
"onFilledContainer": 4293981951,
"filledContainer": 4278324509,
"tonalContainer": 637999929,
"overlayPrimary": 4293981951,
"onOverlayFilledContainer": 4278191631,
"overlayFilledTouchLight": 4293981951,
"overlayFilledHoverLight": 4292864764,
"overlayFilledStateLayer": 4288855783,
"overlayFilledContainer": 4291222006,
"onDisabledContainer": 1711277583,
"temporaryOutline": 856631890
},
"darkThemePalette": {
"baseBackground": 4278324509,
"raisedBackground": 4278655801,
"additiveBackground": 725973961,
"textPrimary": 4292864764,
"textSecondary": 4288855783,
"outline": 1069996534,
"keyLight": 4280040297,
"collectionStack": 4280831104,
"invertedBackground": 4286359267,
"textPrimaryInverse": 4278324509,
"textDisabled": 2143738358,
"dropShadow": 423984073,
"cardOutline": 172325833,
"overlayBackground": 3423017785,
"overlayTextPrimary": 4293981951,
"solidWash": 4292864764,
"solidRim": 4293981951,
"overlaySolidWash": 4292864764,
"overlaySolidRim": 4293981951,
"tonalWash": 214358518,
"tonalRim": 217118463,
"overlayTonalWash": 435222271,
"overlayTonalRim": 435222271,
"outlineRim": 868669942,
"overlayOutlineRim": 868669942,
"solidWashInverse": 4280831104,
"solidRimInverse": 868669942,
"tonalWashInverse": 871429887,
"tonalRimInverse": 866303719,
"outlineRimInverse": 1290860287,
"solidBackground": 4286094803,
"solidBackgroundInverse": 4279183954,
"tonalBackground": 432462326,
"tonalBackgroundInverse": 863542739,
"outlineInverse": 856631890,
"directionalLightStopPrimary": 4278655801,
"directionalLightStopSecondary": 4278324509,
"primary": 4293981951,
"filledStateLayer": 4288855783,
"tonalTouchLight": 1071639292,
"tonalHoverLight": 652208892,
"tonalStateLayer": 435222271,
"overlayDisabledContainer": 868669942,
"overlayTonalTouchLight": 1289743100,
"overlayTonalHoverLight": 1289743100,
"overlayTonalStateLayer": 435222271,
"disabledContainer": 435222271,
"filledTouchLight": 4293981951,
"filledHoverLight": 4292864764,
"onOverlayDisabledContainer": 2582705919,
"overlayTonalContainer": 2581588732,
"overlayOutline": 871429887,
"onFilledContainer": 4278191631,
"filledContainer": 4291222006,
"tonalContainer": 432462326,
"overlayPrimary": 4293981951,
"onOverlayFilledContainer": 4278191631,
"overlayFilledTouchLight": 4293981951,
"overlayFilledHoverLight": 4292864764,
"overlayFilledStateLayer": 4288855783,
"overlayFilledContainer": 4291222006,
"onDisabledContainer": 1727067903,
"temporaryOutline": 868669942
}
}
}
},
{
"continuationItemRenderer": {
"trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN",
"continuationEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"sendPost": true,
"apiUrl": "/youtubei/v1/next"
}
},
"continuationCommand": {
"token": "CCgSDRILU2RUWkFNREtyTlkYACqPDE1nb0lodVQ4eG9EdXVLSUYyczZMNnd6NENBcjFDQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhQ2dQeVBnQUtEc0ktQ3dqaDRleUUzTFAzdV93QkNnUHlQZ0FLRGNJLUNnaU5zcmptOXBENTJEOEtBX0ktQUFvTndqNEtDTExudjR5ZXg3U1dTUW9EOGo0QUNnM0NQZ29JNnZHODhaR1JpTWNnQ2dQeVBnQUtEc0ktQ3dpRF85S3Z5WS1hbThJQkNnUHlQZ0FLRHNJLUN3alppNmJYbWR6YWxZb0JDZ1B5UGdBS0RzSS1Dd2lkemJfS2dPdWN0YUFCQ2dQeVBnQUtEY0ktQ2dqMXktMjYyb2JQeFdNS0FfSS1BQW9Pd2o0TENPLUE4SWVaMUlESXJRRUtBX0ktQUFvTndqNEtDUHlZNDVhUzhkeWpJZ29EOGo0QUNnM0NQZ29Jb191aWtfSE05ZHdTQ2dQeVBnQUtEc0ktQ3dqaTBiemlrdnpBbkxvQkNnUHlQZ0FLRHNJLUN3aW05cXI1eXNtTWlzOEJDZ1B5UGdBS0RjSS1DZ2k3M2V5NXlzQ1pzMG9LQV9JLUFBb093ajRMQ0xmT281WFU1X2JPbFFFS0FfSS1BQW9Od2o0S0NLTEVwYXlXOTliWkdnb0Q4ajRBQ2c3Q1Bnc0k2dWZSMnRPM3hLem9BUW9EOGo0QUNnN0NQZ3NJN2ZlZXZQRHV2WlRXQVFvRDhqNEFDZzNDUGdvSXFibS0zYXVBdDVWcUNnUHlQZ0FLRGNJLUNnaUc1UHpHZ082NG9nVVNLQUFDQkFZSUNnd09FQklVRmhnYUhCNGdJaVFtS0Nvc0xqQXlORFk0T2p3LVFFSkVSa2hLVEU0YUJBZ0FFQUVhQkFnQ0VBTWFCQWdFRUFVYUJBZ0dFQWNhQkFnSUVBa2FCQWdLRUFzYUJBZ01FQTBhQkFnT0VBOGFCQWdRRUJFYUJBZ1NFQk1hQkFnVUVCVWFCQWdXRUJjYUJBZ1lFQmthQkFnYUVCc2FCQWdjRUIwYUJBZ2VFQjhhQkFnZ0VDRWFCQWdpRUNNYUJBZ2tFQ1VhQkFnbUVDY2FCQWdvRUNrYUJBZ3FFQ3NhQkFnc0VDMGFCQWd1RUM4YUJBZ3dFREVhQkFneUVETWFCQWcwRURVYUJBZzJFRGNhQkFnNEVEa2FCQWc2RURzYUJBZzhFRDBhQkFnLUVEOGFCQWhBRUVFYUJBaENFRU1hQkFoRUVFVWFCQWhHRUVjYUJBaElFRWthQkFoS0VFc2FCQWhNRUUwYUJBaE9FRThxS0FBQ0JBWUlDZ3dPRUJJVUZoZ2FIQjRnSWlRbUtDb3NMakF5TkRZNE9qdy1RRUpFUmtoS1RFNGoPd2F0Y2gtbmV4dC1mZWVk",
"request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT"
}
}
}
}
]
}
}
]
}
backend/tests/fixtures/innertube_next_first.json
Vendored
+805
View File
@@ -0,0 +1,805 @@
{
"contents": {
"twoColumnWatchNextResults": {
"secondaryResults": {
"secondaryResults": {
"results": [
{
"lockupViewModel": {
"contentImage": {
"thumbnailViewModel": {
"image": {
"sources": [
{
"url": "https://i.ytimg.com/vi/to-lHJfK4pw/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLBKP_oFChvoqvwVrKAy_PG9bn2_AA",
"width": 168,
"height": 94
},
{
"url": "https://i.ytimg.com/vi/to-lHJfK4pw/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDyH3Z8DJhXMmFJ5eKEWMufk7ql9g",
"width": 336,
"height": 188
}
]
},
"overlays": [
{
"thumbnailBottomOverlayViewModel": {
"badges": [
{
"thumbnailBadgeViewModel": {
"text": "1:02:00",
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT",
"animationActivationTargetId": "to-lHJfK4pw",
"animatedText": "Now playing",
"animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE"
}
}
]
}
}
]
}
},
"metadata": {
"lockupMetadataViewModel": {
"title": {
"content": "MIT 6.S191 (2025): Reinforcement Learning"
},
"image": {
"decoratedAvatarViewModel": {
"avatar": {
"avatarViewModel": {
"image": {
"sources": [
{
"url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj",
"width": 68,
"height": 68
}
]
},
"avatarImageSize": "AVATAR_SIZE_M"
}
},
"a11yLabel": "Go to channel Alexander Amini",
"rendererContext": {
"commandContext": {
"onTap": {
"innertubeCommand": {
"browseEndpoint": {
"browseId": "UCtslD4DGH6PKyG_1gFAX7sg"
}
}
}
}
}
}
},
"metadata": {
"contentMetadataViewModel": {
"metadataRows": [
{
"metadataParts": [
{
"text": {
"content": "Alexander Amini"
}
}
]
},
{
"metadataParts": [
{
"text": {
"content": "128K"
},
"accessibilityLabel": "128 thousand views",
"leadingIcon": {
"name": "PLAY_ARROW_OUTLINED",
"height": 12,
"width": 12,
"marginEndToken": {
"spacing": 2
}
}
},
{
"text": {
"content": "1y ago"
},
"accessibilityLabel": "1 year ago"
}
]
}
],
"delimiter": " "
}
}
}
},
"contentId": "to-lHJfK4pw",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"themedPalette": {
"lightThemePalette": {
"baseBackground": 4293981439,
"raisedBackground": 4293981439,
"additiveBackground": 558196425,
"textPrimary": 4278850870,
"textSecondary": 4282009454,
"outline": 857088843,
"keyLight": 4289509597,
"collectionStack": 4287338944,
"invertedBackground": 4279842703,
"textPrimaryInverse": 4293981439,
"textDisabled": 2132157259,
"dropShadow": 423978697,
"cardOutline": 172320457,
"overlayBackground": 3423212854,
"overlayTextPrimary": 4293981439,
"solidWash": 4279640907,
"solidRim": 4278850870,
"overlaySolidWash": 4292929530,
"overlaySolidRim": 4293981439,
"tonalWash": 2582705407,
"tonalRim": 201327631,
"overlayTonalWash": 435221759,
"overlayTonalRim": 435221759,
"outlineRim": 1276519243,
"overlayOutlineRim": 868930802,
"solidWashInverse": 4291482866,
"solidRimInverse": 2145445882,
"tonalWashInverse": 653325567,
"tonalRimInverse": 652273658,
"outlineRimInverse": 653325567,
"solidBackground": 4279640907,
"solidBackgroundInverse": 4287338944,
"tonalBackground": 864786880,
"tonalBackgroundInverse": 432723186,
"outlineInverse": 870377466,
"directionalLightStopPrimary": 4292929530,
"directionalLightStopSecondary": 4293981439,
"primary": 4278191119,
"filledStateLayer": 4278850870,
"tonalTouchLight": 4293981439,
"tonalHoverLight": 3437291514,
"tonalStateLayer": 419431439,
"overlayDisabledContainer": 868930802,
"overlayTonalTouchLight": 1289807866,
"overlayTonalHoverLight": 1289807866,
"overlayTonalStateLayer": 435221759,
"disabledContainer": 419431439,
"filledTouchLight": 4282009454,
"filledHoverLight": 4280825181,
"onOverlayDisabledContainer": 2582705407,
"overlayTonalContainer": 2581653498,
"overlayOutline": 871429375,
"onFilledContainer": 4293981439,
"filledContainer": 4278323484,
"tonalContainer": 638194998,
"overlayPrimary": 4293981439,
"onOverlayFilledContainer": 4278191119,
"overlayFilledTouchLight": 4293981439,
"overlayFilledHoverLight": 4292929530,
"overlayFilledStateLayer": 4289509597,
"overlayFilledContainer": 4291482866,
"onDisabledContainer": 1711277071,
"temporaryOutline": 857088843
},
"darkThemePalette": {
"baseBackground": 4278323484,
"raisedBackground": 4278850870,
"additiveBackground": 725968585,
"textPrimary": 4292929530,
"textSecondary": 4289509597,
"outline": 1070257394,
"keyLight": 4280825181,
"collectionStack": 4282009454,
"invertedBackground": 4286355427,
"textPrimaryInverse": 4278323484,
"textDisabled": 2143999218,
"dropShadow": 423978697,
"cardOutline": 172320457,
"overlayBackground": 3423212854,
"overlayTextPrimary": 4293981439,
"solidWash": 4292929530,
"solidRim": 4293981439,
"overlaySolidWash": 4292929530,
"overlaySolidRim": 4293981439,
"tonalWash": 214619378,
"tonalRim": 217117951,
"overlayTonalWash": 435221759,
"overlayTonalRim": 435221759,
"outlineRim": 868930802,
"overlayOutlineRim": 868930802,
"solidWashInverse": 4282009454,
"solidRimInverse": 868930802,
"tonalWashInverse": 871429375,
"tonalRimInverse": 866957533,
"outlineRimInverse": 1290859775,
"solidBackground": 4287338944,
"solidBackgroundInverse": 4279640907,
"tonalBackground": 432723186,
"tonalBackgroundInverse": 864786880,
"outlineInverse": 857088843,
"directionalLightStopPrimary": 4278850870,
"directionalLightStopSecondary": 4278323484,
"primary": 4293981439,
"filledStateLayer": 4289509597,
"tonalTouchLight": 1071704058,
"tonalHoverLight": 652273658,
"tonalStateLayer": 435221759,
"overlayDisabledContainer": 868930802,
"overlayTonalTouchLight": 1289807866,
"overlayTonalHoverLight": 1289807866,
"overlayTonalStateLayer": 435221759,
"disabledContainer": 435221759,
"filledTouchLight": 4293981439,
"filledHoverLight": 4292929530,
"onOverlayDisabledContainer": 2582705407,
"overlayTonalContainer": 2581653498,
"overlayOutline": 871429375,
"onFilledContainer": 4278191119,
"filledContainer": 4291482866,
"tonalContainer": 432723186,
"overlayPrimary": 4293981439,
"onOverlayFilledContainer": 4278191119,
"overlayFilledTouchLight": 4293981439,
"overlayFilledHoverLight": 4292929530,
"overlayFilledStateLayer": 4289509597,
"overlayFilledContainer": 4291482866,
"onDisabledContainer": 1727067391,
"temporaryOutline": 868930802
}
}
}
},
{
"lockupViewModel": {
"contentImage": {
"thumbnailViewModel": {
"image": {
"sources": [
{
"url": "https://i.ytimg.com/vi/R8V8CbuxryI/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLDqXMjrpgDyQKHL2lWrxczwxKoefw",
"width": 168,
"height": 94
},
{
"url": "https://i.ytimg.com/vi/R8V8CbuxryI/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLB7JX9Z238s6DGqIS5-fhcEIkUxKw",
"width": 336,
"height": 188
}
]
},
"overlays": [
{
"thumbnailBottomOverlayViewModel": {
"badges": [
{
"thumbnailBadgeViewModel": {
"text": "49:36",
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT",
"animationActivationTargetId": "R8V8CbuxryI",
"animatedText": "Now playing",
"animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE"
}
}
]
}
}
]
}
},
"metadata": {
"lockupMetadataViewModel": {
"title": {
"content": "MIT 6.S191: Deep Generative Modeling"
},
"image": {
"decoratedAvatarViewModel": {
"avatar": {
"avatarViewModel": {
"image": {
"sources": [
{
"url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj",
"width": 68,
"height": 68
}
]
},
"avatarImageSize": "AVATAR_SIZE_M"
}
},
"a11yLabel": "Go to channel Alexander Amini",
"rendererContext": {
"commandContext": {
"onTap": {
"innertubeCommand": {
"browseEndpoint": {
"browseId": "UCtslD4DGH6PKyG_1gFAX7sg"
}
}
}
}
}
}
},
"metadata": {
"contentMetadataViewModel": {
"metadataRows": [
{
"metadataParts": [
{
"text": {
"content": "Alexander Amini"
}
}
]
},
{
"metadataParts": [
{
"text": {
"content": "38K"
},
"accessibilityLabel": "38 thousand views",
"leadingIcon": {
"name": "PLAY_ARROW_OUTLINED",
"height": 12,
"width": 12,
"marginEndToken": {
"spacing": 2
}
}
},
{
"text": {
"content": "6mo ago"
},
"accessibilityLabel": "6 months ago"
}
]
}
],
"delimiter": " "
}
}
}
},
"contentId": "R8V8CbuxryI",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"themedPalette": {
"lightThemePalette": {
"baseBackground": 4294965488,
"raisedBackground": 4294965488,
"additiveBackground": 566856261,
"textPrimary": 4281737226,
"textSecondary": 4285420858,
"outline": 860565782,
"keyLight": 4292724140,
"collectionStack": 4290815883,
"invertedBackground": 4287583769,
"textPrimaryInverse": 4294965488,
"textDisabled": 2135634198,
"dropShadow": 432638533,
"cardOutline": 180980293,
"overlayBackground": 3426099210,
"overlayTextPrimary": 4294965488,
"solidWash": 4283117846,
"solidRim": 4281737226,
"overlaySolidWash": 4294635232,
"overlaySolidRim": 4294965488,
"tonalWash": 2583689456,
"tonalRim": 202311680,
"overlayTonalWash": 436205808,
"overlayTonalRim": 436205808,
"outlineRim": 1279996182,
"overlayOutlineRim": 871554762,
"solidWashInverse": 4294106826,
"solidRimInverse": 2147151584,
"tonalWashInverse": 654309616,
"tonalRimInverse": 653979360,
"outlineRimInverse": 654309616,
"solidBackground": 4283117846,
"solidBackgroundInverse": 4290815883,
"tonalBackground": 868263819,
"tonalBackgroundInverse": 435347146,
"outlineInverse": 872083168,
"directionalLightStopPrimary": 4294635232,
"directionalLightStopSecondary": 4294965488,
"primary": 4279175168,
"filledStateLayer": 4281737226,
"tonalTouchLight": 4294965488,
"tonalHoverLight": 3438997216,
"tonalStateLayer": 420415488,
"overlayDisabledContainer": 871554762,
"overlayTonalTouchLight": 1291513568,
"overlayTonalHoverLight": 1291513568,
"overlayTonalStateLayer": 436205808,
"disabledContainer": 420415488,
"filledTouchLight": 4285420858,
"filledHoverLight": 4284302120,
"onOverlayDisabledContainer": 2583689456,
"overlayTonalContainer": 2583359200,
"overlayOutline": 872413424,
"onFilledContainer": 4294965488,
"filledContainer": 4280029186,
"tonalContainer": 641081354,
"overlayPrimary": 4294965488,
"onOverlayFilledContainer": 4279175168,
"overlayFilledTouchLight": 4294965488,
"overlayFilledHoverLight": 4294635232,
"overlayFilledStateLayer": 4292724140,
"overlayFilledContainer": 4294106826,
"onDisabledContainer": 1712261120,
"temporaryOutline": 860565782
},
"darkThemePalette": {
"baseBackground": 4280029186,
"raisedBackground": 4281737226,
"additiveBackground": 734628421,
"textPrimary": 4294635232,
"textSecondary": 4292724140,
"outline": 1072881354,
"keyLight": 4284302120,
"collectionStack": 4285420858,
"invertedBackground": 4293112700,
"textPrimaryInverse": 4280029186,
"textDisabled": 2146623178,
"dropShadow": 432638533,
"cardOutline": 180980293,
"overlayBackground": 3426099210,
"overlayTextPrimary": 4294965488,
"solidWash": 4294635232,
"solidRim": 4294965488,
"overlaySolidWash": 4294635232,
"overlaySolidRim": 4294965488,
"tonalWash": 217243338,
"tonalRim": 218102000,
"overlayTonalWash": 436205808,
"overlayTonalRim": 436205808,
"outlineRim": 871554762,
"overlayOutlineRim": 871554762,
"solidWashInverse": 4285420858,
"solidRimInverse": 871554762,
"tonalWashInverse": 872413424,
"tonalRimInverse": 870172076,
"outlineRimInverse": 1291843824,
"solidBackground": 4290815883,
"solidBackgroundInverse": 4283117846,
"tonalBackground": 435347146,
"tonalBackgroundInverse": 868263819,
"outlineInverse": 860565782,
"directionalLightStopPrimary": 4281737226,
"directionalLightStopSecondary": 4280029186,
"primary": 4294965488,
"filledStateLayer": 4292724140,
"tonalTouchLight": 1073409760,
"tonalHoverLight": 653979360,
"tonalStateLayer": 436205808,
"overlayDisabledContainer": 871554762,
"overlayTonalTouchLight": 1291513568,
"overlayTonalHoverLight": 1291513568,
"overlayTonalStateLayer": 436205808,
"disabledContainer": 436205808,
"filledTouchLight": 4294965488,
"filledHoverLight": 4294635232,
"onOverlayDisabledContainer": 2583689456,
"overlayTonalContainer": 2583359200,
"overlayOutline": 872413424,
"onFilledContainer": 4279175168,
"filledContainer": 4294106826,
"tonalContainer": 435347146,
"overlayPrimary": 4294965488,
"onOverlayFilledContainer": 4279175168,
"overlayFilledTouchLight": 4294965488,
"overlayFilledHoverLight": 4294635232,
"overlayFilledStateLayer": 4292724140,
"overlayFilledContainer": 4294106826,
"onDisabledContainer": 1728051440,
"temporaryOutline": 871554762
}
}
}
},
{
"lockupViewModel": {
"contentImage": {
"thumbnailViewModel": {
"image": {
"sources": [
{
"url": "https://i.ytimg.com/vi/HLKo4fJx_7k/hqdefault.jpg?sqp=-oaymwEiCKgBEF5IWvKriqkDFQgBFQAAAAAYASUAAMhCPQCAokN4AQ==&rs=AOn4CLAQcvMQEX1ny9Z8qu2mIMogpEMaVA",
"width": 168,
"height": 94
},
{
"url": "https://i.ytimg.com/vi/HLKo4fJx_7k/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLCkoPSkHifs5ifpWXXcb96XG_Ly1w",
"width": 336,
"height": 188
}
]
},
"overlays": [
{
"thumbnailBottomOverlayViewModel": {
"badges": [
{
"thumbnailBadgeViewModel": {
"text": "47:36",
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT",
"animationActivationTargetId": "HLKo4fJx_7k",
"animatedText": "Now playing",
"animationActivationEntitySelectorType": "THUMBNAIL_BADGE_ANIMATION_ENTITY_SELECTOR_TYPE_PLAYER_STATE"
}
}
]
}
}
]
}
},
"metadata": {
"lockupMetadataViewModel": {
"title": {
"content": "MIT 6.S191 (2025): Language Models and New Frontiers"
},
"image": {
"decoratedAvatarViewModel": {
"avatar": {
"avatarViewModel": {
"image": {
"sources": [
{
"url": "https://yt3.ggpht.com/ytc/AIdro_k517G0Dt8QgsF86GKzYkG3Rs8jSGCiYGJw04lx4_1KXQ=s68-c-k-c0x00ffffff-no-rj",
"width": 68,
"height": 68
}
]
},
"avatarImageSize": "AVATAR_SIZE_M"
}
},
"a11yLabel": "Go to channel Alexander Amini",
"rendererContext": {
"commandContext": {
"onTap": {
"innertubeCommand": {
"browseEndpoint": {
"browseId": "UCtslD4DGH6PKyG_1gFAX7sg"
}
}
}
}
}
}
},
"metadata": {
"contentMetadataViewModel": {
"metadataRows": [
{
"metadataParts": [
{
"text": {
"content": "Alexander Amini"
}
}
]
},
{
"metadataParts": [
{
"text": {
"content": "63K"
},
"accessibilityLabel": "63 thousand views",
"leadingIcon": {
"name": "PLAY_ARROW_OUTLINED",
"height": 12,
"width": 12,
"marginEndToken": {
"spacing": 2
}
}
},
{
"text": {
"content": "1y ago"
},
"accessibilityLabel": "1 year ago"
}
]
}
],
"delimiter": " "
}
}
}
},
"contentId": "HLKo4fJx_7k",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"themedPalette": {
"lightThemePalette": {
"baseBackground": 4294964208,
"raisedBackground": 4294964208,
"additiveBackground": 566847045,
"textPrimary": 4281734410,
"textSecondary": 4285417018,
"outline": 860561942,
"keyLight": 4292720812,
"collectionStack": 4290812043,
"invertedBackground": 4287575065,
"textPrimaryInverse": 4294964208,
"textDisabled": 2135630358,
"dropShadow": 432629317,
"cardOutline": 180971077,
"overlayBackground": 3426096394,
"overlayTextPrimary": 4294964208,
"solidWash": 4283114006,
"solidRim": 4281734410,
"overlaySolidWash": 4294633184,
"overlaySolidRim": 4294964208,
"tonalWash": 2583688176,
"tonalRim": 202310656,
"overlayTonalWash": 436204528,
"overlayTonalRim": 436204528,
"outlineRim": 1279992342,
"overlayOutlineRim": 871551946,
"solidWashInverse": 4294104010,
"solidRimInverse": 2147149536,
"tonalWashInverse": 654308336,
"tonalRimInverse": 653977312,
"outlineRimInverse": 654308336,
"solidBackground": 4283114006,
"solidBackgroundInverse": 4290812043,
"tonalBackground": 868259979,
"tonalBackgroundInverse": 435344330,
"outlineInverse": 872081120,
"directionalLightStopPrimary": 4294633184,
"directionalLightStopSecondary": 4294964208,
"primary": 4279174144,
"filledStateLayer": 4281734410,
"tonalTouchLight": 4294964208,
"tonalHoverLight": 3438995168,
"tonalStateLayer": 420414464,
"overlayDisabledContainer": 871551946,
"overlayTonalTouchLight": 1291511520,
"overlayTonalHoverLight": 1291511520,
"overlayTonalStateLayer": 436204528,
"disabledContainer": 420414464,
"filledTouchLight": 4285417018,
"filledHoverLight": 4284298280,
"onOverlayDisabledContainer": 2583688176,
"overlayTonalContainer": 2583357152,
"overlayOutline": 872412144,
"onFilledContainer": 4294964208,
"filledContainer": 4280027394,
"tonalContainer": 641078538,
"overlayPrimary": 4294964208,
"onOverlayFilledContainer": 4279174144,
"overlayFilledTouchLight": 4294964208,
"overlayFilledHoverLight": 4294633184,
"overlayFilledStateLayer": 4292720812,
"overlayFilledContainer": 4294104010,
"onDisabledContainer": 1712260096,
"temporaryOutline": 860561942
},
"darkThemePalette": {
"baseBackground": 4280027394,
"raisedBackground": 4281734410,
"additiveBackground": 734619205,
"textPrimary": 4294633184,
"textSecondary": 4292720812,
"outline": 1072878538,
"keyLight": 4284298280,
"collectionStack": 4285417018,
"invertedBackground": 4293105020,
"textPrimaryInverse": 4280027394,
"textDisabled": 2146620362,
"dropShadow": 432629317,
"cardOutline": 180971077,
"overlayBackground": 3426096394,
"overlayTextPrimary": 4294964208,
"solidWash": 4294633184,
"solidRim": 4294964208,
"overlaySolidWash": 4294633184,
"overlaySolidRim": 4294964208,
"tonalWash": 217240522,
"tonalRim": 218100720,
"overlayTonalWash": 436204528,
"overlayTonalRim": 436204528,
"outlineRim": 871551946,
"overlayOutlineRim": 871551946,
"solidWashInverse": 4285417018,
"solidRimInverse": 871551946,
"tonalWashInverse": 872412144,
"tonalRimInverse": 870168748,
"outlineRimInverse": 1291842544,
"solidBackground": 4290812043,
"solidBackgroundInverse": 4283114006,
"tonalBackground": 435344330,
"tonalBackgroundInverse": 868259979,
"outlineInverse": 860561942,
"directionalLightStopPrimary": 4281734410,
"directionalLightStopSecondary": 4280027394,
"primary": 4294964208,
"filledStateLayer": 4292720812,
"tonalTouchLight": 1073407712,
"tonalHoverLight": 653977312,
"tonalStateLayer": 436204528,
"overlayDisabledContainer": 871551946,
"overlayTonalTouchLight": 1291511520,
"overlayTonalHoverLight": 1291511520,
"overlayTonalStateLayer": 436204528,
"disabledContainer": 436204528,
"filledTouchLight": 4294964208,
"filledHoverLight": 4294633184,
"onOverlayDisabledContainer": 2583688176,
"overlayTonalContainer": 2583357152,
"overlayOutline": 872412144,
"onFilledContainer": 4279174144,
"filledContainer": 4294104010,
"tonalContainer": 435344330,
"overlayPrimary": 4294964208,
"onOverlayFilledContainer": 4279174144,
"overlayFilledTouchLight": 4294964208,
"overlayFilledHoverLight": 4294633184,
"overlayFilledStateLayer": 4292720812,
"overlayFilledContainer": 4294104010,
"onDisabledContainer": 1728050160,
"temporaryOutline": 871551946
}
}
}
},
{
"continuationItemRenderer": {
"trigger": "CONTINUATION_TRIGGER_ON_ITEM_SHOWN",
"continuationEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"sendPost": true,
"apiUrl": "/youtubei/v1/next"
}
},
"continuationCommand": {
"token": "CBQSDRILU2RUWkFNREtyTlkYACqXBk1nb0l5ZGl1dXRUNDdQc2EyczZMNnd5LUJBcTdCQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D",
"request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT"
}
},
"button": {
"buttonRenderer": {
"style": "STYLE_SUGGESTIVE",
"size": "SIZE_DEFAULT",
"isDisabled": false,
"text": {
"runs": [
{
"text": "Show more"
}
]
},
"command": {
"commandMetadata": {
"webCommandMetadata": {
"sendPost": true,
"apiUrl": "/youtubei/v1/next"
}
},
"continuationCommand": {
"token": "CBQSDRILU2RUWkFNREtyTlkYACqXBk1nb0l5ZGl1dXRUNDdQc2EyczZMNnd5LUJBcTdCQW9EOGo0QUNnN0NQZ3NJbk1XcnZzbWo2Y2UyQVFvRDhqNEFDZzNDUGdvSW90N0czWnVCMy1KSENnUHlQZ0FLRGNJLUNnaTVfOGVUbjV5cTJSd0tBX0ktQUFvT3dqNExDSkQydWJfNXp2ZXZfUUVLQV9JLUFBb053ajRLQ1BXWjVQakt2cjZWU2dvRDhqNEFDZzNDUGdvSW9aTDd1OUtGdF85NkNnUHlQZ0FLRGNJLUNnaUh3N2FLMzd6YzVIZ0tBX0ktQUFvT3dqNExDUE94aHRQTWtfVzB1UUVLQV9JLUFBb093ajRMQ092eTBzaXNsSWZScGdFS0FfSS1BQW9Pd2o0TENKS1FxdHpBeE5lZHJBRUtBX0ktQUFvTndqNEtDTFdteWE2cHNPMkJPZ29EOGo0QUNnM0NQZ29JanYtdy1wQ3k1YVozQ2dQeVBnQUtEc0ktQ3dqMmtmVF9tYlB1XzRvQkNnUHlQZ0FLRHNJLUN3aVpuYWZjc0tQVmpZUUJDZ1B5UGdBS0RjSS1DZ2lxM3BxM3FyX2s0U0VLQV9JLUFBb053ajRLQ0phNXU5eVJvTmJOS0FvRDhqNEFDZzNDUGdvSXFMeWR3T1NhNGJkV0NnUHlQZ0FLRGNJLUNnakh3X2ZucktlSDdXUUtBX0ktQUFvTndqNEtDUG4xak1mcXg4QzJNQW9EOGo0QUNnM0NQZ29JeWRpdXV0VDQ3UHNhRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D",
"request": "CONTINUATION_REQUEST_TYPE_WATCH_NEXT"
}
}
}
}
}
}
]
}
}
}
}
}
backend/tests/test_related_route.py
+298
View File
@@ -0,0 +1,298 @@
"""`GET /api/search/related/{video_id}` — a video's related videos, without what the user has seen.
The scrape is replaced by pages the test hands out, and YouTube's videos.list by an offline enrich
(30-minute videos, so no Shorts probe goes out). What is held: the list leaves out what this user
watched or hid — here or in their Takeout history, unless taken back to "new" since — blocked
channels and the video itself; it pages on until it has enough; it lands in the catalog the way
search finds do; and a failure, or the day's cap, says so instead of answering an empty list."""
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select
from app import quota, sysconfig
from app.catalog.models import BlockedChannel, Channel, Video
from app.models import QuotaEvent
from app.reader.models import TakeoutItem, TakeoutKind, VideoState, VideoUnmark
from app.routes import search as search_routes
from app.youtube.innertube import ScrapeError
T0 = datetime(2026, 10, 1, tzinfo=timezone.utc)
SOURCE = "srcvideo001"
def _vid(n: int) -> str:
return f"rel{n:08d}"
def _item(n: int, channel: str = "UCrelated01") -> dict:
return {
"id": _vid(n),
"channel_id": channel,
"channel_title": f"Chan {channel}",
"title": f"Related {n}",
"published_at": None,
"thumbnail_url": None,
"live_broadcast": "none",
}
class _FakeYT:
def __init__(self, *_a, **_kw):
pass
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def get_channels(self, ids):
return []
def get_videos(self, ids):
return [
{
"id": i,
"snippet": {"title": f"T {i}", "publishedAt": "2025-01-01T00:00:00Z"},
"contentDetails": {"duration": "PT30M"},
"statistics": {},
"status": {},
}
for i in ids
]
@pytest.fixture
def pages(monkeypatch, db):
"""The scrape answers with the pages in `state["pages"]`, one per call, recording each call."""
sysconfig.set_value(db, "shorts_probe_max_seconds", 60)
state: dict = {"pages": [], "calls": []}
def related(video_id, continuation=None, *, hl="en", gl="US"):
state["calls"].append((video_id, continuation, hl, gl))
page = state["pages"].pop(0)
if isinstance(page, Exception):
raise page
return page
monkeypatch.setattr(search_routes.related_scrape, "related", related)
monkeypatch.setattr(search_routes, "YouTubeClient", _FakeYT)
return state
def _page(items: list[dict], token: str | None = "NEXT") -> dict:
return {"items": items, "next_page_token": token}
def _ids(resp) -> list[str]:
assert resp.status_code == 200, resp.text
return [it["id"] for it in resp.json()["items"]]
def _known(db, *ns: int, channel: str = "UCrelated01") -> None:
"""These related videos already in the catalog (a state needs the row)."""
if db.get(Channel, channel) is None:
db.add(Channel(id=channel, title="Chan"))
for n in ns:
db.add(Video(id=_vid(n), channel_id=channel, title=f"Related {n}", published_at=T0))
db.commit()
def test_a_related_list_comes_into_the_catalog_as_search_finds_do(db, make_user, as_user, pages):
user = make_user()
pages["pages"] = [_page([_item(n) for n in range(1, 21)])]
resp = as_user(user).get(f"/api/search/related/{SOURCE}")
assert _ids(resp) == [_vid(n) for n in range(1, 21)]
assert resp.json()["next_cursor"] == "NEXT" and resp.json()["has_more"] is True
video = db.get(Video, _vid(1))
# Flagged like a search find: hidden from the shared Library, reclaimed when nobody keeps it.
assert video.via_search is True and video.enriched_at is not None
assert pages["calls"] == [(SOURCE, None, "en", "US")]
def test_what_the_user_watched_or_hid_here_is_left_out(db, make_user, as_user, pages):
user = make_user()
other = make_user()
_known(db, 1, 2, 3, 4)
db.add_all(
[
VideoState(user_id=user.id, video_id=_vid(1), status="watched", watched_at=T0),
VideoState(user_id=user.id, video_id=_vid(2), status="hidden"),
# In progress is not "seen": it stays, to be finished.
VideoState(user_id=user.id, video_id=_vid(3), status="new", position_seconds=120),
# Someone else's watch is theirs.
VideoState(user_id=other.id, video_id=_vid(4), status="watched", watched_at=T0),
]
)
db.commit()
pages["pages"] = [_page([_item(n) for n in range(1, 6)], token=None)]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(3), _vid(4), _vid(5)]
def test_what_the_takeout_history_shows_watched_is_left_out_unless_taken_back(
db, make_user, as_user, pages
):
user = make_user()
other = make_user()
_known(db, 2)
db.add_all(
[
# Not in the catalog at all: the history alone names it.
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0),
# Watched on YouTube, then taken back to "new" here: it may come again.
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(2), at=T0),
VideoUnmark(user_id=user.id, video_id=_vid(2), at=T0 + timedelta(days=1)),
# Only in Watch later: not watched.
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCH_LATER, video_id=_vid(3), at=T0),
# Another user's history.
TakeoutItem(user_id=other.id, kind=TakeoutKind.WATCHED, video_id=_vid(4), at=T0),
]
)
db.commit()
pages["pages"] = [_page([_item(n) for n in range(1, 5)], token=None)]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2), _vid(3), _vid(4)]
def test_an_unmark_older_than_the_history_row_does_not_bring_it_back(db, make_user, as_user, pages):
user = make_user()
_known(db, 1)
db.add_all(
[
VideoUnmark(user_id=user.id, video_id=_vid(1), at=T0 - timedelta(days=1)),
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0),
]
)
db.commit()
pages["pages"] = [_page([_item(1), _item(2)], token=None)]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)]
def test_another_users_unmark_does_not_bring_it_back(db, make_user, as_user, pages):
user = make_user()
other = make_user()
_known(db, 1)
db.add_all(
[
TakeoutItem(user_id=user.id, kind=TakeoutKind.WATCHED, video_id=_vid(1), at=T0),
VideoUnmark(user_id=other.id, video_id=_vid(1), at=T0 + timedelta(days=1)),
]
)
db.commit()
pages["pages"] = [_page([_item(1), _item(2)], token=None)]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(2)]
def test_blocked_channels_and_the_video_itself_are_left_out_and_never_stored(
db, make_user, as_user, pages
):
user = make_user()
db.add(Channel(id="UCblocked01", title="Blocked"))
db.commit()
db.add(BlockedChannel(user_id=user.id, channel_id="UCblocked01"))
db.commit()
own = {**_item(9), "id": SOURCE}
pages["pages"] = [_page([_item(1), _item(2, "UCblocked01"), own], token=None)]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1)]
assert db.get(Video, _vid(2)) is None
def test_it_pages_on_until_enough_unseen_videos_are_gathered(db, make_user, as_user, pages):
user = make_user()
_known(db, *range(1, 16))
for n in range(1, 16):
db.add(VideoState(user_id=user.id, video_id=_vid(n), status="watched", watched_at=T0))
db.commit()
pages["pages"] = [
_page([_item(n) for n in range(1, 21)], token="page2"), # 15 seen, 5 new
_page([_item(n) for n in range(21, 41)], token="page3"), # 20 new: 25 now
_page([_item(n) for n in range(41, 61)], token="page4"), # never asked
]
ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}"))
assert ids == [_vid(n) for n in range(16, 41)]
assert [c[1] for c in pages["calls"]] == [None, "page2"]
assert resp.json()["next_cursor"] == "page3"
def test_a_repeat_on_a_later_page_counts_once(db, make_user, as_user, pages):
user = make_user()
pages["pages"] = [
_page([_item(n) for n in range(1, 11)], token="page2"),
_page([_item(n) for n in range(5, 25)], token=None),
]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [
_vid(n) for n in range(1, 25)
]
def test_it_reads_at_most_four_pages(db, make_user, as_user, pages):
user = make_user()
pages["pages"] = [_page([_item(n)], token=f"page{n + 1}") for n in range(1, 7)]
ids = _ids(resp := as_user(user).get(f"/api/search/related/{SOURCE}"))
assert ids == [_vid(n) for n in range(1, 5)]
assert len(pages["calls"]) == 4 and resp.json()["next_cursor"] == "page5"
def test_the_cursor_and_the_hungarian_locale_reach_the_scrape(db, make_user, as_user, pages):
user = make_user()
pages["pages"] = [_page([_item(1)], token=None)]
resp = as_user(user).get(f"/api/search/related/{SOURCE}?cursor=C1&region=hu")
assert _ids(resp) == [_vid(1)]
assert pages["calls"] == [(SOURCE, "C1", "hu", "HU")]
assert resp.json()["has_more"] is False
def test_an_unknown_region_is_refused(db, make_user, as_user, pages):
user = make_user()
assert as_user(user).get(f"/api/search/related/{SOURCE}?region=xx").status_code == 422
assert pages["calls"] == []
def test_a_failed_scrape_says_youtube_is_unavailable(db, make_user, as_user, pages):
user = make_user()
pages["pages"] = [ScrapeError("blocked")]
resp = as_user(user).get(f"/api/search/related/{SOURCE}")
assert resp.status_code == 422
assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"}
def test_a_failure_on_a_later_page_keeps_what_came_before(db, make_user, as_user, pages):
user = make_user()
pages["pages"] = [_page([_item(1), _item(2)], token="page2"), ScrapeError("blocked")]
assert _ids(as_user(user).get(f"/api/search/related/{SOURCE}")) == [_vid(1), _vid(2)]
def test_one_list_is_one_event_against_the_days_cap(db, make_user, as_user, pages):
user = make_user()
sysconfig.set_value(db, "related_daily_limit_per_user", 1)
pages["pages"] = [
_page([_item(1)], token="page2"),
_page([_item(2)], token=None),
]
client = as_user(user)
assert client.get(f"/api/search/related/{SOURCE}").status_code == 200
events = db.scalars(
select(QuotaEvent).where(
QuotaEvent.user_id == user.id, QuotaEvent.action == quota.QuotaAction.VIDEOS_RELATED
)
).all()
assert len(events) == 1
refused = client.get(f"/api/search/related/{SOURCE}")
assert refused.status_code == 429
assert refused.json()["detail"] == {
"code": "refused",
"reason": "related_daily_limit",
"limit": 1,
}
assert len(pages["calls"]) == 2 # the refused one asked YouTube nothing
def test_reading_further_is_the_same_list_not_another_one(db, make_user, as_user, pages):
user = make_user()
sysconfig.set_value(db, "related_daily_limit_per_user", 2)
pages["pages"] = [_page([_item(1)], token=None), _page([_item(2)], token=None)]
client = as_user(user)
assert client.get(f"/api/search/related/{SOURCE}").status_code == 200
assert client.get(f"/api/search/related/{SOURCE}?cursor=more").status_code == 200
assert quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED) == 1
backend/tests/test_related_scrape.py
+212
View File
@@ -0,0 +1,212 @@
"""youtube/related_scrape — a video's related videos from the InnerTube `next` endpoint.
The two pages under `fixtures/` are real answers (2026-10-24), trimmed to three items and the
continuation: they hold the `lockupViewModel` shape YouTube draws today. Everything here breaks
silently when YouTube moves — a parser that stops finding videos, a cursor that stops paging — so a
shape it does not know must say so, never read as "nothing related". The network is an httpx
MockTransport."""
import copy
import json
from pathlib import Path
import httpx
import pytest
from app.youtube import innertube, related_scrape
FIXTURES = Path(__file__).parent / "fixtures"
FIRST = json.loads((FIXTURES / "innertube_next_first.json").read_text(encoding="utf-8"))
CONTINUATION = json.loads(
(FIXTURES / "innertube_next_continuation.json").read_text(encoding="utf-8")
)
RESULTS_HTML = (
'<script>{"INNERTUBE_API_KEY":"live-key","INNERTUBE_CLIENT_VERSION":"2.20261001"}</script>'
)
def _results(page: dict) -> list:
return page["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"][
"results"
]
@pytest.fixture
def net(monkeypatch):
"""Route every httpx.Client the InnerTube module opens through a handler the test sets."""
calls: list[httpx.Request] = []
state: dict = {"handler": None, "calls": calls}
def transport(request: httpx.Request) -> httpx.Response:
calls.append(request)
if request.url.path == "/results":
return httpx.Response(200, text=RESULTS_HTML)
return state["handler"](request)
real = httpx.Client
monkeypatch.setattr(
innertube.httpx,
"Client",
lambda **kw: real(transport=httpx.MockTransport(transport), **kw),
)
monkeypatch.setattr(innertube, "_config", None)
return state
def _answer(net, page, status=200):
net["handler"] = lambda request: (
httpx.Response(status, text=page)
if isinstance(page, str)
else httpx.Response(status, json=page)
)
def test_a_first_page_gives_its_videos_with_their_channel_and_the_next_cursor(net):
_answer(net, FIRST)
page = related_scrape.related("SdTZAMDKrNY")
assert [it["id"] for it in page["items"]] == ["to-lHJfK4pw", "R8V8CbuxryI", "HLKo4fJx_7k"]
first = page["items"][0]
assert first["channel_id"] == "UCtslD4DGH6PKyG_1gFAX7sg"
assert first["channel_title"] == "Alexander Amini"
assert first["title"] == "MIT 6.S191 (2025): Reinforcement Learning"
assert first["thumbnail_url"].startswith("https://i.ytimg.com/vi/to-lHJfK4pw/")
assert first["published_at"] is None and first["live_broadcast"] == "none"
assert page["next_page_token"]
body = json.loads(net["calls"][-1].content)
assert body["videoId"] == "SdTZAMDKrNY" and "continuation" not in body
assert net["calls"][-1].url.path == "/youtubei/v1/next"
def test_the_largest_thumbnail_is_taken(net):
_answer(net, FIRST)
item = related_scrape.related("SdTZAMDKrNY")["items"][0]
sources = _results(FIRST)[0]["lockupViewModel"]["contentImage"]["thumbnailViewModel"]["image"][
"sources"
]
assert item["thumbnail_url"] == sources[-1]["url"]
def test_a_continuation_page_reads_its_videos_and_sends_only_the_cursor(net):
_answer(net, CONTINUATION)
page = related_scrape.related("SdTZAMDKrNY", continuation="TOKEN", hl="hu", gl="HU")
assert len(page["items"]) == 2
assert all(it["channel_id"].startswith("UC") for it in page["items"])
assert page["next_page_token"]
body = json.loads(net["calls"][-1].content)
assert body["continuation"] == "TOKEN" and "videoId" not in body
assert body["context"]["client"]["hl"] == "hu" and body["context"]["client"]["gl"] == "HU"
def test_the_locale_defaults_to_the_topical_one(net):
_answer(net, FIRST)
related_scrape.related("SdTZAMDKrNY")
client = json.loads(net["calls"][-1].content)["context"]["client"]
assert (client["hl"], client["gl"]) == ("en", "US")
def test_a_video_by_several_channels_is_left_out(net):
page = copy.deepcopy(FIRST)
meta = _results(page)[1]["lockupViewModel"]["metadata"]["lockupMetadataViewModel"]
meta["image"] = {"avatarStackViewModel": {"avatars": []}}
_answer(net, page)
ids = [it["id"] for it in related_scrape.related("SdTZAMDKrNY")["items"]]
assert ids == ["to-lHJfK4pw", "HLKo4fJx_7k"]
def test_what_is_not_a_video_is_left_out_and_a_repeat_counts_once(net):
page = copy.deepcopy(FIRST)
results = _results(page)
playlist = copy.deepcopy(results[0])
playlist["lockupViewModel"]["contentType"] = "LOCKUP_CONTENT_TYPE_PLAYLIST"
playlist["lockupViewModel"]["contentId"] = "PLxyz"
results.insert(0, playlist)
results.insert(2, copy.deepcopy(results[1]))
results.insert(0, {"adSlotRenderer": {}})
_answer(net, page)
ids = [it["id"] for it in related_scrape.related("SdTZAMDKrNY")["items"]]
assert ids == ["to-lHJfK4pw", "R8V8CbuxryI", "HLKo4fJx_7k"]
def test_the_previous_compact_renderer_is_still_read(net):
page = copy.deepcopy(FIRST)
_results(page).insert(
0,
{
"compactVideoRenderer": {
"videoId": "oldshape001",
"title": {"simpleText": "Old shape"},
"longBylineText": {
"runs": [
{
"text": "Old Chan",
"navigationEndpoint": {"browseEndpoint": {"browseId": "UColdchan"}},
}
]
},
"thumbnail": {"thumbnails": [{"url": "s.jpg"}, {"url": "b.jpg"}]},
}
},
)
_answer(net, page)
first = related_scrape.related("SdTZAMDKrNY")["items"][0]
assert first == {
"id": "oldshape001",
"channel_id": "UColdchan",
"channel_title": "Old Chan",
"title": "Old shape",
"published_at": None,
"thumbnail_url": "b.jpg",
"live_broadcast": "none",
}
def test_the_last_page_has_no_cursor(net):
page = copy.deepcopy(FIRST)
_results(page).pop() # the continuationItemRenderer
_answer(net, page)
assert related_scrape.related("SdTZAMDKrNY")["next_page_token"] is None
def test_a_video_youtube_no_longer_has_is_an_empty_page(net):
_answer(
net,
{
"contents": {
"twoColumnWatchNextResults": {
"results": {
"results": {
"contents": [
{
"itemSectionRenderer": {
"contents": [{"backgroundPromoRenderer": {}}]
}
}
]
}
}
}
}
},
)
assert related_scrape.related("aaaaaaaaaaa") == {"items": [], "next_page_token": None}
@pytest.mark.parametrize(
"answer",
[
{"responseContext": {}}, # no watch page frame at all
{"onResponseReceivedEndpoints": [{"somethingElse": {}}]}, # a continuation we do not know
"<html>consent</html>", # not JSON
[1, 2], # JSON, not an object
],
)
def test_an_answer_of_an_unknown_shape_is_an_error_not_an_empty_list(net, answer):
_answer(net, answer)
with pytest.raises(related_scrape.ScrapeError):
related_scrape.related("SdTZAMDKrNY")
def test_a_refused_request_is_an_error_whatever_its_body_says(net):
_answer(net, FIRST, status=429)
with pytest.raises(related_scrape.ScrapeError):
related_scrape.related("SdTZAMDKrNY")
backend/tests/test_search_scrape.py
+4 -4
View File
@@ -9,7 +9,7 @@ import json
import httpx
import pytest
from app.youtube import search_scrape
from app.youtube import innertube, search_scrape
RESULTS_HTML = (
'<script>{"INNERTUBE_API_KEY":"live-key","INNERTUBE_CLIENT_VERSION":"2.20261001"}</script>'
@@ -79,11 +79,11 @@ def net(monkeypatch):
real = httpx.Client
monkeypatch.setattr(
search_scrape.httpx,
innertube.httpx,
"Client",
lambda **kw: real(transport=httpx.MockTransport(transport), **kw),
)
monkeypatch.setattr(search_scrape, "_config", None)
monkeypatch.setattr(innertube, "_config", None)
state["calls"] = calls
return state
@@ -171,7 +171,7 @@ def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, re
with no error anywhere."""
net["handler"] = _serve(results_html=results_html)
search_scrape.search("a")
assert net["calls"][-1].url.params["key"] == search_scrape._FALLBACK_KEY
assert net["calls"][-1].url.params["key"] == innertube._FALLBACK_KEY
net["handler"] = _serve()
search_scrape.search("b")