peter
8b37c470ba
GET /api/search/related/{video_id} reads the list YouTube shows beside a
video from the website's InnerTube "next" endpoint (no quota), and brings
it into the catalog the way live-search results come in. It leaves out
what the user watched or hid, here or in their Takeout history (unless
taken back to "new" since), blocked channels and the video itself, and
pages on until 20 are gathered (at most 4 pages). region=topical|hu picks
the locale the list is filled in. Its own per-user daily cap
(related_daily_limit_per_user, one event per opened list); a failed
scrape answers 422 youtube/unavailable.
The InnerTube request (live key, consent cookie, errors) moves into
youtube/innertube.py, shared by the search scrape and the related one.
112 lines
4.6 KiB
Python
112 lines
4.6 KiB
Python
"""The website's own InnerTube API (`youtubei/v1/*`), read the way the website reads it — no quota.
|
|
|
|
The zero-quota search (`search_scrape`) and the related videos (`related_scrape`) both POST here with
|
|
the public WEB key and a client version, read out of a live results page. This deliberately calls an
|
|
undocumented endpoint: if YouTube changes the JSON shape or blocks the host, each caller says so as a
|
|
`ScrapeError`, never as an empty answer.
|
|
|
|
The SOCS=CAI consent-skip cookie is sent like the Shorts probe's, so a server-side request isn't
|
|
bounced to consent.youtube.com."""
|
|
|
|
import re
|
|
|
|
import httpx
|
|
|
|
from app import logs
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
_API_URL = "https://www.youtube.com/youtubei/v1/"
|
|
_RESULTS_URL = "https://www.youtube.com/results"
|
|
# Public WEB InnerTube key + a recent client version. These are also embedded in every results
|
|
# page; we fetch the live pair once and cache it, falling back to these constants.
|
|
_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
|
_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"
|
|
|
|
_HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
|
|
# SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
|
|
# redirected to consent.youtube.com).
|
|
"Cookie": "SOCS=CAI",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
}
|
|
|
|
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
|
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
|
|
|
# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
|
|
_config: tuple[str, str] | None = None
|
|
|
|
|
|
class ScrapeError(Exception):
|
|
"""An InnerTube read could not be completed (network, blocked, or unexpected shape)."""
|
|
|
|
|
|
def runs_text(node: dict | None) -> str | None:
|
|
"""The text of an InnerTube text node: a `simpleText`, or its `runs` joined."""
|
|
if not node:
|
|
return None
|
|
if node.get("simpleText"):
|
|
return node["simpleText"]
|
|
runs = node.get("runs") or []
|
|
if runs:
|
|
return "".join(r.get("text", "") for r in runs)
|
|
return None
|
|
|
|
|
|
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
|
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
|
|
|
|
The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
|
|
keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
|
|
the life of the process — and when those went stale, search returned nothing, with no error
|
|
anywhere. Now the next call simply tries the live pair again."""
|
|
global _config
|
|
if _config is not None:
|
|
return _config
|
|
try:
|
|
html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
|
|
except httpx.HTTPError as exc:
|
|
log.warning(
|
|
"the InnerTube config could not be fetched; falling back to the pinned key",
|
|
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
|
|
)
|
|
return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
|
key = _KEY_RE.search(html)
|
|
ver = _VER_RE.search(html)
|
|
if key and ver:
|
|
_config = (key.group(1), ver.group(1))
|
|
return _config
|
|
log.warning(
|
|
"the results page carried no InnerTube config; falling back to the pinned key",
|
|
extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
|
|
)
|
|
return (
|
|
key.group(1) if key else _FALLBACK_KEY,
|
|
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
|
)
|
|
|
|
|
|
def post(endpoint: str, body: dict, *, hl: str = "en", gl: str = "US") -> dict:
|
|
"""POST `body` to `youtubei/v1/<endpoint>` as the WEB client in the `hl`/`gl` locale and return
|
|
the decoded JSON object. A network failure, an HTTP error or a body that is not a JSON object (a
|
|
block, consent or captcha page) raises `ScrapeError`; the caller judges the shape."""
|
|
with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
|
|
api_key, client_version = _get_config(client)
|
|
context = {
|
|
"client": {"clientName": "WEB", "clientVersion": client_version, "hl": hl, "gl": gl}
|
|
}
|
|
try:
|
|
resp = client.post(
|
|
_API_URL + endpoint, params={"key": api_key}, json={"context": context, **body}
|
|
)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
except httpx.HTTPError as exc:
|
|
raise ScrapeError(f"InnerTube request failed: {exc}") from exc
|
|
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
|
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
|
if not isinstance(data, dict):
|
|
raise ScrapeError("InnerTube returned an unexpected shape")
|
|
return data
|