siftlode/backend/app/youtube/innertube.py

"""The website's own InnerTube API (`youtubei/v1/*`), read the way the website reads it — no quota.

The zero-quota search (`search_scrape`) and the related videos (`related_scrape`) both POST here with
the public WEB key and a client version, read out of a live results page. This deliberately calls an
undocumented endpoint: if YouTube changes the JSON shape or blocks the host, each caller says so as a
`ScrapeError`, never as an empty answer.

The SOCS=CAI consent-skip cookie is sent like the Shorts probe's, so a server-side request isn't
bounced to consent.youtube.com."""

import re

import httpx

from app import logs

log = logs.get_logger(__name__)

_API_URL = "https://www.youtube.com/youtubei/v1/"
_RESULTS_URL = "https://www.youtube.com/results"
# Public WEB InnerTube key + a recent client version. These are also embedded in every results
# page; we fetch the live pair once and cache it, falling back to these constants.
_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"

_HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
    # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
    # redirected to consent.youtube.com).
    "Cookie": "SOCS=CAI",
    "Accept-Language": "en-US,en;q=0.9",
}

_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')

# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
_config: tuple[str, str] | None = None


class ScrapeError(Exception):
    """An InnerTube read could not be completed (network, blocked, or unexpected shape)."""


def runs_text(node: dict | None) -> str | None:
    """The text of an InnerTube text node: a `simpleText`, or its `runs` joined."""
    if not node:
        return None
    if node.get("simpleText"):
        return node["simpleText"]
    runs = node.get("runs") or []
    if runs:
        return "".join(r.get("text", "") for r in runs)
    return None


def scraped_video(
    vid: str, channel_id: object, channel_title: object, title: object, thumbnails: object
) -> dict:
    """One scraped video in the shape `YouTubeClient.search_videos` answers, so a page of either
    source is ingested the same way. `published_at` is None (the website shows only a relative
    "3 years ago"; the enrich fills the real date) and `live_broadcast` "none" (live/upcoming are
    judged after the enrich, by `live_status`). The largest thumbnail is the last."""
    thumbs = thumbnails if isinstance(thumbnails, list) else []
    last = thumbs[-1] if thumbs else None
    return {
        "id": vid,
        "channel_id": channel_id if isinstance(channel_id, str) and channel_id else None,
        "channel_title": channel_title if isinstance(channel_title, str) else None,
        "title": title if isinstance(title, str) else None,
        "published_at": None,
        "thumbnail_url": last.get("url") if isinstance(last, dict) else None,
        "live_broadcast": "none",
    }


def video_renderer(node: dict) -> dict | None:
    """A `videoRenderer` (search) or `compactVideoRenderer` (the older related column) flattened —
    the two draw a video the same way. None without a video id; the channel is None when the byline
    names none."""
    vid = node.get("videoId")
    if not isinstance(vid, str) or not vid:
        return None
    owner = node.get("ownerText") or node.get("longBylineText") or {}
    runs = owner.get("runs") if isinstance(owner, dict) else None
    first = runs[0] if isinstance(runs, list) and runs and isinstance(runs[0], dict) else {}
    endpoint = (first.get("navigationEndpoint") or {}).get("browseEndpoint") or {}
    return scraped_video(
        vid,
        endpoint.get("browseId"),
        first.get("text"),
        runs_text(node.get("title")),
        (node.get("thumbnail") or {}).get("thumbnails"),
    )


def continuation_token(node: object) -> str | None:
    """The "load more" cursor of a `continuationItemRenderer`, or None when `node` is not one."""
    if not isinstance(node, dict):
        return None
    command = (node.get("continuationEndpoint") or {}).get("continuationCommand") or {}
    token = command.get("token")
    return token if isinstance(token, str) and token else None


def _get_config(client: httpx.Client) -> tuple[str, str]:
    """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.

    The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
    keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
    the life of the process — and when those went stale, search returned nothing, with no error
    anywhere. Now the next call simply tries the live pair again."""
    global _config
    if _config is not None:
        return _config
    try:
        html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
    except httpx.HTTPError as exc:
        log.warning(
            "the InnerTube config could not be fetched; falling back to the pinned key",
            extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
        )
        return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
    key = _KEY_RE.search(html)
    ver = _VER_RE.search(html)
    if key and ver:
        _config = (key.group(1), ver.group(1))
        return _config
    log.warning(
        "the results page carried no InnerTube config; falling back to the pinned key",
        extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
    )
    return (
        key.group(1) if key else _FALLBACK_KEY,
        ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
    )


def post(endpoint: str, body: dict, *, hl: str = "en", gl: str = "US") -> dict:
    """POST `body` to `youtubei/v1/<endpoint>` as the WEB client in the `hl`/`gl` locale and return
    the decoded JSON object. A network failure, an HTTP error or a body that is not a JSON object (a
    block, consent or captcha page) raises `ScrapeError`; the caller judges the shape."""
    with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
        api_key, client_version = _get_config(client)
        context = {
            "client": {"clientName": "WEB", "clientVersion": client_version, "hl": hl, "gl": gl}
        }
        try:
            resp = client.post(
                _API_URL + endpoint, params={"key": api_key}, json={"context": context, **body}
            )
            resp.raise_for_status()
            data = resp.json()
        except httpx.HTTPError as exc:
            raise ScrapeError(f"InnerTube request failed: {exc}") from exc
        except ValueError as exc:  # non-JSON body (blocked / consent / captcha)
            raise ScrapeError("InnerTube returned a non-JSON response") from exc
    if not isinstance(data, dict):
        raise ScrapeError("InnerTube returned an unexpected shape")
    return data