siftlode/backend/app/youtube/related_scrape.py

"""A video's related videos, read from the watch page's InnerTube `next` endpoint — no quota.

The Data API has no related-videos call any more, so this is the only source: the same list the
website shows beside a playing video. A page holds about 20 videos and a continuation token for the
next 20. The returned page has the search scrape's shape (``{"items": [...], "next_page_token"}``)
so the catalog ingests it the same way (`catalog.search.ingest_candidates`).

YouTube draws each item as a `lockupViewModel` now; the older `compactVideoRenderer` is read too, so
a rollback on its side does not empty the list. An item that names no single channel — a video by
several channels draws an avatar stack, not one avatar — is left out: the catalog row needs its
channel up front, and the stack does not say which one uploaded it.

`hl`/`gl` pick the locale YouTube fills the list in: ``en``/``US`` keeps it on the video's topic,
``hu``/``HU`` mixes in what is trending in Hungary (measured)."""

from app.youtube.innertube import ScrapeError, post, runs_text

__all__ = ["ScrapeError", "related"]

# Where a first page and a continuation page put their item list.
_FIRST = (
    "contents",
    "twoColumnWatchNextResults",
    "secondaryResults",
    "secondaryResults",
    "results",
)
_APPENDS = ("appendContinuationItemsAction", "reloadContinuationItemsCommand")


def _at(node: object, *path: str | int) -> object:
    """The value under `path` in nested dicts/lists, or None where the path stops."""
    for key in path:
        if isinstance(node, dict) and isinstance(key, str):
            node = node.get(key)
        elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node):
            node = node[key]
        else:
            return None
    return node


def _item(vid: str, channel_id: object, channel_title: object, title: object, sources: object):
    if not isinstance(channel_id, str) or not channel_id:
        return None
    thumbs = sources if isinstance(sources, list) else []
    return {
        "id": vid,
        "channel_id": channel_id,
        "channel_title": channel_title if isinstance(channel_title, str) else None,
        "title": title if isinstance(title, str) else None,
        "published_at": None,  # only a relative "1y ago" here; the enrich fills the real date
        "thumbnail_url": _at(thumbs, -1, "url"),
        "live_broadcast": "none",  # live/upcoming are judged after the enrich, by live_status
    }


def _lockup(node: dict) -> dict | None:
    if node.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO" or not node.get("contentId"):
        return None
    meta = _at(node, "metadata", "lockupMetadataViewModel")
    channel_id = _at(
        meta,
        "image",
        "decoratedAvatarViewModel",
        "rendererContext",
        "commandContext",
        "onTap",
        "innertubeCommand",
        "browseEndpoint",
        "browseId",
    )
    channel_title = _at(
        meta, "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0
    )
    return _item(
        node["contentId"],
        channel_id,
        _at(channel_title, "text", "content"),
        _at(meta, "title", "content"),
        _at(node, "contentImage", "thumbnailViewModel", "image", "sources"),
    )


def _compact(node: dict) -> dict | None:
    vid = node.get("videoId")
    if not vid:
        return None
    byline = _at(node, "longBylineText", "runs", 0)
    return _item(
        vid,
        _at(byline, "navigationEndpoint", "browseEndpoint", "browseId"),
        _at(byline, "text"),
        runs_text(node.get("title")),
        _at(node, "thumbnail", "thumbnails"),
    )


def _items(data: dict) -> list | None:
    """The page's item list (first page or continuation), or None when it has neither frame."""
    first = _at(data, *_FIRST)
    if isinstance(first, list):
        return first
    # A video YouTube no longer has keeps the watch page's frame but draws an "isn't available"
    # promo in it and no related column: nothing related, said by YouTube itself.
    if isinstance(_at(data, *_FIRST[:2]), dict):
        return []
    for endpoint in data.get("onResponseReceivedEndpoints") or []:
        for key in _APPENDS:
            items = _at(endpoint, key, "continuationItems")
            if isinstance(items, list):
                return items
    return None


def related(
    video_id: str, continuation: str | None = None, *, hl: str = "en", gl: str = "US"
) -> dict:
    """One page of `video_id`'s related videos (`continuation` = the token a previous page gave).

    A video YouTube no longer has is an empty page. An answer without the watch page's frame is a
    `ScrapeError` — not an empty page, which would read as "nothing related"."""
    body = {"continuation": continuation} if continuation else {"videoId": video_id}
    data = post("next", body, hl=hl, gl=gl)
    items = _items(data)
    if items is None:
        raise ScrapeError("InnerTube returned no related list")
    out: list[dict] = []
    seen: set[str] = set()
    token = None
    for entry in items:
        if not isinstance(entry, dict):
            continue
        if "lockupViewModel" in entry:
            item = _lockup(entry["lockupViewModel"])
        elif "compactVideoRenderer" in entry:
            item = _compact(entry["compactVideoRenderer"])
        else:
            token = token or _at(
                entry,
                "continuationItemRenderer",
                "continuationEndpoint",
                "continuationCommand",
                "token",
            )
            continue
        if item and item["id"] not in seen:
            seen.add(item["id"])
            out.append(item)
    return {"items": out, "next_page_token": token if isinstance(token, str) else None}