"""A video's related videos, read from the watch page's InnerTube `next` endpoint — no quota. The Data API has no related-videos call any more, so this is the only source: the same list the website shows beside a playing video. A page holds about 20 videos and a continuation token for the next 20. The returned page has the search scrape's shape (``{"items": [...], "next_page_token"}``) so the catalog ingests it the same way (`catalog.search.ingest_candidates`). YouTube draws each item as a `lockupViewModel` now; the older `compactVideoRenderer` is read too, so a rollback on its side does not empty the list. An item that names no single channel — a video by several channels draws an avatar stack, not one avatar — is left out: the catalog row needs its channel up front, and the stack does not say which one uploaded it. `hl`/`gl` pick the locale YouTube fills the list in: ``en``/``US`` keeps it on the video's topic, ``hu``/``HU`` mixes in what is trending in Hungary (measured).""" from app.youtube.innertube import ScrapeError, post, runs_text __all__ = ["ScrapeError", "related"] # Where a first page and a continuation page put their item list. _FIRST = ( "contents", "twoColumnWatchNextResults", "secondaryResults", "secondaryResults", "results", ) _APPENDS = ("appendContinuationItemsAction", "reloadContinuationItemsCommand") def _at(node: object, *path: str | int) -> object: """The value under `path` in nested dicts/lists, or None where the path stops.""" for key in path: if isinstance(node, dict) and isinstance(key, str): node = node.get(key) elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node): node = node[key] else: return None return node def _item(vid: str, channel_id: object, channel_title: object, title: object, sources: object): if not isinstance(channel_id, str) or not channel_id: return None thumbs = sources if isinstance(sources, list) else [] return { "id": vid, "channel_id": channel_id, "channel_title": channel_title if isinstance(channel_title, str) else None, "title": title if isinstance(title, str) else None, "published_at": None, # only a relative "1y ago" here; the enrich fills the real date "thumbnail_url": _at(thumbs, -1, "url"), "live_broadcast": "none", # live/upcoming are judged after the enrich, by live_status } def _lockup(node: dict) -> dict | None: if node.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO" or not node.get("contentId"): return None meta = _at(node, "metadata", "lockupMetadataViewModel") channel_id = _at( meta, "image", "decoratedAvatarViewModel", "rendererContext", "commandContext", "onTap", "innertubeCommand", "browseEndpoint", "browseId", ) channel_title = _at( meta, "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0 ) return _item( node["contentId"], channel_id, _at(channel_title, "text", "content"), _at(meta, "title", "content"), _at(node, "contentImage", "thumbnailViewModel", "image", "sources"), ) def _compact(node: dict) -> dict | None: vid = node.get("videoId") if not vid: return None byline = _at(node, "longBylineText", "runs", 0) return _item( vid, _at(byline, "navigationEndpoint", "browseEndpoint", "browseId"), _at(byline, "text"), runs_text(node.get("title")), _at(node, "thumbnail", "thumbnails"), ) def _items(data: dict) -> list | None: """The page's item list (first page or continuation), or None when it has neither frame.""" first = _at(data, *_FIRST) if isinstance(first, list): return first # A video YouTube no longer has keeps the watch page's frame but draws an "isn't available" # promo in it and no related column: nothing related, said by YouTube itself. if isinstance(_at(data, *_FIRST[:2]), dict): return [] for endpoint in data.get("onResponseReceivedEndpoints") or []: for key in _APPENDS: items = _at(endpoint, key, "continuationItems") if isinstance(items, list): return items return None def related( video_id: str, continuation: str | None = None, *, hl: str = "en", gl: str = "US" ) -> dict: """One page of `video_id`'s related videos (`continuation` = the token a previous page gave). A video YouTube no longer has is an empty page. An answer without the watch page's frame is a `ScrapeError` — not an empty page, which would read as "nothing related".""" body = {"continuation": continuation} if continuation else {"videoId": video_id} data = post("next", body, hl=hl, gl=gl) items = _items(data) if items is None: raise ScrapeError("InnerTube returned no related list") out: list[dict] = [] seen: set[str] = set() token = None for entry in items: if not isinstance(entry, dict): continue if "lockupViewModel" in entry: item = _lockup(entry["lockupViewModel"]) elif "compactVideoRenderer" in entry: item = _compact(entry["compactVideoRenderer"]) else: token = token or _at( entry, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token", ) continue if item and item["id"] not in seen: seen.add(item["id"]) out.append(item) return {"items": out, "next_page_token": token if isinstance(token, str) else None}