fix(youtube): a related page is empty only on YouTube's own word
The watch frame alone read as "nothing related" — so a related column YouTube moved, or a list drawn by renderers this does not read, answered an empty list instead of an error. Now only the "isn't available" promo YouTube draws for a video it no longer has is an empty page; anything else it cannot read is a ScrapeError. The flattening of a video renderer and the read of the continuation token are one each, in innertube, for the search and the related scrape.
This commit is contained in:
@@ -54,6 +54,55 @@ def runs_text(node: dict | None) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def scraped_video(
|
||||
vid: str, channel_id: object, channel_title: object, title: object, thumbnails: object
|
||||
) -> dict:
|
||||
"""One scraped video in the shape `YouTubeClient.search_videos` answers, so a page of either
|
||||
source is ingested the same way. `published_at` is None (the website shows only a relative
|
||||
"3 years ago"; the enrich fills the real date) and `live_broadcast` "none" (live/upcoming are
|
||||
judged after the enrich, by `live_status`). The largest thumbnail is the last."""
|
||||
thumbs = thumbnails if isinstance(thumbnails, list) else []
|
||||
last = thumbs[-1] if thumbs else None
|
||||
return {
|
||||
"id": vid,
|
||||
"channel_id": channel_id if isinstance(channel_id, str) and channel_id else None,
|
||||
"channel_title": channel_title if isinstance(channel_title, str) else None,
|
||||
"title": title if isinstance(title, str) else None,
|
||||
"published_at": None,
|
||||
"thumbnail_url": last.get("url") if isinstance(last, dict) else None,
|
||||
"live_broadcast": "none",
|
||||
}
|
||||
|
||||
|
||||
def video_renderer(node: dict) -> dict | None:
|
||||
"""A `videoRenderer` (search) or `compactVideoRenderer` (the older related column) flattened —
|
||||
the two draw a video the same way. None without a video id; the channel is None when the byline
|
||||
names none."""
|
||||
vid = node.get("videoId")
|
||||
if not isinstance(vid, str) or not vid:
|
||||
return None
|
||||
owner = node.get("ownerText") or node.get("longBylineText") or {}
|
||||
runs = owner.get("runs") if isinstance(owner, dict) else None
|
||||
first = runs[0] if isinstance(runs, list) and runs and isinstance(runs[0], dict) else {}
|
||||
endpoint = (first.get("navigationEndpoint") or {}).get("browseEndpoint") or {}
|
||||
return scraped_video(
|
||||
vid,
|
||||
endpoint.get("browseId"),
|
||||
first.get("text"),
|
||||
runs_text(node.get("title")),
|
||||
(node.get("thumbnail") or {}).get("thumbnails"),
|
||||
)
|
||||
|
||||
|
||||
def continuation_token(node: object) -> str | None:
|
||||
"""The "load more" cursor of a `continuationItemRenderer`, or None when `node` is not one."""
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
command = (node.get("continuationEndpoint") or {}).get("continuationCommand") or {}
|
||||
token = command.get("token")
|
||||
return token if isinstance(token, str) and token else None
|
||||
|
||||
|
||||
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
||||
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user