fix(youtube): a related page is empty only on YouTube's own word
The watch frame alone read as "nothing related" — so a related column YouTube moved, or a list drawn by renderers this does not read, answered an empty list instead of an error. Now only the "isn't available" promo YouTube draws for a video it no longer has is an empty page; anything else it cannot read is a ScrapeError. The flattening of a video renderer and the read of the continuation token are one each, in innertube, for the search and the related scrape.
This commit is contained in:
@@ -17,7 +17,7 @@ only difference is the cursor is an InnerTube *continuation token*, not a Data A
|
||||
the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"``
|
||||
and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path)."""
|
||||
|
||||
from app.youtube.innertube import ScrapeError, post, runs_text
|
||||
from app.youtube.innertube import ScrapeError, continuation_token, post, video_renderer
|
||||
|
||||
__all__ = ["ScrapeError", "search"]
|
||||
|
||||
@@ -29,29 +29,6 @@ _TYPE_VIDEO = "EgIQAQ=="
|
||||
_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"}
|
||||
|
||||
|
||||
def _video_item(vr: dict) -> dict | None:
|
||||
"""Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos."""
|
||||
vid = vr.get("videoId")
|
||||
if not vid:
|
||||
return None
|
||||
owner = vr.get("ownerText") or vr.get("longBylineText") or {}
|
||||
runs = owner.get("runs") or []
|
||||
channel_id = channel_title = None
|
||||
if runs:
|
||||
channel_title = runs[0].get("text")
|
||||
channel_id = runs[0].get("navigationEndpoint", {}).get("browseEndpoint", {}).get("browseId")
|
||||
thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or []
|
||||
return {
|
||||
"id": vid,
|
||||
"channel_id": channel_id,
|
||||
"channel_title": channel_title,
|
||||
"title": runs_text(vr.get("title")),
|
||||
"published_at": None, # scrape exposes only a relative time; enrich fills the real date
|
||||
"thumbnail_url": thumbs[-1]["url"] if thumbs else None,
|
||||
"live_broadcast": "none", # live/upcoming filtered post-enrich via live_status
|
||||
}
|
||||
|
||||
|
||||
def _collect_videos(obj, out: list[dict], seen: set[str]) -> None:
|
||||
"""Walk the response collecting videoRenderer items in document (relevance) order. Only
|
||||
`videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are
|
||||
@@ -59,7 +36,7 @@ def _collect_videos(obj, out: list[dict], seen: set[str]) -> None:
|
||||
if isinstance(obj, dict):
|
||||
vr = obj.get("videoRenderer")
|
||||
if isinstance(vr, dict):
|
||||
item = _video_item(vr)
|
||||
item = video_renderer(vr)
|
||||
if item and item["id"] not in seen:
|
||||
seen.add(item["id"])
|
||||
out.append(item)
|
||||
@@ -73,11 +50,9 @@ def _collect_videos(obj, out: list[dict], seen: set[str]) -> None:
|
||||
def _find_continuation(obj) -> str | None:
|
||||
"""First continuationItemRenderer token (the 'load more' cursor), or None at the end."""
|
||||
if isinstance(obj, dict):
|
||||
cir = obj.get("continuationItemRenderer")
|
||||
if isinstance(cir, dict):
|
||||
tok = cir.get("continuationEndpoint", {}).get("continuationCommand", {}).get("token")
|
||||
if tok:
|
||||
return tok
|
||||
tok = continuation_token(obj.get("continuationItemRenderer"))
|
||||
if tok:
|
||||
return tok
|
||||
for v in obj.values():
|
||||
tok = _find_continuation(v)
|
||||
if tok:
|
||||
|
||||
Reference in New Issue
Block a user