"""youtube/search_scrape — the zero-quota InnerTube search (the module was 20% covered). Everything here breaks silently when YouTube moves: a parser that stops finding videos, a cursor that stops paging, a fallback key that outlives its usefulness. The network is an httpx MockTransport. """ import json import httpx import pytest from app.youtube import innertube, search_scrape RESULTS_HTML = ( '' ) def _video(vid: str, title: str, *, simple: bool = False) -> dict: return { "videoRenderer": { "videoId": vid, "title": {"simpleText": title} if simple else {"runs": [{"text": title[:3]}, {"text": title[3:]}]}, "ownerText": { "runs": [ { "text": "Chan", "navigationEndpoint": {"browseEndpoint": {"browseId": "UCchan"}}, } ] }, "thumbnail": {"thumbnails": [{"url": "small.jpg"}, {"url": "big.jpg"}]}, } } PAGE = { "contents": { "sectionListRenderer": { "contents": [ { "itemSectionRenderer": { "contents": [ _video("v1", "First video"), {"channelRenderer": {"channelId": "UCx"}}, # not a video: skipped { "reelShelfRenderer": { "items": [{"reelItemRenderer": {"videoId": "s1"}}] } }, _video("v2", "Second", simple=True), _video("v1", "First video"), # a repeat in another shelf: once {"videoRenderer": {"title": {"simpleText": "no id"}}}, ] } }, { "continuationItemRenderer": { "continuationEndpoint": {"continuationCommand": {"token": "NEXT"}} } }, ] } } } @pytest.fixture def net(monkeypatch): """Route every httpx.Client the module opens through a handler the test sets.""" calls: list[httpx.Request] = [] state: dict = {"handler": None} def transport(request: httpx.Request) -> httpx.Response: calls.append(request) return state["handler"](request) real = httpx.Client monkeypatch.setattr( innertube.httpx, "Client", lambda **kw: real(transport=httpx.MockTransport(transport), **kw), ) monkeypatch.setattr(innertube, "_config", None) state["calls"] = calls return state def _serve(results_html=RESULTS_HTML, search=PAGE, status=200): def handler(request: httpx.Request) -> httpx.Response: if request.url.path == "/results": if isinstance(results_html, Exception): raise results_html return httpx.Response(200, text=results_html) if isinstance(search, str): return httpx.Response(status, text=search) return httpx.Response(status, json=search) return handler def test_a_page_flattens_its_videos_in_order_and_hands_back_the_cursor(net): net["handler"] = _serve() page = search_scrape.search("cats") assert page["next_page_token"] == "NEXT" assert [i["id"] for i in page["items"]] == ["v1", "v2"] assert page["items"][0] == { "id": "v1", "channel_id": "UCchan", "channel_title": "Chan", "title": "First video", "published_at": None, "thumbnail_url": "big.jpg", "live_broadcast": "none", } assert page["items"][1]["title"] == "Second" post = net["calls"][-1] assert post.url.params["key"] == "live-key" body = json.loads(post.content) assert body["query"] == "cats" and body["params"] == search_scrape._TYPE_VIDEO assert body["context"]["client"]["clientVersion"] == "2.20261001" def test_the_next_page_sends_the_cursor_not_the_query_and_the_end_has_none(net): net["handler"] = _serve(search={"onResponseReceivedCommands": [_video("v9", "Last")]}) page = search_scrape.search("cats", continuation="NEXT") body = json.loads(net["calls"][-1].content) assert body["continuation"] == "NEXT" and "query" not in body assert [i["id"] for i in page["items"]] == ["v9"] assert page["next_page_token"] is None @pytest.mark.parametrize( ("search", "status"), [ ("consent", 200), ({"error": "blocked"}, 429), ({"responseContext": {}, "alerts": ["unusual traffic"]}, 200), ([], 200), ], ids=["not-json", "http-error", "unknown-shape", "not-an-object"], ) def test_a_blocked_or_garbled_answer_is_an_error_not_an_empty_page(net, search, status): net["handler"] = _serve(search=search, status=status) with pytest.raises(search_scrape.ScrapeError): search_scrape.search("cats") def test_a_search_with_no_results_is_an_empty_page_not_an_error(net): net["handler"] = _serve(search={"contents": {"sectionListRenderer": {"contents": []}}}) assert search_scrape.search("zzzzqqq") == {"items": [], "next_page_token": None} def test_the_live_key_is_fetched_once_and_cached(net): net["handler"] = _serve() search_scrape.search("a") search_scrape.search("b") assert [c.url.path for c in net["calls"]].count("/results") == 1 @pytest.mark.parametrize( "results_html", [httpx.ConnectError("down"), "a consent page, no keys"], ids=["fetch-failed", "no-key-on-page"], ) def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, results_html): """The fallback pair used to be CACHED for the life of the process: one failed fetch pinned every later search to a hard-coded key, and once that went stale search returned nothing, with no error anywhere.""" net["handler"] = _serve(results_html=results_html) search_scrape.search("a") assert net["calls"][-1].url.params["key"] == innertube._FALLBACK_KEY net["handler"] = _serve() search_scrape.search("b") assert net["calls"][-1].url.params["key"] == "live-key"