diff --git a/backend/app/youtube/search_scrape.py b/backend/app/youtube/search_scrape.py index cfdbc95e..391416c8 100644 --- a/backend/app/youtube/search_scrape.py +++ b/backend/app/youtube/search_scrape.py @@ -42,6 +42,9 @@ _HEADERS = { "Accept-Language": "en-US,en;q=0.9", } +# The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands) +# put their result list under. +_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"} _KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') _VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') @@ -65,29 +68,36 @@ def _runs_text(node: dict | None) -> str | None: def _get_config(client: httpx.Client) -> tuple[str, str]: - """The current InnerTube (api_key, client_version), cached after the first fetch.""" + """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them. + + The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no + keys on it) at the wrong moment pinned every later search to a hard-coded key and version for + the life of the process — and when those went stale, search returned nothing, with no error + anywhere (E32 S2). Now the next search simply tries the live pair again.""" global _config if _config is not None: return _config try: - resp = client.get(_RESULTS_URL, params={"search_query": "music"}) - html = resp.text - key = _KEY_RE.search(html) - ver = _VER_RE.search(html) - _config = ( - key.group(1) if key else _FALLBACK_KEY, - ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, - ) + html = client.get(_RESULTS_URL, params={"search_query": "music"}).text except httpx.HTTPError as exc: - # The fallback pair is CACHED in `_config` for the life of the process, so one failed - # fetch at the wrong moment pins every later search to a hard-coded key and version — - # and when those go stale, search returns nothing at all, with no error anywhere. log.warning( "the InnerTube config could not be fetched; falling back to the pinned key", extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__), ) - _config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) - return _config + return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) + key = _KEY_RE.search(html) + ver = _VER_RE.search(html) + if key and ver: + _config = (key.group(1), ver.group(1)) + return _config + log.warning( + "the results page carried no InnerTube config; falling back to the pinned key", + extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"), + ) + return ( + key.group(1) if key else _FALLBACK_KEY, + ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, + ) def _video_item(vr: dict) -> dict | None: @@ -182,6 +192,10 @@ def search(q: str, continuation: str | None = None) -> dict: except ValueError as exc: # non-JSON body (blocked / consent / captcha) raise ScrapeError("InnerTube returned a non-JSON response") from exc + # Even a search with no results carries its result list's frame. Without one this is not "no + # results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so. + if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys(): + raise ScrapeError("InnerTube returned an unexpected shape") items: list[dict] = [] _collect_videos(data, items, set()) return {"items": items, "next_page_token": _find_continuation(data)} diff --git a/backend/tests/test_search_scrape.py b/backend/tests/test_search_scrape.py new file mode 100644 index 00000000..b2dc0d79 --- /dev/null +++ b/backend/tests/test_search_scrape.py @@ -0,0 +1,178 @@ +"""youtube/search_scrape — the zero-quota InnerTube search (E32 S2: the module was 20% covered). + +Everything here breaks silently when YouTube moves: a parser that stops finding videos, a cursor that +stops paging, a fallback key that outlives its usefulness. The network is an httpx MockTransport. +""" + +import json + +import httpx +import pytest + +from app.youtube import search_scrape + +RESULTS_HTML = ( + '' +) + + +def _video(vid: str, title: str, *, simple: bool = False) -> dict: + return { + "videoRenderer": { + "videoId": vid, + "title": {"simpleText": title} + if simple + else {"runs": [{"text": title[:3]}, {"text": title[3:]}]}, + "ownerText": { + "runs": [ + { + "text": "Chan", + "navigationEndpoint": {"browseEndpoint": {"browseId": "UCchan"}}, + } + ] + }, + "thumbnail": {"thumbnails": [{"url": "small.jpg"}, {"url": "big.jpg"}]}, + } + } + + +PAGE = { + "contents": { + "sectionListRenderer": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [ + _video("v1", "First video"), + {"channelRenderer": {"channelId": "UCx"}}, # not a video: skipped + { + "reelShelfRenderer": { + "items": [{"reelItemRenderer": {"videoId": "s1"}}] + } + }, + _video("v2", "Second", simple=True), + _video("v1", "First video"), # a repeat in another shelf: once + {"videoRenderer": {"title": {"simpleText": "no id"}}}, + ] + } + }, + { + "continuationItemRenderer": { + "continuationEndpoint": {"continuationCommand": {"token": "NEXT"}} + } + }, + ] + } + } +} + + +@pytest.fixture +def net(monkeypatch): + """Route every httpx.Client the module opens through a handler the test sets.""" + calls: list[httpx.Request] = [] + state: dict = {"handler": None} + + def transport(request: httpx.Request) -> httpx.Response: + calls.append(request) + return state["handler"](request) + + real = httpx.Client + monkeypatch.setattr( + search_scrape.httpx, + "Client", + lambda **kw: real(transport=httpx.MockTransport(transport), **kw), + ) + monkeypatch.setattr(search_scrape, "_config", None) + state["calls"] = calls + return state + + +def _serve(results_html=RESULTS_HTML, search=PAGE, status=200): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/results": + if isinstance(results_html, Exception): + raise results_html + return httpx.Response(200, text=results_html) + if isinstance(search, str): + return httpx.Response(status, text=search) + return httpx.Response(status, json=search) + + return handler + + +def test_a_page_flattens_its_videos_in_order_and_hands_back_the_cursor(net): + net["handler"] = _serve() + page = search_scrape.search("cats") + assert page["next_page_token"] == "NEXT" + assert [i["id"] for i in page["items"]] == ["v1", "v2"] + assert page["items"][0] == { + "id": "v1", + "channel_id": "UCchan", + "channel_title": "Chan", + "title": "First video", + "published_at": None, + "thumbnail_url": "big.jpg", + "live_broadcast": "none", + } + assert page["items"][1]["title"] == "Second" + post = net["calls"][-1] + assert post.url.params["key"] == "live-key" + body = json.loads(post.content) + assert body["query"] == "cats" and body["params"] == search_scrape._TYPE_VIDEO + assert body["context"]["client"]["clientVersion"] == "2.20261001" + + +def test_the_next_page_sends_the_cursor_not_the_query_and_the_end_has_none(net): + net["handler"] = _serve(search={"onResponseReceivedCommands": [_video("v9", "Last")]}) + page = search_scrape.search("cats", continuation="NEXT") + body = json.loads(net["calls"][-1].content) + assert body["continuation"] == "NEXT" and "query" not in body + assert [i["id"] for i in page["items"]] == ["v9"] + assert page["next_page_token"] is None + + +@pytest.mark.parametrize( + ("search", "status"), + [ + ("consent", 200), + ({"error": "blocked"}, 429), + ({"responseContext": {}, "alerts": ["unusual traffic"]}, 200), + ([], 200), + ], + ids=["not-json", "http-error", "unknown-shape", "not-an-object"], +) +def test_a_blocked_or_garbled_answer_is_an_error_not_an_empty_page(net, search, status): + net["handler"] = _serve(search=search, status=status) + with pytest.raises(search_scrape.ScrapeError): + search_scrape.search("cats") + + +def test_a_search_with_no_results_is_an_empty_page_not_an_error(net): + net["handler"] = _serve(search={"contents": {"sectionListRenderer": {"contents": []}}}) + assert search_scrape.search("zzzzqqq") == {"items": [], "next_page_token": None} + + +def test_the_live_key_is_fetched_once_and_cached(net): + net["handler"] = _serve() + search_scrape.search("a") + search_scrape.search("b") + assert [c.url.path for c in net["calls"]].count("/results") == 1 + + +@pytest.mark.parametrize( + "results_html", + [httpx.ConnectError("down"), "a consent page, no keys"], + ids=["fetch-failed", "no-key-on-page"], +) +def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, results_html): + """The fallback pair used to be CACHED for the life of the process: one failed fetch pinned + every later search to a hard-coded key, and once that went stale search returned nothing, + with no error anywhere (E32 S2).""" + net["handler"] = _serve(results_html=results_html) + search_scrape.search("a") + assert net["calls"][-1].url.params["key"] == search_scrape._FALLBACK_KEY + + net["handler"] = _serve() + search_scrape.search("b") + assert net["calls"][-1].url.params["key"] == "live-key" diff --git a/backend/tests/test_shorts_probe.py b/backend/tests/test_shorts_probe.py new file mode 100644 index 00000000..a05cc7b1 --- /dev/null +++ b/backend/tests/test_shorts_probe.py @@ -0,0 +1,44 @@ +"""youtube/shorts.probe_is_short — three answers, and the third is not the second (E32 S2: 33% covered). + +A Short answers 200 at /shorts/; a regular video redirects to /watch. Anything else — an error +page, a network failure — is "undetermined, ask again", never "not a Short": that would file a Short +in the normal feed for good. +""" + +import httpx +import pytest + +from app.youtube import shorts + + +def _client(respond) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(respond)) + + +@pytest.mark.parametrize( + ("status", "expected"), + [(200, True), (301, False), (302, False), (303, False), (307, False), (308, False)] + + [(404, None), (429, None), (500, None)], +) +def test_the_status_decides_short_regular_or_undetermined(status, expected): + seen: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(status, headers={"location": "https://www.youtube.com/watch?v=x"}) + + assert shorts.probe_is_short(_client(respond), "abc123") is expected + assert str(seen[0].url) == "https://www.youtube.com/shorts/abc123" + assert len(seen) == 1 # a redirect is the answer, never followed + + +def test_a_network_failure_is_undetermined_not_regular(): + def respond(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectTimeout("slow", request=request) + + assert shorts.probe_is_short(_client(respond), "abc123") is None + + +def test_the_client_skips_the_consent_interstitial(): + with shorts.make_client() as c: + assert "SOCS=" in c.headers["cookie"]