fix(youtube): the scrape search never pins a stale key, and an unknown shape is an error
A failed config fetch or a keyless consent page cached the fallback InnerTube key for the life of the process; once it went stale, search returned nothing with no error. The fallback is now used for that call only. A 200 JSON without a result frame returned an empty page that read as no results; it is a ScrapeError now. Parser, cursor and Shorts-probe tests added.
This commit is contained in:
@@ -42,6 +42,9 @@ _HEADERS = {
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
# The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands)
|
||||
# put their result list under.
|
||||
_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"}
|
||||
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
||||
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
||||
|
||||
@@ -65,29 +68,36 @@ def _runs_text(node: dict | None) -> str | None:
|
||||
|
||||
|
||||
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
||||
"""The current InnerTube (api_key, client_version), cached after the first fetch."""
|
||||
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
|
||||
|
||||
The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
|
||||
keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
|
||||
the life of the process — and when those went stale, search returned nothing, with no error
|
||||
anywhere (E32 S2). Now the next search simply tries the live pair again."""
|
||||
global _config
|
||||
if _config is not None:
|
||||
return _config
|
||||
try:
|
||||
resp = client.get(_RESULTS_URL, params={"search_query": "music"})
|
||||
html = resp.text
|
||||
key = _KEY_RE.search(html)
|
||||
ver = _VER_RE.search(html)
|
||||
_config = (
|
||||
key.group(1) if key else _FALLBACK_KEY,
|
||||
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
||||
)
|
||||
html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
|
||||
except httpx.HTTPError as exc:
|
||||
# The fallback pair is CACHED in `_config` for the life of the process, so one failed
|
||||
# fetch at the wrong moment pins every later search to a hard-coded key and version —
|
||||
# and when those go stale, search returns nothing at all, with no error anywhere.
|
||||
log.warning(
|
||||
"the InnerTube config could not be fetched; falling back to the pinned key",
|
||||
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
|
||||
)
|
||||
_config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
||||
return _config
|
||||
return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
||||
key = _KEY_RE.search(html)
|
||||
ver = _VER_RE.search(html)
|
||||
if key and ver:
|
||||
_config = (key.group(1), ver.group(1))
|
||||
return _config
|
||||
log.warning(
|
||||
"the results page carried no InnerTube config; falling back to the pinned key",
|
||||
extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
|
||||
)
|
||||
return (
|
||||
key.group(1) if key else _FALLBACK_KEY,
|
||||
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
||||
)
|
||||
|
||||
|
||||
def _video_item(vr: dict) -> dict | None:
|
||||
@@ -182,6 +192,10 @@ def search(q: str, continuation: str | None = None) -> dict:
|
||||
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
||||
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
||||
|
||||
# Even a search with no results carries its result list's frame. Without one this is not "no
|
||||
# results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so.
|
||||
if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys():
|
||||
raise ScrapeError("InnerTube returned an unexpected shape")
|
||||
items: list[dict] = []
|
||||
_collect_videos(data, items, set())
|
||||
return {"items": items, "next_page_token": _find_continuation(data)}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""youtube/search_scrape — the zero-quota InnerTube search (E32 S2: the module was 20% covered).
|
||||
|
||||
Everything here breaks silently when YouTube moves: a parser that stops finding videos, a cursor that
|
||||
stops paging, a fallback key that outlives its usefulness. The network is an httpx MockTransport.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.youtube import search_scrape
|
||||
|
||||
RESULTS_HTML = (
|
||||
'<script>{"INNERTUBE_API_KEY":"live-key","INNERTUBE_CLIENT_VERSION":"2.20261001"}</script>'
|
||||
)
|
||||
|
||||
|
||||
def _video(vid: str, title: str, *, simple: bool = False) -> dict:
|
||||
return {
|
||||
"videoRenderer": {
|
||||
"videoId": vid,
|
||||
"title": {"simpleText": title}
|
||||
if simple
|
||||
else {"runs": [{"text": title[:3]}, {"text": title[3:]}]},
|
||||
"ownerText": {
|
||||
"runs": [
|
||||
{
|
||||
"text": "Chan",
|
||||
"navigationEndpoint": {"browseEndpoint": {"browseId": "UCchan"}},
|
||||
}
|
||||
]
|
||||
},
|
||||
"thumbnail": {"thumbnails": [{"url": "small.jpg"}, {"url": "big.jpg"}]},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
PAGE = {
|
||||
"contents": {
|
||||
"sectionListRenderer": {
|
||||
"contents": [
|
||||
{
|
||||
"itemSectionRenderer": {
|
||||
"contents": [
|
||||
_video("v1", "First video"),
|
||||
{"channelRenderer": {"channelId": "UCx"}}, # not a video: skipped
|
||||
{
|
||||
"reelShelfRenderer": {
|
||||
"items": [{"reelItemRenderer": {"videoId": "s1"}}]
|
||||
}
|
||||
},
|
||||
_video("v2", "Second", simple=True),
|
||||
_video("v1", "First video"), # a repeat in another shelf: once
|
||||
{"videoRenderer": {"title": {"simpleText": "no id"}}},
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"continuationItemRenderer": {
|
||||
"continuationEndpoint": {"continuationCommand": {"token": "NEXT"}}
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def net(monkeypatch):
|
||||
"""Route every httpx.Client the module opens through a handler the test sets."""
|
||||
calls: list[httpx.Request] = []
|
||||
state: dict = {"handler": None}
|
||||
|
||||
def transport(request: httpx.Request) -> httpx.Response:
|
||||
calls.append(request)
|
||||
return state["handler"](request)
|
||||
|
||||
real = httpx.Client
|
||||
monkeypatch.setattr(
|
||||
search_scrape.httpx,
|
||||
"Client",
|
||||
lambda **kw: real(transport=httpx.MockTransport(transport), **kw),
|
||||
)
|
||||
monkeypatch.setattr(search_scrape, "_config", None)
|
||||
state["calls"] = calls
|
||||
return state
|
||||
|
||||
|
||||
def _serve(results_html=RESULTS_HTML, search=PAGE, status=200):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/results":
|
||||
if isinstance(results_html, Exception):
|
||||
raise results_html
|
||||
return httpx.Response(200, text=results_html)
|
||||
if isinstance(search, str):
|
||||
return httpx.Response(status, text=search)
|
||||
return httpx.Response(status, json=search)
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_page_flattens_its_videos_in_order_and_hands_back_the_cursor(net):
|
||||
net["handler"] = _serve()
|
||||
page = search_scrape.search("cats")
|
||||
assert page["next_page_token"] == "NEXT"
|
||||
assert [i["id"] for i in page["items"]] == ["v1", "v2"]
|
||||
assert page["items"][0] == {
|
||||
"id": "v1",
|
||||
"channel_id": "UCchan",
|
||||
"channel_title": "Chan",
|
||||
"title": "First video",
|
||||
"published_at": None,
|
||||
"thumbnail_url": "big.jpg",
|
||||
"live_broadcast": "none",
|
||||
}
|
||||
assert page["items"][1]["title"] == "Second"
|
||||
post = net["calls"][-1]
|
||||
assert post.url.params["key"] == "live-key"
|
||||
body = json.loads(post.content)
|
||||
assert body["query"] == "cats" and body["params"] == search_scrape._TYPE_VIDEO
|
||||
assert body["context"]["client"]["clientVersion"] == "2.20261001"
|
||||
|
||||
|
||||
def test_the_next_page_sends_the_cursor_not_the_query_and_the_end_has_none(net):
|
||||
net["handler"] = _serve(search={"onResponseReceivedCommands": [_video("v9", "Last")]})
|
||||
page = search_scrape.search("cats", continuation="NEXT")
|
||||
body = json.loads(net["calls"][-1].content)
|
||||
assert body["continuation"] == "NEXT" and "query" not in body
|
||||
assert [i["id"] for i in page["items"]] == ["v9"]
|
||||
assert page["next_page_token"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("search", "status"),
|
||||
[
|
||||
("<html>consent</html>", 200),
|
||||
({"error": "blocked"}, 429),
|
||||
({"responseContext": {}, "alerts": ["unusual traffic"]}, 200),
|
||||
([], 200),
|
||||
],
|
||||
ids=["not-json", "http-error", "unknown-shape", "not-an-object"],
|
||||
)
|
||||
def test_a_blocked_or_garbled_answer_is_an_error_not_an_empty_page(net, search, status):
|
||||
net["handler"] = _serve(search=search, status=status)
|
||||
with pytest.raises(search_scrape.ScrapeError):
|
||||
search_scrape.search("cats")
|
||||
|
||||
|
||||
def test_a_search_with_no_results_is_an_empty_page_not_an_error(net):
|
||||
net["handler"] = _serve(search={"contents": {"sectionListRenderer": {"contents": []}}})
|
||||
assert search_scrape.search("zzzzqqq") == {"items": [], "next_page_token": None}
|
||||
|
||||
|
||||
def test_the_live_key_is_fetched_once_and_cached(net):
|
||||
net["handler"] = _serve()
|
||||
search_scrape.search("a")
|
||||
search_scrape.search("b")
|
||||
assert [c.url.path for c in net["calls"]].count("/results") == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"results_html",
|
||||
[httpx.ConnectError("down"), "<html>a consent page, no keys</html>"],
|
||||
ids=["fetch-failed", "no-key-on-page"],
|
||||
)
|
||||
def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, results_html):
|
||||
"""The fallback pair used to be CACHED for the life of the process: one failed fetch pinned
|
||||
every later search to a hard-coded key, and once that went stale search returned nothing,
|
||||
with no error anywhere (E32 S2)."""
|
||||
net["handler"] = _serve(results_html=results_html)
|
||||
search_scrape.search("a")
|
||||
assert net["calls"][-1].url.params["key"] == search_scrape._FALLBACK_KEY
|
||||
|
||||
net["handler"] = _serve()
|
||||
search_scrape.search("b")
|
||||
assert net["calls"][-1].url.params["key"] == "live-key"
|
||||
@@ -0,0 +1,44 @@
|
||||
"""youtube/shorts.probe_is_short — three answers, and the third is not the second (E32 S2: 33% covered).
|
||||
|
||||
A Short answers 200 at /shorts/<id>; a regular video redirects to /watch. Anything else — an error
|
||||
page, a network failure — is "undetermined, ask again", never "not a Short": that would file a Short
|
||||
in the normal feed for good.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.youtube import shorts
|
||||
|
||||
|
||||
def _client(respond) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(respond))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "expected"),
|
||||
[(200, True), (301, False), (302, False), (303, False), (307, False), (308, False)]
|
||||
+ [(404, None), (429, None), (500, None)],
|
||||
)
|
||||
def test_the_status_decides_short_regular_or_undetermined(status, expected):
|
||||
seen: list[httpx.Request] = []
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
return httpx.Response(status, headers={"location": "https://www.youtube.com/watch?v=x"})
|
||||
|
||||
assert shorts.probe_is_short(_client(respond), "abc123") is expected
|
||||
assert str(seen[0].url) == "https://www.youtube.com/shorts/abc123"
|
||||
assert len(seen) == 1 # a redirect is the answer, never followed
|
||||
|
||||
|
||||
def test_a_network_failure_is_undetermined_not_regular():
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectTimeout("slow", request=request)
|
||||
|
||||
assert shorts.probe_is_short(_client(respond), "abc123") is None
|
||||
|
||||
|
||||
def test_the_client_skips_the_consent_interstitial():
|
||||
with shorts.make_client() as c:
|
||||
assert "SOCS=" in c.headers["cookie"]
|
||||
Reference in New Issue
Block a user