fix(youtube): the scrape search never pins a stale key, and an unknown shape is an error

A failed config fetch or a keyless consent page cached the fallback InnerTube key for the life
of the process; once it went stale, search returned nothing with no error. The fallback is now
used for that call only. A 200 JSON without a result frame returned an empty page that read as
no results; it is a ScrapeError now. Parser, cursor and Shorts-probe tests added.

backend/app/youtube/search_scrape.py

@@ -42,6 +42,9 @@ _HEADERS = {
     "Accept-Language": "en-US,en;q=0.9",
 }
 
+# The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands)
+# put their result list under.
+_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"}
 _KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
 _VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
 
@@ -65,29 +68,36 @@ def _runs_text(node: dict | None) -> str | None:
 
 
 def _get_config(client: httpx.Client) -> tuple[str, str]:
-    """The current InnerTube (api_key, client_version), cached after the first fetch."""
+    """The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
+
+    The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
+    keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
+    the life of the process — and when those went stale, search returned nothing, with no error
+    anywhere (E32 S2). Now the next search simply tries the live pair again."""
     global _config
     if _config is not None:
         return _config
     try:
-        resp = client.get(_RESULTS_URL, params={"search_query": "music"})
-        html = resp.text
-        key = _KEY_RE.search(html)
-        ver = _VER_RE.search(html)
-        _config = (
-            key.group(1) if key else _FALLBACK_KEY,
-            ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
-        )
+        html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
     except httpx.HTTPError as exc:
-        # The fallback pair is CACHED in `_config` for the life of the process, so one failed
-        # fetch at the wrong moment pins every later search to a hard-coded key and version —
-        # and when those go stale, search returns nothing at all, with no error anywhere.
         log.warning(
             "the InnerTube config could not be fetched; falling back to the pinned key",
             extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
         )
-        _config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
-    return _config
+        return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
+    key = _KEY_RE.search(html)
+    ver = _VER_RE.search(html)
+    if key and ver:
+        _config = (key.group(1), ver.group(1))
+        return _config
+    log.warning(
+        "the results page carried no InnerTube config; falling back to the pinned key",
+        extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
+    )
+    return (
+        key.group(1) if key else _FALLBACK_KEY,
+        ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
+    )
 
 
 def _video_item(vr: dict) -> dict | None:
@@ -182,6 +192,10 @@ def search(q: str, continuation: str | None = None) -> dict:
         except ValueError as exc:  # non-JSON body (blocked / consent / captcha)
             raise ScrapeError("InnerTube returned a non-JSON response") from exc
 
+    # Even a search with no results carries its result list's frame. Without one this is not "no
+    # results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so.
+    if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys():
+        raise ScrapeError("InnerTube returned an unexpected shape")
     items: list[dict] = []
     _collect_videos(data, items, set())
     return {"items": items, "next_page_token": _find_continuation(data)}