"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date. `videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request 500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken. The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and a placeholder never becomes the video's permanent date. """ from datetime import datetime, timezone from app import sysconfig from app.models import User, Video from app.routes.search import _ingest_candidates class _FakeYT: """Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives); video enrich returns a real publishedAt plus a long duration so Shorts classification finalises without the youtube.com/shorts probe (no network in the DB lane).""" def __init__(self, enrich: dict[str, dict]): self._enrich = enrich def get_channels(self, ids): return [] def get_videos(self, ids): return [self._enrich[i] for i in ids if i in self._enrich] def _candidate(vid: str, ch: str) -> dict: # The scrape shape (see search_scrape._video_item): published_at is ALWAYS None. return { "id": vid, "channel_id": ch, "channel_title": f"Chan {ch}", "title": f"Title {vid}", "published_at": None, "thumbnail_url": None, "live_broadcast": "none", } def _enrich_item(vid: str, published_iso: str) -> dict: # PT30M keeps the video well above shorts_probe_max_seconds so no probe fires. return { "id": vid, "snippet": {"title": f"Title {vid}", "publishedAt": published_iso}, "contentDetails": {"duration": "PT30M"}, "statistics": {}, "status": {}, } def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db): # Small cap so the 1800s enrich duration keeps the shorts step offline. sysconfig.set_value(db, "shorts_probe_max_seconds", 60) user = User(email="searcher@example.com") db.add(user) db.commit() candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")] enrich = { "vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"), "vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"), } kept = _ingest_candidates(db, _FakeYT(enrich), user, candidates) # No IntegrityError, and both non-Short/non-live videos are kept in input order. assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"] a = db.get(Video, "vid_aaaaaaa") # The now() placeholder was replaced by the video's real snippet date, not left as "now". assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc) assert a.via_search is True assert a.enriched_at is not None def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db): # If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL # constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None). user = User(email="searcher2@example.com") db.add(user) db.commit() kept = _ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")]) assert kept == [] c = db.get(Video, "vid_ccccccc") assert c.published_at is not None # placeholder wrote a real value → no crash assert c.enriched_at is None def test_search_youtube_route_returns_422_when_ingest_fails(db, monkeypatch): # Route-level guard (B-core-1): an ingest failure with nothing yet collected must surface as a 422 # + message on this JSON endpoint (the FE hides 5xx behind a generic "connection lost" + auto-retry, # burning a second quota-costed search), and the session is rolled back. Here the page fetch succeeds # (one candidate) and _ingest_candidates raises → the endpoint returns 422. from fastapi.testclient import TestClient import app.state as state from app.auth import require_human from app.db import get_db from app.main import app from app.routes import search as search_mod user = User(email="route422@example.com") db.add(user) db.commit() monkeypatch.setattr(state, "_configured_cache", True) monkeypatch.setattr( search_mod.sysconfig, "get_str", lambda d, k: "scrape" if k == "search_source" else "" ) monkeypatch.setattr(search_mod.sysconfig, "get_int", lambda d, k: 100) # daily cap not reached monkeypatch.setattr(search_mod.quota, "actions_today", lambda d, uid, action: 0) class _FakeYTCM: def __init__(self, *a, **k): pass def __enter__(self): return self def __exit__(self, *a): return False monkeypatch.setattr(search_mod, "YouTubeClient", _FakeYTCM) monkeypatch.setattr( search_mod.search_scrape, "search", lambda term, continuation=None: { "items": [_candidate("vid_route12", "UC_route_a")], "next_page_token": None, }, ) def _boom(db_, yt, user_, candidates): raise RuntimeError("ingest exploded") monkeypatch.setattr(search_mod, "_ingest_candidates", _boom) app.dependency_overrides[get_db] = lambda: db app.dependency_overrides[require_human] = lambda: user try: resp = TestClient(app).get("/api/search/youtube?q=test+term") finally: app.dependency_overrides.clear() assert resp.status_code == 422, resp.text assert "YouTube search failed" in resp.json()["detail"]