"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date. `videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request 500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken. The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and a placeholder never becomes the video's permanent date. """ from datetime import datetime, timezone import pytest from app import sysconfig from app.models import User from app.catalog.models import Video from app.catalog.search import ingest_candidates from app.youtube.client import YouTubeError class _FakeYT: """Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives); video enrich returns a real publishedAt plus a long duration so Shorts classification finalises without the youtube.com/shorts probe (no network in the DB lane).""" def __init__(self, enrich: dict[str, dict]): self._enrich = enrich def get_channels(self, ids): return [] def get_videos(self, ids): return [self._enrich[i] for i in ids if i in self._enrich] def _candidate(vid: str, ch: str) -> dict: # The scrape shape (see search_scrape._video_item): published_at is ALWAYS None. return { "id": vid, "channel_id": ch, "channel_title": f"Chan {ch}", "title": f"Title {vid}", "published_at": None, "thumbnail_url": None, "live_broadcast": "none", } def _enrich_item(vid: str, published_iso: str) -> dict: # PT30M keeps the video well above shorts_probe_max_seconds so no probe fires. return { "id": vid, "snippet": {"title": f"Title {vid}", "publishedAt": published_iso}, "contentDetails": {"duration": "PT30M"}, "statistics": {}, "status": {}, } def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db): # Small cap so the 1800s enrich duration keeps the shorts step offline. sysconfig.set_value(db, "shorts_probe_max_seconds", 60) user = User(email="searcher@example.com") db.add(user) db.commit() candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")] enrich = { "vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"), "vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"), } kept = ingest_candidates(db, _FakeYT(enrich), user, candidates) # No IntegrityError, and both non-Short/non-live videos are kept in input order. assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"] a = db.get(Video, "vid_aaaaaaa") # The now() placeholder was replaced by the video's real snippet date, not left as "now". assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc) assert a.via_search is True assert a.enriched_at is not None def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db): # If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL # constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None). user = User(email="searcher2@example.com") db.add(user) db.commit() kept = ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")]) assert kept == [] c = db.get(Video, "vid_ccccccc") assert c.published_at is not None # placeholder wrote a real value → no crash assert c.enriched_at is None class _FailingYT(_FakeYT): def get_videos(self, ids): raise YouTubeError("videos -> 403", reason="quotaExceeded", path="videos") def test_a_videos_list_failure_raises_and_keeps_the_stubs(db): # Swallowed, it answered as a page with nothing on it — "no results" for a search whose lookup # never ran. The stubs stay: the FK targets, backfilled later. user = User(email="searcher3@example.com") db.add(user) db.commit() with pytest.raises(YouTubeError): ingest_candidates(db, _FailingYT({}), user, [_candidate("vid_ddddddd", "UC_chan_d")]) db.rollback() # what the caller does: the stubs were committed before the lookup assert db.get(Video, "vid_ddddddd") is not None def test_search_youtube_route_returns_422_when_ingest_fails(db, monkeypatch, override_dep): # Route-level guard (B-core-1): an ingest failure with nothing yet collected must surface as a 422 # + message on this JSON endpoint (the FE hides 5xx behind a generic "connection lost" + auto-retry, # burning a second quota-costed search), and the session is rolled back. Here the page fetch succeeds # (one candidate) and ingest_candidates raises → the endpoint returns 422. from fastapi.testclient import TestClient import app.state as state from app.auth import current_user from app.db import get_db from app.main import app from app.routes import search as search_mod user = User(email="route422@example.com") db.add(user) db.commit() monkeypatch.setattr(state, "_configured_cache", True) monkeypatch.setattr( search_mod.sysconfig, "get_str", lambda d, k: "scrape" if k == "search_source" else "" ) monkeypatch.setattr(search_mod.sysconfig, "get_int", lambda d, k: 100) # daily cap not reached monkeypatch.setattr(search_mod.quota, "actions_today", lambda d, uid, action: 0) class _FakeYTCM: def __init__(self, *a, **k): pass def __enter__(self): return self def __exit__(self, *a): return False monkeypatch.setattr(search_mod, "YouTubeClient", _FakeYTCM) monkeypatch.setattr( search_mod.search_scrape, "search", lambda term, continuation=None: { "items": [_candidate("vid_route12", "UC_route_a")], "next_page_token": None, }, ) def _boom(db_, yt, user_, candidates): raise RuntimeError("ingest exploded") monkeypatch.setattr(search_mod.catalog_search, "ingest_candidates", _boom) override_dep(get_db, db.info["request"]) override_dep(current_user, lambda: user) resp = TestClient(app).get("/api/search/youtube?q=test+term") assert resp.status_code == 422, resp.text assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"}