"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date. `videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request 500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken. The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and a placeholder never becomes the video's permanent date. """ from datetime import datetime, timezone from app import sysconfig from app.models import User, Video from app.routes.search import _ingest_candidates class _FakeYT: """Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives); video enrich returns a real publishedAt plus a long duration so Shorts classification finalises without the youtube.com/shorts probe (no network in the DB lane).""" def __init__(self, enrich: dict[str, dict]): self._enrich = enrich def get_channels(self, ids): return [] def get_videos(self, ids): return [self._enrich[i] for i in ids if i in self._enrich] def _candidate(vid: str, ch: str) -> dict: # The scrape shape (see search_scrape._video_item): published_at is ALWAYS None. return { "id": vid, "channel_id": ch, "channel_title": f"Chan {ch}", "title": f"Title {vid}", "published_at": None, "thumbnail_url": None, "live_broadcast": "none", } def _enrich_item(vid: str, published_iso: str) -> dict: # PT30M keeps the video well above shorts_probe_max_seconds so no probe fires. return { "id": vid, "snippet": {"title": f"Title {vid}", "publishedAt": published_iso}, "contentDetails": {"duration": "PT30M"}, "statistics": {}, "status": {}, } def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db): # Small cap so the 1800s enrich duration keeps the shorts step offline. sysconfig.set_value(db, "shorts_probe_max_seconds", 60) user = User(email="searcher@example.com") db.add(user) db.commit() candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")] enrich = { "vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"), "vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"), } kept = _ingest_candidates(db, _FakeYT(enrich), user, candidates) # No IntegrityError, and both non-Short/non-live videos are kept in input order. assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"] a = db.get(Video, "vid_aaaaaaa") # The now() placeholder was replaced by the video's real snippet date, not left as "now". assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc) assert a.via_search is True assert a.enriched_at is not None def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db): # If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL # constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None). user = User(email="searcher2@example.com") db.add(user) db.commit() kept = _ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")]) assert kept == [] c = db.get(Video, "vid_ccccccc") assert c.published_at is not None # placeholder wrote a real value → no crash assert c.enriched_at is None