diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index 75131d2e..ac96fdfa 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -136,6 +136,10 @@ def _ingest_candidates( # 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO # NOTHING), so an organically-synced video stays organic. + # published_at is NOT NULL (migration 0068); the scrape source exposes no publish date + # (only a relative "3 years ago"), so fall back to now() to satisfy the constraint — the + # enrich in step 3 overwrites it with the video's real snippet date. + now = datetime.now(timezone.utc) insert_stubs( db, [ @@ -143,7 +147,7 @@ def _ingest_candidates( "id": it["id"], "channel_id": it["channel_id"], "title": it["title"], - "published_at": parse_dt(it["published_at"]), + "published_at": parse_dt(it["published_at"]) or now, "thumbnail_url": it["thumbnail_url"], "via_search": True, } @@ -158,7 +162,6 @@ def _ingest_candidates( detail_by_id = {it["id"]: it for it in yt.get_videos(ids)} except YouTubeError: detail_by_id = {} - now = datetime.now(timezone.utc) for v in videos: item = detail_by_id.get(v.id) if item is None: diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index a36417b9..e4653546 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -245,9 +245,15 @@ def apply_video_details(video: Video, item: dict) -> None: video.original_title = raw_title video.title = normalize_title(raw_title) video.description = snippet.get("description") - if not video.published_at: - # NOT NULL (0068): a fresh insert with no date yet falls back to now() (enrich normally has one). - video.published_at = parse_dt(snippet.get("publishedAt")) or _now() + # published_at is NOT NULL (0068). Prefer the enrich snippet's real date whenever it has one — + # this also corrects a now() placeholder written at stub time (e.g. a scrape-search stub, which + # carries no publish date). Only fall back to now() when the row still has nothing at all; never + # clobber an existing real date with now() (a re-enrich whose snippet lacks a date). + snippet_published = parse_dt(snippet.get("publishedAt")) + if snippet_published: + video.published_at = snippet_published + elif not video.published_at: + video.published_at = _now() video.thumbnail_url = best_thumbnail(snippet.get("thumbnails")) or video.thumbnail_url video.duration_seconds = parse_iso8601_duration(content.get("duration")) video.view_count = int(stats["viewCount"]) if stats.get("viewCount") else None diff --git a/backend/tests/test_search_scrape_ingest.py b/backend/tests/test_search_scrape_ingest.py new file mode 100644 index 00000000..0f754f05 --- /dev/null +++ b/backend/tests/test_search_scrape_ingest.py @@ -0,0 +1,95 @@ +"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date. + +`videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a +relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that +None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error +is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request +500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken. + +The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the +video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and +a placeholder never becomes the video's permanent date. +""" +from datetime import datetime, timezone + +from app import sysconfig +from app.models import User, Video +from app.routes.search import _ingest_candidates + + +class _FakeYT: + """Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives); + video enrich returns a real publishedAt plus a long duration so Shorts classification finalises + without the youtube.com/shorts probe (no network in the DB lane).""" + + def __init__(self, enrich: dict[str, dict]): + self._enrich = enrich + + def get_channels(self, ids): + return [] + + def get_videos(self, ids): + return [self._enrich[i] for i in ids if i in self._enrich] + + +def _candidate(vid: str, ch: str) -> dict: + # The scrape shape (see search_scrape._video_item): published_at is ALWAYS None. + return { + "id": vid, + "channel_id": ch, + "channel_title": f"Chan {ch}", + "title": f"Title {vid}", + "published_at": None, + "thumbnail_url": None, + "live_broadcast": "none", + } + + +def _enrich_item(vid: str, published_iso: str) -> dict: + # PT30M keeps the video well above shorts_probe_max_seconds so no probe fires. + return { + "id": vid, + "snippet": {"title": f"Title {vid}", "publishedAt": published_iso}, + "contentDetails": {"duration": "PT30M"}, + "statistics": {}, + "status": {}, + } + + +def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db): + # Small cap so the 1800s enrich duration keeps the shorts step offline. + sysconfig.set_value(db, "shorts_probe_max_seconds", 60) + user = User(email="searcher@example.com") + db.add(user) + db.commit() + + candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")] + enrich = { + "vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"), + "vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"), + } + + kept = _ingest_candidates(db, _FakeYT(enrich), user, candidates) + + # No IntegrityError, and both non-Short/non-live videos are kept in input order. + assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"] + a = db.get(Video, "vid_aaaaaaa") + # The now() placeholder was replaced by the video's real snippet date, not left as "now". + assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc) + assert a.via_search is True + assert a.enriched_at is not None + + +def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db): + # If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL + # constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None). + user = User(email="searcher2@example.com") + db.add(user) + db.commit() + + kept = _ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")]) + + assert kept == [] + c = db.get(Video, "vid_ccccccc") + assert c.published_at is not None # placeholder wrote a real value → no crash + assert c.enriched_at is None