fix: live YouTube search 500 on the scrape source (published_at NOT NULL)
The scrape source exposes no publish date, so its search stub carried
published_at=None. Migration 0068 (PERF-15, E2/0.68.0) made
videos.published_at NOT NULL, so insert_stubs raised IntegrityError —
outside the route's page-fetch try/except — and the whole
/api/search/youtube request 500'd ("YouTube search failed."). Every
scrape search (the default source) had been broken since 0.68.0.
- routes/search._ingest_candidates: stub published_at falls back to
now() (mirrors stub_from_playlist_item), satisfying the constraint.
- sync.videos.apply_video_details: prefer the enrich snippet's real
date whenever present, so the now() placeholder is corrected instead
of becoming the video's permanent date (the guard only filled a NULL,
which a non-null placeholder no longer is). Never clobbers a real
date with now().
Regression tests (DB lane): dateless scrape stub ingests and takes the
real enrich date; an enrich miss keeps a non-null placeholder, unkept.
This commit is contained in:
@@ -136,6 +136,10 @@ def _ingest_candidates(
|
||||
|
||||
# 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO
|
||||
# NOTHING), so an organically-synced video stays organic.
|
||||
# published_at is NOT NULL (migration 0068); the scrape source exposes no publish date
|
||||
# (only a relative "3 years ago"), so fall back to now() to satisfy the constraint — the
|
||||
# enrich in step 3 overwrites it with the video's real snippet date.
|
||||
now = datetime.now(timezone.utc)
|
||||
insert_stubs(
|
||||
db,
|
||||
[
|
||||
@@ -143,7 +147,7 @@ def _ingest_candidates(
|
||||
"id": it["id"],
|
||||
"channel_id": it["channel_id"],
|
||||
"title": it["title"],
|
||||
"published_at": parse_dt(it["published_at"]),
|
||||
"published_at": parse_dt(it["published_at"]) or now,
|
||||
"thumbnail_url": it["thumbnail_url"],
|
||||
"via_search": True,
|
||||
}
|
||||
@@ -158,7 +162,6 @@ def _ingest_candidates(
|
||||
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
|
||||
except YouTubeError:
|
||||
detail_by_id = {}
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in videos:
|
||||
item = detail_by_id.get(v.id)
|
||||
if item is None:
|
||||
|
||||
@@ -245,9 +245,15 @@ def apply_video_details(video: Video, item: dict) -> None:
|
||||
video.original_title = raw_title
|
||||
video.title = normalize_title(raw_title)
|
||||
video.description = snippet.get("description")
|
||||
if not video.published_at:
|
||||
# NOT NULL (0068): a fresh insert with no date yet falls back to now() (enrich normally has one).
|
||||
video.published_at = parse_dt(snippet.get("publishedAt")) or _now()
|
||||
# published_at is NOT NULL (0068). Prefer the enrich snippet's real date whenever it has one —
|
||||
# this also corrects a now() placeholder written at stub time (e.g. a scrape-search stub, which
|
||||
# carries no publish date). Only fall back to now() when the row still has nothing at all; never
|
||||
# clobber an existing real date with now() (a re-enrich whose snippet lacks a date).
|
||||
snippet_published = parse_dt(snippet.get("publishedAt"))
|
||||
if snippet_published:
|
||||
video.published_at = snippet_published
|
||||
elif not video.published_at:
|
||||
video.published_at = _now()
|
||||
video.thumbnail_url = best_thumbnail(snippet.get("thumbnails")) or video.thumbnail_url
|
||||
video.duration_seconds = parse_iso8601_duration(content.get("duration"))
|
||||
video.view_count = int(stats["viewCount"]) if stats.get("viewCount") else None
|
||||
|
||||
Reference in New Issue
Block a user