peter
ff45db70ae
The scrape source exposes no publish date, so its search stub carried
published_at=None. Migration 0068 (PERF-15, E2/0.68.0) made
videos.published_at NOT NULL, so insert_stubs raised IntegrityError —
outside the route's page-fetch try/except — and the whole
/api/search/youtube request 500'd ("YouTube search failed."). Every
scrape search (the default source) had been broken since 0.68.0.
- routes/search._ingest_candidates: stub published_at falls back to
now() (mirrors stub_from_playlist_item), satisfying the constraint.
- sync.videos.apply_video_details: prefer the enrich snippet's real
date whenever present, so the now() placeholder is corrected instead
of becoming the video's permanent date (the guard only filled a NULL,
which a non-null placeholder no longer is). Never clobbers a real
date with now().
Regression tests (DB lane): dateless scrape stub ingests and takes the
real enrich date; an enrich miss keeps a non-null placeholder, unkept.
96 lines
3.8 KiB
Python
96 lines
3.8 KiB
Python
"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date.
|
|
|
|
`videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a
|
|
relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that
|
|
None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error
|
|
is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request
|
|
500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken.
|
|
|
|
The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the
|
|
video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and
|
|
a placeholder never becomes the video's permanent date.
|
|
"""
|
|
from datetime import datetime, timezone
|
|
|
|
from app import sysconfig
|
|
from app.models import User, Video
|
|
from app.routes.search import _ingest_candidates
|
|
|
|
|
|
class _FakeYT:
|
|
"""Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives);
|
|
video enrich returns a real publishedAt plus a long duration so Shorts classification finalises
|
|
without the youtube.com/shorts probe (no network in the DB lane)."""
|
|
|
|
def __init__(self, enrich: dict[str, dict]):
|
|
self._enrich = enrich
|
|
|
|
def get_channels(self, ids):
|
|
return []
|
|
|
|
def get_videos(self, ids):
|
|
return [self._enrich[i] for i in ids if i in self._enrich]
|
|
|
|
|
|
def _candidate(vid: str, ch: str) -> dict:
|
|
# The scrape shape (see search_scrape._video_item): published_at is ALWAYS None.
|
|
return {
|
|
"id": vid,
|
|
"channel_id": ch,
|
|
"channel_title": f"Chan {ch}",
|
|
"title": f"Title {vid}",
|
|
"published_at": None,
|
|
"thumbnail_url": None,
|
|
"live_broadcast": "none",
|
|
}
|
|
|
|
|
|
def _enrich_item(vid: str, published_iso: str) -> dict:
|
|
# PT30M keeps the video well above shorts_probe_max_seconds so no probe fires.
|
|
return {
|
|
"id": vid,
|
|
"snippet": {"title": f"Title {vid}", "publishedAt": published_iso},
|
|
"contentDetails": {"duration": "PT30M"},
|
|
"statistics": {},
|
|
"status": {},
|
|
}
|
|
|
|
|
|
def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db):
|
|
# Small cap so the 1800s enrich duration keeps the shorts step offline.
|
|
sysconfig.set_value(db, "shorts_probe_max_seconds", 60)
|
|
user = User(email="searcher@example.com")
|
|
db.add(user)
|
|
db.commit()
|
|
|
|
candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")]
|
|
enrich = {
|
|
"vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"),
|
|
"vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"),
|
|
}
|
|
|
|
kept = _ingest_candidates(db, _FakeYT(enrich), user, candidates)
|
|
|
|
# No IntegrityError, and both non-Short/non-live videos are kept in input order.
|
|
assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"]
|
|
a = db.get(Video, "vid_aaaaaaa")
|
|
# The now() placeholder was replaced by the video's real snippet date, not left as "now".
|
|
assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc)
|
|
assert a.via_search is True
|
|
assert a.enriched_at is not None
|
|
|
|
|
|
def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db):
|
|
# If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL
|
|
# constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None).
|
|
user = User(email="searcher2@example.com")
|
|
db.add(user)
|
|
db.commit()
|
|
|
|
kept = _ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")])
|
|
|
|
assert kept == []
|
|
c = db.get(Video, "vid_ccccccc")
|
|
assert c.published_at is not None # placeholder wrote a real value → no crash
|
|
assert c.enriched_at is None
|