siftlode/backend/tests/test_search_scrape_ingest.py

"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date.

`videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a
relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that
None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error
is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request
500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken.

The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the
video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and
a placeholder never becomes the video's permanent date.
"""

from datetime import datetime, timezone

from app import sysconfig
from app.models import User
from app.catalog.models import Video
from app.catalog.search import ingest_candidates


class _FakeYT:
    """Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives);
    video enrich returns a real publishedAt plus a long duration so Shorts classification finalises
    without the youtube.com/shorts probe (no network in the DB lane)."""

    def __init__(self, enrich: dict[str, dict]):
        self._enrich = enrich

    def get_channels(self, ids):
        return []

    def get_videos(self, ids):
        return [self._enrich[i] for i in ids if i in self._enrich]


def _candidate(vid: str, ch: str) -> dict:
    # The scrape shape (see search_scrape._video_item): published_at is ALWAYS None.
    return {
        "id": vid,
        "channel_id": ch,
        "channel_title": f"Chan {ch}",
        "title": f"Title {vid}",
        "published_at": None,
        "thumbnail_url": None,
        "live_broadcast": "none",
    }


def _enrich_item(vid: str, published_iso: str) -> dict:
    # PT30M keeps the video well above shorts_probe_max_seconds so no probe fires.
    return {
        "id": vid,
        "snippet": {"title": f"Title {vid}", "publishedAt": published_iso},
        "contentDetails": {"duration": "PT30M"},
        "statistics": {},
        "status": {},
    }


def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db):
    # Small cap so the 1800s enrich duration keeps the shorts step offline.
    sysconfig.set_value(db, "shorts_probe_max_seconds", 60)
    user = User(email="searcher@example.com")
    db.add(user)
    db.commit()

    candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")]
    enrich = {
        "vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"),
        "vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"),
    }

    kept = ingest_candidates(db, _FakeYT(enrich), user, candidates)

    # No IntegrityError, and both non-Short/non-live videos are kept in input order.
    assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"]
    a = db.get(Video, "vid_aaaaaaa")
    # The now() placeholder was replaced by the video's real snippet date, not left as "now".
    assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc)
    assert a.via_search is True
    assert a.enriched_at is not None


def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db):
    # If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL
    # constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None).
    user = User(email="searcher2@example.com")
    db.add(user)
    db.commit()

    kept = ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")])

    assert kept == []
    c = db.get(Video, "vid_ccccccc")
    assert c.published_at is not None  # placeholder wrote a real value → no crash
    assert c.enriched_at is None


def test_search_youtube_route_returns_422_when_ingest_fails(db, monkeypatch, override_dep):
    # Route-level guard (B-core-1): an ingest failure with nothing yet collected must surface as a 422
    # + message on this JSON endpoint (the FE hides 5xx behind a generic "connection lost" + auto-retry,
    # burning a second quota-costed search), and the session is rolled back. Here the page fetch succeeds
    # (one candidate) and ingest_candidates raises → the endpoint returns 422.
    from fastapi.testclient import TestClient

    import app.state as state
    from app.auth import current_user
    from app.db import get_db
    from app.main import app
    from app.routes import search as search_mod

    user = User(email="route422@example.com")
    db.add(user)
    db.commit()

    monkeypatch.setattr(state, "_configured_cache", True)
    monkeypatch.setattr(
        search_mod.sysconfig, "get_str", lambda d, k: "scrape" if k == "search_source" else ""
    )
    monkeypatch.setattr(search_mod.sysconfig, "get_int", lambda d, k: 100)  # daily cap not reached
    monkeypatch.setattr(search_mod.quota, "actions_today", lambda d, uid, action: 0)

    class _FakeYTCM:
        def __init__(self, *a, **k):
            pass

        def __enter__(self):
            return self

        def __exit__(self, *a):
            return False

    monkeypatch.setattr(search_mod, "YouTubeClient", _FakeYTCM)
    monkeypatch.setattr(
        search_mod.search_scrape,
        "search",
        lambda term, continuation=None: {
            "items": [_candidate("vid_route12", "UC_route_a")],
            "next_page_token": None,
        },
    )

    def _boom(db_, yt, user_, candidates):
        raise RuntimeError("ingest exploded")

    monkeypatch.setattr(search_mod.catalog_search, "ingest_candidates", _boom)

    override_dep(get_db, db.info["request"])
    override_dep(current_user, lambda: user)
    resp = TestClient(app).get("/api/search/youtube?q=test+term")

    assert resp.status_code == 422, resp.text
    assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"}