siftlode/backend/tests/test_search_scrape_ingest.py

"""Regression: live YouTube search ingest must survive the scrape source carrying no publish date.

`videos.published_at` is NOT NULL (migration 0068, PERF-15). The scrape source exposes only a
relative time ("3 years ago"), so its stub has `published_at=None`; the ingest used to pass that
None straight through, which the NOT NULL constraint rejected with an IntegrityError. That error
is raised OUTSIDE the route's page-fetch try/except, so the whole `/api/search/youtube` request
500'd and the UI showed the generic "YouTube search failed." — every scrape search was broken.

The stub now falls back to now() (a non-null placeholder), and the enrich overwrites it with the
video's real snippet date. These DB-lane tests lock both halves: no crash on a date-less stub, and
a placeholder never becomes the video's permanent date.
"""

from datetime import datetime, timezone

import pytest

from app import sysconfig
from app.models import User
from app.catalog.models import Video
from app.catalog.search import ingest_candidates
from app.youtube.client import YouTubeError


class _FakeYT:
    """Offline stand-in for YouTubeClient. Channel enrich is a no-op (the from_search stub survives);
    video enrich returns a real publishedAt plus a long duration so Shorts classification finalises
    without the youtube.com/shorts probe (no network in the DB lane)."""

    def __init__(self, enrich: dict[str, dict]):
        self._enrich = enrich

    def get_channels(self, ids):
        return []

    def get_videos(self, ids):
        return [self._enrich[i] for i in ids if i in self._enrich]


def _candidate(vid: str, ch: str) -> dict:
    # The scrape shape (see search_scrape._video_item): published_at is ALWAYS None.
    return {
        "id": vid,
        "channel_id": ch,
        "channel_title": f"Chan {ch}",
        "title": f"Title {vid}",
        "published_at": None,
        "thumbnail_url": None,
        "live_broadcast": "none",
    }


def _enrich_item(vid: str, published_iso: str) -> dict:
    # PT30M keeps the video well above shorts_probe_max_seconds so no probe fires.
    return {
        "id": vid,
        "snippet": {"title": f"Title {vid}", "publishedAt": published_iso},
        "contentDetails": {"duration": "PT30M"},
        "statistics": {},
        "status": {},
    }


def test_scrape_ingest_dateless_stub_ingests_and_takes_real_enrich_date(db):
    # Small cap so the 1800s enrich duration keeps the shorts step offline.
    sysconfig.set_value(db, "shorts_probe_max_seconds", 60)
    user = User(email="searcher@example.com")
    db.add(user)
    db.commit()

    candidates = [_candidate("vid_aaaaaaa", "UC_chan_a"), _candidate("vid_bbbbbbb", "UC_chan_a")]
    enrich = {
        "vid_aaaaaaa": _enrich_item("vid_aaaaaaa", "2021-02-12T19:00:07Z"),
        "vid_bbbbbbb": _enrich_item("vid_bbbbbbb", "2023-10-30T17:00:08Z"),
    }

    kept = ingest_candidates(db, _FakeYT(enrich), user, candidates)

    # No IntegrityError, and both non-Short/non-live videos are kept in input order.
    assert kept == ["vid_aaaaaaa", "vid_bbbbbbb"]
    a = db.get(Video, "vid_aaaaaaa")
    # The now() placeholder was replaced by the video's real snippet date, not left as "now".
    assert a.published_at == datetime(2021, 2, 12, 19, 0, 7, tzinfo=timezone.utc)
    assert a.via_search is True
    assert a.enriched_at is not None


def test_scrape_ingest_enrich_miss_keeps_nonnull_placeholder(db):
    # If enrich yields nothing (proxy blip / omitted row), the stub must STILL satisfy the NOT NULL
    # constraint (a non-null placeholder) — it simply isn't kept (enriched_at stays None).
    user = User(email="searcher2@example.com")
    db.add(user)
    db.commit()

    kept = ingest_candidates(db, _FakeYT({}), user, [_candidate("vid_ccccccc", "UC_chan_c")])

    assert kept == []
    c = db.get(Video, "vid_ccccccc")
    assert c.published_at is not None  # placeholder wrote a real value → no crash
    assert c.enriched_at is None


class _FailingYT(_FakeYT):
    def get_videos(self, ids):
        raise YouTubeError("videos -> 403", reason="quotaExceeded", path="videos")


def test_a_videos_list_failure_raises_and_keeps_the_stubs(db):
    # Swallowed, it answered as a page with nothing on it — "no results" for a search whose lookup
    # never ran. The stubs stay: the FK targets, backfilled later.
    user = User(email="searcher3@example.com")
    db.add(user)
    db.commit()

    with pytest.raises(YouTubeError):
        ingest_candidates(db, _FailingYT({}), user, [_candidate("vid_ddddddd", "UC_chan_d")])

    db.rollback()  # what the caller does: the stubs were committed before the lookup
    assert db.get(Video, "vid_ddddddd") is not None


def test_search_youtube_route_returns_422_when_ingest_fails(db, monkeypatch, override_dep):
    # Route-level guard (B-core-1): an ingest failure with nothing yet collected must surface as a 422
    # + message on this JSON endpoint (the FE hides 5xx behind a generic "connection lost" + auto-retry,
    # burning a second quota-costed search), and the session is rolled back. Here the page fetch succeeds
    # (one candidate) and ingest_candidates raises → the endpoint returns 422.
    from fastapi.testclient import TestClient

    import app.state as state
    from app.auth import current_user
    from app.db import get_db
    from app.main import app
    from app.routes import search as search_mod

    user = User(email="route422@example.com")
    db.add(user)
    db.commit()

    monkeypatch.setattr(state, "_configured_cache", True)
    monkeypatch.setattr(
        search_mod.sysconfig, "get_str", lambda d, k: "scrape" if k == "search_source" else ""
    )
    monkeypatch.setattr(search_mod.sysconfig, "get_int", lambda d, k: 100)  # daily cap not reached
    monkeypatch.setattr(search_mod.quota, "actions_today", lambda d, uid, action: 0)

    class _FakeYTCM:
        def __init__(self, *a, **k):
            pass

        def __enter__(self):
            return self

        def __exit__(self, *a):
            return False

    monkeypatch.setattr(search_mod, "YouTubeClient", _FakeYTCM)
    monkeypatch.setattr(
        search_mod.search_scrape,
        "search",
        lambda term, continuation=None: {
            "items": [_candidate("vid_route12", "UC_route_a")],
            "next_page_token": None,
        },
    )

    def _boom(db_, yt, user_, candidates):
        raise RuntimeError("ingest exploded")

    monkeypatch.setattr(search_mod.catalog_search, "ingest_candidates", _boom)

    override_dep(get_db, db.info["request"])
    override_dep(current_user, lambda: user)
    resp = TestClient(app).get("/api/search/youtube?q=test+term")

    assert resp.status_code == 422, resp.text
    assert resp.json()["detail"] == {"code": "youtube", "reason": "unavailable"}