siftlode/backend/app/routes/search.py

"""Live YouTube search.

Searches YouTube directly and materialises the results into the shared catalog so they render
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
pre-check. The result ids are enriched through the cheap shared videos.list path (1 unit / 50)
regardless of source.

Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
dropped from the search response up front).

A video's related videos (`/related/{video_id}`, the list YouTube shows beside it) come in the same
way: read from the website (`related_scrape`), ingested as search finds are, under their own daily
cap — with what this user has already seen left out.

This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the
catalog is `catalog/search`."""

from typing import Literal

from fastapi import APIRouter, Depends, Path
from pydantic import BaseModel
from sqlalchemy.orm import Session

from app import logs, quota, sysconfig
from app.auth import current_user
from app.catalog import queries
from app.catalog import search as catalog_search
from app.db import get_db
from app.models import User
from app.reader import queries as reader_queries
from app.youtube import related_scrape, search_scrape
from app.youtube.client import YouTubeClient, YouTubeError
from app.refusal import refused
from app.routes._common import youtube_refusal

log = logs.get_logger(__name__)
router = APIRouter(prefix="/api/search", tags=["search"])

# search.list costs this many quota units per page.
SEARCH_COST = 100

# A related list is filled to this many unseen videos, reading at most this many pages (about 20
# videos each, measured) — the ones the user has seen are left out, so a page alone can fall short.
RELATED_TARGET = 20
RELATED_MAX_PAGES = 4
# The locale YouTube fills a related list in: on the video's topic, or with what trends in Hungary.
RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")}


@router.get("/youtube")
def search_youtube(
    q: str | None = None,
    cursor: str | None = None,
    limit: int = 20,
    user: User = Depends(current_user),
    db: Session = Depends(get_db),
) -> dict:
    """Live YouTube search results materialised into the catalog. `limit` is how many results to
    gather (the count selector); with the free scrape source we page through continuations until
    that many are collected, so there's no manual "load more". The API source costs 100 units per
    page, so it returns a single page (≤50) regardless. `cursor` lets the caller fetch further."""
    term = (q or "").strip()
    if not term:
        raise refused(400, "search_term_required")
    limit = max(1, min(limit, 100))

    # Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
    source = sysconfig.get_str(db, "search_source")

    # Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
    # since the search itself spends no quota.
    daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
    used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
    if used >= daily_limit:
        # The 429 is in the request line either way; which of the two caps produced it is not, and
        # they are answered by different things — this one by tomorrow, the other by the admin.
        log.info(
            "a live search was refused: the per-user daily cap is spent",
            extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit),
        )
        raise refused(429, "search_daily_limit", limit=daily_limit)
    # The 100-unit budget pre-check only applies to the API source.
    if source == "api" and not quota.can_spend(db, SEARCH_COST):
        log.info(
            "a live search was refused: the shared YouTube budget is spent",
            extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
        )
        raise refused(429, "quota_spent")

    # Channels this user has blocked are dropped before ingestion — never shown, never stored.
    blocked = queries.blocked_ids(db, user)

    collected: list[str] = []
    next_cursor = cursor
    MAX_PAGES = 12  # scrape safety bound
    with YouTubeClient(db, user) as yt:
        for _ in range(MAX_PAGES):
            try:
                if source == "api":
                    with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
                        page = yt.search_videos(term, page_token=next_cursor)
                else:
                    page = search_scrape.search(term, continuation=next_cursor)
            except (YouTubeError, search_scrape.ScrapeError) as exc:
                if collected:
                    break  # keep what we already gathered
                # 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway
                # dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
                # which hides the real message and burns a second quota-costed search. Upstream failure
                # on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
                # A Data API failure is logged where it happened; the scraper's is not, and its text
                # no longer rides the answer — so it is said here.
                if isinstance(exc, search_scrape.ScrapeError):
                    log.warning(
                        "the YouTube search scrape failed",
                        extra=logs.event("search.scrape_failed", error=str(exc)),
                    )
                raise youtube_refusal(exc) from exc

            # Drop currently-live/upcoming, channel-less, and blocked-channel results.
            candidates = [
                it
                for it in page["items"]
                if it["live_broadcast"] == "none"
                and it["channel_id"]
                and it["channel_id"] not in blocked
            ]
            # Ingest (DB writes + enrich) runs outside the page-fetch try above; guard it too so a
            # write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a
            # bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
            try:
                collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
            except Exception as exc:  # any ingest failure, not just YouTubeError
                db.rollback()
                # The term is NOT in this line. A search string is exactly what the privacy
                # policy keeps out of the log unless the user turned Teljes/TRACE on for
                # themselves, and `redact_text` does not cover it — it scrubs addresses and
                # credential params, not what someone typed into the search box. What the
                # ERROR line carries instead is the shape of the failure, which is what tells
                # an ingest bug from an upstream one; the term follows at TRACE, and only
                # under this user's own consent — not whenever the process runs at TRACE.
                log.exception(
                    "live search ingest failed",
                    extra=logs.event(
                        "search.live.ingest_failed",
                        source=source,
                        candidates=len(candidates),
                        collected=len(collected),
                    ),
                )
                if logs.consented_trace():
                    log.log(
                        logs.TRACE,
                        "the term the ingest failed on",
                        # log-pii-ok: only under this user's own Teljes consent
                        extra=logs.event("search.live.term", term=term),
                    )
                if collected:
                    break  # keep what earlier pages already gathered
                raise youtube_refusal(exc) from exc
            next_cursor = page["next_page_token"]
            # The API source pages cost 100 units each — never auto-page; one page per request.
            if source == "api" or len(collected) >= limit or not next_cursor:
                break

    # Count this as ONE per-user search against the daily cap. The scrape source spends no
    # quota, so it never logs an event on its own; the API source already logged exactly one
    # (via record_usage — it never auto-pages). Logging once here, after a search that produced
    # at least one page (a total failure raises 502 above and never reaches this), keeps the cap
    # counting user searches instead of internal continuation pages.
    if source != "api":
        quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH)

    ordered = collected[:limit]

    if ordered:
        # Record these as THIS user's search finds (Mine feed / Source filter), and fold the
        # query into each result's search_terms so local full-text search inherits YouTube's
        # relevance. Idempotent; the generated search_vector updates automatically.
        catalog_search.record_finds(db, user, ordered, term)
        db.commit()

    ordered = catalog_search.new_first(db, user, ordered)
    return {
        "items": reader_queries.feed_rows_by_ids(db, user, ordered),
        "next_cursor": next_cursor,
        "has_more": bool(next_cursor),
        "source": source,
    }


class ClearSearchIn(BaseModel):
    video_ids: list[str] = []


@router.post("/clear")
def clear_search(
    body: ClearSearchIn,
    user: User = Depends(current_user),
    db: Session = Depends(get_db),
) -> dict:
    """Discard the given live-search results "as if never added": drop this user's search-finds
    for them, then hard-delete the now-orphaned ones — a via_search video nobody KEPT (no watch
    state, not playlisted, channel not subscribed) and that no other user still has as a search
    find. Kept videos and organically-owned ones are left untouched."""
    ids = body.video_ids
    if not ids:
        return {"ok": True, "removed": 0}
    removed = catalog_search.clear(db, user, ids)
    db.commit()
    return {"ok": True, "removed": removed}


@router.get("/related/{video_id}")
def related_videos(
    video_id: str = Path(max_length=16),
    cursor: str | None = None,
    region: Literal["topical", "hu"] = "topical",
    user: User = Depends(current_user),
    db: Session = Depends(get_db),
) -> dict:
    """The videos YouTube shows beside this one, materialised into the catalog like search results,
    without the ones this user has watched or hidden here or in their Takeout history, and without
    blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads
    further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`)."""
    daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user")
    used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED)
    if used >= daily_limit:
        log.info(
            "a related list was refused: the per-user daily cap is spent",
            extra=logs.event("related.refused", reason="daily_cap", limit=daily_limit),
        )
        raise refused(429, "related_daily_limit", limit=daily_limit)

    hl, gl = RELATED_REGIONS[region]
    blocked = queries.blocked_ids(db, user)
    collected: list[str] = []
    next_cursor = cursor
    with YouTubeClient(db, user) as yt:
        for _ in range(RELATED_MAX_PAGES):
            try:
                page = related_scrape.related(video_id, continuation=next_cursor, hl=hl, gl=gl)
            except related_scrape.ScrapeError as exc:
                if collected:
                    break  # keep what earlier pages already gathered
                log.warning(
                    "the related-videos scrape failed",
                    extra=logs.event("related.scrape_failed", error=str(exc)),
                )
                raise youtube_refusal(exc) from exc
            ids = [it["id"] for it in page["items"]]
            drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id}
            candidates = [
                it
                for it in page["items"]
                if it["id"] not in drop and it["channel_id"] not in blocked
            ]
            # As in the search above: an ingest failure answers 422, not a bare 500.
            try:
                collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
            except Exception as exc:  # any ingest failure, not just YouTubeError
                db.rollback()
                log.exception(
                    "a related list's ingest failed",
                    extra=logs.event(
                        "related.ingest_failed",
                        candidates=len(candidates),
                        collected=len(collected),
                    ),
                )
                if collected:
                    break
                raise youtube_refusal(exc) from exc
            next_cursor = page["next_page_token"]
            if len(collected) >= RELATED_TARGET or not next_cursor:
                break

    # One opened list is one event against the cap, however many pages it read; reading further
    # (a cursor) is the same list.
    if cursor is None:
        quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED)
    return {
        "items": reader_queries.feed_rows_by_ids(db, user, collected),
        "next_cursor": next_cursor,
        "has_more": bool(next_cursor),
    }