peter
5229fd96be
Channels, search and sync do no database work in their routes any more: reads go through catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search. What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed projection of results) it asks reader/queries and reader/service. The scheduler and the runs page read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest jobs commit only where they are named.
204 lines
9.8 KiB
Python
204 lines
9.8 KiB
Python
"""Live YouTube search.
|
|
|
|
Searches YouTube directly and materialises the results into the shared catalog so they render
|
|
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
|
|
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
|
|
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
|
|
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
|
|
pre-check. The result ids are enriched through the cheap shared videos.list path (1 unit / 50)
|
|
regardless of source.
|
|
|
|
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
|
|
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
|
|
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
|
|
dropped from the search response up front).
|
|
|
|
This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the
|
|
catalog is `catalog/search`."""
|
|
|
|
from fastapi import APIRouter, Depends
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs, quota, sysconfig
|
|
from app.auth import current_user
|
|
from app.catalog import queries
|
|
from app.catalog import search as catalog_search
|
|
from app.db import get_db
|
|
from app.models import User
|
|
from app.reader import queries as reader_queries
|
|
from app.youtube import search_scrape
|
|
from app.youtube.client import YouTubeClient, YouTubeError
|
|
from app.refusal import refused
|
|
from app.routes._common import youtube_refusal
|
|
|
|
log = logs.get_logger(__name__)
|
|
router = APIRouter(prefix="/api/search", tags=["search"])
|
|
|
|
# search.list costs this many quota units per page.
|
|
SEARCH_COST = 100
|
|
|
|
|
|
@router.get("/youtube")
|
|
def search_youtube(
|
|
q: str | None = None,
|
|
cursor: str | None = None,
|
|
limit: int = 20,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Live YouTube search results materialised into the catalog. `limit` is how many results to
|
|
gather (the count selector); with the free scrape source we page through continuations until
|
|
that many are collected, so there's no manual "load more". The API source costs 100 units per
|
|
page, so it returns a single page (≤50) regardless. `cursor` lets the caller fetch further."""
|
|
term = (q or "").strip()
|
|
if not term:
|
|
raise refused(400, "search_term_required")
|
|
limit = max(1, min(limit, 100))
|
|
|
|
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
|
source = sysconfig.get_str(db, "search_source")
|
|
|
|
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
|
# since the search itself spends no quota.
|
|
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
|
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
|
if used >= daily_limit:
|
|
# The 429 is in the request line either way; which of the two caps produced it is not, and
|
|
# they are answered by different things — this one by tomorrow, the other by the admin.
|
|
log.info(
|
|
"a live search was refused: the per-user daily cap is spent",
|
|
extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit),
|
|
)
|
|
raise refused(429, "search_daily_limit", limit=daily_limit)
|
|
# The 100-unit budget pre-check only applies to the API source.
|
|
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
|
log.info(
|
|
"a live search was refused: the shared YouTube budget is spent",
|
|
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
|
)
|
|
raise refused(429, "quota_spent")
|
|
|
|
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
|
|
blocked = queries.blocked_ids(db, user)
|
|
|
|
collected: list[str] = []
|
|
next_cursor = cursor
|
|
MAX_PAGES = 12 # scrape safety bound
|
|
with YouTubeClient(db, user) as yt:
|
|
for _ in range(MAX_PAGES):
|
|
try:
|
|
if source == "api":
|
|
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
|
page = yt.search_videos(term, page_token=next_cursor)
|
|
else:
|
|
page = search_scrape.search(term, continuation=next_cursor)
|
|
except (YouTubeError, search_scrape.ScrapeError) as exc:
|
|
if collected:
|
|
break # keep what we already gathered
|
|
# 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway
|
|
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
|
|
# which hides the real message and burns a second quota-costed search. Upstream failure
|
|
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
|
|
# A Data API failure is logged where it happened; the scraper's is not, and its text
|
|
# no longer rides the answer — so it is said here.
|
|
if isinstance(exc, search_scrape.ScrapeError):
|
|
log.warning(
|
|
"the YouTube search scrape failed",
|
|
extra=logs.event("search.scrape_failed", error=str(exc)),
|
|
)
|
|
raise youtube_refusal(exc) from exc
|
|
|
|
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
|
candidates = [
|
|
it
|
|
for it in page["items"]
|
|
if it["live_broadcast"] == "none"
|
|
and it["channel_id"]
|
|
and it["channel_id"] not in blocked
|
|
]
|
|
# Ingest (DB writes + enrich) runs outside the page-fetch try above; guard it too so a
|
|
# write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a
|
|
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
|
try:
|
|
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
|
|
except Exception as exc: # any ingest failure, not just YouTubeError
|
|
db.rollback()
|
|
# The term is NOT in this line. A search string is exactly what the privacy
|
|
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
|
# themselves, and `redact_text` does not cover it — it scrubs addresses and
|
|
# credential params, not what someone typed into the search box. What the
|
|
# ERROR line carries instead is the shape of the failure, which is what tells
|
|
# an ingest bug from an upstream one; the term follows at TRACE, and only
|
|
# under this user's own consent — not whenever the process runs at TRACE.
|
|
log.exception(
|
|
"live search ingest failed",
|
|
extra=logs.event(
|
|
"search.live.ingest_failed",
|
|
source=source,
|
|
candidates=len(candidates),
|
|
collected=len(collected),
|
|
),
|
|
)
|
|
if logs.consented_trace():
|
|
log.log(
|
|
logs.TRACE,
|
|
"the term the ingest failed on",
|
|
# log-pii-ok: only under this user's own Teljes consent
|
|
extra=logs.event("search.live.term", term=term),
|
|
)
|
|
if collected:
|
|
break # keep what earlier pages already gathered
|
|
raise youtube_refusal(exc) from exc
|
|
next_cursor = page["next_page_token"]
|
|
# The API source pages cost 100 units each — never auto-page; one page per request.
|
|
if source == "api" or len(collected) >= limit or not next_cursor:
|
|
break
|
|
|
|
# Count this as ONE per-user search against the daily cap. The scrape source spends no
|
|
# quota, so it never logs an event on its own; the API source already logged exactly one
|
|
# (via record_usage — it never auto-pages). Logging once here, after a search that produced
|
|
# at least one page (a total failure raises 502 above and never reaches this), keeps the cap
|
|
# counting user searches instead of internal continuation pages.
|
|
if source != "api":
|
|
quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
|
|
|
ordered = collected[:limit]
|
|
|
|
if ordered:
|
|
# Record these as THIS user's search finds (Mine feed / Source filter), and fold the
|
|
# query into each result's search_terms so local full-text search inherits YouTube's
|
|
# relevance. Idempotent; the generated search_vector updates automatically.
|
|
catalog_search.record_finds(db, user, ordered, term)
|
|
db.commit()
|
|
|
|
ordered = catalog_search.new_first(db, user, ordered)
|
|
return {
|
|
"items": reader_queries.feed_rows_by_ids(db, user, ordered),
|
|
"next_cursor": next_cursor,
|
|
"has_more": bool(next_cursor),
|
|
"source": source,
|
|
}
|
|
|
|
|
|
class ClearSearchIn(BaseModel):
|
|
video_ids: list[str] = []
|
|
|
|
|
|
@router.post("/clear")
|
|
def clear_search(
|
|
body: ClearSearchIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Discard the given live-search results "as if never added": drop this user's search-finds
|
|
for them, then hard-delete the now-orphaned ones — a via_search video nobody KEPT (no watch
|
|
state, not playlisted, channel not subscribed) and that no other user still has as a search
|
|
find. Kept videos and organically-owned ones are left untouched."""
|
|
ids = body.video_ids
|
|
if not ids:
|
|
return {"ok": True, "removed": 0}
|
|
removed = catalog_search.clear(db, user, ids)
|
|
db.commit()
|
|
return {"ok": True, "removed": removed}
|