peter
686c673fb2
The player shows what YouTube lists beside the video, without what the user has watched or hidden. Wide, a Related tab beside the description; stacked, a button beside the info opens them in a drawer. A row plays in the same player (Back returns) and the list stays the opened video's. A switch picks on-topic or Hungarian-trending lists, kept on the account. Each related row says whether the user follows its channel; the row is the video card's narrow form with its whole menu. VideoMenu moves to features/video, where the player can reach it.
295 lines
14 KiB
Python
295 lines
14 KiB
Python
"""Live YouTube search.
|
|
|
|
Searches YouTube directly and materialises the results into the shared catalog so they render
|
|
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
|
|
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
|
|
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
|
|
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
|
|
pre-check. The result ids are enriched through the cheap shared videos.list path (1 unit / 50)
|
|
regardless of source.
|
|
|
|
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
|
|
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
|
|
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
|
|
dropped from the search response up front).
|
|
|
|
A video's related videos (`/related/{video_id}`, the list YouTube shows beside it) come in the same
|
|
way: read from the website (`related_scrape`), ingested as search finds are, under their own daily
|
|
cap — with what this user has already seen left out.
|
|
|
|
This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the
|
|
catalog is `catalog/search`."""
|
|
|
|
from typing import Literal
|
|
|
|
from fastapi import APIRouter, Depends, Path
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs, quota, sysconfig
|
|
from app.auth import current_user
|
|
from app.catalog import queries
|
|
from app.catalog import search as catalog_search
|
|
from app.db import get_db
|
|
from app.models import User
|
|
from app.reader import queries as reader_queries
|
|
from app.youtube import related_scrape, search_scrape
|
|
from app.youtube.client import YouTubeClient, YouTubeError
|
|
from app.refusal import refused
|
|
from app.routes._common import youtube_refusal
|
|
|
|
log = logs.get_logger(__name__)
|
|
router = APIRouter(prefix="/api/search", tags=["search"])
|
|
|
|
# search.list costs this many quota units per page.
|
|
SEARCH_COST = 100
|
|
|
|
# A related list is filled to this many unseen videos, reading at most this many pages (about 20
|
|
# videos each, measured) — the ones the user has seen are left out, so a page alone can fall short.
|
|
RELATED_TARGET = 20
|
|
RELATED_MAX_PAGES = 4
|
|
# The locale YouTube fills a related list in: on the video's topic, or with what trends in Hungary.
|
|
RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")}
|
|
|
|
|
|
@router.get("/youtube")
|
|
def search_youtube(
|
|
q: str | None = None,
|
|
cursor: str | None = None,
|
|
limit: int = 20,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Live YouTube search results materialised into the catalog. `limit` is how many results to
|
|
gather (the count selector); with the free scrape source we page through continuations until
|
|
that many are collected, so there's no manual "load more". The API source costs 100 units per
|
|
page, so it returns a single page (≤50) regardless. `cursor` lets the caller fetch further."""
|
|
term = (q or "").strip()
|
|
if not term:
|
|
raise refused(400, "search_term_required")
|
|
limit = max(1, min(limit, 100))
|
|
|
|
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
|
source = sysconfig.get_str(db, "search_source")
|
|
|
|
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
|
# since the search itself spends no quota.
|
|
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
|
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
|
if used >= daily_limit:
|
|
# The 429 is in the request line either way; which of the two caps produced it is not, and
|
|
# they are answered by different things — this one by tomorrow, the other by the admin.
|
|
log.info(
|
|
"a live search was refused: the per-user daily cap is spent",
|
|
extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit),
|
|
)
|
|
raise refused(429, "search_daily_limit", limit=daily_limit)
|
|
# The 100-unit budget pre-check only applies to the API source.
|
|
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
|
log.info(
|
|
"a live search was refused: the shared YouTube budget is spent",
|
|
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
|
)
|
|
raise refused(429, "quota_spent")
|
|
|
|
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
|
|
blocked = queries.blocked_ids(db, user)
|
|
|
|
collected: list[str] = []
|
|
next_cursor = cursor
|
|
MAX_PAGES = 12 # scrape safety bound
|
|
with YouTubeClient(db, user) as yt:
|
|
for _ in range(MAX_PAGES):
|
|
try:
|
|
if source == "api":
|
|
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
|
page = yt.search_videos(term, page_token=next_cursor)
|
|
else:
|
|
page = search_scrape.search(term, continuation=next_cursor)
|
|
except (YouTubeError, search_scrape.ScrapeError) as exc:
|
|
if collected:
|
|
break # keep what we already gathered
|
|
# 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway
|
|
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
|
|
# which hides the real message and burns a second quota-costed search. Upstream failure
|
|
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
|
|
# A Data API failure is logged where it happened; the scraper's is not, and its text
|
|
# no longer rides the answer — so it is said here.
|
|
if isinstance(exc, search_scrape.ScrapeError):
|
|
log.warning(
|
|
"the YouTube search scrape failed",
|
|
extra=logs.event("search.scrape_failed", error=str(exc)),
|
|
)
|
|
raise youtube_refusal(exc) from exc
|
|
|
|
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
|
candidates = [
|
|
it
|
|
for it in page["items"]
|
|
if it["live_broadcast"] == "none"
|
|
and it["channel_id"]
|
|
and it["channel_id"] not in blocked
|
|
]
|
|
# Ingest (DB writes + enrich) runs outside the page-fetch try above; guard it too so a
|
|
# write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a
|
|
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
|
try:
|
|
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
|
|
except Exception as exc: # any ingest failure, not just YouTubeError
|
|
db.rollback()
|
|
# The term is NOT in this line. A search string is exactly what the privacy
|
|
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
|
# themselves, and `redact_text` does not cover it — it scrubs addresses and
|
|
# credential params, not what someone typed into the search box. What the
|
|
# ERROR line carries instead is the shape of the failure, which is what tells
|
|
# an ingest bug from an upstream one; the term follows at TRACE, and only
|
|
# under this user's own consent — not whenever the process runs at TRACE.
|
|
log.exception(
|
|
"live search ingest failed",
|
|
extra=logs.event(
|
|
"search.live.ingest_failed",
|
|
source=source,
|
|
candidates=len(candidates),
|
|
collected=len(collected),
|
|
),
|
|
)
|
|
if logs.consented_trace():
|
|
log.log(
|
|
logs.TRACE,
|
|
"the term the ingest failed on",
|
|
# log-pii-ok: only under this user's own Teljes consent
|
|
extra=logs.event("search.live.term", term=term),
|
|
)
|
|
if collected:
|
|
break # keep what earlier pages already gathered
|
|
raise youtube_refusal(exc) from exc
|
|
next_cursor = page["next_page_token"]
|
|
# The API source pages cost 100 units each — never auto-page; one page per request.
|
|
if source == "api" or len(collected) >= limit or not next_cursor:
|
|
break
|
|
|
|
# Count this as ONE per-user search against the daily cap. The scrape source spends no
|
|
# quota, so it never logs an event on its own; the API source already logged exactly one
|
|
# (via record_usage — it never auto-pages). Logging once here, after a search that produced
|
|
# at least one page (a total failure raises 502 above and never reaches this), keeps the cap
|
|
# counting user searches instead of internal continuation pages.
|
|
if source != "api":
|
|
quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
|
|
|
ordered = collected[:limit]
|
|
|
|
if ordered:
|
|
# Record these as THIS user's search finds (Mine feed / Source filter), and fold the
|
|
# query into each result's search_terms so local full-text search inherits YouTube's
|
|
# relevance. Idempotent; the generated search_vector updates automatically.
|
|
catalog_search.record_finds(db, user, ordered, term)
|
|
db.commit()
|
|
|
|
ordered = catalog_search.new_first(db, user, ordered)
|
|
return {
|
|
"items": reader_queries.feed_rows_by_ids(db, user, ordered),
|
|
"next_cursor": next_cursor,
|
|
"has_more": bool(next_cursor),
|
|
"source": source,
|
|
}
|
|
|
|
|
|
class ClearSearchIn(BaseModel):
|
|
video_ids: list[str] = []
|
|
|
|
|
|
@router.post("/clear")
|
|
def clear_search(
|
|
body: ClearSearchIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Discard the given live-search results "as if never added": drop this user's search-finds
|
|
for them, then hard-delete the now-orphaned ones — a via_search video nobody KEPT (no watch
|
|
state, not playlisted, channel not subscribed) and that no other user still has as a search
|
|
find. Kept videos and organically-owned ones are left untouched."""
|
|
ids = body.video_ids
|
|
if not ids:
|
|
return {"ok": True, "removed": 0}
|
|
removed = catalog_search.clear(db, user, ids)
|
|
db.commit()
|
|
return {"ok": True, "removed": removed}
|
|
|
|
|
|
@router.get("/related/{video_id}")
|
|
def related_videos(
|
|
video_id: str = Path(max_length=16),
|
|
cursor: str | None = None,
|
|
region: Literal["topical", "hu"] = "topical",
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""The videos YouTube shows beside this one, materialised into the catalog like search results,
|
|
without the ones this user has watched or hidden here or in their Takeout history, and without
|
|
blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads
|
|
further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`)."""
|
|
daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user")
|
|
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED)
|
|
if used >= daily_limit:
|
|
log.info(
|
|
"a related list was refused: the per-user daily cap is spent",
|
|
extra=logs.event("related.refused", reason="daily_cap", limit=daily_limit),
|
|
)
|
|
raise refused(429, "related_daily_limit", limit=daily_limit)
|
|
|
|
hl, gl = RELATED_REGIONS[region]
|
|
blocked = queries.blocked_ids(db, user)
|
|
collected: list[str] = []
|
|
next_cursor = cursor
|
|
with YouTubeClient(db, user) as yt:
|
|
for _ in range(RELATED_MAX_PAGES):
|
|
try:
|
|
page = related_scrape.related(video_id, continuation=next_cursor, hl=hl, gl=gl)
|
|
except related_scrape.ScrapeError as exc:
|
|
if collected:
|
|
break # keep what earlier pages already gathered
|
|
log.warning(
|
|
"the related-videos scrape failed",
|
|
extra=logs.event("related.scrape_failed", error=str(exc)),
|
|
)
|
|
raise youtube_refusal(exc) from exc
|
|
ids = [it["id"] for it in page["items"]]
|
|
drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id}
|
|
candidates = [
|
|
it
|
|
for it in page["items"]
|
|
if it["id"] not in drop and it["channel_id"] not in blocked
|
|
]
|
|
# As in the search above: an ingest failure answers 422, not a bare 500.
|
|
try:
|
|
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
|
|
except Exception as exc: # any ingest failure, not just YouTubeError
|
|
db.rollback()
|
|
log.exception(
|
|
"a related list's ingest failed",
|
|
extra=logs.event(
|
|
"related.ingest_failed",
|
|
candidates=len(candidates),
|
|
collected=len(collected),
|
|
),
|
|
)
|
|
if collected:
|
|
break
|
|
raise youtube_refusal(exc) from exc
|
|
next_cursor = page["next_page_token"]
|
|
if len(collected) >= RELATED_TARGET or not next_cursor:
|
|
break
|
|
|
|
# One opened list is one event against the cap, however many pages it read; reading further
|
|
# (a cursor) is the same list.
|
|
if cursor is None:
|
|
quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED)
|
|
rows = reader_queries.feed_rows_by_ids(db, user, collected)
|
|
# A related list mixes channels the user follows with ones they never met: each row says which.
|
|
followed = queries.followed_among(db, user, {r["channel_id"] for r in rows})
|
|
return {
|
|
"items": [{**r, "followed": r["channel_id"] in followed} for r in rows],
|
|
"next_cursor": next_cursor,
|
|
"has_more": bool(next_cursor),
|
|
}
|