fix(search): every related and live-search request takes a slot, and a failed one gives it back
A related list's cursor cost nothing, and a cursor is a free string: a replayed one read and enriched without end. Every request now takes a slot of the user's daily cap — taken under a lock before the work (`quota.claim_action`), so requests sent together cannot all pass the last free one, and given back when the request fails outright (`quota.refund_action`). The live search's scrape source takes its slots the same way. A limit of 0 turns the feature off: a distinct 409 `related_disabled` / `search_disabled`, and `/api/me` says `related_enabled` / `search_enabled` so the UI can leave it out. A videos.list that fails no longer reads as an empty page: `ingest_candidates` raises, and the search and the related list answer the refusal (the shared budget spent says "quota") instead of "no results". The two routes' page loops are one (`_gather`). `clear` deletes only what the caller found: a related list's rows carry no find, and any user could name them.
This commit is contained in:
@@ -79,6 +79,10 @@ def get_me(
|
||||
# Which live-search source runs, so the search button can say what a search costs: "scrape"
|
||||
# spends no API quota, "api" spends 100 units a page.
|
||||
"search_source": sysconfig.get_str(db, "search_source"),
|
||||
# Whether live search and related videos are on: an admin turns either off with a daily
|
||||
# limit of 0, and the UI then leaves out what would only be refused.
|
||||
"search_enabled": sysconfig.get_int(db, "search_daily_limit_per_user") > 0,
|
||||
"related_enabled": sysconfig.get_int(db, "related_daily_limit_per_user") > 0,
|
||||
"can_read": has_read_scope(user),
|
||||
"can_write": has_write_scope(user),
|
||||
"pending_invites": pending_invites,
|
||||
|
||||
+200
-146
@@ -20,6 +20,7 @@ cap — with what this user has already seen left out.
|
||||
This is the HTTP side — the paging, the caps and the refusals; what a page of results leaves in the
|
||||
catalog is `catalog/search`."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Path
|
||||
@@ -34,6 +35,7 @@ from app.db import get_db
|
||||
from app.models import User
|
||||
from app.reader import queries as reader_queries
|
||||
from app.youtube import related_scrape, search_scrape
|
||||
from app.youtube.innertube import ScrapeError
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
from app.routes._common import youtube_refusal
|
||||
@@ -43,6 +45,8 @@ router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
|
||||
# search.list costs this many quota units per page.
|
||||
SEARCH_COST = 100
|
||||
# The scrape source reads at most this many pages for one search.
|
||||
SEARCH_MAX_PAGES = 12
|
||||
|
||||
# A related list is filled to this many unseen videos, reading at most this many pages (about 20
|
||||
# videos each, measured) — the ones the user has seen are left out, so a page alone can fall short.
|
||||
@@ -52,6 +56,66 @@ RELATED_MAX_PAGES = 4
|
||||
RELATED_REGIONS = {"topical": ("en", "US"), "hu": ("hu", "HU")}
|
||||
|
||||
|
||||
def _gather(
|
||||
db: Session,
|
||||
yt: YouTubeClient,
|
||||
user: User,
|
||||
*,
|
||||
cursor: str | None,
|
||||
max_pages: int,
|
||||
fetch: Callable[[str | None], dict],
|
||||
candidates_of: Callable[[list[dict], list[str]], list[dict]],
|
||||
enough: Callable[[list[str]], bool],
|
||||
scrape_failed: Callable[[Exception], None],
|
||||
ingest_failed: Callable[[int, int], None],
|
||||
) -> tuple[list[str], str | None]:
|
||||
"""Page through a YouTube list into the catalog: fetch a page, keep its `candidates_of`, ingest
|
||||
them, and go on from its continuation until `enough` or `max_pages`. Returns the kept ids and
|
||||
the cursor to read further from. A page that fails after earlier ones gathered something ends
|
||||
the list with what it has; a first page that fails raises the 422 refusal.
|
||||
|
||||
422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway dropout
|
||||
(auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY), which hides the real
|
||||
message and repeats the work. A Data API failure is logged where it happened; the scraper's is
|
||||
not, and its text does not ride the answer — so `scrape_failed` says it. An ingest (DB writes +
|
||||
enrich) fails the same way, guarded for any exception so a write or shape failure is a 422 with
|
||||
a message too, not a bare 500; `ingest_failed` logs it inside the `except`, with the trace."""
|
||||
collected: list[str] = []
|
||||
next_cursor = cursor
|
||||
for _ in range(max_pages):
|
||||
try:
|
||||
page = fetch(next_cursor)
|
||||
except (YouTubeError, ScrapeError) as exc:
|
||||
if collected:
|
||||
break # keep what earlier pages already gathered
|
||||
if isinstance(exc, ScrapeError):
|
||||
scrape_failed(exc)
|
||||
raise youtube_refusal(exc) from exc
|
||||
candidates = candidates_of(page["items"], collected)
|
||||
try:
|
||||
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
|
||||
except Exception as exc: # any ingest failure, not just YouTubeError
|
||||
db.rollback()
|
||||
ingest_failed(len(candidates), len(collected))
|
||||
if collected:
|
||||
break
|
||||
raise youtube_refusal(exc) from exc
|
||||
next_cursor = page["next_page_token"]
|
||||
if enough(collected) or not next_cursor:
|
||||
break
|
||||
return collected, next_cursor
|
||||
|
||||
|
||||
def _cap_spent(feature: str, limit: int) -> None:
|
||||
"""Say that a request was refused for the per-user daily cap. The 429 is in the request line
|
||||
either way; which of the caps produced it is not, and they are answered by different things —
|
||||
this one by tomorrow, the shared budget by the admin."""
|
||||
log.info(
|
||||
"a request was refused: the per-user daily cap is spent",
|
||||
extra=logs.event(f"{feature}.refused", reason="daily_cap", limit=limit),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/youtube")
|
||||
def search_youtube(
|
||||
q: str | None = None,
|
||||
@@ -72,109 +136,103 @@ def search_youtube(
|
||||
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
||||
source = sysconfig.get_str(db, "search_source")
|
||||
|
||||
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
||||
# since the search itself spends no quota.
|
||||
# Every request takes a slot of the per-user daily cap, a cursor's too (anti-abuse); with
|
||||
# scrape it's the only guard since the search itself spends no quota. A limit of 0 turns live
|
||||
# search off on this instance — a refusal tomorrow will not change.
|
||||
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
||||
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
if used >= daily_limit:
|
||||
# The 429 is in the request line either way; which of the two caps produced it is not, and
|
||||
# they are answered by different things — this one by tomorrow, the other by the admin.
|
||||
log.info(
|
||||
"a live search was refused: the per-user daily cap is spent",
|
||||
extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit),
|
||||
)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
||||
log.info(
|
||||
"a live search was refused: the shared YouTube budget is spent",
|
||||
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
||||
)
|
||||
raise refused(429, "quota_spent")
|
||||
if daily_limit <= 0:
|
||||
raise refused(409, "search_disabled")
|
||||
claimed: int | None = None
|
||||
if source == "api":
|
||||
# The one search.list call logs this action's event itself (record_usage, attributed
|
||||
# below), so here the cap is only checked: a claimed slot would count it twice.
|
||||
if quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) >= daily_limit:
|
||||
_cap_spent("search", daily_limit)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if not quota.can_spend(db, SEARCH_COST):
|
||||
log.info(
|
||||
"a live search was refused: the shared YouTube budget is spent",
|
||||
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
||||
)
|
||||
raise refused(429, "quota_spent")
|
||||
else:
|
||||
# The check and the take are one step (`quota.claim_action`); a search that fails
|
||||
# outright gives the slot back below.
|
||||
claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_SEARCH, daily_limit)
|
||||
if claimed is None:
|
||||
_cap_spent("search", daily_limit)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
|
||||
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
|
||||
blocked = queries.blocked_ids(db, user)
|
||||
|
||||
collected: list[str] = []
|
||||
next_cursor = cursor
|
||||
MAX_PAGES = 12 # scrape safety bound
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for _ in range(MAX_PAGES):
|
||||
try:
|
||||
if source == "api":
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
||||
page = yt.search_videos(term, page_token=next_cursor)
|
||||
else:
|
||||
page = search_scrape.search(term, continuation=next_cursor)
|
||||
except (YouTubeError, search_scrape.ScrapeError) as exc:
|
||||
if collected:
|
||||
break # keep what we already gathered
|
||||
# 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway
|
||||
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
|
||||
# which hides the real message and burns a second quota-costed search. Upstream failure
|
||||
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
|
||||
# A Data API failure is logged where it happened; the scraper's is not, and its text
|
||||
# no longer rides the answer — so it is said here.
|
||||
if isinstance(exc, search_scrape.ScrapeError):
|
||||
log.warning(
|
||||
"the YouTube search scrape failed",
|
||||
extra=logs.event("search.scrape_failed", error=str(exc)),
|
||||
)
|
||||
raise youtube_refusal(exc) from exc
|
||||
def fetch(token: str | None) -> dict:
|
||||
if source == "api":
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
||||
return yt.search_videos(term, page_token=token)
|
||||
return search_scrape.search(term, continuation=token)
|
||||
|
||||
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
||||
candidates = [
|
||||
it
|
||||
for it in page["items"]
|
||||
if it["live_broadcast"] == "none"
|
||||
and it["channel_id"]
|
||||
and it["channel_id"] not in blocked
|
||||
]
|
||||
# Ingest (DB writes + enrich) runs outside the page-fetch try above; guard it too so a
|
||||
# write/shape failure degrades to a 422 with a message on this JSON endpoint instead of a
|
||||
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
||||
try:
|
||||
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
|
||||
except Exception as exc: # any ingest failure, not just YouTubeError
|
||||
db.rollback()
|
||||
# The term is NOT in this line. A search string is exactly what the privacy
|
||||
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
||||
# themselves, and `redact_text` does not cover it — it scrubs addresses and
|
||||
# credential params, not what someone typed into the search box. What the
|
||||
# ERROR line carries instead is the shape of the failure, which is what tells
|
||||
# an ingest bug from an upstream one; the term follows at TRACE, and only
|
||||
# under this user's own consent — not whenever the process runs at TRACE.
|
||||
log.exception(
|
||||
"live search ingest failed",
|
||||
extra=logs.event(
|
||||
"search.live.ingest_failed",
|
||||
source=source,
|
||||
candidates=len(candidates),
|
||||
collected=len(collected),
|
||||
),
|
||||
)
|
||||
if logs.consented_trace():
|
||||
log.log(
|
||||
logs.TRACE,
|
||||
"the term the ingest failed on",
|
||||
# log-pii-ok: only under this user's own Teljes consent
|
||||
extra=logs.event("search.live.term", term=term),
|
||||
)
|
||||
if collected:
|
||||
break # keep what earlier pages already gathered
|
||||
raise youtube_refusal(exc) from exc
|
||||
next_cursor = page["next_page_token"]
|
||||
# The API source pages cost 100 units each — never auto-page; one page per request.
|
||||
if source == "api" or len(collected) >= limit or not next_cursor:
|
||||
break
|
||||
def candidates_of(items: list[dict], _collected: list[str]) -> list[dict]:
|
||||
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
||||
return [
|
||||
it
|
||||
for it in items
|
||||
if it["live_broadcast"] == "none"
|
||||
and it["channel_id"]
|
||||
and it["channel_id"] not in blocked
|
||||
]
|
||||
|
||||
# Count this as ONE per-user search against the daily cap. The scrape source spends no
|
||||
# quota, so it never logs an event on its own; the API source already logged exactly one
|
||||
# (via record_usage — it never auto-pages). Logging once here, after a search that produced
|
||||
# at least one page (a total failure raises 502 above and never reaches this), keeps the cap
|
||||
# counting user searches instead of internal continuation pages.
|
||||
if source != "api":
|
||||
quota.log_action(user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
def scrape_failed(exc: Exception) -> None:
|
||||
log.warning(
|
||||
"the YouTube search scrape failed",
|
||||
extra=logs.event("search.scrape_failed", error=str(exc)),
|
||||
)
|
||||
|
||||
def ingest_failed(candidates: int, collected: int) -> None:
|
||||
# The term is NOT in this line. A search string is exactly what the privacy policy keeps
|
||||
# out of the log unless the user turned Teljes/TRACE on for themselves, and `redact_text`
|
||||
# does not cover it — it scrubs addresses and credential params, not what someone typed
|
||||
# into the search box. What the ERROR line carries instead is the shape of the failure,
|
||||
# which is what tells an ingest bug from an upstream one; the term follows at TRACE, and
|
||||
# only under this user's own consent — not whenever the process runs at TRACE.
|
||||
log.exception(
|
||||
"live search ingest failed",
|
||||
extra=logs.event(
|
||||
"search.live.ingest_failed",
|
||||
source=source,
|
||||
candidates=candidates,
|
||||
collected=collected,
|
||||
),
|
||||
)
|
||||
if logs.consented_trace():
|
||||
log.log(
|
||||
logs.TRACE,
|
||||
"the term the ingest failed on",
|
||||
# log-pii-ok: only under this user's own Teljes consent
|
||||
extra=logs.event("search.live.term", term=term),
|
||||
)
|
||||
|
||||
try:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
collected, next_cursor = _gather(
|
||||
db,
|
||||
yt,
|
||||
user,
|
||||
cursor=cursor,
|
||||
# The API source pages cost 100 units each — never auto-page; one page per request.
|
||||
max_pages=1 if source == "api" else SEARCH_MAX_PAGES,
|
||||
fetch=fetch,
|
||||
candidates_of=candidates_of,
|
||||
enough=lambda got: len(got) >= limit,
|
||||
scrape_failed=scrape_failed,
|
||||
ingest_failed=ingest_failed,
|
||||
)
|
||||
except Exception:
|
||||
# A search that showed the user nothing does not count against them.
|
||||
if claimed is not None:
|
||||
quota.refund_action(claimed)
|
||||
raise
|
||||
|
||||
ordered = collected[:limit]
|
||||
|
||||
@@ -228,62 +286,58 @@ def related_videos(
|
||||
without the ones this user has watched or hidden here or in their Takeout history, and without
|
||||
blocked channels. Pages on (`cursor`) until `RELATED_TARGET` are gathered; `next_cursor` reads
|
||||
further. `region` picks the locale YouTube fills the list in (`RELATED_REGIONS`)."""
|
||||
# Every request takes a slot, a cursor's too: each one scrapes pages and enriches what they
|
||||
# hold, and a cursor is a free string — counting only first pages let a replayed cursor read
|
||||
# without end. The check and the take are one step (`quota.claim_action`); a request that fails
|
||||
# outright gives its slot back. A limit of 0 turns related videos off on this instance.
|
||||
daily_limit = sysconfig.get_int(db, "related_daily_limit_per_user")
|
||||
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_RELATED)
|
||||
if used >= daily_limit:
|
||||
log.info(
|
||||
"a related list was refused: the per-user daily cap is spent",
|
||||
extra=logs.event("related.refused", reason="daily_cap", limit=daily_limit),
|
||||
)
|
||||
if daily_limit <= 0:
|
||||
raise refused(409, "related_disabled")
|
||||
claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_RELATED, daily_limit)
|
||||
if claimed is None:
|
||||
_cap_spent("related", daily_limit)
|
||||
raise refused(429, "related_daily_limit", limit=daily_limit)
|
||||
|
||||
hl, gl = RELATED_REGIONS[region]
|
||||
blocked = queries.blocked_ids(db, user)
|
||||
collected: list[str] = []
|
||||
next_cursor = cursor
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for _ in range(RELATED_MAX_PAGES):
|
||||
try:
|
||||
page = related_scrape.related(video_id, continuation=next_cursor, hl=hl, gl=gl)
|
||||
except related_scrape.ScrapeError as exc:
|
||||
if collected:
|
||||
break # keep what earlier pages already gathered
|
||||
log.warning(
|
||||
"the related-videos scrape failed",
|
||||
extra=logs.event("related.scrape_failed", error=str(exc)),
|
||||
)
|
||||
raise youtube_refusal(exc) from exc
|
||||
ids = [it["id"] for it in page["items"]]
|
||||
drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id}
|
||||
candidates = [
|
||||
it
|
||||
for it in page["items"]
|
||||
if it["id"] not in drop and it["channel_id"] not in blocked
|
||||
]
|
||||
# As in the search above: an ingest failure answers 422, not a bare 500.
|
||||
try:
|
||||
collected.extend(catalog_search.ingest_candidates(db, yt, user, candidates))
|
||||
except Exception as exc: # any ingest failure, not just YouTubeError
|
||||
db.rollback()
|
||||
log.exception(
|
||||
"a related list's ingest failed",
|
||||
extra=logs.event(
|
||||
"related.ingest_failed",
|
||||
candidates=len(candidates),
|
||||
collected=len(collected),
|
||||
),
|
||||
)
|
||||
if collected:
|
||||
break
|
||||
raise youtube_refusal(exc) from exc
|
||||
next_cursor = page["next_page_token"]
|
||||
if len(collected) >= RELATED_TARGET or not next_cursor:
|
||||
break
|
||||
|
||||
# One opened list is one event against the cap, however many pages it read; reading further
|
||||
# (a cursor) is the same list.
|
||||
if cursor is None:
|
||||
quota.log_action(user.id, quota.QuotaAction.VIDEOS_RELATED)
|
||||
def candidates_of(items: list[dict], collected: list[str]) -> list[dict]:
|
||||
ids = [it["id"] for it in items]
|
||||
drop = reader_queries.seen_by(db, user, ids) | set(collected) | {video_id}
|
||||
return [it for it in items if it["id"] not in drop and it["channel_id"] not in blocked]
|
||||
|
||||
def scrape_failed(exc: Exception) -> None:
|
||||
log.warning(
|
||||
"the related-videos scrape failed",
|
||||
extra=logs.event("related.scrape_failed", error=str(exc)),
|
||||
)
|
||||
|
||||
def ingest_failed(candidates: int, collected: int) -> None:
|
||||
log.exception(
|
||||
"a related list's ingest failed",
|
||||
extra=logs.event("related.ingest_failed", candidates=candidates, collected=collected),
|
||||
)
|
||||
|
||||
try:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
collected, next_cursor = _gather(
|
||||
db,
|
||||
yt,
|
||||
user,
|
||||
cursor=cursor,
|
||||
max_pages=RELATED_MAX_PAGES,
|
||||
fetch=lambda token: related_scrape.related(
|
||||
video_id, continuation=token, hl=hl, gl=gl
|
||||
),
|
||||
candidates_of=candidates_of,
|
||||
enough=lambda got: len(got) >= RELATED_TARGET,
|
||||
scrape_failed=scrape_failed,
|
||||
ingest_failed=ingest_failed,
|
||||
)
|
||||
except Exception:
|
||||
quota.refund_action(claimed)
|
||||
raise
|
||||
|
||||
rows = reader_queries.feed_rows_by_ids(db, user, collected)
|
||||
# A related list mixes channels the user follows with ones they never met: each row says which.
|
||||
followed = queries.followed_among(db, user, {r["channel_id"] for r in rows})
|
||||
|
||||
Reference in New Issue
Block a user