fix(search): the API-sourced live search takes its slot under the lock too
Checked first and logged after search.list, requests sent together at the last free slot all passed, and each spent 100 units of the shared budget. Every live search now claims its slot (quota.claim_action) whatever the source, and gives it back when it showed nothing. The cap counts the zero-unit claims only, so search.list's own spend event is accounting, not a second slot. log_action, the unlocked writer, goes. Tests: the API source reads one page per request, is refused before the call when the budget cannot pay, counts one slot per search and refunds a failed one; the lock test now proves the count happens after the lock.
This commit is contained in:
+12
-19
@@ -143,7 +143,7 @@ def backfill_reserve_reached() -> bool:
|
||||
condition for every backfill/explore path (recent, deep, immediate full-history, explore, and
|
||||
rolling revalidation).
|
||||
|
||||
Reads in its OWN short-lived session (like record_usage / log_action), for two reasons: it
|
||||
Reads in its OWN short-lived session (like record_usage / claim_action), for two reasons: it
|
||||
replaces the remaining/reserve comparison that was open-coded at every call site, and — critically
|
||||
for the per-page guards in paging loops — a guard must not hold the caller's transaction open
|
||||
across the network fetch that follows it. Quota is committed independently of any caller, so a
|
||||
@@ -152,26 +152,17 @@ def backfill_reserve_reached() -> bool:
|
||||
return remaining_today(qs) <= sysconfig.get_int(qs, "backfill_quota_reserve")
|
||||
|
||||
|
||||
def log_action(user_id: int, action: str) -> None:
|
||||
"""Record a zero-cost action event so per-user daily caps still count it.
|
||||
|
||||
`record_usage` only logs when units are actually charged; a scrape-based search spends no
|
||||
quota yet must still be rate-limited per user, so it logs its event here directly.
|
||||
|
||||
Writes in its OWN session (like record_usage) — quota accounting is durable independent of the
|
||||
caller's transaction and must never commit the caller's session (that would flush a job's
|
||||
partial state mid-run, defeating its own rollback)."""
|
||||
with SessionLocal() as qs:
|
||||
qs.add(QuotaEvent(user_id=user_id, action=action, units=0))
|
||||
qs.commit()
|
||||
|
||||
|
||||
def claim_action(user_id: int, action: str, limit: int) -> int | None:
|
||||
"""Take one of this user's `limit` daily `action` slots, or None when they are all taken — the
|
||||
check and the take in one transaction, under a lock per user and action. Checked first and
|
||||
logged after the work, N requests sent together at the last free slot all passed the check
|
||||
(the work in between is seconds of scraping). Returns the event's id, for `refund_action` when
|
||||
the work then produced nothing. Own session, as `log_action`."""
|
||||
the work then produced nothing.
|
||||
|
||||
The ONE way to count a capped action: a claim is a zero-unit event, and `actions_today` counts
|
||||
those alone, so a call the action then pays for (search.list's own `record_usage` event, under
|
||||
the same action) is accounting, never a second slot. Own session, committed at once — quota
|
||||
accounting is durable whatever the caller's transaction does next, and must never commit it."""
|
||||
with SessionLocal() as qs:
|
||||
qs.execute(select(func.pg_advisory_xact_lock(func.hashtext(action), user_id)))
|
||||
if actions_today(qs, user_id, action) >= limit:
|
||||
@@ -191,9 +182,10 @@ def refund_action(event_id: int) -> None:
|
||||
|
||||
|
||||
def actions_today(db: Session, user_id: int, action: str) -> int:
|
||||
"""How many quota events of `action` this user has logged so far in the current Pacific
|
||||
day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events,
|
||||
not units, so it only works for actions charged exactly once per user action."""
|
||||
"""How many slots of `action` this user has claimed so far in the current Pacific day — for
|
||||
per-user, per-action daily caps (the live search, the related lists). Counts the zero-unit
|
||||
claim events (`claim_action`) only: `record_usage`'s events for units the action spent are not
|
||||
slots (it logs only when units were charged)."""
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count())
|
||||
@@ -201,6 +193,7 @@ def actions_today(db: Session, user_id: int, action: str) -> int:
|
||||
.where(
|
||||
owned_by(QuotaEvent.user_id, user_id),
|
||||
QuotaEvent.action == action,
|
||||
QuotaEvent.units == 0,
|
||||
QuotaEvent.created_at >= pacific_day_start_utc(),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -142,27 +142,20 @@ def search_youtube(
|
||||
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
||||
if daily_limit <= 0:
|
||||
raise refused(409, "search_disabled")
|
||||
claimed: int | None = None
|
||||
if source == "api":
|
||||
# The one search.list call logs this action's event itself (record_usage, attributed
|
||||
# below), so here the cap is only checked: a claimed slot would count it twice.
|
||||
if quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) >= daily_limit:
|
||||
_cap_spent("search", daily_limit)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if not quota.can_spend(db, SEARCH_COST):
|
||||
log.info(
|
||||
"a live search was refused: the shared YouTube budget is spent",
|
||||
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
||||
)
|
||||
raise refused(429, "quota_spent")
|
||||
else:
|
||||
# The check and the take are one step (`quota.claim_action`); a search that fails
|
||||
# outright gives the slot back below.
|
||||
claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_SEARCH, daily_limit)
|
||||
if claimed is None:
|
||||
_cap_spent("search", daily_limit)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
||||
log.info(
|
||||
"a live search was refused: the shared YouTube budget is spent",
|
||||
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
||||
)
|
||||
raise refused(429, "quota_spent")
|
||||
# The check and the take are one step (`quota.claim_action`), whichever the source: checked
|
||||
# first and logged after, requests sent together at the last free slot all passed — and with
|
||||
# the API source each of them spent 100 units. A search that fails outright gives it back below.
|
||||
claimed = quota.claim_action(user.id, quota.QuotaAction.VIDEOS_SEARCH, daily_limit)
|
||||
if claimed is None:
|
||||
_cap_spent("search", daily_limit)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
|
||||
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
|
||||
blocked = queries.blocked_ids(db, user)
|
||||
@@ -230,8 +223,7 @@ def search_youtube(
|
||||
)
|
||||
except Exception:
|
||||
# A search that showed the user nothing does not count against them.
|
||||
if claimed is not None:
|
||||
quota.refund_action(claimed)
|
||||
quota.refund_action(claimed)
|
||||
raise
|
||||
|
||||
ordered = collected[:limit]
|
||||
|
||||
Reference in New Issue
Block a user