refactor(backfill): centralize the reserve-yield guard and drop the per-page connection hold
Addresses the code-review findings on the immediate full-history change: - Extract quota.backfill_reserve_reached(): the remaining/reserve compare was open-coded at seven sites (recent/deep/immediate backfill, explore, rolling revalidation, and the new per-page guard). It reads in its OWN short-lived session (like record_usage/log_action), so a per-page guard no longer holds the caller's transaction open across the YouTube fetch that follows — the 400-page immediate run no longer pins a pooled connection idle-in-transaction through each network round-trip. Migrated all call sites; dropped the now-dead sysconfig imports in channels.py and maintenance.py. - Simplify the deep-backfill loop: channel.backfill_done = not page_token (done is False on entry) plus one break, replacing the doubled `if not page_token`. - Tests patch the backfill_reserve_reached seam directly instead of mocking remaining_today internals, removing the fragile dead-value budget iterator. Full backend suite green; ruff clean.
This commit is contained in:
@@ -227,7 +227,7 @@ class TestDeepBackfillPaging:
|
||||
return ch
|
||||
|
||||
def test_cursor_is_committed_each_page_and_a_later_run_resumes(self, db, monkeypatch):
|
||||
monkeypatch.setattr(videos.quota, "remaining_today", lambda _db: 10_000)
|
||||
monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False) # budget ok
|
||||
ch = self._channel(db)
|
||||
yt = _PagedYt(
|
||||
{
|
||||
@@ -258,11 +258,11 @@ class TestDeepBackfillPaging:
|
||||
assert ch.backfill_cursor is None
|
||||
|
||||
def test_yields_when_the_shared_budget_hits_the_reserve(self, db, monkeypatch):
|
||||
# Budget is above the 2000 reserve for the first page, then drops to the reserve — the guard
|
||||
# is checked BEFORE each page, so the run fetches exactly one page and then yields, leaving
|
||||
# the tail for the paced scheduler (cursor parked, backfill_done False).
|
||||
budget = iter([10_000, 2_000, 2_000])
|
||||
monkeypatch.setattr(videos.quota, "remaining_today", lambda _db: next(budget))
|
||||
# Budget is fine for the first page, then hits the reserve — the guard is checked BEFORE each
|
||||
# page, so the run fetches exactly one page and then yields, leaving the tail for the paced
|
||||
# scheduler (cursor parked, backfill_done False).
|
||||
reached = iter([False, True])
|
||||
monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: next(reached))
|
||||
ch = self._channel(db)
|
||||
yt = _PagedYt(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user