refactor(backfill): centralize the reserve-yield guard and drop the per-page connection hold

Addresses the code-review findings on the immediate full-history change:

- Extract quota.backfill_reserve_reached(): the remaining/reserve compare was
  open-coded at seven sites (recent/deep/immediate backfill, explore, rolling
  revalidation, and the new per-page guard). It reads in its OWN short-lived
  session (like record_usage/log_action), so a per-page guard no longer holds
  the caller's transaction open across the YouTube fetch that follows — the
  400-page immediate run no longer pins a pooled connection idle-in-transaction
  through each network round-trip. Migrated all call sites; dropped the now-dead
  sysconfig imports in channels.py and maintenance.py.
- Simplify the deep-backfill loop: channel.backfill_done = not page_token (done
  is False on entry) plus one break, replacing the doubled `if not page_token`.
- Tests patch the backfill_reserve_reached seam directly instead of mocking
  remaining_today internals, removing the fragile dead-value budget iterator.

Full backend suite green; ruff clean.
This commit is contained in:
2026-08-16 02:41:49 +02:00
parent e3ee1baa94
commit 48285a715b
6 changed files with 30 additions and 18 deletions
+3 -3
View File
@@ -25,7 +25,7 @@ from datetime import timedelta
from sqlalchemy import select
from sqlalchemy.orm import Session
from app import progress, quota, sysconfig
from app import progress, quota
from app.config import settings
from app.models import Playlist, PlaylistItem, Video
from app.notifications import create_notification
@@ -188,7 +188,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
now = _now()
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
while checked < batch:
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
if quota.backfill_reserve_reached():
break
page = (
db.execute(
@@ -228,7 +228,7 @@ def run_maintenance(db: Session) -> dict:
phase1 = _recheck_flagged(db, yt)
phase2 = (
_revalidate_rolling(db, yt)
if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
if not quota.backfill_reserve_reached()
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
)
return {
+3 -3
View File
@@ -183,7 +183,7 @@ def run_recent_backfill(
videos_added = 0
with YouTubeClient(db, user) as yt:
for channel in channels:
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
if quota.backfill_reserve_reached():
break
try:
videos_added += backfill_channel_recent(db, yt, channel)
@@ -229,7 +229,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
videos_added = 0
with YouTubeClient(db, user) as yt:
for channel in channels:
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
if quota.backfill_reserve_reached():
break
# Claim the channel (or skip it if an interactive backfill already owns it) so the two
# never advance the same cursor concurrently — symmetric with backfill_channel_now.
@@ -294,7 +294,7 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) ->
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
if quota.backfill_reserve_reached():
return
try:
# Attribute the spend to the requesting user under the full-backfill action, so it
+2 -4
View File
@@ -168,9 +168,8 @@ def backfill_channel_deep(
return 0
page_token = channel.backfill_cursor
total = 0
reserve = sysconfig.get_int(db, "backfill_quota_reserve")
for _ in range(max_pages):
if quota.remaining_today(db) <= reserve:
if quota.backfill_reserve_reached():
break
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
rows = [
@@ -183,8 +182,7 @@ def backfill_channel_deep(
total += insert_stubs(db, rows)
page_token = data.get("nextPageToken")
channel.backfill_cursor = page_token
if not page_token:
channel.backfill_done = True
channel.backfill_done = not page_token
db.add(channel)
db.commit()
if not page_token: