refactor(backfill): centralize the reserve-yield guard and drop the per-page connection hold
Addresses the code-review findings on the immediate full-history change: - Extract quota.backfill_reserve_reached(): the remaining/reserve compare was open-coded at seven sites (recent/deep/immediate backfill, explore, rolling revalidation, and the new per-page guard). It reads in its OWN short-lived session (like record_usage/log_action), so a per-page guard no longer holds the caller's transaction open across the YouTube fetch that follows — the 400-page immediate run no longer pins a pooled connection idle-in-transaction through each network round-trip. Migrated all call sites; dropped the now-dead sysconfig imports in channels.py and maintenance.py. - Simplify the deep-backfill loop: channel.backfill_done = not page_token (done is False on entry) plus one break, replacing the doubled `if not page_token`. - Tests patch the backfill_reserve_reached seam directly instead of mocking remaining_today internals, removing the fragile dead-value budget iterator. Full backend suite green; ruff clean.
This commit is contained in:
@@ -25,7 +25,7 @@ from datetime import timedelta
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota, sysconfig
|
||||
from app import progress, quota
|
||||
from app.config import settings
|
||||
from app.models import Playlist, PlaylistItem, Video
|
||||
from app.notifications import create_notification
|
||||
@@ -188,7 +188,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
||||
now = _now()
|
||||
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
||||
while checked < batch:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
page = (
|
||||
db.execute(
|
||||
@@ -228,7 +228,7 @@ def run_maintenance(db: Session) -> dict:
|
||||
phase1 = _recheck_flagged(db, yt)
|
||||
phase2 = (
|
||||
_revalidate_rolling(db, yt)
|
||||
if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
|
||||
if not quota.backfill_reserve_reached()
|
||||
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
||||
)
|
||||
return {
|
||||
|
||||
@@ -183,7 +183,7 @@ def run_recent_backfill(
|
||||
videos_added = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
try:
|
||||
videos_added += backfill_channel_recent(db, yt, channel)
|
||||
@@ -229,7 +229,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
videos_added = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
# Claim the channel (or skip it if an interactive backfill already owns it) so the two
|
||||
# never advance the same cursor concurrently — symmetric with backfill_channel_now.
|
||||
@@ -294,7 +294,7 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) ->
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
return
|
||||
try:
|
||||
# Attribute the spend to the requesting user under the full-backfill action, so it
|
||||
|
||||
@@ -168,9 +168,8 @@ def backfill_channel_deep(
|
||||
return 0
|
||||
page_token = channel.backfill_cursor
|
||||
total = 0
|
||||
reserve = sysconfig.get_int(db, "backfill_quota_reserve")
|
||||
for _ in range(max_pages):
|
||||
if quota.remaining_today(db) <= reserve:
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
rows = [
|
||||
@@ -183,8 +182,7 @@ def backfill_channel_deep(
|
||||
total += insert_stubs(db, rows)
|
||||
page_token = data.get("nextPageToken")
|
||||
channel.backfill_cursor = page_token
|
||||
if not page_token:
|
||||
channel.backfill_done = True
|
||||
channel.backfill_done = not page_token
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
if not page_token:
|
||||
|
||||
Reference in New Issue
Block a user