feat(channel): backfill a channel's full history immediately on request

Clicking "get full history" now kicks an immediate background deep
backfill of that one channel (backfill_channel_now: own session, up to
~3000 videos in one pass, quota-guarded) instead of waiting for the
scheduler's paced 10-minute ticks (5 pages/channel/tick). The request
returns instantly and the channel page's live poll shows the count fill
in seconds. A very large channel's tail still falls to the scheduler
(deep_requested stays set).
This commit is contained in:
2026-08-15 06:01:42 +02:00
parent be738d3791
commit a342e15e05
2 changed files with 36 additions and 4 deletions
+25
View File
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
from app import progress, quota, sysconfig
log = logging.getLogger("siftlode.sync")
from app.db import SessionLocal
from app.models import Channel, OAuthToken, Subscription, User
from app.sync.subscriptions import import_subscriptions
from app.sync.videos import (
@@ -244,6 +245,30 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
}
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
background task with its OWN session, so the full catalogue lands in seconds instead of over the
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
since deep_requested stays set. Best-effort — logs and swallows failures."""
with SessionLocal() as db:
user = get_service_user(db)
if user is None:
return
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
return
try:
with YouTubeClient(db, user) as yt:
backfill_channel_deep(db, yt, channel, max_pages)
reconcile_full_history(db)
except Exception:
db.rollback()
log.exception("Immediate backfill failed for channel %s", channel_id)
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
estimated API units / the daily budget left for backfill. Deliberately approximate —