feat(channel): backfill a channel's full history immediately on request
Clicking "get full history" now kicks an immediate background deep backfill of that one channel (backfill_channel_now: own session, up to ~3000 videos in one pass, quota-guarded) instead of waiting for the scheduler's paced 10-minute ticks (5 pages/channel/tick). The request returns instantly and the channel page's live poll shows the count fill in seconds. A very large channel's tail still falls to the scheduler (deep_requested stays set).
This commit is contained in:
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
|
||||
from app import progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
@@ -244,6 +245,30 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
}
|
||||
|
||||
|
||||
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
|
||||
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
|
||||
background task with its OWN session, so the full catalogue lands in seconds instead of over the
|
||||
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
|
||||
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
|
||||
since deep_requested stays set. Best-effort — logs and swallows failures."""
|
||||
with SessionLocal() as db:
|
||||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
return
|
||||
try:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconcile_full_history(db)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Immediate backfill failed for channel %s", channel_id)
|
||||
|
||||
|
||||
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
||||
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
|
||||
estimated API units / the daily budget left for backfill. Deliberately approximate —
|
||||
|
||||
Reference in New Issue
Block a user