feat(channel): backfill a channel's full history immediately on request

Clicking "get full history" now kicks an immediate background deep
backfill of that one channel (backfill_channel_now: own session, up to
~3000 videos in one pass, quota-guarded) instead of waiting for the
scheduler's paced 10-minute ticks (5 pages/channel/tick). The request
returns instantly and the channel page's live poll shows the count fill
in seconds. A very large channel's tail still falls to the scheduler
(deep_requested stays set).
This commit is contained in:
2026-08-15 06:01:42 +02:00
parent be738d3791
commit a342e15e05
2 changed files with 36 additions and 4 deletions
+11 -4
View File
@@ -4,7 +4,7 @@ hidden, tags) and per-channel sync state. Read + local-write only — YouTube-si
import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy import func, select, update
from sqlalchemy.orm import Session
@@ -28,7 +28,7 @@ from app.models import (
from app.routes._common import owned_or_404
from app.routes.admin import admin_user
from app.sync.explore import explore_ingest_page
from app.sync.runner import run_recent_backfill
from app.sync.runner import backfill_channel_now, run_recent_backfill
from app.sync.subscriptions import apply_channel_details
from app.utils import iso
from app.youtube.client import YouTubeClient, YouTubeError
@@ -449,6 +449,7 @@ class ChannelUpdateIn(BaseModel):
def update_channel(
channel_id: str,
body: ChannelUpdateIn,
background: BackgroundTasks,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
@@ -473,12 +474,18 @@ def update_channel(
# Opting a channel into full history should give an immediate feed: if we haven't even
# fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for
# the scheduler. Deep paging still follows on the scheduler's next run (recent-then-deep).
# (deep_turned_on is only ever True for a non-demo user — see the guard above.)
# the scheduler. (deep_turned_on is only ever True for a non-demo user — see the guard above.)
if deep_turned_on and channel is not None and channel.recent_synced_at is None:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
run_recent_backfill(db, [channel], max_channels=1)
# …and kick the FULL deep backfill for this one channel immediately in the background, so the
# whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The
# request returns right away; the channel page's live poll shows the count climb. The scheduler
# still finishes a very large channel's tail on its normal runs (deep_requested stays set).
if deep_turned_on:
background.add_task(backfill_channel_now, channel_id)
return {
"id": channel_id,
"priority": sub.priority,
+25
View File
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
from app import progress, quota, sysconfig
log = logging.getLogger("siftlode.sync")
from app.db import SessionLocal
from app.models import Channel, OAuthToken, Subscription, User
from app.sync.subscriptions import import_subscriptions
from app.sync.videos import (
@@ -244,6 +245,30 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
}
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
background task with its OWN session, so the full catalogue lands in seconds instead of over the
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
since deep_requested stays set. Best-effort — logs and swallows failures."""
with SessionLocal() as db:
user = get_service_user(db)
if user is None:
return
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
return
try:
with YouTubeClient(db, user) as yt:
backfill_channel_deep(db, yt, channel, max_pages)
reconcile_full_history(db)
except Exception:
db.rollback()
log.exception("Immediate backfill failed for channel %s", channel_id)
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
estimated API units / the daily budget left for backfill. Deliberately approximate —