feat(channel): backfill a channel's full history immediately on request
Clicking "get full history" now kicks an immediate background deep backfill of that one channel (backfill_channel_now: own session, up to ~3000 videos in one pass, quota-guarded) instead of waiting for the scheduler's paced 10-minute ticks (5 pages/channel/tick). The request returns instantly and the channel page's live poll shows the count fill in seconds. A very large channel's tail still falls to the scheduler (deep_requested stays set).
This commit is contained in:
@@ -4,7 +4,7 @@ hidden, tags) and per-channel sync state. Read + local-write only — YouTube-si
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -28,7 +28,7 @@ from app.models import (
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes.admin import admin_user
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import run_recent_backfill
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
@@ -449,6 +449,7 @@ class ChannelUpdateIn(BaseModel):
|
||||
def update_channel(
|
||||
channel_id: str,
|
||||
body: ChannelUpdateIn,
|
||||
background: BackgroundTasks,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -473,12 +474,18 @@ def update_channel(
|
||||
|
||||
# Opting a channel into full history should give an immediate feed: if we haven't even
|
||||
# fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for
|
||||
# the scheduler. Deep paging still follows on the scheduler's next run (recent-then-deep).
|
||||
# (deep_turned_on is only ever True for a non-demo user — see the guard above.)
|
||||
# the scheduler. (deep_turned_on is only ever True for a non-demo user — see the guard above.)
|
||||
if deep_turned_on and channel is not None and channel.recent_synced_at is None:
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
|
||||
# …and kick the FULL deep backfill for this one channel immediately in the background, so the
|
||||
# whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The
|
||||
# request returns right away; the channel page's live poll shows the count climb. The scheduler
|
||||
# still finishes a very large channel's tail on its normal runs (deep_requested stays set).
|
||||
if deep_turned_on:
|
||||
background.add_task(backfill_channel_now, channel_id)
|
||||
|
||||
return {
|
||||
"id": channel_id,
|
||||
"priority": sub.priority,
|
||||
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
|
||||
from app import progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
@@ -244,6 +245,30 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
}
|
||||
|
||||
|
||||
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
|
||||
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
|
||||
background task with its OWN session, so the full catalogue lands in seconds instead of over the
|
||||
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
|
||||
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
|
||||
since deep_requested stays set. Best-effort — logs and swallows failures."""
|
||||
with SessionLocal() as db:
|
||||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
return
|
||||
try:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconcile_full_history(db)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Immediate backfill failed for channel %s", channel_id)
|
||||
|
||||
|
||||
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
||||
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
|
||||
estimated API units / the daily budget left for backfill. Deliberately approximate —
|
||||
|
||||
Reference in New Issue
Block a user