diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index 5220cdfc..462c69f9 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -4,7 +4,7 @@ hidden, tags) and per-channel sync state. Read + local-write only — YouTube-si import logging from datetime import datetime, timezone -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException from pydantic import BaseModel from sqlalchemy import func, select, update from sqlalchemy.orm import Session @@ -28,7 +28,7 @@ from app.models import ( from app.routes._common import owned_or_404 from app.routes.admin import admin_user from app.sync.explore import explore_ingest_page -from app.sync.runner import run_recent_backfill +from app.sync.runner import backfill_channel_now, run_recent_backfill from app.sync.subscriptions import apply_channel_details from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError @@ -449,6 +449,7 @@ class ChannelUpdateIn(BaseModel): def update_channel( channel_id: str, body: ChannelUpdateIn, + background: BackgroundTasks, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: @@ -473,12 +474,18 @@ def update_channel( # Opting a channel into full history should give an immediate feed: if we haven't even # fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for - # the scheduler. Deep paging still follows on the scheduler's next run (recent-then-deep). - # (deep_turned_on is only ever True for a non-demo user — see the guard above.) + # the scheduler. (deep_turned_on is only ever True for a non-demo user — see the guard above.) if deep_turned_on and channel is not None and channel.recent_synced_at is None: with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT): run_recent_backfill(db, [channel], max_channels=1) + # …and kick the FULL deep backfill for this one channel immediately in the background, so the + # whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The + # request returns right away; the channel page's live poll shows the count climb. The scheduler + # still finishes a very large channel's tail on its normal runs (deep_requested stays set). + if deep_turned_on: + background.add_task(backfill_channel_now, channel_id) + return { "id": channel_id, "priority": sub.priority, diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index dca35641..ae7fe21c 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -13,6 +13,7 @@ from sqlalchemy.orm import Session from app import progress, quota, sysconfig log = logging.getLogger("siftlode.sync") +from app.db import SessionLocal from app.models import Channel, OAuthToken, Subscription, User from app.sync.subscriptions import import_subscriptions from app.sync.videos import ( @@ -244,6 +245,30 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - } +def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None: + """User-initiated "get full history": deep-backfill ONE channel to completion right now, in a + background task with its OWN session, so the full catalogue lands in seconds instead of over the + paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large + channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs, + since deep_requested stays set. Best-effort — logs and swallows failures.""" + with SessionLocal() as db: + user = get_service_user(db) + if user is None: + return + channel = db.get(Channel, channel_id) + if channel is None or channel.backfill_done or not channel.uploads_playlist_id: + return + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + return + try: + with YouTubeClient(db, user) as yt: + backfill_channel_deep(db, yt, channel, max_pages) + reconcile_full_history(db) + except Exception: + db.rollback() + log.exception("Immediate backfill failed for channel %s", channel_id) + + def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: """Rough ETA to finish full-history backfill of `channels`. Quota-bound: total estimated API units / the daily budget left for backfill. Deliberately approximate —