fix(channels): address R14 code-review findings
- list_channels no longer computes the Normal/Shorts/Live aggregation the frontend stopped reading when R14 dropped the N/S/L table column; the dead ManagedChannel.count_* fields are gone too. - Immediate "get full history" backfill now coordinates with the paced scheduler via a small in-flight set: capped at 2 concurrent + per-channel dedup, and run_deep_backfill skips a channel an immediate backfill owns — killing the cursor race and bounding the quota-drain burst, without losing the speed win. - Add test_channel_backfill.py: the channelDetail N/S/L counts + the guard's no-op / cap / scheduler-skip behaviour. - Inline the syncedChannels useMemo (its dep is a fresh array each render, so it never cached); fix the stale "~2-min ticks" comment (it's 10 minutes).
This commit is contained in:
+41
-19
@@ -5,6 +5,7 @@ user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configu
|
||||
public reads.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -230,6 +231,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
break
|
||||
if channel.id in _immediate_backfill:
|
||||
continue # an interactive backfill_channel_now is paging this one — no cursor race
|
||||
try:
|
||||
videos_added += backfill_channel_deep(db, yt, channel, max_pages)
|
||||
processed += 1
|
||||
@@ -245,28 +248,47 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
}
|
||||
|
||||
|
||||
# Channels an interactive "get full history" is backfilling RIGHT NOW (backfill_channel_now runs in a
|
||||
# FastAPI BackgroundTasks threadpool thread; the scheduler runs in its own thread — hence the lock).
|
||||
# The scheduler's deep pass skips these so the two never page the same channel's cursor concurrently,
|
||||
# and the size cap bounds how much immediate backfill one burst can kick, letting the rest fall back
|
||||
# to the scheduler's fair, quota-paced runs.
|
||||
_immediate_backfill: set[str] = set()
|
||||
_immediate_lock = threading.Lock()
|
||||
_MAX_CONCURRENT_IMMEDIATE = 2
|
||||
|
||||
|
||||
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
|
||||
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
|
||||
background task with its OWN session, so the full catalogue lands in seconds instead of over the
|
||||
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
|
||||
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
|
||||
since deep_requested stays set. Best-effort — logs and swallows failures."""
|
||||
with SessionLocal() as db:
|
||||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
return
|
||||
try:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconcile_full_history(db)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Immediate backfill failed for channel %s", channel_id)
|
||||
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded, and capped to
|
||||
_MAX_CONCURRENT_IMMEDIATE in flight so one user can't drain the shared backfill budget by opting
|
||||
many channels in at once — the overflow (and any very large channel's tail) is finished by the
|
||||
scheduler on its normal runs, since deep_requested stays set. Best-effort — logs and swallows."""
|
||||
with _immediate_lock:
|
||||
if channel_id in _immediate_backfill or len(_immediate_backfill) >= _MAX_CONCURRENT_IMMEDIATE:
|
||||
return # already running, or too many in flight — leave it to the scheduler
|
||||
_immediate_backfill.add(channel_id)
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
return
|
||||
try:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconcile_full_history(db)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Immediate backfill failed for channel %s", channel_id)
|
||||
finally:
|
||||
with _immediate_lock:
|
||||
_immediate_backfill.discard(channel_id)
|
||||
|
||||
|
||||
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user