fix(channels): address R14 code-review findings

- list_channels no longer computes the Normal/Shorts/Live aggregation the
  frontend stopped reading when R14 dropped the N/S/L table column; the dead
  ManagedChannel.count_* fields are gone too.
- Immediate "get full history" backfill now coordinates with the paced
  scheduler via a small in-flight set: capped at 2 concurrent + per-channel
  dedup, and run_deep_backfill skips a channel an immediate backfill owns —
  killing the cursor race and bounding the quota-drain burst, without losing
  the speed win.
- Add test_channel_backfill.py: the channelDetail N/S/L counts + the guard's
  no-op / cap / scheduler-skip behaviour.
- Inline the syncedChannels useMemo (its dep is a fresh array each render, so
  it never cached); fix the stale "~2-min ticks" comment (it's 10 minutes).
This commit is contained in:
2026-08-15 06:38:19 +02:00
parent a342e15e05
commit 2cc77757d9
7 changed files with 179 additions and 42 deletions
+41 -19
View File
@@ -5,6 +5,7 @@ user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configu
public reads.
"""
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select
@@ -230,6 +231,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
for channel in channels:
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break
if channel.id in _immediate_backfill:
continue # an interactive backfill_channel_now is paging this one — no cursor race
try:
videos_added += backfill_channel_deep(db, yt, channel, max_pages)
processed += 1
@@ -245,28 +248,47 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
}
# Channels an interactive "get full history" is backfilling RIGHT NOW (backfill_channel_now runs in a
# FastAPI BackgroundTasks threadpool thread; the scheduler runs in its own thread — hence the lock).
# The scheduler's deep pass skips these so the two never page the same channel's cursor concurrently,
# and the size cap bounds how much immediate backfill one burst can kick, letting the rest fall back
# to the scheduler's fair, quota-paced runs.
_immediate_backfill: set[str] = set()
_immediate_lock = threading.Lock()
_MAX_CONCURRENT_IMMEDIATE = 2
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
background task with its OWN session, so the full catalogue lands in seconds instead of over the
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
since deep_requested stays set. Best-effort — logs and swallows failures."""
with SessionLocal() as db:
user = get_service_user(db)
if user is None:
return
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
return
try:
with YouTubeClient(db, user) as yt:
backfill_channel_deep(db, yt, channel, max_pages)
reconcile_full_history(db)
except Exception:
db.rollback()
log.exception("Immediate backfill failed for channel %s", channel_id)
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded, and capped to
_MAX_CONCURRENT_IMMEDIATE in flight so one user can't drain the shared backfill budget by opting
many channels in at once — the overflow (and any very large channel's tail) is finished by the
scheduler on its normal runs, since deep_requested stays set. Best-effort — logs and swallows."""
with _immediate_lock:
if channel_id in _immediate_backfill or len(_immediate_backfill) >= _MAX_CONCURRENT_IMMEDIATE:
return # already running, or too many in flight — leave it to the scheduler
_immediate_backfill.add(channel_id)
try:
with SessionLocal() as db:
user = get_service_user(db)
if user is None:
return
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
return
try:
with YouTubeClient(db, user) as yt:
backfill_channel_deep(db, yt, channel, max_pages)
reconcile_full_history(db)
except Exception:
db.rollback()
log.exception("Immediate backfill failed for channel %s", channel_id)
finally:
with _immediate_lock:
_immediate_backfill.discard(channel_id)
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: