fix(channels): address R14 code-review findings

- list_channels no longer computes the Normal/Shorts/Live aggregation the
  frontend stopped reading when R14 dropped the N/S/L table column; the dead
  ManagedChannel.count_* fields are gone too.
- Immediate "get full history" backfill now coordinates with the paced
  scheduler via a small in-flight set: capped at 2 concurrent + per-channel
  dedup, and run_deep_backfill skips a channel an immediate backfill owns —
  killing the cursor race and bounding the quota-drain burst, without losing
  the speed win.
- Add test_channel_backfill.py: the channelDetail N/S/L counts + the guard's
  no-op / cap / scheduler-skip behaviour.
- Inline the syncedChannels useMemo (its dep is a fresh array each render, so
  it never cached); fix the stale "~2-min ticks" comment (it's 10 minutes).
This commit is contained in:
2026-08-15 06:38:19 +02:00
parent a342e15e05
commit 2cc77757d9
7 changed files with 179 additions and 42 deletions
+4 -11
View File
@@ -50,20 +50,19 @@ def list_channels(
).all()
# Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last
# upload, total duration, and a Normal/Shorts/Live breakdown. Cheap — a single grouped scan
# (~35 ms over the full 233k-row catalog on the dev DB), so computed on read rather than
# upload, total duration. Cheap — a single grouped scan, so computed on read rather than
# denormalised (measure-first: reads aren't frequent/hot enough to warrant cached columns).
# (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer
# computed here — the manager table dropped its N/S/L column.)
channel_ids = [c.id for _, c in rows]
agg: dict[str, dict] = {}
if channel_ids:
for cid, n, last_up, total_dur, shorts, live in db.execute(
for cid, n, last_up, total_dur in db.execute(
select(
Video.channel_id,
func.count(Video.id),
func.max(Video.published_at),
func.coalesce(func.sum(Video.duration_seconds), 0),
func.count(Video.id).filter(Video.is_short.is_(True)),
func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
)
.where(Video.channel_id.in_(channel_ids))
.group_by(Video.channel_id)
@@ -72,9 +71,6 @@ def list_channels(
"stored": n,
"last_video_at": iso(last_up),
"total_duration_seconds": int(total_dur or 0),
"count_short": int(shorts),
"count_live": int(live),
"count_normal": max(0, int(n) - int(shorts) - int(live)),
}
# Channels already in the *global* deep-backfill queue — i.e. at least one user (any
@@ -108,9 +104,6 @@ def list_channels(
"stored_videos": (agg.get(ch.id) or {}).get("stored", 0),
"last_video_at": (agg.get(ch.id) or {}).get("last_video_at"),
"total_duration_seconds": (agg.get(ch.id) or {}).get("total_duration_seconds", 0),
"count_normal": (agg.get(ch.id) or {}).get("count_normal", 0),
"count_short": (agg.get(ch.id) or {}).get("count_short", 0),
"count_live": (agg.get(ch.id) or {}).get("count_live", 0),
"priority": sub.priority,
"hidden": sub.hidden,
"deep_requested": sub.deep_requested,
+41 -19
View File
@@ -5,6 +5,7 @@ user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configu
public reads.
"""
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select
@@ -230,6 +231,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
for channel in channels:
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
break
if channel.id in _immediate_backfill:
continue # an interactive backfill_channel_now is paging this one — no cursor race
try:
videos_added += backfill_channel_deep(db, yt, channel, max_pages)
processed += 1
@@ -245,28 +248,47 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
}
# Channels an interactive "get full history" is backfilling RIGHT NOW (backfill_channel_now runs in a
# FastAPI BackgroundTasks threadpool thread; the scheduler runs in its own thread — hence the lock).
# The scheduler's deep pass skips these so the two never page the same channel's cursor concurrently,
# and the size cap bounds how much immediate backfill one burst can kick, letting the rest fall back
# to the scheduler's fair, quota-paced runs.
_immediate_backfill: set[str] = set()
_immediate_lock = threading.Lock()
_MAX_CONCURRENT_IMMEDIATE = 2
def backfill_channel_now(channel_id: str, max_pages: int = 60) -> None:
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
background task with its OWN session, so the full catalogue lands in seconds instead of over the
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded; a very large
channel's tail (beyond ~max_pages*50 videos) is finished by the scheduler on its normal runs,
since deep_requested stays set. Best-effort — logs and swallows failures."""
with SessionLocal() as db:
user = get_service_user(db)
if user is None:
return
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
return
try:
with YouTubeClient(db, user) as yt:
backfill_channel_deep(db, yt, channel, max_pages)
reconcile_full_history(db)
except Exception:
db.rollback()
log.exception("Immediate backfill failed for channel %s", channel_id)
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded, and capped to
_MAX_CONCURRENT_IMMEDIATE in flight so one user can't drain the shared backfill budget by opting
many channels in at once — the overflow (and any very large channel's tail) is finished by the
scheduler on its normal runs, since deep_requested stays set. Best-effort — logs and swallows."""
with _immediate_lock:
if channel_id in _immediate_backfill or len(_immediate_backfill) >= _MAX_CONCURRENT_IMMEDIATE:
return # already running, or too many in flight — leave it to the scheduler
_immediate_backfill.add(channel_id)
try:
with SessionLocal() as db:
user = get_service_user(db)
if user is None:
return
channel = db.get(Channel, channel_id)
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
return
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
return
try:
with YouTubeClient(db, user) as yt:
backfill_channel_deep(db, yt, channel, max_pages)
reconcile_full_history(db)
except Exception:
db.rollback()
log.exception("Immediate backfill failed for channel %s", channel_id)
finally:
with _immediate_lock:
_immediate_backfill.discard(channel_id)
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: