release: 0.65.1 (fix/backfill-immediate-full-history)

VERSION

@@ -1 +1 @@
-0.65.0
+0.65.1

backend/app/quota.py

@@ -118,6 +118,20 @@ def can_spend(db: Session, units: int) -> bool:
     return remaining_today(db) >= units
 
 
+def backfill_reserve_reached() -> bool:
+    """True when today's remaining budget is at or below the backfill reserve — the shared yield
+    condition for every backfill/explore path (recent, deep, immediate full-history, explore, and
+    rolling revalidation).
+
+    Reads in its OWN short-lived session (like record_usage / log_action), for two reasons: it
+    replaces the remaining/reserve comparison that was open-coded at every call site, and — critically
+    for the per-page guards in paging loops — a guard must not hold the caller's transaction open
+    across the network fetch that follows it. Quota is committed independently of any caller, so a
+    fresh session reads the same numbers."""
+    with SessionLocal() as qs:
+        return remaining_today(qs) <= sysconfig.get_int(qs, "backfill_quota_reserve")
+
+
 def log_action(user_id: int, action: str) -> None:
     """Record a zero-cost action event so per-user daily caps still count it.
 

backend/app/routes/channels.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel
 from sqlalchemy import func, select, update
 from sqlalchemy.orm import Session
 
-from app import quota, sysconfig
+from app import quota
 from app.auth import current_user, require_human, require_write_scope
 from app.db import get_db
 from app.models import (
@@ -358,7 +358,7 @@ def explore_channel(
         raise HTTPException(status_code=404, detail="Unknown channel")
     if _is_blocked(db, user, channel_id):
         raise HTTPException(status_code=403, detail="You've blocked this channel.")
-    if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
+    if quota.backfill_reserve_reached():
         raise HTTPException(
             status_code=429,
             detail="The shared daily quota reserve is reached — try exploring later.",

backend/app/sync/maintenance.py

@@ -25,7 +25,7 @@ from datetime import timedelta
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from app import progress, quota, sysconfig
+from app import progress, quota
 from app.config import settings
 from app.models import Playlist, PlaylistItem, Video
 from app.notifications import create_notification
@@ -188,7 +188,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
     now = _now()
     # Process in videos.list-sized pages so we commit incrementally and stop on low quota.
     while checked < batch:
-        if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
+        if quota.backfill_reserve_reached():
             break
         page = (
             db.execute(
@@ -228,7 +228,7 @@ def run_maintenance(db: Session) -> dict:
         phase1 = _recheck_flagged(db, yt)
         phase2 = (
             _revalidate_rolling(db, yt)
-            if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
+            if not quota.backfill_reserve_reached()
             else {"checked": 0, "flagged": 0, "skipped": "low quota"}
         )
     return {

backend/app/sync/runner.py

@@ -183,7 +183,7 @@ def run_recent_backfill(
     videos_added = 0
     with YouTubeClient(db, user) as yt:
         for channel in channels:
-            if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
+            if quota.backfill_reserve_reached():
                 break
             try:
                 videos_added += backfill_channel_recent(db, yt, channel)
@@ -229,7 +229,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
     videos_added = 0
     with YouTubeClient(db, user) as yt:
         for channel in channels:
-            if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
+            if quota.backfill_reserve_reached():
                 break
             # Claim the channel (or skip it if an interactive backfill already owns it) so the two
             # never advance the same cursor concurrently — symmetric with backfill_channel_now.
@@ -267,14 +267,19 @@ _backfill_lock = threading.Lock()
 _MAX_CONCURRENT_IMMEDIATE = 2
 
 
-def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 60) -> None:
+def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None:
     """User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
     background task with its OWN session, so the full catalogue lands in seconds instead of over the
-    paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded, capped to
-    _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the shared backfill budget by
-    opting many channels in at once (the overflow, and any very large channel's tail, is finished by
-    the scheduler since deep_requested stays set), and mutually excluded per-channel with the scheduler
-    via _backfill_in_progress. Best-effort — logs and swallows failures."""
+    paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish
+    any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69
+    pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in
+    over the scheduler's 10-minute ticks and looked stuck). backfill_channel_deep re-checks the shared
+    budget before each page and yields at the reserve, so a high cap still can't drain the day's quota.
+    Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the
+    shared backfill budget by opting many channels in at once (the overflow, and any channel too large
+    to finish under the reserve, is finished by the scheduler since deep_requested stays set), and
+    mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and
+    swallows failures."""
     global _immediate_running
     with _backfill_lock:
         if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE:
@@ -289,7 +294,7 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 60) ->
             channel = db.get(Channel, channel_id)
             if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
                 return
-            if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
+            if quota.backfill_reserve_reached():
                 return
             try:
                 # Attribute the spend to the requesting user under the full-backfill action, so it

backend/app/sync/videos.py

@@ -10,7 +10,7 @@ from sqlalchemy import and_, func, or_, select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
 
-from app import progress, sysconfig
+from app import progress, quota, sysconfig
 from app.models import LIVE_OR_UPCOMING, Channel, Video
 from app.titles import normalize_title
 from app.youtube.client import YouTubeClient, best_thumbnail
@@ -156,12 +156,21 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
 def backfill_channel_deep(
     db: Session, yt: YouTubeClient, channel: Channel, max_pages: int = 5
 ) -> int:
-    """Continue paging older uploads from the stored cursor (bounded per run)."""
+    """Continue paging older uploads from the stored cursor (bounded per run).
+
+    Commits the advanced cursor after every page (not just at the end) so progress
+    is durable: a restart resumes from the last stored page instead of re-fetching —
+    and re-charging quota for — pages already ingested. Re-checks the shared budget
+    before each page and yields once it hits the backfill reserve, so a single large
+    channel can't drain the day's quota in one run even with a high max_pages; the
+    paced scheduler finishes the tail on later ticks (deep_requested stays set)."""
     if channel.backfill_done or not channel.uploads_playlist_id:
         return 0
     page_token = channel.backfill_cursor
     total = 0
     for _ in range(max_pages):
+        if quota.backfill_reserve_reached():
+            break
         data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
         rows = [
             s
@@ -173,11 +182,11 @@ def backfill_channel_deep(
         total += insert_stubs(db, rows)
         page_token = data.get("nextPageToken")
         channel.backfill_cursor = page_token
+        channel.backfill_done = not page_token
+        db.add(channel)
+        db.commit()
         if not page_token:
-            channel.backfill_done = True
             break
-    db.add(channel)
-    db.commit()
     return total
 
 

backend/tests/test_channel_backfill.py

@@ -3,6 +3,7 @@ the update_channel -> immediate-backfill wiring (incl. the demo guard), and the
 "get full history" guard that keeps it from racing the scheduler's cursor or draining the budget."""
 from datetime import datetime, timezone
 
+import pytest
 from fastapi.testclient import TestClient
 
 import app.routes.channels as channels_route
@@ -11,7 +12,7 @@ from app.auth import current_user
 from app.db import get_db
 from app.main import app
 from app.models import Channel, Subscription, User, Video
-from app.sync import runner
+from app.sync import runner, videos
 
 _PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
 
@@ -176,3 +177,142 @@ class _NullYt:
 
     def __exit__(self, *a):
         return False
+
+
+def _playlist_page(ids, next_token=None):
+    """A playlistItems.list response shaped for stub_from_playlist_item."""
+    page = {
+        "items": [
+            {
+                "contentDetails": {"videoId": vid, "videoPublishedAt": "2020-01-01T00:00:00Z"},
+                "snippet": {"title": vid},
+            }
+            for vid in ids
+        ]
+    }
+    if next_token is not None:
+        page["nextPageToken"] = next_token
+    return page
+
+
+class _PagedYt:
+    """Serves a fixed page-token -> page map and records the tokens it was asked for, so a test can
+    assert exactly how many pages a deep-backfill run fetched (and where it resumed from)."""
+
+    def __init__(self, pages):
+        self.pages = pages
+        self.calls: list[str | None] = []
+
+    def get_playlist_items(self, playlist_id, page_token=None):
+        self.calls.append(page_token)
+        return self.pages[page_token]
+
+
+class _RaisingYt:
+    """Serves pages like _PagedYt but raises when asked for `raise_on_token`, to simulate a
+    network/API failure that lands BETWEEN pages of a deep backfill."""
+
+    def __init__(self, pages, raise_on_token):
+        self.pages = pages
+        self.raise_on_token = raise_on_token
+        self.calls: list[str | None] = []
+
+    def get_playlist_items(self, playlist_id, page_token=None):
+        self.calls.append(page_token)
+        if page_token == self.raise_on_token:
+            raise RuntimeError("simulated mid-backfill failure")
+        return self.pages[page_token]
+
+
+class TestDeepBackfillPaging:
+    """backfill_channel_deep must commit the advanced cursor after every page (durable resume, no
+    re-charging quota for pages already stored) and yield the moment the shared budget hits the
+    backfill reserve — the two properties that make a high immediate max_pages safe."""
+
+    @staticmethod
+    def _channel(db):
+        ch = Channel(
+            id="UC_deep",
+            title="Deep",
+            recent_synced_at=_PUB,
+            uploads_playlist_id="UU_deep",
+            backfill_done=False,
+            backfill_cursor=None,
+        )
+        db.add(ch)
+        db.commit()
+        return ch
+
+    def test_cursor_is_committed_each_page_and_a_later_run_resumes(self, db, monkeypatch):
+        monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False)  # budget ok
+        ch = self._channel(db)
+        yt = _PagedYt(
+            {
+                None: _playlist_page(["a1", "a2"], next_token="t1"),
+                "t1": _playlist_page(["b1", "b2"], next_token="t2"),
+                "t2": _playlist_page(["c1"], next_token=None),  # last page: no nextPageToken
+            }
+        )
+
+        # First run is bounded to 2 pages: it must stop mid-catalogue with the cursor durably parked
+        # on the next unfetched page and backfill_done still False.
+        added = videos.backfill_channel_deep(db, yt, ch, max_pages=2)
+        assert added == 4
+        assert yt.calls == [None, "t1"]
+        assert ch.backfill_cursor == "t2"
+        assert ch.backfill_done is False
+        # Durably persisted (committed per page), not just set on the in-memory object.
+        db.expire(ch)
+        assert ch.backfill_cursor == "t2"
+        assert db.query(Video).filter(Video.channel_id == ch.id).count() == 4
+
+        # A later run resumes FROM the stored cursor (not from the start — no wasted re-fetch) and
+        # finishes: reaching the token-less last page flips backfill_done and clears the cursor.
+        added2 = videos.backfill_channel_deep(db, yt, ch, max_pages=5)
+        assert added2 == 1
+        assert yt.calls == [None, "t1", "t2"]  # resumed at "t2", did not re-fetch None/"t1"
+        assert ch.backfill_done is True
+        assert ch.backfill_cursor is None
+
+    def test_yields_when_the_shared_budget_hits_the_reserve(self, db, monkeypatch):
+        # Budget is fine for the first page, then hits the reserve — the guard is checked BEFORE each
+        # page, so the run fetches exactly one page and then yields, leaving the tail for the paced
+        # scheduler (cursor parked, backfill_done False).
+        reached = iter([False, True])
+        monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: next(reached))
+        ch = self._channel(db)
+        yt = _PagedYt(
+            {
+                None: _playlist_page(["a1", "a2"], next_token="t1"),
+                "t1": _playlist_page(["b1"], next_token="t2"),
+            }
+        )
+
+        added = videos.backfill_channel_deep(db, yt, ch, max_pages=50)
+        assert added == 2
+        assert yt.calls == [None]  # stopped before fetching the second page
+        assert ch.backfill_cursor == "t1"
+        assert ch.backfill_done is False
+
+    def test_a_mid_run_failure_leaves_the_prior_page_durably_committed(self, db, monkeypatch):
+        # The whole point of committing per page (vs one commit after the loop): if paging blows up
+        # mid-run, the pages already ingested stay durable, so the next run resumes from the stored
+        # cursor instead of re-fetching — and re-charging quota for — them. A single end-of-loop
+        # commit would lose the advanced cursor on the crash (it never reaches the post-loop commit),
+        # so this asserts the property the per-page commit actually buys.
+        monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False)  # budget ok
+        ch = self._channel(db)
+        yt = _RaisingYt(
+            {None: _playlist_page(["a1", "a2"], next_token="t1")},
+            raise_on_token="t1",  # page 1 succeeds; fetching page 2 (token 't1') fails
+        )
+
+        with pytest.raises(RuntimeError):
+            videos.backfill_channel_deep(db, yt, ch, max_pages=5)
+
+        # Page 1's cursor AND rows were committed before the page-2 crash — re-read from the DB
+        # (expire) to prove it's persisted, not just set on the in-memory object.
+        db.expire(ch)
+        assert ch.backfill_cursor == "t1"
+        assert ch.backfill_done is False
+        assert db.query(Video).filter(Video.channel_id == ch.id).count() == 2

backend/tests/test_quota_ownership.py

@@ -55,3 +55,15 @@ def test_log_action_survives_caller_rollback(db):
     assert events[0].units == 0
     # ... and the caller's uncommitted user is gone.
     assert db.scalar(select(func.count()).select_from(User)) == 1
+
+
+def test_backfill_reserve_reached_tracks_the_boundary(db):
+    # The shared yield gate for every backfill/explore/revalidation path: True once today's remaining
+    # budget is at or below the backfill reserve. It reads in its OWN session, so it must reflect
+    # record_usage's independently-committed spend — and the boundary is inclusive (<=). Defaults:
+    # quota_daily_budget 9000, backfill_quota_reserve 2000 -> reached once 7000 units are used.
+    assert quota.backfill_reserve_reached() is False  # 0 used, remaining 9000
+    quota.record_usage(6999)                          # remaining 2001, just above the reserve
+    assert quota.backfill_reserve_reached() is False
+    quota.record_usage(1)                             # remaining 2000, exactly at the reserve
+    assert quota.backfill_reserve_reached() is True   # boundary is <=, so the reserve is reached

frontend/src/components/ChannelPage.tsx

@@ -422,13 +422,18 @@ export default function ChannelPage({
                 </div>
               )}
               {/* Only the recent subset is stored → offer to pull the channel's full back-catalogue.
-                Once queued (deep_requested) it shows as loading; hidden once the backfill is done. */}
+                Once queued (deep_requested) it shows as loading; hidden once the backfill is done.
+                While loading, the label carries the live stored/total count (the detail poll refreshes
+                it) so a large channel's paced tail reads as visible progress rather than a stuck
+                spinner. */}
               {ch?.subscribed && !ch.backfill_done && !me.is_demo && (
                 <button
                   onClick={() => !ch.deep_requested && deepBackfill.mutate()}
                   disabled={ch.deep_requested || deepBackfill.isPending}
                   title={
-                    ch.deep_requested ? t("channel.fullHistoryLoading") : t("channel.fullHistory")
+                    ch.deep_requested
+                      ? t("channel.fullHistoryLoadingHint")
+                      : t("channel.fullHistory")
                   }
                   className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm border border-border text-muted hover:text-accent hover:border-accent disabled:opacity-60 disabled:hover:text-muted disabled:hover:border-border transition"
                 >
@@ -438,7 +443,14 @@ export default function ChannelPage({
                     <History className="w-4 h-4" />
                   )}
                   <span className="hidden sm:inline whitespace-nowrap">
-                    {ch.deep_requested ? t("channel.fullHistoryLoading") : t("channel.fullHistory")}
+                    {!ch.deep_requested
+                      ? t("channel.fullHistory")
+                      : ch.video_count != null && ch.stored_videos < ch.video_count
+                        ? t("channel.fullHistoryProgress", {
+                            stored: ch.stored_videos,
+                            total: ch.video_count,
+                          })
+                        : t("channel.fullHistoryLoading")}
                   </span>
                 </button>
               )}

frontend/src/i18n/locales/en/channel.json

@@ -29,6 +29,8 @@
   "showInFeed": "Show in my feed",
   "fullHistory": "Full history",
   "fullHistoryLoading": "Fetching history…",
+  "fullHistoryProgress": "Fetching history… {{stored}} / {{total}}",
+  "fullHistoryLoadingHint": "Fetching the full back-catalogue — videos appear as they're found; a very large channel's tail is finished in the background.",
   "fullHistoryQueued": "Fetching this channel's full history — new videos will appear as they're found.",
   "fullHistoryFailed": "Couldn't start the full-history fetch."
 }

frontend/src/i18n/locales/hu/channel.json

@@ -29,6 +29,8 @@
   "showInFeed": "Mutasd a hírfolyamomban",
   "fullHistory": "Teljes előzmény",
   "fullHistoryLoading": "Előzmény letöltése…",
+  "fullHistoryProgress": "Előzmény letöltése… {{stored}} / {{total}}",
+  "fullHistoryLoadingHint": "A teljes előzmény letöltése folyamatban — a videók feltűnnek, ahogy előkerülnek; egy nagyon nagy csatorna maradékát a háttérben fejezi be.",
   "fullHistoryQueued": "A csatorna teljes előzményének letöltése elindult — az új videók feltűnnek, ahogy előkerülnek.",
   "fullHistoryFailed": "Nem sikerült elindítani a teljes előzmény letöltését."
 }

frontend/src/lib/releaseNotes.ts

@@ -14,6 +14,18 @@ export interface ReleaseEntry {
 }
 
 export const RELEASE_NOTES: ReleaseEntry[] = [
+  {
+    version: "0.65.1",
+    date: "2026-08-16",
+    summary:
+      "“Get full history” now finishes very large channels in one pass instead of appearing to stall.",
+    fixes: [
+      "Asking for a channel's full history now completes very large channels (thousands of videos) in a single pass. Previously a channel just over ~3,000 videos would look stuck — the button kept spinning while the rest of the catalogue trickled in over the background schedule for many minutes. The button now also shows a live “stored / total” count while it fills.",
+    ],
+    chores: [
+      "Backend hardening: the full-history backfill now saves its progress after every page (so an interrupted run resumes where it left off instead of re-fetching and re-spending API quota), and the shared daily-quota guard was centralized so a long run no longer holds a database connection open during each network call. Added automated tests that lock in both behaviours.",
+    ],
+  },
   {
     version: "0.65.0",
     date: "2026-08-15",