fix(backfill): finish large channels in one immediate full-history run

The user-triggered "get full history" (backfill_channel_now) capped at
max_pages=60 = 3000 videos. A channel just over that (BBC Earth: 3403
uploads = 69 pages) stopped one page short, so its tail dribbled in over
the scheduler's paced 10-minute ticks (250 videos/tick) and the button
spinner looked stuck for ~18 minutes with no visible progress.

- Raise the immediate cap to max_pages=400 (20 000 videos) so any
  realistic channel completes in the single background run.
- Make a high cap safe: backfill_channel_deep now re-checks the shared
  quota before each page and yields at the backfill reserve (previously
  quota was checked once by the caller before the whole run), and commits
  the advanced cursor after every page so a restart resumes instead of
  re-fetching — and re-charging quota for — pages already stored.
- Channel page: while loading, the button shows the live stored/total
  count and a tooltip explaining the tail finishes in the background, so a
  very large channel's paced remainder reads as progress, not a hang.

Adds deep-backfill paging tests (per-page cursor commit + resume, and the
per-page quota yield).
This commit is contained in:
2026-08-15 17:13:52 +02:00
parent a7b4b2ed79
commit e3ee1baa94
6 changed files with 146 additions and 14 deletions
+101 -1
View File
@@ -11,7 +11,7 @@ from app.auth import current_user
from app.db import get_db
from app.main import app
from app.models import Channel, Subscription, User, Video
from app.sync import runner
from app.sync import runner, videos
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
@@ -176,3 +176,103 @@ class _NullYt:
def __exit__(self, *a):
return False
def _playlist_page(ids, next_token=None):
"""A playlistItems.list response shaped for stub_from_playlist_item."""
page = {
"items": [
{
"contentDetails": {"videoId": vid, "videoPublishedAt": "2020-01-01T00:00:00Z"},
"snippet": {"title": vid},
}
for vid in ids
]
}
if next_token is not None:
page["nextPageToken"] = next_token
return page
class _PagedYt:
"""Serves a fixed page-token -> page map and records the tokens it was asked for, so a test can
assert exactly how many pages a deep-backfill run fetched (and where it resumed from)."""
def __init__(self, pages):
self.pages = pages
self.calls: list[str | None] = []
def get_playlist_items(self, playlist_id, page_token=None):
self.calls.append(page_token)
return self.pages[page_token]
class TestDeepBackfillPaging:
"""backfill_channel_deep must commit the advanced cursor after every page (durable resume, no
re-charging quota for pages already stored) and yield the moment the shared budget hits the
backfill reserve — the two properties that make a high immediate max_pages safe."""
@staticmethod
def _channel(db):
ch = Channel(
id="UC_deep",
title="Deep",
recent_synced_at=_PUB,
uploads_playlist_id="UU_deep",
backfill_done=False,
backfill_cursor=None,
)
db.add(ch)
db.commit()
return ch
def test_cursor_is_committed_each_page_and_a_later_run_resumes(self, db, monkeypatch):
monkeypatch.setattr(videos.quota, "remaining_today", lambda _db: 10_000)
ch = self._channel(db)
yt = _PagedYt(
{
None: _playlist_page(["a1", "a2"], next_token="t1"),
"t1": _playlist_page(["b1", "b2"], next_token="t2"),
"t2": _playlist_page(["c1"], next_token=None), # last page: no nextPageToken
}
)
# First run is bounded to 2 pages: it must stop mid-catalogue with the cursor durably parked
# on the next unfetched page and backfill_done still False.
added = videos.backfill_channel_deep(db, yt, ch, max_pages=2)
assert added == 4
assert yt.calls == [None, "t1"]
assert ch.backfill_cursor == "t2"
assert ch.backfill_done is False
# Durably persisted (committed per page), not just set on the in-memory object.
db.expire(ch)
assert ch.backfill_cursor == "t2"
assert db.query(Video).filter(Video.channel_id == ch.id).count() == 4
# A later run resumes FROM the stored cursor (not from the start — no wasted re-fetch) and
# finishes: reaching the token-less last page flips backfill_done and clears the cursor.
added2 = videos.backfill_channel_deep(db, yt, ch, max_pages=5)
assert added2 == 1
assert yt.calls == [None, "t1", "t2"] # resumed at "t2", did not re-fetch None/"t1"
assert ch.backfill_done is True
assert ch.backfill_cursor is None
def test_yields_when_the_shared_budget_hits_the_reserve(self, db, monkeypatch):
# Budget is above the 2000 reserve for the first page, then drops to the reserve — the guard
# is checked BEFORE each page, so the run fetches exactly one page and then yields, leaving
# the tail for the paced scheduler (cursor parked, backfill_done False).
budget = iter([10_000, 2_000, 2_000])
monkeypatch.setattr(videos.quota, "remaining_today", lambda _db: next(budget))
ch = self._channel(db)
yt = _PagedYt(
{
None: _playlist_page(["a1", "a2"], next_token="t1"),
"t1": _playlist_page(["b1"], next_token="t2"),
}
)
added = videos.backfill_channel_deep(db, yt, ch, max_pages=50)
assert added == 2
assert yt.calls == [None] # stopped before fetching the second page
assert ch.backfill_cursor == "t1"
assert ch.backfill_done is False