fix(backfill): finish large channels in one immediate full-history run
The user-triggered "get full history" (backfill_channel_now) capped at max_pages=60 = 3000 videos. A channel just over that (BBC Earth: 3403 uploads = 69 pages) stopped one page short, so its tail dribbled in over the scheduler's paced 10-minute ticks (250 videos/tick) and the button spinner looked stuck for ~18 minutes with no visible progress. - Raise the immediate cap to max_pages=400 (20 000 videos) so any realistic channel completes in the single background run. - Make a high cap safe: backfill_channel_deep now re-checks the shared quota before each page and yields at the backfill reserve (previously quota was checked once by the caller before the whole run), and commits the advanced cursor after every page so a restart resumes instead of re-fetching — and re-charging quota for — pages already stored. - Channel page: while loading, the button shows the live stored/total count and a tooltip explaining the tail finishes in the background, so a very large channel's paced remainder reads as progress, not a hang. Adds deep-backfill paging tests (per-page cursor commit + resume, and the per-page quota yield).
This commit is contained in:
@@ -11,7 +11,7 @@ from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.sync import runner
|
||||
from app.sync import runner, videos
|
||||
|
||||
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
@@ -176,3 +176,103 @@ class _NullYt:
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
|
||||
def _playlist_page(ids, next_token=None):
|
||||
"""A playlistItems.list response shaped for stub_from_playlist_item."""
|
||||
page = {
|
||||
"items": [
|
||||
{
|
||||
"contentDetails": {"videoId": vid, "videoPublishedAt": "2020-01-01T00:00:00Z"},
|
||||
"snippet": {"title": vid},
|
||||
}
|
||||
for vid in ids
|
||||
]
|
||||
}
|
||||
if next_token is not None:
|
||||
page["nextPageToken"] = next_token
|
||||
return page
|
||||
|
||||
|
||||
class _PagedYt:
|
||||
"""Serves a fixed page-token -> page map and records the tokens it was asked for, so a test can
|
||||
assert exactly how many pages a deep-backfill run fetched (and where it resumed from)."""
|
||||
|
||||
def __init__(self, pages):
|
||||
self.pages = pages
|
||||
self.calls: list[str | None] = []
|
||||
|
||||
def get_playlist_items(self, playlist_id, page_token=None):
|
||||
self.calls.append(page_token)
|
||||
return self.pages[page_token]
|
||||
|
||||
|
||||
class TestDeepBackfillPaging:
|
||||
"""backfill_channel_deep must commit the advanced cursor after every page (durable resume, no
|
||||
re-charging quota for pages already stored) and yield the moment the shared budget hits the
|
||||
backfill reserve — the two properties that make a high immediate max_pages safe."""
|
||||
|
||||
@staticmethod
|
||||
def _channel(db):
|
||||
ch = Channel(
|
||||
id="UC_deep",
|
||||
title="Deep",
|
||||
recent_synced_at=_PUB,
|
||||
uploads_playlist_id="UU_deep",
|
||||
backfill_done=False,
|
||||
backfill_cursor=None,
|
||||
)
|
||||
db.add(ch)
|
||||
db.commit()
|
||||
return ch
|
||||
|
||||
def test_cursor_is_committed_each_page_and_a_later_run_resumes(self, db, monkeypatch):
|
||||
monkeypatch.setattr(videos.quota, "remaining_today", lambda _db: 10_000)
|
||||
ch = self._channel(db)
|
||||
yt = _PagedYt(
|
||||
{
|
||||
None: _playlist_page(["a1", "a2"], next_token="t1"),
|
||||
"t1": _playlist_page(["b1", "b2"], next_token="t2"),
|
||||
"t2": _playlist_page(["c1"], next_token=None), # last page: no nextPageToken
|
||||
}
|
||||
)
|
||||
|
||||
# First run is bounded to 2 pages: it must stop mid-catalogue with the cursor durably parked
|
||||
# on the next unfetched page and backfill_done still False.
|
||||
added = videos.backfill_channel_deep(db, yt, ch, max_pages=2)
|
||||
assert added == 4
|
||||
assert yt.calls == [None, "t1"]
|
||||
assert ch.backfill_cursor == "t2"
|
||||
assert ch.backfill_done is False
|
||||
# Durably persisted (committed per page), not just set on the in-memory object.
|
||||
db.expire(ch)
|
||||
assert ch.backfill_cursor == "t2"
|
||||
assert db.query(Video).filter(Video.channel_id == ch.id).count() == 4
|
||||
|
||||
# A later run resumes FROM the stored cursor (not from the start — no wasted re-fetch) and
|
||||
# finishes: reaching the token-less last page flips backfill_done and clears the cursor.
|
||||
added2 = videos.backfill_channel_deep(db, yt, ch, max_pages=5)
|
||||
assert added2 == 1
|
||||
assert yt.calls == [None, "t1", "t2"] # resumed at "t2", did not re-fetch None/"t1"
|
||||
assert ch.backfill_done is True
|
||||
assert ch.backfill_cursor is None
|
||||
|
||||
def test_yields_when_the_shared_budget_hits_the_reserve(self, db, monkeypatch):
|
||||
# Budget is above the 2000 reserve for the first page, then drops to the reserve — the guard
|
||||
# is checked BEFORE each page, so the run fetches exactly one page and then yields, leaving
|
||||
# the tail for the paced scheduler (cursor parked, backfill_done False).
|
||||
budget = iter([10_000, 2_000, 2_000])
|
||||
monkeypatch.setattr(videos.quota, "remaining_today", lambda _db: next(budget))
|
||||
ch = self._channel(db)
|
||||
yt = _PagedYt(
|
||||
{
|
||||
None: _playlist_page(["a1", "a2"], next_token="t1"),
|
||||
"t1": _playlist_page(["b1"], next_token="t2"),
|
||||
}
|
||||
)
|
||||
|
||||
added = videos.backfill_channel_deep(db, yt, ch, max_pages=50)
|
||||
assert added == 2
|
||||
assert yt.calls == [None] # stopped before fetching the second page
|
||||
assert ch.backfill_cursor == "t1"
|
||||
assert ch.backfill_done is False
|
||||
|
||||
Reference in New Issue
Block a user