diff --git a/VERSION b/VERSION index aa4ceb30..ad4ae449 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.65.0 \ No newline at end of file +0.65.1 \ No newline at end of file diff --git a/backend/app/quota.py b/backend/app/quota.py index 9d317503..a9c85736 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -118,6 +118,20 @@ def can_spend(db: Session, units: int) -> bool: return remaining_today(db) >= units +def backfill_reserve_reached() -> bool: + """True when today's remaining budget is at or below the backfill reserve — the shared yield + condition for every backfill/explore path (recent, deep, immediate full-history, explore, and + rolling revalidation). + + Reads in its OWN short-lived session (like record_usage / log_action), for two reasons: it + replaces the remaining/reserve comparison that was open-coded at every call site, and — critically + for the per-page guards in paging loops — a guard must not hold the caller's transaction open + across the network fetch that follows it. Quota is committed independently of any caller, so a + fresh session reads the same numbers.""" + with SessionLocal() as qs: + return remaining_today(qs) <= sysconfig.get_int(qs, "backfill_quota_reserve") + + def log_action(user_id: int, action: str) -> None: """Record a zero-cost action event so per-user daily caps still count it. diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index 70e62c69..7b81fb49 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -9,7 +9,7 @@ from pydantic import BaseModel from sqlalchemy import func, select, update from sqlalchemy.orm import Session -from app import quota, sysconfig +from app import quota from app.auth import current_user, require_human, require_write_scope from app.db import get_db from app.models import ( @@ -358,7 +358,7 @@ def explore_channel( raise HTTPException(status_code=404, detail="Unknown channel") if _is_blocked(db, user, channel_id): raise HTTPException(status_code=403, detail="You've blocked this channel.") - if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + if quota.backfill_reserve_reached(): raise HTTPException( status_code=429, detail="The shared daily quota reserve is reached — try exploring later.", diff --git a/backend/app/sync/maintenance.py b/backend/app/sync/maintenance.py index 1d350032..1eb93302 100644 --- a/backend/app/sync/maintenance.py +++ b/backend/app/sync/maintenance.py @@ -25,7 +25,7 @@ from datetime import timedelta from sqlalchemy import select from sqlalchemy.orm import Session -from app import progress, quota, sysconfig +from app import progress, quota from app.config import settings from app.models import Playlist, PlaylistItem, Video from app.notifications import create_notification @@ -188,7 +188,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict: now = _now() # Process in videos.list-sized pages so we commit incrementally and stop on low quota. while checked < batch: - if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + if quota.backfill_reserve_reached(): break page = ( db.execute( @@ -228,7 +228,7 @@ def run_maintenance(db: Session) -> dict: phase1 = _recheck_flagged(db, yt) phase2 = ( _revalidate_rolling(db, yt) - if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve") + if not quota.backfill_reserve_reached() else {"checked": 0, "flagged": 0, "skipped": "low quota"} ) return { diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index 8221a5fd..e204ee7a 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -183,7 +183,7 @@ def run_recent_backfill( videos_added = 0 with YouTubeClient(db, user) as yt: for channel in channels: - if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + if quota.backfill_reserve_reached(): break try: videos_added += backfill_channel_recent(db, yt, channel) @@ -229,7 +229,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - videos_added = 0 with YouTubeClient(db, user) as yt: for channel in channels: - if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + if quota.backfill_reserve_reached(): break # Claim the channel (or skip it if an interactive backfill already owns it) so the two # never advance the same cursor concurrently — symmetric with backfill_channel_now. @@ -267,14 +267,19 @@ _backfill_lock = threading.Lock() _MAX_CONCURRENT_IMMEDIATE = 2 -def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 60) -> None: +def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None: """User-initiated "get full history": deep-backfill ONE channel to completion right now, in a background task with its OWN session, so the full catalogue lands in seconds instead of over the - paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded, capped to - _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the shared backfill budget by - opting many channels in at once (the overflow, and any very large channel's tail, is finished by - the scheduler since deep_requested stays set), and mutually excluded per-channel with the scheduler - via _backfill_in_progress. Best-effort — logs and swallows failures.""" + paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish + any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69 + pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in + over the scheduler's 10-minute ticks and looked stuck). backfill_channel_deep re-checks the shared + budget before each page and yields at the reserve, so a high cap still can't drain the day's quota. + Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the + shared backfill budget by opting many channels in at once (the overflow, and any channel too large + to finish under the reserve, is finished by the scheduler since deep_requested stays set), and + mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and + swallows failures.""" global _immediate_running with _backfill_lock: if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE: @@ -289,7 +294,7 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 60) -> channel = db.get(Channel, channel_id) if channel is None or channel.backfill_done or not channel.uploads_playlist_id: return - if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + if quota.backfill_reserve_reached(): return try: # Attribute the spend to the requesting user under the full-backfill action, so it diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index 3b2f605f..c520c37b 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -10,7 +10,7 @@ from sqlalchemy import and_, func, or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session -from app import progress, sysconfig +from app import progress, quota, sysconfig from app.models import LIVE_OR_UPCOMING, Channel, Video from app.titles import normalize_title from app.youtube.client import YouTubeClient, best_thumbnail @@ -156,12 +156,21 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> def backfill_channel_deep( db: Session, yt: YouTubeClient, channel: Channel, max_pages: int = 5 ) -> int: - """Continue paging older uploads from the stored cursor (bounded per run).""" + """Continue paging older uploads from the stored cursor (bounded per run). + + Commits the advanced cursor after every page (not just at the end) so progress + is durable: a restart resumes from the last stored page instead of re-fetching — + and re-charging quota for — pages already ingested. Re-checks the shared budget + before each page and yields once it hits the backfill reserve, so a single large + channel can't drain the day's quota in one run even with a high max_pages; the + paced scheduler finishes the tail on later ticks (deep_requested stays set).""" if channel.backfill_done or not channel.uploads_playlist_id: return 0 page_token = channel.backfill_cursor total = 0 for _ in range(max_pages): + if quota.backfill_reserve_reached(): + break data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) rows = [ s @@ -173,11 +182,11 @@ def backfill_channel_deep( total += insert_stubs(db, rows) page_token = data.get("nextPageToken") channel.backfill_cursor = page_token + channel.backfill_done = not page_token + db.add(channel) + db.commit() if not page_token: - channel.backfill_done = True break - db.add(channel) - db.commit() return total diff --git a/backend/tests/test_channel_backfill.py b/backend/tests/test_channel_backfill.py index 8da7c72d..b32ccb33 100644 --- a/backend/tests/test_channel_backfill.py +++ b/backend/tests/test_channel_backfill.py @@ -3,6 +3,7 @@ the update_channel -> immediate-backfill wiring (incl. the demo guard), and the "get full history" guard that keeps it from racing the scheduler's cursor or draining the budget.""" from datetime import datetime, timezone +import pytest from fastapi.testclient import TestClient import app.routes.channels as channels_route @@ -11,7 +12,7 @@ from app.auth import current_user from app.db import get_db from app.main import app from app.models import Channel, Subscription, User, Video -from app.sync import runner +from app.sync import runner, videos _PUB = datetime(2020, 1, 1, tzinfo=timezone.utc) @@ -176,3 +177,142 @@ class _NullYt: def __exit__(self, *a): return False + + +def _playlist_page(ids, next_token=None): + """A playlistItems.list response shaped for stub_from_playlist_item.""" + page = { + "items": [ + { + "contentDetails": {"videoId": vid, "videoPublishedAt": "2020-01-01T00:00:00Z"}, + "snippet": {"title": vid}, + } + for vid in ids + ] + } + if next_token is not None: + page["nextPageToken"] = next_token + return page + + +class _PagedYt: + """Serves a fixed page-token -> page map and records the tokens it was asked for, so a test can + assert exactly how many pages a deep-backfill run fetched (and where it resumed from).""" + + def __init__(self, pages): + self.pages = pages + self.calls: list[str | None] = [] + + def get_playlist_items(self, playlist_id, page_token=None): + self.calls.append(page_token) + return self.pages[page_token] + + +class _RaisingYt: + """Serves pages like _PagedYt but raises when asked for `raise_on_token`, to simulate a + network/API failure that lands BETWEEN pages of a deep backfill.""" + + def __init__(self, pages, raise_on_token): + self.pages = pages + self.raise_on_token = raise_on_token + self.calls: list[str | None] = [] + + def get_playlist_items(self, playlist_id, page_token=None): + self.calls.append(page_token) + if page_token == self.raise_on_token: + raise RuntimeError("simulated mid-backfill failure") + return self.pages[page_token] + + +class TestDeepBackfillPaging: + """backfill_channel_deep must commit the advanced cursor after every page (durable resume, no + re-charging quota for pages already stored) and yield the moment the shared budget hits the + backfill reserve — the two properties that make a high immediate max_pages safe.""" + + @staticmethod + def _channel(db): + ch = Channel( + id="UC_deep", + title="Deep", + recent_synced_at=_PUB, + uploads_playlist_id="UU_deep", + backfill_done=False, + backfill_cursor=None, + ) + db.add(ch) + db.commit() + return ch + + def test_cursor_is_committed_each_page_and_a_later_run_resumes(self, db, monkeypatch): + monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False) # budget ok + ch = self._channel(db) + yt = _PagedYt( + { + None: _playlist_page(["a1", "a2"], next_token="t1"), + "t1": _playlist_page(["b1", "b2"], next_token="t2"), + "t2": _playlist_page(["c1"], next_token=None), # last page: no nextPageToken + } + ) + + # First run is bounded to 2 pages: it must stop mid-catalogue with the cursor durably parked + # on the next unfetched page and backfill_done still False. + added = videos.backfill_channel_deep(db, yt, ch, max_pages=2) + assert added == 4 + assert yt.calls == [None, "t1"] + assert ch.backfill_cursor == "t2" + assert ch.backfill_done is False + # Durably persisted (committed per page), not just set on the in-memory object. + db.expire(ch) + assert ch.backfill_cursor == "t2" + assert db.query(Video).filter(Video.channel_id == ch.id).count() == 4 + + # A later run resumes FROM the stored cursor (not from the start — no wasted re-fetch) and + # finishes: reaching the token-less last page flips backfill_done and clears the cursor. + added2 = videos.backfill_channel_deep(db, yt, ch, max_pages=5) + assert added2 == 1 + assert yt.calls == [None, "t1", "t2"] # resumed at "t2", did not re-fetch None/"t1" + assert ch.backfill_done is True + assert ch.backfill_cursor is None + + def test_yields_when_the_shared_budget_hits_the_reserve(self, db, monkeypatch): + # Budget is fine for the first page, then hits the reserve — the guard is checked BEFORE each + # page, so the run fetches exactly one page and then yields, leaving the tail for the paced + # scheduler (cursor parked, backfill_done False). + reached = iter([False, True]) + monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: next(reached)) + ch = self._channel(db) + yt = _PagedYt( + { + None: _playlist_page(["a1", "a2"], next_token="t1"), + "t1": _playlist_page(["b1"], next_token="t2"), + } + ) + + added = videos.backfill_channel_deep(db, yt, ch, max_pages=50) + assert added == 2 + assert yt.calls == [None] # stopped before fetching the second page + assert ch.backfill_cursor == "t1" + assert ch.backfill_done is False + + def test_a_mid_run_failure_leaves_the_prior_page_durably_committed(self, db, monkeypatch): + # The whole point of committing per page (vs one commit after the loop): if paging blows up + # mid-run, the pages already ingested stay durable, so the next run resumes from the stored + # cursor instead of re-fetching — and re-charging quota for — them. A single end-of-loop + # commit would lose the advanced cursor on the crash (it never reaches the post-loop commit), + # so this asserts the property the per-page commit actually buys. + monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False) # budget ok + ch = self._channel(db) + yt = _RaisingYt( + {None: _playlist_page(["a1", "a2"], next_token="t1")}, + raise_on_token="t1", # page 1 succeeds; fetching page 2 (token 't1') fails + ) + + with pytest.raises(RuntimeError): + videos.backfill_channel_deep(db, yt, ch, max_pages=5) + + # Page 1's cursor AND rows were committed before the page-2 crash — re-read from the DB + # (expire) to prove it's persisted, not just set on the in-memory object. + db.expire(ch) + assert ch.backfill_cursor == "t1" + assert ch.backfill_done is False + assert db.query(Video).filter(Video.channel_id == ch.id).count() == 2 diff --git a/backend/tests/test_quota_ownership.py b/backend/tests/test_quota_ownership.py index fd7138ef..2153ec4d 100644 --- a/backend/tests/test_quota_ownership.py +++ b/backend/tests/test_quota_ownership.py @@ -55,3 +55,15 @@ def test_log_action_survives_caller_rollback(db): assert events[0].units == 0 # ... and the caller's uncommitted user is gone. assert db.scalar(select(func.count()).select_from(User)) == 1 + + +def test_backfill_reserve_reached_tracks_the_boundary(db): + # The shared yield gate for every backfill/explore/revalidation path: True once today's remaining + # budget is at or below the backfill reserve. It reads in its OWN session, so it must reflect + # record_usage's independently-committed spend — and the boundary is inclusive (<=). Defaults: + # quota_daily_budget 9000, backfill_quota_reserve 2000 -> reached once 7000 units are used. + assert quota.backfill_reserve_reached() is False # 0 used, remaining 9000 + quota.record_usage(6999) # remaining 2001, just above the reserve + assert quota.backfill_reserve_reached() is False + quota.record_usage(1) # remaining 2000, exactly at the reserve + assert quota.backfill_reserve_reached() is True # boundary is <=, so the reserve is reached diff --git a/frontend/src/components/ChannelPage.tsx b/frontend/src/components/ChannelPage.tsx index ca420456..b3516082 100644 --- a/frontend/src/components/ChannelPage.tsx +++ b/frontend/src/components/ChannelPage.tsx @@ -422,13 +422,18 @@ export default function ChannelPage({ )} {/* Only the recent subset is stored → offer to pull the channel's full back-catalogue. - Once queued (deep_requested) it shows as loading; hidden once the backfill is done. */} + Once queued (deep_requested) it shows as loading; hidden once the backfill is done. + While loading, the label carries the live stored/total count (the detail poll refreshes + it) so a large channel's paced tail reads as visible progress rather than a stuck + spinner. */} {ch?.subscribed && !ch.backfill_done && !me.is_demo && ( )} diff --git a/frontend/src/i18n/locales/en/channel.json b/frontend/src/i18n/locales/en/channel.json index b6bab16a..56f1ec5f 100644 --- a/frontend/src/i18n/locales/en/channel.json +++ b/frontend/src/i18n/locales/en/channel.json @@ -29,6 +29,8 @@ "showInFeed": "Show in my feed", "fullHistory": "Full history", "fullHistoryLoading": "Fetching history…", + "fullHistoryProgress": "Fetching history… {{stored}} / {{total}}", + "fullHistoryLoadingHint": "Fetching the full back-catalogue — videos appear as they're found; a very large channel's tail is finished in the background.", "fullHistoryQueued": "Fetching this channel's full history — new videos will appear as they're found.", "fullHistoryFailed": "Couldn't start the full-history fetch." } diff --git a/frontend/src/i18n/locales/hu/channel.json b/frontend/src/i18n/locales/hu/channel.json index ddaee097..814903b6 100644 --- a/frontend/src/i18n/locales/hu/channel.json +++ b/frontend/src/i18n/locales/hu/channel.json @@ -29,6 +29,8 @@ "showInFeed": "Mutasd a hírfolyamomban", "fullHistory": "Teljes előzmény", "fullHistoryLoading": "Előzmény letöltése…", + "fullHistoryProgress": "Előzmény letöltése… {{stored}} / {{total}}", + "fullHistoryLoadingHint": "A teljes előzmény letöltése folyamatban — a videók feltűnnek, ahogy előkerülnek; egy nagyon nagy csatorna maradékát a háttérben fejezi be.", "fullHistoryQueued": "A csatorna teljes előzményének letöltése elindult — az új videók feltűnnek, ahogy előkerülnek.", "fullHistoryFailed": "Nem sikerült elindítani a teljes előzmény letöltését." } diff --git a/frontend/src/lib/releaseNotes.ts b/frontend/src/lib/releaseNotes.ts index 761ce00f..f7428d84 100644 --- a/frontend/src/lib/releaseNotes.ts +++ b/frontend/src/lib/releaseNotes.ts @@ -14,6 +14,18 @@ export interface ReleaseEntry { } export const RELEASE_NOTES: ReleaseEntry[] = [ + { + version: "0.65.1", + date: "2026-08-16", + summary: + "“Get full history” now finishes very large channels in one pass instead of appearing to stall.", + fixes: [ + "Asking for a channel's full history now completes very large channels (thousands of videos) in a single pass. Previously a channel just over ~3,000 videos would look stuck — the button kept spinning while the rest of the catalogue trickled in over the background schedule for many minutes. The button now also shows a live “stored / total” count while it fills.", + ], + chores: [ + "Backend hardening: the full-history backfill now saves its progress after every page (so an interrupted run resumes where it left off instead of re-fetching and re-spending API quota), and the shared daily-quota guard was centralized so a long run no longer holds a database connection open during each network call. Added automated tests that lock in both behaviours.", + ], + }, { version: "0.65.0", date: "2026-08-15",