fix(sync): a channel with no uploads backfills as done, not as a failure

YouTube answers the uploads list of a channel with no uploads as
playlistNotFound. The recent and deep backfills raised on it, so the channel
was never stamped and was picked again, charged a quota unit and logged an
ERROR on every run. Both now read it as an empty last page, as explore already
does; any other failure still raises.
This commit is contained in:
2026-10-04 04:51:25 +02:00
parent 0314c4cce2
commit 40b27f2982
2 changed files with 68 additions and 5 deletions
+20 -4
View File
@@ -11,12 +11,14 @@ from sqlalchemy import and_, func, or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app import progress, quota, sysconfig
from app import logs, progress, quota, sysconfig
from app.models import LIVE_OR_UPCOMING, Channel, Video
from app.titles import normalize_title
from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail, is_playlist_gone
from app.youtube.shorts import make_client, probe_is_short
log = logs.get_logger(__name__)
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
@@ -104,6 +106,20 @@ def stub_from_playlist_item(item: dict, channel_id: str) -> dict | None:
}
def _uploads_page(yt: YouTubeClient, channel_id: str, uploads: str, page_token: str | None) -> dict:
"""One page of the channel's uploads. YouTube answers the uploads list of a channel with no
uploads at all as not found: an empty back-catalogue, so an empty last page — not a failure
that left `recent_synced_at` unstamped and charged the channel again on every run (E32 review).
Any other failure raises."""
try:
return yt.get_playlist_items(uploads, page_token)
except YouTubeError as exc:
if not is_playlist_gone(exc):
raise
log.info("backfill channel=%s has no uploads playlist", channel_id)
return {}
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
"""First pass: newest videos up to the configured count or age cutoff. Records a
resume cursor for the later deep backfill."""
@@ -121,7 +137,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
while True:
page_start = page_token # token that fetched THIS page (None = first page)
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
stopped_mid_page = False
for item in data.get("items", []):
stub = stub_from_playlist_item(item, channel.id)
@@ -175,7 +191,7 @@ def backfill_channel_deep(
for _ in range(max_pages):
if quota.backfill_reserve_reached():
break
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
rows = [
s
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
+48 -1
View File
@@ -8,8 +8,9 @@ import httpx
import pytest
from app.auth import has_read_scope
from app.models import OAuthToken
from app.models import Channel, OAuthToken
from app.sync.explore import explore_ingest_page
from app.sync.videos import backfill_channel_deep, backfill_channel_recent
from app.youtube import client as yt
from app.youtube.client import YouTubeError, refusal_reason
@@ -57,6 +58,52 @@ def test_a_channel_without_uploads_explores_as_an_empty_page():
assert explore_ingest_page(None, _Yt(), channel) == {"ingested": 0, "next_page_token": None}
class _NoUploads:
"""A client whose every uploads read answers "playlist not found" (a channel with no uploads)."""
def __init__(self):
self.calls = 0
def get_playlist_items(self, *_):
self.calls += 1
raise YouTubeError("m", reason="playlistNotFound", path="playlistItems")
def test_a_channel_without_uploads_backfills_as_done_and_is_not_charged_again(db):
# The recent pass stamped nothing on this answer, so the channel was picked — and charged a
# quota unit, with an ERROR traceback — on every backfill run, forever (E32 review round 3).
ch = Channel(id="UCempty", title="Empty", uploads_playlist_id="UUempty")
db.add(ch)
db.commit()
yt = _NoUploads()
assert backfill_channel_recent(db, yt, ch) == 0
db.refresh(ch)
assert ch.recent_synced_at is not None and ch.backfill_done and ch.backfill_cursor is None
assert backfill_channel_deep(db, yt, ch) == 0
assert yt.calls == 1 # the deep pass has nothing left to ask
def test_a_deep_backfill_whose_uploads_vanish_ends_instead_of_failing(db):
ch = Channel(id="UCgone", title="Gone", uploads_playlist_id="UUgone", backfill_cursor="page2")
db.add(ch)
db.commit()
assert backfill_channel_deep(db, _NoUploads(), ch) == 0
db.refresh(ch)
assert ch.backfill_done and ch.backfill_cursor is None
def test_any_other_backfill_failure_still_raises(db):
class _Yt:
def get_playlist_items(self, *_):
raise YouTubeError("m", reason="quotaExceeded", path="playlistItems")
ch = Channel(id="UCq", title="Q", uploads_playlist_id="UUq")
db.add(ch)
db.commit()
with pytest.raises(YouTubeError):
backfill_channel_recent(db, _Yt(), ch)
def test_any_other_explore_failure_still_raises():
class _Yt:
def get_playlist_items(self, *_):