diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index a257c92a..d725df3f 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -11,12 +11,14 @@ from sqlalchemy import and_, func, or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session -from app import progress, quota, sysconfig +from app import logs, progress, quota, sysconfig from app.models import LIVE_OR_UPCOMING, Channel, Video from app.titles import normalize_title -from app.youtube.client import YouTubeClient, best_thumbnail +from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail, is_playlist_gone from app.youtube.shorts import make_client, probe_is_short +log = logs.get_logger(__name__) + _DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE) @@ -104,6 +106,20 @@ def stub_from_playlist_item(item: dict, channel_id: str) -> dict | None: } +def _uploads_page(yt: YouTubeClient, channel_id: str, uploads: str, page_token: str | None) -> dict: + """One page of the channel's uploads. YouTube answers the uploads list of a channel with no + uploads at all as not found: an empty back-catalogue, so an empty last page — not a failure + that left `recent_synced_at` unstamped and charged the channel again on every run (E32 review). + Any other failure raises.""" + try: + return yt.get_playlist_items(uploads, page_token) + except YouTubeError as exc: + if not is_playlist_gone(exc): + raise + log.info("backfill channel=%s has no uploads playlist", channel_id) + return {} + + def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int: """First pass: newest videos up to the configured count or age cutoff. Records a resume cursor for the later deep backfill.""" @@ -121,7 +137,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> while True: page_start = page_token # token that fetched THIS page (None = first page) - data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) + data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token) stopped_mid_page = False for item in data.get("items", []): stub = stub_from_playlist_item(item, channel.id) @@ -175,7 +191,7 @@ def backfill_channel_deep( for _ in range(max_pages): if quota.backfill_reserve_reached(): break - data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) + data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token) rows = [ s for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", [])) diff --git a/backend/tests/test_youtube_refusals.py b/backend/tests/test_youtube_refusals.py index 356e67f8..c41b9765 100644 --- a/backend/tests/test_youtube_refusals.py +++ b/backend/tests/test_youtube_refusals.py @@ -8,8 +8,9 @@ import httpx import pytest from app.auth import has_read_scope -from app.models import OAuthToken +from app.models import Channel, OAuthToken from app.sync.explore import explore_ingest_page +from app.sync.videos import backfill_channel_deep, backfill_channel_recent from app.youtube import client as yt from app.youtube.client import YouTubeError, refusal_reason @@ -57,6 +58,52 @@ def test_a_channel_without_uploads_explores_as_an_empty_page(): assert explore_ingest_page(None, _Yt(), channel) == {"ingested": 0, "next_page_token": None} +class _NoUploads: + """A client whose every uploads read answers "playlist not found" (a channel with no uploads).""" + + def __init__(self): + self.calls = 0 + + def get_playlist_items(self, *_): + self.calls += 1 + raise YouTubeError("m", reason="playlistNotFound", path="playlistItems") + + +def test_a_channel_without_uploads_backfills_as_done_and_is_not_charged_again(db): + # The recent pass stamped nothing on this answer, so the channel was picked — and charged a + # quota unit, with an ERROR traceback — on every backfill run, forever (E32 review round 3). + ch = Channel(id="UCempty", title="Empty", uploads_playlist_id="UUempty") + db.add(ch) + db.commit() + yt = _NoUploads() + assert backfill_channel_recent(db, yt, ch) == 0 + db.refresh(ch) + assert ch.recent_synced_at is not None and ch.backfill_done and ch.backfill_cursor is None + assert backfill_channel_deep(db, yt, ch) == 0 + assert yt.calls == 1 # the deep pass has nothing left to ask + + +def test_a_deep_backfill_whose_uploads_vanish_ends_instead_of_failing(db): + ch = Channel(id="UCgone", title="Gone", uploads_playlist_id="UUgone", backfill_cursor="page2") + db.add(ch) + db.commit() + assert backfill_channel_deep(db, _NoUploads(), ch) == 0 + db.refresh(ch) + assert ch.backfill_done and ch.backfill_cursor is None + + +def test_any_other_backfill_failure_still_raises(db): + class _Yt: + def get_playlist_items(self, *_): + raise YouTubeError("m", reason="quotaExceeded", path="playlistItems") + + ch = Channel(id="UCq", title="Q", uploads_playlist_id="UUq") + db.add(ch) + db.commit() + with pytest.raises(YouTubeError): + backfill_channel_recent(db, _Yt(), ch) + + def test_any_other_explore_failure_still_raises(): class _Yt: def get_playlist_items(self, *_):