fix(sync): a channel with no uploads backfills as done, not as a failure
YouTube answers the uploads list of a channel with no uploads as playlistNotFound. The recent and deep backfills raised on it, so the channel was never stamped and was picked again, charged a quota unit and logged an ERROR on every run. Both now read it as an empty last page, as explore already does; any other failure still raises.
This commit is contained in:
@@ -11,12 +11,14 @@ from sqlalchemy import and_, func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota, sysconfig
|
||||
from app import logs, progress, quota, sysconfig
|
||||
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
||||
from app.titles import normalize_title
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail, is_playlist_gone
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||
|
||||
|
||||
@@ -104,6 +106,20 @@ def stub_from_playlist_item(item: dict, channel_id: str) -> dict | None:
|
||||
}
|
||||
|
||||
|
||||
def _uploads_page(yt: YouTubeClient, channel_id: str, uploads: str, page_token: str | None) -> dict:
|
||||
"""One page of the channel's uploads. YouTube answers the uploads list of a channel with no
|
||||
uploads at all as not found: an empty back-catalogue, so an empty last page — not a failure
|
||||
that left `recent_synced_at` unstamped and charged the channel again on every run (E32 review).
|
||||
Any other failure raises."""
|
||||
try:
|
||||
return yt.get_playlist_items(uploads, page_token)
|
||||
except YouTubeError as exc:
|
||||
if not is_playlist_gone(exc):
|
||||
raise
|
||||
log.info("backfill channel=%s has no uploads playlist", channel_id)
|
||||
return {}
|
||||
|
||||
|
||||
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
|
||||
"""First pass: newest videos up to the configured count or age cutoff. Records a
|
||||
resume cursor for the later deep backfill."""
|
||||
@@ -121,7 +137,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
||||
|
||||
while True:
|
||||
page_start = page_token # token that fetched THIS page (None = first page)
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
|
||||
stopped_mid_page = False
|
||||
for item in data.get("items", []):
|
||||
stub = stub_from_playlist_item(item, channel.id)
|
||||
@@ -175,7 +191,7 @@ def backfill_channel_deep(
|
||||
for _ in range(max_pages):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
|
||||
rows = [
|
||||
s
|
||||
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
||||
|
||||
Reference in New Issue
Block a user