fix(sync): R5 S2 — sync guards
- never prune subscriptions on an empty fetch: one anomalous empty-but-200 response wiped every subscription (and its deep_requested flag) in one run - push_playlist: stop after 5 consecutive write failures — a quota-dead or scope-revoked account otherwise burns 50 units per remaining item - RSS: raise RssError instead of returning [] on a transport/HTTP failure, so a failed poll no longer stamps last_rss_at and a 404 feed stops looking healthy - worker: exit when the schema never appears instead of "continuing" into a crash one call later
This commit is contained in:
@@ -25,7 +25,7 @@ from app.sync.videos import (
|
||||
run_shorts_classification,
|
||||
)
|
||||
from app.youtube.client import YouTubeClient
|
||||
from app.youtube.rss import fetch_channel_feed
|
||||
from app.youtube.rss import RssError, fetch_channel_feed
|
||||
|
||||
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
||||
RSS_POLL_WORKERS = 16
|
||||
@@ -54,6 +54,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int:
|
||||
total = len(channels)
|
||||
new = 0
|
||||
done = 0
|
||||
failed = 0
|
||||
# Fetch feeds concurrently (network-bound, no DB), then apply inserts on this thread as
|
||||
# each fetch returns — a SQLAlchemy session isn't thread-safe. Pass the channel id (a
|
||||
# plain str) to the workers; the ORM instance is touched only here on the main thread.
|
||||
@@ -65,9 +66,20 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int:
|
||||
channel = futures[fut]
|
||||
try:
|
||||
entries = fut.result()
|
||||
except RssError as exc:
|
||||
# A failed poll is NOT an empty feed: skip the apply so `last_rss_at` stays where
|
||||
# it was. Stamping it here made a permanently-broken feed look freshly polled.
|
||||
log.warning("RSS poll skipped for channel %s: %s", channel.id, exc)
|
||||
failed += 1
|
||||
done += 1
|
||||
progress.report(done, total, "rss_poll")
|
||||
continue
|
||||
except Exception:
|
||||
log.exception("RSS fetch failed for channel %s", channel.id)
|
||||
entries = []
|
||||
failed += 1
|
||||
done += 1
|
||||
progress.report(done, total, "rss_poll")
|
||||
continue
|
||||
try:
|
||||
new += apply_rss_feed(db, channel, entries)
|
||||
except Exception:
|
||||
@@ -75,6 +87,8 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int:
|
||||
log.exception("RSS apply failed for channel %s", channel.id)
|
||||
done += 1
|
||||
progress.report(done, total, "rss_poll")
|
||||
if failed:
|
||||
log.warning("RSS poll: %d/%d channel feeds could not be read", failed, total)
|
||||
return new
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user