fix(sync): try a channel's feed again before counting it as failed
YouTube's feed endpoint answers the same channel with 404, 500 and 200 in turn: on prod 182, then 296 of 396 channels failed a poll, and a probe of 24 channels saw 15 fail a first request while 16 had succeeded by the fifth. A channel that failed once waited for the next poll, 20 minutes later, so new uploads showed up late. A feed is now tried up to three times (after 1 s and 2 s) on a transport error, a 404, a 429 or a 5xx; a refusal such as 403 is not retried. The last failure is what gets counted. A feed that is really gone costs two more requests per poll.
This commit is contained in:
+33
-10
@@ -1,6 +1,7 @@
|
||||
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
|
||||
~15 uploads as lightweight stubs for fast fresh-video detection."""
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import feedparser
|
||||
@@ -8,6 +9,15 @@ import httpx
|
||||
|
||||
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
||||
|
||||
# The waits before a second and a third try. YouTube's feed endpoint answers the SAME channel with
|
||||
# 404, 500 and 200 in turn (measured 2026-09-19: 15 of 24 channels failed a first request, 16 of them
|
||||
# had succeeded by the fifth, one second apart), so one failure is not an answer. A feed that is
|
||||
# really gone costs two more requests per poll.
|
||||
RETRY_DELAYS = (1.0, 2.0)
|
||||
# A 404 is in the set because that is how the flaky endpoint fails most often; a 403/410 is not,
|
||||
# and neither is anything else that says the request itself is wrong.
|
||||
_RETRYABLE_STATUS = frozenset({404, 429, 500, 502, 503, 504})
|
||||
|
||||
|
||||
class RssError(Exception):
|
||||
"""The feed could not be read (transport failure or a non-200 answer).
|
||||
@@ -24,21 +34,34 @@ class RssError(Exception):
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
||||
"""The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure
|
||||
raises RssError so the caller can leave the channel un-stamped and log it."""
|
||||
def _get_feed(channel_id: str) -> bytes:
|
||||
"""The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised."""
|
||||
url = RSS_URL.format(channel_id=channel_id)
|
||||
try:
|
||||
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
|
||||
except httpx.HTTPError as exc:
|
||||
raise RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport") from exc
|
||||
if resp.status_code != 200:
|
||||
raise RssError(
|
||||
for attempt in range(len(RETRY_DELAYS) + 1):
|
||||
if attempt:
|
||||
time.sleep(RETRY_DELAYS[attempt - 1])
|
||||
try:
|
||||
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
|
||||
# silent-ok: not silent — the last failure is raised after the final try.
|
||||
except httpx.HTTPError as exc:
|
||||
error = RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport")
|
||||
error.__cause__ = exc
|
||||
continue
|
||||
if resp.status_code == 200:
|
||||
return resp.content
|
||||
error = RssError(
|
||||
f"RSS fetch for {channel_id} returned HTTP {resp.status_code}",
|
||||
f"http_{resp.status_code}",
|
||||
)
|
||||
if resp.status_code not in _RETRYABLE_STATUS:
|
||||
break
|
||||
raise error
|
||||
|
||||
parsed = feedparser.parse(resp.content)
|
||||
|
||||
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
||||
"""The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure
|
||||
(after the retries) raises RssError so the caller can leave the channel un-stamped and log it."""
|
||||
parsed = feedparser.parse(_get_feed(channel_id))
|
||||
out: list[dict] = []
|
||||
for entry in parsed.entries:
|
||||
video_id = entry.get("yt_videoid")
|
||||
|
||||
Reference in New Issue
Block a user