"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent ~15 uploads as lightweight stubs for fast fresh-video detection.""" import time from datetime import datetime, timezone import feedparser import httpx RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}" # The waits before a second and a third try. YouTube's feed endpoint answers the SAME channel with # 404, 500 and 200 in turn (measured 2026-09-19: 15 of 24 channels failed a first request, 16 of them # had succeeded by the fifth, one second apart), so one failure is not an answer. A feed that is # really gone costs two more requests per poll. RETRY_DELAYS = (1.0, 2.0) # A 404 is in the set because that is how the flaky endpoint fails most often; a 403/410 is not, # and neither is anything else that says the request itself is wrong. _RETRYABLE_STATUS = frozenset({404, 429, 500, 502, 503, 504}) class RssError(Exception): """The feed could not be read (transport failure or a non-200 answer). Deliberately NOT the same as an empty feed: a failed poll used to return `[]`, which the caller stamped as a successful `last_rss_at` — so a channel whose feed 404s permanently (deleted, renamed id) looked like it was being polled fine forever, and nothing surfaced it. `reason` is a short stable code (`http_404`, `transport`) so a poll over many channels can report WHAT failed as counts instead of one line per channel.""" def __init__(self, message: str, reason: str): super().__init__(message) self.reason = reason def _get_feed(channel_id: str) -> bytes: """The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised.""" url = RSS_URL.format(channel_id=channel_id) for attempt in range(len(RETRY_DELAYS) + 1): if attempt: time.sleep(RETRY_DELAYS[attempt - 1]) try: resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"}) # silent-ok: not silent — the last failure is raised after the final try. except httpx.HTTPError as exc: error = RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport") error.__cause__ = exc continue if resp.status_code == 200: return resp.content error = RssError( f"RSS fetch for {channel_id} returned HTTP {resp.status_code}", f"http_{resp.status_code}", ) if resp.status_code not in _RETRYABLE_STATUS: break raise error def fetch_channel_feed(channel_id: str) -> list[dict]: """The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure (after the retries) raises RssError so the caller can leave the channel un-stamped and log it.""" parsed = feedparser.parse(_get_feed(channel_id)) out: list[dict] = [] for entry in parsed.entries: video_id = entry.get("yt_videoid") if not video_id: continue published = None if entry.get("published_parsed"): published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) thumbnail = None media = entry.get("media_thumbnail") or [] if media: thumbnail = media[0].get("url") out.append( { "id": video_id, "channel_id": channel_id, "title": entry.get("title"), "published_at": published, "thumbnail_url": thumbnail, } ) return out