peter
8cfab498cf
YouTube's feed endpoint answers the same channel with 404, 500 and 200 in turn: on prod 182, then 296 of 396 channels failed a poll, and a probe of 24 channels saw 15 fail a first request while 16 had succeeded by the fifth. A channel that failed once waited for the next poll, 20 minutes later, so new uploads showed up late. A feed is now tried up to three times (after 1 s and 2 s) on a transport error, a 404, a 429 or a 5xx; a refusal such as 403 is not retried. The last failure is what gets counted. A feed that is really gone costs two more requests per poll.
87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
|
|
~15 uploads as lightweight stubs for fast fresh-video detection."""
|
|
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
import feedparser
|
|
import httpx
|
|
|
|
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
|
|
|
# The waits before a second and a third try. YouTube's feed endpoint answers the SAME channel with
|
|
# 404, 500 and 200 in turn (measured 2026-09-19: 15 of 24 channels failed a first request, 16 of them
|
|
# had succeeded by the fifth, one second apart), so one failure is not an answer. A feed that is
|
|
# really gone costs two more requests per poll.
|
|
RETRY_DELAYS = (1.0, 2.0)
|
|
# A 404 is in the set because that is how the flaky endpoint fails most often; a 403/410 is not,
|
|
# and neither is anything else that says the request itself is wrong.
|
|
_RETRYABLE_STATUS = frozenset({404, 429, 500, 502, 503, 504})
|
|
|
|
|
|
class RssError(Exception):
|
|
"""The feed could not be read (transport failure or a non-200 answer).
|
|
|
|
Deliberately NOT the same as an empty feed: a failed poll used to return `[]`, which the
|
|
caller stamped as a successful `last_rss_at` — so a channel whose feed 404s permanently
|
|
(deleted, renamed id) looked like it was being polled fine forever, and nothing surfaced it.
|
|
|
|
`reason` is a short stable code (`http_404`, `transport`) so a poll over many channels can report
|
|
WHAT failed as counts instead of one line per channel."""
|
|
|
|
def __init__(self, message: str, reason: str):
|
|
super().__init__(message)
|
|
self.reason = reason
|
|
|
|
|
|
def _get_feed(channel_id: str) -> bytes:
|
|
"""The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised."""
|
|
url = RSS_URL.format(channel_id=channel_id)
|
|
for attempt in range(len(RETRY_DELAYS) + 1):
|
|
if attempt:
|
|
time.sleep(RETRY_DELAYS[attempt - 1])
|
|
try:
|
|
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
|
|
# silent-ok: not silent — the last failure is raised after the final try.
|
|
except httpx.HTTPError as exc:
|
|
error = RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport")
|
|
error.__cause__ = exc
|
|
continue
|
|
if resp.status_code == 200:
|
|
return resp.content
|
|
error = RssError(
|
|
f"RSS fetch for {channel_id} returned HTTP {resp.status_code}",
|
|
f"http_{resp.status_code}",
|
|
)
|
|
if resp.status_code not in _RETRYABLE_STATUS:
|
|
break
|
|
raise error
|
|
|
|
|
|
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
|
"""The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure
|
|
(after the retries) raises RssError so the caller can leave the channel un-stamped and log it."""
|
|
parsed = feedparser.parse(_get_feed(channel_id))
|
|
out: list[dict] = []
|
|
for entry in parsed.entries:
|
|
video_id = entry.get("yt_videoid")
|
|
if not video_id:
|
|
continue
|
|
published = None
|
|
if entry.get("published_parsed"):
|
|
published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
|
|
thumbnail = None
|
|
media = entry.get("media_thumbnail") or []
|
|
if media:
|
|
thumbnail = media[0].get("url")
|
|
out.append(
|
|
{
|
|
"id": video_id,
|
|
"channel_id": channel_id,
|
|
"title": entry.get("title"),
|
|
"published_at": published,
|
|
"thumbnail_url": thumbnail,
|
|
}
|
|
)
|
|
return out
|