From 8cfab498cf89641855969e2d5d467ff46c4aae24 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 01:58:40 +0200 Subject: [PATCH] fix(sync): try a channel's feed again before counting it as failed YouTube's feed endpoint answers the same channel with 404, 500 and 200 in turn: on prod 182, then 296 of 396 channels failed a poll, and a probe of 24 channels saw 15 fail a first request while 16 had succeeded by the fifth. A channel that failed once waited for the next poll, 20 minutes later, so new uploads showed up late. A feed is now tried up to three times (after 1 s and 2 s) on a transport error, a 404, a 429 or a 5xx; a refusal such as 403 is not retried. The last failure is what gets counted. A feed that is really gone costs two more requests per poll. --- backend/app/youtube/rss.py | 43 +++++++++++++++++++++------- backend/tests/test_sync_guards.py | 47 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/backend/app/youtube/rss.py b/backend/app/youtube/rss.py index e4624e76..b83d4b17 100644 --- a/backend/app/youtube/rss.py +++ b/backend/app/youtube/rss.py @@ -1,6 +1,7 @@ """Free per-channel RSS feed reader (no API quota). Returns the channel's most recent ~15 uploads as lightweight stubs for fast fresh-video detection.""" +import time from datetime import datetime, timezone import feedparser @@ -8,6 +9,15 @@ import httpx RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}" +# The waits before a second and a third try. YouTube's feed endpoint answers the SAME channel with +# 404, 500 and 200 in turn (measured 2026-09-19: 15 of 24 channels failed a first request, 16 of them +# had succeeded by the fifth, one second apart), so one failure is not an answer. A feed that is +# really gone costs two more requests per poll. +RETRY_DELAYS = (1.0, 2.0) +# A 404 is in the set because that is how the flaky endpoint fails most often; a 403/410 is not, +# and neither is anything else that says the request itself is wrong. +_RETRYABLE_STATUS = frozenset({404, 429, 500, 502, 503, 504}) + class RssError(Exception): """The feed could not be read (transport failure or a non-200 answer). @@ -24,21 +34,34 @@ class RssError(Exception): self.reason = reason -def fetch_channel_feed(channel_id: str) -> list[dict]: - """The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure - raises RssError so the caller can leave the channel un-stamped and log it.""" +def _get_feed(channel_id: str) -> bytes: + """The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised.""" url = RSS_URL.format(channel_id=channel_id) - try: - resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"}) - except httpx.HTTPError as exc: - raise RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport") from exc - if resp.status_code != 200: - raise RssError( + for attempt in range(len(RETRY_DELAYS) + 1): + if attempt: + time.sleep(RETRY_DELAYS[attempt - 1]) + try: + resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"}) + # silent-ok: not silent — the last failure is raised after the final try. + except httpx.HTTPError as exc: + error = RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport") + error.__cause__ = exc + continue + if resp.status_code == 200: + return resp.content + error = RssError( f"RSS fetch for {channel_id} returned HTTP {resp.status_code}", f"http_{resp.status_code}", ) + if resp.status_code not in _RETRYABLE_STATUS: + break + raise error - parsed = feedparser.parse(resp.content) + +def fetch_channel_feed(channel_id: str) -> list[dict]: + """The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure + (after the retries) raises RssError so the caller can leave the channel un-stamped and log it.""" + parsed = feedparser.parse(_get_feed(channel_id)) out: list[dict] = [] for entry in parsed.entries: video_id = entry.get("yt_videoid") diff --git a/backend/tests/test_sync_guards.py b/backend/tests/test_sync_guards.py index ef688baf..f58cda15 100644 --- a/backend/tests/test_sync_guards.py +++ b/backend/tests/test_sync_guards.py @@ -246,6 +246,53 @@ class TestRssErrorVsEmpty: ) monkeypatch.setattr(rss.httpx, "get", fake_get) + monkeypatch.setattr(rss.time, "sleep", lambda s: None) + + def _answers(self, monkeypatch, *answers): + """The endpoint answers each try with the next of `answers` (a status or an exception); + returns the list of tries made and the waits between them.""" + tries, waits = [], [] + queue = list(answers) + + def fake_get(url, **_): + answer = queue.pop(0) + tries.append(answer) + if isinstance(answer, Exception): + raise answer + body = self.FEED if answer == 200 else b"" + return httpx.Response(answer, content=body, request=httpx.Request("GET", url)) + + monkeypatch.setattr(rss.httpx, "get", fake_get) + monkeypatch.setattr(rss.time, "sleep", waits.append) + return tries, waits + + FEED = b""" + + abc123Hello + """ + + def test_a_feed_that_fails_and_then_answers_is_read(self, monkeypatch): + # The endpoint's measured behaviour: the same channel 404s, 500s, then answers. + tries, waits = self._answers(monkeypatch, 404, 500, 200) + assert [e["id"] for e in rss.fetch_channel_feed("UC_flaky")] == ["abc123"] + assert (len(tries), waits) == (3, list(rss.RETRY_DELAYS)) + + def test_a_dropped_connection_is_tried_again_too(self, monkeypatch): + tries, _ = self._answers(monkeypatch, httpx.ConnectError("reset"), 200) + assert rss.fetch_channel_feed("UC_blip") + assert len(tries) == 2 + + def test_a_feed_that_keeps_failing_raises_its_last_failure(self, monkeypatch): + tries, _ = self._answers(monkeypatch, 500, 404, 404) + with pytest.raises(rss.RssError) as err: + rss.fetch_channel_feed("UC_gone") + assert (len(tries), err.value.reason) == (3, "http_404") + + def test_a_refusal_is_not_tried_again(self, monkeypatch): + tries, waits = self._answers(monkeypatch, 403) + with pytest.raises(rss.RssError) as err: + rss.fetch_channel_feed("UC_forbidden") + assert (len(tries), waits, err.value.reason) == (1, [], "http_403") def test_non_200_raises(self, monkeypatch): self._patch(monkeypatch, status=404)