fix(sync): try a channel's feed again before counting it as failed

YouTube's feed endpoint answers the same channel with 404, 500 and 200 in
turn: on prod 182, then 296 of 396 channels failed a poll, and a probe of
24 channels saw 15 fail a first request while 16 had succeeded by the
fifth. A channel that failed once waited for the next poll, 20 minutes
later, so new uploads showed up late.

A feed is now tried up to three times (after 1 s and 2 s) on a transport
error, a 404, a 429 or a 5xx; a refusal such as 403 is not retried. The
last failure is what gets counted. A feed that is really gone costs two
more requests per poll.
This commit is contained in:
2026-09-19 01:58:40 +02:00
parent b5996a0e56
commit 8cfab498cf
2 changed files with 80 additions and 10 deletions
+47
View File
@@ -246,6 +246,53 @@ class TestRssErrorVsEmpty:
)
monkeypatch.setattr(rss.httpx, "get", fake_get)
monkeypatch.setattr(rss.time, "sleep", lambda s: None)
def _answers(self, monkeypatch, *answers):
"""The endpoint answers each try with the next of `answers` (a status or an exception);
returns the list of tries made and the waits between them."""
tries, waits = [], []
queue = list(answers)
def fake_get(url, **_):
answer = queue.pop(0)
tries.append(answer)
if isinstance(answer, Exception):
raise answer
body = self.FEED if answer == 200 else b""
return httpx.Response(answer, content=body, request=httpx.Request("GET", url))
monkeypatch.setattr(rss.httpx, "get", fake_get)
monkeypatch.setattr(rss.time, "sleep", waits.append)
return tries, waits
FEED = b"""<?xml version="1.0"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:yt="http://www.youtube.com/xml/schemas/2015">
<entry><yt:videoId>abc123</yt:videoId><title>Hello</title></entry>
</feed>"""
def test_a_feed_that_fails_and_then_answers_is_read(self, monkeypatch):
# The endpoint's measured behaviour: the same channel 404s, 500s, then answers.
tries, waits = self._answers(monkeypatch, 404, 500, 200)
assert [e["id"] for e in rss.fetch_channel_feed("UC_flaky")] == ["abc123"]
assert (len(tries), waits) == (3, list(rss.RETRY_DELAYS))
def test_a_dropped_connection_is_tried_again_too(self, monkeypatch):
tries, _ = self._answers(monkeypatch, httpx.ConnectError("reset"), 200)
assert rss.fetch_channel_feed("UC_blip")
assert len(tries) == 2
def test_a_feed_that_keeps_failing_raises_its_last_failure(self, monkeypatch):
tries, _ = self._answers(monkeypatch, 500, 404, 404)
with pytest.raises(rss.RssError) as err:
rss.fetch_channel_feed("UC_gone")
assert (len(tries), err.value.reason) == (3, "http_404")
def test_a_refusal_is_not_tried_again(self, monkeypatch):
tries, waits = self._answers(monkeypatch, 403)
with pytest.raises(rss.RssError) as err:
rss.fetch_channel_feed("UC_forbidden")
assert (len(tries), waits, err.value.reason) == (1, [], "http_403")
def test_non_200_raises(self, monkeypatch):
self._patch(monkeypatch, status=404)