fix(sync): try a channel's feed again before counting it as failed
YouTube's feed endpoint answers the same channel with 404, 500 and 200 in turn: on prod 182, then 296 of 396 channels failed a poll, and a probe of 24 channels saw 15 fail a first request while 16 had succeeded by the fifth. A channel that failed once waited for the next poll, 20 minutes later, so new uploads showed up late. A feed is now tried up to three times (after 1 s and 2 s) on a transport error, a 404, a 429 or a 5xx; a refusal such as 403 is not retried. The last failure is what gets counted. A feed that is really gone costs two more requests per poll.
This commit is contained in:
@@ -246,6 +246,53 @@ class TestRssErrorVsEmpty:
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rss.httpx, "get", fake_get)
|
||||
monkeypatch.setattr(rss.time, "sleep", lambda s: None)
|
||||
|
||||
def _answers(self, monkeypatch, *answers):
|
||||
"""The endpoint answers each try with the next of `answers` (a status or an exception);
|
||||
returns the list of tries made and the waits between them."""
|
||||
tries, waits = [], []
|
||||
queue = list(answers)
|
||||
|
||||
def fake_get(url, **_):
|
||||
answer = queue.pop(0)
|
||||
tries.append(answer)
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
body = self.FEED if answer == 200 else b""
|
||||
return httpx.Response(answer, content=body, request=httpx.Request("GET", url))
|
||||
|
||||
monkeypatch.setattr(rss.httpx, "get", fake_get)
|
||||
monkeypatch.setattr(rss.time, "sleep", waits.append)
|
||||
return tries, waits
|
||||
|
||||
FEED = b"""<?xml version="1.0"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:yt="http://www.youtube.com/xml/schemas/2015">
|
||||
<entry><yt:videoId>abc123</yt:videoId><title>Hello</title></entry>
|
||||
</feed>"""
|
||||
|
||||
def test_a_feed_that_fails_and_then_answers_is_read(self, monkeypatch):
|
||||
# The endpoint's measured behaviour: the same channel 404s, 500s, then answers.
|
||||
tries, waits = self._answers(monkeypatch, 404, 500, 200)
|
||||
assert [e["id"] for e in rss.fetch_channel_feed("UC_flaky")] == ["abc123"]
|
||||
assert (len(tries), waits) == (3, list(rss.RETRY_DELAYS))
|
||||
|
||||
def test_a_dropped_connection_is_tried_again_too(self, monkeypatch):
|
||||
tries, _ = self._answers(monkeypatch, httpx.ConnectError("reset"), 200)
|
||||
assert rss.fetch_channel_feed("UC_blip")
|
||||
assert len(tries) == 2
|
||||
|
||||
def test_a_feed_that_keeps_failing_raises_its_last_failure(self, monkeypatch):
|
||||
tries, _ = self._answers(monkeypatch, 500, 404, 404)
|
||||
with pytest.raises(rss.RssError) as err:
|
||||
rss.fetch_channel_feed("UC_gone")
|
||||
assert (len(tries), err.value.reason) == (3, "http_404")
|
||||
|
||||
def test_a_refusal_is_not_tried_again(self, monkeypatch):
|
||||
tries, waits = self._answers(monkeypatch, 403)
|
||||
with pytest.raises(rss.RssError) as err:
|
||||
rss.fetch_channel_feed("UC_forbidden")
|
||||
assert (len(tries), waits, err.value.reason) == (1, [], "http_403")
|
||||
|
||||
def test_non_200_raises(self, monkeypatch):
|
||||
self._patch(monkeypatch, status=404)
|
||||
|
||||
Reference in New Issue
Block a user