fix(sync): don't try a timed-out feed again
A request that hung for the full timeout says the way out is hanging, not that the endpoint is flaky; three of them per channel stretched a poll over a dead egress to about three times its interval.
This commit is contained in:
@@ -12,8 +12,11 @@ RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
|||||||
# The waits before a second and a third try. YouTube's feed endpoint answers the SAME channel with
|
# The waits before a second and a third try. YouTube's feed endpoint answers the SAME channel with
|
||||||
# 404, 500 and 200 in turn (measured 2026-09-19: 15 of 24 channels failed a first request, 16 of them
|
# 404, 500 and 200 in turn (measured 2026-09-19: 15 of 24 channels failed a first request, 16 of them
|
||||||
# had succeeded by the fifth, one second apart), so one failure is not an answer. A feed that is
|
# had succeeded by the fifth, one second apart), so one failure is not an answer. A feed that is
|
||||||
# really gone costs two more requests per poll.
|
# really gone costs two more requests per poll. A TIMEOUT is not tried again: the flaky endpoint
|
||||||
|
# answers fast, and a request that hung for `TIMEOUT` says the way out is hanging — three of them
|
||||||
|
# per channel would stretch a poll over a dead egress to three times its interval.
|
||||||
RETRY_DELAYS = (1.0, 2.0)
|
RETRY_DELAYS = (1.0, 2.0)
|
||||||
|
TIMEOUT = 20.0
|
||||||
# A 404 is in the set because that is how the flaky endpoint fails most often; a 403/410 is not,
|
# A 404 is in the set because that is how the flaky endpoint fails most often; a 403/410 is not,
|
||||||
# and neither is anything else that says the request itself is wrong.
|
# and neither is anything else that says the request itself is wrong.
|
||||||
_RETRYABLE_STATUS = frozenset({404, 429, 500, 502, 503, 504})
|
_RETRYABLE_STATUS = frozenset({404, 429, 500, 502, 503, 504})
|
||||||
@@ -41,11 +44,13 @@ def _get_feed(channel_id: str) -> bytes:
|
|||||||
if attempt:
|
if attempt:
|
||||||
time.sleep(RETRY_DELAYS[attempt - 1])
|
time.sleep(RETRY_DELAYS[attempt - 1])
|
||||||
try:
|
try:
|
||||||
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
|
resp = httpx.get(url, timeout=TIMEOUT, headers={"User-Agent": "Siftlode/1.0"})
|
||||||
# silent-ok: not silent — the last failure is raised after the final try.
|
# silent-ok: not silent — the last failure is raised after the final try.
|
||||||
except httpx.HTTPError as exc:
|
except httpx.HTTPError as exc:
|
||||||
error = RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport")
|
error = RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport")
|
||||||
error.__cause__ = exc
|
error.__cause__ = exc
|
||||||
|
if isinstance(exc, httpx.TimeoutException):
|
||||||
|
break
|
||||||
continue
|
continue
|
||||||
if resp.status_code == 200:
|
if resp.status_code == 200:
|
||||||
return resp.content
|
return resp.content
|
||||||
|
|||||||
@@ -282,6 +282,14 @@ class TestRssErrorVsEmpty:
|
|||||||
assert rss.fetch_channel_feed("UC_blip")
|
assert rss.fetch_channel_feed("UC_blip")
|
||||||
assert len(tries) == 2
|
assert len(tries) == 2
|
||||||
|
|
||||||
|
def test_a_timeout_is_not_tried_again(self, monkeypatch):
|
||||||
|
# A hung request already cost the full timeout; retrying it would triple a poll over a
|
||||||
|
# dead egress.
|
||||||
|
tries, waits = self._answers(monkeypatch, httpx.ReadTimeout("hung"), 200)
|
||||||
|
with pytest.raises(rss.RssError) as err:
|
||||||
|
rss.fetch_channel_feed("UC_hung")
|
||||||
|
assert (len(tries), waits, err.value.reason) == (1, [], "transport")
|
||||||
|
|
||||||
def test_a_feed_that_keeps_failing_raises_its_last_failure(self, monkeypatch):
|
def test_a_feed_that_keeps_failing_raises_its_last_failure(self, monkeypatch):
|
||||||
tries, _ = self._answers(monkeypatch, 500, 404, 404)
|
tries, _ = self._answers(monkeypatch, 500, 404, 404)
|
||||||
with pytest.raises(rss.RssError) as err:
|
with pytest.raises(rss.RssError) as err:
|
||||||
|
|||||||
Reference in New Issue
Block a user