fix(sync): read a dark RSS feed through the API instead

YouTube's RSS goes dark per channel for hours (prod 2026-09-18/19: 393 of
396 feeds answered 404 for four hours, sticky per channel, so a retry does
not help), and nothing else read a followed channel's new uploads. A
channel whose feed has not answered for `rss_fallback_hours` (admin knob,
default 2, 0 = off) now gets one page of its uploads playlist through the
Data API (1 unit), at most once per window, yielding to the backfill
reserve. Spend is attributed to its own quota action.
This commit is contained in:
2026-09-19 04:51:31 +02:00
parent 16a44dec79
commit 29094f5f51
13 changed files with 302 additions and 11 deletions
+44 -6
View File
@@ -112,11 +112,15 @@ class TestRssPollResult:
def _run(self, monkeypatch, *, fetch, apply_, channels=("UC1", "UC2")):
monkeypatch.setattr(runner, "fetch_channel_feed", fetch)
monkeypatch.setattr(runner, "apply_rss_feed", apply_)
# The API fallback has its own DB-lane tests (test_feed_fallback.py); here it reads nothing.
monkeypatch.setattr(
runner, "run_feed_fallback", lambda db, ids: {"read": 0, "new": 0, "failed": 0}
)
return runner.run_rss_poll(_StubDb(), [_StubChannel(c) for c in channels])
def test_a_healthy_run_reports_no_failures(self, monkeypatch):
out = self._run(monkeypatch, fetch=lambda cid: [{"id": "v"}], apply_=lambda db, ch, e: 1)
assert out == {"new": 2, "failed": 0}
assert out == {"new": 2, "failed": 0, "fallback": 0}
def test_unreadable_feeds_are_counted(self, monkeypatch):
# One dead channel among healthy ones: counted, but the run itself is still a run.
@@ -128,7 +132,7 @@ class TestRssPollResult:
out = self._run(
monkeypatch, fetch=fetch, apply_=lambda db, ch, e: 1, channels=("UC1", "UC_dead")
)
assert out == {"new": 1, "failed": 1}
assert out == {"new": 1, "failed": 1, "fallback": 0}
def test_apply_failures_are_counted_too(self, monkeypatch):
# A per-row write failure (a constraint clash on one channel) while the rest store fine.
@@ -143,7 +147,9 @@ class TestRssPollResult:
apply_=apply_,
channels=("UC1", "UC_bad"),
)
assert out == {"new": 1, "failed": 1}, "an unstored channel must not read as ok"
assert out == {"new": 1, "failed": 1, "fallback": 0}, (
"an unstored channel must not read as ok"
)
def test_many_failures_write_one_warning_with_counts_not_one_per_channel(
self, monkeypatch, caplog
@@ -163,7 +169,7 @@ class TestRssPollResult:
caplog.set_level(logging.DEBUG, logger="siftlode.sync")
out = self._run(monkeypatch, fetch=fetch, apply_=lambda db, ch, e: 1, channels=channels)
assert out == {"new": 5, "failed": 35}
assert out == {"new": 5, "failed": 35, "fallback": 0}
warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
assert len(warnings) == 1
assert warnings[0].event == "sync.rss.failed"
@@ -195,7 +201,11 @@ class TestRssPollResult:
def test_a_quiet_hour_is_distinguishable_from_an_outage(self, monkeypatch):
quiet = self._run(monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0)
assert quiet == {"new": 0, "failed": 0} # same "new", different "failed" — that's the point
assert quiet == {
"new": 0,
"failed": 0,
"fallback": 0,
} # same "new", different "failed" — that's the point
def test_a_total_outage_raises_instead_of_returning_ok(self, monkeypatch):
# Every feed unreachable (the tinyproxy-lost-the-boot-race incident). Returning a result
@@ -225,10 +235,38 @@ class TestRssPollResult:
with pytest.raises(RuntimeError, match="read-only transaction"):
self._run(monkeypatch, fetch=lambda cid: [{"id": "v"}], apply_=boom)
def test_only_the_unreadable_feeds_go_to_the_api_and_what_it_reads_counts(self, monkeypatch):
# A feed that answered 404 is the fallback's; a channel whose rows failed to STORE is not —
# the API would hand the same rows to the same broken write.
seen = []
def fetch(cid):
if cid == "UC_dark":
raise rss.RssError("404", "http_404")
return [{"id": "v"}]
def apply_(db, ch, entries):
if ch.id == "UC_badrow":
raise RuntimeError("constraint")
return 1
def fallback(db, ids):
seen.extend(ids)
return {"read": 1, "new": 3, "failed": 0}
monkeypatch.setattr(runner, "fetch_channel_feed", fetch)
monkeypatch.setattr(runner, "apply_rss_feed", apply_)
monkeypatch.setattr(runner, "run_feed_fallback", fallback)
out = runner.run_rss_poll(
_StubDb(), [_StubChannel(c) for c in ("UC_ok", "UC_dark", "UC_badrow")]
)
assert seen == ["UC_dark"]
assert out == {"new": 4, "failed": 2, "fallback": 1}
def test_no_channels_at_all_is_not_an_outage(self, monkeypatch):
# A fresh instance with nothing subscribed must not report an error every poll.
out = self._run(monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0, channels=())
assert out == {"new": 0, "failed": 0}
assert out == {"new": 0, "failed": 0, "fallback": 0}
class TestRssErrorVsEmpty: