feat: store warnings and request lines in log_events

The log now has a stored, queryable copy for the diagnostics console:
every WARNING and above plus the request line of every request, kept
for log_retention_days (default 7, no "forever") and pruned hourly by
the new log_gc job.

Rows are built on the logging thread from the same redacted fields as
the stdout line, queued without blocking and written in batches by one
background thread on its own connection. A full queue or a failed write
drops and reports instead of retrying. Flood control keeps 30 of any
one message per minute plus a count of the rest; anonymous request
lines are capped at 60 a minute per client, and the client address is
never stored. Deleting an account deletes its stored events.

Measured on prod first: a YouTube RSS outage wrote 11k warnings in four
hours, one per channel per poll. The poll now writes one warning with
per-reason counts and a few examples, and a failure on our side gets
one traceback per poll instead of one per channel.
This commit is contained in:
2026-09-17 00:20:14 +02:00
parent 88a166e4aa
commit b59fd38a9c
23 changed files with 832 additions and 67 deletions
+53 -3
View File
@@ -1,6 +1,8 @@
"""Guards that stop one bad YouTube answer from cascading: the subscription prune guard, the
playlist-push circuit breaker, and RSS telling a failure apart from an empty feed."""
import logging
import httpx
import pytest
@@ -120,7 +122,7 @@ class TestRssPollResult:
# One dead channel among healthy ones: counted, but the run itself is still a run.
def fetch(cid):
if cid == "UC_dead":
raise rss.RssError("404")
raise rss.RssError("404", "http_404")
return [{"id": "v"}]
out = self._run(
@@ -143,6 +145,54 @@ class TestRssPollResult:
)
assert out == {"new": 1, "failed": 1}, "an unstored channel must not read as ok"
def test_many_failures_write_one_warning_with_counts_not_one_per_channel(
self, monkeypatch, caplog
):
# YouTube's RSS outage of 2026-09-16: ~900 of 979 feeds failed on every 20-minute poll and
# each wrote its own WARNING — 11k lines in four hours, which the log_events table would
# have stored too. One summary per poll, per-channel detail at DEBUG.
channels = [f"UC{i}" for i in range(40)]
def fetch(cid):
n = int(cid[2:])
if n < 30:
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404", "http_404")
if n < 35:
raise rss.RssError(f"RSS fetch failed for {cid}: timed out", "transport")
return [{"id": "v"}]
caplog.set_level(logging.DEBUG, logger="siftlode.sync")
out = self._run(monkeypatch, fetch=fetch, apply_=lambda db, ch, e: 1, channels=channels)
assert out == {"new": 5, "failed": 35}
warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
assert len(warnings) == 1
assert warnings[0].event == "sync.rss.failed"
assert warnings[0].attrs["reasons"] == {"http_404": 30, "transport": 5}
assert len(warnings[0].attrs["examples"]) == runner.RSS_FAILURE_EXAMPLES
assert sum(r.levelno == logging.DEBUG for r in caplog.records) == 35
def test_a_failure_on_our_side_gets_one_traceback_per_poll_not_one_per_channel(
self, monkeypatch, caplog
):
# A read-only database fails every channel's apply identically. The first gets a traceback
# at ERROR; repeating it for each of 979 channels would be the RSS flood all over again.
def apply_(db, ch, entries):
raise RuntimeError("read-only transaction")
caplog.set_level(logging.DEBUG, logger="siftlode.sync")
with pytest.raises(RuntimeError):
self._run(
monkeypatch,
fetch=lambda cid: [{"id": "v"}],
apply_=apply_,
channels=("A", "B", "C"),
)
errors = [r for r in caplog.records if r.levelno == logging.ERROR]
assert len(errors) == 1 and errors[0].exc_info is not None
assert (
sum(r.levelno == logging.DEBUG and r.exc_info is not None for r in caplog.records) == 2
)
def test_a_quiet_hour_is_distinguishable_from_an_outage(self, monkeypatch):
quiet = self._run(monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0)
assert quiet == {"new": 0, "failed": 0} # same "new", different "failed" — that's the point
@@ -152,7 +202,7 @@ class TestRssPollResult:
# here recorded the scheduler run as "ok", with the outage only visible inside the summary
# text — a green pill for a run that did nothing at all.
def boom(cid):
raise rss.RssError("Connection refused")
raise rss.RssError("Connection refused", "transport")
with pytest.raises(RuntimeError, match="all 2 channels"):
self._run(monkeypatch, fetch=boom, apply_=lambda db, ch, e: 1)
@@ -162,7 +212,7 @@ class TestRssPollResult:
# The card has to say which: `HTTP 404` = fix the subscription, `Connection refused` =
# the egress is down.
def dead(cid):
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404")
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404", "http_404")
with pytest.raises(RuntimeError, match="HTTP 404"):
self._run(monkeypatch, fetch=dead, apply_=lambda db, ch, e: 1, channels=("UC_dead",))