peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: store warnings and request lines in log_events

Browse Source 
The log now has a stored, queryable copy for the diagnostics console:
every WARNING and above plus the request line of every request, kept
for log_retention_days (default 7, no "forever") and pruned hourly by
the new log_gc job.

Rows are built on the logging thread from the same redacted fields as
the stdout line, queued without blocking and written in batches by one
background thread on its own connection. A full queue or a failed write
drops and reports instead of retrying. Flood control keeps 30 of any
one message per minute plus a count of the rest; anonymous request
lines are capped at 60 a minute per client, and the client address is
never stored. Deleting an account deletes its stored events.

Measured on prod first: a YouTube RSS outage wrote 11k warnings in four
hours, one per channel per poll. The poll now writes one warning with
per-reason counts and a few examples, and a failure on our side gets
one traceback per poll instead of one per channel.
This commit is contained in:
peter
2026-09-17 00:20:14 +02:00
parent 88a166e4aa
commit b59fd38a9c
23 changed files with 832 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sync/runner.py
+49 -13
View File
@@ -7,12 +7,13 @@ public reads.
import logging
import threading
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select
from sqlalchemy.orm import Session
from app import progress, quota, sysconfig
from app import logs, progress, quota, sysconfig
log = logging.getLogger("siftlode.sync")
from app.db import SessionLocal
@@ -32,6 +33,8 @@ from app.youtube.rss import RssError, fetch_channel_feed
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
RSS_POLL_WORKERS = 16
# Failed channels named in a poll's summary warning; the rest are counted by reason.
RSS_FAILURE_EXAMPLES = 5
def token_users(db: Session) -> list[User]:
@@ -76,6 +79,27 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# outage from one permanently dead feed id — which is the whole difference on an instance with
# a single subscription, where BOTH render as 100% failed.
first_error: str | None = None
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
reasons: Counter[str] = Counter()
examples: list[str] = []
def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None:
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
tell nobody anything the first one didn't. Call it from inside the `except` block."""
nonlocal failed, first_error
if unexpected is not None:
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
log.log(level, unexpected, exc_info=True)
failed += 1
reasons[reason] += 1
first_error = first_error or detail
if len(examples) < RSS_FAILURE_EXAMPLES:
examples.append(detail)
# Fetch feeds concurrently (network-bound, no DB), then apply inserts on this thread as
# each fetch returns — a SQLAlchemy session isn't thread-safe. Pass the channel id (a
# plain str) to the workers; the ORM instance is touched only here on the main thread.
@@ -90,16 +114,15 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# A failed poll is NOT an empty feed: skip the apply so `last_rss_at` stays where it
# was (stamping it made a permanently-broken feed look freshly polled). The shared
# counter/progress tail lives after the statement, so the two outcomes can stay two
# clauses: an RssError is EXPECTED (404, timeout) and gets one warning line; anything
# else is a bug on our side and earns a traceback.
# clauses: an RssError is EXPECTED (404, timeout) and is only counted; anything else is a
# bug on our side and earns a traceback (the first of its kind per poll).
except RssError as exc:
log.warning("RSS poll skipped for channel %s: %s", channel.id, exc)
failed += 1
first_error = first_error or f"{channel.id}: {exc}"
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
fail(exc.reason, f"{channel.id}: {exc}")
except Exception as exc:
log.exception("RSS fetch failed for channel %s", channel.id)
failed += 1
first_error = first_error or f"{channel.id}: {exc}"
fail(
"error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}"
)
else:
try:
new += apply_rss_feed(db, channel, entries)
@@ -107,14 +130,27 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# A write failure (read-only DB, failover) leaves the channel just as un-polled
# as an unreachable feed — count it, or a wholly broken run reports "failed=0".
db.rollback()
failed += 1
first_error = first_error or f"{channel.id} (apply): {exc}"
log.exception("RSS apply failed for channel %s", channel.id)
fail(
"apply",
f"{channel.id} (apply): {exc}",
unexpected=f"RSS apply failed for {channel.id}",
)
# One place that advances the counter, whatever happened above.
done += 1
progress.report(done, total, "rss_poll")
if failed:
log.warning("RSS poll: %d/%d channels could not be polled", failed, total)
log.warning(
"RSS poll: %d/%d channels could not be polled",
failed,
total,
extra=logs.event(
"sync.rss.failed",
failed=failed,
total=total,
reasons=dict(reasons),
examples=examples,
),
)
if total and failed == total:
# Nothing was polled at all — an egress outage (the tinyproxy-after-reboot incident), a
# dead DNS, a read-only database. Returning normally would record the run as `ok` with the