peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: store warnings and request lines in log_events

Browse Source 
The log now has a stored, queryable copy for the diagnostics console:
every WARNING and above plus the request line of every request, kept
for log_retention_days (default 7, no "forever") and pruned hourly by
the new log_gc job.

Rows are built on the logging thread from the same redacted fields as
the stdout line, queued without blocking and written in batches by one
background thread on its own connection. A full queue or a failed write
drops and reports instead of retrying. Flood control keeps 30 of any
one message per minute plus a count of the rest; anonymous request
lines are capped at 60 a minute per client, and the client address is
never stored. Deleting an account deletes its stored events.

Measured on prod first: a YouTube RSS outage wrote 11k warnings in four
hours, one per channel per poll. The poll now writes one warning with
per-reason counts and a few examples, and a failure on our side gets
one traceback per poll instead of one per channel.
This commit is contained in:
peter
2026-09-17 00:20:14 +02:00
parent 88a166e4aa
commit b59fd38a9c
23 changed files with 832 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/logs.py
+84 -40
View File
@@ -41,6 +41,7 @@ from datetime import datetime, timezone
from typing import Any
from app.config import settings
from app.ratelimit import RateLimiter, client_key
# One level below DEBUG for the "Teljes / TRACE" diagnostics level (search strings, redacted request
# bodies). Registered on import so `logging.getLogger(...).log(logs.TRACE, …)` and a `LOG_LEVEL=TRACE`
@@ -157,6 +158,9 @@ _SECRET_PARAM_RE = re.compile(
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
REDACTED = "[redacted]"
# `extra={PERSIST: False}` keeps a line out of the `log_events` table while stdout still gets it —
# how `log_requests` caps anonymous traffic per client (app/logsink.py reads it).
PERSIST = "persist"
REDACTED_EMAIL = "[email]"
@@ -189,7 +193,7 @@ def redact(value: Any, *, key: str = "") -> Any:
# attribute and is merged into `attrs` (so `extra={"job_id": 7}` works as well as `event(...)`).
_RECORD_STANDARD = frozenset(
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
| {"message", "asctime", "taskName", "event", "attrs"}
| {"message", "asctime", "taskName", "event", "attrs", PERSIST}
)
@@ -222,33 +226,43 @@ def _safe_message(record: logging.LogRecord) -> str:
return f"{record.msg!r} (log formatting failed: {exc})"
def record_fields(record: logging.LogRecord) -> dict[str, Any]:
"""The event as structured fields, REDACTED — the one definition both the stdout line and the
`log_events` row are built from, so the two sinks can never disagree about what is safe to keep.
Absent optional fields are left out; `ts` is an aware datetime."""
fields: dict[str, Any] = {
"ts": datetime.fromtimestamp(record.created, timezone.utc),
"level": record.levelname,
"sev": SEVERITY.get(record.levelname, 9),
"source": _source,
"logger": record.name,
"msg": redact_text(_safe_message(record)),
}
code = getattr(record, "event", None)
if code:
fields["event"] = code
for name in _CONTEXT_VARS:
value = getattr(record, name, None)
if value is not None:
fields[name] = value
attrs = _record_attrs(record)
if attrs:
fields["attrs"] = redact(attrs)
if record.exc_info:
fields["exc"] = redact_text(logging.Formatter().formatException(record.exc_info))
elif record.exc_text:
# A QueueHandler-style hand-off formats the traceback into exc_text and drops exc_info.
fields["exc"] = redact_text(record.exc_text)
fields["version"] = settings.app_version
return fields
class JsonFormatter(logging.Formatter):
"""One JSON object per line — the shape Loki/the `log_events` table both ingest."""
"""One JSON object per line — the shape Loki and the `log_events` table both ingest."""
def format(self, record: logging.LogRecord) -> str:
payload: dict[str, Any] = {
"ts": datetime.fromtimestamp(record.created, timezone.utc).isoformat(
timespec="milliseconds"
),
"level": record.levelname,
"sev": SEVERITY.get(record.levelname, 9),
"source": _source,
"logger": record.name,
"msg": redact_text(_safe_message(record)),
}
code = getattr(record, "event", None)
if code:
payload["event"] = code
for name in _CONTEXT_VARS:
value = getattr(record, name, None)
if value is not None:
payload[name] = value
attrs = _record_attrs(record)
if attrs:
payload["attrs"] = redact(attrs)
if record.exc_info:
payload["exc"] = redact_text(self.formatException(record.exc_info))
payload["version"] = settings.app_version
payload = record_fields(record)
payload["ts"] = payload["ts"].isoformat(timespec="milliseconds")
return json.dumps(payload, ensure_ascii=False, default=str)
@@ -352,6 +366,11 @@ def current_level() -> str:
return logging.getLevelName(logging.getLogger().getEffectiveLevel())
def current_source() -> str:
"""The process name `configure_logging` was given (api, worker, runner, alembic)."""
return _source
# --- request logging -----------------------------------------------------------------------
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
@@ -388,13 +407,15 @@ async def log_requests(request, call_next):
# access-log entry at all.
_request_log.exception(
"request failed",
extra=event(
"http.request",
method=request.method,
path=_route_template(request),
duration_ms=round((time.perf_counter() - started) * 1000, 1),
user_id=getattr(request.state, "user_id", None),
),
extra={
**event(
"http.request",
method=request.method,
path=_route_template(request),
duration_ms=round((time.perf_counter() - started) * 1000, 1),
),
**_request_owner(request, logging.ERROR),
},
)
_request_id.reset(token)
raise
@@ -413,20 +434,43 @@ async def log_requests(request, call_next):
request.method,
_route_template(request),
status,
extra=event(
"http.request",
method=request.method,
path=_route_template(request),
status=status,
duration_ms=duration_ms,
user_id=getattr(request.state, "user_id", None),
),
extra={
**event(
"http.request",
method=request.method,
path=_route_template(request),
status=status,
duration_ms=duration_ms,
),
**_request_owner(request, level),
},
)
response.headers["X-Request-Id"] = rid
_request_id.reset(token)
return response
# Anonymous request lines kept per client per minute. Signed-in traffic is not capped: it is a known
# account and exactly what an admin needs to see. Anonymous traffic is where the volume can come from
# nobody in particular — a scanner walking `/wp-login.php`, a bot on a share link — and each of its
# 404s is a WARNING that would otherwise land in `log_events` without limit. The client key is an
# in-memory counter key only; no address is ever written to a log line or a row.
ANONYMOUS_LINES_PER_MINUTE = 60
_anonymous_lines = RateLimiter(ANONYMOUS_LINES_PER_MINUTE, 60.0)
def _request_owner(request, level: int) -> dict[str, Any]:
"""`extra=` fields naming who the request belongs to, and whether its line may be stored. The
account comes from `request.state` (filled by the auth dependency), so it lands in the line's
top-level `user_id` — the correlation field — rather than in `attrs`."""
user_id = getattr(request.state, "user_id", None)
if user_id is not None:
return {"user_id": user_id}
if level < logging.INFO:
return {} # a quiet line is never stored anyway — don't spend the client's allowance on it
return {} if _anonymous_lines.allow(client_key(request)) else {PERSIST: False}
def _route_template(request) -> str:
"""The matched route's template (`/api/videos/{video_id}`), else the raw path for a 404 or a
static file. Never the query string."""