fix: stored logs keep bad rows, erased accounts and unconsented queries out

Deep-review round 1 of E22.

- A NUL character (`GET /assets/%00` logs the decoded path) made Postgres
  refuse the row and with it the sink's whole batch, every user's rows
  included. `to_row` replaces NUL in every text column and in attrs.
- An erased account's lines kept arriving after purge_user's DELETE: the
  erasing request's own access line, a queued row, a background task.
  `logs.forget_user` runs first; later rows are stored without the account,
  and the check and the insert share a lock with the erasure.
- An admin setting the whole process to TRACE logged every user's query
  string. The query is now logged only under the user's own Teljes consent,
  and decoded before redaction, so `%40` no longer hides an address.
- The browser log ingest has an hourly storage budget per account (5 MiB,
  one notice when spent) and reads only a JSON Content-Type, so a
  same-site text/plain form post cannot write into a user's log.
  RateLimiter takes a per-attempt cost for the byte budget.
- The root level is applied under the elevation lock: two concurrent
  changes could leave it at the base while an elevation was active.
- Tests for the running sink (level gate, drain on stop) and for the
  admin log level applied at runtime and at boot.
This commit is contained in:
2026-09-17 03:53:29 +02:00
parent 9c9b1cf6d5
commit fac5c4465a
9 changed files with 349 additions and 39 deletions
+73 -1
View File
@@ -12,7 +12,7 @@ from datetime import datetime, timedelta, timezone
import pytest
from fastapi import BackgroundTasks
from sqlalchemy import create_engine, select
from sqlalchemy import create_engine, func, select
from app import logs, logsink, scheduler
from app.auth import purge_user
@@ -173,3 +173,75 @@ def test_deleting_an_account_deletes_its_stored_log_events(db, make_user):
db.commit()
purge_user(db, gone, BackgroundTasks())
assert [r.user_id for r in db.execute(select(LogEvent)).scalars()] == [kept.id]
def test_an_erased_accounts_late_lines_are_stored_without_it(stored, db, make_user):
"""The DELETE in purge_user cannot reach a row still on its way: one queued before the erasure,
or the erasing request's own access line, logged after the handler committed."""
_sink, log, rows = stored
user = make_user()
try:
log.warning("queued before the erasure", extra={"user_id": user.id})
purge_user(db, user, BackgroundTasks())
log.info(
"DELETE /api/me/account 200", extra={**logs.event("http.request"), "user_id": user.id}
)
assert [(r.msg, r.user_id) for r in rows()] == [
("queued before the erasure", None),
("DELETE /api/me/account 200", None),
]
finally:
logs._erased.clear()
def test_a_nul_character_costs_nothing_but_itself(stored):
"""Postgres refuses NUL in text and JSONB, and a refused row fails its whole batch — every other
user's rows with it. `GET /assets/%00` logs exactly that."""
_sink, log, rows = stored
log.warning("before")
log.warning("GET /assets/\x00 404", extra=logs.event("http.request", path="/assets/\x00"))
log.warning("after")
before, nul, after = rows()
assert (before.msg, after.msg) == ("before", "after")
assert nul.msg == "GET /assets/� 404" and nul.attrs["path"] == "/assets/�"
def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
sink = logsink.LogSink(engine)
log = logging.getLogger("siftlode.sinktest.drain")
log.addHandler(sink)
try:
for i in range(logsink.BATCH_MAX + 5):
log.warning(f"line {i}") # distinct templates: no flood limiter involved
finally:
log.removeHandler(sink)
sink._stopping.set()
sink._run() # stopped before its first wait: everything goes through the drain
assert (
db.execute(select(func.count()).select_from(LogEvent)).scalar_one() == logsink.BATCH_MAX + 5
)
def test_the_running_sink_keeps_the_base_level_for_everyone_the_elevation_is_not_for(
db, monkeypatch
):
"""`logsink.start()` as production runs it. One user's elevation lowers the ROOT logger, so an
ordinary request line below the base level reaches every handler — the sink's own level gate is
what keeps it out of the table, as stdout's keeps it off the terminal."""
root = logging.getLogger()
saved = (logs._base_level, dict(logs._elevations), logs._root_applied, root.level)
monkeypatch.setattr(logsink.settings, "log_db_sink", True)
logs.set_level("WARNING")
logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
log = logging.getLogger("siftlode.sinktest.live")
try:
logsink.start()
log.info("GET /api/feed 200", extra={**logs.event("http.request"), "user_id": 2})
log.warning("a warning")
finally:
logsink.stop()
logs._base_level, logs._root_applied = saved[0], saved[2]
logs._elevations.clear()
logs._elevations.update(saved[1])
root.setLevel(saved[3])
assert [r.msg for r in db.execute(select(LogEvent)).scalars()] == ["a warning"]