fix: stored logs keep bad rows, erased accounts and unconsented queries out
Deep-review round 1 of E22. - A NUL character (`GET /assets/%00` logs the decoded path) made Postgres refuse the row and with it the sink's whole batch, every user's rows included. `to_row` replaces NUL in every text column and in attrs. - An erased account's lines kept arriving after purge_user's DELETE: the erasing request's own access line, a queued row, a background task. `logs.forget_user` runs first; later rows are stored without the account, and the check and the insert share a lock with the erasure. - An admin setting the whole process to TRACE logged every user's query string. The query is now logged only under the user's own Teljes consent, and decoded before redaction, so `%40` no longer hides an address. - The browser log ingest has an hourly storage budget per account (5 MiB, one notice when spent) and reads only a JSON Content-Type, so a same-site text/plain form post cannot write into a user's log. RateLimiter takes a per-attempt cost for the byte budget. - The root level is applied under the elevation lock: two concurrent changes could leave it at the base while an elevation was active. - Tests for the running sink (level gate, drain on stop) and for the admin log level applied at runtime and at boot.
This commit is contained in:
@@ -12,7 +12,7 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from fastapi import BackgroundTasks
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy import create_engine, func, select
|
||||
|
||||
from app import logs, logsink, scheduler
|
||||
from app.auth import purge_user
|
||||
@@ -173,3 +173,75 @@ def test_deleting_an_account_deletes_its_stored_log_events(db, make_user):
|
||||
db.commit()
|
||||
purge_user(db, gone, BackgroundTasks())
|
||||
assert [r.user_id for r in db.execute(select(LogEvent)).scalars()] == [kept.id]
|
||||
|
||||
|
||||
def test_an_erased_accounts_late_lines_are_stored_without_it(stored, db, make_user):
|
||||
"""The DELETE in purge_user cannot reach a row still on its way: one queued before the erasure,
|
||||
or the erasing request's own access line, logged after the handler committed."""
|
||||
_sink, log, rows = stored
|
||||
user = make_user()
|
||||
try:
|
||||
log.warning("queued before the erasure", extra={"user_id": user.id})
|
||||
purge_user(db, user, BackgroundTasks())
|
||||
log.info(
|
||||
"DELETE /api/me/account 200", extra={**logs.event("http.request"), "user_id": user.id}
|
||||
)
|
||||
assert [(r.msg, r.user_id) for r in rows()] == [
|
||||
("queued before the erasure", None),
|
||||
("DELETE /api/me/account 200", None),
|
||||
]
|
||||
finally:
|
||||
logs._erased.clear()
|
||||
|
||||
|
||||
def test_a_nul_character_costs_nothing_but_itself(stored):
|
||||
"""Postgres refuses NUL in text and JSONB, and a refused row fails its whole batch — every other
|
||||
user's rows with it. `GET /assets/%00` logs exactly that."""
|
||||
_sink, log, rows = stored
|
||||
log.warning("before")
|
||||
log.warning("GET /assets/\x00 404", extra=logs.event("http.request", path="/assets/\x00"))
|
||||
log.warning("after")
|
||||
before, nul, after = rows()
|
||||
assert (before.msg, after.msg) == ("before", "after")
|
||||
assert nul.msg == "GET /assets/� 404" and nul.attrs["path"] == "/assets/�"
|
||||
|
||||
|
||||
def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
|
||||
sink = logsink.LogSink(engine)
|
||||
log = logging.getLogger("siftlode.sinktest.drain")
|
||||
log.addHandler(sink)
|
||||
try:
|
||||
for i in range(logsink.BATCH_MAX + 5):
|
||||
log.warning(f"line {i}") # distinct templates: no flood limiter involved
|
||||
finally:
|
||||
log.removeHandler(sink)
|
||||
sink._stopping.set()
|
||||
sink._run() # stopped before its first wait: everything goes through the drain
|
||||
assert (
|
||||
db.execute(select(func.count()).select_from(LogEvent)).scalar_one() == logsink.BATCH_MAX + 5
|
||||
)
|
||||
|
||||
|
||||
def test_the_running_sink_keeps_the_base_level_for_everyone_the_elevation_is_not_for(
|
||||
db, monkeypatch
|
||||
):
|
||||
"""`logsink.start()` as production runs it. One user's elevation lowers the ROOT logger, so an
|
||||
ordinary request line below the base level reaches every handler — the sink's own level gate is
|
||||
what keeps it out of the table, as stdout's keeps it off the terminal."""
|
||||
root = logging.getLogger()
|
||||
saved = (logs._base_level, dict(logs._elevations), logs._root_applied, root.level)
|
||||
monkeypatch.setattr(logsink.settings, "log_db_sink", True)
|
||||
logs.set_level("WARNING")
|
||||
logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
log = logging.getLogger("siftlode.sinktest.live")
|
||||
try:
|
||||
logsink.start()
|
||||
log.info("GET /api/feed 200", extra={**logs.event("http.request"), "user_id": 2})
|
||||
log.warning("a warning")
|
||||
finally:
|
||||
logsink.stop()
|
||||
logs._base_level, logs._root_applied = saved[0], saved[2]
|
||||
logs._elevations.clear()
|
||||
logs._elevations.update(saved[1])
|
||||
root.setLevel(saved[3])
|
||||
assert [r.msg for r in db.execute(select(LogEvent)).scalars()] == ["a warning"]
|
||||
|
||||
Reference in New Issue
Block a user