fix: stored logs keep bad rows, erased accounts and unconsented queries out

Deep-review round 1 of E22.

- A NUL character (`GET /assets/%00` logs the decoded path) made Postgres
  refuse the row and with it the sink's whole batch, every user's rows
  included. `to_row` replaces NUL in every text column and in attrs.
- An erased account's lines kept arriving after purge_user's DELETE: the
  erasing request's own access line, a queued row, a background task.
  `logs.forget_user` runs first; later rows are stored without the account,
  and the check and the insert share a lock with the erasure.
- An admin setting the whole process to TRACE logged every user's query
  string. The query is now logged only under the user's own Teljes consent,
  and decoded before redaction, so `%40` no longer hides an address.
- The browser log ingest has an hourly storage budget per account (5 MiB,
  one notice when spent) and reads only a JSON Content-Type, so a
  same-site text/plain form post cannot write into a user's log.
  RateLimiter takes a per-attempt cost for the byte budget.
- The root level is applied under the elevation lock: two concurrent
  changes could leave it at the base while an elevation was active.
- Tests for the running sink (level gate, drain on stop) and for the
  admin log level applied at runtime and at boot.

backend/tests/test_diagnostics.py

@@ -13,6 +13,7 @@ Three parts:
 
 import json
 import logging
+import threading
 from datetime import datetime, timedelta, timezone
 
 import pytest
@@ -20,7 +21,8 @@ from fastapi import Depends, FastAPI
 from fastapi.testclient import TestClient
 from sqlalchemy import select
 
-from app import diagnostics, logs, logsink
+from app import diagnostics, logs, logsink, sysconfig
+from app.config import settings
 from app.models import LogEvent, User
 from app.routes import diagnostics as diagnostics_routes
 
@@ -235,6 +237,38 @@ def test_the_pace_is_capped_per_user(elevated):
     assert codes[:30] == [200] * 30 and codes[30] == 429
 
 
+def test_what_an_account_stores_per_hour_is_capped_and_said_once(elevated, monkeypatch):
+    user, client, rows = elevated("DEBUG")
+    monkeypatch.setattr(
+        diagnostics_routes, "_stored_bytes", diagnostics_routes.RateLimiter(3_000, 3600)
+    )
+    batch = {"events": [_event(msg="m" * 1_000)]}
+    stored = [client.post("/api/diagnostics/events", json=batch).json()["stored"] for _ in range(4)]
+    assert stored == [1, 1, 0, 0]
+    events = [row.event for row in rows()]
+    # Two batches, then ONE notice for the hour, not one per refused batch.
+    assert events == ["nav.view", "nav.view", "diagnostics.budget_spent"]
+    assert rows()[-1].user_id == user.id
+
+
+def test_only_a_json_body_is_read(elevated):
+    _user, client, rows = elevated("DEBUG")
+    # What a same-site `<form enctype="text/plain">` can send without a preflight.
+    r = client.post(
+        "/api/diagnostics/events",
+        content=json.dumps({"events": [_event()]}),
+        headers={"content-type": "text/plain"},
+    )
+    assert r.status_code == 415 and r.json()["detail"]["code"] == "json_required"
+    assert rows() == []
+    ok = client.post(
+        "/api/diagnostics/events",
+        content=json.dumps({"events": [_event()]}),
+        headers={"content-type": "application/json; charset=utf-8"},
+    )
+    assert ok.json()["stored"] == 1
+
+
 # --- the server side of an elevation ---------------------------------------------------------
 
 
@@ -330,6 +364,82 @@ def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated
     assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
 
 
+def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
+    _app().get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
+    (line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
+    assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
+
+
+def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
+    logs.set_level("TRACE")
+    _app().get("/api/thing/7", params={"q": "something private"})
+    (line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
+    assert "query" not in line.attrs
+
+
+def test_the_admin_level_applies_to_the_running_process_when_set_and_when_reset(
+    gated, db, make_user, as_user
+):
+    admin = as_user(make_user(role="admin"))
+    r = admin.patch("/api/admin/config/log_level", json={"value": "DEBUG"})
+    assert r.status_code == 200, r.text
+    assert logs.current_level() == "DEBUG"
+    assert admin.delete("/api/admin/config/log_level").status_code == 200
+    assert logs.current_level() == settings.log_level
+
+
+def test_the_stored_admin_level_is_applied_at_boot(gated, db, monkeypatch):
+    import asyncio
+
+    import app.main as app_main
+
+    sysconfig.set_value(db, "log_level", "DEBUG")
+    db.commit()
+    monkeypatch.setattr(app_main.state, "is_configured", lambda db: True)
+    monkeypatch.setattr(app_main, "start_scheduler", lambda: None)
+    monkeypatch.setattr(app_main, "shutdown_scheduler", lambda: None)
+    monkeypatch.setattr(app_main.plex_subtitles, "shutdown", lambda: None)
+
+    async def _no_sweep():
+        return None
+
+    monkeypatch.setattr(app_main, "_sweep_hls_orphans", _no_sweep)
+    levels: list[str] = []
+
+    async def _cycle():
+        async with app_main.lifespan(app_main.app):
+            levels.append(logs.current_level())
+
+    asyncio.run(_cycle())
+    assert levels == ["DEBUG"]
+
+
+def test_the_root_level_is_applied_in_the_order_the_elevations_changed(gated, monkeypatch):
+    """Two requests change the elevations at once: A's user leaves (the root may go back to INFO),
+    B's user arrives (it must go to DEBUG). B's change is the later one, so DEBUG must be what stays
+    — even when A is paused between working out its level and applying it."""
+    root = logging.getLogger()
+    logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
+    real_set = root.setLevel
+    other: list[threading.Thread] = []
+
+    def paused_set(level):
+        if level == logging.INFO and not other:
+            b = threading.Thread(
+                target=logs._track_elevation,
+                args=(2, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1)),
+            )
+            other.append(b)
+            b.start()
+            b.join(0.3)  # B runs to completion here unless A's level change holds it off
+        real_set(level)
+
+    monkeypatch.setattr(root, "setLevel", paused_set)
+    logs._track_elevation(1, None, None)  # A
+    other[0].join(5)
+    assert root.level == logging.DEBUG and 2 in logs._elevations
+
+
 def test_a_real_sign_in_carries_the_consented_level_into_the_process(gated, client, db, make_user):
     user = _signed_in(client, make_user)
     assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200

backend/tests/test_logs.py

@@ -102,7 +102,7 @@ def test_an_email_in_an_attribute_value_is_scrubbed_too():
     assert logs.redact({"to": "a.b+tag@mail.example.org"})["to"] == logs.REDACTED_EMAIL
 
 
-# The SAME cases frontend/src/lib/log.redaction.test.ts runs against the browser's copy of redact():
+# The SAME cases frontend/src/lib/log.contract.test.ts runs against the browser's copy of redact():
 # the one thing that keeps the two logs' idea of "safe to keep" from drifting apart.
 _REDACTION_CASES = json.loads(
     (Path(__file__).parent / "fixtures" / "log_redaction_cases.json").read_text(encoding="utf-8")

backend/tests/test_logsink.py

@@ -12,7 +12,7 @@ from datetime import datetime, timedelta, timezone
 
 import pytest
 from fastapi import BackgroundTasks
-from sqlalchemy import create_engine, select
+from sqlalchemy import create_engine, func, select
 
 from app import logs, logsink, scheduler
 from app.auth import purge_user
@@ -173,3 +173,75 @@ def test_deleting_an_account_deletes_its_stored_log_events(db, make_user):
     db.commit()
     purge_user(db, gone, BackgroundTasks())
     assert [r.user_id for r in db.execute(select(LogEvent)).scalars()] == [kept.id]
+
+
+def test_an_erased_accounts_late_lines_are_stored_without_it(stored, db, make_user):
+    """The DELETE in purge_user cannot reach a row still on its way: one queued before the erasure,
+    or the erasing request's own access line, logged after the handler committed."""
+    _sink, log, rows = stored
+    user = make_user()
+    try:
+        log.warning("queued before the erasure", extra={"user_id": user.id})
+        purge_user(db, user, BackgroundTasks())
+        log.info(
+            "DELETE /api/me/account 200", extra={**logs.event("http.request"), "user_id": user.id}
+        )
+        assert [(r.msg, r.user_id) for r in rows()] == [
+            ("queued before the erasure", None),
+            ("DELETE /api/me/account 200", None),
+        ]
+    finally:
+        logs._erased.clear()
+
+
+def test_a_nul_character_costs_nothing_but_itself(stored):
+    """Postgres refuses NUL in text and JSONB, and a refused row fails its whole batch — every other
+    user's rows with it. `GET /assets/%00` logs exactly that."""
+    _sink, log, rows = stored
+    log.warning("before")
+    log.warning("GET /assets/\x00 404", extra=logs.event("http.request", path="/assets/\x00"))
+    log.warning("after")
+    before, nul, after = rows()
+    assert (before.msg, after.msg) == ("before", "after")
+    assert nul.msg == "GET /assets/� 404" and nul.attrs["path"] == "/assets/�"
+
+
+def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
+    sink = logsink.LogSink(engine)
+    log = logging.getLogger("siftlode.sinktest.drain")
+    log.addHandler(sink)
+    try:
+        for i in range(logsink.BATCH_MAX + 5):
+            log.warning(f"line {i}")  # distinct templates: no flood limiter involved
+    finally:
+        log.removeHandler(sink)
+    sink._stopping.set()
+    sink._run()  # stopped before its first wait: everything goes through the drain
+    assert (
+        db.execute(select(func.count()).select_from(LogEvent)).scalar_one() == logsink.BATCH_MAX + 5
+    )
+
+
+def test_the_running_sink_keeps_the_base_level_for_everyone_the_elevation_is_not_for(
+    db, monkeypatch
+):
+    """`logsink.start()` as production runs it. One user's elevation lowers the ROOT logger, so an
+    ordinary request line below the base level reaches every handler — the sink's own level gate is
+    what keeps it out of the table, as stdout's keeps it off the terminal."""
+    root = logging.getLogger()
+    saved = (logs._base_level, dict(logs._elevations), logs._root_applied, root.level)
+    monkeypatch.setattr(logsink.settings, "log_db_sink", True)
+    logs.set_level("WARNING")
+    logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
+    log = logging.getLogger("siftlode.sinktest.live")
+    try:
+        logsink.start()
+        log.info("GET /api/feed 200", extra={**logs.event("http.request"), "user_id": 2})
+        log.warning("a warning")
+    finally:
+        logsink.stop()
+        logs._base_level, logs._root_applied = saved[0], saved[2]
+        logs._elevations.clear()
+        logs._elevations.update(saved[1])
+        root.setLevel(saved[3])
+    assert [r.msg for r in db.execute(select(LogEvent)).scalars()] == ["a warning"]