fix: stored logs keep bad rows, erased accounts and unconsented queries out
Deep-review round 1 of E22. - A NUL character (`GET /assets/%00` logs the decoded path) made Postgres refuse the row and with it the sink's whole batch, every user's rows included. `to_row` replaces NUL in every text column and in attrs. - An erased account's lines kept arriving after purge_user's DELETE: the erasing request's own access line, a queued row, a background task. `logs.forget_user` runs first; later rows are stored without the account, and the check and the insert share a lock with the erasure. - An admin setting the whole process to TRACE logged every user's query string. The query is now logged only under the user's own Teljes consent, and decoded before redaction, so `%40` no longer hides an address. - The browser log ingest has an hourly storage budget per account (5 MiB, one notice when spent) and reads only a JSON Content-Type, so a same-site text/plain form post cannot write into a user's log. RateLimiter takes a per-attempt cost for the byte budget. - The root level is applied under the elevation lock: two concurrent changes could leave it at the base while an elevation was active. - Tests for the running sink (level gate, drain on stop) and for the admin log level applied at runtime and at boot.
This commit is contained in:
@@ -13,6 +13,7 @@ Three parts:
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
@@ -20,7 +21,8 @@ from fastapi import Depends, FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import diagnostics, logs, logsink
|
||||
from app import diagnostics, logs, logsink, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import LogEvent, User
|
||||
from app.routes import diagnostics as diagnostics_routes
|
||||
|
||||
@@ -235,6 +237,38 @@ def test_the_pace_is_capped_per_user(elevated):
|
||||
assert codes[:30] == [200] * 30 and codes[30] == 429
|
||||
|
||||
|
||||
def test_what_an_account_stores_per_hour_is_capped_and_said_once(elevated, monkeypatch):
|
||||
user, client, rows = elevated("DEBUG")
|
||||
monkeypatch.setattr(
|
||||
diagnostics_routes, "_stored_bytes", diagnostics_routes.RateLimiter(3_000, 3600)
|
||||
)
|
||||
batch = {"events": [_event(msg="m" * 1_000)]}
|
||||
stored = [client.post("/api/diagnostics/events", json=batch).json()["stored"] for _ in range(4)]
|
||||
assert stored == [1, 1, 0, 0]
|
||||
events = [row.event for row in rows()]
|
||||
# Two batches, then ONE notice for the hour, not one per refused batch.
|
||||
assert events == ["nav.view", "nav.view", "diagnostics.budget_spent"]
|
||||
assert rows()[-1].user_id == user.id
|
||||
|
||||
|
||||
def test_only_a_json_body_is_read(elevated):
|
||||
_user, client, rows = elevated("DEBUG")
|
||||
# What a same-site `<form enctype="text/plain">` can send without a preflight.
|
||||
r = client.post(
|
||||
"/api/diagnostics/events",
|
||||
content=json.dumps({"events": [_event()]}),
|
||||
headers={"content-type": "text/plain"},
|
||||
)
|
||||
assert r.status_code == 415 and r.json()["detail"]["code"] == "json_required"
|
||||
assert rows() == []
|
||||
ok = client.post(
|
||||
"/api/diagnostics/events",
|
||||
content=json.dumps({"events": [_event()]}),
|
||||
headers={"content-type": "application/json; charset=utf-8"},
|
||||
)
|
||||
assert ok.json()["stored"] == 1
|
||||
|
||||
|
||||
# --- the server side of an elevation ---------------------------------------------------------
|
||||
|
||||
|
||||
@@ -330,6 +364,82 @@ def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated
|
||||
assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
|
||||
|
||||
|
||||
def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
|
||||
_app().get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
|
||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||
assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
|
||||
|
||||
|
||||
def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
|
||||
logs.set_level("TRACE")
|
||||
_app().get("/api/thing/7", params={"q": "something private"})
|
||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||
assert "query" not in line.attrs
|
||||
|
||||
|
||||
def test_the_admin_level_applies_to_the_running_process_when_set_and_when_reset(
|
||||
gated, db, make_user, as_user
|
||||
):
|
||||
admin = as_user(make_user(role="admin"))
|
||||
r = admin.patch("/api/admin/config/log_level", json={"value": "DEBUG"})
|
||||
assert r.status_code == 200, r.text
|
||||
assert logs.current_level() == "DEBUG"
|
||||
assert admin.delete("/api/admin/config/log_level").status_code == 200
|
||||
assert logs.current_level() == settings.log_level
|
||||
|
||||
|
||||
def test_the_stored_admin_level_is_applied_at_boot(gated, db, monkeypatch):
|
||||
import asyncio
|
||||
|
||||
import app.main as app_main
|
||||
|
||||
sysconfig.set_value(db, "log_level", "DEBUG")
|
||||
db.commit()
|
||||
monkeypatch.setattr(app_main.state, "is_configured", lambda db: True)
|
||||
monkeypatch.setattr(app_main, "start_scheduler", lambda: None)
|
||||
monkeypatch.setattr(app_main, "shutdown_scheduler", lambda: None)
|
||||
monkeypatch.setattr(app_main.plex_subtitles, "shutdown", lambda: None)
|
||||
|
||||
async def _no_sweep():
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_main, "_sweep_hls_orphans", _no_sweep)
|
||||
levels: list[str] = []
|
||||
|
||||
async def _cycle():
|
||||
async with app_main.lifespan(app_main.app):
|
||||
levels.append(logs.current_level())
|
||||
|
||||
asyncio.run(_cycle())
|
||||
assert levels == ["DEBUG"]
|
||||
|
||||
|
||||
def test_the_root_level_is_applied_in_the_order_the_elevations_changed(gated, monkeypatch):
|
||||
"""Two requests change the elevations at once: A's user leaves (the root may go back to INFO),
|
||||
B's user arrives (it must go to DEBUG). B's change is the later one, so DEBUG must be what stays
|
||||
— even when A is paused between working out its level and applying it."""
|
||||
root = logging.getLogger()
|
||||
logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
real_set = root.setLevel
|
||||
other: list[threading.Thread] = []
|
||||
|
||||
def paused_set(level):
|
||||
if level == logging.INFO and not other:
|
||||
b = threading.Thread(
|
||||
target=logs._track_elevation,
|
||||
args=(2, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1)),
|
||||
)
|
||||
other.append(b)
|
||||
b.start()
|
||||
b.join(0.3) # B runs to completion here unless A's level change holds it off
|
||||
real_set(level)
|
||||
|
||||
monkeypatch.setattr(root, "setLevel", paused_set)
|
||||
logs._track_elevation(1, None, None) # A
|
||||
other[0].join(5)
|
||||
assert root.level == logging.DEBUG and 2 in logs._elevations
|
||||
|
||||
|
||||
def test_a_real_sign_in_carries_the_consented_level_into_the_process(gated, client, db, make_user):
|
||||
user = _signed_in(client, make_user)
|
||||
assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200
|
||||
|
||||
Reference in New Issue
Block a user