peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(logs): deep-review round 2 — a stack names code, not data, and survives a failed write

Browse Source 
- A failure keeps its traceback unless it is yt-dlp's or our own StatedFailure (a
  message that is the whole story). A plain RuntimeError, RecursionError or OSError
  is also what a bug of ours raises, and its stack is the only lead.
- The stack is scrubbed link by link (logs.scrubbed_stack): a chained DownloadError
  under a DB error no longer prints yt-dlp's text unscrubbed.
- Engine hide_parameters: a DB error's message no longer carries the title or path
  it was writing.
- The download and edit failure lines go out even when their DB write fails.
- The worker's child_exit/crash/supervisor lines say download.job.failed only when
  the job failed there; a crash the DB will not record no longer ends the thread.
  Stopping a paused child is download.child.stopped, not a second abort.
- log_safe also covers generic ids with spaces, playlist titles and bare ids.
- The ffmpeg image steps log the failure kind, not the argv with its path.
- The consent contract accepts only the body of a plain consented_trace() check,
  and any level that may be TRACE.
- Tests for each; every one mutation-checked.
This commit is contained in:
peter
2026-09-18 22:50:37 +02:00
parent 54d19cc386
commit 0b672ed6e7
14 changed files with 729 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_log_pii.py
+118 -12
View File
@@ -127,15 +127,31 @@ def test_a_scope_with_no_elevation_is_not_consent():
assert not logs.consented_trace() # a scheduler thread: no request at all
def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink):
def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink, monkeypatch):
"""The token endpoint's error is `{"error": "<code>", "error_description": <prose>}` — not the
Data API's shape. It goes through the same door, so it gets the same treatment."""
Data API's shape. The refresh goes through the same door, so it gets the same treatment."""
from types import SimpleNamespace
from app.youtube import client as yt
resp = httpx.Response(
400,
json={"error": "invalid_grant", "error_description": "Token has been expired or revoked."},
request=httpx.Request("POST", "https://oauth2.googleapis.com/token"),
)
err = _upstream_failure("POST", "oauth2/token", resp)
client = yt.YouTubeClient.__new__(yt.YouTubeClient) # no HTTP client: `_send` is the stub
client.db = None
client.user = SimpleNamespace(
token=SimpleNamespace(access_token=None, expiry=None, refresh_token_enc=b"x")
)
monkeypatch.setattr(yt, "decrypt_optional", lambda value: None)
monkeypatch.setattr(yt, "decrypt", lambda value: "a-refresh-token")
monkeypatch.setattr(yt.sysconfig, "get_str", lambda db, key: "")
monkeypatch.setattr(client, "_send", lambda *a, **k: resp)
with pytest.raises(YouTubeError) as caught:
client._access_token()
err = caught.value
(line,) = sink.at("WARNING")
assert line["attrs"]["reason"] == "invalid_grant"
@@ -267,16 +283,88 @@ def test_no_log_call_passes_what_a_person_typed():
)
_AT_LEAST_DEBUG = {"DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"}
def _may_run_at_trace(call: ast.Call) -> bool:
"""A `log.log(level, …)` whose level is not provably DEBUG or above: `logs.TRACE`, a bare
`TRACE`, a literal 5 — or a variable, which could be any of them."""
if call.func.attr != "log" or not call.args:
return False
level = call.args[0]
if isinstance(level, ast.Attribute | ast.Name):
return (level.attr if isinstance(level, ast.Attribute) else level.id) not in _AT_LEAST_DEBUG
if isinstance(level, ast.Constant) and isinstance(level.value, int):
return level.value < logging.DEBUG
return True
def _is_consent_check(test: ast.expr) -> bool:
"""Exactly `…consented_trace()`: not `not …`, not `x or …` — each of those runs the body when
there is no consent."""
return (
isinstance(test, ast.Call)
and isinstance(test.func, ast.Attribute | ast.Name)
and (test.func.attr if isinstance(test.func, ast.Attribute) else test.func.id)
== "consented_trace"
)
def _guarded_by_consent(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> bool:
"""The node runs only inside the BODY of an `if <consented_trace()>:` — not its `else`."""
while node in parents:
node = parents[node]
if isinstance(node, ast.If):
for sub in ast.walk(node.test):
if isinstance(sub, ast.Attribute) and sub.attr == "consented_trace":
return True
parent = parents[node]
if isinstance(parent, ast.If) and _is_consent_check(parent.test) and node in parent.body:
return True
node = parent
return False
def _parents(tree: ast.AST) -> dict[ast.AST, ast.AST]:
return {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
_GATE_SHAPES = {
"the body of the check": ("if logs.consented_trace():\n log.log(logs.TRACE, 'x')", True),
"its else": (
"if logs.consented_trace():\n pass\nelse:\n log.log(logs.TRACE, 'x')",
False,
),
"a negated check": ("if not logs.consented_trace():\n log.log(logs.TRACE, 'x')", False),
"an or": ("if debug or logs.consented_trace():\n log.log(logs.TRACE, 'x')", False),
"a nested block in the body": (
"if logs.consented_trace():\n for t in ts:\n log.log(logs.TRACE, 'x')",
True,
),
}
@pytest.mark.parametrize("shape", list(_GATE_SHAPES))
def test_the_consent_gate_is_the_body_of_the_check_and_nothing_else(shape):
src, gated = _GATE_SHAPES[shape]
tree = ast.parse(src)
(call,) = [n for n in ast.walk(tree) if isinstance(n, ast.Call) and _is_log_call(n)]
assert _guarded_by_consent(call, _parents(tree)) is gated
@pytest.mark.parametrize(
("level", "trace"),
[
("logs.TRACE", True),
("TRACE", True),
("5", True),
("level", True),
("levels[scope]", True),
("logging.DEBUG", False),
("logging.WARNING", False),
("20", False),
],
)
def test_a_level_that_may_be_trace_is_treated_as_trace(level, trace):
(call,) = [n for n in ast.walk(ast.parse(f"log.log({level}, 'x')")) if isinstance(n, ast.Call)]
assert _may_run_at_trace(call) is trace
def test_every_escaped_trace_line_is_gated_on_the_users_own_consent():
"""`log-pii-ok` on a TRACE line means "this is what the user's Teljes consent is for". The TRACE
level alone does not hold that promise: an admin can run the whole process at TRACE, and a
@@ -287,15 +375,33 @@ def test_every_escaped_trace_line_is_gated_on_the_users_own_consent():
src = path.read_text(encoding="utf-8")
lines = src.splitlines()
tree = ast.parse(src)
parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
parents = _parents(tree)
for node in ast.walk(tree):
if not _is_log_call(node) or not _offending(node):
continue
first = node.args[0] if node.args else None
at_trace = isinstance(first, ast.Attribute) and first.attr == "TRACE"
span = " ".join(lines[max(0, node.lineno - 2) : (node.end_lineno or node.lineno)])
if at_trace and _ESCAPE in span and not _guarded_by_consent(node, parents):
if (
_may_run_at_trace(node)
and _ESCAPE in span
and not _guarded_by_consent(node, parents)
):
offenders.append(f"{path.relative_to(root)}:{node.lineno}")
assert offenders == [], (
f"a TRACE line with user content is not under consented_trace(): {offenders}"
)
def test_a_database_error_does_not_carry_the_values_it_was_given(db):
"""DB lane. A failed write's message is logged and stored as the job's error text; with its
bound parameters in it, that is the title or the file path the row was about."""
from sqlalchemy import exc as sa_exc
from sqlalchemy import text
# Postgres's own text for this one names no value ("value too long for type character
# varying(3)"), so what is left to leak is SQLAlchemy's `[parameters: …]` suffix.
db.execute(text("CREATE TEMP TABLE pii_probe (x varchar(3))"))
with pytest.raises(sa_exc.DBAPIError) as caught:
db.execute(text("INSERT INTO pii_probe VALUES (:v)"), {"v": "A_Private_Title"})
db.rollback()
assert "value too long" in str(caught.value)
assert "A_Private_Title" not in str(caught.value)