fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere

- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
  `formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
  file paths, and the job-failed, refused, edit-failed and backfill lines use
  it. Done lines log the asset id, not the title-built path. The PII contract
  now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
  failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
  is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
  set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
  a process's last batch closes the repeat window, so a short-lived download
  child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
  `download.job.failed` with a cause; they left no line at all before. The
  crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
  and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
  not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
  carry reason `transport`. Subscribe recognises a duplicate by reason code
  only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
  directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
  were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
This commit is contained in:
2026-09-18 03:16:15 +02:00
parent 04b58e5c6f
commit 54d19cc386
22 changed files with 834 additions and 141 deletions
+92 -4
View File
@@ -87,15 +87,62 @@ def test_an_upstream_failure_logs_the_reason_code_and_not_the_body(sink):
assert err.reason == "quotaExceeded"
def test_the_body_is_kept_for_the_one_user_who_consented_to_it(sink):
@pytest.fixture
def consented():
"""The request scope of a user who turned Teljes (TRACE) on for themselves."""
scope = logs._RequestScope()
scope.user_id, scope.level = 7, logs.TRACE
token = logs._scope.set(scope)
try:
yield
finally:
logs._scope.reset(token)
def test_the_body_is_kept_for_the_one_user_who_consented_to_it(sink, consented):
_upstream_failure("GET", "search", _youtube_403(TERM))
# TRACE is emitted only while the user has turned Teljes on for their own session, which is
# exactly the consent the privacy policy describes. The diagnostic is not lost, it is gated.
# The diagnostic is not lost, it is gated — on the consent the privacy policy describes.
(trace,) = sink.at("TRACE")
assert TERM in trace["attrs"]["body"]
def test_a_process_running_at_trace_is_not_anyones_consent(sink):
"""`sink` runs the whole tree at TRACE — what an admin setting `log_level` to TRACE does. That
buys more detail, not every user's search terms quoted back (`logs._traced`)."""
_upstream_failure("GET", "search", _youtube_403(TERM))
assert sink.at("TRACE") == []
assert TERM not in json.dumps(sink.objects())
def test_a_scope_with_no_elevation_is_not_consent():
scope = logs._RequestScope()
scope.user_id = 7
token = logs._scope.set(scope)
try:
assert not logs.consented_trace()
finally:
logs._scope.reset(token)
assert not logs.consented_trace() # a scheduler thread: no request at all
def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink):
"""The token endpoint's error is `{"error": "<code>", "error_description": <prose>}` — not the
Data API's shape. It goes through the same door, so it gets the same treatment."""
resp = httpx.Response(
400,
json={"error": "invalid_grant", "error_description": "Token has been expired or revoked."},
request=httpx.Request("POST", "https://oauth2.googleapis.com/token"),
)
err = _upstream_failure("POST", "oauth2/token", resp)
(line,) = sink.at("WARNING")
assert line["attrs"]["reason"] == "invalid_grant"
assert err.reason == "invalid_grant"
assert "revoked" not in str(err) and "revoked" not in json.dumps(line)
def test_an_unparseable_body_still_names_the_status(sink):
"""A proxy answering HTML has no `reason` to take — the line must still be worth reading, and
must still not fall back to printing the body."""
@@ -153,9 +200,16 @@ _BANNED_NAMES = frozenset(
"source_url",
"source_webpage_url",
"password",
# A download's path is built from its channel and its title (`storage.rel_path`), so it
# names what someone downloaded exactly as the URL does. Log the asset id.
"rel",
"rel_path",
"title",
}
)
_BANNED_ATTRS = frozenset({"text", "content", "email", "url", "query"})
_BANNED_ATTRS = frozenset(
{"text", "content", "email", "url", "query", "rel_path", "title", "display_name"}
)
_ESCAPE = "log-pii-ok"
_LOG_METHODS = frozenset(
{"debug", "info", "warning", "warn", "error", "exception", "critical", "log"}
@@ -211,3 +265,37 @@ def test_no_log_call_passes_what_a_person_typed():
f"the argument (or the line directly above the call) `# {_ESCAPE}: <why it is "
f"safe>` — {offenders}"
)
def _guarded_by_consent(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> bool:
while node in parents:
node = parents[node]
if isinstance(node, ast.If):
for sub in ast.walk(node.test):
if isinstance(sub, ast.Attribute) and sub.attr == "consented_trace":
return True
return False
def test_every_escaped_trace_line_is_gated_on_the_users_own_consent():
"""`log-pii-ok` on a TRACE line means "this is what the user's Teljes consent is for". The TRACE
level alone does not hold that promise: an admin can run the whole process at TRACE, and a
scheduler thread has no user at all. So the line must sit under `logs.consented_trace()`."""
root = pathlib.Path(__file__).resolve().parents[1] / "app"
offenders = []
for path in sorted(root.rglob("*.py")):
src = path.read_text(encoding="utf-8")
lines = src.splitlines()
tree = ast.parse(src)
parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
for node in ast.walk(tree):
if not _is_log_call(node) or not _offending(node):
continue
first = node.args[0] if node.args else None
at_trace = isinstance(first, ast.Attribute) and first.attr == "TRACE"
span = " ".join(lines[max(0, node.lineno - 2) : (node.end_lineno or node.lineno)])
if at_trace and _ESCAPE in span and not _guarded_by_consent(node, parents):
offenders.append(f"{path.relative_to(root)}:{node.lineno}")
assert offenders == [], (
f"a TRACE line with user content is not under consented_trace(): {offenders}"
)