fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere
- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
`formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
file paths, and the job-failed, refused, edit-failed and backfill lines use
it. Done lines log the asset id, not the title-built path. The PII contract
now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
a process's last batch closes the repeat window, so a short-lived download
child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
`download.job.failed` with a cause; they left no line at all before. The
crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
carry reason `transport`. Subscribe recognises a duplicate by reason code
only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
This commit is contained in:
@@ -87,15 +87,62 @@ def test_an_upstream_failure_logs_the_reason_code_and_not_the_body(sink):
|
||||
assert err.reason == "quotaExceeded"
|
||||
|
||||
|
||||
def test_the_body_is_kept_for_the_one_user_who_consented_to_it(sink):
|
||||
@pytest.fixture
|
||||
def consented():
|
||||
"""The request scope of a user who turned Teljes (TRACE) on for themselves."""
|
||||
scope = logs._RequestScope()
|
||||
scope.user_id, scope.level = 7, logs.TRACE
|
||||
token = logs._scope.set(scope)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
logs._scope.reset(token)
|
||||
|
||||
|
||||
def test_the_body_is_kept_for_the_one_user_who_consented_to_it(sink, consented):
|
||||
_upstream_failure("GET", "search", _youtube_403(TERM))
|
||||
|
||||
# TRACE is emitted only while the user has turned Teljes on for their own session, which is
|
||||
# exactly the consent the privacy policy describes. The diagnostic is not lost, it is gated.
|
||||
# The diagnostic is not lost, it is gated — on the consent the privacy policy describes.
|
||||
(trace,) = sink.at("TRACE")
|
||||
assert TERM in trace["attrs"]["body"]
|
||||
|
||||
|
||||
def test_a_process_running_at_trace_is_not_anyones_consent(sink):
|
||||
"""`sink` runs the whole tree at TRACE — what an admin setting `log_level` to TRACE does. That
|
||||
buys more detail, not every user's search terms quoted back (`logs._traced`)."""
|
||||
_upstream_failure("GET", "search", _youtube_403(TERM))
|
||||
|
||||
assert sink.at("TRACE") == []
|
||||
assert TERM not in json.dumps(sink.objects())
|
||||
|
||||
|
||||
def test_a_scope_with_no_elevation_is_not_consent():
|
||||
scope = logs._RequestScope()
|
||||
scope.user_id = 7
|
||||
token = logs._scope.set(scope)
|
||||
try:
|
||||
assert not logs.consented_trace()
|
||||
finally:
|
||||
logs._scope.reset(token)
|
||||
assert not logs.consented_trace() # a scheduler thread: no request at all
|
||||
|
||||
|
||||
def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink):
|
||||
"""The token endpoint's error is `{"error": "<code>", "error_description": <prose>}` — not the
|
||||
Data API's shape. It goes through the same door, so it gets the same treatment."""
|
||||
resp = httpx.Response(
|
||||
400,
|
||||
json={"error": "invalid_grant", "error_description": "Token has been expired or revoked."},
|
||||
request=httpx.Request("POST", "https://oauth2.googleapis.com/token"),
|
||||
)
|
||||
err = _upstream_failure("POST", "oauth2/token", resp)
|
||||
|
||||
(line,) = sink.at("WARNING")
|
||||
assert line["attrs"]["reason"] == "invalid_grant"
|
||||
assert err.reason == "invalid_grant"
|
||||
assert "revoked" not in str(err) and "revoked" not in json.dumps(line)
|
||||
|
||||
|
||||
def test_an_unparseable_body_still_names_the_status(sink):
|
||||
"""A proxy answering HTML has no `reason` to take — the line must still be worth reading, and
|
||||
must still not fall back to printing the body."""
|
||||
@@ -153,9 +200,16 @@ _BANNED_NAMES = frozenset(
|
||||
"source_url",
|
||||
"source_webpage_url",
|
||||
"password",
|
||||
# A download's path is built from its channel and its title (`storage.rel_path`), so it
|
||||
# names what someone downloaded exactly as the URL does. Log the asset id.
|
||||
"rel",
|
||||
"rel_path",
|
||||
"title",
|
||||
}
|
||||
)
|
||||
_BANNED_ATTRS = frozenset({"text", "content", "email", "url", "query"})
|
||||
_BANNED_ATTRS = frozenset(
|
||||
{"text", "content", "email", "url", "query", "rel_path", "title", "display_name"}
|
||||
)
|
||||
_ESCAPE = "log-pii-ok"
|
||||
_LOG_METHODS = frozenset(
|
||||
{"debug", "info", "warning", "warn", "error", "exception", "critical", "log"}
|
||||
@@ -211,3 +265,37 @@ def test_no_log_call_passes_what_a_person_typed():
|
||||
f"the argument (or the line directly above the call) `# {_ESCAPE}: <why it is "
|
||||
f"safe>` — {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def _guarded_by_consent(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> bool:
|
||||
while node in parents:
|
||||
node = parents[node]
|
||||
if isinstance(node, ast.If):
|
||||
for sub in ast.walk(node.test):
|
||||
if isinstance(sub, ast.Attribute) and sub.attr == "consented_trace":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def test_every_escaped_trace_line_is_gated_on_the_users_own_consent():
|
||||
"""`log-pii-ok` on a TRACE line means "this is what the user's Teljes consent is for". The TRACE
|
||||
level alone does not hold that promise: an admin can run the whole process at TRACE, and a
|
||||
scheduler thread has no user at all. So the line must sit under `logs.consented_trace()`."""
|
||||
root = pathlib.Path(__file__).resolve().parents[1] / "app"
|
||||
offenders = []
|
||||
for path in sorted(root.rglob("*.py")):
|
||||
src = path.read_text(encoding="utf-8")
|
||||
lines = src.splitlines()
|
||||
tree = ast.parse(src)
|
||||
parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
|
||||
for node in ast.walk(tree):
|
||||
if not _is_log_call(node) or not _offending(node):
|
||||
continue
|
||||
first = node.args[0] if node.args else None
|
||||
at_trace = isinstance(first, ast.Attribute) and first.attr == "TRACE"
|
||||
span = " ".join(lines[max(0, node.lineno - 2) : (node.end_lineno or node.lineno)])
|
||||
if at_trace and _ESCAPE in span and not _guarded_by_consent(node, parents):
|
||||
offenders.append(f"{path.relative_to(root)}:{node.lineno}")
|
||||
assert offenders == [], (
|
||||
f"a TRACE line with user content is not under consented_trace(): {offenders}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user