fix(logs): keep what a person typed out of the backend log lines
The E22 redactor is a safety net over addresses and credential params, and every one of these call sites was leaning on it to do the whole job. What it does not cover went in whole, and what it does cover came out as a constant. Not covered, so a real leak: - routes/search.py logged the raw search term at ERROR. The privacy policy shipped in 0.88.0 says a search string is kept only at Teljes/TRACE, with the user's own consent. The ERROR line now carries the shape of the failure (source, candidate and collected counts) and the term follows on a TRACE line, which nothing emits unless that consent is in force. - downloads/backfill.py logged the full source URL at INFO — a row of the user's download history. Now the asset id and the host. - youtube/client.py logged the upstream error body, which is prose that quotes the request back. Every failing YouTube call now goes through one _upstream_failure() that logs the API's `reason` code, keeps the body for the same consented TRACE line, and raises a YouTubeError carrying `reason` as a field — callers put that error in log lines of their own (routes/channels.py) and in a 422 answered to the user, so the body must not travel in it either. Covered, so merely useless: routes/auth.py named the account by an address that reached both sinks as `[email]`, and app/email.py printed recipient lists that read as `['[email]']`. Both now carry the uid (or a count) and a reason code, which is also the first half of E22.5 S4's failed-sign-in coverage. routes/channels.py branched on the text of that error message to recognise an already-existing subscription. It now branches on `exc.reason`, with the substring check kept beside it for a body that carried no reason at all. tests/test_log_pii.py pins the call sites and the family: a source-level contract test walks every log call in app/ and fails on an argument naming user-typed content unless the line says why it is safe (`# log-pii-ok: <why>`). Eight sites carry that marker today, each with its reason. Gate 8/8 green; all five cases mutation-checked.
This commit is contained in:
@@ -12,7 +12,7 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
|
||||
from app import quota, sysconfig
|
||||
from app import logs, quota, sysconfig
|
||||
from app.models import User
|
||||
from app.security import decrypt, decrypt_optional, encrypt
|
||||
|
||||
@@ -23,7 +23,58 @@ API_BASE = "https://www.googleapis.com/youtube/v3"
|
||||
|
||||
|
||||
class YouTubeError(Exception):
|
||||
pass
|
||||
"""An upstream YouTube Data API failure.
|
||||
|
||||
`reason` is the API's own error reason code (`quotaExceeded`, `subscriptionDuplicate`,
|
||||
`videoNotFound`, …) when the body carried one. Branch on THAT, never on the message: the
|
||||
message is prose, it changes under us, and it is deliberately the poorer of the two here."""
|
||||
|
||||
def __init__(self, message: str, *, reason: str = ""):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _error_reason(resp: httpx.Response) -> str:
|
||||
"""The `reason` YouTube puts on an API error. That is the whole diagnostic value of an error
|
||||
body; the `message` beside it is prose, and upstream prose can quote the request back — a
|
||||
search term, a playlist title."""
|
||||
try:
|
||||
errors = resp.json().get("error", {}).get("errors") or []
|
||||
reasons = ",".join(e.get("reason", "") for e in errors if e.get("reason"))
|
||||
except Exception: # noqa: BLE001 — any non-JSON body (an HTML error page from a proxy)
|
||||
return "<unparseable>"
|
||||
return reasons or "<none>"
|
||||
|
||||
|
||||
def _upstream_failure(method: str, path: str, resp: httpx.Response) -> YouTubeError:
|
||||
"""Report one upstream failure and build the error to raise. Every failing YouTube call goes
|
||||
through here, so what may appear in a log line cannot drift apart between the read, the write
|
||||
and the delete path — and the error a caller raises carries the reason code rather than the
|
||||
body, because callers put it in a log line (`routes/channels.py`) and in a 422 answered to the
|
||||
user. The body itself is TRACE-only: it is the request quoted back, which the privacy policy
|
||||
keeps out of the log unless the user turned Teljes on for themselves."""
|
||||
reason = _error_reason(resp)
|
||||
log.warning(
|
||||
"YouTube %s %s -> %s (%s)",
|
||||
method,
|
||||
path,
|
||||
resp.status_code,
|
||||
reason,
|
||||
extra=logs.event(
|
||||
"youtube.api.error",
|
||||
method=method,
|
||||
path=path,
|
||||
status=resp.status_code,
|
||||
reason=reason,
|
||||
),
|
||||
)
|
||||
log.log(
|
||||
logs.TRACE,
|
||||
"the upstream body behind it",
|
||||
# log-pii-ok: TRACE is the consented level, and this is what it is for
|
||||
extra=logs.event("youtube.api.body", body=resp.text[:500]),
|
||||
)
|
||||
return YouTubeError(f"{method} {path} -> {resp.status_code}: {reason}", reason=reason)
|
||||
|
||||
|
||||
def _chunks(items: list, size: int) -> Iterator[list]:
|
||||
@@ -62,8 +113,8 @@ class YouTubeClient:
|
||||
YouTubeError — degrade gracefully, log, and move on — so wrapping here keeps a
|
||||
transient network fault (e.g. the fixed-IP egress proxy being unreachable) from
|
||||
escaping as a raw httpx error and turning into an uncaught 500. HTTP status errors
|
||||
are NOT touched here (we don't use raise_for_status): callers inspect resp.status_code
|
||||
themselves and raise their own YouTubeError with the response body."""
|
||||
are NOT touched here (we don't use raise_for_status): each call site inspects
|
||||
resp.status_code itself and hands the response to `_upstream_failure`."""
|
||||
try:
|
||||
return self._http.request(method, url, **kwargs)
|
||||
except httpx.HTTPError as exc:
|
||||
@@ -121,8 +172,7 @@ class YouTubeClient:
|
||||
resp = self._send("GET", f"{API_BASE}/{path}", params=p, headers=headers)
|
||||
quota.record_usage(cost)
|
||||
if resp.status_code != 200:
|
||||
log.warning("YouTube API %s -> %s: %s", path, resp.status_code, resp.text[:200])
|
||||
raise YouTubeError(f"GET {path} -> {resp.status_code}: {resp.text[:300]}")
|
||||
raise _upstream_failure("GET", path, resp)
|
||||
return resp.json()
|
||||
|
||||
# --- endpoints ---
|
||||
@@ -247,12 +297,7 @@ class YouTubeClient:
|
||||
)
|
||||
quota.record_usage(50)
|
||||
if resp.status_code not in (200, 204):
|
||||
log.warning(
|
||||
"YouTube subscriptions.delete -> %s: %s",
|
||||
resp.status_code,
|
||||
resp.text[:200],
|
||||
)
|
||||
raise YouTubeError(f"DELETE subscriptions -> {resp.status_code}: {resp.text[:300]}")
|
||||
raise _upstream_failure("DELETE", "subscriptions", resp)
|
||||
|
||||
def insert_subscription(self, channel_id: str) -> str:
|
||||
"""Subscribe to a channel on YouTube. Requires the write scope and the user's
|
||||
@@ -282,8 +327,7 @@ class YouTubeClient:
|
||||
resp = self._send(method, f"{API_BASE}/{path}", params=params, json=json, headers=headers)
|
||||
quota.record_usage(50)
|
||||
if resp.status_code not in (200, 204):
|
||||
log.warning("YouTube %s %s -> %s: %s", method, path, resp.status_code, resp.text[:200])
|
||||
raise YouTubeError(f"{method} {path} -> {resp.status_code}: {resp.text[:300]}")
|
||||
raise _upstream_failure(method, path, resp)
|
||||
return resp.json() if resp.content else {}
|
||||
|
||||
def get_playlist_snippet(self, playlist_id: str) -> dict | None:
|
||||
|
||||
Reference in New Issue
Block a user