fix(logs): deep-review round 2 — a stack names code, not data, and survives a failed write

- A failure keeps its traceback unless it is yt-dlp's or our own StatedFailure (a
  message that is the whole story). A plain RuntimeError, RecursionError or OSError
  is also what a bug of ours raises, and its stack is the only lead.
- The stack is scrubbed link by link (logs.scrubbed_stack): a chained DownloadError
  under a DB error no longer prints yt-dlp's text unscrubbed.
- Engine hide_parameters: a DB error's message no longer carries the title or path
  it was writing.
- The download and edit failure lines go out even when their DB write fails.
- The worker's child_exit/crash/supervisor lines say download.job.failed only when
  the job failed there; a crash the DB will not record no longer ends the thread.
  Stopping a paused child is download.child.stopped, not a second abort.
- log_safe also covers generic ids with spaces, playlist titles and bare ids.
- The ffmpeg image steps log the failure kind, not the argv with its path.
- The consent contract accepts only the body of a plain consented_trace() check,
  and any level that may be TRACE.
- Tests for each; every one mutation-checked.
This commit is contained in:
2026-09-18 22:50:37 +02:00
parent 54d19cc386
commit 0b672ed6e7
14 changed files with 729 additions and 138 deletions
+85 -66
View File
@@ -23,6 +23,7 @@ from app.downloads.jobdb import (
_staging_dir,
_touch_job,
)
from app.downloads.stated import StatedFailure
from app.downloads.status import JobStatus
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
@@ -307,7 +308,7 @@ def _produced_file(info: dict, staging: Path) -> Path:
and not _YTDLP_TEMP_RE.search(p.name)
]
if not cands:
raise RuntimeError("yt-dlp produced no output file")
raise StatedFailure("yt-dlp produced no output file")
return max(cands, key=lambda p: p.stat().st_size)
@@ -481,7 +482,7 @@ def _download(
try:
urlguard.check_url(url)
except urlguard.BlockedUrl as e:
raise RuntimeError(f"Refusing to download from this address ({e.reason}).") from e
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
# this asset is that user's private, salted identity (the double gate lives in credentials).
remaining_bytes: int | None = None
@@ -653,25 +654,30 @@ def _download(
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
asset.status = "error"
asset.error = msg
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
# with `error` (error also holds the asset → would re-open the double-release class).
service.finalize_if_running(
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=diagnosis.code,
error_detail=diagnosis.detail,
phase=None,
)
db.commit()
# silent-ok: not silent — `_log_download_failed` is the report.
_log_download_failed(job_id, msg, diagnosis.code, exc)
# The line goes out whether or not the write below succeeds: when the database is what
# failed, that write fails too, and the failure would otherwise leave no line at all.
try:
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
asset.status = "error"
asset.error = msg
# Atomic fail: only while still running, so a route-set cancel/pause is never
# overwritten with `error` (error also holds the asset → would re-open the
# double-release class).
service.finalize_if_running(
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=diagnosis.code,
error_detail=diagnosis.detail,
phase=None,
)
db.commit()
finally:
# silent-ok: not silent — `_log_download_failed` is the report.
_log_download_failed(job_id, msg, diagnosis.code, exc)
finally:
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
@@ -680,13 +686,20 @@ def _download(
shutil.rmtree(staging, ignore_errors=True)
def _stack_worth_logging(exc: BaseException) -> bool:
"""Whether a failure's traceback goes with its line: only when it is our own code failing (a
TypeError on an odd info dict, a KeyError). yt-dlp's DownloadError, a RuntimeError we raised
with the whole story in its message (`ffmpeg failed (1): …`, `yt-dlp produced no output
file`) and an OSError naming its file all have the same stack — the `except` that caught them
— and the message at its foot is the unscrubbed text: the one copy `log_safe` cannot reach."""
return not isinstance(exc, (yt_dlp.utils.DownloadError, RuntimeError, OSError))
def _failure_stack(exc: BaseException) -> dict[str, str]:
"""The `extra=` fragment that puts a failure's traceback on its line — or nothing.
Nothing for yt-dlp's DownloadError and for our own `StatedFailure` (`ffmpeg failed (1): …`):
their message is the whole story, and their stack is only the `except` that caught them.
Everything else keeps it — a TypeError on an odd info dict, a plain RuntimeError, an OSError,
a database error: that is our code or our platform failing, and the stack is the lead.
Scrubbed, link by link (`logs.scrubbed_stack`): a stack's messages are data — an OSError names
its file, and a database error raised inside `except DownloadError` carries yt-dlp's
unscrubbed text as its context."""
if isinstance(exc, (yt_dlp.utils.DownloadError, StatedFailure)):
return {}
return logs.scrubbed_stack(exc, formats.log_safe)
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
@@ -694,14 +707,20 @@ def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseExcep
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
place the failure: a recognised one is fully described by its code."""
unplaced = code in (None, "", "unknown")
log.warning(
"job %s failed: %s",
job_id,
formats.log_safe(msg),
extra=logs.event(
"download.job.failed", job_id=job_id, error_code=code, error_type=type(exc).__name__
),
exc_info=code in (None, "", "unknown") and _stack_worth_logging(exc),
extra={
**logs.event(
"download.job.failed",
job_id=job_id,
error_code=code,
error_type=type(exc).__name__,
),
**(_failure_stack(exc) if unplaced else {}),
},
)
@@ -709,41 +728,41 @@ def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None =
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
where its line and its event code live. `exc` is the exception when there was one.
Its traceback goes with the line only when it is our own code failing
(`_stack_worth_logging`)."""
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
asset.status = "error"
asset.error = msg
# Atomic fail: only while still running (see the download error path).
service.finalize_if_running(
db,
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
The line goes out whether or not the write succeeds (see the download error path)."""
try:
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
asset.status = "error"
asset.error = msg
# Atomic fail: only while still running (see the download error path).
service.finalize_if_running(
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=dlerrors.classify(msg),
phase=None,
)
db.commit()
finally:
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose
# path is built from the channel and the title.
log.warning(
"edit job %s failed: %s",
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=dlerrors.classify(msg),
phase=None,
formats.log_safe(msg),
extra={
**logs.event(
"edit.job.failed",
job_id=job_id,
asset_id=asset_id,
**({"error_type": type(exc).__name__} if exc is not None else {}),
),
**(_failure_stack(exc) if exc is not None else {}),
},
)
db.commit()
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
# built from the channel and the title. The traceback frames name code, not data.
log.warning(
"edit job %s failed: %s",
job_id,
formats.log_safe(msg),
extra=logs.event(
"edit.job.failed",
job_id=job_id,
asset_id=asset_id,
**({"error_type": type(exc).__name__} if exc is not None else {}),
),
exc_info=(
(type(exc), exc, exc.__traceback__)
if exc is not None and _stack_worth_logging(exc)
else None
),
)
def _process_edit(job_id: int, asset_id: int) -> None:
@@ -795,7 +814,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
should_cancel=_cancel_poll(job_id),
)
if not dest_staging.exists():
raise RuntimeError("ffmpeg produced no output file")
raise StatedFailure("ffmpeg produced no output file")
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
storage.place_file(dest_staging, settings.download_root, rel)