peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(logs): deep-review round 2 — a stack names code, not data, and survives a failed write

Browse Source 
- A failure keeps its traceback unless it is yt-dlp's or our own StatedFailure (a
  message that is the whole story). A plain RuntimeError, RecursionError or OSError
  is also what a bug of ours raises, and its stack is the only lead.
- The stack is scrubbed link by link (logs.scrubbed_stack): a chained DownloadError
  under a DB error no longer prints yt-dlp's text unscrubbed.
- Engine hide_parameters: a DB error's message no longer carries the title or path
  it was writing.
- The download and edit failure lines go out even when their DB write fails.
- The worker's child_exit/crash/supervisor lines say download.job.failed only when
  the job failed there; a crash the DB will not record no longer ends the thread.
  Stopping a paused child is download.child.stopped, not a second abort.
- log_safe also covers generic ids with spaces, playlist titles and bare ids.
- The ffmpeg image steps log the failure kind, not the argv with its path.
- The consent contract accepts only the body of a plain consented_trace() check,
  and any level that may be TRACE.
- Tests for each; every one mutation-checked.
This commit is contained in:
peter
2026-09-18 22:50:37 +02:00
parent 54d19cc386
commit 0b672ed6e7
14 changed files with 729 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/worker.py
+74 -40
View File
@@ -23,6 +23,7 @@ import subprocess
import sys
import threading
import time
from collections.abc import Callable
from datetime import datetime, timezone
from sqlalchemy import text
@@ -267,11 +268,11 @@ def _fail_after_kill(
error: str,
error_code: str,
cause: str | None = None,
) -> None:
) -> bool:
"""`_stop_and_free` + mark the job errored — for the watchdog's stall/timeout kills and the
supervisor-error fallback."""
supervisor-error fallback. Returns whether the job failed here (`_fail_running`)."""
_stop_and_free(job_id, asset_id, proc, grace_s)
_fail_running(job_id, error=error, error_code=error_code, cause=cause)
return _fail_running(job_id, error=error, error_code=error_code, cause=cause)
def _fail_running(
@@ -281,17 +282,19 @@ def _fail_running(
error_code: str | None,
error_detail: str | None = None,
cause: str | None = None,
) -> None:
exit_code: int | None = None,
) -> bool:
"""Fail a claimed job — only while it is still `running` (`service.finalize_if_running`), so a
cancel/pause a route landed first is never turned back into `error`, which holds the asset the
route already released. `error_detail` is written every time: a failure never shows an earlier
failure's explanation.
`cause` is the worker's half of `download.job.failed`: the failures the child never got to
report because it was killed or never ran — a stall, the wall-clock cap. Without it those left
no line at all. A caller that already logs its own coded line (a crash, with its stack) passes
none. Logged only when the job actually failed here: a route's cancel that landed first is not
a failure."""
report because it was killed, never ran or died — a stall, the wall-clock cap, a non-zero exit
(with its `exit_code`). Without it those left no line at all. A caller that logs its own line
(a worker error, with its stack — `_report_worker_error`) passes none and uses the returned
verdict. Logged only when the job actually failed here: a route's cancel that landed first is
not a failure. Returns whether it failed here."""
with SessionLocal() as db:
failed = service.finalize_if_running(
db,
@@ -309,7 +312,37 @@ def _fail_running(
job_id,
cause,
extra=logs.event(
"download.job.failed", job_id=job_id, error_code=error_code, cause=cause
"download.job.failed",
job_id=job_id,
error_code=error_code,
cause=cause,
**({"exit_code": exit_code} if exit_code is not None else {}),
),
)
return failed
def _report_worker_error(job_id: int, exc: Exception, cause: str, fail: Callable[[], bool]) -> None:
"""A worker error on a job (the supervisor loop, or processing the job, raised): fail the job,
then ONE line with the stack. Its event says what the error did to the job —
`download.job.failed` when the job failed with it, `download.worker.error` when a route had
already stopped it (a cancel that landed first is not a failure, `_fail_running`).
The line goes out even when failing the job raises (the database is down): as
`download.worker.error`, since the job was not failed, and the raise goes on to the caller."""
failed = False
try:
failed = fail()
finally:
log.error(
"job %s: worker error (%s)",
job_id,
cause,
exc_info=exc,
extra=logs.event(
"download.job.failed" if failed else "download.worker.error",
job_id=job_id,
cause=cause,
),
)
@@ -366,19 +399,20 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
# else: a terminal status under the cap (done/error, or a re-queued contended asset) means
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
# drop its completion notice).
except Exception: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
log.exception(
"supervisor loop failed for job %s; stopping the child",
except Exception as exc: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
# silent-ok: not silent — `_report_worker_error` is the report.
_report_worker_error(
job_id,
extra=logs.event("download.job.failed", job_id=job_id, cause="supervisor_error"),
)
_fail_after_kill(
job_id,
asset_id,
proc,
grace_s,
"The download was stopped after a worker error.",
"unknown",
exc,
"supervisor_error",
lambda: _fail_after_kill(
job_id,
asset_id,
proc,
grace_s,
"The download was stopped after a worker error.",
"unknown",
),
)
return
@@ -397,17 +431,8 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
job_id,
error=f"The download process exited unexpectedly (code {proc.returncode}).",
error_code="unknown",
)
log.warning(
"job %s: child exited %s without finishing",
job_id,
proc.returncode,
extra=logs.event(
"download.job.failed",
job_id=job_id,
cause="child_exit",
exit_code=proc.returncode,
),
cause="child_exit",
exit_code=proc.returncode,
)
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
log.exception("post-exit reconciliation failed for job %s", job_id)
@@ -446,10 +471,13 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
extra=logs.event("download.child.abandoned", job_id=job_id),
)
else: # cancelled
# Not `download.job.aborted`: that is the child's line, logged when it notices the pause
# itself — and it often has, and is only tearing down when this poll comes round. This one
# says the worker had to stop it.
log.info(
"job %s: child stopped (route pause/cancel)",
job_id,
extra=logs.event("download.job.aborted", job_id=job_id),
extra=logs.event("download.child.stopped", job_id=job_id),
)
@@ -559,13 +587,19 @@ def _worker_loop(idx: int) -> None:
with logs.bind(job_id=job_id):
try:
_process_job(job_id)
except Exception: # noqa: BLE001
log.exception(
"processing job %s crashed",
job_id,
extra=logs.event("download.job.failed", job_id=job_id, cause="crash"),
)
_fail_running(job_id, error="Worker error", error_code=None)
except Exception as exc: # noqa: BLE001
try:
_report_worker_error(
job_id,
exc,
"crash",
lambda: _fail_running(job_id, error="Worker error", error_code=None),
)
# silent-ok: not silent — `_report_worker_error` logged the crash before this
# re-raise. Failing the job hit the database, and letting that out would end this
# worker thread for good; boot recovery settles a job left `running`.
except Exception: # noqa: BLE001
log.exception("could not fail job %s after its crash", job_id)
def main() -> None: