fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere

- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
  `formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
  file paths, and the job-failed, refused, edit-failed and backfill lines use
  it. Done lines log the asset id, not the title-built path. The PII contract
  now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
  failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
  is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
  set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
  a process's last batch closes the repeat window, so a short-lived download
  child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
  `download.job.failed` with a cause; they left no line at all before. The
  crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
  and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
  not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
  carry reason `transport`. Subscribe recognises a duplicate by reason code
  only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
  directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
  were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
This commit is contained in:
2026-09-18 03:16:15 +02:00
parent 04b58e5c6f
commit 54d19cc386
22 changed files with 834 additions and 141 deletions
+62 -11
View File
@@ -260,23 +260,40 @@ def _stop_and_free(job_id: int, asset_id: int, proc: subprocess.Popen, grace_s:
def _fail_after_kill(
job_id: int, asset_id: int, proc: subprocess.Popen, grace_s: float, error: str, error_code: str
job_id: int,
asset_id: int,
proc: subprocess.Popen,
grace_s: float,
error: str,
error_code: str,
cause: str | None = None,
) -> None:
"""`_stop_and_free` + mark the job errored — for the watchdog's stall/timeout kills and the
supervisor-error fallback."""
_stop_and_free(job_id, asset_id, proc, grace_s)
_fail_running(job_id, error=error, error_code=error_code)
_fail_running(job_id, error=error, error_code=error_code, cause=cause)
def _fail_running(
job_id: int, *, error: str, error_code: str | None, error_detail: str | None = None
job_id: int,
*,
error: str,
error_code: str | None,
error_detail: str | None = None,
cause: str | None = None,
) -> None:
"""Fail a claimed job — only while it is still `running` (`service.finalize_if_running`), so a
cancel/pause a route landed first is never turned back into `error`, which holds the asset the
route already released. `error_detail` is written every time: a failure never shows an earlier
failure's explanation."""
failure's explanation.
`cause` is the worker's half of `download.job.failed`: the failures the child never got to
report because it was killed or never ran — a stall, the wall-clock cap. Without it those left
no line at all. A caller that already logs its own coded line (a crash, with its stack) passes
none. Logged only when the job actually failed here: a route's cancel that landed first is not
a failure."""
with SessionLocal() as db:
service.finalize_if_running(
failed = service.finalize_if_running(
db,
job_id,
status=JobStatus.ERROR,
@@ -286,6 +303,15 @@ def _fail_running(
phase=None,
)
db.commit()
if cause is not None and failed:
log.warning(
"job %s failed in the worker: %s",
job_id,
cause,
extra=logs.event(
"download.job.failed", job_id=job_id, error_code=error_code, cause=cause
),
)
def _run_supervised(job_id: int, asset_id: int) -> None:
@@ -341,7 +367,11 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
# drop its completion notice).
except Exception: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
log.exception("supervisor loop failed for job %s; stopping the child", job_id)
log.exception(
"supervisor loop failed for job %s; stopping the child",
job_id,
extra=logs.event("download.job.failed", job_id=job_id, cause="supervisor_error"),
)
_fail_after_kill(
job_id,
asset_id,
@@ -369,7 +399,15 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
error_code="unknown",
)
log.warning(
"job %s: child exited %s without finishing", job_id, proc.returncode
"job %s: child exited %s without finishing",
job_id,
proc.returncode,
extra=logs.event(
"download.job.failed",
job_id=job_id,
cause="child_exit",
exit_code=proc.returncode,
),
)
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
log.exception("post-exit reconciliation failed for job %s", job_id)
@@ -383,6 +421,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
grace_s,
f"The download stalled (no progress for {int(stall_s)}s) and was stopped.",
"network",
cause="stalled",
)
return
if reason == "timeout":
@@ -393,6 +432,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
grace_s,
f"The download exceeded the {int(wall_s)}s time limit and was stopped.",
"network",
cause="timeout",
)
return
# cancelled / abandon / shutdown: stop + free the asset, but leave the job's own DB status.
@@ -401,10 +441,16 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
_requeue_if_running(job_id, None) # resume on the next boot
elif reason == "abandon":
log.warning(
"job %s: killed a child wedged past the wall-clock; DB status left intact", job_id
"job %s: killed a child wedged past the wall-clock; DB status left intact",
job_id,
extra=logs.event("download.child.abandoned", job_id=job_id),
)
else: # cancelled
log.info("job %s: child stopped (route pause/cancel)", job_id)
log.info(
"job %s: child stopped (route pause/cancel)",
job_id,
extra=logs.event("download.job.aborted", job_id=job_id),
)
def _cleanup_staging(asset_id: int) -> None:
@@ -444,7 +490,7 @@ def _process_job(job_id: int) -> None:
del cookie # only whether there is one matters here
if asset is None:
_fail_running(job_id, error="No media asset", error_code=None)
_fail_running(job_id, error="No media asset", error_code=None, cause="no_asset")
return
if asset_status == "ready":
@@ -456,6 +502,7 @@ def _process_job(job_id: int) -> None:
error=asset_error or "Download failed",
error_code=diagnosis.code,
error_detail=diagnosis.detail,
cause="asset_failed",
)
return
if asset_status == "downloading":
@@ -513,7 +560,11 @@ def _worker_loop(idx: int) -> None:
try:
_process_job(job_id)
except Exception: # noqa: BLE001
log.exception("processing job %s crashed", job_id)
log.exception(
"processing job %s crashed",
job_id,
extra=logs.event("download.job.failed", job_id=job_id, cause="crash"),
)
_fail_running(job_id, error="Worker error", error_code=None)