fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere
- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
`formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
file paths, and the job-failed, refused, edit-failed and backfill lines use
it. Done lines log the asset id, not the title-built path. The PII contract
now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
a process's last batch closes the repeat window, so a short-lived download
child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
`download.job.failed` with a cause; they left no line at all before. The
crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
carry reason `transport`. Subscribe recognises a duplicate by reason code
only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
This commit is contained in:
+62
-11
@@ -260,23 +260,40 @@ def _stop_and_free(job_id: int, asset_id: int, proc: subprocess.Popen, grace_s:
|
||||
|
||||
|
||||
def _fail_after_kill(
|
||||
job_id: int, asset_id: int, proc: subprocess.Popen, grace_s: float, error: str, error_code: str
|
||||
job_id: int,
|
||||
asset_id: int,
|
||||
proc: subprocess.Popen,
|
||||
grace_s: float,
|
||||
error: str,
|
||||
error_code: str,
|
||||
cause: str | None = None,
|
||||
) -> None:
|
||||
"""`_stop_and_free` + mark the job errored — for the watchdog's stall/timeout kills and the
|
||||
supervisor-error fallback."""
|
||||
_stop_and_free(job_id, asset_id, proc, grace_s)
|
||||
_fail_running(job_id, error=error, error_code=error_code)
|
||||
_fail_running(job_id, error=error, error_code=error_code, cause=cause)
|
||||
|
||||
|
||||
def _fail_running(
|
||||
job_id: int, *, error: str, error_code: str | None, error_detail: str | None = None
|
||||
job_id: int,
|
||||
*,
|
||||
error: str,
|
||||
error_code: str | None,
|
||||
error_detail: str | None = None,
|
||||
cause: str | None = None,
|
||||
) -> None:
|
||||
"""Fail a claimed job — only while it is still `running` (`service.finalize_if_running`), so a
|
||||
cancel/pause a route landed first is never turned back into `error`, which holds the asset the
|
||||
route already released. `error_detail` is written every time: a failure never shows an earlier
|
||||
failure's explanation."""
|
||||
failure's explanation.
|
||||
|
||||
`cause` is the worker's half of `download.job.failed`: the failures the child never got to
|
||||
report because it was killed or never ran — a stall, the wall-clock cap. Without it those left
|
||||
no line at all. A caller that already logs its own coded line (a crash, with its stack) passes
|
||||
none. Logged only when the job actually failed here: a route's cancel that landed first is not
|
||||
a failure."""
|
||||
with SessionLocal() as db:
|
||||
service.finalize_if_running(
|
||||
failed = service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
@@ -286,6 +303,15 @@ def _fail_running(
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
if cause is not None and failed:
|
||||
log.warning(
|
||||
"job %s failed in the worker: %s",
|
||||
job_id,
|
||||
cause,
|
||||
extra=logs.event(
|
||||
"download.job.failed", job_id=job_id, error_code=error_code, cause=cause
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
@@ -341,7 +367,11 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
|
||||
# drop its completion notice).
|
||||
except Exception: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
|
||||
log.exception("supervisor loop failed for job %s; stopping the child", job_id)
|
||||
log.exception(
|
||||
"supervisor loop failed for job %s; stopping the child",
|
||||
job_id,
|
||||
extra=logs.event("download.job.failed", job_id=job_id, cause="supervisor_error"),
|
||||
)
|
||||
_fail_after_kill(
|
||||
job_id,
|
||||
asset_id,
|
||||
@@ -369,7 +399,15 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
error_code="unknown",
|
||||
)
|
||||
log.warning(
|
||||
"job %s: child exited %s without finishing", job_id, proc.returncode
|
||||
"job %s: child exited %s without finishing",
|
||||
job_id,
|
||||
proc.returncode,
|
||||
extra=logs.event(
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
cause="child_exit",
|
||||
exit_code=proc.returncode,
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
|
||||
log.exception("post-exit reconciliation failed for job %s", job_id)
|
||||
@@ -383,6 +421,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
grace_s,
|
||||
f"The download stalled (no progress for {int(stall_s)}s) and was stopped.",
|
||||
"network",
|
||||
cause="stalled",
|
||||
)
|
||||
return
|
||||
if reason == "timeout":
|
||||
@@ -393,6 +432,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
grace_s,
|
||||
f"The download exceeded the {int(wall_s)}s time limit and was stopped.",
|
||||
"network",
|
||||
cause="timeout",
|
||||
)
|
||||
return
|
||||
# cancelled / abandon / shutdown: stop + free the asset, but leave the job's own DB status.
|
||||
@@ -401,10 +441,16 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
_requeue_if_running(job_id, None) # resume on the next boot
|
||||
elif reason == "abandon":
|
||||
log.warning(
|
||||
"job %s: killed a child wedged past the wall-clock; DB status left intact", job_id
|
||||
"job %s: killed a child wedged past the wall-clock; DB status left intact",
|
||||
job_id,
|
||||
extra=logs.event("download.child.abandoned", job_id=job_id),
|
||||
)
|
||||
else: # cancelled
|
||||
log.info("job %s: child stopped (route pause/cancel)", job_id)
|
||||
log.info(
|
||||
"job %s: child stopped (route pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("download.job.aborted", job_id=job_id),
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_staging(asset_id: int) -> None:
|
||||
@@ -444,7 +490,7 @@ def _process_job(job_id: int) -> None:
|
||||
del cookie # only whether there is one matters here
|
||||
|
||||
if asset is None:
|
||||
_fail_running(job_id, error="No media asset", error_code=None)
|
||||
_fail_running(job_id, error="No media asset", error_code=None, cause="no_asset")
|
||||
return
|
||||
|
||||
if asset_status == "ready":
|
||||
@@ -456,6 +502,7 @@ def _process_job(job_id: int) -> None:
|
||||
error=asset_error or "Download failed",
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
cause="asset_failed",
|
||||
)
|
||||
return
|
||||
if asset_status == "downloading":
|
||||
@@ -513,7 +560,11 @@ def _worker_loop(idx: int) -> None:
|
||||
try:
|
||||
_process_job(job_id)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("processing job %s crashed", job_id)
|
||||
log.exception(
|
||||
"processing job %s crashed",
|
||||
job_id,
|
||||
extra=logs.event("download.job.failed", job_id=job_id, cause="crash"),
|
||||
)
|
||||
_fail_running(job_id, error="Worker error", error_code=None)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user