fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere

- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
  `formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
  file paths, and the job-failed, refused, edit-failed and backfill lines use
  it. Done lines log the asset id, not the title-built path. The PII contract
  now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
  failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
  is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
  set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
  a process's last batch closes the repeat window, so a short-lived download
  child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
  `download.job.failed` with a cause; they left no line at all before. The
  crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
  and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
  not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
  carry reason `transport`. Subscribe recognises a duplicate by reason code
  only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
  directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
  were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
This commit is contained in:
2026-09-18 03:16:15 +02:00
parent 04b58e5c6f
commit 54d19cc386
22 changed files with 834 additions and 141 deletions
+63 -27
View File
@@ -383,7 +383,7 @@ def _extract_with_fallback(
"job %s: clients %s failed (%s); retrying with fallback",
job_id,
clients,
str(exc)[:70],
formats.log_safe(exc)[:70],
)
for p in list(staging.iterdir()): # drop partials before the retry
try:
@@ -603,11 +603,12 @@ def _download(
db.commit()
if finalized:
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
# The asset id, not `rel`: the path is built from the channel and the title.
log.info(
"job %s done: %s",
"job %s done: asset %s",
job_id,
rel,
extra=logs.event("download.job.done", job_id=job_id),
asset_id,
extra=logs.event("download.job.done", job_id=job_id, asset_id=asset_id),
)
except (_Aborted, editmod.EditAborted):
@@ -642,7 +643,7 @@ def _download(
log.info(
"job %s refused for its owner: %s",
job_id,
msg,
formats.log_safe(msg),
extra=logs.event("download.job.refused", job_id=job_id),
)
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
@@ -669,16 +670,8 @@ def _download(
phase=None,
)
db.commit()
# The traceback only when the classifier could not place it: a recognised yt-dlp
# failure is fully described by its code, and the stack under it is always this same
# `except`. An UNRECOGNISED one leaves nothing else to work from.
log.warning(
"job %s failed: %s",
job_id,
msg,
extra=logs.event("download.job.failed", job_id=job_id, error_code=diagnosis.code),
exc_info=diagnosis.code in (None, "", "unknown"),
)
# silent-ok: not silent — `_log_download_failed` is the report.
_log_download_failed(job_id, msg, diagnosis.code, exc)
finally:
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
@@ -687,7 +680,37 @@ def _download(
shutil.rmtree(staging, ignore_errors=True)
def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
def _stack_worth_logging(exc: BaseException) -> bool:
"""Whether a failure's traceback goes with its line: only when it is our own code failing (a
TypeError on an odd info dict, a KeyError). yt-dlp's DownloadError, a RuntimeError we raised
with the whole story in its message (`ffmpeg failed (1): …`, `yt-dlp produced no output
file`) and an OSError naming its file all have the same stack — the `except` that caught them
— and the message at its foot is the unscrubbed text: the one copy `log_safe` cannot reach."""
return not isinstance(exc, (yt_dlp.utils.DownloadError, RuntimeError, OSError))
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
"""The child's `download.job.failed` line. `msg` is the user's copy and keeps their URL; the
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
place the failure: a recognised one is fully described by its code."""
log.warning(
"job %s failed: %s",
job_id,
formats.log_safe(msg),
extra=logs.event(
"download.job.failed", job_id=job_id, error_code=code, error_type=type(exc).__name__
),
exc_info=code in (None, "", "unknown") and _stack_worth_logging(exc),
)
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
where its line and its event code live. `exc` is the exception when there was one.
Its traceback goes with the line only when it is our own code failing
(`_stack_worth_logging`)."""
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset:
@@ -703,7 +726,24 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
phase=None,
)
db.commit()
log.warning("edit job %s failed: %s", job_id, msg)
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
# built from the channel and the title. The traceback frames name code, not data.
log.warning(
"edit job %s failed: %s",
job_id,
formats.log_safe(msg),
extra=logs.event(
"edit.job.failed",
job_id=job_id,
asset_id=asset_id,
**({"error_type": type(exc).__name__} if exc is not None else {}),
),
exc_info=(
(type(exc), exc, exc.__traceback__)
if exc is not None and _stack_worth_logging(exc)
else None
),
)
def _process_edit(job_id: int, asset_id: int) -> None:
@@ -791,11 +831,12 @@ def _process_edit(job_id: int, asset_id: int) -> None:
db.commit()
if finalized:
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
# The asset id, not `rel`: the path is built from the channel and the title.
log.info(
"edit job %s done: %s",
"edit job %s done: asset %s",
job_id,
rel,
extra=logs.event("edit.job.done", job_id=job_id),
asset_id,
extra=logs.event("edit.job.done", job_id=job_id, asset_id=asset_id),
)
except editmod.EditAborted:
@@ -807,12 +848,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
extra=logs.event("edit.job.aborted", job_id=job_id),
)
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
# `_fail_edit` stores `str(exc)` for the user, which is the wrong half for whoever has to
# fix it: an edit reaching here is by definition unclassified, so the stack is all there is.
log.exception(
"the edit job failed",
extra=logs.event("edit.job.failed", job_id=job_id, asset_id=asset_id),
)
_fail_edit(job_id, asset_id, str(exc)[:500])
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
finally:
shutil.rmtree(staging, ignore_errors=True)