fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere
- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter:
`formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and
file paths, and the job-failed, refused, edit-failed and backfill lines use
it. Done lines log the asset id, not the title-built path. The PII contract
now bans `rel`/`rel_path`/`title`.
- A traceback goes with a download/edit failure only when it is our own code
failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot
is the unscrubbed message.
- yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client
set recovers from; the job's verdict is `download.job.failed`.
- The sink groups a bare `%s` pass-through by its text (numbers folded), and
a process's last batch closes the repeat window, so a short-lived download
child still reports what it dropped.
- Worker-side failures (stall, wall-clock, no asset, cached asset error) log
`download.job.failed` with a cause; they left no line at all before. The
crash, supervisor-error and non-zero-exit lines carry the code too.
- An edit failure is one coded line in `_fail_edit`, for every path.
- `logs.get_logger("__main__")` resolves to the `-m` module, so the worker
and the download child keep their names.
- TRACE lines carrying user content are gated on `logs.consented_trace()`,
not on the process level; a contract test pins it.
- The OAuth token refresh goes through `_upstream_failure`; transport errors
carry reason `transport`. Subscribe recognises a duplicate by reason code
only (the prose fallback was dead).
- Once-only guards for a stale sysconfig override and a refusing HLS session
directory; item_detail splits Plex failures from our own parse bugs.
- The playlist breaker line names the playlist and the action; comments that
were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
This commit is contained in:
@@ -383,7 +383,7 @@ def _extract_with_fallback(
|
||||
"job %s: clients %s failed (%s); retrying with fallback",
|
||||
job_id,
|
||||
clients,
|
||||
str(exc)[:70],
|
||||
formats.log_safe(exc)[:70],
|
||||
)
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
@@ -603,11 +603,12 @@ def _download(
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"job %s done: %s",
|
||||
"job %s done: asset %s",
|
||||
job_id,
|
||||
rel,
|
||||
extra=logs.event("download.job.done", job_id=job_id),
|
||||
asset_id,
|
||||
extra=logs.event("download.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
@@ -642,7 +643,7 @@ def _download(
|
||||
log.info(
|
||||
"job %s refused for its owner: %s",
|
||||
job_id,
|
||||
msg,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event("download.job.refused", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||
@@ -669,16 +670,8 @@ def _download(
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
# The traceback only when the classifier could not place it: a recognised yt-dlp
|
||||
# failure is fully described by its code, and the stack under it is always this same
|
||||
# `except`. An UNRECOGNISED one leaves nothing else to work from.
|
||||
log.warning(
|
||||
"job %s failed: %s",
|
||||
job_id,
|
||||
msg,
|
||||
extra=logs.event("download.job.failed", job_id=job_id, error_code=diagnosis.code),
|
||||
exc_info=diagnosis.code in (None, "", "unknown"),
|
||||
)
|
||||
# silent-ok: not silent — `_log_download_failed` is the report.
|
||||
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
@@ -687,7 +680,37 @@ def _download(
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
def _stack_worth_logging(exc: BaseException) -> bool:
|
||||
"""Whether a failure's traceback goes with its line: only when it is our own code failing (a
|
||||
TypeError on an odd info dict, a KeyError). yt-dlp's DownloadError, a RuntimeError we raised
|
||||
with the whole story in its message (`ffmpeg failed (1): …`, `yt-dlp produced no output
|
||||
file`) and an OSError naming its file all have the same stack — the `except` that caught them
|
||||
— and the message at its foot is the unscrubbed text: the one copy `log_safe` cannot reach."""
|
||||
return not isinstance(exc, (yt_dlp.utils.DownloadError, RuntimeError, OSError))
|
||||
|
||||
|
||||
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
|
||||
"""The child's `download.job.failed` line. `msg` is the user's copy and keeps their URL; the
|
||||
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
|
||||
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
|
||||
place the failure: a recognised one is fully described by its code."""
|
||||
log.warning(
|
||||
"job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event(
|
||||
"download.job.failed", job_id=job_id, error_code=code, error_type=type(exc).__name__
|
||||
),
|
||||
exc_info=code in (None, "", "unknown") and _stack_worth_logging(exc),
|
||||
)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
|
||||
Its traceback goes with the line only when it is our own code failing
|
||||
(`_stack_worth_logging`)."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
@@ -703,7 +726,24 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title. The traceback frames name code, not data.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event(
|
||||
"edit.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
),
|
||||
exc_info=(
|
||||
(type(exc), exc, exc.__traceback__)
|
||||
if exc is not None and _stack_worth_logging(exc)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
@@ -791,11 +831,12 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"edit job %s done: %s",
|
||||
"edit job %s done: asset %s",
|
||||
job_id,
|
||||
rel,
|
||||
extra=logs.event("edit.job.done", job_id=job_id),
|
||||
asset_id,
|
||||
extra=logs.event("edit.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except editmod.EditAborted:
|
||||
@@ -807,12 +848,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
extra=logs.event("edit.job.aborted", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
# `_fail_edit` stores `str(exc)` for the user, which is the wrong half for whoever has to
|
||||
# fix it: an edit reaching here is by definition unclassified, so the stack is all there is.
|
||||
log.exception(
|
||||
"the edit job failed",
|
||||
extra=logs.event("edit.job.failed", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500])
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
Reference in New Issue
Block a user