fix(logs): deep-review round 2 — a stack names code, not data, and survives a failed write
- A failure keeps its traceback unless it is yt-dlp's or our own StatedFailure (a message that is the whole story). A plain RuntimeError, RecursionError or OSError is also what a bug of ours raises, and its stack is the only lead. - The stack is scrubbed link by link (logs.scrubbed_stack): a chained DownloadError under a DB error no longer prints yt-dlp's text unscrubbed. - Engine hide_parameters: a DB error's message no longer carries the title or path it was writing. - The download and edit failure lines go out even when their DB write fails. - The worker's child_exit/crash/supervisor lines say download.job.failed only when the job failed there; a crash the DB will not record no longer ends the thread. Stopping a paused child is download.child.stopped, not a second abort. - log_safe also covers generic ids with spaces, playlist titles and bare ids. - The ffmpeg image steps log the failure kind, not the argv with its path. - The consent contract accepts only the body of a plain consented_trace() check, and any level that may be TRACE. - Tests for each; every one mutation-checked.
This commit is contained in:
@@ -20,6 +20,10 @@ engine = create_engine(
|
||||
max_overflow=30,
|
||||
pool_timeout=30,
|
||||
future=True,
|
||||
# A database error renders its bound parameters into its message — a title, an uploader, a
|
||||
# file path, a URL — and that message is logged and stored as a job's error text. The SQL
|
||||
# stays; the values are data, and the ids an operator needs are on the line already.
|
||||
hide_parameters=True,
|
||||
)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import yt_dlp
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||||
@@ -178,11 +179,11 @@ class FacebookAdapter(SourceAdapter):
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise RuntimeError(POST_NOT_SHOWN_MESSAGE)
|
||||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||||
video_id = facebook_post_video_id(page, post_id)
|
||||
if video_id is None:
|
||||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||||
raise RuntimeError(POST_WITHOUT_VIDEO_MESSAGE)
|
||||
raise StatedFailure(POST_WITHOUT_VIDEO_MESSAGE)
|
||||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
|
||||
@@ -34,6 +34,7 @@ from pathlib import Path
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -451,11 +452,11 @@ def run_ffmpeg(
|
||||
if should_cancel():
|
||||
raise EditAborted
|
||||
if time.monotonic() - last_output > stall_timeout_s:
|
||||
raise RuntimeError(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
raise StatedFailure(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
try:
|
||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||
raise RuntimeError("ffmpeg did not exit after closing its output")
|
||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
||||
except BaseException:
|
||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||
@@ -470,7 +471,7 @@ def run_ffmpeg(
|
||||
if pipe:
|
||||
pipe.close()
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
|
||||
|
||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
||||
@@ -723,6 +724,15 @@ def _run_ffmpeg_best_effort(cmd: list[str], timeout_s: int) -> Exception | None:
|
||||
return exc
|
||||
|
||||
|
||||
def _step_failure(err: BaseException) -> str:
|
||||
"""What a best-effort image step's failure line says: the kind and, for ffmpeg, its exit
|
||||
status. Never `str(err)`: a CalledProcessError or TimeoutExpired quotes ffmpeg's argv, and an
|
||||
OSError its file name — the source path either way, which is built from the channel and the
|
||||
title."""
|
||||
code = getattr(err, "returncode", None)
|
||||
return f"{type(err).__name__} {code}" if code is not None else type(err).__name__
|
||||
|
||||
|
||||
def _is_jpeg(path: Path) -> bool:
|
||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
||||
try:
|
||||
@@ -750,13 +760,15 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
shutil.copyfile(src, dest)
|
||||
return dest
|
||||
except OSError as exc:
|
||||
log.info("thumbnail copy failed (%s); poster falls back to a cut frame", str(exc)[:120])
|
||||
log.info(
|
||||
"thumbnail copy failed (%s); poster falls back to a cut frame", _step_failure(exc)
|
||||
)
|
||||
return None
|
||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||
if err is not None:
|
||||
log.info(
|
||||
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||
str(err)[:120],
|
||||
_step_failure(err),
|
||||
)
|
||||
return None
|
||||
return dest if dest.exists() else None
|
||||
@@ -781,7 +793,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
at = min(3.0, (asset.duration_s or 10) * 0.1)
|
||||
err = _run_ffmpeg_best_effort(build_poster_cmd(src, dest, at), timeout_s)
|
||||
if err is not None:
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
return rel if dest.exists() else None
|
||||
|
||||
@@ -812,7 +824,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
||||
cmd = build_storyboard_cmd(src, dest, asset.duration_s, geom)
|
||||
err = _run_ffmpeg_best_effort(cmd, timeout_s)
|
||||
if err is not None:
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
if not dest.exists():
|
||||
return None
|
||||
|
||||
@@ -26,16 +26,29 @@ from app import logs
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# yt-dlp names what it is working on in three shapes, and each one is a row of someone's download
|
||||
# history: the source URL, the `[extractor] <id>:` prefix on nearly every warning and error (for
|
||||
# the generic extractor the "id" is the URL's own basename), and the file path it writes to, which
|
||||
# `storage.rel_path` builds from the channel and the title. The job id is already on every line
|
||||
# (`logs.set_job` in the download child), so none of the three adds anything an operator needs.
|
||||
# What stays: the host (it says which extractor is failing) and the extractor tag.
|
||||
# yt-dlp names what it is working on in these shapes, and each one is a row of someone's download
|
||||
# history: the source URL, the `[extractor] <id>:` prefix on nearly every warning and error, the
|
||||
# file path it writes to (`storage.rel_path` builds it from the channel and the title), a
|
||||
# playlist's title, and an id quoted mid-sentence. The job id is already on every line
|
||||
# (`logs.set_job` in the download child), so none of them adds anything an operator needs. What
|
||||
# stays: the host (it says which extractor is failing) and the extractor tag. Every pattern below
|
||||
# is a shape measured in yt-dlp's own source (2026.07.04), pinned in test_ytdlp_logging.py.
|
||||
_URL_RE = re.compile(r"https?://([^/\s]+)\S*")
|
||||
# `Destination:` is yt-dlp's own word in `[download] Destination: <path>`, not an id — the path
|
||||
# after it is what the third pattern takes out.
|
||||
# after it is what `_PATH_RE` takes out.
|
||||
_ID_PREFIX_RE = re.compile(r"(\[[\w:.-]+\] )(?!Destination:)[^\s:\[\]]+:")
|
||||
# The generic extractor's "id" is the URL's basename, percent-DECODED: `My%20Holiday.mp4` is
|
||||
# `My Holiday` — spaces and all, which the pattern above stops at. Only this extractor: every
|
||||
# other one names a site id, and letting the prefix run to `: ` everywhere would take the words
|
||||
# out of lines like `[VideoConvertor] Converting video from webm to mp4; Destination: …`.
|
||||
_GENERIC_ID_RE = re.compile(r"(\[generic\] )(?!Destination:)[^\[\]\n]+?:(?= |$)", re.M)
|
||||
# `[youtube:tab] Playlist <title>: Downloading 5 items` — the title is not an id, and has spaces.
|
||||
_PLAYLIST_TITLE_RE = re.compile(r"(\] Playlist ).+?(?=: Downloading )")
|
||||
# `[download] Finished downloading playlist: <title>`, `[download] Downloading playlist: <title>`,
|
||||
# `… Skipping already downloaded playlist: <title>` — the title runs to the end of the line.
|
||||
_TITLE_TAIL_RE = re.compile(r"(playlist: |\[download\] Downloading \w+: ).*", re.I)
|
||||
# An id quoted mid-sentence, not as a prefix.
|
||||
_BARE_ID_RE = re.compile(r"((?:subtitles not available for|Downloading just video) )[^\s,;]+")
|
||||
# Two segments or more, and not glued to a word: `www.youtube.com/…` is what the URL pattern
|
||||
# leaves behind, and it must survive this one.
|
||||
_PATH_RE = re.compile(r"(?<![\w.…])/(?:[^/\s\"']+/)+[^/\s\"']*")
|
||||
@@ -48,6 +61,10 @@ def log_safe(text: object) -> str:
|
||||
it would put back one line later exactly what the adapter took out."""
|
||||
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", str(text))
|
||||
text = _ID_PREFIX_RE.sub(r"\1…:", text)
|
||||
text = _GENERIC_ID_RE.sub(r"\1…:", text)
|
||||
text = _PLAYLIST_TITLE_RE.sub(r"\1…", text)
|
||||
text = _TITLE_TAIL_RE.sub(r"\1…", text)
|
||||
text = _BARE_ID_RE.sub(r"\1…", text)
|
||||
return _PATH_RE.sub("…", text)
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
_touch_job,
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
@@ -307,7 +308,7 @@ def _produced_file(info: dict, staging: Path) -> Path:
|
||||
and not _YTDLP_TEMP_RE.search(p.name)
|
||||
]
|
||||
if not cands:
|
||||
raise RuntimeError("yt-dlp produced no output file")
|
||||
raise StatedFailure("yt-dlp produced no output file")
|
||||
return max(cands, key=lambda p: p.stat().st_size)
|
||||
|
||||
|
||||
@@ -481,7 +482,7 @@ def _download(
|
||||
try:
|
||||
urlguard.check_url(url)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise RuntimeError(f"Refusing to download from this address ({e.reason}).") from e
|
||||
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
@@ -653,25 +654,30 @@ def _download(
|
||||
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
|
||||
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
# silent-ok: not silent — `_log_download_failed` is the report.
|
||||
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
||||
# The line goes out whether or not the write below succeeds: when the database is what
|
||||
# failed, that write fails too, and the failure would otherwise leave no line at all.
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never
|
||||
# overwritten with `error` (error also holds the asset → would re-open the
|
||||
# double-release class).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
# silent-ok: not silent — `_log_download_failed` is the report.
|
||||
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
@@ -680,13 +686,20 @@ def _download(
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _stack_worth_logging(exc: BaseException) -> bool:
|
||||
"""Whether a failure's traceback goes with its line: only when it is our own code failing (a
|
||||
TypeError on an odd info dict, a KeyError). yt-dlp's DownloadError, a RuntimeError we raised
|
||||
with the whole story in its message (`ffmpeg failed (1): …`, `yt-dlp produced no output
|
||||
file`) and an OSError naming its file all have the same stack — the `except` that caught them
|
||||
— and the message at its foot is the unscrubbed text: the one copy `log_safe` cannot reach."""
|
||||
return not isinstance(exc, (yt_dlp.utils.DownloadError, RuntimeError, OSError))
|
||||
def _failure_stack(exc: BaseException) -> dict[str, str]:
|
||||
"""The `extra=` fragment that puts a failure's traceback on its line — or nothing.
|
||||
|
||||
Nothing for yt-dlp's DownloadError and for our own `StatedFailure` (`ffmpeg failed (1): …`):
|
||||
their message is the whole story, and their stack is only the `except` that caught them.
|
||||
Everything else keeps it — a TypeError on an odd info dict, a plain RuntimeError, an OSError,
|
||||
a database error: that is our code or our platform failing, and the stack is the lead.
|
||||
|
||||
Scrubbed, link by link (`logs.scrubbed_stack`): a stack's messages are data — an OSError names
|
||||
its file, and a database error raised inside `except DownloadError` carries yt-dlp's
|
||||
unscrubbed text as its context."""
|
||||
if isinstance(exc, (yt_dlp.utils.DownloadError, StatedFailure)):
|
||||
return {}
|
||||
return logs.scrubbed_stack(exc, formats.log_safe)
|
||||
|
||||
|
||||
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
|
||||
@@ -694,14 +707,20 @@ def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseExcep
|
||||
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
|
||||
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
|
||||
place the failure: a recognised one is fully described by its code."""
|
||||
unplaced = code in (None, "", "unknown")
|
||||
log.warning(
|
||||
"job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event(
|
||||
"download.job.failed", job_id=job_id, error_code=code, error_type=type(exc).__name__
|
||||
),
|
||||
exc_info=code in (None, "", "unknown") and _stack_worth_logging(exc),
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
error_code=code,
|
||||
error_type=type(exc).__name__,
|
||||
),
|
||||
**(_failure_stack(exc) if unplaced else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -709,41 +728,41 @@ def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None =
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
|
||||
Its traceback goes with the line only when it is our own code failing
|
||||
(`_stack_worth_logging`)."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running (see the download error path).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||
The line goes out whether or not the write succeeds (see the download error path)."""
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running (see the download error path).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose
|
||||
# path is built from the channel and the title.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
phase=None,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"edit.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
),
|
||||
**(_failure_stack(exc) if exc is not None else {}),
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title. The traceback frames name code, not data.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event(
|
||||
"edit.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
),
|
||||
exc_info=(
|
||||
(type(exc), exc, exc.__traceback__)
|
||||
if exc is not None and _stack_worth_logging(exc)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
@@ -795,7 +814,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
if not dest_staging.exists():
|
||||
raise RuntimeError("ffmpeg produced no output file")
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""The failures the download pipeline raises itself, with the whole story in the message.
|
||||
|
||||
`ffmpeg failed (1): …`, `yt-dlp produced no output file`, a post that shows no video: the message
|
||||
says what happened, and the stack is only the `except` that caught it. A failure line leaves that
|
||||
stack off (`pipeline._failure_stack`) — but ONLY for these. A plain `RuntimeError` is also what
|
||||
Python raises for a bug of ours (a dict changed size during iteration, a generator's
|
||||
StopIteration, `RecursionError`, `NotImplementedError`), and that one's stack is the only lead.
|
||||
"""
|
||||
|
||||
|
||||
class StatedFailure(RuntimeError):
|
||||
"""A failure we raised on purpose, whose message is its full explanation. A `RuntimeError` so
|
||||
every `except RuntimeError` that caught these before still does."""
|
||||
+49
-1
@@ -37,7 +37,9 @@ import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
@@ -263,11 +265,53 @@ def redact(value: Any, *, key: str = "") -> Any:
|
||||
|
||||
|
||||
# --- formatters ----------------------------------------------------------------------------
|
||||
_SCRUBBED_STACK = "scrubbed_stack"
|
||||
|
||||
|
||||
def scrubbed_stack(exc: BaseException, scrub: Callable[[BaseException], str]) -> dict[str, str]:
|
||||
"""`extra=` fragment: `exc`'s traceback, rendered where an `exc_info` stack goes, with the
|
||||
message of every exception in it put through `scrub`.
|
||||
|
||||
For a failure whose message can name someone's data (a download's URL, a file path, a video
|
||||
id). The frames name code and are kept whole; the text at the foot of each link is the data.
|
||||
A plain `exc_info` renders the chain too — the `__cause__`/`__context__` a DB error raised
|
||||
inside `except DownloadError` carries — so scrubbing only the line's own message is not
|
||||
enough. Merge it into the call's `extra`: `{**event(...), **scrubbed_stack(...)}`."""
|
||||
links: list[BaseException] = []
|
||||
link: BaseException | None = exc
|
||||
while link is not None and all(link is not seen for seen in links):
|
||||
links.append(link)
|
||||
link = link.__cause__ or (None if link.__suppress_context__ else link.__context__)
|
||||
parts: list[str] = []
|
||||
for i in range(len(links) - 1, -1, -1): # the oldest first, as Python prints a chain
|
||||
link = links[i]
|
||||
kind = type(link)
|
||||
name = kind.__qualname__
|
||||
if kind.__module__ not in ("builtins", "__main__"):
|
||||
name = f"{kind.__module__}.{name}"
|
||||
try:
|
||||
message = scrub(link)
|
||||
# silent-ok: the message is what could not be rendered, and the placeholder says so — the
|
||||
# caller is in the middle of reporting one failure and must not get a second from here.
|
||||
except Exception: # noqa: BLE001 — a `__str__` that raises, of any shape
|
||||
message = "<unprintable>"
|
||||
frames = "".join(traceback.format_tb(link.__traceback__))
|
||||
head = f"Traceback (most recent call last):\n{frames}" if frames else ""
|
||||
parts.append(f"{head}{name}: {message}" if message else f"{head}{name}")
|
||||
if i:
|
||||
parts.append(
|
||||
"\nThe above exception was the direct cause of the following exception:\n"
|
||||
if links[i - 1].__cause__ is link
|
||||
else "\nDuring handling of the above exception, another exception occurred:\n"
|
||||
)
|
||||
return {_SCRUBBED_STACK: "\n".join(parts)}
|
||||
|
||||
|
||||
# Attributes every LogRecord carries; anything else a caller passed via `extra=` is a structured
|
||||
# attribute and is merged into `attrs` (so `extra={"job_id": 7}` works as well as `event(...)`).
|
||||
_RECORD_STANDARD = frozenset(
|
||||
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
|
||||
| {"message", "asctime", "taskName", "event", "attrs", PERSIST, "elevated"}
|
||||
| {"message", "asctime", "taskName", "event", "attrs", PERSIST, "elevated", _SCRUBBED_STACK}
|
||||
)
|
||||
|
||||
|
||||
@@ -348,6 +392,8 @@ def record_fields(record: logging.LogRecord) -> dict[str, Any]:
|
||||
fields["attrs"] = redact(attrs)
|
||||
if record.exc_info:
|
||||
fields["exc"] = redact_text(logging.Formatter().formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
fields["exc"] = redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
elif record.exc_text:
|
||||
# A QueueHandler-style hand-off formats the traceback into exc_text and drops exc_info.
|
||||
fields["exc"] = redact_text(record.exc_text)
|
||||
@@ -381,6 +427,8 @@ class TextFormatter(logging.Formatter):
|
||||
line += f" {json.dumps(redact(attrs), ensure_ascii=False, default=str)}"
|
||||
if record.exc_info:
|
||||
line += "\n" + redact_text(self.formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
line += "\n" + redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
return line
|
||||
|
||||
|
||||
|
||||
+74
-40
@@ -23,6 +23,7 @@ import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import text
|
||||
@@ -267,11 +268,11 @@ def _fail_after_kill(
|
||||
error: str,
|
||||
error_code: str,
|
||||
cause: str | None = None,
|
||||
) -> None:
|
||||
) -> bool:
|
||||
"""`_stop_and_free` + mark the job errored — for the watchdog's stall/timeout kills and the
|
||||
supervisor-error fallback."""
|
||||
supervisor-error fallback. Returns whether the job failed here (`_fail_running`)."""
|
||||
_stop_and_free(job_id, asset_id, proc, grace_s)
|
||||
_fail_running(job_id, error=error, error_code=error_code, cause=cause)
|
||||
return _fail_running(job_id, error=error, error_code=error_code, cause=cause)
|
||||
|
||||
|
||||
def _fail_running(
|
||||
@@ -281,17 +282,19 @@ def _fail_running(
|
||||
error_code: str | None,
|
||||
error_detail: str | None = None,
|
||||
cause: str | None = None,
|
||||
) -> None:
|
||||
exit_code: int | None = None,
|
||||
) -> bool:
|
||||
"""Fail a claimed job — only while it is still `running` (`service.finalize_if_running`), so a
|
||||
cancel/pause a route landed first is never turned back into `error`, which holds the asset the
|
||||
route already released. `error_detail` is written every time: a failure never shows an earlier
|
||||
failure's explanation.
|
||||
|
||||
`cause` is the worker's half of `download.job.failed`: the failures the child never got to
|
||||
report because it was killed or never ran — a stall, the wall-clock cap. Without it those left
|
||||
no line at all. A caller that already logs its own coded line (a crash, with its stack) passes
|
||||
none. Logged only when the job actually failed here: a route's cancel that landed first is not
|
||||
a failure."""
|
||||
report because it was killed, never ran or died — a stall, the wall-clock cap, a non-zero exit
|
||||
(with its `exit_code`). Without it those left no line at all. A caller that logs its own line
|
||||
(a worker error, with its stack — `_report_worker_error`) passes none and uses the returned
|
||||
verdict. Logged only when the job actually failed here: a route's cancel that landed first is
|
||||
not a failure. Returns whether it failed here."""
|
||||
with SessionLocal() as db:
|
||||
failed = service.finalize_if_running(
|
||||
db,
|
||||
@@ -309,7 +312,37 @@ def _fail_running(
|
||||
job_id,
|
||||
cause,
|
||||
extra=logs.event(
|
||||
"download.job.failed", job_id=job_id, error_code=error_code, cause=cause
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
error_code=error_code,
|
||||
cause=cause,
|
||||
**({"exit_code": exit_code} if exit_code is not None else {}),
|
||||
),
|
||||
)
|
||||
return failed
|
||||
|
||||
|
||||
def _report_worker_error(job_id: int, exc: Exception, cause: str, fail: Callable[[], bool]) -> None:
|
||||
"""A worker error on a job (the supervisor loop, or processing the job, raised): fail the job,
|
||||
then ONE line with the stack. Its event says what the error did to the job —
|
||||
`download.job.failed` when the job failed with it, `download.worker.error` when a route had
|
||||
already stopped it (a cancel that landed first is not a failure, `_fail_running`).
|
||||
|
||||
The line goes out even when failing the job raises (the database is down): as
|
||||
`download.worker.error`, since the job was not failed, and the raise goes on to the caller."""
|
||||
failed = False
|
||||
try:
|
||||
failed = fail()
|
||||
finally:
|
||||
log.error(
|
||||
"job %s: worker error (%s)",
|
||||
job_id,
|
||||
cause,
|
||||
exc_info=exc,
|
||||
extra=logs.event(
|
||||
"download.job.failed" if failed else "download.worker.error",
|
||||
job_id=job_id,
|
||||
cause=cause,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -366,19 +399,20 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
# else: a terminal status under the cap (done/error, or a re-queued contended asset) means
|
||||
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
|
||||
# drop its completion notice).
|
||||
except Exception: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
|
||||
log.exception(
|
||||
"supervisor loop failed for job %s; stopping the child",
|
||||
except Exception as exc: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
|
||||
# silent-ok: not silent — `_report_worker_error` is the report.
|
||||
_report_worker_error(
|
||||
job_id,
|
||||
extra=logs.event("download.job.failed", job_id=job_id, cause="supervisor_error"),
|
||||
)
|
||||
_fail_after_kill(
|
||||
job_id,
|
||||
asset_id,
|
||||
proc,
|
||||
grace_s,
|
||||
"The download was stopped after a worker error.",
|
||||
"unknown",
|
||||
exc,
|
||||
"supervisor_error",
|
||||
lambda: _fail_after_kill(
|
||||
job_id,
|
||||
asset_id,
|
||||
proc,
|
||||
grace_s,
|
||||
"The download was stopped after a worker error.",
|
||||
"unknown",
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
@@ -397,17 +431,8 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
job_id,
|
||||
error=f"The download process exited unexpectedly (code {proc.returncode}).",
|
||||
error_code="unknown",
|
||||
)
|
||||
log.warning(
|
||||
"job %s: child exited %s without finishing",
|
||||
job_id,
|
||||
proc.returncode,
|
||||
extra=logs.event(
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
cause="child_exit",
|
||||
exit_code=proc.returncode,
|
||||
),
|
||||
cause="child_exit",
|
||||
exit_code=proc.returncode,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
|
||||
log.exception("post-exit reconciliation failed for job %s", job_id)
|
||||
@@ -446,10 +471,13 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
extra=logs.event("download.child.abandoned", job_id=job_id),
|
||||
)
|
||||
else: # cancelled
|
||||
# Not `download.job.aborted`: that is the child's line, logged when it notices the pause
|
||||
# itself — and it often has, and is only tearing down when this poll comes round. This one
|
||||
# says the worker had to stop it.
|
||||
log.info(
|
||||
"job %s: child stopped (route pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("download.job.aborted", job_id=job_id),
|
||||
extra=logs.event("download.child.stopped", job_id=job_id),
|
||||
)
|
||||
|
||||
|
||||
@@ -559,13 +587,19 @@ def _worker_loop(idx: int) -> None:
|
||||
with logs.bind(job_id=job_id):
|
||||
try:
|
||||
_process_job(job_id)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception(
|
||||
"processing job %s crashed",
|
||||
job_id,
|
||||
extra=logs.event("download.job.failed", job_id=job_id, cause="crash"),
|
||||
)
|
||||
_fail_running(job_id, error="Worker error", error_code=None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
try:
|
||||
_report_worker_error(
|
||||
job_id,
|
||||
exc,
|
||||
"crash",
|
||||
lambda: _fail_running(job_id, error="Worker error", error_code=None),
|
||||
)
|
||||
# silent-ok: not silent — `_report_worker_error` logged the crash before this
|
||||
# re-raise. Failing the job hit the database, and letting that out would end this
|
||||
# worker thread for good; boot recovery settles a job left `running`.
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("could not fail job %s after its crash", job_id)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
Reference in New Issue
Block a user