fix(logs): deep-review round 3 — a failure nobody could record is one scrubbed line

- The download and edit failure lines go back after their DB write: a job the
  write could not fail is not a failed job, and the worker's child_exit line
  already says it failed.
- The download child's runner is the last stop before Python's hook: an
  escaping exception is one download.child.failed line with its chain scrubbed,
  then exit 1, instead of a raw stderr traceback carrying the URL.
- A crash the database will not record carries its stack once, and the
  fail-write line has an event code.
- The job's own generic-extractor name is taken out as a literal (formats.hiding),
  so brackets and colons in a direct link's file name no longer leak; the
  pattern stays as the fallback. `Downloading just the video <id>` is covered.
- Tests: scrubbed_stack's cause, suppress, unprintable and text-format paths,
  the image steps' failure kinds, the watchdog error through the real
  _fail_running, exact poster/storyboard messages, ffmpeg skip marker.
This commit is contained in:
2026-09-18 23:44:42 +02:00
parent 0b672ed6e7
commit 89c77ae6b5
9 changed files with 407 additions and 81 deletions
+43 -6
View File
@@ -17,10 +17,15 @@ spec shape (all keys optional; defaults + browser-playability clamp applied by `
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
"""
import contextlib
import contextvars
import hashlib
import json
import logging
import os
import re
import urllib.parse
from collections.abc import Iterator
from app import logs
@@ -38,28 +43,60 @@ _URL_RE = re.compile(r"https?://([^/\s]+)\S*")
# after it is what `_PATH_RE` takes out.
_ID_PREFIX_RE = re.compile(r"(\[[\w:.-]+\] )(?!Destination:)[^\s:\[\]]+:")
# The generic extractor's "id" is the URL's basename, percent-DECODED: `My%20Holiday.mp4` is
# `My Holiday` — spaces and all, which the pattern above stops at. Only this extractor: every
# other one names a site id, and letting the prefix run to `: ` everywhere would take the words
# out of lines like `[VideoConvertor] Converting video from webm to mp4; Destination: …`.
# `My Holiday` — spaces, brackets, colons and all, which no pattern can bound (`Our Trip: Day 1`
# ends at which `: `?). The job knows its own URL, so the exact name is taken out as a literal
# (`hiding`). This pattern is the fallback for a line logged outside that scope: up to the first
# `: `. Only this extractor: every other one names a site id, and letting the prefix run to `: `
# everywhere would take the words out of lines like `[VideoConvertor] Converting …; Destination:`.
_GENERIC_ID_RE = re.compile(r"(\[generic\] )(?!Destination:)[^\[\]\n]+?:(?= |$)", re.M)
# `[youtube:tab] Playlist <title>: Downloading 5 items` — the title is not an id, and has spaces.
_PLAYLIST_TITLE_RE = re.compile(r"(\] Playlist ).+?(?=: Downloading )")
# `[download] Finished downloading playlist: <title>`, `[download] Downloading playlist: <title>`,
# `… Skipping already downloaded playlist: <title>` — the title runs to the end of the line.
_TITLE_TAIL_RE = re.compile(r"(playlist: |\[download\] Downloading \w+: ).*", re.I)
# An id quoted mid-sentence, not as a prefix.
_BARE_ID_RE = re.compile(r"((?:subtitles not available for|Downloading just video) )[^\s,;]+")
# An id quoted mid-sentence, not as a prefix: `… not available for <id>`, and `--no-playlist`'s
# `Downloading just the video <id>` (`_yes_playlist`) / `Downloading just video <id>` (multifeed).
_BARE_ID_RE = re.compile(
r"((?:subtitles not available for|Downloading just (?:the )?\w+) )[^\s,;]+"
)
# Two segments or more, and not glued to a word: `www.youtube.com/…` is what the URL pattern
# leaves behind, and it must survive this one.
_PATH_RE = re.compile(r"(?<![\w.…])/(?:[^/\s\"']+/)+[^/\s\"']*")
_HIDDEN: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
"formats_hidden", default=()
)
def generic_id(url: str) -> str:
"""The name yt-dlp's generic extractor gives a URL: its basename, extension off, percent-
decoded — `InfoExtractor._generic_id` (2026.07.04), pinned against it in the tests."""
return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
@contextlib.contextmanager
def hiding(url: str | None) -> Iterator[None]:
"""For the work on one URL: `log_safe` also takes out the name yt-dlp's generic extractor
gives it, exactly, where yt-dlp puts it (`[generic] <name>:`). A context variable, so it
holds for yt-dlp's logger calls too — they run on the same thread as the extraction."""
name = generic_id(url) if url else ""
token = _HIDDEN.set((*_HIDDEN.get(), name) if name else _HIDDEN.get())
try:
yield
finally:
_HIDDEN.reset(token)
def log_safe(text: object) -> str:
"""yt-dlp's text (its own lines, or a DownloadError's message) with the parts that name a
person's download taken out. Every log line that carries yt-dlp's words goes through here,
not just the adapter's: yt-dlp raises the same text it has just logged, so a line that skipped
it would put back one line later exactly what the adapter took out."""
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", str(text))
text = str(text)
for name in _HIDDEN.get():
text = text.replace(f"[generic] {name}:", "[generic] …:")
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", text)
text = _ID_PREFIX_RE.sub(r"\1…:", text)
text = _GENERIC_ID_RE.sub(r"\1…:", text)
text = _PLAYLIST_TITLE_RE.sub(r"\1…", text)