fix(logs): deep-review round 3 — a failure nobody could record is one scrubbed line
- The download and edit failure lines go back after their DB write: a job the write could not fail is not a failed job, and the worker's child_exit line already says it failed. - The download child's runner is the last stop before Python's hook: an escaping exception is one download.child.failed line with its chain scrubbed, then exit 1, instead of a raw stderr traceback carrying the URL. - A crash the database will not record carries its stack once, and the fail-write line has an event code. - The job's own generic-extractor name is taken out as a literal (formats.hiding), so brackets and colons in a direct link's file name no longer leak; the pattern stays as the fallback. `Downloading just the video <id>` is covered. - Tests: scrubbed_stack's cause, suppress, unprintable and text-format paths, the image steps' failure kinds, the watchdog error through the real _fail_running, exact poster/storyboard messages, ffmpeg skip marker.
This commit is contained in:
@@ -17,10 +17,15 @@ spec shape (all keys optional; defaults + browser-playability clamp applied by `
|
||||
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from collections.abc import Iterator
|
||||
|
||||
from app import logs
|
||||
|
||||
@@ -38,28 +43,60 @@ _URL_RE = re.compile(r"https?://([^/\s]+)\S*")
|
||||
# after it is what `_PATH_RE` takes out.
|
||||
_ID_PREFIX_RE = re.compile(r"(\[[\w:.-]+\] )(?!Destination:)[^\s:\[\]]+:")
|
||||
# The generic extractor's "id" is the URL's basename, percent-DECODED: `My%20Holiday.mp4` is
|
||||
# `My Holiday` — spaces and all, which the pattern above stops at. Only this extractor: every
|
||||
# other one names a site id, and letting the prefix run to `: ` everywhere would take the words
|
||||
# out of lines like `[VideoConvertor] Converting video from webm to mp4; Destination: …`.
|
||||
# `My Holiday` — spaces, brackets, colons and all, which no pattern can bound (`Our Trip: Day 1`
|
||||
# ends at which `: `?). The job knows its own URL, so the exact name is taken out as a literal
|
||||
# (`hiding`). This pattern is the fallback for a line logged outside that scope: up to the first
|
||||
# `: `. Only this extractor: every other one names a site id, and letting the prefix run to `: `
|
||||
# everywhere would take the words out of lines like `[VideoConvertor] Converting …; Destination:`.
|
||||
_GENERIC_ID_RE = re.compile(r"(\[generic\] )(?!Destination:)[^\[\]\n]+?:(?= |$)", re.M)
|
||||
# `[youtube:tab] Playlist <title>: Downloading 5 items` — the title is not an id, and has spaces.
|
||||
_PLAYLIST_TITLE_RE = re.compile(r"(\] Playlist ).+?(?=: Downloading )")
|
||||
# `[download] Finished downloading playlist: <title>`, `[download] Downloading playlist: <title>`,
|
||||
# `… Skipping already downloaded playlist: <title>` — the title runs to the end of the line.
|
||||
_TITLE_TAIL_RE = re.compile(r"(playlist: |\[download\] Downloading \w+: ).*", re.I)
|
||||
# An id quoted mid-sentence, not as a prefix.
|
||||
_BARE_ID_RE = re.compile(r"((?:subtitles not available for|Downloading just video) )[^\s,;]+")
|
||||
# An id quoted mid-sentence, not as a prefix: `… not available for <id>`, and `--no-playlist`'s
|
||||
# `Downloading just the video <id>` (`_yes_playlist`) / `Downloading just video <id>` (multifeed).
|
||||
_BARE_ID_RE = re.compile(
|
||||
r"((?:subtitles not available for|Downloading just (?:the )?\w+) )[^\s,;]+"
|
||||
)
|
||||
# Two segments or more, and not glued to a word: `www.youtube.com/…` is what the URL pattern
|
||||
# leaves behind, and it must survive this one.
|
||||
_PATH_RE = re.compile(r"(?<![\w.…])/(?:[^/\s\"']+/)+[^/\s\"']*")
|
||||
|
||||
|
||||
_HIDDEN: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
|
||||
"formats_hidden", default=()
|
||||
)
|
||||
|
||||
|
||||
def generic_id(url: str) -> str:
|
||||
"""The name yt-dlp's generic extractor gives a URL: its basename, extension off, percent-
|
||||
decoded — `InfoExtractor._generic_id` (2026.07.04), pinned against it in the tests."""
|
||||
return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hiding(url: str | None) -> Iterator[None]:
|
||||
"""For the work on one URL: `log_safe` also takes out the name yt-dlp's generic extractor
|
||||
gives it, exactly, where yt-dlp puts it (`[generic] <name>:`). A context variable, so it
|
||||
holds for yt-dlp's logger calls too — they run on the same thread as the extraction."""
|
||||
name = generic_id(url) if url else ""
|
||||
token = _HIDDEN.set((*_HIDDEN.get(), name) if name else _HIDDEN.get())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_HIDDEN.reset(token)
|
||||
|
||||
|
||||
def log_safe(text: object) -> str:
|
||||
"""yt-dlp's text (its own lines, or a DownloadError's message) with the parts that name a
|
||||
person's download taken out. Every log line that carries yt-dlp's words goes through here,
|
||||
not just the adapter's: yt-dlp raises the same text it has just logged, so a line that skipped
|
||||
it would put back one line later exactly what the adapter took out."""
|
||||
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", str(text))
|
||||
text = str(text)
|
||||
for name in _HIDDEN.get():
|
||||
text = text.replace(f"[generic] {name}:", "[generic] …:")
|
||||
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", text)
|
||||
text = _ID_PREFIX_RE.sub(r"\1…:", text)
|
||||
text = _GENERIC_ID_RE.sub(r"\1…:", text)
|
||||
text = _PLAYLIST_TITLE_RE.sub(r"\1…", text)
|
||||
|
||||
Reference in New Issue
Block a user