fix(logs): deep-review round 3 — a failure nobody could record is one scrubbed line

- The download and edit failure lines go back after their DB write: a job the write could not fail is not a failed job, and the worker's child_exit line already says it failed. - The download child's runner is the last stop before Python's hook: an escaping exception is one download.child.failed line with its chain scrubbed, then exit 1, instead of a raw stderr traceback carrying the URL. - A crash the database will not record carries its stack once, and the fail-write line has an event code. - The job's own generic-extractor name is taken out as a literal (formats.hiding), so brackets and colons in a direct link's file name no longer leak; the pattern stays as the fallback. `Downloading just the video <id>` is covered. - Tests: scrubbed_stack's cause, suppress, unprintable and text-format paths, the image steps' failure kinds, the watchdog error through the real _fail_running, exact poster/storyboard messages, ffmpeg skip marker.
2026-09-18 23:44:42 +02:00
parent 0b672ed6e7
commit 89c77ae6b5
9 changed files with 407 additions and 81 deletions
@@ -8,6 +8,9 @@ same edit failure twice, a warning per segment request, per settings read).

 import json
 import logging
+import re
+import shutil
+import subprocess
 from pathlib import Path
 from types import SimpleNamespace

@@ -220,9 +223,9 @@ def test_an_edit_that_ffmpeg_refused_names_no_file_and_carries_no_stack(sink, fi
    assert "exc" not in line


-def test_an_edit_failure_is_reported_even_when_its_write_fails(sink, monkeypatch):
-    """The database is often what failed; then the write fails too, and the line must not go
-    down with it."""
+def test_an_edit_the_database_would_not_fail_is_not_reported_as_failed(sink, monkeypatch):
+    """The job is still `running`: the runner reports this failure with the write's, and the
+    worker fails the job (`child_exit`). A line here as well would be a second, and a false one."""

    class _Down:
        def __enter__(self):
@@ -235,7 +238,7 @@ def test_an_edit_failure_is_reported_even_when_its_write_fails(sink, monkeypatch
    with pytest.raises(ConnectionError):
        pipeline._fail_edit(9, 3, "boom", KeyError("x"))

-    assert len(sink.events("edit.job.failed")) == 1
+    assert sink.events("edit.job.failed") == []


 class _EditDb(_NoDb):
@@ -371,6 +374,31 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey
    assert lines and all("dQw4w9WgXcQ" not in json.dumps(o) for o in lines)


+def test_the_metadata_backfill_takes_out_a_generic_name_whatever_it_holds(sink, monkeypatch):
+    from app.downloads import adapters, backfill
+
+    url = "https://example.org/v/Our%20Trip%3A%20Day%201.mp4"
+
+    class _Refusing:
+        def __init__(self, opts):
+            pass
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+        def extract_info(self, target, download):
+            raise yt_dlp.utils.DownloadError("ERROR: [generic] Our Trip: Day 1: Unable to download")
+
+    monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
+    assert backfill._extract_meta(url, adapters.SourceAdapter(), 11) is None
+
+    lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
+    assert lines and all("Day 1" not in json.dumps(o) for o in lines)
+
+
 # --- a repeating failure, once ------------------------------------------------------------------
 def test_a_stale_setting_is_reported_once_not_on_every_read(sink, monkeypatch):
    spec = next(s for s in sysconfig.SPECS if s.type == "int")
@@ -446,6 +474,7 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t
    assert "dQw4w9WgXcQ" not in json.dumps(line)


+@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
 def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
    """ffmpeg's CalledProcessError quotes its argv, and the source path in it is built from the
    channel and the title."""
@@ -459,7 +488,135 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
    assert editmod.ensure_poster(asset, str(tmp_path)) is None
    assert editmod.ensure_storyboard(asset, str(tmp_path)) is None

-    lines = [o for o in sink.objects() if "gen skipped" in o["msg"]]
+    lines = [o["msg"] for o in sink.objects() if "gen skipped" in o["msg"]]
+    # The whole message: a long tmp root would push a leaked argv's path past any length cut.
    assert len(lines) == 2
-    assert all("Some_Channel" not in json.dumps(o) for o in lines)
-    assert all("CalledProcessError" in o["msg"] for o in lines)
+    assert all(
+        re.fullmatch(r"\w+ gen skipped for asset 3: CalledProcessError \d+", m) for m in lines
+    )
+
+
+@pytest.mark.parametrize(
+    "err",
+    [
+        subprocess.TimeoutExpired(
+            ["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30
+        ),
+        FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"),
+        subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]),
+    ],
+    ids=lambda e: type(e).__name__,
+)
+def test_an_image_step_failure_is_its_kind_and_nothing_it_quotes(err):
+    from app.downloads import edit as editmod
+
+    code = getattr(err, "returncode", None)
+    assert editmod._step_failure(err) == " ".join(
+        [type(err).__name__, *([str(code)] if code is not None else [])]
+    )
+
+
+def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_path):
+    from app.downloads import edit as editmod
+
+    src = tmp_path / "thumb.jpg"
+    src.write_bytes(b"\xff\xd8\xff rest")
+
+    def refuse(a, b):
+        raise PermissionError(
+            13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg"
+        )
+
+    monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
+    assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
+
+    (line,) = [o for o in sink.objects() if "thumbnail copy failed" in o["msg"]]
+    assert "Some_Channel" not in json.dumps(line)
+
+
+# --- the child's last stop ------------------------------------------------------------------------
+def test_a_failure_the_child_cannot_record_is_one_scrubbed_line_and_a_non_zero_exit(
+    sink, monkeypatch
+):
+    """The pipeline's failure write was refused. Python's own hook would print the chain raw —
+    and its context is the download's failure, the URL in it. The worker fails the job on the
+    exit code (`child_exit`), so this line does not claim the job failed."""
+    from app.downloads import runner
+
+    url = "https://example.org/v/Our%20Trip%3A%20Day%201.mp4"
+    text = f"ERROR: [generic] Our Trip: Day 1: Unable to download webpage ({url})"
+
+    def download(*a):
+        try:
+            raise yt_dlp.utils.DownloadError(text)
+        except yt_dlp.utils.DownloadError:
+            raise ConnectionError("the database went away")  # noqa: B904
+
+    monkeypatch.setattr(pipeline, "_download", download)
+    with pytest.raises(SystemExit) as exited:
+        runner._run(5, 7, "download", "url", url, {}, 1)
+
+    assert exited.value.code == 1
+    (line,) = [o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]
+    assert line["event"] == "download.child.failed"
+    assert "Unable to download webpage" in line["exc"]
+    # "Day 1" too: the fallback pattern alone stops at the first `: ` and leaves it.
+    assert "Trip" not in json.dumps(line) and "Day 1" not in json.dumps(line)
+
+
+def test_a_crash_the_database_will_not_record_carries_its_stack_once(monkeypatch, sink):
+    claims = iter([42])
+
+    def claim():
+        job = next(claims, None)
+        if job is None:
+            worker._stop.set()
+        return job
+
+    def crash(job_id):
+        raise RuntimeError("the one crash")
+
+    def db_down(*a, **k):
+        raise ConnectionError("the database went away")
+
+    monkeypatch.setattr(worker, "_claim_job", claim)
+    monkeypatch.setattr(worker, "_process_job", crash)
+    monkeypatch.setattr(worker, "_fail_running", db_down)
+    monkeypatch.setattr(worker.settings, "download_poll_seconds", 0)
+    worker._stop.clear()
+    try:
+        worker._worker_loop(0)
+    finally:
+        worker._stop.clear()
+
+    stacks = [o["exc"] for o in sink.objects() if "exc" in o]
+    assert sum("the one crash" in s for s in stacks) == 1
+    (write,) = sink.events("download.worker.fail_write_failed")
+    assert "the database went away" in write["exc"]
+
+
+def test_a_watchdog_error_on_a_job_it_failed_is_that_jobs_failure_line(
+    sink, finalized, monkeypatch
+):
+    """Through the real `_fail_after_kill` → `_fail_running`: the verdict it returns picks the
+    event, so a lost `return` would log every such failure as a mere worker error."""
+
+    class _Alive:
+        pid = 4242
+        returncode = None
+
+        def wait(self, timeout=None):
+            raise subprocess.TimeoutExpired("child", timeout or 0)
+
+    def broken(job_id):
+        raise KeyError("a watchdog bug")
+
+    monkeypatch.setattr(worker, "_spawn_download_child", lambda job_id: _Alive())
+    monkeypatch.setattr(worker, "_job_liveness", broken)
+    monkeypatch.setattr(worker, "_stop_and_free", lambda *a: None)
+
+    worker._run_supervised(42, 7)
+
+    (line,) = [o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]
+    assert (line["event"], line["attrs"]["cause"]) == ("download.job.failed", "supervisor_error")
+    assert "a watchdog bug" in line["exc"]