fix(logs): deep-review round 3 — a failure nobody could record is one scrubbed line

- The download and edit failure lines go back after their DB write: a job the
  write could not fail is not a failed job, and the worker's child_exit line
  already says it failed.
- The download child's runner is the last stop before Python's hook: an
  escaping exception is one download.child.failed line with its chain scrubbed,
  then exit 1, instead of a raw stderr traceback carrying the URL.
- A crash the database will not record carries its stack once, and the
  fail-write line has an event code.
- The job's own generic-extractor name is taken out as a literal (formats.hiding),
  so brackets and colons in a direct link's file name no longer leak; the
  pattern stays as the fallback. `Downloading just the video <id>` is covered.
- Tests: scrubbed_stack's cause, suppress, unprintable and text-format paths,
  the image steps' failure kinds, the watchdog error through the real
  _fail_running, exact poster/storyboard messages, ffmpeg skip marker.

backend/tests/test_download_failure_catalog.py

@@ -125,9 +125,9 @@ def test_the_download_child_knows_the_cookie_went_with_it(db, monkeypatch, tmp_p
     assert (job.error_code, job.error_detail) == ("login_required", "facebook.cookie_rejected")
 
 
-def test_the_failure_line_goes_out_even_when_its_write_fails(db, monkeypatch, tmp_path):
-    """The database is often what failed; the write of the failure then fails too, and the line
-    that says so must not go down with it."""
+def test_a_failure_the_database_would_not_record_is_not_claimed(db, monkeypatch, tmp_path):
+    """The job is still `running`: the runner reports this with the write's error, and the worker
+    fails the job. A `download.job.failed` here as well would be a second line, and a false one."""
     u = _user(db, "fb-dbdown@example.com")
     a, j = _asset_and_job(db, u, sig="dbdown", asset_status="downloading", job_status="running")
     _fail_the_child(monkeypatch, tmp_path, cookie=False)
@@ -141,7 +141,7 @@ def test_the_failure_line_goes_out_even_when_its_write_fails(db, monkeypatch, tm
 
     with pytest.raises(ConnectionError):
         pipeline._download(j.id, a.id, "url", _REEL, {}, u.id)
-    assert said == [j.id]
+    assert said == []
 
 
 def test_a_job_meeting_a_failed_shared_asset_gets_the_same_explanation(db):

backend/tests/test_failure_lines.py

@@ -8,6 +8,9 @@ same edit failure twice, a warning per segment request, per settings read).
 
 import json
 import logging
+import re
+import shutil
+import subprocess
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -220,9 +223,9 @@ def test_an_edit_that_ffmpeg_refused_names_no_file_and_carries_no_stack(sink, fi
     assert "exc" not in line
 
 
-def test_an_edit_failure_is_reported_even_when_its_write_fails(sink, monkeypatch):
-    """The database is often what failed; then the write fails too, and the line must not go
-    down with it."""
+def test_an_edit_the_database_would_not_fail_is_not_reported_as_failed(sink, monkeypatch):
+    """The job is still `running`: the runner reports this failure with the write's, and the
+    worker fails the job (`child_exit`). A line here as well would be a second, and a false one."""
 
     class _Down:
         def __enter__(self):
@@ -235,7 +238,7 @@ def test_an_edit_failure_is_reported_even_when_its_write_fails(sink, monkeypatch
     with pytest.raises(ConnectionError):
         pipeline._fail_edit(9, 3, "boom", KeyError("x"))
 
-    assert len(sink.events("edit.job.failed")) == 1
+    assert sink.events("edit.job.failed") == []
 
 
 class _EditDb(_NoDb):
@@ -371,6 +374,31 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey
     assert lines and all("dQw4w9WgXcQ" not in json.dumps(o) for o in lines)
 
 
+def test_the_metadata_backfill_takes_out_a_generic_name_whatever_it_holds(sink, monkeypatch):
+    from app.downloads import adapters, backfill
+
+    url = "https://example.org/v/Our%20Trip%3A%20Day%201.mp4"
+
+    class _Refusing:
+        def __init__(self, opts):
+            pass
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+        def extract_info(self, target, download):
+            raise yt_dlp.utils.DownloadError("ERROR: [generic] Our Trip: Day 1: Unable to download")
+
+    monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
+    assert backfill._extract_meta(url, adapters.SourceAdapter(), 11) is None
+
+    lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
+    assert lines and all("Day 1" not in json.dumps(o) for o in lines)
+
+
 # --- a repeating failure, once ------------------------------------------------------------------
 def test_a_stale_setting_is_reported_once_not_on_every_read(sink, monkeypatch):
     spec = next(s for s in sysconfig.SPECS if s.type == "int")
@@ -446,6 +474,7 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t
     assert "dQw4w9WgXcQ" not in json.dumps(line)
 
 
+@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
 def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
     """ffmpeg's CalledProcessError quotes its argv, and the source path in it is built from the
     channel and the title."""
@@ -459,7 +488,135 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
     assert editmod.ensure_poster(asset, str(tmp_path)) is None
     assert editmod.ensure_storyboard(asset, str(tmp_path)) is None
 
-    lines = [o for o in sink.objects() if "gen skipped" in o["msg"]]
+    lines = [o["msg"] for o in sink.objects() if "gen skipped" in o["msg"]]
+    # The whole message: a long tmp root would push a leaked argv's path past any length cut.
     assert len(lines) == 2
-    assert all("Some_Channel" not in json.dumps(o) for o in lines)
-    assert all("CalledProcessError" in o["msg"] for o in lines)
+    assert all(
+        re.fullmatch(r"\w+ gen skipped for asset 3: CalledProcessError \d+", m) for m in lines
+    )
+
+
+@pytest.mark.parametrize(
+    "err",
+    [
+        subprocess.TimeoutExpired(
+            ["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30
+        ),
+        FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"),
+        subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]),
+    ],
+    ids=lambda e: type(e).__name__,
+)
+def test_an_image_step_failure_is_its_kind_and_nothing_it_quotes(err):
+    from app.downloads import edit as editmod
+
+    code = getattr(err, "returncode", None)
+    assert editmod._step_failure(err) == " ".join(
+        [type(err).__name__, *([str(code)] if code is not None else [])]
+    )
+
+
+def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_path):
+    from app.downloads import edit as editmod
+
+    src = tmp_path / "thumb.jpg"
+    src.write_bytes(b"\xff\xd8\xff rest")
+
+    def refuse(a, b):
+        raise PermissionError(
+            13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg"
+        )
+
+    monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
+    assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
+
+    (line,) = [o for o in sink.objects() if "thumbnail copy failed" in o["msg"]]
+    assert "Some_Channel" not in json.dumps(line)
+
+
+# --- the child's last stop ------------------------------------------------------------------------
+def test_a_failure_the_child_cannot_record_is_one_scrubbed_line_and_a_non_zero_exit(
+    sink, monkeypatch
+):
+    """The pipeline's failure write was refused. Python's own hook would print the chain raw —
+    and its context is the download's failure, the URL in it. The worker fails the job on the
+    exit code (`child_exit`), so this line does not claim the job failed."""
+    from app.downloads import runner
+
+    url = "https://example.org/v/Our%20Trip%3A%20Day%201.mp4"
+    text = f"ERROR: [generic] Our Trip: Day 1: Unable to download webpage ({url})"
+
+    def download(*a):
+        try:
+            raise yt_dlp.utils.DownloadError(text)
+        except yt_dlp.utils.DownloadError:
+            raise ConnectionError("the database went away")  # noqa: B904
+
+    monkeypatch.setattr(pipeline, "_download", download)
+    with pytest.raises(SystemExit) as exited:
+        runner._run(5, 7, "download", "url", url, {}, 1)
+
+    assert exited.value.code == 1
+    (line,) = [o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]
+    assert line["event"] == "download.child.failed"
+    assert "Unable to download webpage" in line["exc"]
+    # "Day 1" too: the fallback pattern alone stops at the first `: ` and leaves it.
+    assert "Trip" not in json.dumps(line) and "Day 1" not in json.dumps(line)
+
+
+def test_a_crash_the_database_will_not_record_carries_its_stack_once(monkeypatch, sink):
+    claims = iter([42])
+
+    def claim():
+        job = next(claims, None)
+        if job is None:
+            worker._stop.set()
+        return job
+
+    def crash(job_id):
+        raise RuntimeError("the one crash")
+
+    def db_down(*a, **k):
+        raise ConnectionError("the database went away")
+
+    monkeypatch.setattr(worker, "_claim_job", claim)
+    monkeypatch.setattr(worker, "_process_job", crash)
+    monkeypatch.setattr(worker, "_fail_running", db_down)
+    monkeypatch.setattr(worker.settings, "download_poll_seconds", 0)
+    worker._stop.clear()
+    try:
+        worker._worker_loop(0)
+    finally:
+        worker._stop.clear()
+
+    stacks = [o["exc"] for o in sink.objects() if "exc" in o]
+    assert sum("the one crash" in s for s in stacks) == 1
+    (write,) = sink.events("download.worker.fail_write_failed")
+    assert "the database went away" in write["exc"]
+
+
+def test_a_watchdog_error_on_a_job_it_failed_is_that_jobs_failure_line(
+    sink, finalized, monkeypatch
+):
+    """Through the real `_fail_after_kill` → `_fail_running`: the verdict it returns picks the
+    event, so a lost `return` would log every such failure as a mere worker error."""
+
+    class _Alive:
+        pid = 4242
+        returncode = None
+
+        def wait(self, timeout=None):
+            raise subprocess.TimeoutExpired("child", timeout or 0)
+
+    def broken(job_id):
+        raise KeyError("a watchdog bug")
+
+    monkeypatch.setattr(worker, "_spawn_download_child", lambda job_id: _Alive())
+    monkeypatch.setattr(worker, "_job_liveness", broken)
+    monkeypatch.setattr(worker, "_stop_and_free", lambda *a: None)
+
+    worker._run_supervised(42, 7)
+
+    (line,) = [o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]
+    assert (line["event"], line["attrs"]["cause"]) == ("download.job.failed", "supervisor_error")
+    assert "a watchdog bug" in line["exc"]

backend/tests/test_logs.py

@@ -318,3 +318,68 @@ def test_a_failing_request_is_logged_by_status_and_an_unhandled_error_is_logged_
     assert payload["level"] == "ERROR"
     assert payload["event"] == "http.request"
     assert "RuntimeError" in payload["exc"]
+
+
+# --- a stack whose messages are data (`logs.scrubbed_stack`) -----------------------------------
+def _upper(exc: BaseException) -> str:
+    return str(exc).upper()
+
+
+def _raised(make):
+    try:
+        make()
+    except Exception as exc:  # noqa: BLE001 — whatever the case raised is the case
+        return exc
+    raise AssertionError("the case did not raise")
+
+
+def test_a_raise_from_is_printed_as_its_direct_cause():
+    def make():
+        try:
+            raise KeyError("inner")
+        except KeyError as inner:
+            raise ValueError("outer") from inner
+
+    stack = logs.scrubbed_stack(_raised(make), _upper)["scrubbed_stack"]
+    assert "The above exception was the direct cause" in stack
+    assert "During handling" not in stack
+    assert "'INNER'" in stack and "OUTER" in stack
+
+
+def test_a_raise_from_none_hides_the_link_its_author_hid():
+    def make():
+        try:
+            raise KeyError("hidden")
+        except KeyError:
+            raise ValueError("shown") from None
+
+    stack = logs.scrubbed_stack(_raised(make), _upper)["scrubbed_stack"]
+    assert "HIDDEN" not in stack and "SHOWN" in stack
+
+
+def test_a_chain_that_loops_back_ends():
+    a, b = ValueError("a"), ValueError("b")
+    a.__context__, b.__context__ = b, a
+    assert "A" in logs.scrubbed_stack(a, _upper)["scrubbed_stack"]
+
+
+def test_a_message_that_cannot_be_rendered_does_not_fail_the_report():
+    def refuse(exc):
+        raise RuntimeError("no")
+
+    assert (
+        logs.scrubbed_stack(ValueError("x"), refuse)["scrubbed_stack"]
+        == "ValueError: <unprintable>"
+    )
+
+
+def test_the_text_format_carries_the_scrubbed_stack_too():
+    title = "a private title"  # a variable: a frame prints its source line, which is code
+    exc = ValueError(title)
+    record = logging.LogRecord("siftlode.x", logging.WARNING, __file__, 1, "failed", None, None)
+    for key, value in logs.scrubbed_stack(exc, lambda e: "…").items():
+        setattr(record, key, value)
+
+    line = logs.TextFormatter().format(record)
+    assert "ValueError: …" in line and "private" not in line
+    assert "scrubbed_stack" not in line  # the stack, not an attribute

backend/tests/test_ytdlp_logging.py

@@ -134,9 +134,15 @@ def test_the_source_url_does_not_travel_with_the_explanation(sink):
             "hu subtitles not available for …",
             "dQw4w9WgXcQ",
         ),
+        # `watch?v=…&list=…` under `noplaylist` (`_yes_playlist`), and the multifeed shape.
         (
-            "[youtube:tab] Downloading just video dQw4w9WgXcQ because of --no-playlist",
-            "[youtube:tab] Downloading just video … because of --no-playlist",
+            "[youtube:tab] Downloading just the video dQw4w9WgXcQ because of --no-playlist",
+            "[youtube:tab] Downloading just the video … because of --no-playlist",
+            "dQw4w9WgXcQ",
+        ),
+        (
+            "[youtube] Downloading just video dQw4w9WgXcQ because of --no-playlist",
+            "[youtube] Downloading just video … because of --no-playlist",
             "dQw4w9WgXcQ",
         ),
         # `Destination` is yt-dlp's word, not an id; the path after it is built from the title.
@@ -155,6 +161,31 @@ def test_what_names_the_download_comes_out_of_real_ytdlp_shapes(sink, line, kept
     assert gone not in json.dumps(obj)
 
 
+@pytest.mark.parametrize(
+    ("url", "name"),
+    [
+        ("https://example.org/v/Wedding%20%5B2024%5D.mp4", "Wedding [2024]"),
+        ("https://example.org/v/Our%20Trip%3A%20Day%201.mp4", "Our Trip: Day 1"),
+    ],
+)
+def test_the_jobs_own_generic_name_is_taken_out_whatever_it_holds(url, name):
+    """Brackets and `: ` bound no pattern; the job knows its URL, so the exact name goes."""
+    line = f"ERROR: [generic] {name}: Unable to download webpage: HTTP Error 404: Not Found"
+
+    assert formats.generic_id(url) == name
+    with formats.hiding(url):
+        assert formats.log_safe(line) == (
+            "ERROR: [generic] …: Unable to download webpage: HTTP Error 404: Not Found"
+        )
+
+
+def test_the_generic_name_is_yt_dlps_own():
+    from yt_dlp.extractor.common import InfoExtractor
+
+    for url in ("https://x.org/a/My%20Holiday.mp4", "https://x.org/a/clip/", "https://x.org/b"):
+        assert formats.generic_id(url) == InfoExtractor._generic_id(url)
+
+
 def test_a_download_error_is_scrubbed_the_same_way_wherever_it_is_logged():
     """yt-dlp raises the text it has just logged, and the pipeline logs the exception again one
     line later (`download.job.failed`). The same scrub, or that line puts the URL back."""