fix(logs): deep-review round 1 — say each failure once, and scrub it everywhere

- yt-dlp's text is scrubbed wherever it is logged, not only in the adapter: `formats.log_safe` takes out the URL, the `[extractor] <id>:` prefix and file paths, and the job-failed, refused, edit-failed and backfill lines use it. Done lines log the asset id, not the title-built path. The PII contract now bans `rel`/`rel_path`/`title`. - A traceback goes with a download/edit failure only when it is our own code failing; for yt-dlp's DownloadError, our RuntimeErrors and OSErrors its foot is the unscrubbed message. - yt-dlp's `error` maps to WARNING: it fires on attempts the fallback client set recovers from; the job's verdict is `download.job.failed`. - The sink groups a bare `%s` pass-through by its text (numbers folded), and a process's last batch closes the repeat window, so a short-lived download child still reports what it dropped. - Worker-side failures (stall, wall-clock, no asset, cached asset error) log `download.job.failed` with a cause; they left no line at all before. The crash, supervisor-error and non-zero-exit lines carry the code too. - An edit failure is one coded line in `_fail_edit`, for every path. - `logs.get_logger("__main__")` resolves to the `-m` module, so the worker and the download child keep their names. - TRACE lines carrying user content are gated on `logs.consented_trace()`, not on the process level; a contract test pins it. - The OAuth token refresh goes through `_upstream_failure`; transport errors carry reason `transport`. Subscribe recognises a duplicate by reason code only (the prose fallback was dead). - Once-only guards for a stale sysconfig override and a refusing HLS session directory; item_detail splits Plex failures from our own parse bugs. - The playlist breaker line names the playlist and the action; comments that were false (runner noqa, `_merge_vtt`, sink grouping) are corrected.
2026-09-18 03:16:15 +02:00
parent 04b58e5c6f
commit 54d19cc386
22 changed files with 834 additions and 141 deletions
@@ -0,0 +1,220 @@
+"""E22.5 review round 1: a failure is reported once, with its code, and a repeating one is not a
+flood.
+
+Each case here is a place where the first sweep got the line wrong in one of two directions: a
+failure that left no line at all (a download the watchdog killed), or one that left too many (the
+same edit failure twice, a warning per segment request, per settings read).
+"""
+
+import json
+import logging
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yt_dlp
+
+from app import logs, sysconfig, worker
+from app.downloads import pipeline
+from app.plex import stream as plex_stream
+
+
+class _Sink(logging.Handler):
+    def __init__(self):
+        super().__init__(level=logging.NOTSET)
+        self.lines: list[str] = []
+        self.setFormatter(logs.JsonFormatter())
+        self.addFilter(logs.ContextFilter())
+
+    def emit(self, record):
+        self.lines.append(self.format(record))
+
+    def objects(self) -> list[dict]:
+        return [json.loads(line) for line in self.lines]
+
+    def events(self, code: str) -> list[dict]:
+        return [o for o in self.objects() if o.get("event") == code]
+
+
+@pytest.fixture
+def sink():
+    logger = logging.getLogger("siftlode")
+    handler = _Sink()
+    previous = logger.level
+    logger.addHandler(handler)
+    logger.setLevel(logs.TRACE)
+    try:
+        yield handler
+    finally:
+        logger.removeHandler(handler)
+        logger.setLevel(previous)
+
+
+class _NoDb:
+    """A session that finds nothing and commits nothing — these cases are about the line."""
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+    def get(self, *a, **k):
+        return None
+
+    def commit(self):
+        pass
+
+
+@pytest.fixture
+def finalized(monkeypatch):
+    """Whether `finalize_if_running` fires: True unless a case says a route got there first."""
+    state = {"fires": True}
+    monkeypatch.setattr(worker, "SessionLocal", _NoDb)
+    monkeypatch.setattr(pipeline, "SessionLocal", _NoDb)
+    fire = lambda db, job_id, **fields: state["fires"]  # noqa: E731
+    monkeypatch.setattr(worker.service, "finalize_if_running", fire)
+    return state
+
+
+# --- the worker's own failures ---------------------------------------------------------------
+@pytest.mark.parametrize("cause", ["stalled", "timeout"])
+def test_a_download_the_watchdog_kills_leaves_a_coded_line(sink, finalized, cause):
+    """The child is killed, so it never reports; before this the worker did not either, and a
+    stalled download failed with no line anywhere."""
+    worker._fail_running(42, error="stopped", error_code="network", cause=cause)
+
+    (line,) = sink.events("download.job.failed")
+    assert line["level"] == "WARNING"
+    assert line["attrs"] == {"job_id": 42, "error_code": "network", "cause": cause}
+
+
+def test_a_job_a_route_already_stopped_is_not_reported_as_failed(sink, finalized):
+    finalized["fires"] = False
+    worker._fail_running(42, error="stopped", error_code="network", cause="stalled")
+
+    assert sink.events("download.job.failed") == []
+
+
+def test_a_caller_with_its_own_line_is_not_reported_twice(sink, finalized):
+    worker._fail_running(42, error="Worker error", error_code=None)
+
+    assert sink.events("download.job.failed") == []
+
+
+# --- an edit failure, once ---------------------------------------------------------------------
+def test_an_edit_failure_is_one_line_with_its_code_and_our_own_stack(sink, finalized):
+    try:
+        raise TypeError("a spec shape nobody expected")
+    except TypeError as exc:
+        pipeline._fail_edit(9, 3, str(exc), exc)
+
+    (line,) = sink.events("edit.job.failed")
+    assert line["attrs"] == {"job_id": 9, "asset_id": 3, "error_type": "TypeError"}
+    assert "TypeError" in line["exc"]
+    assert len([o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]) == 1
+
+
+def test_an_edit_that_ffmpeg_refused_names_no_file_and_carries_no_stack(sink, finalized):
+    """ffmpeg's stderr tail names its input, whose path is the channel and the title. The message
+    is scrubbed; the stack is left off, because its last line is the same text unscrubbed."""
+    msg = "ffmpeg failed (1): /downloads/Some_Channel/Season_2024/A_Private_Title_a3.mp4: Invalid data"
+    pipeline._fail_edit(9, 3, msg, RuntimeError(msg))
+
+    (line,) = sink.events("edit.job.failed")
+    assert "A_Private_Title" not in json.dumps(line)
+    assert "exc" not in line
+
+
+def test_an_edit_whose_source_is_gone_is_found_by_the_same_code(sink, finalized):
+    pipeline._fail_edit(9, 3, "The source file is missing.")
+
+    (line,) = sink.events("edit.job.failed")
+    assert line["attrs"] == {"job_id": 9, "asset_id": 3}
+
+
+# --- a download failure, scrubbed ---------------------------------------------------------------
+def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink):
+    """yt-dlp logs its error through the adapter (scrubbed) and then raises the same text, which
+    the pipeline logs again. Unscrubbed, that second line is the URL the first one hid."""
+    msg = "ERROR: Unsupported URL: https://example.org/private/path?id=123"
+    pipeline._log_download_failed(5, msg, "unknown", yt_dlp.utils.DownloadError(msg))
+
+    (line,) = sink.events("download.job.failed")
+    assert "private/path" not in json.dumps(line)
+    assert "example.org/…" in line["msg"]
+    # Unclassified, but yt-dlp's own error: its stack is the `except`, its foot the raw text.
+    assert "exc" not in line
+
+
+def test_an_unclassified_failure_of_our_own_keeps_its_stack(sink):
+    try:
+        raise KeyError("requested_formats")
+    except KeyError as exc:
+        pipeline._log_download_failed(5, str(exc), "unknown", exc)
+
+    (line,) = sink.events("download.job.failed")
+    assert "KeyError" in line["exc"]
+    assert line["attrs"]["error_type"] == "KeyError"
+
+
+def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkeypatch):
+    from app.downloads import adapters, backfill
+
+    url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
+
+    class _Refusing:
+        def __init__(self, opts):
+            pass
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+        def extract_info(self, target, download):
+            raise yt_dlp.utils.DownloadError(
+                f"ERROR: [youtube] dQw4w9WgXcQ: Video unavailable ({target})"
+            )
+
+    monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing)
+    assert backfill._extract_meta(url, adapters.YouTubeAdapter(), 11) is None
+
+    lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]]
+    assert lines and all("dQw4w9WgXcQ" not in json.dumps(o) for o in lines)
+
+
+# --- a repeating failure, once ------------------------------------------------------------------
+def test_a_stale_setting_is_reported_once_not_on_every_read(sink, monkeypatch):
+    spec = next(s for s in sysconfig.SPECS if s.type == "int")
+    monkeypatch.setattr(sysconfig, "_raw_override", lambda db, key: "not-a-number")
+    monkeypatch.setattr(sysconfig, "_coerce_warned", set())
+
+    for _ in range(3):
+        sysconfig.get(None, spec.key)
+
+    assert len(sink.events("sysconfig.override_uncoercible")) == 1
+
+
+class _RefusingDir:
+    """A session directory on a disk that has gone away: every probe raises."""
+
+    def __truediv__(self, name):
+        return SimpleNamespace(exists=self._refuse)
+
+    @staticmethod
+    def _refuse():
+        raise PermissionError("the disk went read-only")
+
+
+def test_a_refusing_session_directory_is_reported_once_per_session(sink):
+    """Every segment request probes, and hls.js retries each 404 — a line per probe is a flood."""
+    session = plex_stream.HlsSession("k", Path("."), None, 0.0, ())
+    session.dir = _RefusingDir()
+
+    for seq in (3, 4, 5):
+        assert not plex_stream.artifact_may_arrive(session, plex_stream.segment_name(seq, None))
+
+    (line,) = sink.events("plex.stream.segment_probe_failed")
+    assert line["attrs"]["reason"] == "PermissionError"