"""E22.5 review round 1: a failure is reported once, with its code, and a repeating one is not a flood. Each case here is a place where the first sweep got the line wrong in one of two directions: a failure that left no line at all (a download the watchdog killed), or one that left too many (the same edit failure twice, a warning per segment request, per settings read). """ import json import logging from pathlib import Path from types import SimpleNamespace import pytest import yt_dlp from app import logs, sysconfig, worker from app.downloads import pipeline from app.plex import stream as plex_stream class _Sink(logging.Handler): def __init__(self): super().__init__(level=logging.NOTSET) self.lines: list[str] = [] self.setFormatter(logs.JsonFormatter()) self.addFilter(logs.ContextFilter()) def emit(self, record): self.lines.append(self.format(record)) def objects(self) -> list[dict]: return [json.loads(line) for line in self.lines] def events(self, code: str) -> list[dict]: return [o for o in self.objects() if o.get("event") == code] @pytest.fixture def sink(): logger = logging.getLogger("siftlode") handler = _Sink() previous = logger.level logger.addHandler(handler) logger.setLevel(logs.TRACE) try: yield handler finally: logger.removeHandler(handler) logger.setLevel(previous) class _NoDb: """A session that finds nothing and commits nothing — these cases are about the line.""" def __enter__(self): return self def __exit__(self, *exc): return False def get(self, *a, **k): return None def commit(self): pass @pytest.fixture def finalized(monkeypatch): """Whether `finalize_if_running` fires: True unless a case says a route got there first.""" state = {"fires": True} monkeypatch.setattr(worker, "SessionLocal", _NoDb) monkeypatch.setattr(pipeline, "SessionLocal", _NoDb) fire = lambda db, job_id, **fields: state["fires"] # noqa: E731 monkeypatch.setattr(worker.service, "finalize_if_running", fire) return state # --- the worker's own failures --------------------------------------------------------------- @pytest.mark.parametrize("cause", ["stalled", "timeout"]) def test_a_download_the_watchdog_kills_leaves_a_coded_line(sink, finalized, cause): """The child is killed, so it never reports; before this the worker did not either, and a stalled download failed with no line anywhere.""" worker._fail_running(42, error="stopped", error_code="network", cause=cause) (line,) = sink.events("download.job.failed") assert line["level"] == "WARNING" assert line["attrs"] == {"job_id": 42, "error_code": "network", "cause": cause} def test_a_job_a_route_already_stopped_is_not_reported_as_failed(sink, finalized): finalized["fires"] = False worker._fail_running(42, error="stopped", error_code="network", cause="stalled") assert sink.events("download.job.failed") == [] def test_a_caller_with_its_own_line_is_not_reported_twice(sink, finalized): worker._fail_running(42, error="Worker error", error_code=None) assert sink.events("download.job.failed") == [] # --- an edit failure, once --------------------------------------------------------------------- def test_an_edit_failure_is_one_line_with_its_code_and_our_own_stack(sink, finalized): try: raise TypeError("a spec shape nobody expected") except TypeError as exc: pipeline._fail_edit(9, 3, str(exc), exc) (line,) = sink.events("edit.job.failed") assert line["attrs"] == {"job_id": 9, "asset_id": 3, "error_type": "TypeError"} assert "TypeError" in line["exc"] assert len([o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]) == 1 def test_an_edit_that_ffmpeg_refused_names_no_file_and_carries_no_stack(sink, finalized): """ffmpeg's stderr tail names its input, whose path is the channel and the title. The message is scrubbed; the stack is left off, because its last line is the same text unscrubbed.""" msg = "ffmpeg failed (1): /downloads/Some_Channel/Season_2024/A_Private_Title_a3.mp4: Invalid data" pipeline._fail_edit(9, 3, msg, RuntimeError(msg)) (line,) = sink.events("edit.job.failed") assert "A_Private_Title" not in json.dumps(line) assert "exc" not in line def test_an_edit_whose_source_is_gone_is_found_by_the_same_code(sink, finalized): pipeline._fail_edit(9, 3, "The source file is missing.") (line,) = sink.events("edit.job.failed") assert line["attrs"] == {"job_id": 9, "asset_id": 3} # --- a download failure, scrubbed --------------------------------------------------------------- def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink): """yt-dlp logs its error through the adapter (scrubbed) and then raises the same text, which the pipeline logs again. Unscrubbed, that second line is the URL the first one hid.""" msg = "ERROR: Unsupported URL: https://example.org/private/path?id=123" pipeline._log_download_failed(5, msg, "unknown", yt_dlp.utils.DownloadError(msg)) (line,) = sink.events("download.job.failed") assert "private/path" not in json.dumps(line) assert "example.org/…" in line["msg"] # Unclassified, but yt-dlp's own error: its stack is the `except`, its foot the raw text. assert "exc" not in line def test_an_unclassified_failure_of_our_own_keeps_its_stack(sink): try: raise KeyError("requested_formats") except KeyError as exc: pipeline._log_download_failed(5, str(exc), "unknown", exc) (line,) = sink.events("download.job.failed") assert "KeyError" in line["exc"] assert line["attrs"]["error_type"] == "KeyError" def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkeypatch): from app.downloads import adapters, backfill url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" class _Refusing: def __init__(self, opts): pass def __enter__(self): return self def __exit__(self, *exc): return False def extract_info(self, target, download): raise yt_dlp.utils.DownloadError( f"ERROR: [youtube] dQw4w9WgXcQ: Video unavailable ({target})" ) monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing) assert backfill._extract_meta(url, adapters.YouTubeAdapter(), 11) is None lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]] assert lines and all("dQw4w9WgXcQ" not in json.dumps(o) for o in lines) # --- a repeating failure, once ------------------------------------------------------------------ def test_a_stale_setting_is_reported_once_not_on_every_read(sink, monkeypatch): spec = next(s for s in sysconfig.SPECS if s.type == "int") monkeypatch.setattr(sysconfig, "_raw_override", lambda db, key: "not-a-number") monkeypatch.setattr(sysconfig, "_coerce_warned", set()) for _ in range(3): sysconfig.get(None, spec.key) assert len(sink.events("sysconfig.override_uncoercible")) == 1 class _RefusingDir: """A session directory on a disk that has gone away: every probe raises.""" def __truediv__(self, name): return SimpleNamespace(exists=self._refuse) @staticmethod def _refuse(): raise PermissionError("the disk went read-only") def test_a_refusing_session_directory_is_reported_once_per_session(sink): """Every segment request probes, and hls.js retries each 404 — a line per probe is a flood.""" session = plex_stream.HlsSession("k", Path("."), None, 0.0, ()) session.dir = _RefusingDir() for seq in (3, 4, 5): assert not plex_stream.artifact_may_arrive(session, plex_stream.segment_name(seq, None)) (line,) = sink.events("plex.stream.segment_probe_failed") assert line["attrs"]["reason"] == "PermissionError"