fix(logs): deep-review round 2 — a stack names code, not data, and survives a failed write
- A failure keeps its traceback unless it is yt-dlp's or our own StatedFailure (a message that is the whole story). A plain RuntimeError, RecursionError or OSError is also what a bug of ours raises, and its stack is the only lead. - The stack is scrubbed link by link (logs.scrubbed_stack): a chained DownloadError under a DB error no longer prints yt-dlp's text unscrubbed. - Engine hide_parameters: a DB error's message no longer carries the title or path it was writing. - The download and edit failure lines go out even when their DB write fails. - The worker's child_exit/crash/supervisor lines say download.job.failed only when the job failed there; a crash the DB will not record no longer ends the thread. Stopping a paused child is download.child.stopped, not a second abort. - log_safe also covers generic ids with spaces, playlist titles and bare ids. - The ffmpeg image steps log the failure kind, not the argv with its path. - The consent contract accepts only the body of a plain consented_trace() check, and any level that may be TRACE. - Tests for each; every one mutation-checked.
This commit is contained in:
@@ -125,6 +125,25 @@ def test_the_download_child_knows_the_cookie_went_with_it(db, monkeypatch, tmp_p
|
||||
assert (job.error_code, job.error_detail) == ("login_required", "facebook.cookie_rejected")
|
||||
|
||||
|
||||
def test_the_failure_line_goes_out_even_when_its_write_fails(db, monkeypatch, tmp_path):
|
||||
"""The database is often what failed; the write of the failure then fails too, and the line
|
||||
that says so must not go down with it."""
|
||||
u = _user(db, "fb-dbdown@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="dbdown", asset_status="downloading", job_status="running")
|
||||
_fail_the_child(monkeypatch, tmp_path, cookie=False)
|
||||
said = []
|
||||
monkeypatch.setattr(pipeline, "_log_download_failed", lambda job_id, *rest: said.append(job_id))
|
||||
|
||||
def db_down(*a, **k):
|
||||
raise ConnectionError("the database went away")
|
||||
|
||||
monkeypatch.setattr(pipeline.service, "finalize_if_running", db_down)
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
pipeline._download(j.id, a.id, "url", _REEL, {}, u.id)
|
||||
assert said == [j.id]
|
||||
|
||||
|
||||
def test_a_job_meeting_a_failed_shared_asset_gets_the_same_explanation(db):
|
||||
# Before the catalog this path wrote the text and NO code: "Download failed" for the second
|
||||
# job, "Facebook sign-in required" for the first.
|
||||
|
||||
@@ -16,6 +16,8 @@ import yt_dlp
|
||||
|
||||
from app import logs, sysconfig, worker
|
||||
from app.downloads import pipeline
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.plex import stream as plex_stream
|
||||
|
||||
|
||||
@@ -102,6 +104,98 @@ def test_a_caller_with_its_own_line_is_not_reported_twice(sink, finalized):
|
||||
assert sink.events("download.job.failed") == []
|
||||
|
||||
|
||||
def test_a_child_that_died_is_reported_with_its_exit_code(sink, finalized):
|
||||
worker._fail_running(
|
||||
42, error="exited", error_code="unknown", cause="child_exit", exit_code=137
|
||||
)
|
||||
|
||||
(line,) = sink.events("download.job.failed")
|
||||
assert line["attrs"]["exit_code"] == 137
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fires", "code"), [(True, "download.job.failed"), (False, "download.worker.error")]
|
||||
)
|
||||
def test_a_worker_error_is_one_line_that_says_what_it_did_to_the_job(sink, fires, code):
|
||||
"""A route's cancel that landed first is not a failure — the error is still reported, with
|
||||
its stack, but not as the job failing."""
|
||||
try:
|
||||
raise KeyError("a worker bug")
|
||||
except KeyError as exc:
|
||||
worker._report_worker_error(42, exc, "crash", lambda: fires)
|
||||
|
||||
(line,) = [o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]
|
||||
assert line["event"] == code
|
||||
assert "KeyError" in line["exc"]
|
||||
|
||||
|
||||
def test_a_worker_error_is_reported_even_when_failing_the_job_raises(sink):
|
||||
def db_down():
|
||||
raise ConnectionError("the database went away")
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
worker._report_worker_error(42, KeyError("a worker bug"), "crash", db_down)
|
||||
|
||||
(line,) = sink.events("download.worker.error")
|
||||
assert line["attrs"]["cause"] == "crash"
|
||||
|
||||
|
||||
def test_a_crash_the_database_will_not_record_does_not_end_the_worker_thread(monkeypatch, sink):
|
||||
"""The crash handler writes to the database; if that raises too, the thread must go on to
|
||||
the next job — a dead thread is a download slot gone until the worker restarts."""
|
||||
claims = iter([42, 43])
|
||||
seen = []
|
||||
|
||||
def claim():
|
||||
job = next(claims, None)
|
||||
if job is None:
|
||||
worker._stop.set()
|
||||
return job
|
||||
|
||||
def crash(job_id):
|
||||
seen.append(job_id)
|
||||
raise RuntimeError("boom")
|
||||
|
||||
def db_down(*a, **k):
|
||||
raise ConnectionError("the database went away")
|
||||
|
||||
monkeypatch.setattr(worker, "_claim_job", claim)
|
||||
monkeypatch.setattr(worker, "_process_job", crash)
|
||||
monkeypatch.setattr(worker, "_fail_running", db_down)
|
||||
monkeypatch.setattr(worker.settings, "download_poll_seconds", 0)
|
||||
worker._stop.clear()
|
||||
try:
|
||||
worker._worker_loop(0)
|
||||
finally:
|
||||
worker._stop.clear()
|
||||
assert seen == [42, 43]
|
||||
|
||||
|
||||
class _StoppedProc:
|
||||
"""A child that is still tearing down when the supervisor polls, after a route paused it."""
|
||||
|
||||
pid = 4242
|
||||
returncode = None
|
||||
|
||||
def wait(self, timeout=None):
|
||||
import subprocess
|
||||
|
||||
raise subprocess.TimeoutExpired("child", timeout or 0)
|
||||
|
||||
|
||||
def test_the_worker_stopping_a_paused_child_is_not_a_second_abort_line(monkeypatch, sink):
|
||||
"""The child logs `download.job.aborted` when it notices the pause itself; the worker's line
|
||||
is its own fact — it had to stop the child — so one pause is never two aborts."""
|
||||
monkeypatch.setattr(worker, "_spawn_download_child", lambda job_id: _StoppedProc())
|
||||
monkeypatch.setattr(worker, "_job_liveness", lambda job_id: ("paused", 1.0))
|
||||
monkeypatch.setattr(worker, "_stop_and_free", lambda *a: None)
|
||||
|
||||
worker._run_supervised(42, 7)
|
||||
|
||||
assert sink.events("download.job.aborted") == []
|
||||
assert len(sink.events("download.child.stopped")) == 1
|
||||
|
||||
|
||||
# --- an edit failure, once ---------------------------------------------------------------------
|
||||
def test_an_edit_failure_is_one_line_with_its_code_and_our_own_stack(sink, finalized):
|
||||
try:
|
||||
@@ -117,15 +211,65 @@ def test_an_edit_failure_is_one_line_with_its_code_and_our_own_stack(sink, final
|
||||
|
||||
def test_an_edit_that_ffmpeg_refused_names_no_file_and_carries_no_stack(sink, finalized):
|
||||
"""ffmpeg's stderr tail names its input, whose path is the channel and the title. The message
|
||||
is scrubbed; the stack is left off, because its last line is the same text unscrubbed."""
|
||||
is scrubbed; the stack is left off, because the message is the whole story."""
|
||||
msg = "ffmpeg failed (1): /downloads/Some_Channel/Season_2024/A_Private_Title_a3.mp4: Invalid data"
|
||||
pipeline._fail_edit(9, 3, msg, RuntimeError(msg))
|
||||
pipeline._fail_edit(9, 3, msg, StatedFailure(msg))
|
||||
|
||||
(line,) = sink.events("edit.job.failed")
|
||||
assert "A_Private_Title" not in json.dumps(line)
|
||||
assert "exc" not in line
|
||||
|
||||
|
||||
def test_an_edit_failure_is_reported_even_when_its_write_fails(sink, monkeypatch):
|
||||
"""The database is often what failed; then the write fails too, and the line must not go
|
||||
down with it."""
|
||||
|
||||
class _Down:
|
||||
def __enter__(self):
|
||||
raise ConnectionError("the database went away")
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(pipeline, "SessionLocal", _Down)
|
||||
with pytest.raises(ConnectionError):
|
||||
pipeline._fail_edit(9, 3, "boom", KeyError("x"))
|
||||
|
||||
assert len(sink.events("edit.job.failed")) == 1
|
||||
|
||||
|
||||
class _EditDb(_NoDb):
|
||||
"""An edit job whose source is ready, for `_process_edit` to get as far as its encode."""
|
||||
|
||||
def get(self, model, _id):
|
||||
if model is DownloadJob:
|
||||
return SimpleNamespace(source_asset_id=7, edit_spec={}, user_id=1)
|
||||
if model is MediaAsset:
|
||||
return SimpleNamespace(
|
||||
status="ready", rel_path="src.mp4", width=640, height=360, duration_s=10
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def test_an_edit_that_crashes_is_said_once(sink, finalized, monkeypatch, tmp_path):
|
||||
"""The line `_process_edit`'s handler used to log before `_fail_edit` logged it again."""
|
||||
(tmp_path / "src.mp4").write_bytes(b"x")
|
||||
monkeypatch.setattr(pipeline, "SessionLocal", _EditDb)
|
||||
monkeypatch.setattr(pipeline.settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging")
|
||||
|
||||
def odd_spec(*a, **k):
|
||||
raise TypeError("a spec shape nobody expected")
|
||||
|
||||
monkeypatch.setattr(pipeline, "_set_job", odd_spec)
|
||||
|
||||
pipeline._process_edit(9, 3)
|
||||
|
||||
(line,) = [o for o in sink.objects() if o["level"] in ("WARNING", "ERROR")]
|
||||
assert line["event"] == "edit.job.failed"
|
||||
assert "TypeError" in line["exc"]
|
||||
|
||||
|
||||
def test_an_edit_whose_source_is_gone_is_found_by_the_same_code(sink, finalized):
|
||||
pipeline._fail_edit(9, 3, "The source file is missing.")
|
||||
|
||||
@@ -147,6 +291,48 @@ def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink
|
||||
assert "exc" not in line
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
RuntimeError("dictionary changed size during iteration"),
|
||||
RecursionError("maximum recursion depth exceeded"),
|
||||
NotImplementedError(),
|
||||
FileNotFoundError(2, "No such file", "/downloads/Some_Channel/Season_2024/A_Private_Title"),
|
||||
],
|
||||
ids=lambda e: type(e).__name__,
|
||||
)
|
||||
def test_a_bug_that_python_raises_keeps_its_stack_and_names_no_file(sink, exc):
|
||||
"""Only the failures WE raise with the whole story in the message go without a stack. The
|
||||
same types are what Python raises for a bug of ours, and then the stack is the only lead."""
|
||||
try:
|
||||
raise exc
|
||||
except (RuntimeError, OSError) as caught:
|
||||
pipeline._log_download_failed(5, str(caught), "unknown", caught)
|
||||
|
||||
(line,) = sink.events("download.job.failed")
|
||||
assert type(exc).__name__ in line["exc"]
|
||||
assert "A_Private_Title" not in json.dumps(line)
|
||||
|
||||
|
||||
def test_a_stack_does_not_carry_the_download_error_it_was_raised_over(sink):
|
||||
"""A database error raised inside `except DownloadError` has yt-dlp's text as its context,
|
||||
and a plain `exc_info` prints that link unscrubbed."""
|
||||
# A variable, as in the pipeline: a frame prints its SOURCE line, and that is code, not data.
|
||||
text = "ERROR: [youtube] dQw4w9WgXcQ: Sign in to confirm"
|
||||
try:
|
||||
try:
|
||||
raise yt_dlp.utils.DownloadError(text)
|
||||
except yt_dlp.utils.DownloadError:
|
||||
raise ConnectionError("server closed the connection unexpectedly") # noqa: B904
|
||||
except ConnectionError as exc:
|
||||
pipeline._log_download_failed(5, str(exc), "unknown", exc)
|
||||
|
||||
(line,) = sink.events("download.job.failed")
|
||||
assert "During handling of the above exception" in line["exc"]
|
||||
assert "Sign in to confirm" in line["exc"]
|
||||
assert "dQw4w9WgXcQ" not in json.dumps(line)
|
||||
|
||||
|
||||
def test_an_unclassified_failure_of_our_own_keeps_its_stack(sink):
|
||||
try:
|
||||
raise KeyError("requested_formats")
|
||||
@@ -218,3 +404,62 @@ def test_a_refusing_session_directory_is_reported_once_per_session(sink):
|
||||
|
||||
(line,) = sink.events("plex.stream.segment_probe_failed")
|
||||
assert line["attrs"]["reason"] == "PermissionError"
|
||||
|
||||
|
||||
def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, tmp_path):
|
||||
"""The first attempt's DownloadError is logged when the fallback set takes over — one line
|
||||
before the adapter's scrubbed copy of the same text would have hidden it."""
|
||||
url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
attempts = iter(
|
||||
[yt_dlp.utils.DownloadError(f"ERROR: [youtube] dQw4w9WgXcQ: Sign in ({url})"), {"id": "x"}]
|
||||
)
|
||||
|
||||
class _YDL:
|
||||
def __init__(self, opts):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def extract_info(self, target, download):
|
||||
outcome = next(attempts)
|
||||
if isinstance(outcome, Exception):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
adapter = SimpleNamespace(
|
||||
client_sets=lambda authed: [["web"], ["tv"]],
|
||||
resolve_url=lambda target, opts: target,
|
||||
ydl_opts_extra=lambda clients: {},
|
||||
retriable=lambda text: True,
|
||||
)
|
||||
monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", _YDL)
|
||||
monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None)
|
||||
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **fields: None)
|
||||
|
||||
assert pipeline._extract_with_fallback(1, 1, url, {}, tmp_path, adapter) == {"id": "x"}
|
||||
|
||||
(line,) = [o for o in sink.objects() if "retrying with fallback" in o["msg"]]
|
||||
assert "dQw4w9WgXcQ" not in json.dumps(line)
|
||||
|
||||
|
||||
def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
|
||||
"""ffmpeg's CalledProcessError quotes its argv, and the source path in it is built from the
|
||||
channel and the title."""
|
||||
from app.downloads import edit as editmod
|
||||
|
||||
rel = "Some_Channel/Season_2024/Some_Channel_-_20240101_-_A_Private_Title_a3.mp4"
|
||||
(tmp_path / rel).parent.mkdir(parents=True)
|
||||
(tmp_path / rel).write_bytes(b"not a video")
|
||||
asset = SimpleNamespace(id=3, rel_path=rel, duration_s=10.0, storyboard_path=None)
|
||||
|
||||
assert editmod.ensure_poster(asset, str(tmp_path)) is None
|
||||
assert editmod.ensure_storyboard(asset, str(tmp_path)) is None
|
||||
|
||||
lines = [o for o in sink.objects() if "gen skipped" in o["msg"]]
|
||||
assert len(lines) == 2
|
||||
assert all("Some_Channel" not in json.dumps(o) for o in lines)
|
||||
assert all("CalledProcessError" in o["msg"] for o in lines)
|
||||
|
||||
+118
-12
@@ -127,15 +127,31 @@ def test_a_scope_with_no_elevation_is_not_consent():
|
||||
assert not logs.consented_trace() # a scheduler thread: no request at all
|
||||
|
||||
|
||||
def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink):
|
||||
def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink, monkeypatch):
|
||||
"""The token endpoint's error is `{"error": "<code>", "error_description": <prose>}` — not the
|
||||
Data API's shape. It goes through the same door, so it gets the same treatment."""
|
||||
Data API's shape. The refresh goes through the same door, so it gets the same treatment."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.youtube import client as yt
|
||||
|
||||
resp = httpx.Response(
|
||||
400,
|
||||
json={"error": "invalid_grant", "error_description": "Token has been expired or revoked."},
|
||||
request=httpx.Request("POST", "https://oauth2.googleapis.com/token"),
|
||||
)
|
||||
err = _upstream_failure("POST", "oauth2/token", resp)
|
||||
client = yt.YouTubeClient.__new__(yt.YouTubeClient) # no HTTP client: `_send` is the stub
|
||||
client.db = None
|
||||
client.user = SimpleNamespace(
|
||||
token=SimpleNamespace(access_token=None, expiry=None, refresh_token_enc=b"x")
|
||||
)
|
||||
monkeypatch.setattr(yt, "decrypt_optional", lambda value: None)
|
||||
monkeypatch.setattr(yt, "decrypt", lambda value: "a-refresh-token")
|
||||
monkeypatch.setattr(yt.sysconfig, "get_str", lambda db, key: "")
|
||||
monkeypatch.setattr(client, "_send", lambda *a, **k: resp)
|
||||
|
||||
with pytest.raises(YouTubeError) as caught:
|
||||
client._access_token()
|
||||
err = caught.value
|
||||
|
||||
(line,) = sink.at("WARNING")
|
||||
assert line["attrs"]["reason"] == "invalid_grant"
|
||||
@@ -267,16 +283,88 @@ def test_no_log_call_passes_what_a_person_typed():
|
||||
)
|
||||
|
||||
|
||||
_AT_LEAST_DEBUG = {"DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"}
|
||||
|
||||
|
||||
def _may_run_at_trace(call: ast.Call) -> bool:
|
||||
"""A `log.log(level, …)` whose level is not provably DEBUG or above: `logs.TRACE`, a bare
|
||||
`TRACE`, a literal 5 — or a variable, which could be any of them."""
|
||||
if call.func.attr != "log" or not call.args:
|
||||
return False
|
||||
level = call.args[0]
|
||||
if isinstance(level, ast.Attribute | ast.Name):
|
||||
return (level.attr if isinstance(level, ast.Attribute) else level.id) not in _AT_LEAST_DEBUG
|
||||
if isinstance(level, ast.Constant) and isinstance(level.value, int):
|
||||
return level.value < logging.DEBUG
|
||||
return True
|
||||
|
||||
|
||||
def _is_consent_check(test: ast.expr) -> bool:
|
||||
"""Exactly `…consented_trace()`: not `not …`, not `x or …` — each of those runs the body when
|
||||
there is no consent."""
|
||||
return (
|
||||
isinstance(test, ast.Call)
|
||||
and isinstance(test.func, ast.Attribute | ast.Name)
|
||||
and (test.func.attr if isinstance(test.func, ast.Attribute) else test.func.id)
|
||||
== "consented_trace"
|
||||
)
|
||||
|
||||
|
||||
def _guarded_by_consent(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> bool:
|
||||
"""The node runs only inside the BODY of an `if <consented_trace()>:` — not its `else`."""
|
||||
while node in parents:
|
||||
node = parents[node]
|
||||
if isinstance(node, ast.If):
|
||||
for sub in ast.walk(node.test):
|
||||
if isinstance(sub, ast.Attribute) and sub.attr == "consented_trace":
|
||||
return True
|
||||
parent = parents[node]
|
||||
if isinstance(parent, ast.If) and _is_consent_check(parent.test) and node in parent.body:
|
||||
return True
|
||||
node = parent
|
||||
return False
|
||||
|
||||
|
||||
def _parents(tree: ast.AST) -> dict[ast.AST, ast.AST]:
|
||||
return {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
|
||||
|
||||
|
||||
_GATE_SHAPES = {
|
||||
"the body of the check": ("if logs.consented_trace():\n log.log(logs.TRACE, 'x')", True),
|
||||
"its else": (
|
||||
"if logs.consented_trace():\n pass\nelse:\n log.log(logs.TRACE, 'x')",
|
||||
False,
|
||||
),
|
||||
"a negated check": ("if not logs.consented_trace():\n log.log(logs.TRACE, 'x')", False),
|
||||
"an or": ("if debug or logs.consented_trace():\n log.log(logs.TRACE, 'x')", False),
|
||||
"a nested block in the body": (
|
||||
"if logs.consented_trace():\n for t in ts:\n log.log(logs.TRACE, 'x')",
|
||||
True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", list(_GATE_SHAPES))
|
||||
def test_the_consent_gate_is_the_body_of_the_check_and_nothing_else(shape):
|
||||
src, gated = _GATE_SHAPES[shape]
|
||||
tree = ast.parse(src)
|
||||
(call,) = [n for n in ast.walk(tree) if isinstance(n, ast.Call) and _is_log_call(n)]
|
||||
assert _guarded_by_consent(call, _parents(tree)) is gated
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("level", "trace"),
|
||||
[
|
||||
("logs.TRACE", True),
|
||||
("TRACE", True),
|
||||
("5", True),
|
||||
("level", True),
|
||||
("levels[scope]", True),
|
||||
("logging.DEBUG", False),
|
||||
("logging.WARNING", False),
|
||||
("20", False),
|
||||
],
|
||||
)
|
||||
def test_a_level_that_may_be_trace_is_treated_as_trace(level, trace):
|
||||
(call,) = [n for n in ast.walk(ast.parse(f"log.log({level}, 'x')")) if isinstance(n, ast.Call)]
|
||||
assert _may_run_at_trace(call) is trace
|
||||
|
||||
|
||||
def test_every_escaped_trace_line_is_gated_on_the_users_own_consent():
|
||||
"""`log-pii-ok` on a TRACE line means "this is what the user's Teljes consent is for". The TRACE
|
||||
level alone does not hold that promise: an admin can run the whole process at TRACE, and a
|
||||
@@ -287,15 +375,33 @@ def test_every_escaped_trace_line_is_gated_on_the_users_own_consent():
|
||||
src = path.read_text(encoding="utf-8")
|
||||
lines = src.splitlines()
|
||||
tree = ast.parse(src)
|
||||
parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
|
||||
parents = _parents(tree)
|
||||
for node in ast.walk(tree):
|
||||
if not _is_log_call(node) or not _offending(node):
|
||||
continue
|
||||
first = node.args[0] if node.args else None
|
||||
at_trace = isinstance(first, ast.Attribute) and first.attr == "TRACE"
|
||||
span = " ".join(lines[max(0, node.lineno - 2) : (node.end_lineno or node.lineno)])
|
||||
if at_trace and _ESCAPE in span and not _guarded_by_consent(node, parents):
|
||||
if (
|
||||
_may_run_at_trace(node)
|
||||
and _ESCAPE in span
|
||||
and not _guarded_by_consent(node, parents)
|
||||
):
|
||||
offenders.append(f"{path.relative_to(root)}:{node.lineno}")
|
||||
assert offenders == [], (
|
||||
f"a TRACE line with user content is not under consented_trace(): {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_database_error_does_not_carry_the_values_it_was_given(db):
|
||||
"""DB lane. A failed write's message is logged and stored as the job's error text; with its
|
||||
bound parameters in it, that is the title or the file path the row was about."""
|
||||
from sqlalchemy import exc as sa_exc
|
||||
from sqlalchemy import text
|
||||
|
||||
# Postgres's own text for this one names no value ("value too long for type character
|
||||
# varying(3)"), so what is left to leak is SQLAlchemy's `[parameters: …]` suffix.
|
||||
db.execute(text("CREATE TEMP TABLE pii_probe (x varchar(3))"))
|
||||
with pytest.raises(sa_exc.DBAPIError) as caught:
|
||||
db.execute(text("INSERT INTO pii_probe VALUES (:v)"), {"v": "A_Private_Title"})
|
||||
db.rollback()
|
||||
assert "value too long" in str(caught.value)
|
||||
assert "A_Private_Title" not in str(caught.value)
|
||||
|
||||
@@ -271,6 +271,23 @@ def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
|
||||
)
|
||||
|
||||
|
||||
def test_a_stopped_writer_says_what_it_dropped_before_it_goes(db):
|
||||
"""The download child's exit is `stop()` → `_run`'s drain: the notice for the repeats it
|
||||
dropped is written there, or a child that lived under a minute never reports them."""
|
||||
sink = logsink.LogSink(engine)
|
||||
log = logging.getLogger("siftlode.sinktest.dropped")
|
||||
log.addHandler(sink)
|
||||
try:
|
||||
for i in range(logsink.REPEATS_PER_MINUTE + 4):
|
||||
log.warning("RSS poll skipped for channel %s: HTTP 404", f"UC{i}")
|
||||
finally:
|
||||
log.removeHandler(sink)
|
||||
sink._stopping.set()
|
||||
sink._run()
|
||||
last = db.execute(select(LogEvent).order_by(LogEvent.id.desc()).limit(1)).scalar_one()
|
||||
assert (last.event, last.attrs["suppressed"]) == ("log.repeats_suppressed", 4)
|
||||
|
||||
|
||||
def test_the_running_sink_keeps_the_base_level_for_everyone_the_elevation_is_not_for(
|
||||
db, monkeypatch
|
||||
):
|
||||
|
||||
@@ -87,6 +87,7 @@ def test_no_progress_stall_kills_and_fails_the_job(monkeypatch, rec):
|
||||
fail = next(c for c in rec if c[0] == "fail")
|
||||
assert fail[2]["error_code"] == "network"
|
||||
assert "stalled" in fail[2]["error"].lower()
|
||||
assert fail[2]["cause"] == "stalled" # the line the killed child never gets to write
|
||||
|
||||
|
||||
def test_route_pause_kills_but_keeps_the_route_status(monkeypatch, rec):
|
||||
@@ -120,6 +121,7 @@ def test_wall_clock_cap_kills_even_while_progressing(monkeypatch, rec):
|
||||
worker._run_supervised(1, 99)
|
||||
fail = next(c for c in rec if c[0] == "fail")
|
||||
assert "time limit" in fail[2]["error"].lower()
|
||||
assert fail[2]["cause"] == "timeout"
|
||||
|
||||
|
||||
def test_child_crash_without_finishing_is_failed(monkeypatch, rec):
|
||||
@@ -131,6 +133,8 @@ def test_child_crash_without_finishing_is_failed(monkeypatch, rec):
|
||||
assert "reset_asset" in names
|
||||
fail = next(c for c in rec if c[0] == "fail")
|
||||
assert "unexpectedly" in fail[2]["error"].lower()
|
||||
# Its line is `_fail_running`'s, so it is logged only if the job really failed here.
|
||||
assert (fail[2]["cause"], fail[2]["exit_code"]) == ("child_exit", 1)
|
||||
|
||||
|
||||
def test_child_nonzero_exit_that_already_set_state_is_left_alone(monkeypatch, rec):
|
||||
@@ -258,7 +262,7 @@ def test_a_crashed_jobs_crash_line_and_cleanup_carry_the_job_id(monkeypatch):
|
||||
monkeypatch.setattr(worker, "_claim_job", claim)
|
||||
monkeypatch.setattr(worker, "_process_job", crash)
|
||||
monkeypatch.setattr(
|
||||
worker.log, "exception", lambda *a, **k: seen.setdefault("crash", logs._job_id.get())
|
||||
worker.log, "error", lambda *a, **k: seen.setdefault("crash", logs._job_id.get())
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
worker, "_fail_running", lambda *a, **k: seen.setdefault("cleanup", logs._job_id.get())
|
||||
|
||||
@@ -101,6 +101,44 @@ def test_the_source_url_does_not_travel_with_the_explanation(sink):
|
||||
"WARNING: [generic] …: Falling back on generic information",
|
||||
"my-private-holiday-clip",
|
||||
),
|
||||
# ...percent-decoded, so it can have spaces: `https://x/My%20Holiday%20in%20Crete.mp4`.
|
||||
(
|
||||
"ERROR: [generic] My Holiday in Crete: Unable to download webpage: HTTP Error 404",
|
||||
"ERROR: [generic] …: Unable to download webpage: HTTP Error 404",
|
||||
"Holiday",
|
||||
),
|
||||
(
|
||||
"[generic] My Holiday in Crete: Downloading webpage",
|
||||
"[generic] …: Downloading webpage",
|
||||
"Holiday",
|
||||
),
|
||||
# A playlist URL is extracted too (`playlist_items: "1"`), and its title is printed.
|
||||
(
|
||||
"[youtube:tab] Playlist Our Wedding Videos: Downloading 1 items of 12",
|
||||
"[youtube:tab] Playlist …: Downloading 1 items of 12",
|
||||
"Wedding",
|
||||
),
|
||||
(
|
||||
"[download] Finished downloading playlist: Our Wedding Videos",
|
||||
"[download] Finished downloading playlist: …",
|
||||
"Wedding",
|
||||
),
|
||||
(
|
||||
"[download] Downloading playlist: Our Wedding Videos",
|
||||
"[download] Downloading playlist: …",
|
||||
"Wedding",
|
||||
),
|
||||
# An id quoted mid-sentence.
|
||||
(
|
||||
"hu subtitles not available for dQw4w9WgXcQ",
|
||||
"hu subtitles not available for …",
|
||||
"dQw4w9WgXcQ",
|
||||
),
|
||||
(
|
||||
"[youtube:tab] Downloading just video dQw4w9WgXcQ because of --no-playlist",
|
||||
"[youtube:tab] Downloading just video … because of --no-playlist",
|
||||
"dQw4w9WgXcQ",
|
||||
),
|
||||
# `Destination` is yt-dlp's word, not an id; the path after it is built from the title.
|
||||
(
|
||||
"[download] Destination: /downloads/Some_Channel/Season_2024/A_Private_Title_a7.mp4",
|
||||
@@ -125,6 +163,20 @@ def test_a_download_error_is_scrubbed_the_same_way_wherever_it_is_logged():
|
||||
assert formats.log_safe(text) == "ERROR: Unsupported URL: example.org/…"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"line",
|
||||
[
|
||||
"nsig extraction failed: Some formats may be missing",
|
||||
"[VideoConvertor] Converting video from webm to mp4; Destination: x",
|
||||
"[download] 53.2% of ~ 12.00MiB at 1.20MiB/s ETA 00:05",
|
||||
"[download] 100% of 12.00MiB in 00:00:03 at 3.90MiB/s",
|
||||
],
|
||||
)
|
||||
def test_a_line_that_names_nothing_keeps_its_words(line):
|
||||
"""The scrub takes out what names a download, and nothing an operator reads the line for."""
|
||||
assert formats.log_safe(line) == line
|
||||
|
||||
|
||||
def test_a_line_with_no_url_is_left_alone(sink):
|
||||
formats._YDL_LOG.warning("nsig extraction failed: Some formats may be missing")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user