peter
0ed82fe6b6
Defense-in-depth after the ReDoS incident. The heavy work (yt-dlp download / ffmpeg edit) now runs in a child process (app.downloads.runner) supervised by the worker thread, not in the thread itself. A thread can't be killed and a C-level spin can't even be cooperatively cancelled, so one wedged download used to pin a core, hold the GIL and freeze the whole queue; a child process can be SIGTERM/SIGKILLed. The supervisor kills the child (and its process group, so ffmpeg grandchildren die too) when: a route pauses/cancels the job, it makes no progress for download_stall_timeout_seconds (150s), it runs past download_wall_clock_max_seconds (2h), the worker is shutting down, or the child crashes without setting state. On a kill it frees the asset and fails the job with a retryable network error (stalls are not auto-requeued). A clean exit is left alone — the child already wrote the terminal state, including a cooperative pause via the progress hook. - config.py: stall / wall-clock / supervisor-poll / kill-grace settings. - worker.py: _run_supervised + _spawn_download_child / _job_liveness / _stop_child / _cleanup_staging; _process_job claims the asset, then hands the heavy work to the child. - app/downloads/runner.py: the child entrypoint. - docker-compose.localdev.yml: init:true on the worker so tini (PID 1) reaps orphaned ffmpeg grandchildren left by a hard-kill (the worker reaps its own direct children). - tests: watchdog decision logic + SIGTERM->SIGKILL escalation (fake proc, no real subprocess).
176 lines
6.5 KiB
Python
176 lines
6.5 KiB
Python
"""The download supervisor/watchdog: `_run_supervised` decides when to hard-kill a child, and
|
|
`_stop_child` escalates SIGTERM→SIGKILL. No real subprocess or DB here — a fake proc stands in and
|
|
the DB-liveness lookup + side effects are monkeypatched, so the decision logic is exercised on any
|
|
machine. These are regression tests for the class of bug that motivated the whole change: a wedged
|
|
download that a thread could neither kill nor cancel.
|
|
"""
|
|
import signal
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
from app import worker
|
|
|
|
|
|
class _FakeProc:
|
|
"""Stands in for the child Popen. `waits` scripts each wait() call: "timeout" = still alive,
|
|
("exit", rc) = the child finished with that code."""
|
|
|
|
def __init__(self, waits, pid=4242):
|
|
self._waits = list(waits)
|
|
self.pid = pid
|
|
self.returncode = None
|
|
self.signals = []
|
|
|
|
def wait(self, timeout=None):
|
|
step = self._waits.pop(0) if self._waits else "timeout"
|
|
if step == "timeout":
|
|
raise subprocess.TimeoutExpired("child", timeout or 0)
|
|
self.returncode = step[1]
|
|
return step[1]
|
|
|
|
def poll(self):
|
|
return self.returncode
|
|
|
|
def send_signal(self, sig):
|
|
self.signals.append(sig)
|
|
|
|
|
|
@pytest.fixture
|
|
def rec(monkeypatch):
|
|
"""Record supervisor side effects and stub out the real spawn/kill/liveness/staging so a test
|
|
drives them. `calls` collects (name, kwargs) so assertions read the decisions, not the plumbing."""
|
|
calls = []
|
|
|
|
def record(name):
|
|
def _f(*args, **kwargs):
|
|
calls.append((name, args, kwargs))
|
|
return _f
|
|
|
|
monkeypatch.setattr(worker, "_reset_asset_pending", record("reset_asset"))
|
|
monkeypatch.setattr(worker, "_requeue_if_running", record("requeue"))
|
|
monkeypatch.setattr(worker, "_set_job", record("set_job"))
|
|
monkeypatch.setattr(worker, "_cleanup_staging", record("cleanup"))
|
|
monkeypatch.setattr(worker, "_stop_child", record("stop_child"))
|
|
monkeypatch.setattr(worker._stop, "is_set", lambda: False)
|
|
return calls
|
|
|
|
|
|
def _names(calls):
|
|
return [c[0] for c in calls]
|
|
|
|
|
|
def _spawn(monkeypatch, proc):
|
|
monkeypatch.setattr(worker, "_spawn_download_child", lambda job_id: proc)
|
|
|
|
|
|
def _liveness(monkeypatch, value):
|
|
monkeypatch.setattr(worker, "_job_liveness", lambda job_id: value)
|
|
|
|
|
|
def test_clean_child_exit_leaves_the_outcome_alone(monkeypatch, rec):
|
|
# Child finished on its own (exit 0) — it wrote the terminal state; the supervisor must not kill,
|
|
# reset, or re-fail anything.
|
|
_spawn(monkeypatch, _FakeProc([("exit", 0)]))
|
|
worker._run_supervised(1, 99)
|
|
assert _names(rec) == []
|
|
|
|
|
|
def test_no_progress_stall_kills_and_fails_the_job(monkeypatch, rec):
|
|
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
|
_liveness(monkeypatch, ("running", 9999.0)) # far past the stall timeout
|
|
worker._run_supervised(1, 99)
|
|
names = _names(rec)
|
|
assert "stop_child" in names and "reset_asset" in names
|
|
set_job = next(c for c in rec if c[0] == "set_job")
|
|
assert set_job[2]["status"] == "error" and set_job[2]["error_code"] == "network"
|
|
assert "stalled" in set_job[2]["error"].lower()
|
|
|
|
|
|
def test_route_pause_kills_but_keeps_the_route_status(monkeypatch, rec):
|
|
# A route flipped the job to 'paused'/'cancelled'. Kill + free the asset, but do NOT overwrite the
|
|
# status with an error and do NOT requeue.
|
|
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
|
_liveness(monkeypatch, ("paused", 1.0))
|
|
worker._run_supervised(1, 99)
|
|
names = _names(rec)
|
|
assert "stop_child" in names and "reset_asset" in names
|
|
assert "set_job" not in names and "requeue" not in names
|
|
|
|
|
|
def test_wall_clock_cap_kills_even_while_progressing(monkeypatch, rec):
|
|
monkeypatch.setattr(worker.settings, "download_wall_clock_max_seconds", 0)
|
|
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
|
_liveness(monkeypatch, ("running", 1.0)) # fresh progress, but over the absolute cap
|
|
worker._run_supervised(1, 99)
|
|
set_job = next(c for c in rec if c[0] == "set_job")
|
|
assert set_job[2]["status"] == "error"
|
|
assert "time limit" in set_job[2]["error"].lower()
|
|
|
|
|
|
def test_child_crash_without_finishing_is_failed(monkeypatch, rec):
|
|
# Non-zero exit and the job is still 'running' → the child died (crash/OOM) before writing state.
|
|
_spawn(monkeypatch, _FakeProc([("exit", 1)]))
|
|
_liveness(monkeypatch, ("running", 1.0))
|
|
worker._run_supervised(1, 99)
|
|
names = _names(rec)
|
|
assert "reset_asset" in names
|
|
set_job = next(c for c in rec if c[0] == "set_job")
|
|
assert set_job[2]["status"] == "error" and "unexpectedly" in set_job[2]["error"].lower()
|
|
|
|
|
|
def test_child_nonzero_exit_that_already_set_state_is_left_alone(monkeypatch, rec):
|
|
# Child exited non-zero but the job is no longer 'running' (it recorded its own error) → no double.
|
|
_spawn(monkeypatch, _FakeProc([("exit", 1)]))
|
|
_liveness(monkeypatch, ("error", None))
|
|
worker._run_supervised(1, 99)
|
|
assert "set_job" not in _names(rec)
|
|
|
|
|
|
def test_spawn_failure_frees_the_asset_and_requeues(monkeypatch, rec):
|
|
def boom(job_id):
|
|
raise OSError("no fork")
|
|
|
|
monkeypatch.setattr(worker, "_spawn_download_child", boom)
|
|
worker._run_supervised(1, 99)
|
|
names = _names(rec)
|
|
assert "reset_asset" in names and "requeue" in names
|
|
|
|
|
|
def test_shutdown_kills_and_requeues_for_next_boot(monkeypatch, rec):
|
|
monkeypatch.setattr(worker._stop, "is_set", lambda: True)
|
|
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
|
_liveness(monkeypatch, ("running", 1.0))
|
|
worker._run_supervised(1, 99)
|
|
names = _names(rec)
|
|
assert "stop_child" in names and "requeue" in names and "set_job" not in names
|
|
|
|
|
|
# --- _stop_child escalation -----------------------------------------------------------------
|
|
|
|
@pytest.fixture
|
|
def killpg_rec(monkeypatch):
|
|
sent = []
|
|
monkeypatch.setattr(worker.os, "getpgid", lambda pid: pid)
|
|
monkeypatch.setattr(worker.os, "killpg", lambda pgid, sig: sent.append(sig))
|
|
return sent
|
|
|
|
|
|
def test_stop_child_sigterm_is_enough(killpg_rec):
|
|
proc = _FakeProc([("exit", 0)]) # dies on the SIGTERM wait
|
|
worker._stop_child(proc, grace_s=0.01)
|
|
assert killpg_rec == [signal.SIGTERM]
|
|
|
|
|
|
def test_stop_child_escalates_to_sigkill(killpg_rec):
|
|
proc = _FakeProc(["timeout", ("exit", -9)]) # survives SIGTERM, dies on SIGKILL
|
|
worker._stop_child(proc, grace_s=0.01)
|
|
assert killpg_rec == [signal.SIGTERM, signal.SIGKILL]
|
|
|
|
|
|
def test_stop_child_noop_if_already_exited(killpg_rec):
|
|
proc = _FakeProc([])
|
|
proc.returncode = 0 # already gone
|
|
worker._stop_child(proc, grace_s=0.01)
|
|
assert killpg_rec == []
|