refactor(downloads): address /code-review on the download supervisor
- worker.py: wrap the watchdog loop so a failure in it (e.g. a transient DB error in _job_liveness) still hard-kills the child + fails the job, instead of leaving the child running unsupervised — the exact wedge the feature prevents (#1). - worker.py: _touch_job heartbeat bumps updated_at at each extraction attempt, so a slow / multi-client-set fallback extraction (which writes no progress, and re-writes identical values so ORM onupdate does not fire) is not false-killed as "stalled" (#2). - edit.py: extract escalate_kill (terminate->wait->kill escalation); _stop and worker's _stop_child (process-group variant) now share it (#3). - worker.py: document the edit path's second, tighter run_ffmpeg watchdog and that the supervisor is POSIX/Linux-container-only (#4, #6). - tests: a long download stays alive while progress is fresh, and a watchdog-loop error still stops the child (#1, #5).
This commit is contained in:
@@ -146,6 +146,32 @@ def test_shutdown_kills_and_requeues_for_next_boot(monkeypatch, rec):
|
||||
assert "stop_child" in names and "requeue" in names and "set_job" not in names
|
||||
|
||||
|
||||
def test_a_long_download_survives_while_progress_stays_fresh(monkeypatch, rec):
|
||||
# A download that outlives the stall timeout must NOT be killed as long as it keeps writing
|
||||
# progress (updated_at stays fresh via the model's onupdate). Model: alive across several polls,
|
||||
# liveness always fresh, then a clean exit — nothing killed, nothing failed.
|
||||
monkeypatch.setattr(worker.settings, "download_stall_timeout_seconds", 10)
|
||||
_spawn(monkeypatch, _FakeProc(["timeout", "timeout", "timeout", ("exit", 0)]))
|
||||
_liveness(monkeypatch, ("running", 2.0)) # always well under the 10s stall
|
||||
worker._run_supervised(1, 99)
|
||||
assert _names(rec) == []
|
||||
|
||||
|
||||
def test_a_watchdog_error_still_stops_the_child(monkeypatch, rec):
|
||||
# If the liveness probe raises (e.g. a transient DB error), the child must NOT be left running
|
||||
# unsupervised — the feature's whole point. It is killed and the job is failed.
|
||||
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
||||
|
||||
def boom(job_id):
|
||||
raise RuntimeError("db down")
|
||||
|
||||
monkeypatch.setattr(worker, "_job_liveness", boom)
|
||||
worker._run_supervised(1, 99)
|
||||
names = _names(rec)
|
||||
assert "stop_child" in names and "reset_asset" in names
|
||||
assert next(c for c in rec if c[0] == "set_job")[2]["status"] == "error"
|
||||
|
||||
|
||||
# --- _stop_child escalation -----------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
Reference in New Issue
Block a user