fix(downloads): supervisor edge-cases from review round 3
- worker.py: keep an absolute wall-clock backstop for ANY still-alive child. A child that
reached a terminal state (done/error) but wedges in teardown is now killed to free the worker
slot ("abandon"), with its DB status left intact; only a still-'running' over-cap job is failed.
Previously gating the wall-clock on status=='running' let a wedged finisher loop forever (#1).
- worker.py: reconcile the claimed asset on ANY clean child exit (not only non-zero), so a child
that exits without finalizing it — e.g. an early return on a job DELETEd mid-flight — no longer
leaves the asset stuck 'downloading' until boot recovery (#2).
- worker.py: extract _stop_and_free + _fail_after_kill; the exception / stall / timeout / cancel /
shutdown / abandon paths now share one kill+free (+fail) tail instead of three copies (#3).
- tests: a wedged-finished child is killed by the wall-clock without a status change; clean exits
now assert the reconciliation no-op (#1, #4).
This commit is contained in:
@@ -69,11 +69,11 @@ def _liveness(monkeypatch, value):
|
||||
|
||||
|
||||
def test_clean_child_exit_leaves_the_outcome_alone(monkeypatch, rec):
|
||||
# Child finished on its own (exit 0) — it wrote the terminal state; the supervisor must not kill,
|
||||
# reset, or re-fail anything.
|
||||
# Child finished on its own (exit 0) — it wrote the terminal state. The supervisor only reconciles
|
||||
# the asset (a no-op when the child finalized it) and must not kill or re-fail anything.
|
||||
_spawn(monkeypatch, _FakeProc([("exit", 0)]))
|
||||
worker._run_supervised(1, 99)
|
||||
assert _names(rec) == []
|
||||
assert _names(rec) == ["reset_asset"]
|
||||
|
||||
|
||||
def test_no_progress_stall_kills_and_fails_the_job(monkeypatch, rec):
|
||||
@@ -154,7 +154,7 @@ def test_a_long_download_survives_while_progress_stays_fresh(monkeypatch, rec):
|
||||
_spawn(monkeypatch, _FakeProc(["timeout", "timeout", "timeout", ("exit", 0)]))
|
||||
_liveness(monkeypatch, ("running", 2.0)) # always well under the 10s stall
|
||||
worker._run_supervised(1, 99)
|
||||
assert _names(rec) == []
|
||||
assert _names(rec) == ["reset_asset"] # only the clean-exit reconciliation (a no-op), no kill
|
||||
|
||||
|
||||
def test_a_finishing_child_is_not_mistaken_for_a_cancel(monkeypatch, rec):
|
||||
@@ -164,7 +164,19 @@ def test_a_finishing_child_is_not_mistaken_for_a_cancel(monkeypatch, rec):
|
||||
_spawn(monkeypatch, _FakeProc(["timeout", ("exit", 0)])) # alive one poll, then a clean exit
|
||||
_liveness(monkeypatch, ("done", 1.0))
|
||||
worker._run_supervised(1, 99)
|
||||
assert _names(rec) == [] # never killed, asset/job untouched
|
||||
assert _names(rec) == ["reset_asset"] # not killed; only the clean-exit reconciliation (no-op)
|
||||
|
||||
|
||||
def test_a_wedged_finished_child_is_killed_by_the_wall_clock(monkeypatch, rec):
|
||||
# A child that reached a terminal state (done) but never exits (e.g. wedged in teardown on a stuck
|
||||
# mount) must still be killed by the absolute wall-clock cap to free the worker slot — WITHOUT
|
||||
# overwriting its DB status (it already succeeded).
|
||||
monkeypatch.setattr(worker.settings, "download_wall_clock_max_seconds", 0)
|
||||
_spawn(monkeypatch, _FakeProc(["timeout"])) # never exits within the test
|
||||
_liveness(monkeypatch, ("done", 1.0)) # finished, but the process lingers
|
||||
worker._run_supervised(1, 99)
|
||||
names = _names(rec)
|
||||
assert "stop_child" in names and "set_job" not in names # killed to free the slot, status kept
|
||||
|
||||
|
||||
def test_a_watchdog_error_still_stops_the_child(monkeypatch, rec):
|
||||
|
||||
Reference in New Issue
Block a user