fix(downloads): supervisor edge-cases from review round 3

- worker.py: keep an absolute wall-clock backstop for ANY still-alive child. A child that
  reached a terminal state (done/error) but wedges in teardown is now killed to free the worker
  slot ("abandon"), with its DB status left intact; only a still-'running' over-cap job is failed.
  Previously gating the wall-clock on status=='running' let a wedged finisher loop forever (#1).
- worker.py: reconcile the claimed asset on ANY clean child exit (not only non-zero), so a child
  that exits without finalizing it — e.g. an early return on a job DELETEd mid-flight — no longer
  leaves the asset stuck 'downloading' until boot recovery (#2).
- worker.py: extract _stop_and_free + _fail_after_kill; the exception / stall / timeout / cancel /
  shutdown / abandon paths now share one kill+free (+fail) tail instead of three copies (#3).
- tests: a wedged-finished child is killed by the wall-clock without a status change; clean exits
  now assert the reconciliation no-op (#1, #4).
This commit is contained in:
2026-08-12 23:38:37 +02:00
parent b42b340594
commit cce862cde7
2 changed files with 73 additions and 42 deletions
+17 -5
View File
@@ -69,11 +69,11 @@ def _liveness(monkeypatch, value):
def test_clean_child_exit_leaves_the_outcome_alone(monkeypatch, rec):
# Child finished on its own (exit 0) — it wrote the terminal state; the supervisor must not kill,
# reset, or re-fail anything.
# Child finished on its own (exit 0) — it wrote the terminal state. The supervisor only reconciles
# the asset (a no-op when the child finalized it) and must not kill or re-fail anything.
_spawn(monkeypatch, _FakeProc([("exit", 0)]))
worker._run_supervised(1, 99)
assert _names(rec) == []
assert _names(rec) == ["reset_asset"]
def test_no_progress_stall_kills_and_fails_the_job(monkeypatch, rec):
@@ -154,7 +154,7 @@ def test_a_long_download_survives_while_progress_stays_fresh(monkeypatch, rec):
_spawn(monkeypatch, _FakeProc(["timeout", "timeout", "timeout", ("exit", 0)]))
_liveness(monkeypatch, ("running", 2.0)) # always well under the 10s stall
worker._run_supervised(1, 99)
assert _names(rec) == []
assert _names(rec) == ["reset_asset"] # only the clean-exit reconciliation (a no-op), no kill
def test_a_finishing_child_is_not_mistaken_for_a_cancel(monkeypatch, rec):
@@ -164,7 +164,19 @@ def test_a_finishing_child_is_not_mistaken_for_a_cancel(monkeypatch, rec):
_spawn(monkeypatch, _FakeProc(["timeout", ("exit", 0)])) # alive one poll, then a clean exit
_liveness(monkeypatch, ("done", 1.0))
worker._run_supervised(1, 99)
assert _names(rec) == [] # never killed, asset/job untouched
assert _names(rec) == ["reset_asset"] # not killed; only the clean-exit reconciliation (no-op)
def test_a_wedged_finished_child_is_killed_by_the_wall_clock(monkeypatch, rec):
# A child that reached a terminal state (done) but never exits (e.g. wedged in teardown on a stuck
# mount) must still be killed by the absolute wall-clock cap to free the worker slot — WITHOUT
# overwriting its DB status (it already succeeded).
monkeypatch.setattr(worker.settings, "download_wall_clock_max_seconds", 0)
_spawn(monkeypatch, _FakeProc(["timeout"])) # never exits within the test
_liveness(monkeypatch, ("done", 1.0)) # finished, but the process lingers
worker._run_supervised(1, 99)
names = _names(rec)
assert "stop_child" in names and "set_job" not in names # killed to free the slot, status kept
def test_a_watchdog_error_still_stops_the_child(monkeypatch, rec):