refactor(downloads): address /code-review on the download supervisor
- worker.py: wrap the watchdog loop so a failure in it (e.g. a transient DB error in _job_liveness) still hard-kills the child + fails the job, instead of leaving the child running unsupervised — the exact wedge the feature prevents (#1). - worker.py: _touch_job heartbeat bumps updated_at at each extraction attempt, so a slow / multi-client-set fallback extraction (which writes no progress, and re-writes identical values so ORM onupdate does not fire) is not false-killed as "stalled" (#2). - edit.py: extract escalate_kill (terminate->wait->kill escalation); _stop and worker's _stop_child (process-group variant) now share it (#3). - worker.py: document the edit path's second, tighter run_ffmpeg watchdog and that the supervisor is POSIX/Linux-container-only (#4, #6). - tests: a long download stays alive while progress is fresh, and a watchdog-loop error still stops the child (#1, #5).
This commit is contained in:
@@ -288,20 +288,30 @@ def _drain_tail(stream, sink: list[str]) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _stop(proc: subprocess.Popen) -> None:
|
||||
"""Terminate, then kill — never wait unbounded on a process we've given up on. Safe to call on
|
||||
a process that already exited (the single exit path below may reach it after a clean wait)."""
|
||||
def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label: str) -> None:
|
||||
"""terminate → wait(grace) → kill → wait(grace) escalation for a process we've given up on, so it
|
||||
is never waited on unboundedly. `terminate`/`kill` deliver the stop to whatever scope the caller
|
||||
owns — a single process (`proc.terminate`/`proc.kill`) or its whole group (`os.killpg`). No-op if
|
||||
the process already exited."""
|
||||
if proc.poll() is not None:
|
||||
return
|
||||
proc.terminate()
|
||||
terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
proc.wait(timeout=grace_s)
|
||||
return
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("ffmpeg pid %s survived SIGKILL", proc.pid)
|
||||
pass
|
||||
kill()
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid)
|
||||
|
||||
|
||||
def _stop(proc: subprocess.Popen) -> None:
|
||||
"""Terminate, then kill — never wait unbounded on a process we've given up on. Safe to call on
|
||||
a process that already exited."""
|
||||
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
|
||||
|
||||
|
||||
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
|
||||
|
||||
Reference in New Issue
Block a user