peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(downloads): split worker.py parent/child into pipeline.py + jobdb.py (E3 S2, B-sub-9)

Browse Source 
worker.py (~1090 L) mixed the PARENT supervisor (claim + supervise + the process
loop) with the CHILD pipeline (the yt-dlp/ffmpeg download+edit that runs in the
spawned subprocess). Extract them:

  worker.py  1090 -> 475   parent: claim/supervise/loop
  downloads/pipeline.py  (new 592)  the child pipeline (_download/_process_edit +
                                    hooks/progress/ffmpeg helpers + _Aborted)
  downloads/jobdb.py     (new  73)  the short-lived-session helpers both sides
                                    share (_set_job/_job_status/_touch_job/
                                    _requeue_if_running/_reset_asset_pending +
                                    the staging-dir paths)

The graph is one-directional (runner -> pipeline -> jobdb; worker -> jobdb; no
worker<->pipeline edge) — the parent never calls the child directly, it spawns
runner.py as a subprocess. Consumers rewired: runner import, and two tests follow
the moved symbols (_media_info/_produced_file -> pipeline; the ffmpeg-callback
patches target pipeline where the names are looked up).

Pure code motion, verbatim bodies. ruff clean, full lane 363 pass (unchanged).
This commit is contained in:
peter
2026-08-18 21:58:23 +02:00
parent 0fab0ce779
commit b2013a079a
6 changed files with 696 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_ffmpeg_callbacks.py
+19 -19
View File
@@ -4,20 +4,20 @@ run_ffmpeg calls `on_progress`/`should_cancel` once per ffmpeg output LINE (`-pr
emits a block of ~12 every half second), and both callbacks are database round-trips: `_set_job`
opens a session and UPDATE+COMMITs, `_job_status` opens one and SELECTs. Unthrottled, a long
re-encode is ~10^5 of each."""
from app import worker
import app.downloads.pipeline as pipeline
class TestProgressWriter:
def test_writes_the_first_value_immediately(self, monkeypatch):
writes = []
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
worker._ffmpeg_progress_writer(7, "processing")(12.4)
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **f: writes.append(f))
pipeline._ffmpeg_progress_writer(7, "processing")(12.4)
assert writes == [{"progress": 12, "phase": "processing"}]
def test_a_burst_of_lines_costs_one_write(self, monkeypatch):
writes = []
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
write = worker._ffmpeg_progress_writer(7, "processing")
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **f: writes.append(f))
write = pipeline._ffmpeg_progress_writer(7, "processing")
for pct in range(0, 40): # what one second of ffmpeg output looks like
write(pct)
assert len(writes) == 1
@@ -25,8 +25,8 @@ class TestProgressWriter:
def test_the_next_second_writes_again(self, monkeypatch):
writes = []
clock = {"t": 1000.0}
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
write = worker._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **f: writes.append(f))
write = pipeline._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
write(10)
write(11) # same second → dropped
clock["t"] += 1.5
@@ -36,9 +36,9 @@ class TestProgressWriter:
def test_an_unchanged_whole_percent_is_never_rewritten(self, monkeypatch):
writes = []
clock = {"t": 1000.0}
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
monkeypatch.setattr(worker, "_touch_job", lambda job_id: None)
write = worker._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **f: writes.append(f))
monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None)
write = pipeline._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
write(10.1)
clock["t"] += 5
write(10.9) # still 10% — the bar shows integers
@@ -50,9 +50,9 @@ class TestProgressWriter:
progress, so the stall watchdog keeps seeing a fresh updated_at."""
writes, touches = [], []
clock = {"t": 1000.0}
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
monkeypatch.setattr(worker, "_touch_job", lambda job_id: touches.append(job_id))
write = worker._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **f: writes.append(f))
monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: touches.append(job_id))
write = pipeline._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
write(10.1) # first real write → progress 10
for _ in range(3): # three more intervals stuck on 10%
clock["t"] += 1.5
@@ -64,8 +64,8 @@ class TestProgressWriter:
class TestCancelPoll:
def test_asks_once_then_reuses_the_answer(self, monkeypatch):
calls = []
monkeypatch.setattr(worker, "_job_status", lambda job_id: calls.append(job_id) or "running")
cancelled = worker._cancel_poll(7)
monkeypatch.setattr(pipeline, "_job_status", lambda job_id: calls.append(job_id) or "running")
cancelled = pipeline._cancel_poll(7)
for _ in range(40):
assert cancelled() is False
assert len(calls) == 1
@@ -78,8 +78,8 @@ class TestCancelPoll:
calls.append(job_id)
return "cancelled"
monkeypatch.setattr(worker, "_job_status", status)
cancelled = worker._cancel_poll(7, now=lambda: clock["t"])
monkeypatch.setattr(pipeline, "_job_status", status)
cancelled = pipeline._cancel_poll(7, now=lambda: clock["t"])
assert cancelled() is True
clock["t"] += 60 # cancellation is sticky: no re-query, and no un-cancelling
assert cancelled() is True
@@ -88,8 +88,8 @@ class TestCancelPoll:
def test_a_cancel_arriving_later_is_seen_on_the_next_tick(self, monkeypatch):
clock = {"t": 1000.0}
state = {"status": "running"}
monkeypatch.setattr(worker, "_job_status", lambda job_id: state["status"])
cancelled = worker._cancel_poll(7, now=lambda: clock["t"])
monkeypatch.setattr(pipeline, "_job_status", lambda job_id: state["status"])
cancelled = pipeline._cancel_poll(7, now=lambda: clock["t"])
assert cancelled() is False
state["status"] = "paused"
clock["t"] += 1.0 # the throttle must not delay a cancel by more than its interval