refactor(downloads): split worker.py parent/child into pipeline.py + jobdb.py (E3 S2, B-sub-9)
worker.py (~1090 L) mixed the PARENT supervisor (claim + supervise + the process
loop) with the CHILD pipeline (the yt-dlp/ffmpeg download+edit that runs in the
spawned subprocess). Extract them:
worker.py 1090 -> 475 parent: claim/supervise/loop
downloads/pipeline.py (new 592) the child pipeline (_download/_process_edit +
hooks/progress/ffmpeg helpers + _Aborted)
downloads/jobdb.py (new 73) the short-lived-session helpers both sides
share (_set_job/_job_status/_touch_job/
_requeue_if_running/_reset_asset_pending +
the staging-dir paths)
The graph is one-directional (runner -> pipeline -> jobdb; worker -> jobdb; no
worker<->pipeline edge) — the parent never calls the child directly, it spawns
runner.py as a subprocess. Consumers rewired: runner import, and two tests follow
the moved symbols (_media_info/_produced_file -> pipeline; the ffmpeg-callback
patches target pipeline where the names are looked up).
Pure code motion, verbatim bodies. ruff clean, full lane 363 pass (unchanged).
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||
|
||||
def _set_job(job_id: int, **fields) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
return
|
||||
for k, v in fields.items():
|
||||
setattr(job, k, v)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _job_status(job_id: int) -> str | None:
|
||||
with SessionLocal() as db:
|
||||
return db.execute(
|
||||
text("SELECT status FROM download_jobs WHERE id = :id"), {"id": job_id}
|
||||
).scalar()
|
||||
|
||||
|
||||
def _touch_job(job_id: int) -> None:
|
||||
"""Bump a running job's `updated_at` without changing any value — a stall-watchdog heartbeat for
|
||||
phases that legitimately write no progress. Extraction writes nothing, and a multi-client-set
|
||||
fallback re-writes identical values (progress 0 / phase 'downloading'), so ORM `onupdate` would
|
||||
NOT fire and two slow extraction attempts could otherwise cross the stall timeout and be killed
|
||||
mid-extraction though they'd have succeeded."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(
|
||||
text("UPDATE download_jobs SET updated_at=now() WHERE id=:id AND status='running'"),
|
||||
{"id": job_id},
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _requeue_if_running(job_id: int, phase: str) -> None:
|
||||
"""Put the job back to 'queued' for a later retry, but ONLY while it is still 'running'. The
|
||||
route flips status to paused/cancelled to stop a job; a blind `status='queued'` here would
|
||||
clobber that and silently revive a job the user just cancelled. The conditional WHERE makes the
|
||||
requeue a no-op unless we still own a running job."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE download_jobs SET status='queued', phase=:phase, updated_at=now() "
|
||||
"WHERE id=:id AND status='running'"
|
||||
),
|
||||
{"id": job_id, "phase": phase},
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _reset_asset_pending(asset_id: int) -> None:
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset and asset.status == "downloading":
|
||||
asset.status = "pending"
|
||||
db.commit()
|
||||
|
||||
|
||||
def _staging_dir(asset_id: int) -> Path:
|
||||
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
|
||||
|
||||
|
||||
def _edit_staging_dir(asset_id: int) -> Path:
|
||||
return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
|
||||
Reference in New Issue
Block a user