fix(worker): obey the stored concurrency and log level, live
The admin page's download_worker_concurrency and log_level were stored but never reached the worker: it read the env once at boot (worker.py) and never looked again. The worker now re-reads both every 10 s (_sync_knobs): extra loop threads start at once, surplus ones retire after their current job, and a changed level is applied to the process. An unreadable database keeps what already runs.
This commit is contained in:
+59
-10
@@ -5,7 +5,8 @@ from the API so long yt-dlp downloads + ffmpeg postprocessing never block web re
|
||||
|
||||
Design:
|
||||
* N loop threads (download_worker_concurrency) each claim a `queued` DownloadJob with
|
||||
FOR UPDATE SKIP LOCKED and process it.
|
||||
FOR UPDATE SKIP LOCKED and process it. N and the log level follow the settings page while the
|
||||
worker runs (`_sync_knobs`), not only the env it booted with.
|
||||
* The shared MediaAsset is the dedup unit. A job whose asset is already `ready` finishes
|
||||
instantly (no re-download); a fresh asset is claimed pending→downloading by exactly one
|
||||
worker (a DB compare-and-set), so two jobs for the same video+format never double-download.
|
||||
@@ -48,12 +49,60 @@ from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_stop = threading.Event()
|
||||
# How many loop threads may claim jobs. A thread whose index is at or above it stops after its
|
||||
# current job, so lowering the knob never abandons a download halfway.
|
||||
_target = 1
|
||||
# How often the worker re-reads the knobs it obeys; the settings page says nothing about a restart.
|
||||
_KNOB_SECONDS = 10.0
|
||||
|
||||
|
||||
def _handle_signal(signum, frame):
|
||||
_stop.set()
|
||||
|
||||
|
||||
def _read_knobs() -> tuple[int, str] | None:
|
||||
"""The stored concurrency and log level, or None when the database cannot answer — the caller
|
||||
then keeps what already runs."""
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
return (
|
||||
max(1, sysconfig.get_int(db, "download_worker_concurrency")),
|
||||
sysconfig.get_str(db, "log_level"),
|
||||
)
|
||||
except Exception: # noqa: BLE001 — any DB or config failure keeps the knobs already in force
|
||||
log.warning(
|
||||
"could not read the worker's settings; keeping the ones in force",
|
||||
exc_info=True,
|
||||
extra=logs.event("download.worker.knobs_unreadable"),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _sync_knobs(threads: dict[int, threading.Thread], level: str | None) -> str | None:
|
||||
"""Apply the stored knobs: run loop threads up to the concurrency (surplus ones retire after
|
||||
their job, see `_target`) and re-apply the log level when it changed. Returns the level in force."""
|
||||
global _target
|
||||
knobs = _read_knobs()
|
||||
if knobs is not None:
|
||||
target, stored_level = knobs
|
||||
if target != _target:
|
||||
log.info(
|
||||
"download concurrency is now %d",
|
||||
target,
|
||||
extra=logs.event("download.worker.concurrency", before=_target, after=target),
|
||||
)
|
||||
_target = target
|
||||
if stored_level != level:
|
||||
logs.set_level(stored_level)
|
||||
level = stored_level
|
||||
for i in range(_target):
|
||||
t = threads.get(i)
|
||||
if t is None or not t.is_alive():
|
||||
threads[i] = threading.Thread(target=_worker_loop, args=(i,), daemon=True)
|
||||
threads[i].start()
|
||||
return level
|
||||
|
||||
|
||||
# --- claim + recovery -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -576,7 +625,7 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
||||
|
||||
|
||||
def _worker_loop(idx: int) -> None:
|
||||
while not _stop.is_set():
|
||||
while not _stop.is_set() and idx < _target:
|
||||
try:
|
||||
job_id = _claim_job()
|
||||
except Exception:
|
||||
@@ -634,15 +683,15 @@ def main() -> None:
|
||||
# has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks.
|
||||
logsink.start()
|
||||
_recover_orphans()
|
||||
n = max(1, settings.download_worker_concurrency)
|
||||
log.info("Download worker started (concurrency=%d, root=%s).", n, settings.download_root)
|
||||
threads = [threading.Thread(target=_worker_loop, args=(i,), daemon=True) for i in range(n)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
while not _stop.is_set():
|
||||
_stop.wait(1.0)
|
||||
global _target
|
||||
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
|
||||
threads: dict[int, threading.Thread] = {}
|
||||
level = _sync_knobs(threads, None)
|
||||
log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.download_root)
|
||||
while not _stop.wait(_KNOB_SECONDS):
|
||||
level = _sync_knobs(threads, level)
|
||||
log.info("Download worker stopping…")
|
||||
for t in threads:
|
||||
for t in threads.values():
|
||||
t.join(timeout=5)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user