peter
815083bcc4
- The queue keeps a done job by when it became done (ready_at), not by its last write: renaming, keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished. - GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a second release that frees a file another user still holds. - A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit whose parent expired still goes back to its folder, not to Uploads. - A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes that file home too, instead of failing on the foreign key. - The hash route reads at most two files at once (429 past that) and holds no connection meanwhile. - 0104's backfill dates old files by their request date only: no notice prod wrote names its job. - The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up. - Stale comments about the /downloads mount and the Downloads page.
125 lines
4.5 KiB
Python
125 lines
4.5 KiB
Python
"""The worker obeys the two admin knobs that concern it — `download_worker_concurrency` and
|
|
`log_level` — as stored in the settings page, not only the env it booted with. It used to read
|
|
`settings.download_worker_concurrency` and `settings.log_level` once at start, so the admin page
|
|
showed values the worker never applied. No DB or real threads of work here: the stored values come
|
|
from a patched `sysconfig`, and the loop threads are stand-ins that only record their index."""
|
|
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from app import worker
|
|
|
|
|
|
class _NoSession:
|
|
def __enter__(self):
|
|
return None
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
|
|
@pytest.fixture
|
|
def stored(monkeypatch):
|
|
"""The settings page's values, served to the worker instead of a database."""
|
|
values = {"download_worker_concurrency": 3, "log_level": "DEBUG"}
|
|
monkeypatch.setattr(worker, "SessionLocal", _NoSession)
|
|
monkeypatch.setattr(worker.sysconfig, "get_int", lambda _db, key: int(values[key]))
|
|
monkeypatch.setattr(worker.sysconfig, "get_str", lambda _db, key: str(values[key]))
|
|
return values
|
|
|
|
|
|
@pytest.fixture
|
|
def levels(monkeypatch):
|
|
applied: list[str] = []
|
|
monkeypatch.setattr(worker.logs, "set_level", applied.append)
|
|
return applied
|
|
|
|
|
|
@pytest.fixture
|
|
def loops(monkeypatch):
|
|
"""Stand-in loop threads: each records its index and parks until the worker stops."""
|
|
started: list[int] = []
|
|
lock = threading.Lock()
|
|
|
|
def fake_loop(idx: int) -> None:
|
|
with lock:
|
|
started.append(idx)
|
|
worker._stop.wait()
|
|
|
|
monkeypatch.setattr(worker, "_worker_loop", fake_loop)
|
|
yield started
|
|
worker._stop.set()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_stop():
|
|
worker._stop.clear()
|
|
yield
|
|
worker._stop.clear()
|
|
|
|
|
|
def test_worker_boots_with_the_stored_concurrency_and_log_level(monkeypatch, stored, levels, loops):
|
|
monkeypatch.setattr(worker.settings, "download_worker_concurrency", 2)
|
|
monkeypatch.setattr(worker.settings, "worker_enabled", True)
|
|
monkeypatch.setattr(worker.logs, "configure_logging", lambda _source: None)
|
|
monkeypatch.setattr(worker.signal, "signal", lambda *_a: None)
|
|
monkeypatch.setattr(worker, "_wait_for_schema", lambda: True)
|
|
monkeypatch.setattr(worker.logsink, "start", lambda: None)
|
|
monkeypatch.setattr(worker, "_recover_orphans", lambda: None)
|
|
monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: None)
|
|
|
|
threading.Timer(0.5, worker._stop.set).start()
|
|
worker.main()
|
|
|
|
assert sorted(loops) == [0, 1, 2] # the stored 3, not the env's 2
|
|
assert levels and levels[-1] == "DEBUG"
|
|
|
|
|
|
def test_a_changed_knob_applies_without_a_restart(stored, levels, loops):
|
|
threads: dict[int, threading.Thread] = {}
|
|
level = worker._sync_knobs(threads, None)
|
|
assert level == "DEBUG"
|
|
assert worker._target == 3
|
|
|
|
stored["download_worker_concurrency"] = 5
|
|
stored["log_level"] = "WARNING"
|
|
level = worker._sync_knobs(threads, level)
|
|
|
|
assert worker._target == 5
|
|
assert sorted(threads) == [0, 1, 2, 3, 4]
|
|
assert level == "WARNING" and levels[-1] == "WARNING"
|
|
|
|
# Lowering it retires the surplus threads after their current job: they see the lower target.
|
|
stored["download_worker_concurrency"] = 1
|
|
worker._sync_knobs(threads, level)
|
|
assert worker._target == 1
|
|
|
|
|
|
def test_an_unreadable_database_keeps_what_runs(monkeypatch, levels, loops):
|
|
def boom():
|
|
raise RuntimeError("db down")
|
|
|
|
monkeypatch.setattr(worker, "SessionLocal", boom)
|
|
monkeypatch.setattr(worker, "_target", 2)
|
|
threads: dict[int, threading.Thread] = {}
|
|
|
|
assert worker._sync_knobs(threads, "INFO") == "INFO"
|
|
assert worker._target == 2
|
|
assert sorted(threads) == [0, 1] # the loops still run on the boot value
|
|
assert levels == []
|
|
|
|
|
|
def test_the_worker_will_not_start_on_a_files_root_that_is_not_the_volume(monkeypatch, loops):
|
|
# Files downloaded into the container's own empty /files go with the container: refuse, loudly.
|
|
monkeypatch.setattr(worker.settings, "worker_enabled", True)
|
|
monkeypatch.setattr(worker.logs, "configure_logging", lambda _source: None)
|
|
monkeypatch.setattr(worker.signal, "signal", lambda *_a: None)
|
|
monkeypatch.setattr(worker, "_wait_for_schema", lambda: True)
|
|
monkeypatch.setattr(worker.logsink, "start", lambda: None)
|
|
monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: "not a mounted volume")
|
|
with pytest.raises(SystemExit) as exit_:
|
|
worker.main()
|
|
assert exit_.value.code == 1
|
|
assert loops == []
|