peter
9ae6c47d6c
The download/edit child (yt-dlp + ffmpeg) shares one LAN box with the API, Postgres, and Plex playback. A CPU re-encode pegged every core for ~30s and starved web requests + playback until it finished. Full-scope S4 mitigations: - Child self-de-prioritizes at startup (inherited by every ffmpeg/yt-dlp descendant): CPU nice increment + best-effort idle ionice class + an OMP_NUM_THREADS budget, all config-gated and independently disable-able. - Cap the browser-safe re-encode's encoder threads (`-threads`) so a transcode reserves a core for foreground work instead of claiming all of them. - Throttle the retention GC's whole-tree orphan sweep (yield IO after each batch of files) so its rglob can't monopolize disk IO in the API process. - Idle-back-off the Download Center (2s→15s when nothing queued/running) and UsageBar (2s→15s when running_jobs is 0) polls; both snap back to 2s on the next fetch after an enqueue (invalidateDownloads refetches immediately). Tests: encode_thread_cap (pin/reserve-core/edge), -threads flag presence, _deprioritize (nice/ionice/OMP branches), GC sweep throttle batching.
109 lines
3.8 KiB
Python
109 lines
3.8 KiB
Python
"""Orphan-file sweep: remove on-disk cruft no ready asset owns (old accented dirs, stale posters),
|
|
while never touching system trees, owned files, or freshly-written downloads."""
|
|
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from app.config import settings
|
|
from app.downloads import gc as gcmod
|
|
from app.downloads.gc import _sweep_orphan_files
|
|
|
|
|
|
class _Scalars:
|
|
def __init__(self, rows):
|
|
self._rows = rows
|
|
|
|
def scalars(self):
|
|
return list(self._rows)
|
|
|
|
|
|
class _StubDB:
|
|
"""Minimal stand-in: `_sweep_orphan_files` only issues one select for ready rel_paths."""
|
|
|
|
def __init__(self, rel_paths):
|
|
self._rel_paths = rel_paths
|
|
|
|
def execute(self, _stmt):
|
|
return _Scalars(self._rel_paths)
|
|
|
|
|
|
def _aged(path: Path, hours: float) -> None:
|
|
old = time.time() - hours * 3600
|
|
os.utime(path, (old, old))
|
|
|
|
|
|
def test_sweep_removes_orphans_but_keeps_owned_system_and_fresh(tmp_path: Path):
|
|
root = tmp_path
|
|
# An owned asset (in the keep set) + its sidecars.
|
|
kept_dir = root / "Chan" / "Season_2026"
|
|
kept_dir.mkdir(parents=True)
|
|
kept_media = kept_dir / "Chan_-_2026-01-01_-_Title_vid_a1.mp4"
|
|
for p in (kept_media, kept_media.with_suffix(".nfo"), kept_media.with_suffix(".jpg")):
|
|
p.write_bytes(b"x")
|
|
_aged(p, 5) # old, but owned → must survive
|
|
|
|
# An orphan poster in a stale (e.g. old accented) channel dir — old, unowned → swept.
|
|
orphan_dir = root / "OldChan" / "Season_2026"
|
|
orphan_dir.mkdir(parents=True)
|
|
orphan = orphan_dir / "OldChan_-_old_title_vid.jpg"
|
|
orphan.write_bytes(b"x")
|
|
_aged(orphan, 5)
|
|
|
|
# A system tree (storyboards) — never touched, even when old.
|
|
sysfile = root / ".storyboards" / "asset-1.jpg"
|
|
sysfile.parent.mkdir()
|
|
sysfile.write_bytes(b"x")
|
|
_aged(sysfile, 5)
|
|
|
|
# A brand-new unowned file (an in-flight download's just-placed output) — too young to sweep.
|
|
fresh_dir = root / "FreshChan" / "Season_2026"
|
|
fresh_dir.mkdir(parents=True)
|
|
fresh = fresh_dir / "FreshChan_-_now_vid_a2.mp4"
|
|
fresh.write_bytes(b"x") # mtime = now
|
|
|
|
db = _StubDB(["Chan/Season_2026/Chan_-_2026-01-01_-_Title_vid_a1.mp4"])
|
|
removed = _sweep_orphan_files(db, str(root))
|
|
|
|
assert removed == 1 # only the aged orphan
|
|
assert kept_media.exists() and kept_media.with_suffix(".jpg").exists()
|
|
assert not orphan.exists() and not orphan_dir.exists() # orphan + its emptied dirs pruned
|
|
assert not (root / "OldChan").exists()
|
|
assert sysfile.exists() # system tree untouched
|
|
assert fresh.exists() # too new to sweep
|
|
|
|
|
|
def test_sweep_throttles_after_each_batch(tmp_path: Path, monkeypatch):
|
|
"""The whole-tree walk yields the IO channel after every `batch` files so it can't monopolize
|
|
disk IO in the API process. batch=2 over 5 touched files → 2 pauses (at file 2 and 4)."""
|
|
root = tmp_path
|
|
d = root / "Chan"
|
|
d.mkdir()
|
|
for i in range(5):
|
|
(d / f"orphan_{i}.dat").write_bytes(b"x")
|
|
for f in d.iterdir():
|
|
_aged(f, 5) # old enough that every file is "touched" by the walk
|
|
|
|
monkeypatch.setattr(settings, "download_gc_sweep_pause_ms", 10)
|
|
monkeypatch.setattr(settings, "download_gc_sweep_batch", 2)
|
|
pauses = []
|
|
monkeypatch.setattr(gcmod.time, "sleep", lambda s: pauses.append(s))
|
|
|
|
_sweep_orphan_files(_StubDB([]), str(root))
|
|
|
|
assert pauses == [0.01, 0.01] # 5 files, batch 2 → yield at #2 and #4
|
|
|
|
|
|
def test_sweep_no_throttle_when_pause_disabled(tmp_path: Path, monkeypatch):
|
|
(tmp_path / "Chan").mkdir()
|
|
f = tmp_path / "Chan" / "orphan.dat"
|
|
f.write_bytes(b"x")
|
|
_aged(f, 5)
|
|
monkeypatch.setattr(settings, "download_gc_sweep_pause_ms", 0)
|
|
pauses = []
|
|
monkeypatch.setattr(gcmod.time, "sleep", lambda s: pauses.append(s))
|
|
|
|
_sweep_orphan_files(_StubDB([]), str(tmp_path))
|
|
|
|
assert pauses == []
|