fix(downloads): an upload's conversion is guarded, counted, named and cleaned up

- Every ffmpeg/ffprobe over a library file names the demuxers it may use
  (edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
  demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
  failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
  kept after a failed conversion) — the footprint, the GC's cap and the admin
  view read one ON_DISK predicate — and a conversion that outgrows the cap is
  refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
  uploads shared a file); a converted file placed but not committed is removed,
  and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
  convert_failed / upload_missing, reports the "converting" phase, and notifies
  as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
2026-10-06 03:19:32 +02:00
parent 00886d92fe
commit 68cfc7e25d
15 changed files with 892 additions and 68 deletions
+74
View File
@@ -0,0 +1,74 @@
"""The worker child's dispatch (`runner._run`) and the edit path's last step before it stores a clip.
`runner._run` picks the pipeline entry by the job's kind; the tests elsewhere call those entries
directly, so a kind routed to the wrong one (an upload sent to yt-dlp) would pass all of them.
"""
from pathlib import Path
import pytest
from app.config import settings
from app.downloads import pipeline, runner, service, storage
from app.downloads import probe as probemod
from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset
@pytest.mark.parametrize(
("kind", "entry"),
[("upload", "_convert_upload"), ("edit", "_process_edit"), ("download", "_download")],
)
def test_each_job_kind_reaches_its_own_pipeline_entry(kind, entry, monkeypatch):
called = []
for name in ("_convert_upload", "_process_edit", "_download"):
monkeypatch.setattr(pipeline, name, lambda *a, _n=name: called.append(_n))
runner._run(5, 7, kind, "url", "https://example.org/v", {}, 1)
assert called == [entry]
def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension(
db, make_user, tmp_path, monkeypatch
):
# A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an
# audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file.
monkeypatch.setattr(settings, "download_root", str(tmp_path))
data = {
"streams": [{"codec_type": "audio", "codec_name": "aac"}],
"format": {"format_name": "mov,mp4", "duration": "10"},
}
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
src = tmp_path / "in.bin"
src.write_bytes(b"a" * 10)
u = make_user()
source = uploadmod.ingest_upload(db, u.id, filename="song.m4a", src_path=src, size=10)
# Stored as an .mp4, the way an audio file was kept before every ready file met the rule.
kept = db.get(MediaAsset, source.asset_id)
old = storage.abs_path(str(tmp_path), kept.rel_path)
kept.rel_path, kept.container = kept.rel_path[: -len(".m4a")] + ".mp4", "mp4"
old.rename(storage.abs_path(str(tmp_path), kept.rel_path))
db.commit()
job = service.enqueue_edit(
db, u.id, source, {"trim": {"start_s": 1, "end_s": 3}, "accurate": False}
)
asset = db.get(MediaAsset, job.asset_id)
job.status, asset.status = "running", "downloading"
db.commit()
def _trim(cmd, *_a, **_k):
Path(cmd[-1]).write_bytes(b"t" * 4) # the stream-copied clip, at the path ffmpeg was given
def _remux(job_id, produced, out_dir, phase, total_s=None, **_k):
out = Path(out_dir) / "remuxed.m4a"
out.write_bytes(b"r" * 3)
return out
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _trim)
monkeypatch.setattr(pipeline, "_ensure_phone_safe", _remux)
pipeline._process_edit(job.id, asset.id)
db.expire_all()
asset = db.get(MediaAsset, asset.id)
assert (db.get(DownloadJob, job.id).status, asset.container) == ("done", "m4a")
assert asset.rel_path.endswith(".m4a")
assert storage.abs_path(str(tmp_path), asset.rel_path).read_bytes() == b"rrr"