fix(downloads): an upload's conversion is guarded, counted, named and cleaned up

- Every ffmpeg/ffprobe over a library file names the demuxers it may use
  (edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
  demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
  failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
  kept after a failed conversion) — the footprint, the GC's cap and the admin
  view read one ON_DISK predicate — and a conversion that outgrows the cap is
  refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
  uploads shared a file); a converted file placed but not committed is removed,
  and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
  convert_failed / upload_missing, reports the "converting" phase, and notifies
  as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
2026-10-06 03:19:32 +02:00
parent 00886d92fe
commit 68cfc7e25d
15 changed files with 892 additions and 68 deletions
+163
View File
@@ -0,0 +1,163 @@
"""Every ffmpeg/ffprobe the downloads package runs over a library file names the demuxers it may use
(`edit.INPUT_GUARD`, a downloaded thumbnail's `edit.THUMB_GUARD`).
Without the list, ffmpeg picks a demuxer from the file's content and name, and both are the user's
for an upload (the remote site's for a thumbnail): a playlist-type demuxer opens whatever local paths
its entries name. These tests read the argv each builder hands to ffmpeg — the guard is a flag, so
the flag is what is held — and scan the package for an ffmpeg/ffprobe call written without one.
Not in scope: `app/plex`, whose ffmpeg reads the admin's own Plex library, not user-supplied bytes.
"""
import ast
import shutil
import subprocess
from pathlib import Path
import pytest
from app.downloads import edit, playable
DOWNLOADS = Path(__file__).resolve().parents[1] / "app" / "downloads"
# The demuxers that read paths named inside a file. `concat` is allowed only where the list is ours.
PLAYLIST_DEMUXERS = {"hls", "dash", "concat", "image2", "ffconcat", "tee"}
MEDIA = set(edit.MEDIA_DEMUXERS)
def _whitelists(cmd: list[str]) -> list[set[str] | None]:
"""For each `-i` in `cmd`, the demuxers the `-format_whitelist` given since the previous input
allows — None when that input has none."""
out, current = [], None
for i, tok in enumerate(cmd):
if tok == "-format_whitelist":
current = set(cmd[i + 1].split(","))
elif tok == "-i":
out.append(current)
current = None
return out
def _assert_media_guarded(cmd: list[str], extra: set[str] = frozenset()) -> None:
lists = _whitelists(cmd)
assert lists, cmd # a builder that opens no input is not what this test is about
for allowed in lists:
assert allowed is not None, f"an input with no -format_whitelist: {cmd}"
assert allowed <= MEDIA | extra, allowed - MEDIA - extra
assert not (allowed - extra) & PLAYLIST_DEMUXERS
def test_the_media_list_names_no_demuxer_that_reads_other_paths():
assert not MEDIA & PLAYLIST_DEMUXERS
assert edit.INPUT_GUARD == ["-format_whitelist", ",".join(edit.MEDIA_DEMUXERS)]
def test_the_phone_safe_conversion_is_guarded():
p = playable.Plan("mp4", True, True, True, True)
_assert_media_guarded(playable.build_cmd(Path("in.mov"), Path("out.mp4"), p))
@pytest.mark.parametrize(
"spec",
[
{"trim": {"start_s": 1, "end_s": 3}, "accurate": False},
{"trim": {"start_s": 1, "end_s": 3}, "accurate": True},
{"crop": {"x": 0, "y": 0, "w": 101, "h": 51}},
],
ids=["fast trim", "accurate trim", "crop"],
)
def test_a_clip_is_guarded(spec):
spec = edit.normalize_edit_spec(spec)
_assert_media_guarded(edit.build_edit_cmd(Path("in.mp4"), Path("out.mp4"), spec, "mp4"))
@pytest.mark.parametrize("accurate", [True, False], ids=["accurate join", "fast join"])
def test_a_join_is_guarded(accurate, tmp_path):
segs = [{"start_s": 0, "end_s": 1}, {"start_s": 2, "end_s": 3}]
spec = edit.normalize_edit_spec({"segments": segs, "accurate": accurate})
cmd, _ = edit.build_concat_plan(Path("in.mp4"), Path("out.mp4"), spec, "mp4", tmp_path)
# The fast join reads a concat list WE write; the whitelist rides down to the file it names.
_assert_media_guarded(cmd, extra=set() if accurate else {"concat"})
def test_the_poster_and_the_filmstrip_are_guarded():
_assert_media_guarded(edit.build_poster_cmd(Path("in.mp4"), Path("p.jpg"), 2.0))
geom = edit.storyboard_geometry(60)
_assert_media_guarded(edit.build_storyboard_cmd(Path("in.mp4"), Path("s.jpg"), 60, geom))
def test_a_downloaded_thumbnail_opens_as_a_single_image_only():
(allowed,) = _whitelists(edit.build_thumbnail_jpg_cmd(Path("t.webp"), Path("t.jpg")))
assert allowed == {"png_pipe", "jpeg_pipe", "webp_pipe"}
def test_every_ffprobe_is_guarded(monkeypatch):
seen = []
class _Done:
stdout = "{}"
def _run(args, **_k):
seen.append(args)
return _Done()
monkeypatch.setattr(edit.subprocess, "run", _run)
edit.run_ffprobe_json(Path("x.mp4"), show_streams=True)
edit.probe_duration(Path("x.mp4"))
assert len(seen) == 2
for args in seen:
i = args.index("-format_whitelist")
assert set(args[i + 1].split(",")) == MEDIA
def _unguarded_calls(source: str) -> list[int]:
"""Lines of a list literal that starts an ffmpeg/ffprobe command or hands one an input (`-i`)
and carries no `*…GUARD` / `*guard` splat and no "-format_whitelist" of its own. A builder that
adds its input in a later `+=` must guard that list — that is where the input is."""
bad = []
for node in ast.walk(ast.parse(source)):
if not isinstance(node, ast.List):
continue
consts = {e.value for e in node.elts if isinstance(e, ast.Constant)}
if not consts & {"-i", "ffprobe"}:
continue
guarded = "-format_whitelist" in consts or any(
isinstance(e, ast.Starred)
and isinstance(v := e.value, ast.Name | ast.Attribute)
and (v.id if isinstance(v, ast.Name) else v.attr).upper().endswith("GUARD")
for e in node.elts
)
if not guarded:
bad.append(node.lineno)
return bad
def test_no_ffmpeg_call_in_the_package_opens_an_input_unguarded():
found = {
f"{p.name}:{line}"
for p in sorted(DOWNLOADS.rglob("*.py"))
for line in _unguarded_calls(p.read_text(encoding="utf-8"))
}
assert found == set()
def test_the_scan_sees_an_unguarded_call():
# The scan above is green over nothing if it cannot see one: both shapes it must catch.
assert _unguarded_calls('cmd = ["ffmpeg", "-y", "-i", src]') == [1]
assert _unguarded_calls('a = ["ffprobe", "-v", "error", str(p)]') == [1]
assert _unguarded_calls('cmd = [*INPUT_GUARD, "-i", src]') == []
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
@pytest.mark.parametrize("ext", ["png", "webp", "jpg"])
def test_the_thumbnail_guard_still_opens_every_thumbnail_a_download_keeps(ext, tmp_path):
# `pipeline._THUMB_SUFFIXES`: what the guard must still let through (a .jpg that is really a
# PNG goes the ffmpeg route too — `thumbnail_to_jpg` copies only true JPEG bytes).
src = tmp_path / f"t.{ext}"
fmt = ["-f", "image2", "-c:v", "png"] if ext == "jpg" else []
subprocess.run(
["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=64x48",
"-frames:v", "1", *fmt, str(src)],
check=True,
) # fmt: skip
out = edit.thumbnail_to_jpg(src, tmp_path / "poster.jpg")
assert out is not None and edit._is_jpeg(out)
+73
View File
@@ -0,0 +1,73 @@
"""A libx264 re-encode of an odd-sized frame: 4:2:0 halves the chroma, so libx264 refuses an odd
width or height ("width not divisible by 2"). Measured: a source that is odd AND not 4:2:0 already
(a 4:4:4 or RGB screen capture, 1365x767) failed its conversion outright; a 4:2:0 source never
reaches the encoder odd (ffmpeg's crop rounds to its chroma grid), but the clip then stored the
size the user dragged, not the one the file has. Every re-encode now ends its filter chain on
`edit.EVEN_FRAME` (at most one pixel row or column), and a clip stores its probed frame.
Real ffmpeg on both paths — the refusal is the encoder's, so only the encoder can show it is gone.
"""
import shutil
import subprocess
from pathlib import Path
import pytest
from app.config import settings
from app.downloads import pipeline, playable, probe, service, storage
from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
def _ffmpeg(*args: str) -> None:
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", *args], check=True)
def _frame(path: Path) -> tuple[int | None, int | None]:
_, _, w, h, _ = probe.probe_media(path)
return w, h
def test_an_odd_sized_4_4_4_video_converts_to_the_even_frame_below_it(tmp_path):
src = tmp_path / "in.mkv"
_ffmpeg(
"-f", "lavfi", "-i", "testsrc=size=161x121:rate=10:duration=1",
"-c:v", "libx264", "-pix_fmt", "yuv444p", str(src),
) # fmt: skip
assert _frame(src) == (161, 121) # odd for real (`testsrc2` rounds its own size)
p = playable.plan(probe.probe_json(src), "mkv")
assert p is not None and p.reencode_video
dest = tmp_path / "out.mp4"
subprocess.run(playable.build_cmd(src, dest, p), check=True, capture_output=True)
assert _frame(dest) == (160, 120)
def test_an_odd_crop_makes_a_clip_and_stores_the_frame_it_has(db, make_user, tmp_path, monkeypatch):
monkeypatch.setattr(settings, "download_root", str(tmp_path))
src = tmp_path / "in.mp4"
_ffmpeg(
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2",
"-f", "lavfi", "-i", "sine=duration=2",
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", str(src),
) # fmt: skip
u = make_user()
source = uploadmod.ingest_upload(
db, u.id, filename="clip.mp4", src_path=src, size=src.stat().st_size
)
assert source.status == "done" # already phone-safe: the clip's source as it came
job = service.enqueue_edit(db, u.id, source, {"crop": {"x": 3, "y": 5, "w": 101, "h": 51}})
asset = db.get(MediaAsset, job.asset_id)
job.status, asset.status = "running", "downloading" # the worker's claim
db.commit()
pipeline._process_edit(job.id, asset.id)
db.expire_all()
job, asset = db.get(DownloadJob, job.id), db.get(MediaAsset, job.asset_id)
assert (job.status, job.error, asset.status) == ("done", None, "ready")
final = storage.abs_path(str(tmp_path), asset.rel_path)
assert _frame(final) == (100, 50)
assert (asset.width, asset.height) == (100, 50) # what the file is, not what was asked
+355
View File
@@ -0,0 +1,355 @@
"""An upload after ingest: what it costs its owner from the moment it lands, how the worker's
conversion names, checks and cleans up the file it makes, and what the owner and a share link see.
DB-backed. The conversion's ffmpeg step (`pipeline._ensure_phone_safe`) is stubbed to write a file of
a chosen size — these tests are about what happens around the encode; `test_upload.py` runs the real
one, and `test_playable.py` judges its bytes.
"""
import os
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
from app import sysconfig
from app.config import settings
from app.downloads import gc as gcmod
from app.downloads import pipeline, quota, storage
from app.downloads import probe as probemod
from app.downloads import upload as uploadmod
from app.downloads.stated import StatedFailure
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification
from sqlalchemy import select
@pytest.fixture
def _root(tmp_path, monkeypatch):
monkeypatch.setattr(settings, "download_root", str(tmp_path))
return tmp_path
def _probe(monkeypatch, vcodec="hevc"):
"""ffprobe's view of every file here: `vcodec` + AAC, 8-bit 4:2:0 — HEVC is queued for the
worker, H.264 lands as it came."""
data = {
"streams": [
{"codec_type": "video", "codec_name": vcodec, "pix_fmt": "yuv420p",
"width": 640, "height": 360},
{"codec_type": "audio", "codec_name": "aac"},
],
"format": {"format_name": "mov,mp4", "duration": "6"},
} # fmt: skip
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
def _ingest(db, root, user_id, name="clip.mov", size=10):
src = root / "incoming.bin"
src.write_bytes(b"o" * size)
job = uploadmod.ingest_upload(db, user_id, filename=name, src_path=src, size=size)
return job, db.get(MediaAsset, job.asset_id)
def _claim(db, job, asset):
"""What the worker's claim does before it converts."""
job.status, asset.status = "running", "downloading"
db.commit()
def _converts_to(monkeypatch, size, phases=None):
"""The encode writes a `size`-byte .mp4 into its staging dir (and notes the phase it reports)."""
def _encode(job_id, produced, out_dir, phase, total_s=None, **_k):
if phases is not None:
phases.append(phase)
out = Path(out_dir) / "converted.mp4"
out.write_bytes(b"c" * size)
return out
monkeypatch.setattr(pipeline, "_ensure_phone_safe", _encode)
def _cap(db, user_id, max_bytes):
db.add(
DownloadQuota(
user_id=user_id, max_bytes=max_bytes, max_concurrent=2, max_jobs=50, unlimited=False
)
)
db.commit()
def _uploads_dir(root, user_id) -> Path:
return root / ".uploads" / str(user_id)
# --- what it costs ---------------------------------------------------------------------------
def test_a_queued_upload_counts_against_its_owner_from_the_moment_it_lands(
db, make_user, _root, monkeypatch
):
_probe(monkeypatch)
u = make_user()
job, asset = _ingest(db, _root, u.id, size=10)
assert (job.status, asset.status) == ("queued", "pending")
assert quota.footprint(db, u.id) == 10 # its original is on disk already
def test_a_failed_conversion_still_counts_its_kept_original(db, make_user, _root, monkeypatch):
_probe(monkeypatch)
u = make_user()
job, asset = _ingest(db, _root, u.id, size=10)
job.status, asset.status = "error", "error" # kept for Retry
db.commit()
assert storage.abs_path(str(_root), asset.rel_path).exists()
assert quota.footprint(db, u.id) == 10
def test_a_conversion_that_outgrows_the_cap_is_refused_and_keeps_the_original(
db, make_user, _root, monkeypatch
):
_probe(monkeypatch)
u = make_user()
_cap(db, u.id, max_bytes=100)
job, asset = _ingest(db, _root, u.id, size=10)
original = storage.abs_path(str(_root), asset.rel_path)
_claim(db, job, asset)
_converts_to(monkeypatch, size=500)
pipeline._convert_upload(job.id, asset.id)
db.expire_all()
job = db.get(DownloadJob, job.id)
assert (job.status, job.error_code) == ("error", "quota_exceeded")
assert original.exists()
assert [p.name for p in _uploads_dir(_root, u.id).iterdir()] == [original.name]
def test_a_conversion_that_shrinks_passes_an_owner_at_the_cap(db, make_user, _root, monkeypatch):
_probe(monkeypatch)
u = make_user()
_cap(db, u.id, max_bytes=10)
job, asset = _ingest(db, _root, u.id, size=10)
_claim(db, job, asset)
_converts_to(monkeypatch, size=5)
pipeline._convert_upload(job.id, asset.id)
db.expire_all()
assert db.get(DownloadJob, job.id).status == "done"
assert db.get(MediaAsset, asset.id).size_bytes == 5
# --- the converted file's name and its leftovers -----------------------------------------------
def test_two_dotted_bare_names_keep_a_file_each(db, make_user, _root, monkeypatch):
# 'Holiday.2024.ä': its suffix is no extension (nothing shell-safe is left of it), so the stem's
# own dot once read as one — and the converted name lost `_a{id}`, the one thing keeping two apart.
_probe(monkeypatch)
u = make_user()
_converts_to(monkeypatch, size=5)
finals = []
for _ in range(2):
job, asset = _ingest(db, _root, u.id, name="Holiday.2024.ä")
_claim(db, job, asset)
pipeline._convert_upload(job.id, asset.id)
db.expire_all()
asset = db.get(MediaAsset, asset.id)
assert asset.status == "ready"
assert asset.rel_path.endswith(f"_a{asset.id}.mp4"), asset.rel_path
finals.append(asset.rel_path)
assert finals[0] != finals[1]
assert all(storage.abs_path(str(_root), r).exists() for r in finals)
def test_a_conversion_failing_after_it_placed_its_file_removes_it(
db, make_user, _root, monkeypatch
):
_probe(monkeypatch)
u = make_user()
job, asset = _ingest(db, _root, u.id)
original = storage.abs_path(str(_root), asset.rel_path)
_claim(db, job, asset)
_converts_to(monkeypatch, size=5)
def _probe_fails(_path):
raise RuntimeError("probe crashed")
monkeypatch.setattr(pipeline.probe, "probe_media", _probe_fails)
pipeline._convert_upload(job.id, asset.id)
db.expire_all()
assert db.get(DownloadJob, job.id).status == "error"
assert original.exists() # Retry starts from it
# The row still names the original: a converted file left beside it is one nothing would find.
assert [p.name for p in _uploads_dir(_root, u.id).iterdir()] == [original.name]
# --- what the owner is told ------------------------------------------------------------------
def test_a_failed_encode_is_a_failed_conversion_not_a_download_step(
db, make_user, _root, monkeypatch
):
_probe(monkeypatch)
job, asset = _ingest(db, _root, make_user().id)
_claim(db, job, asset)
def _ffmpeg_fails(*_a, **_k):
raise StatedFailure("ffmpeg failed (1): unsupported")
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _ffmpeg_fails)
pipeline._convert_upload(job.id, asset.id)
db.expire_all()
assert db.get(DownloadJob, job.id).error_code == "convert_failed"
def test_a_missing_original_says_so(db, make_user, _root, monkeypatch):
_probe(monkeypatch)
job, asset = _ingest(db, _root, make_user().id)
_claim(db, job, asset)
storage.abs_path(str(_root), asset.rel_path).unlink()
pipeline._convert_upload(job.id, asset.id)
db.expire_all()
assert db.get(DownloadJob, job.id).error_code == "upload_missing"
def test_the_conversion_reports_its_own_phase(db, make_user, _root, monkeypatch):
_probe(monkeypatch)
job, asset = _ingest(db, _root, make_user().id)
_claim(db, job, asset)
phases = []
_converts_to(monkeypatch, size=5, phases=phases)
pipeline._convert_upload(job.id, asset.id)
assert phases == ["converting"] # not yt-dlp's "processing": the queue row shows ffmpeg's %
def test_the_owner_is_told_their_upload_is_ready(db, make_user, _root, monkeypatch):
_probe(monkeypatch)
u = make_user()
job, asset = _ingest(db, _root, u.id)
_claim(db, job, asset)
_converts_to(monkeypatch, size=5)
pipeline._convert_upload(job.id, asset.id)
notes = db.execute(select(Notification).where(Notification.user_id == u.id)).scalars().all()
assert [n.data.get("kind") for n in notes] == ["upload"]
def test_an_extension_is_judged_as_it_is_stored(db, make_user, _root, monkeypatch):
# 'clip.mp4~' is stored as .mp4; judged by the raw 'mp4~' it queued a conversion of a file that
# was phone-safe already.
_probe(monkeypatch, vcodec="h264")
job, asset = _ingest(db, _root, make_user().id, name="clip.mp4~")
assert (job.status, asset.status, asset.container) == ("done", "ready", "mp4")
# --- the garbage collector ---------------------------------------------------------------------
def _aged(path: Path, hours: float = 5) -> None:
old = time.time() - hours * 3600
os.utime(path, (old, old))
def test_the_sweep_takes_a_stranded_upload_file_and_keeps_a_waiting_original(
db, make_user, _root, monkeypatch
):
_probe(monkeypatch)
u = make_user()
_job, asset = _ingest(db, _root, u.id)
original = storage.abs_path(str(_root), asset.rel_path)
stranded = original.with_name("left_by_a_stopped_conversion_a999.mp4")
stranded.write_bytes(b"x")
_aged(original)
_aged(stranded)
gcmod._sweep_orphan_files(db, str(_root))
assert original.exists() # queued for conversion: its row names it, ready or not
assert not stranded.exists() # no row names it
def test_the_cache_cap_never_evicts_an_upload(db, make_user, _root, monkeypatch):
_probe(monkeypatch, vcodec="h264")
u = make_user()
_job, upload = _ingest(db, _root, u.id, name="mine.mp4", size=10)
db.add(
MediaAsset(
source_kind="youtube", source_ref="vid1", format_sig="best", status="ready",
rel_path="Chan/clip.mp4", size_bytes=10, title="downloaded",
last_access_at=datetime.now(timezone.utc),
)
) # fmt: skip
# The upload is the least recently used: an LRU pass that may take it takes it first.
upload.last_access_at = datetime.now(timezone.utc) - timedelta(days=30)
db.commit()
(_root / "Chan").mkdir()
(_root / "Chan" / "clip.mp4").write_bytes(b"d" * 10)
sysconfig.set_value(db, "download_total_max_bytes", 15) # 20 on disk
db.commit()
gcmod.run_download_gc(db)
db.expire_all()
assert db.get(MediaAsset, upload.id).status == "ready"
assert storage.abs_path(str(_root), upload.rel_path).exists()
# --- a share link ----------------------------------------------------------------------------
@pytest.mark.parametrize("state", ["pending", "downloading"])
def test_a_link_whose_video_is_being_made_again_says_preparing_not_gone(
state, db, as_user, make_user, _root, monkeypatch, upload_headers
):
# A link is made on a finished job; its asset can be (re)made under it later — a shared cache row
# re-fetched, a conversion retried. Until it is ready the page says "try again shortly".
_probe(monkeypatch, vcodec="h264")
c = as_user(make_user())
job = c.post(
"/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
).json()
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
assert c.get(f"/api/public/watch/{link['token']}").status_code == 200
db.get(MediaAsset, job["asset"]["id"]).status = state
db.commit()
r = c.get(f"/api/public/watch/{link['token']}")
assert r.status_code == 404
assert r.json()["detail"]["reason"] == "video_preparing"
def test_the_admin_storage_view_counts_a_waiting_upload(db, as_user, make_user, _root, monkeypatch):
# The same disk the owner's footprint and the GC's cap count (`quota.ON_DISK`).
_probe(monkeypatch)
u = make_user()
_ingest(db, _root, u.id, size=10)
r = as_user(make_user(role="admin")).get("/api/admin/downloads/storage").json()
assert (r["ready_files"], r["total_bytes"]) == (1, 10)
assert {p["user_id"]: p["footprint_bytes"] for p in r["per_user"]} == {u.id: 10}
@pytest.mark.parametrize(
("job_status", "asset_status", "ref_count"),
[
("done", "ready", 1), # a finished upload its owner keeps
("queued", "pending", 1), # one waiting for the worker
("canceled", "downloading", 0), # cancelled mid-conversion: the worker still reads it
],
)
def test_the_upload_reap_leaves_one_still_held_or_still_read(
job_status, asset_status, ref_count, db, make_user, _root, monkeypatch
):
_probe(monkeypatch)
job, asset = _ingest(db, _root, make_user().id)
path = storage.abs_path(str(_root), asset.rel_path)
job.status, asset.status, asset.ref_count = job_status, asset_status, ref_count
db.commit()
gcmod.run_download_gc(db)
db.expire_all()
assert db.get(MediaAsset, asset.id) is not None
assert path.exists()
+74
View File
@@ -0,0 +1,74 @@
"""The worker child's dispatch (`runner._run`) and the edit path's last step before it stores a clip.
`runner._run` picks the pipeline entry by the job's kind; the tests elsewhere call those entries
directly, so a kind routed to the wrong one (an upload sent to yt-dlp) would pass all of them.
"""
from pathlib import Path
import pytest
from app.config import settings
from app.downloads import pipeline, runner, service, storage
from app.downloads import probe as probemod
from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset
@pytest.mark.parametrize(
("kind", "entry"),
[("upload", "_convert_upload"), ("edit", "_process_edit"), ("download", "_download")],
)
def test_each_job_kind_reaches_its_own_pipeline_entry(kind, entry, monkeypatch):
called = []
for name in ("_convert_upload", "_process_edit", "_download"):
monkeypatch.setattr(pipeline, name, lambda *a, _n=name: called.append(_n))
runner._run(5, 7, kind, "url", "https://example.org/v", {}, 1)
assert called == [entry]
def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension(
db, make_user, tmp_path, monkeypatch
):
# A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an
# audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file.
monkeypatch.setattr(settings, "download_root", str(tmp_path))
data = {
"streams": [{"codec_type": "audio", "codec_name": "aac"}],
"format": {"format_name": "mov,mp4", "duration": "10"},
}
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
src = tmp_path / "in.bin"
src.write_bytes(b"a" * 10)
u = make_user()
source = uploadmod.ingest_upload(db, u.id, filename="song.m4a", src_path=src, size=10)
# Stored as an .mp4, the way an audio file was kept before every ready file met the rule.
kept = db.get(MediaAsset, source.asset_id)
old = storage.abs_path(str(tmp_path), kept.rel_path)
kept.rel_path, kept.container = kept.rel_path[: -len(".m4a")] + ".mp4", "mp4"
old.rename(storage.abs_path(str(tmp_path), kept.rel_path))
db.commit()
job = service.enqueue_edit(
db, u.id, source, {"trim": {"start_s": 1, "end_s": 3}, "accurate": False}
)
asset = db.get(MediaAsset, job.asset_id)
job.status, asset.status = "running", "downloading"
db.commit()
def _trim(cmd, *_a, **_k):
Path(cmd[-1]).write_bytes(b"t" * 4) # the stream-copied clip, at the path ffmpeg was given
def _remux(job_id, produced, out_dir, phase, total_s=None, **_k):
out = Path(out_dir) / "remuxed.m4a"
out.write_bytes(b"r" * 3)
return out
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _trim)
monkeypatch.setattr(pipeline, "_ensure_phone_safe", _remux)
pipeline._process_edit(job.id, asset.id)
db.expire_all()
asset = db.get(MediaAsset, asset.id)
assert (db.get(DownloadJob, job.id).status, asset.container) == ("done", "m4a")
assert asset.rel_path.endswith(".m4a")
assert storage.abs_path(str(tmp_path), asset.rel_path).read_bytes() == b"rrr"