fix(downloads): an upload's conversion is guarded, counted, named and cleaned up
- Every ffmpeg/ffprobe over a library file names the demuxers it may use
(edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
kept after a failed conversion) — the footprint, the GC's cap and the admin
view read one ON_DISK predicate — and a conversion that outgrows the cap is
refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
uploads shared a file); a converted file placed but not committed is removed,
and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
convert_failed / upload_missing, reports the "converting" phase, and notifies
as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
"""Every ffmpeg/ffprobe the downloads package runs over a library file names the demuxers it may use
|
||||
(`edit.INPUT_GUARD`, a downloaded thumbnail's `edit.THUMB_GUARD`).
|
||||
|
||||
Without the list, ffmpeg picks a demuxer from the file's content and name, and both are the user's
|
||||
for an upload (the remote site's for a thumbnail): a playlist-type demuxer opens whatever local paths
|
||||
its entries name. These tests read the argv each builder hands to ffmpeg — the guard is a flag, so
|
||||
the flag is what is held — and scan the package for an ffmpeg/ffprobe call written without one.
|
||||
|
||||
Not in scope: `app/plex`, whose ffmpeg reads the admin's own Plex library, not user-supplied bytes.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.downloads import edit, playable
|
||||
|
||||
DOWNLOADS = Path(__file__).resolve().parents[1] / "app" / "downloads"
|
||||
# The demuxers that read paths named inside a file. `concat` is allowed only where the list is ours.
|
||||
PLAYLIST_DEMUXERS = {"hls", "dash", "concat", "image2", "ffconcat", "tee"}
|
||||
MEDIA = set(edit.MEDIA_DEMUXERS)
|
||||
|
||||
|
||||
def _whitelists(cmd: list[str]) -> list[set[str] | None]:
|
||||
"""For each `-i` in `cmd`, the demuxers the `-format_whitelist` given since the previous input
|
||||
allows — None when that input has none."""
|
||||
out, current = [], None
|
||||
for i, tok in enumerate(cmd):
|
||||
if tok == "-format_whitelist":
|
||||
current = set(cmd[i + 1].split(","))
|
||||
elif tok == "-i":
|
||||
out.append(current)
|
||||
current = None
|
||||
return out
|
||||
|
||||
|
||||
def _assert_media_guarded(cmd: list[str], extra: set[str] = frozenset()) -> None:
|
||||
lists = _whitelists(cmd)
|
||||
assert lists, cmd # a builder that opens no input is not what this test is about
|
||||
for allowed in lists:
|
||||
assert allowed is not None, f"an input with no -format_whitelist: {cmd}"
|
||||
assert allowed <= MEDIA | extra, allowed - MEDIA - extra
|
||||
assert not (allowed - extra) & PLAYLIST_DEMUXERS
|
||||
|
||||
|
||||
def test_the_media_list_names_no_demuxer_that_reads_other_paths():
|
||||
assert not MEDIA & PLAYLIST_DEMUXERS
|
||||
assert edit.INPUT_GUARD == ["-format_whitelist", ",".join(edit.MEDIA_DEMUXERS)]
|
||||
|
||||
|
||||
def test_the_phone_safe_conversion_is_guarded():
|
||||
p = playable.Plan("mp4", True, True, True, True)
|
||||
_assert_media_guarded(playable.build_cmd(Path("in.mov"), Path("out.mp4"), p))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spec",
|
||||
[
|
||||
{"trim": {"start_s": 1, "end_s": 3}, "accurate": False},
|
||||
{"trim": {"start_s": 1, "end_s": 3}, "accurate": True},
|
||||
{"crop": {"x": 0, "y": 0, "w": 101, "h": 51}},
|
||||
],
|
||||
ids=["fast trim", "accurate trim", "crop"],
|
||||
)
|
||||
def test_a_clip_is_guarded(spec):
|
||||
spec = edit.normalize_edit_spec(spec)
|
||||
_assert_media_guarded(edit.build_edit_cmd(Path("in.mp4"), Path("out.mp4"), spec, "mp4"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accurate", [True, False], ids=["accurate join", "fast join"])
|
||||
def test_a_join_is_guarded(accurate, tmp_path):
|
||||
segs = [{"start_s": 0, "end_s": 1}, {"start_s": 2, "end_s": 3}]
|
||||
spec = edit.normalize_edit_spec({"segments": segs, "accurate": accurate})
|
||||
cmd, _ = edit.build_concat_plan(Path("in.mp4"), Path("out.mp4"), spec, "mp4", tmp_path)
|
||||
# The fast join reads a concat list WE write; the whitelist rides down to the file it names.
|
||||
_assert_media_guarded(cmd, extra=set() if accurate else {"concat"})
|
||||
|
||||
|
||||
def test_the_poster_and_the_filmstrip_are_guarded():
|
||||
_assert_media_guarded(edit.build_poster_cmd(Path("in.mp4"), Path("p.jpg"), 2.0))
|
||||
geom = edit.storyboard_geometry(60)
|
||||
_assert_media_guarded(edit.build_storyboard_cmd(Path("in.mp4"), Path("s.jpg"), 60, geom))
|
||||
|
||||
|
||||
def test_a_downloaded_thumbnail_opens_as_a_single_image_only():
|
||||
(allowed,) = _whitelists(edit.build_thumbnail_jpg_cmd(Path("t.webp"), Path("t.jpg")))
|
||||
assert allowed == {"png_pipe", "jpeg_pipe", "webp_pipe"}
|
||||
|
||||
|
||||
def test_every_ffprobe_is_guarded(monkeypatch):
|
||||
seen = []
|
||||
|
||||
class _Done:
|
||||
stdout = "{}"
|
||||
|
||||
def _run(args, **_k):
|
||||
seen.append(args)
|
||||
return _Done()
|
||||
|
||||
monkeypatch.setattr(edit.subprocess, "run", _run)
|
||||
edit.run_ffprobe_json(Path("x.mp4"), show_streams=True)
|
||||
edit.probe_duration(Path("x.mp4"))
|
||||
assert len(seen) == 2
|
||||
for args in seen:
|
||||
i = args.index("-format_whitelist")
|
||||
assert set(args[i + 1].split(",")) == MEDIA
|
||||
|
||||
|
||||
def _unguarded_calls(source: str) -> list[int]:
|
||||
"""Lines of a list literal that starts an ffmpeg/ffprobe command or hands one an input (`-i`)
|
||||
and carries no `*…GUARD` / `*guard` splat and no "-format_whitelist" of its own. A builder that
|
||||
adds its input in a later `+=` must guard that list — that is where the input is."""
|
||||
bad = []
|
||||
for node in ast.walk(ast.parse(source)):
|
||||
if not isinstance(node, ast.List):
|
||||
continue
|
||||
consts = {e.value for e in node.elts if isinstance(e, ast.Constant)}
|
||||
if not consts & {"-i", "ffprobe"}:
|
||||
continue
|
||||
guarded = "-format_whitelist" in consts or any(
|
||||
isinstance(e, ast.Starred)
|
||||
and isinstance(v := e.value, ast.Name | ast.Attribute)
|
||||
and (v.id if isinstance(v, ast.Name) else v.attr).upper().endswith("GUARD")
|
||||
for e in node.elts
|
||||
)
|
||||
if not guarded:
|
||||
bad.append(node.lineno)
|
||||
return bad
|
||||
|
||||
|
||||
def test_no_ffmpeg_call_in_the_package_opens_an_input_unguarded():
|
||||
found = {
|
||||
f"{p.name}:{line}"
|
||||
for p in sorted(DOWNLOADS.rglob("*.py"))
|
||||
for line in _unguarded_calls(p.read_text(encoding="utf-8"))
|
||||
}
|
||||
assert found == set()
|
||||
|
||||
|
||||
def test_the_scan_sees_an_unguarded_call():
|
||||
# The scan above is green over nothing if it cannot see one: both shapes it must catch.
|
||||
assert _unguarded_calls('cmd = ["ffmpeg", "-y", "-i", src]') == [1]
|
||||
assert _unguarded_calls('a = ["ffprobe", "-v", "error", str(p)]') == [1]
|
||||
assert _unguarded_calls('cmd = [*INPUT_GUARD, "-i", src]') == []
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
@pytest.mark.parametrize("ext", ["png", "webp", "jpg"])
|
||||
def test_the_thumbnail_guard_still_opens_every_thumbnail_a_download_keeps(ext, tmp_path):
|
||||
# `pipeline._THUMB_SUFFIXES`: what the guard must still let through (a .jpg that is really a
|
||||
# PNG goes the ffmpeg route too — `thumbnail_to_jpg` copies only true JPEG bytes).
|
||||
src = tmp_path / f"t.{ext}"
|
||||
fmt = ["-f", "image2", "-c:v", "png"] if ext == "jpg" else []
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=64x48",
|
||||
"-frames:v", "1", *fmt, str(src)],
|
||||
check=True,
|
||||
) # fmt: skip
|
||||
out = edit.thumbnail_to_jpg(src, tmp_path / "poster.jpg")
|
||||
assert out is not None and edit._is_jpeg(out)
|
||||
@@ -0,0 +1,73 @@
|
||||
"""A libx264 re-encode of an odd-sized frame: 4:2:0 halves the chroma, so libx264 refuses an odd
|
||||
width or height ("width not divisible by 2"). Measured: a source that is odd AND not 4:2:0 already
|
||||
(a 4:4:4 or RGB screen capture, 1365x767) failed its conversion outright; a 4:2:0 source never
|
||||
reaches the encoder odd (ffmpeg's crop rounds to its chroma grid), but the clip then stored the
|
||||
size the user dragged, not the one the file has. Every re-encode now ends its filter chain on
|
||||
`edit.EVEN_FRAME` (at most one pixel row or column), and a clip stores its probed frame.
|
||||
|
||||
Real ffmpeg on both paths — the refusal is the encoder's, so only the encoder can show it is gone.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import pipeline, playable, probe, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
|
||||
|
||||
def _ffmpeg(*args: str) -> None:
|
||||
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", *args], check=True)
|
||||
|
||||
|
||||
def _frame(path: Path) -> tuple[int | None, int | None]:
|
||||
_, _, w, h, _ = probe.probe_media(path)
|
||||
return w, h
|
||||
|
||||
|
||||
def test_an_odd_sized_4_4_4_video_converts_to_the_even_frame_below_it(tmp_path):
|
||||
src = tmp_path / "in.mkv"
|
||||
_ffmpeg(
|
||||
"-f", "lavfi", "-i", "testsrc=size=161x121:rate=10:duration=1",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv444p", str(src),
|
||||
) # fmt: skip
|
||||
assert _frame(src) == (161, 121) # odd for real (`testsrc2` rounds its own size)
|
||||
p = playable.plan(probe.probe_json(src), "mkv")
|
||||
assert p is not None and p.reencode_video
|
||||
dest = tmp_path / "out.mp4"
|
||||
subprocess.run(playable.build_cmd(src, dest, p), check=True, capture_output=True)
|
||||
assert _frame(dest) == (160, 120)
|
||||
|
||||
|
||||
def test_an_odd_crop_makes_a_clip_and_stores_the_frame_it_has(db, make_user, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
src = tmp_path / "in.mp4"
|
||||
_ffmpeg(
|
||||
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2",
|
||||
"-f", "lavfi", "-i", "sine=duration=2",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", str(src),
|
||||
) # fmt: skip
|
||||
u = make_user()
|
||||
source = uploadmod.ingest_upload(
|
||||
db, u.id, filename="clip.mp4", src_path=src, size=src.stat().st_size
|
||||
)
|
||||
assert source.status == "done" # already phone-safe: the clip's source as it came
|
||||
job = service.enqueue_edit(db, u.id, source, {"crop": {"x": 3, "y": 5, "w": 101, "h": 51}})
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
job.status, asset.status = "running", "downloading" # the worker's claim
|
||||
db.commit()
|
||||
|
||||
pipeline._process_edit(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
job, asset = db.get(DownloadJob, job.id), db.get(MediaAsset, job.asset_id)
|
||||
assert (job.status, job.error, asset.status) == ("done", None, "ready")
|
||||
final = storage.abs_path(str(tmp_path), asset.rel_path)
|
||||
assert _frame(final) == (100, 50)
|
||||
assert (asset.width, asset.height) == (100, 50) # what the file is, not what was asked
|
||||
@@ -0,0 +1,355 @@
|
||||
"""An upload after ingest: what it costs its owner from the moment it lands, how the worker's
|
||||
conversion names, checks and cleans up the file it makes, and what the owner and a share link see.
|
||||
|
||||
DB-backed. The conversion's ffmpeg step (`pipeline._ensure_phone_safe`) is stubbed to write a file of
|
||||
a chosen size — these tests are about what happens around the encode; `test_upload.py` runs the real
|
||||
one, and `test_playable.py` judges its bytes.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import gc as gcmod
|
||||
from app.downloads import pipeline, quota, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _probe(monkeypatch, vcodec="hevc"):
|
||||
"""ffprobe's view of every file here: `vcodec` + AAC, 8-bit 4:2:0 — HEVC is queued for the
|
||||
worker, H.264 lands as it came."""
|
||||
data = {
|
||||
"streams": [
|
||||
{"codec_type": "video", "codec_name": vcodec, "pix_fmt": "yuv420p",
|
||||
"width": 640, "height": 360},
|
||||
{"codec_type": "audio", "codec_name": "aac"},
|
||||
],
|
||||
"format": {"format_name": "mov,mp4", "duration": "6"},
|
||||
} # fmt: skip
|
||||
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
|
||||
|
||||
|
||||
def _ingest(db, root, user_id, name="clip.mov", size=10):
|
||||
src = root / "incoming.bin"
|
||||
src.write_bytes(b"o" * size)
|
||||
job = uploadmod.ingest_upload(db, user_id, filename=name, src_path=src, size=size)
|
||||
return job, db.get(MediaAsset, job.asset_id)
|
||||
|
||||
|
||||
def _claim(db, job, asset):
|
||||
"""What the worker's claim does before it converts."""
|
||||
job.status, asset.status = "running", "downloading"
|
||||
db.commit()
|
||||
|
||||
|
||||
def _converts_to(monkeypatch, size, phases=None):
|
||||
"""The encode writes a `size`-byte .mp4 into its staging dir (and notes the phase it reports)."""
|
||||
|
||||
def _encode(job_id, produced, out_dir, phase, total_s=None, **_k):
|
||||
if phases is not None:
|
||||
phases.append(phase)
|
||||
out = Path(out_dir) / "converted.mp4"
|
||||
out.write_bytes(b"c" * size)
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(pipeline, "_ensure_phone_safe", _encode)
|
||||
|
||||
|
||||
def _cap(db, user_id, max_bytes):
|
||||
db.add(
|
||||
DownloadQuota(
|
||||
user_id=user_id, max_bytes=max_bytes, max_concurrent=2, max_jobs=50, unlimited=False
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _uploads_dir(root, user_id) -> Path:
|
||||
return root / ".uploads" / str(user_id)
|
||||
|
||||
|
||||
# --- what it costs ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_queued_upload_counts_against_its_owner_from_the_moment_it_lands(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
assert (job.status, asset.status) == ("queued", "pending")
|
||||
assert quota.footprint(db, u.id) == 10 # its original is on disk already
|
||||
|
||||
|
||||
def test_a_failed_conversion_still_counts_its_kept_original(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
job.status, asset.status = "error", "error" # kept for Retry
|
||||
db.commit()
|
||||
assert storage.abs_path(str(_root), asset.rel_path).exists()
|
||||
assert quota.footprint(db, u.id) == 10
|
||||
|
||||
|
||||
def test_a_conversion_that_outgrows_the_cap_is_refused_and_keeps_the_original(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_cap(db, u.id, max_bytes=100)
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=500)
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, job.id)
|
||||
assert (job.status, job.error_code) == ("error", "quota_exceeded")
|
||||
assert original.exists()
|
||||
assert [p.name for p in _uploads_dir(_root, u.id).iterdir()] == [original.name]
|
||||
|
||||
|
||||
def test_a_conversion_that_shrinks_passes_an_owner_at_the_cap(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_cap(db, u.id, max_bytes=10)
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=5)
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).status == "done"
|
||||
assert db.get(MediaAsset, asset.id).size_bytes == 5
|
||||
|
||||
|
||||
# --- the converted file's name and its leftovers -----------------------------------------------
|
||||
|
||||
|
||||
def test_two_dotted_bare_names_keep_a_file_each(db, make_user, _root, monkeypatch):
|
||||
# 'Holiday.2024.ä': its suffix is no extension (nothing shell-safe is left of it), so the stem's
|
||||
# own dot once read as one — and the converted name lost `_a{id}`, the one thing keeping two apart.
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_converts_to(monkeypatch, size=5)
|
||||
finals = []
|
||||
for _ in range(2):
|
||||
job, asset = _ingest(db, _root, u.id, name="Holiday.2024.ä")
|
||||
_claim(db, job, asset)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
asset = db.get(MediaAsset, asset.id)
|
||||
assert asset.status == "ready"
|
||||
assert asset.rel_path.endswith(f"_a{asset.id}.mp4"), asset.rel_path
|
||||
finals.append(asset.rel_path)
|
||||
assert finals[0] != finals[1]
|
||||
assert all(storage.abs_path(str(_root), r).exists() for r in finals)
|
||||
|
||||
|
||||
def test_a_conversion_failing_after_it_placed_its_file_removes_it(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=5)
|
||||
|
||||
def _probe_fails(_path):
|
||||
raise RuntimeError("probe crashed")
|
||||
|
||||
monkeypatch.setattr(pipeline.probe, "probe_media", _probe_fails)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).status == "error"
|
||||
assert original.exists() # Retry starts from it
|
||||
# The row still names the original: a converted file left beside it is one nothing would find.
|
||||
assert [p.name for p in _uploads_dir(_root, u.id).iterdir()] == [original.name]
|
||||
|
||||
|
||||
# --- what the owner is told ------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_failed_encode_is_a_failed_conversion_not_a_download_step(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
_claim(db, job, asset)
|
||||
|
||||
def _ffmpeg_fails(*_a, **_k):
|
||||
raise StatedFailure("ffmpeg failed (1): unsupported")
|
||||
|
||||
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _ffmpeg_fails)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).error_code == "convert_failed"
|
||||
|
||||
|
||||
def test_a_missing_original_says_so(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
_claim(db, job, asset)
|
||||
storage.abs_path(str(_root), asset.rel_path).unlink()
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).error_code == "upload_missing"
|
||||
|
||||
|
||||
def test_the_conversion_reports_its_own_phase(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
_claim(db, job, asset)
|
||||
phases = []
|
||||
_converts_to(monkeypatch, size=5, phases=phases)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
assert phases == ["converting"] # not yt-dlp's "processing": the queue row shows ffmpeg's %
|
||||
|
||||
|
||||
def test_the_owner_is_told_their_upload_is_ready(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=5)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
notes = db.execute(select(Notification).where(Notification.user_id == u.id)).scalars().all()
|
||||
assert [n.data.get("kind") for n in notes] == ["upload"]
|
||||
|
||||
|
||||
def test_an_extension_is_judged_as_it_is_stored(db, make_user, _root, monkeypatch):
|
||||
# 'clip.mp4~' is stored as .mp4; judged by the raw 'mp4~' it queued a conversion of a file that
|
||||
# was phone-safe already.
|
||||
_probe(monkeypatch, vcodec="h264")
|
||||
job, asset = _ingest(db, _root, make_user().id, name="clip.mp4~")
|
||||
assert (job.status, asset.status, asset.container) == ("done", "ready", "mp4")
|
||||
|
||||
|
||||
# --- the garbage collector ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _aged(path: Path, hours: float = 5) -> None:
|
||||
old = time.time() - hours * 3600
|
||||
os.utime(path, (old, old))
|
||||
|
||||
|
||||
def test_the_sweep_takes_a_stranded_upload_file_and_keeps_a_waiting_original(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_job, asset = _ingest(db, _root, u.id)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
stranded = original.with_name("left_by_a_stopped_conversion_a999.mp4")
|
||||
stranded.write_bytes(b"x")
|
||||
_aged(original)
|
||||
_aged(stranded)
|
||||
|
||||
gcmod._sweep_orphan_files(db, str(_root))
|
||||
|
||||
assert original.exists() # queued for conversion: its row names it, ready or not
|
||||
assert not stranded.exists() # no row names it
|
||||
|
||||
|
||||
def test_the_cache_cap_never_evicts_an_upload(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, vcodec="h264")
|
||||
u = make_user()
|
||||
_job, upload = _ingest(db, _root, u.id, name="mine.mp4", size=10)
|
||||
db.add(
|
||||
MediaAsset(
|
||||
source_kind="youtube", source_ref="vid1", format_sig="best", status="ready",
|
||||
rel_path="Chan/clip.mp4", size_bytes=10, title="downloaded",
|
||||
last_access_at=datetime.now(timezone.utc),
|
||||
)
|
||||
) # fmt: skip
|
||||
# The upload is the least recently used: an LRU pass that may take it takes it first.
|
||||
upload.last_access_at = datetime.now(timezone.utc) - timedelta(days=30)
|
||||
db.commit()
|
||||
(_root / "Chan").mkdir()
|
||||
(_root / "Chan" / "clip.mp4").write_bytes(b"d" * 10)
|
||||
sysconfig.set_value(db, "download_total_max_bytes", 15) # 20 on disk
|
||||
db.commit()
|
||||
|
||||
gcmod.run_download_gc(db)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(MediaAsset, upload.id).status == "ready"
|
||||
assert storage.abs_path(str(_root), upload.rel_path).exists()
|
||||
|
||||
|
||||
# --- a share link ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["pending", "downloading"])
|
||||
def test_a_link_whose_video_is_being_made_again_says_preparing_not_gone(
|
||||
state, db, as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
# A link is made on a finished job; its asset can be (re)made under it later — a shared cache row
|
||||
# re-fetched, a conversion retried. Until it is ready the page says "try again shortly".
|
||||
_probe(monkeypatch, vcodec="h264")
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
|
||||
).json()
|
||||
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
|
||||
assert c.get(f"/api/public/watch/{link['token']}").status_code == 200
|
||||
db.get(MediaAsset, job["asset"]["id"]).status = state
|
||||
db.commit()
|
||||
r = c.get(f"/api/public/watch/{link['token']}")
|
||||
assert r.status_code == 404
|
||||
assert r.json()["detail"]["reason"] == "video_preparing"
|
||||
|
||||
|
||||
def test_the_admin_storage_view_counts_a_waiting_upload(db, as_user, make_user, _root, monkeypatch):
|
||||
# The same disk the owner's footprint and the GC's cap count (`quota.ON_DISK`).
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_ingest(db, _root, u.id, size=10)
|
||||
r = as_user(make_user(role="admin")).get("/api/admin/downloads/storage").json()
|
||||
assert (r["ready_files"], r["total_bytes"]) == (1, 10)
|
||||
assert {p["user_id"]: p["footprint_bytes"] for p in r["per_user"]} == {u.id: 10}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("job_status", "asset_status", "ref_count"),
|
||||
[
|
||||
("done", "ready", 1), # a finished upload its owner keeps
|
||||
("queued", "pending", 1), # one waiting for the worker
|
||||
("canceled", "downloading", 0), # cancelled mid-conversion: the worker still reads it
|
||||
],
|
||||
)
|
||||
def test_the_upload_reap_leaves_one_still_held_or_still_read(
|
||||
job_status, asset_status, ref_count, db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
path = storage.abs_path(str(_root), asset.rel_path)
|
||||
job.status, asset.status, asset.ref_count = job_status, asset_status, ref_count
|
||||
db.commit()
|
||||
|
||||
gcmod.run_download_gc(db)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(MediaAsset, asset.id) is not None
|
||||
assert path.exists()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""The worker child's dispatch (`runner._run`) and the edit path's last step before it stores a clip.
|
||||
|
||||
`runner._run` picks the pipeline entry by the job's kind; the tests elsewhere call those entries
|
||||
directly, so a kind routed to the wrong one (an upload sent to yt-dlp) would pass all of them.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import pipeline, runner, service, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "entry"),
|
||||
[("upload", "_convert_upload"), ("edit", "_process_edit"), ("download", "_download")],
|
||||
)
|
||||
def test_each_job_kind_reaches_its_own_pipeline_entry(kind, entry, monkeypatch):
|
||||
called = []
|
||||
for name in ("_convert_upload", "_process_edit", "_download"):
|
||||
monkeypatch.setattr(pipeline, name, lambda *a, _n=name: called.append(_n))
|
||||
runner._run(5, 7, kind, "url", "https://example.org/v", {}, 1)
|
||||
assert called == [entry]
|
||||
|
||||
|
||||
def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension(
|
||||
db, make_user, tmp_path, monkeypatch
|
||||
):
|
||||
# A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an
|
||||
# audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file.
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
data = {
|
||||
"streams": [{"codec_type": "audio", "codec_name": "aac"}],
|
||||
"format": {"format_name": "mov,mp4", "duration": "10"},
|
||||
}
|
||||
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
|
||||
src = tmp_path / "in.bin"
|
||||
src.write_bytes(b"a" * 10)
|
||||
u = make_user()
|
||||
source = uploadmod.ingest_upload(db, u.id, filename="song.m4a", src_path=src, size=10)
|
||||
# Stored as an .mp4, the way an audio file was kept before every ready file met the rule.
|
||||
kept = db.get(MediaAsset, source.asset_id)
|
||||
old = storage.abs_path(str(tmp_path), kept.rel_path)
|
||||
kept.rel_path, kept.container = kept.rel_path[: -len(".m4a")] + ".mp4", "mp4"
|
||||
old.rename(storage.abs_path(str(tmp_path), kept.rel_path))
|
||||
db.commit()
|
||||
job = service.enqueue_edit(
|
||||
db, u.id, source, {"trim": {"start_s": 1, "end_s": 3}, "accurate": False}
|
||||
)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
job.status, asset.status = "running", "downloading"
|
||||
db.commit()
|
||||
|
||||
def _trim(cmd, *_a, **_k):
|
||||
Path(cmd[-1]).write_bytes(b"t" * 4) # the stream-copied clip, at the path ffmpeg was given
|
||||
|
||||
def _remux(job_id, produced, out_dir, phase, total_s=None, **_k):
|
||||
out = Path(out_dir) / "remuxed.m4a"
|
||||
out.write_bytes(b"r" * 3)
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _trim)
|
||||
monkeypatch.setattr(pipeline, "_ensure_phone_safe", _remux)
|
||||
pipeline._process_edit(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
asset = db.get(MediaAsset, asset.id)
|
||||
assert (db.get(DownloadJob, job.id).status, asset.container) == ("done", "m4a")
|
||||
assert asset.rel_path.endswith(".m4a")
|
||||
assert storage.abs_path(str(tmp_path), asset.rel_path).read_bytes() == b"rrr"
|
||||
Reference in New Issue
Block a user