fix(downloads): an upload's conversion is guarded, counted, named and cleaned up
- Every ffmpeg/ffprobe over a library file names the demuxers it may use
(edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
kept after a failed conversion) — the footprint, the GC's cap and the admin
view read one ON_DISK predicate — and a conversion that outgrows the cap is
refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
uploads shared a file); a converted file placed but not committed is removed,
and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
convert_failed / upload_missing, reports the "converting" phase, and notifies
as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
"""An upload after ingest: what it costs its owner from the moment it lands, how the worker's
|
||||
conversion names, checks and cleans up the file it makes, and what the owner and a share link see.
|
||||
|
||||
DB-backed. The conversion's ffmpeg step (`pipeline._ensure_phone_safe`) is stubbed to write a file of
|
||||
a chosen size — these tests are about what happens around the encode; `test_upload.py` runs the real
|
||||
one, and `test_playable.py` judges its bytes.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import gc as gcmod
|
||||
from app.downloads import pipeline, quota, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _probe(monkeypatch, vcodec="hevc"):
|
||||
"""ffprobe's view of every file here: `vcodec` + AAC, 8-bit 4:2:0 — HEVC is queued for the
|
||||
worker, H.264 lands as it came."""
|
||||
data = {
|
||||
"streams": [
|
||||
{"codec_type": "video", "codec_name": vcodec, "pix_fmt": "yuv420p",
|
||||
"width": 640, "height": 360},
|
||||
{"codec_type": "audio", "codec_name": "aac"},
|
||||
],
|
||||
"format": {"format_name": "mov,mp4", "duration": "6"},
|
||||
} # fmt: skip
|
||||
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
|
||||
|
||||
|
||||
def _ingest(db, root, user_id, name="clip.mov", size=10):
|
||||
src = root / "incoming.bin"
|
||||
src.write_bytes(b"o" * size)
|
||||
job = uploadmod.ingest_upload(db, user_id, filename=name, src_path=src, size=size)
|
||||
return job, db.get(MediaAsset, job.asset_id)
|
||||
|
||||
|
||||
def _claim(db, job, asset):
|
||||
"""What the worker's claim does before it converts."""
|
||||
job.status, asset.status = "running", "downloading"
|
||||
db.commit()
|
||||
|
||||
|
||||
def _converts_to(monkeypatch, size, phases=None):
|
||||
"""The encode writes a `size`-byte .mp4 into its staging dir (and notes the phase it reports)."""
|
||||
|
||||
def _encode(job_id, produced, out_dir, phase, total_s=None, **_k):
|
||||
if phases is not None:
|
||||
phases.append(phase)
|
||||
out = Path(out_dir) / "converted.mp4"
|
||||
out.write_bytes(b"c" * size)
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(pipeline, "_ensure_phone_safe", _encode)
|
||||
|
||||
|
||||
def _cap(db, user_id, max_bytes):
|
||||
db.add(
|
||||
DownloadQuota(
|
||||
user_id=user_id, max_bytes=max_bytes, max_concurrent=2, max_jobs=50, unlimited=False
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _uploads_dir(root, user_id) -> Path:
|
||||
return root / ".uploads" / str(user_id)
|
||||
|
||||
|
||||
# --- what it costs ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_queued_upload_counts_against_its_owner_from_the_moment_it_lands(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
assert (job.status, asset.status) == ("queued", "pending")
|
||||
assert quota.footprint(db, u.id) == 10 # its original is on disk already
|
||||
|
||||
|
||||
def test_a_failed_conversion_still_counts_its_kept_original(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
job.status, asset.status = "error", "error" # kept for Retry
|
||||
db.commit()
|
||||
assert storage.abs_path(str(_root), asset.rel_path).exists()
|
||||
assert quota.footprint(db, u.id) == 10
|
||||
|
||||
|
||||
def test_a_conversion_that_outgrows_the_cap_is_refused_and_keeps_the_original(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_cap(db, u.id, max_bytes=100)
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=500)
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, job.id)
|
||||
assert (job.status, job.error_code) == ("error", "quota_exceeded")
|
||||
assert original.exists()
|
||||
assert [p.name for p in _uploads_dir(_root, u.id).iterdir()] == [original.name]
|
||||
|
||||
|
||||
def test_a_conversion_that_shrinks_passes_an_owner_at_the_cap(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_cap(db, u.id, max_bytes=10)
|
||||
job, asset = _ingest(db, _root, u.id, size=10)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=5)
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).status == "done"
|
||||
assert db.get(MediaAsset, asset.id).size_bytes == 5
|
||||
|
||||
|
||||
# --- the converted file's name and its leftovers -----------------------------------------------
|
||||
|
||||
|
||||
def test_two_dotted_bare_names_keep_a_file_each(db, make_user, _root, monkeypatch):
|
||||
# 'Holiday.2024.ä': its suffix is no extension (nothing shell-safe is left of it), so the stem's
|
||||
# own dot once read as one — and the converted name lost `_a{id}`, the one thing keeping two apart.
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_converts_to(monkeypatch, size=5)
|
||||
finals = []
|
||||
for _ in range(2):
|
||||
job, asset = _ingest(db, _root, u.id, name="Holiday.2024.ä")
|
||||
_claim(db, job, asset)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
asset = db.get(MediaAsset, asset.id)
|
||||
assert asset.status == "ready"
|
||||
assert asset.rel_path.endswith(f"_a{asset.id}.mp4"), asset.rel_path
|
||||
finals.append(asset.rel_path)
|
||||
assert finals[0] != finals[1]
|
||||
assert all(storage.abs_path(str(_root), r).exists() for r in finals)
|
||||
|
||||
|
||||
def test_a_conversion_failing_after_it_placed_its_file_removes_it(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=5)
|
||||
|
||||
def _probe_fails(_path):
|
||||
raise RuntimeError("probe crashed")
|
||||
|
||||
monkeypatch.setattr(pipeline.probe, "probe_media", _probe_fails)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).status == "error"
|
||||
assert original.exists() # Retry starts from it
|
||||
# The row still names the original: a converted file left beside it is one nothing would find.
|
||||
assert [p.name for p in _uploads_dir(_root, u.id).iterdir()] == [original.name]
|
||||
|
||||
|
||||
# --- what the owner is told ------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_failed_encode_is_a_failed_conversion_not_a_download_step(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
_claim(db, job, asset)
|
||||
|
||||
def _ffmpeg_fails(*_a, **_k):
|
||||
raise StatedFailure("ffmpeg failed (1): unsupported")
|
||||
|
||||
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _ffmpeg_fails)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).error_code == "convert_failed"
|
||||
|
||||
|
||||
def test_a_missing_original_says_so(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
_claim(db, job, asset)
|
||||
storage.abs_path(str(_root), asset.rel_path).unlink()
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).error_code == "upload_missing"
|
||||
|
||||
|
||||
def test_the_conversion_reports_its_own_phase(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
_claim(db, job, asset)
|
||||
phases = []
|
||||
_converts_to(monkeypatch, size=5, phases=phases)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
assert phases == ["converting"] # not yt-dlp's "processing": the queue row shows ffmpeg's %
|
||||
|
||||
|
||||
def test_the_owner_is_told_their_upload_is_ready(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
job, asset = _ingest(db, _root, u.id)
|
||||
_claim(db, job, asset)
|
||||
_converts_to(monkeypatch, size=5)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
notes = db.execute(select(Notification).where(Notification.user_id == u.id)).scalars().all()
|
||||
assert [n.data.get("kind") for n in notes] == ["upload"]
|
||||
|
||||
|
||||
def test_an_extension_is_judged_as_it_is_stored(db, make_user, _root, monkeypatch):
|
||||
# 'clip.mp4~' is stored as .mp4; judged by the raw 'mp4~' it queued a conversion of a file that
|
||||
# was phone-safe already.
|
||||
_probe(monkeypatch, vcodec="h264")
|
||||
job, asset = _ingest(db, _root, make_user().id, name="clip.mp4~")
|
||||
assert (job.status, asset.status, asset.container) == ("done", "ready", "mp4")
|
||||
|
||||
|
||||
# --- the garbage collector ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _aged(path: Path, hours: float = 5) -> None:
|
||||
old = time.time() - hours * 3600
|
||||
os.utime(path, (old, old))
|
||||
|
||||
|
||||
def test_the_sweep_takes_a_stranded_upload_file_and_keeps_a_waiting_original(
|
||||
db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_job, asset = _ingest(db, _root, u.id)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
stranded = original.with_name("left_by_a_stopped_conversion_a999.mp4")
|
||||
stranded.write_bytes(b"x")
|
||||
_aged(original)
|
||||
_aged(stranded)
|
||||
|
||||
gcmod._sweep_orphan_files(db, str(_root))
|
||||
|
||||
assert original.exists() # queued for conversion: its row names it, ready or not
|
||||
assert not stranded.exists() # no row names it
|
||||
|
||||
|
||||
def test_the_cache_cap_never_evicts_an_upload(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, vcodec="h264")
|
||||
u = make_user()
|
||||
_job, upload = _ingest(db, _root, u.id, name="mine.mp4", size=10)
|
||||
db.add(
|
||||
MediaAsset(
|
||||
source_kind="youtube", source_ref="vid1", format_sig="best", status="ready",
|
||||
rel_path="Chan/clip.mp4", size_bytes=10, title="downloaded",
|
||||
last_access_at=datetime.now(timezone.utc),
|
||||
)
|
||||
) # fmt: skip
|
||||
# The upload is the least recently used: an LRU pass that may take it takes it first.
|
||||
upload.last_access_at = datetime.now(timezone.utc) - timedelta(days=30)
|
||||
db.commit()
|
||||
(_root / "Chan").mkdir()
|
||||
(_root / "Chan" / "clip.mp4").write_bytes(b"d" * 10)
|
||||
sysconfig.set_value(db, "download_total_max_bytes", 15) # 20 on disk
|
||||
db.commit()
|
||||
|
||||
gcmod.run_download_gc(db)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(MediaAsset, upload.id).status == "ready"
|
||||
assert storage.abs_path(str(_root), upload.rel_path).exists()
|
||||
|
||||
|
||||
# --- a share link ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["pending", "downloading"])
|
||||
def test_a_link_whose_video_is_being_made_again_says_preparing_not_gone(
|
||||
state, db, as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
# A link is made on a finished job; its asset can be (re)made under it later — a shared cache row
|
||||
# re-fetched, a conversion retried. Until it is ready the page says "try again shortly".
|
||||
_probe(monkeypatch, vcodec="h264")
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
|
||||
).json()
|
||||
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
|
||||
assert c.get(f"/api/public/watch/{link['token']}").status_code == 200
|
||||
db.get(MediaAsset, job["asset"]["id"]).status = state
|
||||
db.commit()
|
||||
r = c.get(f"/api/public/watch/{link['token']}")
|
||||
assert r.status_code == 404
|
||||
assert r.json()["detail"]["reason"] == "video_preparing"
|
||||
|
||||
|
||||
def test_the_admin_storage_view_counts_a_waiting_upload(db, as_user, make_user, _root, monkeypatch):
|
||||
# The same disk the owner's footprint and the GC's cap count (`quota.ON_DISK`).
|
||||
_probe(monkeypatch)
|
||||
u = make_user()
|
||||
_ingest(db, _root, u.id, size=10)
|
||||
r = as_user(make_user(role="admin")).get("/api/admin/downloads/storage").json()
|
||||
assert (r["ready_files"], r["total_bytes"]) == (1, 10)
|
||||
assert {p["user_id"]: p["footprint_bytes"] for p in r["per_user"]} == {u.id: 10}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("job_status", "asset_status", "ref_count"),
|
||||
[
|
||||
("done", "ready", 1), # a finished upload its owner keeps
|
||||
("queued", "pending", 1), # one waiting for the worker
|
||||
("canceled", "downloading", 0), # cancelled mid-conversion: the worker still reads it
|
||||
],
|
||||
)
|
||||
def test_the_upload_reap_leaves_one_still_held_or_still_read(
|
||||
job_status, asset_status, ref_count, db, make_user, _root, monkeypatch
|
||||
):
|
||||
_probe(monkeypatch)
|
||||
job, asset = _ingest(db, _root, make_user().id)
|
||||
path = storage.abs_path(str(_root), asset.rel_path)
|
||||
job.status, asset.status, asset.ref_count = job_status, asset_status, ref_count
|
||||
db.commit()
|
||||
|
||||
gcmod.run_download_gc(db)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(MediaAsset, asset.id) is not None
|
||||
assert path.exists()
|
||||
Reference in New Issue
Block a user