feat(downloads): every ready file is phone-safe — uploads and clips too
A share link is opened on whatever the friend holds. Measured on an iPhone and an iPad: AV1, Matroska, HEVC tagged hev1 and MP3 inside an .mp4 fail there; Chrome and Firefox add HEVC and 10-bit / 4:4:4 H.264. The old check judged the codec NAME only and ran on downloads only, so a 10-bit H.264 passed, an upload went out exactly as it came (an iPhone's HEVC .mov, a webm, an mkv) and a fast trim kept its source's codecs. One rule now, `downloads/playable.py`: video is H.264 8-bit 4:2:0 + AAC in .mp4, audio-only is .mp3 or AAC .m4a; `plan` says what must change, `build_cmd` re-encodes only the failing stream (or remuxes). The worker applies it to a download, an editor clip and — new — an upload: media that fails the rule is queued (asset pending, job queued) and the worker converts it, keeping only the converted file (owner's choice). Until then the original sits at rel_path, so Retry starts from it and a delete removes it; an upload that lost its last holder mid-conversion is reaped by the GC. Every libx264 run (edit, concat, conversion) now pins `-pix_fmt yuv420p` — without it a 10-bit source came out High 10. The download cache signature moves to v5 so a cached 10-bit asset re-derives. The upload toast says when the file is still converting (HU/EN). Tests: the plan for every measured class; real ffmpeg turning a 10-bit Matroska + Opus and an mp3-in-mp4 into files the rule accepts (red without the pixel-format pin); the queued upload's delete, cancel-then-GC, failed conversion and real worker conversion (the cleanup tests red without their rules); the toast (red without the branch). Live on localdev: 9 fixture uploads converted, every served file probed H.264 High yuv420p + AAC, or mp3 / m4a.
This commit is contained in:
@@ -7,15 +7,18 @@ guards that keep an upload out of paths that assume a re-fetchable source (edit
|
||||
|
||||
DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is
|
||||
pointed at a per-test tmp dir so ingest's on-disk moves are isolated. The ingest/route tests
|
||||
monkeypatch the shared `probe.probe_media` so they don't shell out to ffprobe; the classifier's OWN
|
||||
monkeypatch the shared `probe.probe_json` so they don't shell out to ffprobe; the classifier's OWN
|
||||
parsing (image → file, cover-art skip, dims, duration) is covered directly by `TestProbeClassifier`
|
||||
below, which stubs one level down (`run_ffprobe_json`) and exercises the real `probe_media`.
|
||||
below, which stubs one level down (`run_ffprobe_json`) and exercises the real `probe_media`. The
|
||||
phone-safe conversion of an upload runs the real ffmpeg (`TestPhoneSafeUpload`).
|
||||
"""
|
||||
|
||||
import ast
|
||||
import asyncio
|
||||
import io
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -25,9 +28,11 @@ from app.config import settings
|
||||
from app.routes._common import capped_stream_to
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.downloads import mediakind, quota, storage
|
||||
from app.downloads import gc as gcmod
|
||||
from app.downloads import mediakind, pipeline, playable, quota, service, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
|
||||
@@ -51,9 +56,20 @@ def _root(tmp_path, monkeypatch):
|
||||
|
||||
|
||||
def _probe(monkeypatch, result):
|
||||
# upload ingest now calls the shared `probe.probe_media` (same ground-truth ffprobe the download
|
||||
# worker uses); patch it at that shared home.
|
||||
monkeypatch.setattr(probemod, "probe_media", lambda _p: result)
|
||||
"""Stub the shared ffprobe call (`probe.probe_json`, the same ground truth the download worker
|
||||
reads) with the JSON a file of these (vcodec, acodec, width, height, duration) would give — its
|
||||
video 8-bit 4:2:0, so an H.264 + AAC .mp4 is already phone-safe and ingests straight to `done`.
|
||||
All-None is a file ffprobe could not read."""
|
||||
v, a, w, h, d = result
|
||||
streams = []
|
||||
if v:
|
||||
streams.append(
|
||||
{"codec_type": "video", "codec_name": v, "pix_fmt": "yuv420p", "width": w, "height": h}
|
||||
)
|
||||
if a:
|
||||
streams.append({"codec_type": "audio", "codec_name": a})
|
||||
data = {"streams": streams, "format": {"format_name": "mov,mp4", "duration": d}}
|
||||
monkeypatch.setattr(probemod, "probe_json", lambda _p: data if streams else None)
|
||||
|
||||
|
||||
def test_ingest_generic_file_is_download_only(db, make_user, _root, monkeypatch):
|
||||
@@ -281,7 +297,7 @@ def test_upload_route_generic_forces_download(
|
||||
|
||||
|
||||
def test_upload_route_media_html_extension_not_served_as_html(
|
||||
as_user, make_user, _root, monkeypatch, upload_headers
|
||||
db, as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
"""The XSS guard: a MEDIA-classified upload (real codecs) named with an executable extension must
|
||||
NOT be served with a guessed text/html type. The served Content-Type comes from the verified
|
||||
@@ -298,6 +314,14 @@ def test_upload_route_media_html_extension_not_served_as_html(
|
||||
assert r.status_code == 200, r.text
|
||||
job = r.json()
|
||||
assert job["asset"]["media_kind"] == "video" and job["asset"]["container"] == "html"
|
||||
# Media in no phone-safe container is queued for conversion to .mp4, so today it is never served
|
||||
# as .html at all. The serving guard below is the second line: a row from before that rule (or
|
||||
# one whose conversion is skipped some day) must still not run as HTML.
|
||||
assert job["status"] == "queued"
|
||||
row = db.get(DownloadJob, job["id"])
|
||||
db.get(MediaAsset, row.asset_id).status = "ready"
|
||||
row.status = "done"
|
||||
db.commit()
|
||||
|
||||
# Authenticated endpoint: never text/html.
|
||||
fr = c.get(f"/api/downloads/{job['id']}/file")
|
||||
@@ -1014,3 +1038,122 @@ class TestResolveMediaFacts:
|
||||
)
|
||||
assert (v, a) == (None, "aac")
|
||||
assert mediakind.derive(v, a) == mediakind.AUDIO
|
||||
|
||||
|
||||
# --- the phone-safe conversion (an iPhone's HEVC, a webm, an mkv) ------------------------------
|
||||
|
||||
|
||||
def _queued_upload(db, make_user, root, monkeypatch, name="clip.mov"):
|
||||
"""An upload no phone could play as it is: probed as HEVC, so ingest queues it."""
|
||||
_probe(monkeypatch, ("hevc", "aac", 640, 360, 6))
|
||||
src = root / "src.bin"
|
||||
src.write_bytes(b"hevc bytes")
|
||||
job = uploadmod.ingest_upload(db, make_user().id, filename=name, src_path=src, size=10)
|
||||
return job, db.get(MediaAsset, job.asset_id)
|
||||
|
||||
|
||||
class TestPhoneSafeUpload:
|
||||
def test_an_unsafe_video_is_queued_for_the_worker_not_done(
|
||||
self, db, make_user, _root, monkeypatch
|
||||
):
|
||||
job, asset = _queued_upload(db, make_user, _root, monkeypatch)
|
||||
assert (job.status, job.progress, asset.status) == ("queued", 0, "pending")
|
||||
assert asset.rel_path.endswith(".mov") # the original waits at rel_path for the worker
|
||||
assert storage.abs_path(str(_root), asset.rel_path).exists()
|
||||
|
||||
def test_a_safe_video_still_lands_done(self, db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, ("h264", "aac", 640, 360, 6))
|
||||
src = _root / "src.bin"
|
||||
src.write_bytes(b"h264 bytes")
|
||||
job = uploadmod.ingest_upload(
|
||||
db, make_user().id, filename="clip.mp4", src_path=src, size=10
|
||||
)
|
||||
assert job.status == "done" and db.get(MediaAsset, job.asset_id).status == "ready"
|
||||
|
||||
def test_deleting_a_queued_upload_removes_its_original(self, db, make_user, _root, monkeypatch):
|
||||
job, asset = _queued_upload(db, make_user, _root, monkeypatch)
|
||||
path, asset_id = storage.abs_path(str(_root), asset.rel_path), asset.id
|
||||
service.delete(db, job)
|
||||
assert not path.exists()
|
||||
assert db.get(MediaAsset, asset_id) is None
|
||||
|
||||
def test_one_cancelled_mid_conversion_is_reaped_once_the_worker_let_go(
|
||||
self, db, make_user, _root, monkeypatch
|
||||
):
|
||||
job, asset = _queued_upload(db, make_user, _root, monkeypatch)
|
||||
path, asset_id = storage.abs_path(str(_root), asset.rel_path), asset.id
|
||||
asset.status = "downloading" # the worker child is reading the original
|
||||
job.status = "running"
|
||||
db.commit()
|
||||
service.cancel(db, job)
|
||||
assert path.exists() # never pulled out from under a running ffmpeg
|
||||
db.get(MediaAsset, asset_id).status = "pending" # the worker stopped the child and let go
|
||||
db.commit()
|
||||
gcmod.run_download_gc(db)
|
||||
db.expire_all()
|
||||
assert not path.exists()
|
||||
assert db.get(MediaAsset, asset_id) is None
|
||||
|
||||
def test_a_failed_conversion_keeps_the_original_for_retry(
|
||||
self, db, make_user, _root, monkeypatch
|
||||
):
|
||||
job, asset = _queued_upload(db, make_user, _root, monkeypatch)
|
||||
path = storage.abs_path(str(_root), asset.rel_path)
|
||||
job.status, asset.status = "running", "downloading"
|
||||
db.commit()
|
||||
|
||||
def _ffmpeg_fails(*_a, **_k):
|
||||
raise StatedFailure("ffmpeg failed (1): unsupported")
|
||||
|
||||
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", _ffmpeg_fails)
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
db.expire_all()
|
||||
assert (db.get(DownloadJob, job.id).status, db.get(MediaAsset, asset.id).status) == (
|
||||
"error",
|
||||
"error",
|
||||
)
|
||||
assert path.exists()
|
||||
service.resume(db, db.get(DownloadJob, job.id))
|
||||
assert db.get(MediaAsset, asset.id).status == "pending" # Retry converts from it again
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
def test_the_worker_converts_it_and_keeps_only_the_safe_file(
|
||||
self, db, make_user, _root, monkeypatch
|
||||
):
|
||||
# Real bytes, real probe, real ffmpeg: a 10-bit H.264 + Opus Matroska, the kind of file
|
||||
# that reached the share page as-is before this rule.
|
||||
src = _root / "src.mkv"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-y", "-loglevel", "error",
|
||||
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=1",
|
||||
"-f", "lavfi", "-i", "sine=duration=1",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p10le", "-c:a", "libopus", str(src),
|
||||
],
|
||||
check=True,
|
||||
) # fmt: skip
|
||||
job = uploadmod.ingest_upload(
|
||||
db, make_user().id, filename="holiday.mkv", src_path=src, size=src.stat().st_size
|
||||
)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
original = storage.abs_path(str(_root), asset.rel_path)
|
||||
assert job.status == "queued"
|
||||
job.status, asset.status = "running", "downloading" # what the worker's claim does
|
||||
db.commit()
|
||||
|
||||
pipeline._convert_upload(job.id, asset.id)
|
||||
|
||||
db.expire_all()
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert db.get(DownloadJob, job.id).status == "done"
|
||||
assert (asset.status, asset.container, asset.vcodec, asset.acodec) == (
|
||||
"ready",
|
||||
"mp4",
|
||||
"h264",
|
||||
"aac",
|
||||
)
|
||||
final = storage.abs_path(str(_root), asset.rel_path)
|
||||
assert final.suffix == ".mp4" and final.exists()
|
||||
assert asset.size_bytes == final.stat().st_size
|
||||
assert not original.exists() # one file per upload
|
||||
assert not playable.plan(probemod.probe_json(final), final.suffix).needs_change
|
||||
|
||||
Reference in New Issue
Block a user