From 5b48018e920d27840cdc8200fda4adc5fe1674cb Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 10 Oct 2026 22:10:56 +0200 Subject: [PATCH] fix(downloads): a produced file is held to the space left by its real size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's fragmented formats never meet it — measured on localdev, a 718 MB download ran to the end with 1 MB left and put its owner four times over the cap. The download now re-reads what is left once its file is made (after the phone-safe pass) and refuses one larger than that as the owner's refusal; an editor clip, which nothing checked, is held the same way, as a player clip already was. --- backend/app/downloads/formats.py | 4 +- backend/app/downloads/pipeline.py | 14 ++++++ backend/tests/test_download_limits.py | 48 ++++++++++++++++++++ backend/tests/test_edit_cap.py | 63 +++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_edit_cap.py diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py index 0d957185..959753c9 100644 --- a/backend/app/downloads/formats.py +++ b/backend/app/downloads/formats.py @@ -307,7 +307,9 @@ def build_ydl_opts( `max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp's HTTP downloader refuses a file announced larger than it rather than filling the disk and only then being caught by the - quota accounting. Its ffmpeg downloader (a `download_ranges` fetch) does not read it.""" + quota accounting. Its ffmpeg downloader (a `download_ranges` fetch) does not read it, nor does + a fragmented download (YouTube's formats: measured, 718 MB fetched with 1 MB left) — so the + pipeline also holds every produced file to what is left by its real size.""" s = normalize(spec) h = s["max_height"] opts: dict = { diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index dee150a3..2b937438 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -563,6 +563,13 @@ def _download( CONVERTING, int(info["duration"]) if info.get("duration") else None, ) + # Held to the cap by its real size, as a clip is: yt-dlp's `max_filesize` only stops a plain + # HTTP file that announces its size, and YouTube's fragmented formats never meet it + # (measured: 718 MB fetched with 1 MB left). Read again — other jobs may have landed. + with SessionLocal() as qdb: + room = quota.remaining_bytes(qdb, user_id) + if room is not None and produced.stat().st_size > room: + raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE) ext = produced.suffix.lstrip(".").lower() # Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's # `info` dict, whose fields are absent for some sources (Facebook among them): deriving @@ -871,6 +878,13 @@ def _process_edit(job_id: int, asset_id: int) -> None: # playable as its source: hold it to the same rule as every other ready file. dest_staging = _ensure_phone_safe(job_id, dest_staging, staging, "editing", total_s) out_ext = dest_staging.suffix.lstrip(".").lower() or out_ext + # New storage on the owner's footprint, held to what is left by its real size, as a + # download and a player clip are. + if user_id is not None: + with SessionLocal() as qdb: + room = quota.remaining_bytes(qdb, user_id) + if room is not None and dest_staging.stat().st_size > room: + raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE) rel = storage.edit_rel_path(user_id, asset_id, out_ext) storage.place_file(dest_staging, settings.download_root, rel) diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py index 19545904..d2dfdc32 100644 --- a/backend/tests/test_download_limits.py +++ b/backend/tests/test_download_limits.py @@ -344,3 +344,51 @@ def test_a_download_over_the_space_left_says_so_and_releases_the_shared_asset( assert (job.status, job.error_code) == ("error", "quota_exceeded") assert job.error == dlerrors.DOWNLOAD_TOO_BIG_MESSAGE assert db.get(MediaAsset, shared.id).status == "pending" + + +def test_a_download_that_outgrows_the_space_left_unannounced_is_refused_and_not_placed( + db, monkeypatch, tmp_path +): + # yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's + # fragmented formats never meet it (measured on localdev: 718 MB fetched with 1 MB left). The + # file it produced is held to what is left by its real size, as a clip is. + monkeypatch.setattr(settings, "download_root", str(tmp_path)) + u = User(email="unannounced@example.com") + db.add(u) + db.flush() + db.add( + DownloadQuota(user_id=u.id, max_bytes=1000, max_concurrent=2, max_jobs=100, unlimited=False) + ) + held = _sized_asset(db, 400, "unannounced-held") + _job(db, u.id, queue_pos=0, status="done", asset_id=held.id) + shared = MediaAsset( + source_kind="youtube", + source_ref="z" * 11, + format_sig="s", + status="downloading", + ref_count=1, + ) + db.add(shared) + db.flush() + mine = _job(db, u.id, queue_pos=1, status="running", asset_id=shared.id) + db.commit() + + def fetched_past_the_cap(job_id, asset_id, url, spec, staging, adapter, **kw): + f = staging / "zzzzzzzzzzz.mp4" + f.write_bytes(b"v" * 2000) # 600 left + return {"id": "z" * 11, "requested_downloads": [{"filepath": str(f)}]} + + monkeypatch.setattr(pipeline, "_extract_with_fallback", fetched_past_the_cap) + monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced) + pipeline._download(mine.id, shared.id, "youtube", "z" * 11, {}, u.id) + + db.expire_all() + job = db.get(DownloadJob, mine.id) + assert (job.status, job.error_code, job.error) == ( + "error", + "quota_exceeded", + dlerrors.DOWNLOAD_TOO_BIG_MESSAGE, + ) + asset = db.get(MediaAsset, shared.id) + assert (asset.status, asset.rel_path) == ("pending", None) + assert not any(p.suffix == ".mp4" for p in tmp_path.rglob("*") if "staging" not in p.parts) diff --git a/backend/tests/test_edit_cap.py b/backend/tests/test_edit_cap.py new file mode 100644 index 00000000..8f78c28d --- /dev/null +++ b/backend/tests/test_edit_cap.py @@ -0,0 +1,63 @@ +"""An editor clip is held to its owner's storage cap by its real size, as a download and a player +clip are: its file is new storage on the owner's footprint, and nothing else checked it. + +Real ffmpeg — the size the check reads is the encoder's. +""" + +import shutil +import subprocess + +import pytest + +from app.config import settings +from app.downloads import errors as dlerrors +from app.downloads import pipeline, quota, service +from app.downloads import upload as uploadmod +from app.models import DownloadJob, DownloadQuota, MediaAsset + +pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") + + +def test_an_editor_clip_larger_than_what_is_left_is_refused_and_not_placed( + db, make_user, tmp_path, monkeypatch +): + monkeypatch.setattr(settings, "download_root", str(tmp_path)) + src = tmp_path / "in.mp4" + subprocess.run( + [ + "ffmpeg", "-y", "-loglevel", "error", + "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2", + "-c:v", "libx264", "-pix_fmt", "yuv420p", str(src), + ], + check=True, + ) # fmt: skip + u = make_user() + source = uploadmod.ingest_upload( + db, u.id, filename="clip.mp4", src_path=src, size=src.stat().st_size + ) + job = service.enqueue_edit(db, u.id, source, {"trim": {"start_s": 0, "end_s": 1.5}}) + asset = db.get(MediaAsset, job.asset_id) + job.status, asset.status = "running", "downloading" # the worker's claim + # Ten bytes left: no clip of a real video fits. + db.add( + DownloadQuota( + user_id=u.id, + max_bytes=quota.footprint(db, u.id) + 10, + max_concurrent=2, + max_jobs=100, + unlimited=False, + ) + ) + db.commit() + + pipeline._process_edit(job.id, asset.id) + + db.expire_all() + job, asset = db.get(DownloadJob, job.id), db.get(MediaAsset, job.asset_id) + assert (job.status, job.error_code, job.error) == ( + "error", + "quota_exceeded", + dlerrors.CLIP_TOO_BIG_MESSAGE, + ) + assert asset.rel_path is None + assert not list(tmp_path.glob(".edits/**/*.mp4"))