fix(downloads): a download larger than the space left says so, not "processing failed"

yt-dlp refuses a file announced over `max_filesize` by logging one line and returning with no file (measured, and pinned in test_formats against a local server): the worker found no output and failed the job as `postprocess`. `formats.watching_size_cap` now sees that line through our yt-dlp logger, and the download (and, should it ever happen, a YouTube clip's range fetch) fails as the owner's refusal with DOWNLOAD_TOO_BIG_MESSAGE, classified `quota_exceeded`. The download's quota copy says it does not fit in what is left — true at a full cap too — instead of "Storage full". A pre-existing type error the change touched is fixed (baseline down by one).
2026-10-10 21:56:47 +02:00
parent 977ba244fa
commit e1d325b942
11 changed files with 195 additions and 34 deletions
@@ -37,9 +37,9 @@ from app.downloads.credentials import host_key_for_url

 # Worker-authored strings that this module owns because each is coupled to `_RULES`.
 #
-# QUOTA_FULL_MESSAGE, CLIP_TOO_BIG_MESSAGE and CONVERTED_TOO_BIG_MESSAGE are raised by the worker and
-# must classify back to `quota_exceeded`; their rules below match a fragment of each, and
-# test_download_errors pins each one, because a message and its rule drifting apart is exactly how a
+# QUOTA_FULL_MESSAGE and the three *_TOO_BIG_MESSAGEs are raised by the worker and must classify
+# back to `quota_exceeded`; their rules below match a fragment of each, and test_download_errors pins
+# each one, because a message and its rule drifting apart is exactly how a
 # category silently becomes unreachable.
 #
 # NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
@@ -54,8 +54,11 @@ from app.downloads.credentials import host_key_for_url
 # constants, imported above; each has a rule below that gives its category back, and a catalog entry
 # that gives the site's sentence.
 QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
-# A cut clip, or an upload's converted file, held to the space left, which is not zero: "full"
-# would be untrue. Both classify by the one phrase they share.
+# A download, a cut clip, or an upload's converted file, held to the space left, which is not zero:
+# "full" would be untrue. All three classify by the one phrase they share.
+DOWNLOAD_TOO_BIG_MESSAGE = (
+    "The video is larger than the download storage you have left — free up space, then try again."
+)
 CLIP_TOO_BIG_MESSAGE = (
    "The clip is larger than the download storage you have left — "
    "free up space or cut a shorter clip, then try again."
@@ -105,6 +105,28 @@ def log_safe(text: object) -> str:
    return _PATH_RE.sub("…", text)


+# yt-dlp's HTTP downloader, refusing a file announced larger than `max_filesize`, logs this line and
+# RETURNS — no exception and no file (measured; test_formats holds it against the pinned yt-dlp).
+_OVER_CAP_LINE = "File is larger than max-filesize"
+_OVER_CAP: contextvars.ContextVar[list[bool] | None] = contextvars.ContextVar(
+    "formats_over_cap", default=None
+)
+
+
+@contextlib.contextmanager
+def watching_size_cap() -> Iterator[list[bool]]:
+    """While yt-dlp works inside it: a one-slot flag, set when it refused a file over the size cap
+    (`build_ydl_opts(max_bytes=…)`). It then returns as if it had downloaded, and all the caller
+    would find is no file — a processing fault, to anyone reading it, for what is the owner's
+    storage. A context variable, like `hiding`: yt-dlp's logger calls run on the extraction's thread."""
+    flag = [False]
+    token = _OVER_CAP.set(flag)
+    try:
+        yield flag
+    finally:
+        _OVER_CAP.reset(token)
+
+
 class _YdlLog:
    """yt-dlp's logger protocol, pointed at ours.

@@ -124,6 +146,8 @@ class _YdlLog:
    is the one line whose level should say so."""

    def _emit(self, level: int, msg: str) -> None:
+        if _OVER_CAP_LINE in msg and (over := _OVER_CAP.get()) is not None:
+            over[0] = True
        # The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
        # `[download]  53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
        # a record that has args, this way the question never arises — and it is the shape ruff's
@@ -535,8 +535,8 @@ def _download(
        # free space, which is what the `quota_exceeded` category tells them.
        if remaining_bytes == 0:
            raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
-        info = _media_info(
-            _extract_with_fallback(
+        with formats.watching_size_cap() as over_cap:
+            fetched = _extract_with_fallback(
                job_id,
                asset_id,
                url,
@@ -546,7 +546,11 @@ def _download(
                cookiefile=str(cookiefile) if cookiefile else None,
                max_bytes=remaining_bytes,
            )
-        )
+        info = _media_info(dict(fetched))
+        # yt-dlp refused a file larger than what is left: it returns with no file, which would
+        # otherwise read as a processing fault. The owner's storage, so the owner's refusal.
+        if over_cap[0]:
+            raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
        # Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
        # identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
        adapter = adapters.resolve_adapter(source_kind, source_ref, info)
@@ -959,17 +963,21 @@ def _clip_source(
                )
        _set_job(job_id, phase="downloading", progress=0)
        try:
-            info = _extract_with_fallback(
-                job_id,
-                asset_id,
-                url,
-                formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
-                staging,
-                adapters.resolve_adapter("youtube", spec["ref"]),
-                cookiefile=str(cookiefile) if cookiefile else None,
-                max_bytes=remaining,
-                section=(spec["start_s"], spec["end_s"]),
-            )
+            with formats.watching_size_cap() as over_cap:
+                info = _extract_with_fallback(
+                    job_id,
+                    asset_id,
+                    url,
+                    formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
+                    staging,
+                    adapters.resolve_adapter("youtube", spec["ref"]),
+                    cookiefile=str(cookiefile) if cookiefile else None,
+                    max_bytes=remaining,
+                    section=(spec["start_s"], spec["end_s"]),
+                )
+            # Should yt-dlp's own cap ever refuse the range (see `_download`): the owner's storage.
+            if over_cap[0]:
+                raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
        except (_Aborted, _OwnerRefused):
            raise
        except Exception as exc:  # re-raised below with the download catalog's diagnosis