peter
e1d325b942
yt-dlp refuses a file announced over `max_filesize` by logging one line and returning with no file (measured, and pinned in test_formats against a local server): the worker found no output and failed the job as `postprocess`. `formats.watching_size_cap` now sees that line through our yt-dlp logger, and the download (and, should it ever happen, a YouTube clip's range fetch) fails as the owner's refusal with DOWNLOAD_TOO_BIG_MESSAGE, classified `quota_exceeded`. The download's quota copy says it does not fit in what is left — true at a full cap too — instead of "Storage full". A pre-existing type error the change touched is fixed (baseline down by one).
329 lines
17 KiB
Python
329 lines
17 KiB
Python
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code — and, where a site's
|
|
failure has been measured, into that site's entry of the failure catalog.
|
|
|
|
The worker stores the raw exception text (truncated) for technical detail, but a raw
|
|
`ERROR: [youtube] <id>: Private video. Sign in…` is noise to a user. `classify()` maps that text
|
|
to one of a small set of category codes; the frontend turns the code into a localized, human
|
|
explanation (and decides whether a retry makes sense). The raw text stays available behind a
|
|
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
|
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
|
|
|
**The failure catalog** (`_CATALOG`). A category is generic: "Sign-in required" cannot tell a
|
|
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
|
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
|
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
|
its own sentence for — why it failed on that site, and what the user can do. `diagnose()` tries the
|
|
catalog first and falls back to the category. How an entry is added, every time a failure is reported:
|
|
1. reproduce it on dev (with and without the user's cookie) and keep the verbatim yt-dlp text;
|
|
2. add the entry below, scoped as narrowly as the measurement allows (a marker that is yt-dlp's
|
|
generic wording for every site must never match without its host; a marker another module also
|
|
decides on — the Facebook Chrome retry's FACEBOOK_PARSE_WALL — is imported, never retyped);
|
|
3. pin the real text in `tests/test_download_errors.py`;
|
|
4. add `downloads.errors.fail.catalog.<detail>.{short,body}` in EN and HU (a contract test checks it;
|
|
the group is not called `detail`, which `actionErrors.contract` reads as a raw `HttpError.detail`).
|
|
"""
|
|
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from typing import NamedTuple
|
|
|
|
from app.downloads.adapters import (
|
|
FACEBOOK_PARSE_WALL,
|
|
POST_NOT_SHOWN_MESSAGE,
|
|
POST_WITHOUT_VIDEO_MESSAGE,
|
|
facebook_group_post_id,
|
|
)
|
|
from app.downloads.credentials import host_key_for_url
|
|
|
|
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
|
#
|
|
# QUOTA_FULL_MESSAGE and the three *_TOO_BIG_MESSAGEs are raised by the worker and must classify
|
|
# back to `quota_exceeded`; their rules below match a fragment of each, and test_download_errors pins
|
|
# each one, because a message and its rule drifting apart is exactly how a
|
|
# category silently becomes unreachable.
|
|
#
|
|
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
|
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
|
# text (`worker._process_job`), so it has a rule of its own that gives the category back.
|
|
#
|
|
# Other worker-raised texts are still inline literals at their raise sites. The ffmpeg ones and
|
|
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
|
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
|
#
|
|
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`) are adapters.py's
|
|
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
|
# that gives the site's sentence.
|
|
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
|
# A download, a cut clip, or an upload's converted file, held to the space left, which is not zero:
|
|
# "full" would be untrue. All three classify by the one phrase they share.
|
|
DOWNLOAD_TOO_BIG_MESSAGE = (
|
|
"The video is larger than the download storage you have left — free up space, then try again."
|
|
)
|
|
CLIP_TOO_BIG_MESSAGE = (
|
|
"The clip is larger than the download storage you have left — "
|
|
"free up space or cut a shorter clip, then try again."
|
|
)
|
|
CONVERTED_TOO_BIG_MESSAGE = (
|
|
"The converted file is larger than the download storage you have left — "
|
|
"free up space, then try again."
|
|
)
|
|
UPLOAD_MISSING_MESSAGE = "The uploaded file is missing."
|
|
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
|
|
|
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
|
# case-insensitive on a substring, against the raw yt-dlp/ffmpeg message.
|
|
_RULES: tuple[tuple[str, str], ...] = (
|
|
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
|
# first — it is our wording, so it can never be a false match on an upstream message.
|
|
("download storage is full", "quota_exceeded"),
|
|
("larger than the download storage you have left", "quota_exceeded"),
|
|
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
|
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
|
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
|
# or the site answered the signed-in read with something that is not the post at all.
|
|
(POST_WITHOUT_VIDEO_MESSAGE.lower(), "format_unavailable"),
|
|
(POST_NOT_SHOWN_MESSAGE.lower(), "login_required"),
|
|
# Login / age / members / bot-check — the user must be signed in (we deliberately are not).
|
|
("sign in to confirm your age", "login_required"),
|
|
("confirm your age", "login_required"),
|
|
("sign in to confirm you're not a bot", "login_required"),
|
|
("not a bot", "login_required"),
|
|
("sign in to confirm", "login_required"),
|
|
("members-only", "login_required"),
|
|
("available to this channel's members", "login_required"),
|
|
("join this channel", "login_required"),
|
|
("this video is only available to", "login_required"),
|
|
# Some extractors (e.g. Reddit) gate content behind a logged-in account: "Account
|
|
# authentication is required." — an auth wall we deliberately don't cross, not a transient error.
|
|
("authentication is required", "login_required"),
|
|
# Closed-group / private-post auth walls (Facebook groups, Instagram private, …). yt-dlp phrases
|
|
# these many ways; the common thread is "you must be signed in" or its own suggestion to supply
|
|
# cookies. Treating the cookie suggestion as an auth signal is what makes a login-walled link read
|
|
# honestly (and stay non-retryable) when the acting user has no usable cookie for that host.
|
|
("log in to", "login_required"),
|
|
("login required", "login_required"),
|
|
("you must be logged in", "login_required"),
|
|
("only available for registered users", "login_required"),
|
|
("use --cookies", "login_required"),
|
|
("--cookies-from-browser", "login_required"),
|
|
# Geo-restricted.
|
|
("in your country", "geo_blocked"),
|
|
("available in your location", "geo_blocked"),
|
|
("not available from your location", "geo_blocked"),
|
|
("geo restricted", "geo_blocked"),
|
|
# An edit's source download went away — our own wording, and MORE specific than the generic
|
|
# "no longer available" below, so it must be matched first.
|
|
("source download is no longer available", "source_missing"),
|
|
# An upload's original went away before its conversion ran — our wording (UPLOAD_MISSING_MESSAGE).
|
|
(UPLOAD_MISSING_MESSAGE.lower().rstrip("."), "upload_missing"),
|
|
("source file is missing", "source_missing"),
|
|
# Private / deleted / removed / terminated.
|
|
("private video", "unavailable"),
|
|
("video unavailable", "unavailable"),
|
|
("no longer available", "unavailable"),
|
|
("has been removed", "unavailable"),
|
|
("removed by the uploader", "unavailable"),
|
|
("account associated with this video has been terminated", "unavailable"),
|
|
("this video is not available", "unavailable"),
|
|
("this video has been removed", "unavailable"),
|
|
# yt-dlp doesn't support this site at all ("Unsupported URL: …") — distinct from an auth wall on a
|
|
# supported site. Non-retryable: no cookie or retry can make an unsupported host downloadable.
|
|
("unsupported url", "unsupported_host"),
|
|
("no video could be found in this url", "unsupported_host"),
|
|
# DRM.
|
|
("drm protected", "drm"),
|
|
("drm-protected", "drm"),
|
|
# No usable media format.
|
|
("requested format is not available", "format_unavailable"),
|
|
("only images are available", "format_unavailable"),
|
|
("no video formats found", "format_unavailable"),
|
|
("unable to extract", "format_unavailable"),
|
|
# Our own post-processing (ffmpeg) step failed.
|
|
("ffmpeg failed", "postprocess"),
|
|
("ffmpeg stalled", "postprocess"),
|
|
("postprocessing", "postprocess"),
|
|
("produced no output file", "postprocess"),
|
|
("no output file", "postprocess"),
|
|
# Network / transport.
|
|
("unable to download", "network"),
|
|
("http error", "network"),
|
|
("timed out", "network"),
|
|
("read timed out", "network"),
|
|
("temporary failure in name resolution", "network"),
|
|
("connection reset", "network"),
|
|
("connection refused", "network"),
|
|
("failed to resolve", "network"),
|
|
)
|
|
|
|
|
|
class Diagnosis(NamedTuple):
|
|
"""What a failure was: the generic `code` (category) and, when a catalog entry matched, its
|
|
host-specific `detail`."""
|
|
|
|
code: str
|
|
detail: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _Entry:
|
|
"""One measured failure of one site. It matches when the download URL belongs to `host`
|
|
(`credentials.SUPPORTED_HOSTS`), every one of `markers` is in the message, the cookie state is
|
|
`cookie` (None: either), and — when `link` is given — it recognises the URL (a truthy result). A
|
|
link predicate is the adapter's own, so the catalog and the download agree on what a link is."""
|
|
|
|
host: str
|
|
markers: tuple[str, ...]
|
|
code: str
|
|
detail: str
|
|
cookie: bool | None = None
|
|
link: Callable[[str], object] | None = None
|
|
|
|
|
|
_CATALOG: tuple[_Entry, ...] = (
|
|
# Facebook answers a signed-out client with a page yt-dlp cannot parse — an 18+ reel, a closed
|
|
# group's post (measured 2026-09-13: prod jobs 145, 156, 256 download once the owner's cookie is
|
|
# attached). "Cannot parse data" is yt-dlp's generic wording for a broken extractor on EVERY site,
|
|
# which is why the entry is scoped to Facebook and never becomes a general rule.
|
|
_Entry(
|
|
host="facebook",
|
|
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
|
code="login_required",
|
|
detail="facebook.login_wall",
|
|
cookie=False,
|
|
),
|
|
# The same text WITH a cookie: an expired / signed-out cookie, or a page that also needs
|
|
# impersonation (prod job 257). Telling this user to add cookies would be wrong — they have.
|
|
_Entry(
|
|
host="facebook",
|
|
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
|
code="login_required",
|
|
detail="facebook.cookie_rejected",
|
|
cookie=True,
|
|
),
|
|
# A group post read with a cookie that Facebook answered with its login page (measured with a
|
|
# refused cookie: no post id anywhere on the page) — the same refused cookie, found by the adapter.
|
|
_Entry(
|
|
host="facebook",
|
|
markers=(POST_NOT_SHOWN_MESSAGE.lower(),),
|
|
code="login_required",
|
|
detail="facebook.cookie_rejected",
|
|
cookie=True,
|
|
),
|
|
# A group post with no single video of its own. With a cookie the adapter reads the post's page
|
|
# and says so itself (POST_WITHOUT_VIDEO_MESSAGE); without one yt-dlp is given the post link and
|
|
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
|
|
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
|
|
# that video, not the post — the category says it. Only for group post links (the adapter's own
|
|
# test of one): a reel with no formats is a different failure.
|
|
_Entry(
|
|
host="facebook",
|
|
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
|
|
code="format_unavailable",
|
|
detail="facebook.group_post_video",
|
|
link=facebook_group_post_id,
|
|
),
|
|
_Entry(
|
|
host="facebook",
|
|
markers=("[facebook]", "no video formats found"),
|
|
code="format_unavailable",
|
|
detail="facebook.group_post_video",
|
|
cookie=False,
|
|
link=facebook_group_post_id,
|
|
),
|
|
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
|
# job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
|
|
# neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
|
|
# identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
|
|
# The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
|
|
# "you're not a bot" with a straight quote never matches the real text.
|
|
_Entry(
|
|
host="youtube",
|
|
markers=("[youtube]", "not a bot"),
|
|
code="login_required",
|
|
detail="youtube.bot_wall",
|
|
cookie=False,
|
|
),
|
|
# The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
|
|
# one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
|
|
# and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
|
|
# longer valid. They have likely been rotated in the browser as a security measure."
|
|
_Entry(
|
|
host="youtube",
|
|
markers=("[youtube]", "not a bot"),
|
|
code="login_required",
|
|
detail="youtube.cookie_rejected",
|
|
cookie=True,
|
|
),
|
|
# The same gate one step later: the player response comes back, and then the CDN refuses the
|
|
# stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
|
|
# very same video, ~90 minutes after it was answering with the bot message — signed out it
|
|
# 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
|
|
# the category this text carries generically, `network`, is a lie here: nothing was wrong with
|
|
# the network. Scoped to a cookie-less YouTube download, where that is what we measured.
|
|
_Entry(
|
|
host="youtube",
|
|
markers=("unable to download video data", "403"),
|
|
code="login_required",
|
|
detail="youtube.media_forbidden",
|
|
cookie=False,
|
|
),
|
|
)
|
|
|
|
|
|
def classify(msg: str | None) -> str:
|
|
"""Return a category code for a raw failure message (``"unknown"`` if nothing matches)."""
|
|
if not msg:
|
|
return "unknown"
|
|
low = msg.lower()
|
|
for marker, category in _RULES:
|
|
if marker in low:
|
|
return category
|
|
return "unknown"
|
|
|
|
|
|
def classify_conversion(msg: str | None) -> str:
|
|
"""The category of an upload's failed conversion. The generic ffmpeg categories speak of a
|
|
download ("the video downloaded but…"), which an upload never was: a failed encode or an
|
|
unrecognised failure of one is `convert_failed`; a category of its own (a full quota, a missing
|
|
original) stays as it is."""
|
|
code = classify(msg)
|
|
return "convert_failed" if code in ("postprocess", "unknown") else code
|
|
|
|
|
|
def diagnose(
|
|
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
|
) -> Diagnosis:
|
|
"""The catalog entry for this failure of this link, else just its category (see the module doc).
|
|
|
|
`url` is the address yt-dlp was given; `cookie_attached` whether the acting user's cookie went
|
|
with it. Without a URL only the category can be known."""
|
|
host = host_key_for_url(url) if msg and url else None
|
|
if host is not None:
|
|
low = msg.lower() # type: ignore[union-attr] — `host` is only set when `msg` is truthy
|
|
for e in _CATALOG:
|
|
if (
|
|
e.host == host
|
|
and all(m in low for m in e.markers)
|
|
and (e.cookie is None or e.cookie == cookie_attached)
|
|
and (e.link is None or e.link(url)) # type: ignore[arg-type] — as `msg` above
|
|
):
|
|
return Diagnosis(e.code, e.detail)
|
|
return Diagnosis(classify(msg))
|
|
|
|
|
|
def user_error_text(source_kind: str, msg: str, code: str) -> str:
|
|
"""The failure text to STORE on the job, given the raw message and its category.
|
|
|
|
SEC-12: for a user-supplied URL, a transport-level failure text distinguishes "connection
|
|
refused" from "HTTP Error 401" from "timed out" per host:port — which is a port scanner with
|
|
extra steps, since the raw text is shown to the user behind the details disclosure. `urlguard`
|
|
stops the fetch aiming at our own network; this stops the remaining readout.
|
|
|
|
Only the network class, and only for ``url`` jobs: a YouTube job has no user-chosen host, and
|
|
every other class (login_required, geo_blocked, drm, …) is real, useful diagnosis. A separate
|
|
function rather than an inline branch in the worker so it can be pinned by a test — the raw
|
|
text leaking back is a security regression that would otherwise pass a green suite."""
|
|
if source_kind == "url" and code == "network":
|
|
return NETWORK_REDACTED_MESSAGE
|
|
return msg
|