feat(downloads): categorize download failures + a readable error dialog (S2)
Failed downloads stored the raw yt-dlp/ffmpeg text and dumped it as a truncated red one-liner in the queue. Classify the failure into a category (unavailable / login_required / geo_blocked / format_unavailable / drm / postprocess / source_missing / network / unknown) and surface it properly: - backend: app/downloads/errors.classify maps the raw message to a category code; the worker stores it in the new download_jobs.error_code (migration 0061) at both failure sites; resume clears it; the serializer exposes it. Raw text kept as technical detail. - frontend: the queue row shows a clean localized reason + a "details" link opening a modal with the explanation, the raw text behind a disclosure, and a Retry only for transient categories (network/postprocess/unknown). - tests: classify() unit coverage over verbatim yt-dlp/ffmpeg message shapes.
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
"""add download_jobs.error_code — a category for a failed download (S2)
|
||||
|
||||
The worker already stores the raw yt-dlp/ffmpeg failure text in `download_jobs.error`. This adds a
|
||||
small category code (`unavailable` / `login_required` / `geo_blocked` / `format_unavailable` /
|
||||
`drm` / `postprocess` / `source_missing` / `network` / `unknown`) computed by
|
||||
`app.downloads.errors.classify`, so the UI can show a clean localized explanation and decide whether
|
||||
a retry makes sense, keeping the raw text as a technical-detail disclosure. Nullable; only set on the
|
||||
error path. Forward-only — an existing errored row simply has NULL until its next failure/retry.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0061_dl_job_error_code"
|
||||
down_revision: Union[str, None] = "0060_plex_jsonb_null_normalize"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("error_code", sa.String(length=24), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "error_code")
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code.
|
||||
|
||||
The worker stores the raw exception text (truncated) for technical detail, but a raw
|
||||
`ERROR: [youtube] <id>: Private video. Sign in…` is noise to a user. `classify()` maps that text
|
||||
to one of a small set of category codes; the frontend turns the code into a localized, human
|
||||
explanation (and decides whether a retry makes sense). The raw text stays available behind a
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
"""
|
||||
|
||||
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
||||
# case-insensitive on a substring, against the raw yt-dlp/ffmpeg message.
|
||||
_RULES: tuple[tuple[str, str], ...] = (
|
||||
# Login / age / members / bot-check — the user must be signed in (we deliberately are not).
|
||||
("sign in to confirm your age", "login_required"),
|
||||
("confirm your age", "login_required"),
|
||||
("sign in to confirm you're not a bot", "login_required"),
|
||||
("not a bot", "login_required"),
|
||||
("sign in to confirm", "login_required"),
|
||||
("members-only", "login_required"),
|
||||
("available to this channel's members", "login_required"),
|
||||
("join this channel", "login_required"),
|
||||
("this video is only available to", "login_required"),
|
||||
# Geo-restricted.
|
||||
("in your country", "geo_blocked"),
|
||||
("available in your location", "geo_blocked"),
|
||||
("not available from your location", "geo_blocked"),
|
||||
("geo restricted", "geo_blocked"),
|
||||
# An edit's source download went away — our own wording, and MORE specific than the generic
|
||||
# "no longer available" below, so it must be matched first.
|
||||
("source download is no longer available", "source_missing"),
|
||||
("source file is missing", "source_missing"),
|
||||
# Private / deleted / removed / terminated.
|
||||
("private video", "unavailable"),
|
||||
("video unavailable", "unavailable"),
|
||||
("no longer available", "unavailable"),
|
||||
("has been removed", "unavailable"),
|
||||
("removed by the uploader", "unavailable"),
|
||||
("account associated with this video has been terminated", "unavailable"),
|
||||
("this video is not available", "unavailable"),
|
||||
("this video has been removed", "unavailable"),
|
||||
# DRM.
|
||||
("drm protected", "drm"),
|
||||
("drm-protected", "drm"),
|
||||
# No usable media format.
|
||||
("requested format is not available", "format_unavailable"),
|
||||
("only images are available", "format_unavailable"),
|
||||
("no video formats found", "format_unavailable"),
|
||||
("unable to extract", "format_unavailable"),
|
||||
# Our own post-processing (ffmpeg) step failed.
|
||||
("ffmpeg failed", "postprocess"),
|
||||
("ffmpeg stalled", "postprocess"),
|
||||
("postprocessing", "postprocess"),
|
||||
("produced no output file", "postprocess"),
|
||||
("no output file", "postprocess"),
|
||||
# Network / transport.
|
||||
("unable to download", "network"),
|
||||
("http error", "network"),
|
||||
("timed out", "network"),
|
||||
("read timed out", "network"),
|
||||
("temporary failure in name resolution", "network"),
|
||||
("connection reset", "network"),
|
||||
("connection refused", "network"),
|
||||
("failed to resolve", "network"),
|
||||
)
|
||||
|
||||
|
||||
def classify(msg: str | None) -> str:
|
||||
"""Return a category code for a raw failure message (``"unknown"`` if nothing matches)."""
|
||||
if not msg:
|
||||
return "unknown"
|
||||
low = msg.lower()
|
||||
for marker, category in _RULES:
|
||||
if marker in low:
|
||||
return category
|
||||
return "unknown"
|
||||
@@ -877,7 +877,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||
phase: Mapped[str | None] = mapped_column(String(32))
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||
error_code: Mapped[str | None] = mapped_column(String(24)) # category, see downloads.errors.classify
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
|
||||
@@ -81,6 +81,7 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"eta_s": job.eta_s,
|
||||
"phase": job.phase,
|
||||
"error": job.error,
|
||||
"error_code": job.error_code,
|
||||
"queue_pos": job.queue_pos,
|
||||
"created_at": iso(job.created_at),
|
||||
"source_kind": job.source_kind,
|
||||
@@ -468,6 +469,7 @@ def resume_download(
|
||||
if job.status in ("paused", "error"):
|
||||
job.progress = 0
|
||||
job.error = None
|
||||
job.error_code = None
|
||||
# If the shared asset itself failed, reset it to pending so the worker actually
|
||||
# re-downloads — otherwise the requeued job would instantly re-inherit the asset's
|
||||
# stale error (the worker short-circuits a job whose asset is already errored).
|
||||
|
||||
@@ -31,6 +31,7 @@ from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.titles import normalize_title
|
||||
@@ -654,6 +655,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
if job:
|
||||
job.status = "error"
|
||||
job.error = msg
|
||||
job.error_code = dlerrors.classify(msg)
|
||||
job.phase = None
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
@@ -685,6 +687,7 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
if job:
|
||||
job.status = "error"
|
||||
job.error = msg
|
||||
job.error_code = dlerrors.classify(msg)
|
||||
job.phase = None
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Category classification of raw yt-dlp / ffmpeg download-failure strings (S2)."""
|
||||
from app.downloads.errors import classify
|
||||
|
||||
|
||||
def test_real_ytdlp_messages_map_to_categories():
|
||||
# Verbatim-shaped yt-dlp / ffmpeg output → the category the UI explains.
|
||||
cases = {
|
||||
"ERROR: [youtube] xxxxxxxxxxx: Video unavailable": "unavailable",
|
||||
"ERROR: [youtube] abc: Private video. Sign in if you've been granted access": "unavailable",
|
||||
"ERROR: [youtube] abc: This video is no longer available because the uploader has closed their "
|
||||
"account": "unavailable",
|
||||
"ERROR: [youtube] abc: Sign in to confirm your age. This video may be inappropriate": "login_required",
|
||||
"ERROR: [youtube] abc: Sign in to confirm you're not a bot": "login_required",
|
||||
"ERROR: [youtube] abc: Join this channel to get access to members-only content": "login_required",
|
||||
"ERROR: [youtube] abc: The uploader has not made this video available in your country": "geo_blocked",
|
||||
"ERROR: [youtube] abc: Requested format is not available": "format_unavailable",
|
||||
"ERROR: [youtube] abc: Only images are available for download": "format_unavailable",
|
||||
"ERROR: [generic] This video is DRM protected": "drm",
|
||||
"ffmpeg failed (1): Conversion failed!": "postprocess",
|
||||
"The source download is no longer available.": "source_missing",
|
||||
"The source file is missing.": "source_missing",
|
||||
"ERROR: unable to download video data: HTTP Error 403: Forbidden": "network",
|
||||
"ERROR: Unable to download webpage: <urlopen error timed out>": "network",
|
||||
}
|
||||
for msg, expected in cases.items():
|
||||
assert classify(msg) == expected, f"{msg!r} → {classify(msg)!r}, expected {expected!r}"
|
||||
|
||||
|
||||
def test_unmatched_and_empty_are_unknown():
|
||||
assert classify("some totally novel failure text") == "unknown"
|
||||
assert classify("") == "unknown"
|
||||
assert classify(None) == "unknown"
|
||||
|
||||
|
||||
def test_matching_is_case_insensitive():
|
||||
assert classify("VIDEO UNAVAILABLE") == "unavailable"
|
||||
assert classify("drm protected") == "drm"
|
||||
|
||||
|
||||
def test_login_beats_generic_unavailable_when_both_present():
|
||||
# A members-only video also reads as "unavailable" to a signed-out client; the sign-in cause is
|
||||
# the more useful, more specific one, so it must win (rule order guards this).
|
||||
msg = (
|
||||
"ERROR: [youtube] abc: Join this channel to get access to members-only content. "
|
||||
"This video is unavailable"
|
||||
)
|
||||
assert classify(msg) == "login_required"
|
||||
Reference in New Issue
Block a user