peter
18c11ca437
A failed Facebook download read "Download failed": yt-dlp words the login wall (18+ reel, closed group) as "Cannot parse data", which no category matched. Measured on dev with the owner's cookie before writing a rule — the same text is the login wall without a cookie and a refused cookie with one. - errors.diagnose(msg, url, cookie_attached) tries a host-scoped catalog first (all-of markers, cookie state, link fragment) and falls back to the category. First entries: facebook.login_wall, facebook.cookie_rejected, facebook.group_post_video. The module doc is the recipe for the next entry. - download_jobs.error_detail (0084) + backfill of the stored Facebook parse failures that were unknown. - The pipeline's failure branch passes the link and whether the cookie went with it; resume and redownload clear code and detail (redownload never cleared error_code). - A job meeting an already-failed asset got the text but no code, so it read "Download failed" where the first job said why. It is diagnosed again, with the cookie state from credentials.is_authed_asset (now shared with the cookie gate); the redacted network text classifies back to network. - FE: failCopyKeys gives the row and the failure window one lookup — catalog copy, else the category. Copy in EN and HU; a contract test pins every backend entry to both locales.
142 lines
7.4 KiB
Python
142 lines
7.4 KiB
Python
"""Category classification of raw yt-dlp / ffmpeg download-failure strings (S2)."""
|
|
|
|
from app.downloads import errors as dlerrors
|
|
from app.downloads.errors import classify
|
|
|
|
|
|
def test_real_ytdlp_messages_map_to_categories():
|
|
# Verbatim-shaped yt-dlp / ffmpeg output → the category the UI explains.
|
|
cases = {
|
|
"ERROR: [youtube] xxxxxxxxxxx: Video unavailable": "unavailable",
|
|
"ERROR: [youtube] abc: Private video. Sign in if you've been granted access": "unavailable",
|
|
"ERROR: [youtube] abc: This video is no longer available because the uploader has closed their "
|
|
"account": "unavailable",
|
|
"ERROR: [youtube] abc: Sign in to confirm your age. This video may be inappropriate": "login_required",
|
|
"ERROR: [youtube] abc: Sign in to confirm you're not a bot": "login_required",
|
|
"ERROR: [youtube] abc: Join this channel to get access to members-only content": "login_required",
|
|
"ERROR: [Reddit] 1vi1x56: Account authentication is required.": "login_required",
|
|
# Closed-group / private-post auth walls: yt-dlp either says to sign in or suggests cookies.
|
|
"ERROR: [facebook] 123: You must be logged in to see this post": "login_required",
|
|
"ERROR: [facebook] 123: Login required. Use --cookies for the authentication.": "login_required",
|
|
"ERROR: [instagram] abc: This content is only available for registered users": "login_required",
|
|
"ERROR: [youtube] abc: Sign in to confirm... Use --cookies-from-browser or --cookies": "login_required",
|
|
# A site yt-dlp has no extractor for at all — distinct from an auth wall, still non-retryable.
|
|
"ERROR: Unsupported URL: https://example.com/whatever": "unsupported_host",
|
|
"ERROR: [youtube] abc: The uploader has not made this video available in your country": "geo_blocked",
|
|
"ERROR: [youtube] abc: Requested format is not available": "format_unavailable",
|
|
"ERROR: [youtube] abc: Only images are available for download": "format_unavailable",
|
|
"ERROR: [generic] This video is DRM protected": "drm",
|
|
"ffmpeg failed (1): Conversion failed!": "postprocess",
|
|
"The source download is no longer available.": "source_missing",
|
|
"The source file is missing.": "source_missing",
|
|
"ERROR: unable to download video data: HTTP Error 403: Forbidden": "network",
|
|
"ERROR: Unable to download webpage: <urlopen error timed out>": "network",
|
|
}
|
|
for msg, expected in cases.items():
|
|
assert classify(msg) == expected, f"{msg!r} → {classify(msg)!r}, expected {expected!r}"
|
|
|
|
|
|
def test_unmatched_and_empty_are_unknown():
|
|
assert classify("some totally novel failure text") == "unknown"
|
|
assert classify("") == "unknown"
|
|
assert classify(None) == "unknown"
|
|
|
|
|
|
def test_matching_is_case_insensitive():
|
|
assert classify("VIDEO UNAVAILABLE") == "unavailable"
|
|
assert classify("drm protected") == "drm"
|
|
|
|
|
|
def test_login_beats_generic_unavailable_when_both_present():
|
|
# A members-only video also reads as "unavailable" to a signed-out client; the sign-in cause is
|
|
# the more useful, more specific one, so it must win (rule order guards this).
|
|
msg = (
|
|
"ERROR: [youtube] abc: Join this channel to get access to members-only content. "
|
|
"This video is unavailable"
|
|
)
|
|
assert classify(msg) == "login_required"
|
|
|
|
|
|
def test_our_own_worker_refusals_classify_as_themselves():
|
|
# The worker raises these itself (not yt-dlp), and the rule that catches each one matches only a
|
|
# fragment of the text. Pinning message and rule together is what stops an edit to the wording
|
|
# from silently orphaning the category — the UI would then fall back to "Download failed".
|
|
assert classify(dlerrors.QUOTA_FULL_MESSAGE) == "quota_exceeded"
|
|
|
|
|
|
def test_the_redacted_network_text_classifies_back_to_network():
|
|
# The redacted text is also what a failed asset keeps, and a second job that meets that asset is
|
|
# diagnosed from it — without its own rule it would read "Download failed".
|
|
assert classify(dlerrors.NETWORK_REDACTED_MESSAGE) == "network"
|
|
|
|
|
|
# Verbatim yt-dlp 2026.07.04 output from prod jobs, measured on dev 2026-09-13 (E11.5).
|
|
_FB_PARSE = (
|
|
"ERROR: [facebook] 1736360027626346: Cannot parse data; please report this issue on "
|
|
"https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template."
|
|
)
|
|
_FB_NO_FORMATS = (
|
|
"ERROR: [facebook] 1755988225525547: No video formats found!; please report this issue on "
|
|
"https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template."
|
|
)
|
|
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
|
_GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?"
|
|
|
|
|
|
def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie():
|
|
for url in (
|
|
_REEL,
|
|
_GROUP_POST,
|
|
"https://fb.watch/Ia_iJYI2Vw/?fs=e",
|
|
"https://m.facebook.com/x",
|
|
):
|
|
assert dlerrors.diagnose(_FB_PARSE, url=url) == ("login_required", "facebook.login_wall")
|
|
|
|
|
|
def test_facebook_cannot_parse_data_with_a_cookie_is_the_cookie_refused():
|
|
# Telling a user who already gave a cookie to add one is the failure this entry exists to avoid.
|
|
assert dlerrors.diagnose(_FB_PARSE, url=_REEL, cookie_attached=True) == (
|
|
"login_required",
|
|
"facebook.cookie_rejected",
|
|
)
|
|
|
|
|
|
def test_a_facebook_group_post_without_formats_names_the_link():
|
|
assert dlerrors.diagnose(_FB_NO_FORMATS, url=_GROUP_POST, cookie_attached=True) == (
|
|
"format_unavailable",
|
|
"facebook.group_post_video",
|
|
)
|
|
# A reel with no formats is another failure: only the category.
|
|
assert dlerrors.diagnose(_FB_NO_FORMATS, url=_REEL, cookie_attached=True) == (
|
|
"format_unavailable",
|
|
None,
|
|
)
|
|
|
|
|
|
def test_a_catalog_entry_never_leaves_its_host():
|
|
# "Cannot parse data" is yt-dlp's wording for a broken extractor on every site.
|
|
other = _FB_PARSE.replace("[facebook]", "[vimeo]")
|
|
assert dlerrors.diagnose(other, url="https://vimeo.com/1") == ("unknown", None)
|
|
# The Facebook text for a link that is not Facebook's (a look-alike host) is not the Facebook wall.
|
|
assert dlerrors.diagnose(_FB_PARSE, url="https://facebook.com.evil/reel/1") == ("unknown", None)
|
|
# No link known: the category only.
|
|
assert dlerrors.diagnose(_FB_PARSE) == ("unknown", None)
|
|
|
|
|
|
def test_diagnose_falls_back_to_the_category():
|
|
msg = "ERROR: [facebook] 123: You must be logged in to see this post"
|
|
assert dlerrors.diagnose(msg, url=_REEL) == ("login_required", None)
|
|
assert dlerrors.diagnose(None, url=_REEL) == ("unknown", None)
|
|
|
|
|
|
def test_a_url_jobs_network_failure_is_redacted_but_youtube_keeps_its_text():
|
|
# SEC-12: the raw transport text for a user-chosen host:port is a port-scanner readout, so it is
|
|
# replaced for `url` jobs. Everything else must survive verbatim, or a real diagnosis is lost.
|
|
raw = "ERROR: Unable to download webpage: <urlopen error [Errno 111] Connection refused>"
|
|
assert dlerrors.user_error_text("url", raw, "network") == dlerrors.NETWORK_REDACTED_MESSAGE
|
|
# Same failure, but the destination was never user-chosen — nothing to leak, keep the detail.
|
|
assert dlerrors.user_error_text("youtube", raw, "network") == raw
|
|
# A user-chosen host, but a category that says something real about the CONTENT, not the network.
|
|
login = "ERROR: [facebook] 123: You must be logged in to see this post"
|
|
assert dlerrors.user_error_text("url", login, "login_required") == login
|