fix(plex): a stray byte keeps a UTF-8 subtitle UTF-8; untagged and Serbian read as cp1250
One invalid byte sent a whole UTF-8 sidecar to a legacy code page, an untagged file fell to Windows-1252 (ő/ű became õ/û) and Serbian was read as Cyrillic. The cache key moves to sub3_ so a track the old rules mis-decoded is read again.
This commit is contained in:
@@ -218,10 +218,11 @@ _FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can
|
||||
|
||||
|
||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||
# `sub2_`: versioned when sidecars in a legacy code page turned out to have been cached as
|
||||
# U+FFFD (`decode_subtitle`). The cache has no eviction, so every track is read again once
|
||||
# rather than a broken one being served for good; the old `sub_` files are just left behind.
|
||||
return f"sub2_{rating_key}_{sub_ord}"
|
||||
# Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code
|
||||
# pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went
|
||||
# to the wrong code page. The cache has no eviction, so every track is read again once rather
|
||||
# than a broken one being served for good; the older files are just left behind.
|
||||
return f"sub3_{rating_key}_{sub_ord}"
|
||||
|
||||
|
||||
def _vtt_path(key: str) -> Path:
|
||||
@@ -909,7 +910,8 @@ def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
||||
|
||||
|
||||
# The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely
|
||||
# saved in; anything not listed falls to Windows-1252.
|
||||
# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in
|
||||
# Windows-1250, like Croatian and Bosnian.
|
||||
_CP1250 = {
|
||||
"hun",
|
||||
"ces",
|
||||
@@ -920,19 +922,47 @@ _CP1250 = {
|
||||
"slv",
|
||||
"hrv",
|
||||
"bos",
|
||||
"srp",
|
||||
"ron",
|
||||
"rum",
|
||||
"sqi",
|
||||
"alb",
|
||||
}
|
||||
_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac", "srp"}
|
||||
_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"}
|
||||
# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to
|
||||
# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű
|
||||
# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian.
|
||||
_CP1252 = {
|
||||
"eng",
|
||||
"fre",
|
||||
"fra",
|
||||
"ger",
|
||||
"deu",
|
||||
"spa",
|
||||
"ita",
|
||||
"por",
|
||||
"dut",
|
||||
"nld",
|
||||
"swe",
|
||||
"dan",
|
||||
"nor",
|
||||
"fin",
|
||||
"ice",
|
||||
"isl",
|
||||
"cat",
|
||||
}
|
||||
# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote),
|
||||
# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real
|
||||
# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte.
|
||||
_UTF8_VALID_PER_STRAY = 10
|
||||
|
||||
|
||||
def decode_subtitle(raw: bytes, language_code: str | None) -> str:
|
||||
"""A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is
|
||||
typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found
|
||||
live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy
|
||||
code page with accents is almost never valid UTF-8); else the language's legacy code page."""
|
||||
code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray
|
||||
byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page."""
|
||||
if raw.startswith(codecs.BOM_UTF8):
|
||||
return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace")
|
||||
if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
||||
@@ -940,10 +970,15 @@ def decode_subtitle(raw: bytes, language_code: str | None) -> str:
|
||||
try:
|
||||
return raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# silent-ok: not UTF-8 is the expected answer for a legacy-code-page file, decoded below.
|
||||
# silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below.
|
||||
pass
|
||||
lenient = raw.decode("utf-8", "replace")
|
||||
stray = lenient.count("�")
|
||||
valid = sum(1 for ch in lenient if ch > "" and ch != "�")
|
||||
if valid >= stray * _UTF8_VALID_PER_STRAY:
|
||||
return lenient
|
||||
lang = (language_code or "").lower()
|
||||
legacy = "cp1250" if lang in _CP1250 else "cp1251" if lang in _CP1251 else "cp1252"
|
||||
legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250"
|
||||
return raw.decode(legacy, "replace")
|
||||
|
||||
|
||||
|
||||
@@ -79,3 +79,37 @@ def test_a_track_cached_before_the_encoding_fix_is_not_served(tmp_path, monkeypa
|
||||
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAz�rt\n", "utf-8"
|
||||
)
|
||||
assert subtitles.cached_track("14881", 1)[0] is None
|
||||
|
||||
|
||||
def test_one_stray_byte_does_not_send_a_utf8_file_to_a_legacy_code_page(monkeypatch):
|
||||
"""A UTF-8 release with ONE bad byte (a cut-and-paste Windows-1252 quote) failed strict UTF-8 as a
|
||||
whole, and the whole file was then read as Windows-1250: every correct ő/é became mojibake. Nearly
|
||||
all of it is valid UTF-8, so it stays UTF-8 and only the one byte is replaced."""
|
||||
raw = (HU * 20).encode("utf-8") + b"\x93quoted\x94\r\n"
|
||||
vtt = _sidecar(monkeypatch, raw, "hun")
|
||||
assert "Miért nem szóltál? Ősz, űr." in vtt
|
||||
assert "Miért" not in vtt
|
||||
|
||||
|
||||
def test_an_untagged_legacy_sidecar_is_read_as_central_european(monkeypatch):
|
||||
"""No language tag: the fallback was Windows-1252, which turns a Hungarian ő/ű into õ/û."""
|
||||
vtt = _sidecar(monkeypatch, HU.encode("cp1250"), None)
|
||||
assert "Ősz, űr." in vtt
|
||||
|
||||
|
||||
def test_a_latin_serbian_sidecar_is_read_as_central_european(monkeypatch):
|
||||
"""Serbian releases are overwhelmingly Latin script in Windows-1250 (like hrv/bos); read as
|
||||
Windows-1251 every č/ć/š/ž/đ turned into Cyrillic noise."""
|
||||
sr = "1\r\n00:00:01,000 --> 00:00:02,000\r\nŠta ćeš, đače? Čekaj.\r\n"
|
||||
vtt = _sidecar(monkeypatch, sr.encode("cp1250"), "srp")
|
||||
assert "Šta ćeš, đače? Čekaj." in vtt
|
||||
|
||||
|
||||
def test_a_track_cached_under_the_previous_decoder_is_not_served(tmp_path, monkeypatch):
|
||||
"""The decoder changed again (stray-byte UTF-8, cp1250 fallbacks): the key is versioned once more
|
||||
so a track the old rules mis-decoded is read again instead of served for good."""
|
||||
monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
|
||||
(tmp_path / "sub2_14881_1.vtt").write_text(
|
||||
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAzõrt\n", "utf-8"
|
||||
)
|
||||
assert subtitles.cached_track("14881", 1)[0] is None
|
||||
|
||||
Reference in New Issue
Block a user