From 7987bd8a3724a1daef9738c64aac563f739f2d36 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 27 Sep 2026 13:58:23 +0200 Subject: [PATCH] fix(plex): a stray byte keeps a UTF-8 subtitle UTF-8; untagged and Serbian read as cp1250 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One invalid byte sent a whole UTF-8 sidecar to a legacy code page, an untagged file fell to Windows-1252 (ő/ű became õ/û) and Serbian was read as Cyrillic. The cache key moves to sub3_ so a track the old rules mis-decoded is read again. --- backend/app/plex/subtitles.py | 53 ++++++++++++++++---- backend/tests/test_plex_subtitle_encoding.py | 34 +++++++++++++ 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/backend/app/plex/subtitles.py b/backend/app/plex/subtitles.py index 8bb38dd5..22f2a3ce 100644 --- a/backend/app/plex/subtitles.py +++ b/backend/app/plex/subtitles.py @@ -218,10 +218,11 @@ _FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can def cache_key(rating_key: str, sub_ord: int) -> str: - # `sub2_`: versioned when sidecars in a legacy code page turned out to have been cached as - # U+FFFD (`decode_subtitle`). The cache has no eviction, so every track is read again once - # rather than a broken one being served for good; the old `sub_` files are just left behind. - return f"sub2_{rating_key}_{sub_ord}" + # Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code + # pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went + # to the wrong code page. The cache has no eviction, so every track is read again once rather + # than a broken one being served for good; the older files are just left behind. + return f"sub3_{rating_key}_{sub_ord}" def _vtt_path(key: str) -> Path: @@ -909,7 +910,8 @@ def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float: # The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely -# saved in; anything not listed falls to Windows-1252. +# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in +# Windows-1250, like Croatian and Bosnian. _CP1250 = { "hun", "ces", @@ -920,19 +922,47 @@ _CP1250 = { "slv", "hrv", "bos", + "srp", "ron", "rum", "sqi", "alb", } -_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac", "srp"} +_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"} +# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to +# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű +# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian. +_CP1252 = { + "eng", + "fre", + "fra", + "ger", + "deu", + "spa", + "ita", + "por", + "dut", + "nld", + "swe", + "dan", + "nor", + "fin", + "ice", + "isl", + "cat", +} +# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote), +# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real +# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte. +_UTF8_VALID_PER_STRAY = 10 def decode_subtitle(raw: bytes, language_code: str | None) -> str: """A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy - code page with accents is almost never valid UTF-8); else the language's legacy code page.""" + code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray + byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page.""" if raw.startswith(codecs.BOM_UTF8): return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace") if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)): @@ -940,10 +970,15 @@ def decode_subtitle(raw: bytes, language_code: str | None) -> str: try: return raw.decode("utf-8") except UnicodeDecodeError: - # silent-ok: not UTF-8 is the expected answer for a legacy-code-page file, decoded below. + # silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below. pass + lenient = raw.decode("utf-8", "replace") + stray = lenient.count("�") + valid = sum(1 for ch in lenient if ch > "" and ch != "�") + if valid >= stray * _UTF8_VALID_PER_STRAY: + return lenient lang = (language_code or "").lower() - legacy = "cp1250" if lang in _CP1250 else "cp1251" if lang in _CP1251 else "cp1252" + legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250" return raw.decode(legacy, "replace") diff --git a/backend/tests/test_plex_subtitle_encoding.py b/backend/tests/test_plex_subtitle_encoding.py index 6222787b..89a40dcc 100644 --- a/backend/tests/test_plex_subtitle_encoding.py +++ b/backend/tests/test_plex_subtitle_encoding.py @@ -79,3 +79,37 @@ def test_a_track_cached_before_the_encoding_fix_is_not_served(tmp_path, monkeypa "WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAz�rt\n", "utf-8" ) assert subtitles.cached_track("14881", 1)[0] is None + + +def test_one_stray_byte_does_not_send_a_utf8_file_to_a_legacy_code_page(monkeypatch): + """A UTF-8 release with ONE bad byte (a cut-and-paste Windows-1252 quote) failed strict UTF-8 as a + whole, and the whole file was then read as Windows-1250: every correct ő/é became mojibake. Nearly + all of it is valid UTF-8, so it stays UTF-8 and only the one byte is replaced.""" + raw = (HU * 20).encode("utf-8") + b"\x93quoted\x94\r\n" + vtt = _sidecar(monkeypatch, raw, "hun") + assert "Miért nem szóltál? Ősz, űr." in vtt + assert "MiĂ©rt" not in vtt + + +def test_an_untagged_legacy_sidecar_is_read_as_central_european(monkeypatch): + """No language tag: the fallback was Windows-1252, which turns a Hungarian ő/ű into õ/û.""" + vtt = _sidecar(monkeypatch, HU.encode("cp1250"), None) + assert "Ősz, űr." in vtt + + +def test_a_latin_serbian_sidecar_is_read_as_central_european(monkeypatch): + """Serbian releases are overwhelmingly Latin script in Windows-1250 (like hrv/bos); read as + Windows-1251 every č/ć/š/ž/đ turned into Cyrillic noise.""" + sr = "1\r\n00:00:01,000 --> 00:00:02,000\r\nŠta ćeš, đače? Čekaj.\r\n" + vtt = _sidecar(monkeypatch, sr.encode("cp1250"), "srp") + assert "Šta ćeš, đače? Čekaj." in vtt + + +def test_a_track_cached_under_the_previous_decoder_is_not_served(tmp_path, monkeypatch): + """The decoder changed again (stray-byte UTF-8, cp1250 fallbacks): the key is versioned once more + so a track the old rules mis-decoded is read again instead of served for good.""" + monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path) + (tmp_path / "sub2_14881_1.vtt").write_text( + "WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAzõrt\n", "utf-8" + ) + assert subtitles.cached_track("14881", 1)[0] is None