fix(plex): a stray byte keeps a UTF-8 subtitle UTF-8; untagged and Serbian read as cp1250

One invalid byte sent a whole UTF-8 sidecar to a legacy code page, an untagged
file fell to Windows-1252 (ő/ű became õ/û) and Serbian was read as Cyrillic.
The cache key moves to sub3_ so a track the old rules mis-decoded is read again.

backend/app/plex/subtitles.py

@@ -218,10 +218,11 @@ _FULL_END_S = 1_000_000_000.0  # stand-in for "to the end of the file" (JSON can
 
 
 def cache_key(rating_key: str, sub_ord: int) -> str:
-    # `sub2_`: versioned when sidecars in a legacy code page turned out to have been cached as
+    # Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code
-    # U+FFFD (`decode_subtitle`). The cache has no eviction, so every track is read again once
+    # pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went
-    # rather than a broken one being served for good; the old `sub_` files are just left behind.
+    # to the wrong code page. The cache has no eviction, so every track is read again once rather
-    return f"sub2_{rating_key}_{sub_ord}"
+    # than a broken one being served for good; the older files are just left behind.
+    return f"sub3_{rating_key}_{sub_ord}"
 
 
 def _vtt_path(key: str) -> Path:
@@ -909,7 +910,8 @@ def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
 
 
 # The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely
-# saved in; anything not listed falls to Windows-1252.
+# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in
+# Windows-1250, like Croatian and Bosnian.
 _CP1250 = {
     "hun",
     "ces",
@@ -920,19 +922,47 @@ _CP1250 = {
     "slv",
     "hrv",
     "bos",
+    "srp",
     "ron",
     "rum",
     "sqi",
     "alb",
 }
-_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac", "srp"}
+_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"}
+# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to
+# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű
+# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian.
+_CP1252 = {
+    "eng",
+    "fre",
+    "fra",
+    "ger",
+    "deu",
+    "spa",
+    "ita",
+    "por",
+    "dut",
+    "nld",
+    "swe",
+    "dan",
+    "nor",
+    "fin",
+    "ice",
+    "isl",
+    "cat",
+}
+# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote),
+# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real
+# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte.
+_UTF8_VALID_PER_STRAY = 10
 
 
 def decode_subtitle(raw: bytes, language_code: str | None) -> str:
     """A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is
     typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found
     live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy
-    code page with accents is almost never valid UTF-8); else the language's legacy code page."""
+    code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray
+    byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page."""
     if raw.startswith(codecs.BOM_UTF8):
         return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace")
     if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
@@ -940,10 +970,15 @@ def decode_subtitle(raw: bytes, language_code: str | None) -> str:
     try:
         return raw.decode("utf-8")
     except UnicodeDecodeError:
-        # silent-ok: not UTF-8 is the expected answer for a legacy-code-page file, decoded below.
+        # silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below.
         pass
+    lenient = raw.decode("utf-8", "replace")
+    stray = lenient.count("�")
+    valid = sum(1 for ch in lenient if ch > "" and ch != "�")
+    if valid >= stray * _UTF8_VALID_PER_STRAY:
+        return lenient
     lang = (language_code or "").lower()
-    legacy = "cp1250" if lang in _CP1250 else "cp1251" if lang in _CP1251 else "cp1252"
+    legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250"
     return raw.decode(legacy, "replace")
 
 

backend/tests/test_plex_subtitle_encoding.py

@@ -79,3 +79,37 @@ def test_a_track_cached_before_the_encoding_fix_is_not_served(tmp_path, monkeypa
         "WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAz�rt\n", "utf-8"
     )
     assert subtitles.cached_track("14881", 1)[0] is None
+
+
+def test_one_stray_byte_does_not_send_a_utf8_file_to_a_legacy_code_page(monkeypatch):
+    """A UTF-8 release with ONE bad byte (a cut-and-paste Windows-1252 quote) failed strict UTF-8 as a
+    whole, and the whole file was then read as Windows-1250: every correct ő/é became mojibake. Nearly
+    all of it is valid UTF-8, so it stays UTF-8 and only the one byte is replaced."""
+    raw = (HU * 20).encode("utf-8") + b"\x93quoted\x94\r\n"
+    vtt = _sidecar(monkeypatch, raw, "hun")
+    assert "Miért nem szóltál? Ősz, űr." in vtt
+    assert "Miért" not in vtt
+
+
+def test_an_untagged_legacy_sidecar_is_read_as_central_european(monkeypatch):
+    """No language tag: the fallback was Windows-1252, which turns a Hungarian ő/ű into õ/û."""
+    vtt = _sidecar(monkeypatch, HU.encode("cp1250"), None)
+    assert "Ősz, űr." in vtt
+
+
+def test_a_latin_serbian_sidecar_is_read_as_central_european(monkeypatch):
+    """Serbian releases are overwhelmingly Latin script in Windows-1250 (like hrv/bos); read as
+    Windows-1251 every č/ć/š/ž/đ turned into Cyrillic noise."""
+    sr = "1\r\n00:00:01,000 --> 00:00:02,000\r\nŠta ćeš, đače? Čekaj.\r\n"
+    vtt = _sidecar(monkeypatch, sr.encode("cp1250"), "srp")
+    assert "Šta ćeš, đače? Čekaj." in vtt
+
+
+def test_a_track_cached_under_the_previous_decoder_is_not_served(tmp_path, monkeypatch):
+    """The decoder changed again (stray-byte UTF-8, cp1250 fallbacks): the key is versioned once more
+    so a track the old rules mis-decoded is read again instead of served for good."""
+    monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
+    (tmp_path / "sub2_14881_1.vtt").write_text(
+        "WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAzõrt\n", "utf-8"
+    )
+    assert subtitles.cached_track("14881", 1)[0] is None