fix(plex): a sidecar subtitle in a legacy code page rendered its accents as U+FFFD

A Hungarian .srt beside the video (Windows-1250) was decoded as UTF-8 with
errors=replace: one episode served 1391 U+FFFD and no accented letter.
decode_subtitle now lets a BOM decide, keeps valid UTF-8, and otherwise
reads the legacy code page of the stream's language (cp1250 for Central
European, cp1251 for Cyrillic, else cp1252) — on the ffmpeg path too.

The cache has no eviction, so its key is versioned (sub2_): a track cached
broken is read again once instead of being served for good.
This commit is contained in:
2026-09-25 15:20:31 +02:00
parent 58878958d8
commit 1d8b83c37c
3 changed files with 133 additions and 6 deletions
@@ -0,0 +1,81 @@
"""A sidecar subtitle in a legacy code page must render its accents, not U+FFFD.
Found live on Arrow S03E15 (a Hungarian .srt beside the .mp4): the served WebVTT held 1391 U+FFFD and
not one accented letter, because `_sidecar_vtt` decoded the bytes as UTF-8 with `errors="replace"`
while the file was Windows-1250 (ő = 0xF5, ű = 0xFB). Legacy-code-page .srt files are the norm for
Hungarian releases, and SubRip carries no charset of its own, so the stream's language is the only hint.
No Postgres: Plex and ffmpeg are stubbed.
"""
from pathlib import Path
from typing import cast
import pytest
from sqlalchemy.orm import Session
from app.plex import subtitles
HU = "1\r\n00:00:01,000 --> 00:00:02,000\r\nMiért nem szóltál? Ősz, űr.\r\n"
FR = "1\r\n00:00:01,000 --> 00:00:02,000\r\nDéjà vu, garçon.\r\n"
RU = "1\r\n00:00:01,000 --> 00:00:02,000\r\nПривет, мир.\r\n"
class _Plex:
def __init__(self, raw: bytes):
self.raw = raw
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def raw_get(self, key: str) -> bytes:
return self.raw
def _sidecar(monkeypatch, raw: bytes, lang: str | None, codec: str = "srt") -> str:
monkeypatch.setattr(subtitles, "PlexClient", lambda db: _Plex(raw))
stream = {"key": "/library/streams/1", "codec": codec, "languageCode": lang}
return subtitles._sidecar_vtt(cast(Session, None), stream)
@pytest.mark.parametrize(
("text", "encoding", "lang"),
[
(HU, "cp1250", "hun"), # the file that was found broken
(FR, "cp1252", "fre"), # a Western legacy file
(RU, "cp1251", "rus"), # a Cyrillic one
(HU, "utf-8", "hun"), # already UTF-8: untouched
(HU, "utf-8-sig", "hun"), # UTF-8 with a BOM
(HU, "utf-16", "hun"), # UTF-16 (carries its BOM)
],
ids=["cp1250-hu", "cp1252-fr", "cp1251-ru", "utf8", "utf8-bom", "utf16"],
)
def test_a_sidecar_srt_is_decoded_in_its_own_encoding(monkeypatch, text, encoding, lang):
vtt = _sidecar(monkeypatch, text.encode(encoding), lang)
assert "\ufffd" not in vtt
assert vtt.startswith("WEBVTT\n\n")
assert text.splitlines()[2] in vtt
def test_a_non_srt_sidecar_reaches_ffmpeg_as_utf8(monkeypatch):
seen: dict[str, bytes] = {}
def fake_ffmpeg(src: str, sub_ord):
seen["bytes"] = Path(src).read_bytes()
return "WEBVTT\n"
monkeypatch.setattr(subtitles, "ffmpeg_to_vtt", fake_ffmpeg)
_sidecar(monkeypatch, "[Events]\nDialogue: Ősz\n".encode("cp1250"), "hun", codec="ass")
assert seen["bytes"].decode("utf-8") == "[Events]\nDialogue: Ősz\n"
def test_a_track_cached_before_the_encoding_fix_is_not_served(tmp_path, monkeypatch):
"""The cache has no eviction, so a mis-decoded track would have stayed broken for good: the key
is versioned, and a track cached under the old one is read again (once) instead of served."""
monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
(tmp_path / "sub_14881_1.vtt").write_text(
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAz�rt\n", "utf-8"
)
assert subtitles.cached_track("14881", 1)[0] is None