peter
7987bd8a37
One invalid byte sent a whole UTF-8 sidecar to a legacy code page, an untagged file fell to Windows-1252 (ő/ű became õ/û) and Serbian was read as Cyrillic. The cache key moves to sub3_ so a track the old rules mis-decoded is read again.
116 lines
4.8 KiB
Python
116 lines
4.8 KiB
Python
"""A sidecar subtitle in a legacy code page must render its accents, not U+FFFD.
|
|
|
|
Found live on Arrow S03E15 (a Hungarian .srt beside the .mp4): the served WebVTT held 1391 U+FFFD and
|
|
not one accented letter, because `_sidecar_vtt` decoded the bytes as UTF-8 with `errors="replace"`
|
|
while the file was Windows-1250 (ő = 0xF5, ű = 0xFB). Legacy-code-page .srt files are the norm for
|
|
Hungarian releases, and SubRip carries no charset of its own, so the stream's language is the only hint.
|
|
No Postgres: Plex and ffmpeg are stubbed.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
import pytest
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.plex import subtitles
|
|
|
|
HU = "1\r\n00:00:01,000 --> 00:00:02,000\r\nMiért nem szóltál? Ősz, űr.\r\n"
|
|
FR = "1\r\n00:00:01,000 --> 00:00:02,000\r\nDéjà vu, garçon.\r\n"
|
|
RU = "1\r\n00:00:01,000 --> 00:00:02,000\r\nПривет, мир.\r\n"
|
|
|
|
|
|
class _Plex:
|
|
def __init__(self, raw: bytes):
|
|
self.raw = raw
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def raw_get(self, key: str) -> bytes:
|
|
return self.raw
|
|
|
|
|
|
def _sidecar(monkeypatch, raw: bytes, lang: str | None, codec: str = "srt") -> str:
|
|
monkeypatch.setattr(subtitles, "PlexClient", lambda db: _Plex(raw))
|
|
stream = {"key": "/library/streams/1", "codec": codec, "languageCode": lang}
|
|
return subtitles._sidecar_vtt(cast(Session, None), stream)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "encoding", "lang"),
|
|
[
|
|
(HU, "cp1250", "hun"), # the file that was found broken
|
|
(FR, "cp1252", "fre"), # a Western legacy file
|
|
(RU, "cp1251", "rus"), # a Cyrillic one
|
|
(HU, "utf-8", "hun"), # already UTF-8: untouched
|
|
(HU, "utf-8-sig", "hun"), # UTF-8 with a BOM
|
|
(HU, "utf-16", "hun"), # UTF-16 (carries its BOM)
|
|
],
|
|
ids=["cp1250-hu", "cp1252-fr", "cp1251-ru", "utf8", "utf8-bom", "utf16"],
|
|
)
|
|
def test_a_sidecar_srt_is_decoded_in_its_own_encoding(monkeypatch, text, encoding, lang):
|
|
vtt = _sidecar(monkeypatch, text.encode(encoding), lang)
|
|
assert "\ufffd" not in vtt
|
|
assert vtt.startswith("WEBVTT\n\n")
|
|
assert text.splitlines()[2] in vtt
|
|
|
|
|
|
def test_a_non_srt_sidecar_reaches_ffmpeg_as_utf8(monkeypatch):
|
|
seen: dict[str, bytes] = {}
|
|
|
|
def fake_ffmpeg(src: str, sub_ord):
|
|
seen["bytes"] = Path(src).read_bytes()
|
|
return "WEBVTT\n"
|
|
|
|
monkeypatch.setattr(subtitles, "ffmpeg_to_vtt", fake_ffmpeg)
|
|
_sidecar(monkeypatch, "[Events]\nDialogue: Ősz\n".encode("cp1250"), "hun", codec="ass")
|
|
assert seen["bytes"].decode("utf-8") == "[Events]\nDialogue: Ősz\n"
|
|
|
|
|
|
def test_a_track_cached_before_the_encoding_fix_is_not_served(tmp_path, monkeypatch):
|
|
"""The cache has no eviction, so a mis-decoded track would have stayed broken for good: the key
|
|
is versioned, and a track cached under the old one is read again (once) instead of served."""
|
|
monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
|
|
(tmp_path / "sub_14881_1.vtt").write_text(
|
|
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAz�rt\n", "utf-8"
|
|
)
|
|
assert subtitles.cached_track("14881", 1)[0] is None
|
|
|
|
|
|
def test_one_stray_byte_does_not_send_a_utf8_file_to_a_legacy_code_page(monkeypatch):
|
|
"""A UTF-8 release with ONE bad byte (a cut-and-paste Windows-1252 quote) failed strict UTF-8 as a
|
|
whole, and the whole file was then read as Windows-1250: every correct ő/é became mojibake. Nearly
|
|
all of it is valid UTF-8, so it stays UTF-8 and only the one byte is replaced."""
|
|
raw = (HU * 20).encode("utf-8") + b"\x93quoted\x94\r\n"
|
|
vtt = _sidecar(monkeypatch, raw, "hun")
|
|
assert "Miért nem szóltál? Ősz, űr." in vtt
|
|
assert "Miért" not in vtt
|
|
|
|
|
|
def test_an_untagged_legacy_sidecar_is_read_as_central_european(monkeypatch):
|
|
"""No language tag: the fallback was Windows-1252, which turns a Hungarian ő/ű into õ/û."""
|
|
vtt = _sidecar(monkeypatch, HU.encode("cp1250"), None)
|
|
assert "Ősz, űr." in vtt
|
|
|
|
|
|
def test_a_latin_serbian_sidecar_is_read_as_central_european(monkeypatch):
|
|
"""Serbian releases are overwhelmingly Latin script in Windows-1250 (like hrv/bos); read as
|
|
Windows-1251 every č/ć/š/ž/đ turned into Cyrillic noise."""
|
|
sr = "1\r\n00:00:01,000 --> 00:00:02,000\r\nŠta ćeš, đače? Čekaj.\r\n"
|
|
vtt = _sidecar(monkeypatch, sr.encode("cp1250"), "srp")
|
|
assert "Šta ćeš, đače? Čekaj." in vtt
|
|
|
|
|
|
def test_a_track_cached_under_the_previous_decoder_is_not_served(tmp_path, monkeypatch):
|
|
"""The decoder changed again (stray-byte UTF-8, cp1250 fallbacks): the key is versioned once more
|
|
so a track the old rules mis-decoded is read again instead of served for good."""
|
|
monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
|
|
(tmp_path / "sub2_14881_1.vtt").write_text(
|
|
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAzõrt\n", "utf-8"
|
|
)
|
|
assert subtitles.cached_track("14881", 1)[0] is None
|