fix(plex): a segment name ffmpeg never spells is refused; teardown no longer holds the registry lock
The artifact guard parsed the sequence with int(), so a zero-padded or non-ASCII-digit name read as a segment the session had reached and parked the sync route for the full 20 s wait. It now also requires the canonical spelling segment_name() produces. Stopping, replacing, culling or reaping a session waited up to 3 s for ffmpeg and deleted a whole segmented film under _lock, which every segment request and checkpoint takes. Teardown now signals the process and moves its directory aside under the lock, and waits/deletes after it. A new session on the same path still waits for the dying ffmpeg first (it writes by path), and the orphan sweep recognises a moved-aside directory left by a crash.
This commit is contained in:
@@ -83,6 +83,7 @@ class _Proc:
|
||||
|
||||
class _Session:
|
||||
def __init__(self, tmp_path, name: str, finished: bool, idle_s: float):
|
||||
self.key = name
|
||||
self.dir = tmp_path / name
|
||||
self.dir.mkdir()
|
||||
self.proc = _Proc(finished)
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Tearing a session down does not stall everybody else's playback.
|
||||
|
||||
Every segment request (`current_session`) and every checkpoint (`touch_session`) takes the one
|
||||
registry `_lock`. Stopping a session waits up to 3 s for its ffmpeg to exit and then deletes a whole
|
||||
segmented film, ~1200 files — and that used to happen under the same lock, so one viewer's seek, a
|
||||
cap cull or a reap pass held up every other viewer's HLS requests for seconds, each one a parked
|
||||
threadpool worker and its pooled DB connection.
|
||||
|
||||
What still has to be atomic is pinned too: a session directory is reused for the same item, offset
|
||||
and tag, so a new session on that path must not start before the old ffmpeg — which writes by path
|
||||
— has exited.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from app.plex import stream
|
||||
|
||||
|
||||
class _SlowProc:
|
||||
"""An ffmpeg that takes a while to exit after TERM: `wait` blocks until `exited` is set."""
|
||||
|
||||
def __init__(self, order: list[str], name: str):
|
||||
self.order = order
|
||||
self.name = name
|
||||
self.waiting = threading.Event()
|
||||
self.exited = threading.Event()
|
||||
|
||||
def poll(self):
|
||||
return 0 if self.exited.is_set() else None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def kill(self):
|
||||
self.exited.set()
|
||||
|
||||
def wait(self, timeout=None):
|
||||
self.waiting.set()
|
||||
if not self.exited.wait(timeout):
|
||||
raise stream.subprocess.TimeoutExpired("ffmpeg", timeout)
|
||||
self.order.append(f"{self.name} exited")
|
||||
return 0
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, directory, proc, key: str):
|
||||
self.id = f"sid-{key}"
|
||||
self.key = key
|
||||
self.dir = directory
|
||||
self.proc = proc
|
||||
self.last_access = time.time()
|
||||
self.transcode = False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sessions(monkeypatch):
|
||||
live: dict = {}
|
||||
monkeypatch.setattr(stream, "_sessions", live)
|
||||
monkeypatch.setattr(stream, "_dying", {}, raising=False)
|
||||
return live
|
||||
|
||||
|
||||
def test_stopping_a_session_does_not_hold_up_another_viewers_requests(tmp_path, sessions):
|
||||
order: list[str] = []
|
||||
slow = _SlowProc(order, "a")
|
||||
d = tmp_path / "1_u1_0_0_+0.00"
|
||||
d.mkdir()
|
||||
(d / "seg_0.ts").write_bytes(b"x")
|
||||
sessions[(1, "a")] = _Session(d, slow, "a")
|
||||
sessions[(2, "b")] = _Session(tmp_path / "2_u2_0_0_+0.00", _SlowProc(order, "b"), "b")
|
||||
|
||||
stopper = threading.Thread(target=stream.stop_session, args=(1, "a", "sid-a"))
|
||||
stopper.start()
|
||||
try:
|
||||
assert slow.waiting.wait(2.0), "the stop never waited for its ffmpeg"
|
||||
got: list = []
|
||||
reader = threading.Thread(target=lambda: got.append(stream.current_session(2, "b")))
|
||||
reader.start()
|
||||
reader.join(1.0)
|
||||
assert not reader.is_alive(), "a segment request waited on another viewer's teardown"
|
||||
assert got and got[0] is sessions[(2, "b")]
|
||||
finally:
|
||||
slow.exited.set()
|
||||
stopper.join(5.0)
|
||||
# The teardown still finishes: the session is gone and nothing of its directory is left behind.
|
||||
assert (1, "a") not in sessions
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_a_reap_pass_does_not_hold_up_another_viewers_requests(tmp_path, sessions):
|
||||
order: list[str] = []
|
||||
slow = _SlowProc(order, "a")
|
||||
d = tmp_path / "1_u1_0_0_+0.00"
|
||||
d.mkdir()
|
||||
gone = _Session(d, slow, "a")
|
||||
gone.last_access = time.time() - stream._SESSION_IDLE_S - 5
|
||||
sessions[(1, "a")] = gone
|
||||
sessions[(2, "b")] = _Session(tmp_path / "2_u2_0_0_+0.00", _SlowProc(order, "b"), "b")
|
||||
|
||||
reaper = threading.Thread(target=stream.reap_idle)
|
||||
reaper.start()
|
||||
try:
|
||||
assert slow.waiting.wait(2.0), "the reap never waited for its ffmpeg"
|
||||
toucher = threading.Thread(target=stream.touch_session, args=(2, "b"))
|
||||
toucher.start()
|
||||
toucher.join(1.0)
|
||||
assert not toucher.is_alive(), "a checkpoint waited on another viewer's teardown"
|
||||
finally:
|
||||
slow.exited.set()
|
||||
reaper.join(5.0)
|
||||
assert list(sessions) == [(2, "b")]
|
||||
assert not d.exists()
|
||||
|
||||
|
||||
class _FakeItem:
|
||||
rating_key = "12345"
|
||||
file_path = "/media/movie.mkv"
|
||||
playable = "remux"
|
||||
codec_video = "h264"
|
||||
|
||||
|
||||
def test_a_new_session_on_a_dying_ones_path_starts_only_after_it_exited(
|
||||
tmp_path, sessions, monkeypatch
|
||||
):
|
||||
"""The same item at the same second gets the same directory. The old ffmpeg writes by PATH, so
|
||||
while it is still winding down after TERM it can write a segment — or, on a clean exit, a final
|
||||
playlist with ENDLIST — into the NEW session's directory."""
|
||||
order: list[str] = []
|
||||
procs = iter([_SlowProc(order, "old"), _SlowProc(order, "new")])
|
||||
|
||||
def popen(*a, **k):
|
||||
order.append("popen")
|
||||
return next(procs)
|
||||
|
||||
monkeypatch.setattr(stream, "_HLS_ROOT", tmp_path)
|
||||
src = tmp_path / "movie.mkv"
|
||||
src.write_bytes(b"x")
|
||||
monkeypatch.setattr(stream.paths, "local_media_path", lambda db, fp: src)
|
||||
monkeypatch.setattr(stream.sysconfig, "get_int", lambda db, k: 2)
|
||||
monkeypatch.setattr(stream.sysconfig, "get_bool", lambda db, k: False)
|
||||
monkeypatch.setattr(stream.subprocess, "Popen", popen)
|
||||
monkeypatch.setattr(stream, "wait_for", lambda *a, **k: True)
|
||||
|
||||
first = stream.start_session(db=None, user_id=7, item=_FakeItem(), start_s=0.0)
|
||||
assert first is not None
|
||||
stopper = threading.Thread(target=stream.stop_session, args=(7, "12345", first.id))
|
||||
stopper.start()
|
||||
assert first.proc.waiting.wait(2.0)
|
||||
threading.Timer(0.3, first.proc.exited.set).start()
|
||||
second = stream.start_session(db=None, user_id=7, item=_FakeItem(), start_s=0.0)
|
||||
stopper.join(5.0)
|
||||
assert second is not None and second.dir == first.dir
|
||||
assert order.index("old exited") < order.index("popen", 1)
|
||||
assert second.dir.is_dir(), "the old teardown deleted the new session's directory"
|
||||
second.proc.exited.set()
|
||||
@@ -198,6 +198,31 @@ def test_a_name_the_session_cannot_produce_fails_at_once(client, tmp_path, sessi
|
||||
assert elapsed < 2.0, f"waited {elapsed:.1f}s for a playlist this session never writes"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "audio_langs"),
|
||||
[
|
||||
("seg_00.ts", None), # `int()` reads it as segment 0, which needs no predecessor
|
||||
("seg_007.ts", None), # …as segment 7, whose predecessor seg_6.ts is on disk
|
||||
("seg_٧.ts", None), # an Arabic-Indic 7: `\d` and `int()` both take it
|
||||
("seg_a0_07.ts", ["hu", "en"]),
|
||||
("seg_0_00.ts", ["hu", "en"]),
|
||||
],
|
||||
)
|
||||
def test_a_segment_name_ffmpeg_never_spells_is_refused(tmp_path, name, audio_langs):
|
||||
"""ffmpeg writes `seg_%d.ts`: no padding, ASCII digits. A name that only PARSES to a sequence the
|
||||
session has reached is still a file that will never appear, and saying yes to it parked the
|
||||
request for the full 20 s wait — the parking this guard exists to end."""
|
||||
d = tmp_path / "m1_0_0_+0.00"
|
||||
d.mkdir()
|
||||
for v in plex_stream.session_variants(audio_langs) or (None,):
|
||||
(d / plex_stream.segment_name(6, v)).write_bytes(b"x")
|
||||
sess = _Session("sid-a", d, running=True, audio_langs=audio_langs)
|
||||
assert not plex_stream.artifact_may_arrive(sess, name)
|
||||
# The canonical spelling of the same next segment is still accepted.
|
||||
v = (plex_stream.session_variants(audio_langs) or (None,))[0]
|
||||
assert plex_stream.artifact_may_arrive(sess, plex_stream.segment_name(7, v))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("digits", [100, 5000])
|
||||
def test_an_absurd_segment_index_is_a_404_not_a_500(client, tmp_path, sessions, digits):
|
||||
"""`int()` on an unbounded digit run raises ValueError past 4300 digits, and only OSError was
|
||||
@@ -213,7 +238,7 @@ def test_an_absurd_segment_index_is_a_404_not_a_500(client, tmp_path, sessions,
|
||||
def test_a_session_torn_down_during_the_read_is_a_404_not_a_500(
|
||||
client, monkeypatch, tmp_path, sessions
|
||||
):
|
||||
"""`_kill` rmtrees the whole directory and nothing holds `_lock` across the wait and the read,
|
||||
"""Teardown moves the whole directory aside and nothing holds `_lock` across the wait and the read,
|
||||
so the file can vanish in between — on every seek-restart, since `loadSession` awaits the new
|
||||
session before destroying the old hls instance. The player's recovery is keyed on the 404
|
||||
STATUS, so a 500 here skipped the reclaim and ended playback with "couldn't play"."""
|
||||
|
||||
@@ -39,6 +39,7 @@ class _Proc:
|
||||
class _Session:
|
||||
def __init__(self, session_id: str, directory):
|
||||
self.id = session_id
|
||||
self.key = "m1"
|
||||
self.dir = directory
|
||||
self.proc = _Proc()
|
||||
self.last_access = time.time()
|
||||
|
||||
@@ -26,6 +26,9 @@ from app.plex.stream import _SESSION_DIR_RE
|
||||
"12345_0_2_+0.00", # current shape, explicit audio_ord
|
||||
"12345_842_multi_-1.25", # multi-audio, negative offset
|
||||
"12345_842_multi_+12.50",
|
||||
# a torn-down session's directory, moved aside to be deleted outside the lock — left behind
|
||||
# only if the process died between the rename and the delete
|
||||
"12345_u7_842_multi_+12.50.dead-0123abcd",
|
||||
],
|
||||
)
|
||||
def test_our_session_directories_are_swept(name):
|
||||
@@ -47,6 +50,9 @@ def test_our_session_directories_are_swept(name):
|
||||
"snapshot_2024_backup_+1.00", # session-ish tail, but the key isn't a rating_key
|
||||
"backup_2024_multi_+0.00",
|
||||
".config",
|
||||
"12345_u7_0_0_+0.00.dead-", # a tombstone suffix with no id
|
||||
"12345_u7_0_0_+0.00.dead-notours!",
|
||||
"backup_u7_0_0_+0.00.dead-0123abcd",
|
||||
],
|
||||
)
|
||||
def test_foreign_directories_are_left_alone(name):
|
||||
@@ -159,7 +165,7 @@ class TestSweepOrphans:
|
||||
|
||||
|
||||
class _FakeProc:
|
||||
"""Stand-in for the ffmpeg Popen — start_session only stores it (and _kill/reap poll it later)."""
|
||||
"""Stand-in for the ffmpeg Popen — start_session only stores it (and teardown/reap poll it later)."""
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user