From d5bc7e8031faad54a297a9648f7465a3f3853540 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 18:35:33 +0100 Subject: [PATCH 1/7] feat(plex): every HLS session reap logs its rule and idle time --- backend/app/plex/stream.py | 8 ++++++++ backend/tests/test_hls_reaper.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index 7c4ec00a..50dcf8a0 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -1185,6 +1185,7 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int: one a whole segmented copy of the film on a volume shared with the download center.""" now = time.time() doomed: list[tuple[HlsSession, Path | None]] = [] + reaped: list[tuple[str, str, int]] = [] with _lock: for reg, s in list(_sessions.items()): done = s.proc.poll() is not None @@ -1192,6 +1193,13 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int: if idle > _SESSION_IDLE_S or (done and idle > finished_idle_s): _sessions.pop(reg, None) doomed.append(_detach(s)) + reaped.append((s.key, "finished_idle" if done else "idle", int(idle))) + # The viewer of a session wrongly taken sees only a 404 later, so this line is the one record of why. + for key, reason, idle_s in reaped: + log.info( + "an idle session was reaped", + extra=logs.event("plex.stream.session_reaped", key=key, reason=reason, idle_s=idle_s), + ) _finish(doomed) return len(doomed) diff --git a/backend/tests/test_hls_reaper.py b/backend/tests/test_hls_reaper.py index 844f5499..fb1ffe2b 100644 --- a/backend/tests/test_hls_reaper.py +++ b/backend/tests/test_hls_reaper.py @@ -116,6 +116,30 @@ def test_the_periodic_pass_still_reaps_what_nobody_has_touched_for_the_idle_limi assert sessions == {} +def test_every_reap_says_which_rule_took_the_session(tmp_path, sessions, caplog): + """A session that vanishes leaves the viewer a 404 and nothing else: with no line here, a live + stall could only ever be blamed on the reaper by inference.""" + sessions["1"] = _Session( + tmp_path, "gone_done", finished=True, idle_s=stream._SESSION_IDLE_S - 5 + ) + sessions["2"] = _Session( + tmp_path, "gone_live", finished=False, idle_s=stream._SESSION_IDLE_S + 5 + ) + sessions["3"] = _Session(tmp_path, "kept", finished=False, idle_s=5) + with caplog.at_level("INFO"): + assert stream.reap_idle() == 2 + reaped = { + r.attrs["key"]: r.attrs + for r in caplog.records + if getattr(r, "event", "") == "plex.stream.session_reaped" + } + assert set(reaped) == {"gone_done", "gone_live"} + assert reaped["gone_done"]["reason"] == "finished_idle" + assert reaped["gone_live"]["reason"] == "idle" + assert reaped["gone_done"]["idle_s"] == pytest.approx(stream._SESSION_IDLE_S - 5, abs=2) + assert reaped["gone_live"]["idle_s"] == pytest.approx(stream._SESSION_IDLE_S + 5, abs=2) + + def test_touching_a_session_keeps_it_through_the_short_rule(tmp_path, sessions): s = _Session(tmp_path, "paused", finished=True, idle_s=90) sessions[(1, "12345")] = s From f69725503f9d604c1c349f615336417c02c772f5 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 19:15:18 +0100 Subject: [PATCH 2/7] fix(plex): a finished HLS session outlives a throttled heartbeat --- backend/app/main.py | 2 +- backend/app/plex/stream.py | 24 ++++++++++++-------- backend/app/routes/plex/playback.py | 3 ++- backend/tests/test_hls_reaper.py | 35 +++++++++++++++++++++++------ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index f11aad8b..30905c8d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -151,7 +151,7 @@ async def _reap_hls_sessions() -> None: somebody pressed play again or the process restarted. A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open - player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer + player keeps checkpointing and that route touches the session, so a paused viewer is no longer invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead held one whole segmented copy of a film per item on the download volume.""" while True: diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index 50dcf8a0..1d36ffb0 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -1160,9 +1160,13 @@ def sweep_orphans(older_than: float | None = None) -> int: return dropped -# How long a session whose ffmpeg has FINISHED may go unfetched before a new session request reaps -# it (see reap_idle). -_FINISHED_IDLE_S = 30 +# The slowest heartbeat a live player sends: a browser cuts the timers of a hidden page that makes no +# sound to one a minute (measured on Chromium: a paused and a muted player, about a minute after +# hiding; a hidden player with sound keeps its 10 s). +_THROTTLED_HEARTBEAT_S = 60 +# How long a session whose ffmpeg has FINISHED may go without a signal before `reap_idle` takes it: +# the beat before the reaper looks, one lost on the way, and the time the next one takes to arrive. +_FINISHED_IDLE_S = 3 * _THROTTLED_HEARTBEAT_S def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int: @@ -1171,15 +1175,17 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int: A finished session is also what a PAUSED viewer has: a remux runs far faster than realtime, so ffmpeg is done a minute or two in and writes ENDLIST; hls.js then stops asking for the playlist, - and with its forward buffer full a paused player fetches nothing at all. With the 30 s rule that - killed a one-minute pause, and playback died ~30 s after resuming. + and with its forward buffer full a paused player fetches nothing at all. - What makes the short rule safe again is that fetching is no longer the only signal: a player - parked MID-ITEM checkpoints its position every 10 s, and that route calls `touch_session`, so a - viewer who is merely paused says so far more often than this rule asks. The heartbeat + What makes the short rule safe is that fetching is not the only signal: a player parked MID-ITEM + checkpoints its position and that route calls `touch_session`. That beat is every 10 s only while + the page is visible or making sound; a hidden, silent one sends it once a minute, which a 30 s + rule killed whenever the reaper happened to run more than 30 s after the beat (measured: a paused + session gone that way, and one that lived on the same cadence because its phase was luckier). + So the rule is sized to the throttled beat (`_THROTTLED_HEARTBEAT_S`). The heartbeat stops once the item is finished or before a position is known (`usePlexProgress.flush`) — both states where reaping is correct, and where the player's 404 → reclaim recovers a viewer who - scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing": + scrubs back in anyway. Do not restate this as "whether or not it is playing": a review found that claim in two comments and not in the code. An earlier blanket `reap_abandoned` bought the same safety by keeping every finished session for ten minutes — each one a whole segmented copy of the film on a volume shared with the download center.""" diff --git a/backend/app/routes/plex/playback.py b/backend/app/routes/plex/playback.py index 043dc26a..fed16d5b 100644 --- a/backend/app/routes/plex/playback.py +++ b/backend/app/routes/plex/playback.py @@ -323,7 +323,8 @@ def item_progress( `_should_push_resume`). `final` (sent on pause / seek / pagehide / unmount) pushes immediately and resets the throttle window, so Plex sees a settled position the instant the user pauses or seeks.""" it = item_or_404(db, rating_key) - # A player parked mid-item — playing or PAUSED — checkpoints every 10 s, so this is the signal + # A player parked mid-item — playing or PAUSED — keeps checkpointing (every 10 s, once a minute + # in a hidden silent page), so this is the signal # that such a viewer still wants their HLS session: the one thing `last_access` could not see, # since it moved only on a file fetch and a paused player fetches nothing (see # stream.reap_idle). It is NOT every open player: `usePlexProgress.flush` sends nothing once the diff --git a/backend/tests/test_hls_reaper.py b/backend/tests/test_hls_reaper.py index fb1ffe2b..c4b14368 100644 --- a/backend/tests/test_hls_reaper.py +++ b/backend/tests/test_hls_reaper.py @@ -5,11 +5,11 @@ POST /session: after the last playback, an abandoned session (its ffmpeg still w film, its directory a whole segmented copy of it) lived until somebody pressed play again. And it must not take a PAUSED viewer's session: a remux finishes long before the film does, and a -paused player with a full buffer fetches nothing, so "finished and unfetched for 30 s" does not mean +paused player with a full buffer fetches nothing, so "finished and unfetched for a while" does not mean "abandoned". An earlier fix bought that safety by keeping EVERY finished session for the full ten-minute idle window — one segmented copy of a film per item, on the volume the download center shares. Its successor replaced the blanket retention with the signal that already existed: a player -parked mid-item checkpoints its position every 10 s, paused or not, and that route says "still +parked mid-item keeps checkpointing its position, paused or not, and that route says "still here" (it stops once the item is finished, where reaping is correct). Both call sites are pinned here, because the bug each time was that only one of them moved: the @@ -81,6 +81,9 @@ class _Proc: return 0 +_PAST_THE_RULE = stream._FINISHED_IDLE_S + 5 + + class _Session: def __init__(self, tmp_path, name: str, finished: bool, idle_s: float): self.key = name @@ -98,11 +101,29 @@ def sessions(monkeypatch): def test_the_periodic_pass_reaps_a_finished_session_nobody_has_claimed(tmp_path, sessions): - sessions["12345"] = _Session(tmp_path, "done", finished=True, idle_s=45) + sessions["12345"] = _Session(tmp_path, "done", finished=True, idle_s=_PAST_THE_RULE) assert stream.reap_idle() == 1 assert sessions == {} +@pytest.mark.parametrize("idle_s", [65, 125]) +def test_a_heartbeat_throttled_to_once_a_minute_keeps_a_finished_session( + tmp_path, sessions, idle_s +): + """A hidden page that makes no sound gets its timers cut to one a minute (measured on a paused and + on a muted player, about a minute after hiding), so that is the viewer's heartbeat there: the + reaper can see a beat over a minute old, and two if one is lost on the way.""" + s = _Session(tmp_path, "hidden", finished=True, idle_s=idle_s) + sessions[(1, "12345")] = s + assert stream.reap_idle() == 0 + assert (1, "12345") in sessions and s.dir.exists() and not s.proc.killed + + +def test_an_abandoned_finished_session_is_still_gone_within_minutes(tmp_path, sessions): + sessions[(1, "12345")] = _Session(tmp_path, "gone", finished=True, idle_s=185) + assert stream.reap_idle() == 1 + + def test_the_periodic_pass_still_reaps_what_nobody_has_touched_for_the_idle_limit( tmp_path, sessions ): @@ -141,7 +162,7 @@ def test_every_reap_says_which_rule_took_the_session(tmp_path, sessions, caplog) def test_touching_a_session_keeps_it_through_the_short_rule(tmp_path, sessions): - s = _Session(tmp_path, "paused", finished=True, idle_s=90) + s = _Session(tmp_path, "paused", finished=True, idle_s=_PAST_THE_RULE) sessions[(1, "12345")] = s stream.touch_session(1, "12345") assert stream.reap_idle() == 0 @@ -184,7 +205,7 @@ def test_a_paused_viewers_checkpoint_keeps_their_session_alive( user = User(email="paused@example.com") db.add(user) db.commit() - s = _Session(tmp_path, "paused", finished=True, idle_s=90) + s = _Session(tmp_path, "paused", finished=True, idle_s=_PAST_THE_RULE) sessions[(user.id, "paused1")] = s resp = _client(db, user, override_dep).post( f"/api/plex/item/{it.rating_key}/progress", @@ -199,7 +220,7 @@ def test_starting_a_session_leaves_another_viewers_paused_one_alone( db, monkeypatch, tmp_path, sessions, override_dep ): """The opportunistic sweep this route used to run was not scoped to the key being started: it - walked the WHOLE registry with the 30 s finished rule, so pressing play on any item killed a + walked the WHOLE registry with the finished rule, so pressing play on any item killed a paused viewer's session — the very case the retention fix set out to protect.""" monkeypatch.setattr(state, "_configured_cache", True) monkeypatch.setattr(plex_paths, "local_media_path", lambda db_, fp: "/local/a.mkv") @@ -207,7 +228,7 @@ def test_starting_a_session_leaves_another_viewers_paused_one_alone( user = User(email="starter@example.com") db.add(user) db.commit() - paused = _Session(tmp_path, "paused", finished=True, idle_s=90) + paused = _Session(tmp_path, "paused", finished=True, idle_s=_PAST_THE_RULE) sessions[(999, "someone-else")] = paused def fake_start(db_, item, user_id, start, audio=None, aoff=0.0, multi=False, transcode=False): From df2997e9f3ba68776c85d5966cf8625067ad3792 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 19:20:22 +0100 Subject: [PATCH 3/7] fix(plex): a lost HLS session is reclaimed at the first answer The route answers 410 for a session it no longer has and keeps 404 for a segment not there yet. hls.js retries even a 4xx on a stream with one video variant, so the player waited about half a minute before asking again, and minutes in a throttled page. --- backend/app/plex/stream.py | 2 +- backend/app/routes/plex/playback.py | 14 +++-- backend/tests/test_plex_hls_file_route.py | 18 +++--- frontend/src/lib/hlsEngine.test.ts | 72 ++++++++++++++++++++++ frontend/src/lib/hlsEngine.ts | 73 ++++++++++++++--------- 5 files changed, 135 insertions(+), 44 deletions(-) diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index 1d36ffb0..e491773b 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -1184,7 +1184,7 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int: session gone that way, and one that lived on the same cadence because its phase was luckier). So the rule is sized to the throttled beat (`_THROTTLED_HEARTBEAT_S`). The heartbeat stops once the item is finished or before a position is known (`usePlexProgress.flush`) — both - states where reaping is correct, and where the player's 404 → reclaim recovers a viewer who + states where reaping is correct, and where the player's 410 → reclaim recovers a viewer who scrubs back in anyway. Do not restate this as "whether or not it is playing": a review found that claim in two comments and not in the code. An earlier blanket `reap_abandoned` bought the same safety by keeping every finished session for ten minutes — each diff --git a/backend/app/routes/plex/playback.py b/backend/app/routes/plex/playback.py index fed16d5b..47477427 100644 --- a/backend/app/routes/plex/playback.py +++ b/backend/app/routes/plex/playback.py @@ -330,7 +330,7 @@ def item_progress( # stream.reap_idle). It is NOT every open player: `usePlexProgress.flush` sends nothing once the # finish latch is up, or before the first position is known. Those are the states in which # reaping is the right answer anyway, and a viewer who scrubs back into a reaped session gets - # the same 404 → reclaim the player already handles. + # the same 410 → reclaim the player already handles. plex_stream.touch_session(user.id, str(rating_key)) position = max(0, int(body.position_seconds)) duration = int(body.duration_seconds) @@ -610,7 +610,9 @@ def stream_hls_file( is per (key, viewer, offset, tag) — so resolving by the key alone handed the old player the new session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong bytes with a 200, which the player has no way to notice. A request for a - session that is no longer the registered one is a 404, and the player reclaims at the playhead. + session that is no longer the registered one is a 410, and the player reclaims at the playhead. + 410, not 404: a 404 here means "that segment is not there yet", worth hls.js's retries, and the + player can only read the status (a gone session retried for half a minute behind that 404). That reclaim is also why a request with no live session is no longer auto-started here: the auto-start could only ever begin at 0, silently throwing away the viewer's position.""" @@ -626,7 +628,7 @@ def stream_hls_file( raise not_found("file") s = plex_stream.current_session(user.id, str(rating_key)) if s is None or s.id != session_id: - raise refused(404, "session_gone") + raise refused(410, "session_gone") path = s.dir / filename # Only a name THIS session can actually produce is worth waiting for: its own playlists, and a # segment of one of its own renditions whose predecessor is already on disk (ffmpeg writes them @@ -649,13 +651,13 @@ def stream_hls_file( except OSError: # The session was torn down between the wait and the read — teardown moves the whole # directory aside and deletes it, and nothing holds `_lock` across those two steps. It has - # to present as the same 404 a gone session does: the player's recovery is keyed on the - # STATUS (`code === 404`), so + # to present as the same 410 a gone session does: the player's recovery is keyed on the + # STATUS, so # a 500 here skipped the reclaim and ended playback with "couldn't play" instead of # restarting at the playhead. Ordinary trigger: `loadSession` awaits the new session — which # kills and rmtrees the old one — before destroying the old hls instance, so the outgoing # instance's in-flight segment request races the rmtree on every seek-restart. - raise refused(404, "session_gone") from None + raise refused(410, "session_gone") from None return Response( content=content, media_type=_HLS_CT[ext], diff --git a/backend/tests/test_plex_hls_file_route.py b/backend/tests/test_plex_hls_file_route.py index bf58d4b4..c60ff471 100644 --- a/backend/tests/test_plex_hls_file_route.py +++ b/backend/tests/test_plex_hls_file_route.py @@ -88,10 +88,10 @@ def test_another_viewers_session_of_the_same_item_is_not_this_viewers(client, tm (d / "index.m3u8").write_bytes(b"#EXTM3U") sessions[(7, "m1")] = _Session("sid-a", d) r = client.get("/api/plex/stream/m1/hls/sid-a/index.m3u8") - assert r.status_code == 404, r.text + assert r.status_code == 410, r.text -def test_a_segment_of_a_replaced_session_is_a_404_not_the_new_sessions_bytes( +def test_a_segment_of_a_replaced_session_is_gone_not_the_new_sessions_bytes( client, tmp_path, sessions ): """Viewer A is at 0:00; viewer B seeks to 25:00 and their session takes the registry slot. A's @@ -102,15 +102,15 @@ def test_a_segment_of_a_replaced_session_is_a_404_not_the_new_sessions_bytes( (theirs / "seg_12.ts").write_bytes(b"theirs") sessions[(VIEWER, "m1")] = _Session("sid-b", theirs) r = client.get("/api/plex/stream/m1/hls/sid-a/seg_12.ts") - assert r.status_code == 404, r.text + assert r.status_code == 410, r.text assert b"theirs" not in r.content -def test_a_playlist_with_no_live_session_is_a_404_not_a_silent_restart_at_zero( +def test_a_playlist_with_no_live_session_is_gone_not_a_silent_restart_at_zero( client, monkeypatch, sessions ): """The entry playlist used to auto-start a session when none was registered — always at 0.0, - silently throwing away the viewer's position. A 404 sends the player through its own reclaim, + silently throwing away the viewer's position. A 410 sends the player through its own reclaim, which restarts at the playhead.""" started: list[float] = [] monkeypatch.setattr( @@ -119,7 +119,7 @@ def test_a_playlist_with_no_live_session_is_a_404_not_a_silent_restart_at_zero( lambda *a, **k: started.append(a[3] if len(a) > 3 else 0.0), ) r = client.get("/api/plex/stream/m1/hls/sid-gone/index.m3u8") - assert r.status_code == 404, r.text + assert r.status_code == 410, r.text assert started == [] @@ -235,12 +235,12 @@ def test_an_absurd_segment_index_is_a_404_not_a_500(client, tmp_path, sessions, assert r.status_code == 404, r.text -def test_a_session_torn_down_during_the_read_is_a_404_not_a_500( +def test_a_session_torn_down_during_the_read_is_gone_not_a_500( client, monkeypatch, tmp_path, sessions ): """Teardown moves the whole directory aside and nothing holds `_lock` across the wait and the read, so the file can vanish in between — on every seek-restart, since `loadSession` awaits the new - session before destroying the old hls instance. The player's recovery is keyed on the 404 + session before destroying the old hls instance. The player's recovery is keyed on the STATUS, so a 500 here skipped the reclaim and ended playback with "couldn't play".""" d = tmp_path / "m1_0_0_+0.00" d.mkdir() @@ -249,7 +249,7 @@ def test_a_session_torn_down_during_the_read_is_a_404_not_a_500( # The file existed when the wait checked, and is gone by the time the route reads it. monkeypatch.setattr(plex_stream, "wait_for", lambda *a, **k: True) r = client.get("/api/plex/stream/m1/hls/sid-a/seg_12.ts") - assert r.status_code == 404, r.text + assert r.status_code == 410, r.text def test_a_dead_producer_is_not_waited_out(client, tmp_path, sessions): diff --git a/frontend/src/lib/hlsEngine.test.ts b/frontend/src/lib/hlsEngine.test.ts index 703933ba..25b3feb2 100644 --- a/frontend/src/lib/hlsEngine.test.ts +++ b/frontend/src/lib/hlsEngine.test.ts @@ -65,6 +65,27 @@ class FakeHls implements HlsPort { lost404() { this.lost(404); } + /** A request answered "this session is gone". hls.js only reports it (not fatal) and retries for + * half a minute: with one video variant it has nowhere else to go. */ + gone() { + this.handlers.onError({ + fatal: false, + network: true, + status: 410, + type: "networkError", + details: "fragLoadError", + }); + } + /** hls.js reports a failed request it will retry by itself. */ + retrying(status: number) { + this.handlers.onError({ + fatal: false, + network: true, + status, + type: "networkError", + details: "fragLoadError", + }); + } /** hls.js gave up on a playlist or fragment with this answer (null: no answer at all). */ lost(status: number | null) { this.handlers.onError({ @@ -418,6 +439,57 @@ describe("a session the server no longer has (404)", () => { }); }); +describe("a session the server says is gone (410)", () => { + async function playing() { + const s = setup(); + s.engine.start("A", 600); + await s.asks[0]!.resolve(hls("A", 600)); + s.hlses[0]!.handlers.onManifestParsed(); + s.media.fire("canplay"); + s.state.position = 1299; + return s; + } + + it("is reclaimed at the first answer, not after hls.js's half minute of retries", async () => { + const { asks, hlses } = await playing(); + hlses[0]!.gone(); + expect(asks[1]!.startAt).toBe(1299); + }); + + it("keeps the running picture until the new session answers", async () => { + const { engine, hlses } = await playing(); + expect(engine.getSnapshot().ready).toBe(true); + hlses[0]!.gone(); + expect(engine.getSnapshot().ready).toBe(true); + }); + + it("asks once, whatever the same instance still reports", async () => { + const { asks, hlses } = await playing(); + hlses[0]!.gone(); + hlses[0]!.gone(); + hlses[0]!.lost(410); + expect(asks).toHaveLength(2); + }); + + it(`shares the reclaim budget, and says so after ${MAX_SESSION_RECLAIMS}`, async () => { + const { engine, asks, hlses } = setup(); + engine.start("A", 600); + for (let i = 0; i <= MAX_SESSION_RECLAIMS; i++) { + await asks[i]!.resolve(hls("A", 600)); + hlses[i]!.gone(); + } + expect(asks).toHaveLength(MAX_SESSION_RECLAIMS + 1); + expect(engine.getSnapshot().error).toBe("generic"); + }); + + it("leaves a segment that is merely not there yet to hls.js's retries", async () => { + const { engine, asks, hlses } = await playing(); + hlses[0]!.retrying(404); + expect(asks).toHaveLength(1); + expect(engine.getSnapshot().error).toBeNull(); + }); +}); + describe("a server that is restarting (a deploy)", () => { // The gateway answers 502 while the app restarts, and nothing at all may answer (status 0): the // stream is not broken, the server is away. The buffer keeps playing meanwhile. diff --git a/frontend/src/lib/hlsEngine.ts b/frontend/src/lib/hlsEngine.ts index 64aa7305..99f14a70 100644 --- a/frontend/src/lib/hlsEngine.ts +++ b/frontend/src/lib/hlsEngine.ts @@ -14,7 +14,7 @@ import type { HlsLevelInfo } from "./plexStats"; // once: twenty presses of ← at key-repeat pace past the buffer were twenty requests, twenty ffmpeg // spawns, and five seconds before the picture moved again. -// Reclaims of a 404'd HLS session, within `RECLAIM_WINDOW_MS`, before the player stops and says so. +// Reclaims of a lost HLS session, within `RECLAIM_WINDOW_MS`, before the player stops and says so. // Two covers the real case (the session was reaped, once) without letting a server that cannot serve // the stream keep the viewer on a spinner (see `onHlsError`). export const MAX_SESSION_RECLAIMS = 2; @@ -33,6 +33,8 @@ export const RECLAIM_WINDOW_MS = 120_000; export const AWAY_WINDOW_MS = 180_000; const AWAY_RETRY_MS = [2_000, 4_000, 8_000, 15_000]; const SERVER_AWAY = new Set([0, 502, 503, 504]); +// The server's answer for a session it no longer has; its 404 means a segment not there yet. +const SESSION_GONE = 410; /** Why a session could not play, in the three buckets the player has a sentence for. */ type EngineError = "transcode" | "notFound" | "generic"; @@ -183,7 +185,7 @@ export class HlsEngine { // the way back to the first item then sent a second request beside the one still out for it. private readonly inFlight = new Map(); private queued: Request | null = null; - // How often a session the server answered 404 for may be reclaimed (see `onHlsError`). + // How often a session the server lost may be reclaimed (see `onHlsError`). private reclaims = { n: 0, at: 0 }; // Since when the server has been away, and how often it was asked since (see `serverAway`). private away: { since: number; tries: number } | null = null; @@ -459,42 +461,57 @@ export class HlsEngine { }; } - // A fatal HLS error (the session died, the file became unreadable) would otherwise leave the - // spinner up forever — it is said instead. + // A fatal HLS error (the file became unreadable) would otherwise leave the spinner up forever — it + // is said instead. // - // Except a 404: the session is gone, not broken. Either the server reaped it (a pause longer than - // its idle limit, a deploy) — hls.js never retries a 4xx, so this used to end playback with - // "couldn't play" after the buffer ran out — or it was superseded before it ever played (two - // requests for one item: the server's LAST spawn wins). The server answers a superseded session - // exactly so the client asks again; the reclaim is that asking. + // Except a session the server no longer has (410): gone, not broken. Either the server reaped it + // or restarted, or it was superseded before it ever played (two requests for one item: the + // server's LAST spawn wins), and the reclaim is the asking again that answer wants. It does not + // wait for hls.js to give up: with one video variant it has nothing to switch to, so it retries + // even a 4xx, six times over about half a minute (measured: seven requests, then the reclaim), and + // in a throttled page once a minute (four minutes). Quiet, since the buffer is still playing. // - // Bounded by the budget, not by whether the session got as far as playing: that belongs to ONE - // session, and every restart builds a fresh one, so it could never bound a loop. The server also answers - // 404 for "that segment is not there YET" — indistinguishable on the wire — so a transcode falling - // behind realtime restarted ffmpeg forever behind a spinner that never became an error. + // And except a fatal 404: a segment still not there after those retries, so a transcode has fallen + // behind, and asking again is all that is left. + // + // Both are bounded by the budget, not by whether the session got as far as playing: that belongs to + // ONE session, and every restart builds a fresh one, so it could never bound a loop. A transcode + // falling behind realtime restarted ffmpeg forever behind a spinner that never became an error. private onHlsError(req: Request, media: HTMLMediaElement, e: HlsFailure): void { this.deps.onHlsError?.(e); - if (!e.fatal || !this.current(req)) return; + if (!this.current(req)) return; + if (e.network && e.status === SESSION_GONE) { + this.reclaim(req, media, this.snapshot.ready); + return; + } + if (!e.fatal) return; if (e.network && SERVER_AWAY.has(e.status ?? 0) && this.serverAway(req)) return; if (e.network && e.status === 404) { - const now = this.deps.now(); - // A new window when the last loss was long enough ago. The anchor moves with EVERY loss: left - // at the window's start, the window tumbled instead of rolling, and any loss period longer - // than half the window reset the budget on every second loss (measured: 200 losses 70 s apart - // → 200 reclaims, never an error). - if (now - this.reclaims.at > RECLAIM_WINDOW_MS) this.reclaims.n = 0; - this.reclaims.at = now; - if (this.reclaims.n < MAX_SESSION_RECLAIMS) { - this.reclaims.n += 1; - // The server dropped it already; nothing to stop. - this.attachedSession = null; - this.start(req.id, this.playhead(), this.wantsPlay(media)); - return; - } + this.reclaim(req, media, false); + return; } this.set({ error: "generic" }); } + /** Ask for `req`'s item again where the viewer is, while the budget lasts; then say it cannot play. */ + private reclaim(req: Request, media: HTMLMediaElement, quiet: boolean): void { + const now = this.deps.now(); + // A new window when the last loss was long enough ago. The anchor moves with EVERY loss: left + // at the window's start, the window tumbled instead of rolling, and any loss period longer + // than half the window reset the budget on every second loss (measured: 200 losses 70 s apart + // → 200 reclaims, never an error). + if (now - this.reclaims.at > RECLAIM_WINDOW_MS) this.reclaims.n = 0; + this.reclaims.at = now; + if (this.reclaims.n >= MAX_SESSION_RECLAIMS) { + this.set({ error: "generic" }); + return; + } + this.reclaims.n += 1; + // The server dropped it already; nothing to stop. + this.attachedSession = null; + this.start(req.id, this.playhead(), this.wantsPlay(media), quiet); + } + /** The server is away (see `AWAY_WINDOW_MS`): ask for `req`'s item again where the viewer is, * after a delay that grows with each try. False once it has been away too long — the caller then * says it cannot play. A restart the viewer asks for meanwhile (a seek, another item) replaces the From 469c8aa4811ee0ad18a203751f7152fcb6f13fdd Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 20:31:41 +0100 Subject: [PATCH 4/7] test(e2e): the drag onto the rail waits for the rail's lists --- frontend/e2e/playlists.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/e2e/playlists.spec.ts b/frontend/e2e/playlists.spec.ts index 08cc28c9..d63ed3dc 100644 --- a/frontend/e2e/playlists.spec.ts +++ b/frontend/e2e/playlists.spec.ts @@ -66,11 +66,13 @@ async function carry(page: Page, grip: Locator, to: { x: number; y: number }) { /** The same onto the rail. An unpinned one is a tab until the drag's own hover peeks it open. */ async function carryToRail(page: Page, grip: Locator, target: Locator) { + const tab = page.getByRole("button", { name: "Pin Playlists open" }); + // The rail's lists answer after the rows: until then a pinned rail shows neither the list nor a tab. + await target.or(tab).first().waitFor(); if (await target.isVisible()) { await carry(page, grip, await centre(target)); return; } - const tab = page.getByRole("button", { name: "Pin Playlists open" }); await carry(page, grip, await centre(tab)); await expect(target).toBeVisible(); await page.mouse.move(...(Object.values(await centre(target)) as [number, number]), { steps: 6 }); From 3f6638bf9609348605d196941054616f3b751594 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 21:10:51 +0100 Subject: [PATCH 5/7] fix(plex): a seek during a quiet reclaim is no longer undone A quiet request leaves `live` alone, so a seek inside the old session's range went native and the answer then jumped back to where it asked from. `seek` now also goes by `pending`; the server-away retry had the same hole. Comments that still named 404 for a lost session say 410, and the hls.js retry comment now tells a finished playlist from a growing one. --- backend/app/plex/stream.py | 8 +++--- backend/tests/test_hls_reaper.py | 2 +- backend/tests/test_plex_hls_file_route.py | 2 +- frontend/src/lib/hlsEngine.test.ts | 27 ++++++++++++++++++- frontend/src/lib/hlsEngine.ts | 19 +++++++------ .../modules/plex/PlexPlayer.hls.dom.test.tsx | 15 +++++------ 6 files changed, 50 insertions(+), 23 deletions(-) diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index e491773b..8e25bbdb 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -125,7 +125,7 @@ class HlsSession: # replaces the viewer's registered session for the item — and the old player's next request # for `seg_12.ts`, resolved by (viewer, rating_key) alone, was answered 200 with the NEW session's twelfth segment, produced from a different offset # under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it. - # A request naming a session that is no longer registered is a 404, + # A request naming a session that is no longer registered is a 410, # which the player recovers from by restarting at the playhead. self.id = secrets.token_urlsafe(9) self.key = key # the item's rating_key (what the logs name) @@ -729,7 +729,7 @@ def _enforce_caps( whole segmented copy of the film under `_HLS_ROOT`, written faster than realtime. Counted only by the transcode cap, N distinct items were N ffmpegs and N film-sized scratch directories. - Somebody's playback stops at each drop. The route turns the missing session into a retryable 404 + Somebody's playback stops at each drop. The route turns the missing session into a 410 and the player restarts at the playhead, so from the server nothing failed — and "my film keeps rebuffering" has no other explanation anywhere, hence the line.""" @@ -918,7 +918,7 @@ def start_session( # stale and stops the session it names (`stop_session`), which stopped the playback that won. # Never None either: the POST /session route turns a None return into a "Media file not found # on disk" 404 — dishonest for a file that exists. A dead session instead degrades to a - # retryable 404 ("No active playback session" / "Not available yet") the client re-requests. + # 410 ("session_gone") on its URLs, and the client asks again. return s s = respawned ready = wait_for(_seg(0), timeout=25.0, proc=s.proc) @@ -1200,7 +1200,7 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int: _sessions.pop(reg, None) doomed.append(_detach(s)) reaped.append((s.key, "finished_idle" if done else "idle", int(idle))) - # The viewer of a session wrongly taken sees only a 404 later, so this line is the one record of why. + # The viewer of a session wrongly taken sees only a 410 later, so this line is the one record of why. for key, reason, idle_s in reaped: log.info( "an idle session was reaped", diff --git a/backend/tests/test_hls_reaper.py b/backend/tests/test_hls_reaper.py index c4b14368..8560ab8d 100644 --- a/backend/tests/test_hls_reaper.py +++ b/backend/tests/test_hls_reaper.py @@ -138,7 +138,7 @@ def test_the_periodic_pass_still_reaps_what_nobody_has_touched_for_the_idle_limi def test_every_reap_says_which_rule_took_the_session(tmp_path, sessions, caplog): - """A session that vanishes leaves the viewer a 404 and nothing else: with no line here, a live + """A session that vanishes leaves the viewer a 410 and nothing else: with no line here, a live stall could only ever be blamed on the reaper by inference.""" sessions["1"] = _Session( tmp_path, "gone_done", finished=True, idle_s=stream._SESSION_IDLE_S - 5 diff --git a/backend/tests/test_plex_hls_file_route.py b/backend/tests/test_plex_hls_file_route.py index c60ff471..24e8ee9f 100644 --- a/backend/tests/test_plex_hls_file_route.py +++ b/backend/tests/test_plex_hls_file_route.py @@ -5,7 +5,7 @@ seek-restart — or a second viewer of the same item — replaces the registered viewer's next request for `seg_12.ts`, resolved by rating_key, was answered 200 with the NEW session's twelfth segment: bytes produced from a different `-ss` offset under `-copyts`, i.e. a timeline the client is not on. Wrong content with a success status is the one failure a player cannot -notice, and the client's reclaim-on-404 could never fire on it. +notice, and the client's reclaim could never fire on it. No `db` fixture: the route reads nothing from the database (its own `Depends(get_db)` went with the removed auto-start, and `current_user` is overridden wholesale), so these must not sit in the DB diff --git a/frontend/src/lib/hlsEngine.test.ts b/frontend/src/lib/hlsEngine.test.ts index 25b3feb2..38c54da3 100644 --- a/frontend/src/lib/hlsEngine.test.ts +++ b/frontend/src/lib/hlsEngine.test.ts @@ -374,7 +374,7 @@ describe("a failed session request", () => { }); }); -describe("a session the server no longer has (404)", () => { +describe("something that never arrives (a fatal 404)", () => { it("is reclaimed where the viewer is", async () => { const { engine, asks, hlses, state } = setup(); engine.start("A", 600); @@ -482,6 +482,17 @@ describe("a session the server says is gone (410)", () => { expect(engine.getSnapshot().error).toBe("generic"); }); + it("keeps a seek made while it is out, inside what the old session still reports", async () => { + const { engine, asks, hlses, media } = await playing(); + media.seekEnd = 800; + hlses[0]!.gone(); + media.currentTime = 42; + engine.seek("A", 1250); + expect(media.currentTime).toBe(42); // the outgoing media stays where it is + await asks[1]!.resolve(hls("A", 1299)); + expect(asks.map((a) => a.startAt)).toEqual([600, 1299, 1250]); + }); + it("leaves a segment that is merely not there yet to hls.js's retries", async () => { const { engine, asks, hlses } = await playing(); hlses[0]!.retrying(404); @@ -493,6 +504,20 @@ describe("a session the server says is gone (410)", () => { describe("a server that is restarting (a deploy)", () => { // The gateway answers 502 while the app restarts, and nothing at all may answer (status 0): the // stream is not broken, the server is away. The buffer keeps playing meanwhile. + it("keeps a seek made while the retry is out", async () => { + const { engine, asks, hlses, state, fire, media } = setup(); + engine.start("A", 600); + await asks[0]!.resolve(hls("A", 600)); + hlses[0]!.handlers.onManifestParsed(); + media.seekEnd = 800; + state.position = 1299; + hlses[0]!.lost(502); + expect(fire()).toBe(true); + engine.seek("A", 1250); + await asks[1]!.resolve(hls("A", 1299)); + expect(asks.map((a) => a.startAt)).toEqual([600, 1299, 1250]); + }); + it.each([502, 503, 504, 0, null])( "is waited out on %s: no error, and the stream comes back where the viewer is", async (status) => { diff --git a/frontend/src/lib/hlsEngine.ts b/frontend/src/lib/hlsEngine.ts index 99f14a70..fc764040 100644 --- a/frontend/src/lib/hlsEngine.ts +++ b/frontend/src/lib/hlsEngine.ts @@ -285,7 +285,7 @@ export class HlsEngine { /** (Re)start `id` at `startAt` seconds. Another item's request goes out at once; one for the item * already being asked for waits for that answer (see the header). `quiet`: the attached stream is - * still playing from its buffer (the server went away under it), so it stays the player's until + * still playing from its buffer (the server went away or lost its session), so it stays the player's until * the answer replaces it — no "Loading…" over a running picture, its clock still true. */ start(id: string, startAt: number, play = true, quiet = false): void { if (!this.deps.media()) return; @@ -467,12 +467,14 @@ export class HlsEngine { // Except a session the server no longer has (410): gone, not broken. Either the server reaped it // or restarted, or it was superseded before it ever played (two requests for one item: the // server's LAST spawn wins), and the reclaim is the asking again that answer wants. It does not - // wait for hls.js to give up: with one video variant it has nothing to switch to, so it retries - // even a 4xx, six times over about half a minute (measured: seven requests, then the reclaim), and - // in a throttled page once a minute (four minutes). Quiet, since the buffer is still playing. + // wait for hls.js to give up. On a finished playlist, with one video variant and nothing to + // switch to, it retries even a 4xx, six times over about half a minute (measured: seven requests, + // then the reclaim), and in a throttled page once a minute (four minutes). On a growing one it + // takes the segment for a gap and moves on, so the error is never fatal. Either way the first + // answer is reported. Quiet, since the buffer is still playing. // - // And except a fatal 404: a segment still not there after those retries, so a transcode has fallen - // behind, and asking again is all that is left. + // And except a fatal 404: hls.js gave up on something the server still does not have, and asking + // again is all that is left. // // Both are bounded by the budget, not by whether the session got as far as playing: that belongs to // ONE session, and every restart builds a fresh one, so it could never bound a loop. A transcode @@ -565,9 +567,10 @@ export class HlsEngine { const rel = T - this.sessionStart.current; // Native only when the attached media IS the current session. While one is still being asked // for, `sessionStart` and `seekable` describe the OLD one: a native seek there left the pending - // restart current, and its answer then jumped to the earlier target. + // restart current, and its answer then jumped to the earlier target. A quiet request leaves + // `live` alone, so `pending` is what says one is out. const end = media.seekable.length ? media.seekable.end(0) : 0; - if (this.live && rel >= 0 && rel <= end + 0.5) { + if (this.live && !this.pending && rel >= 0 && rel <= end + 0.5) { media.currentTime = rel; // keeps playing on its own if (play) media.play().catch(() => {}); } else { diff --git a/frontend/src/modules/plex/PlexPlayer.hls.dom.test.tsx b/frontend/src/modules/plex/PlexPlayer.hls.dom.test.tsx index 40ac9c05..69167485 100644 --- a/frontend/src/modules/plex/PlexPlayer.hls.dom.test.tsx +++ b/frontend/src/modules/plex/PlexPlayer.hls.dom.test.tsx @@ -134,13 +134,13 @@ const lost404 = { fatal: true, type: Hls.ErrorTypes.NETWORK_ERROR, details: "fragLoadError", - response: { code: 404, text: "No active playback session" }, + response: { code: 404, text: "Not available yet" }, }; -describe("an HLS session the server no longer has", () => { +describe("an HLS segment that never arrives (a fatal 404)", () => { it("is restarted where the viewer is, instead of ending playback", async () => { - // The server reaps a session nobody fetches from (a long pause; a deploy). hls.js never retries - // a 4xx, so the next segment's 404 used to end playback with "couldn't play". + // Once hls.js has given up on it, the 404 used to end playback with "couldn't play". A session + // the server no longer has answers 410 instead: `hlsEngine.test.ts` holds that case. const session = mount(); await waitFor(() => expect(hlsInstances).toHaveLength(1)); act(() => hlsInstances[0]!.emit(Hls.Events.MANIFEST_PARSED)); @@ -155,10 +155,9 @@ describe("an HLS session the server no longer has", () => { it("gives up and says so when the reclaimed session keeps 404ing", async () => { // `parsed` cannot bound this: it lives on ONE hls instance and every restart builds a fresh one, - // so a session that parses its manifest and then 404s re-arms the guard every cycle. The server - // also answers 404 for "that segment is not there YET" (a transcode falling behind realtime), - // which is indistinguishable on the wire — so the player respawned ffmpeg forever behind a - // spinner that never became an error. + // so a session that parses its manifest and then 404s re-arms the guard every cycle. A transcode + // falling behind realtime answers 404 for "that segment is not there YET" every time, so the + // player respawned ffmpeg forever behind a spinner that never became an error. const session = mount(); const v = () => document.querySelector("video") as Media; // The first session plus the two it is allowed to reclaim; the third 404 has nothing left. From 38fc95d77269a140fe75a53ec2c2bd77c4cd119e Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 21:22:17 +0100 Subject: [PATCH 6/7] docs(plex): a quiet reclaim's replay is named as accepted --- frontend/src/lib/hlsEngine.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/hlsEngine.ts b/frontend/src/lib/hlsEngine.ts index fc764040..4436c63a 100644 --- a/frontend/src/lib/hlsEngine.ts +++ b/frontend/src/lib/hlsEngine.ts @@ -471,7 +471,9 @@ export class HlsEngine { // switch to, it retries even a 4xx, six times over about half a minute (measured: seven requests, // then the reclaim), and in a throttled page once a minute (four minutes). On a growing one it // takes the segment for a gap and moves on, so the error is never fatal. Either way the first - // answer is reported. Quiet, since the buffer is still playing. + // answer is reported. Quiet, since the buffer is still playing: the picture runs on until the + // answer, which restarts from where it was asked, so what played meanwhile is replayed (accepted: + // a brief loading state marks it, and nothing the viewer has saved is lost). // // And except a fatal 404: hls.js gave up on something the server still does not have, and asking // again is all that is left. From c6831090b5d476bb30860f6c3940e707d0408afc Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 3 Nov 2026 21:35:19 +0100 Subject: [PATCH 7/7] chore(release): 0.116.2 --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- frontend/src/content/release-notes/en.json | 8 ++++++++ frontend/src/content/release-notes/hu.json | 8 ++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5820b3c..2c553c0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.116.2 — 2026-11-03 + +### Fixed + +- A Plex video paused or muted in a background tab no longer loses its session after a minute, and if a session is lost anyway, playback comes back at once instead of after half a minute. + ## 0.116.1 — 2026-11-03 ### Fixed diff --git a/VERSION b/VERSION index 6e7dd678..fdf3a3e5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.116.1 +0.116.2 diff --git a/frontend/src/content/release-notes/en.json b/frontend/src/content/release-notes/en.json index f7d72552..3711573b 100644 --- a/frontend/src/content/release-notes/en.json +++ b/frontend/src/content/release-notes/en.json @@ -1,4 +1,12 @@ [ + { + "version": "0.116.2", + "date": "2026-11-03", + "new": [], + "fixed": [ + "A Plex video paused or muted in a background tab no longer loses its session after a minute, and if a session is lost anyway, playback comes back at once instead of after half a minute." + ] + }, { "version": "0.116.1", "date": "2026-11-03", diff --git a/frontend/src/content/release-notes/hu.json b/frontend/src/content/release-notes/hu.json index 2a52bf70..91085221 100644 --- a/frontend/src/content/release-notes/hu.json +++ b/frontend/src/content/release-notes/hu.json @@ -1,4 +1,12 @@ [ + { + "version": "0.116.2", + "date": "2026-11-03", + "new": [], + "fixed": [ + "A háttérfülön szüneteltetett vagy némított Plex-videó már nem veszíti el a munkamenetét egy perc után, és ha mégis elveszne, a lejátszás azonnal visszatér, nem fél perc múlva." + ] + }, { "version": "0.116.1", "date": "2026-11-03",