diff --git a/frontend/src/lib/hlsEngine.test.ts b/frontend/src/lib/hlsEngine.test.ts index 4f292643..703933ba 100644 --- a/frontend/src/lib/hlsEngine.test.ts +++ b/frontend/src/lib/hlsEngine.test.ts @@ -3,6 +3,7 @@ import { describe, it, expect, vi } from "vitest"; import type { PlexPlaySession } from "./api"; import { + AWAY_WINDOW_MS, HlsEngine, MAX_SESSION_RECLAIMS, RECLAIM_WINDOW_MS, @@ -62,12 +63,16 @@ class FakeHls implements HlsPort { return { index: 0, total: 1 }; } lost404() { + this.lost(404); + } + /** hls.js gave up on a playlist or fragment with this answer (null: no answer at all). */ + lost(status: number | null) { this.handlers.onError({ fatal: true, network: true, - status: 404, + status, type: "networkError", - details: "fragLoadError", + details: "levelLoadError", }); } } @@ -87,6 +92,8 @@ function setup(over: Partial = {}) { const asks: Ask[] = []; const hlses: FakeHls[] = []; const stops: [string, string][] = []; + // The engine's timers, run by hand: `fire` runs the next one still armed. + const timers: { fn: () => void; ms: number; armed: boolean }[] = []; let clock = 1_000_000; const state = { position: 0, duration: 3000, params: { multi: false, audio: null, aoff: 0 } }; const engine = new HlsEngine({ @@ -118,6 +125,11 @@ function setup(over: Partial = {}) { duration: () => state.duration, now: () => clock, statusOf: (e) => (e as { status?: number }).status, + later: (fn, ms) => { + const t = { fn, ms, armed: true }; + timers.push(t); + return () => void (t.armed = false); + }, ...over, }); return { @@ -127,7 +139,17 @@ function setup(over: Partial = {}) { hlses, stops, state, + timers, advance: (ms: number) => void (clock += ms), + /** Let the next armed timer run (its delay passes on the clock too); false when none is. */ + fire: () => { + const t = timers.find((x) => x.armed); + if (!t) return false; + t.armed = false; + clock += t.ms; + t.fn(); + return true; + }, }; } @@ -396,6 +418,101 @@ describe("a session the server no longer has (404)", () => { }); }); +describe("a server that is restarting (a deploy)", () => { + // The gateway answers 502 while the app restarts, and nothing at all may answer (status 0): the + // stream is not broken, the server is away. The buffer keeps playing meanwhile. + it.each([502, 503, 504, 0, null])( + "is waited out on %s: no error, and the stream comes back where the viewer is", + async (status) => { + const { engine, asks, hlses, state, fire, media } = setup(); + engine.start("A", 600); + await asks[0]!.resolve(hls("A", 600)); + hlses[0]!.handlers.onManifestParsed(); + state.position = 1299; + hlses[0]!.lost(status); + expect(engine.getSnapshot().error).toBeNull(); + expect(asks).toHaveLength(1); // not at once: the server is still away + expect(fire()).toBe(true); + expect(asks[1]!.startAt).toBe(1299); + // The buffered picture is still playing while the server is asked: no "Loading…" over it. + expect(engine.getSnapshot().ready).toBe(true); + await asks[1]!.reject({ status: 502 }); // still away + expect(engine.getSnapshot().error).toBeNull(); + state.position = 1310; + expect(fire()).toBe(true); + await asks[2]!.resolve(hls("A", 1310)); + expect(hlses[1]!.loaded).toContain("?t=1310"); + expect(engine.getSnapshot().ready).toBe(false); // the new stream is loading now + hlses[1]!.handlers.onManifestParsed(); + expect(engine.getSnapshot()).toMatchObject({ ready: true, error: null }); + expect(media.plays).toBeGreaterThan(0); + }, + ); + + it("is waited out when the very first request finds it away", async () => { + const { engine, asks, fire } = setup(); + engine.start("A", 0); + await asks[0]!.reject({ status: 0 }); + expect(engine.getSnapshot()).toMatchObject({ ready: false, error: null }); + expect(fire()).toBe(true); + await asks[1]!.resolve(hls("A")); + expect(engine.getSnapshot().error).toBeNull(); + }); + + it("asks less and less often, and says so once it has been away too long", async () => { + const { engine, asks, hlses, fire, timers } = setup(); + engine.start("A", 0); + await asks[0]!.resolve(hls("A")); + hlses[0]!.handlers.onManifestParsed(); + hlses[0]!.lost(502); + let i = 1; + while (fire()) await asks[i++]!.reject({ status: 502 }); + const delays = timers.map((t) => t.ms); + expect(delays[1]!).toBeGreaterThan(delays[0]!); + expect(delays.reduce((a, b) => a + b, 0)).toBeGreaterThan(AWAY_WINDOW_MS); + expect(engine.getSnapshot().error).toBe("generic"); + }); + + it("starts the wait over after the server came back", async () => { + const { engine, asks, hlses, fire, advance } = setup(); + engine.start("A", 0); + await asks[0]!.resolve(hls("A")); + hlses[0]!.handlers.onManifestParsed(); + hlses[0]!.lost(502); + fire(); + await asks[1]!.resolve(hls("A")); + hlses[1]!.handlers.onManifestParsed(); + advance(AWAY_WINDOW_MS + 1); + hlses[1]!.lost(502); // a second deploy, long after the first + expect(engine.getSnapshot().error).toBeNull(); + }); + + it("drops a pending retry once the viewer seeks or leaves the item", async () => { + const { engine, asks, hlses, fire } = setup(); + engine.start("A", 0); + await asks[0]!.resolve(hls("A")); + hlses[0]!.handlers.onManifestParsed(); + hlses[0]!.lost(502); + engine.seek("A", 2000); // the viewer's own restart goes out instead + expect(asks).toHaveLength(2); + fire(); + expect(asks).toHaveLength(2); + await asks[1]!.reject({ status: 502 }); + engine.leaveItem(); + fire(); + expect(asks).toHaveLength(2); + }); + + it("still says a real failure at once", async () => { + const { engine, asks, hlses } = setup(); + engine.start("A", 0); + await asks[0]!.resolve(hls("A")); + hlses[0]!.handlers.onManifestParsed(); + hlses[0]!.lost(500); + expect(engine.getSnapshot().error).toBe("generic"); + }); +}); + describe("play intent while a session is asked for", () => { it("goes into the pending session, and never plays the outgoing media", async () => { const { engine, asks, media, hlses } = setup(); diff --git a/frontend/src/lib/hlsEngine.ts b/frontend/src/lib/hlsEngine.ts index b4c26c43..64aa7305 100644 --- a/frontend/src/lib/hlsEngine.ts +++ b/frontend/src/lib/hlsEngine.ts @@ -25,6 +25,15 @@ export const MAX_SESSION_RECLAIMS = 2; // Two losses inside two minutes is a session that keeps being lost; one an hour apart is not. export const RECLAIM_WINDOW_MS = 120_000; +// A server that is restarting (a deploy) is away, not broken: the gateway in front of it answers +// 502/503/504 while the app comes back, and nothing at all may answer (0). The stream it served is +// gone with it, but the buffer keeps playing — so the engine asks again where the viewer is, less and +// less often, and says it cannot play only once the server has been away this long. It used to say +// so at the first 502, over a picture still playing from the buffer, and never asked again. +export const AWAY_WINDOW_MS = 180_000; +const AWAY_RETRY_MS = [2_000, 4_000, 8_000, 15_000]; +const SERVER_AWAY = new Set([0, 502, 503, 504]); + /** Why a session could not play, in the three buckets the player has a sentence for. */ type EngineError = "transcode" | "notFound" | "generic"; @@ -88,7 +97,10 @@ export interface EngineDeps { /** The item's duration, seconds (0 = unknown). */ duration(): number; now(): number; - /** An HTTP status from a failed session request (undefined = not an HTTP error). */ + /** Run `fn` after `ms`; the returned function cancels it. */ + later(fn: () => void, ms: number): () => void; + /** An HTTP status from a failed session request (0 = nothing answered; undefined = not an HTTP + * error). */ statusOf(e: unknown): number | undefined; /** A request for a session went out: the player clears its stall spinner. */ onLoadStart?(): void; @@ -113,6 +125,7 @@ const UNCONNECTED: EngineDeps = { position: () => 0, duration: () => 0, now: () => 0, + later: () => () => {}, statusOf: () => undefined, }; @@ -121,6 +134,8 @@ interface Request { startAt: number; intent: Intent; gen: number; + /** Asked for while the attached stream still plays from its buffer (see `start`). */ + quiet: boolean; } export class HlsEngine { @@ -170,6 +185,9 @@ export class HlsEngine { private queued: Request | null = null; // How often a session the server answered 404 for may be reclaimed (see `onHlsError`). private reclaims = { n: 0, at: 0 }; + // Since when the server has been away, and how often it was asked since (see `serverAway`). + private away: { since: number; tries: number } | null = null; + private cancelRetry: () => void = () => {}; // Where the current — or still in-flight — session was ASKED to start. Written before the request // goes out, so "where the viewer is" has an answer before the first timeupdate. private loadStart = 0; @@ -244,12 +262,14 @@ export class HlsEngine { * (the player stays mounted across a queue advance). */ newItem(): void { this.reclaims = { n: 0, at: 0 }; + this.away = null; } /** Leaving the item: a session still in flight for it must not land after it, its stream goes, * and the server stops producing it. */ leaveItem(): void { this.gen++; // a queued request of this item is now stale, and is never sent + this.cancelRetry(); this.detachSession(); this.detachSession = () => {}; this.dropHls(); @@ -262,21 +282,28 @@ export class HlsEngine { } /** (Re)start `id` at `startAt` seconds. Another item's request goes out at once; one for the item - * already being asked for waits for that answer (see the header). */ - start(id: string, startAt: number, play = true): void { + * already being asked for waits for that answer (see the header). `quiet`: the attached stream is + * still playing from its buffer (the server went away under it), so it stays the player's until + * the answer replaces it — no "Loading…" over a running picture, its clock still true. */ + start(id: string, startAt: number, play = true, quiet = false): void { if (!this.deps.media()) return; + this.cancelRetry(); // this request is the one a pending retry would have sent const gen = ++this.gen; this.detachSession(); this.detachSession = () => {}; - this.live = false; const intent: Intent = { play }; this.pending = intent; this.loadStart = startAt; - this.deps.onLoadStart?.(); - // Clear the prior session's transcode flag up front, so the cold-start overlay cannot show a - // stale "Preparing stream…" for a session not known yet — set honestly from the answer. - this.set({ ready: false, error: null, transcode: false }); - const req: Request = { id, startAt, intent, gen }; + if (quiet) { + this.set({ error: null }); + } else { + this.live = false; + this.deps.onLoadStart?.(); + // Clear the prior session's transcode flag up front, so the cold-start overlay cannot show a + // stale "Preparing stream…" for a session not known yet — set honestly from the answer. + this.set({ ready: false, error: null, transcode: false }); + } + const req: Request = { id, startAt, intent, gen, quiet }; if (this.inFlight.has(id)) this.queued = req; else this.send(req); } @@ -313,7 +340,9 @@ export class HlsEngine { this.live = this.attachedId === req.id; // Say WHY playback can't start instead of an eternal spinner. 404 = the physical file isn't // reachable (a missing media mount or a wrong path map); 501 = the codec needs a full transcode. + // A server that is away is asked again instead, for a while. const status = this.deps.statusOf(e); + if (status !== undefined && SERVER_AWAY.has(status) && this.serverAway(req)) return; this.set({ error: status === 501 ? "transcode" : status === 404 ? "notFound" : "generic" }); } @@ -327,7 +356,10 @@ export class HlsEngine { } const media = this.deps.media(); if (!media) return; + this.away = null; // the server answered: a later absence is a new one const { intent, startAt, id } = req; + // The stream that kept playing is replaced from here: the new one loads like any other. + if (req.quiet) this.set({ ready: false }); const isTranscode = sess.mode === "hls" && !!sess.transcode; this.hw = isTranscode ? (sess.hw ?? false) : null; this.tonemap = isTranscode ? (sess.tonemap ?? false) : null; @@ -443,6 +475,7 @@ export class HlsEngine { private onHlsError(req: Request, media: HTMLMediaElement, e: HlsFailure): void { this.deps.onHlsError?.(e); if (!e.fatal || !this.current(req)) return; + if (e.network && SERVER_AWAY.has(e.status ?? 0) && this.serverAway(req)) return; if (e.network && e.status === 404) { const now = this.deps.now(); // A new window when the last loss was long enough ago. The anchor moves with EVERY loss: left @@ -462,6 +495,28 @@ export class HlsEngine { this.set({ error: "generic" }); } + /** The server is away (see `AWAY_WINDOW_MS`): ask for `req`'s item again where the viewer is, + * after a delay that grows with each try. False once it has been away too long — the caller then + * says it cannot play. A restart the viewer asks for meanwhile (a seek, another item) replaces the + * retry: `start` and `leaveItem` cancel it, and it is dropped if the generation moved on. */ + private serverAway(req: Request): boolean { + const now = this.deps.now(); + const away = (this.away ??= { since: now, tries: 0 }); + if (now - away.since > AWAY_WINDOW_MS) return false; + const ms = AWAY_RETRY_MS[Math.min(away.tries, AWAY_RETRY_MS.length - 1)]!; + away.tries += 1; + const gen = this.gen; + this.cancelRetry(); + this.cancelRetry = this.deps.later(() => { + this.cancelRetry = () => {}; + const media = this.deps.media(); + if (!this.alive || this.gen !== gen || !media) return; + // Quiet while what was attached had played: its buffer may still be running. + this.start(req.id, this.playhead(), this.wantsPlay(media), this.snapshot.ready); + }, ms); + return true; + } + /** Where the viewer is: the ticked position, or — before playback has ticked (a session still * loading, or a resumed one that has stayed paused) — where the session was ASKED to start. * Anything that restarts "here" must use this: reading the ticked 0 restarted a resume at 20:00 diff --git a/frontend/src/modules/plex/PlexPlayer.tsx b/frontend/src/modules/plex/PlexPlayer.tsx index e77157f2..cc805349 100644 --- a/frontend/src/modules/plex/PlexPlayer.tsx +++ b/frontend/src/modules/plex/PlexPlayer.tsx @@ -232,7 +232,13 @@ export default function PlexPlayer({ itemId, onClose, queue, fromStart = false } position: () => absRef.current, duration: () => durationRef.current, now: () => Date.now(), - statusOf: (e) => (e instanceof HttpError ? e.status : undefined), + later: (fn, ms) => { + const timer = window.setTimeout(fn, ms); + return () => window.clearTimeout(timer); + }, + // `req` rethrows fetch's own TypeError when nothing answered at all: that is a status of 0. + statusOf: (e) => + e instanceof HttpError ? e.status : e instanceof TypeError ? 0 : undefined, onLoadStart: () => setBuffering(false), onFragLoaded: (frag) => fragMeterRef.current.add(frag), // Fatal ones are what the viewer saw fail; the recoverable ones hls.js retries by itself only