fix(plex): a server that restarts mid-episode is waited out, not reported as a stream that cannot play

During a deploy the gateway answers 502 (or nothing answers at all) until the app is back. The
player took the first such answer for a broken stream: it said "Playback couldn't start" over a
picture still playing from its buffer, stopped loading, and never asked again, so the episode froze
where the buffer ended with the error still up. Now a 502/503/504 or no answer is the server being
away: the buffer keeps playing undisturbed, the session is asked for again where the viewer is
(2, 4, 8, then every 15 s), and the error is said only after three minutes away. A seek or another
item replaces a pending retry. Measured on localdev: a 45 s API stop now resumes at the viewer's
position, where it froze with the error before.

frontend/src/lib/hlsEngine.test.ts

@@ -3,6 +3,7 @@
 import { describe, it, expect, vi } from "vitest";
 import type { PlexPlaySession } from "./api";
 import {
+  AWAY_WINDOW_MS,
   HlsEngine,
   MAX_SESSION_RECLAIMS,
   RECLAIM_WINDOW_MS,
@@ -62,12 +63,16 @@ class FakeHls implements HlsPort {
     return { index: 0, total: 1 };
   }
   lost404() {
+    this.lost(404);
+  }
+  /** hls.js gave up on a playlist or fragment with this answer (null: no answer at all). */
+  lost(status: number | null) {
     this.handlers.onError({
       fatal: true,
       network: true,
-      status: 404,
+      status,
       type: "networkError",
-      details: "fragLoadError",
+      details: "levelLoadError",
     });
   }
 }
@@ -87,6 +92,8 @@ function setup(over: Partial<EngineDeps> = {}) {
   const asks: Ask[] = [];
   const hlses: FakeHls[] = [];
   const stops: [string, string][] = [];
+  // The engine's timers, run by hand: `fire` runs the next one still armed.
+  const timers: { fn: () => void; ms: number; armed: boolean }[] = [];
   let clock = 1_000_000;
   const state = { position: 0, duration: 3000, params: { multi: false, audio: null, aoff: 0 } };
   const engine = new HlsEngine({
@@ -118,6 +125,11 @@ function setup(over: Partial<EngineDeps> = {}) {
     duration: () => state.duration,
     now: () => clock,
     statusOf: (e) => (e as { status?: number }).status,
+    later: (fn, ms) => {
+      const t = { fn, ms, armed: true };
+      timers.push(t);
+      return () => void (t.armed = false);
+    },
     ...over,
   });
   return {
@@ -127,7 +139,17 @@ function setup(over: Partial<EngineDeps> = {}) {
     hlses,
     stops,
     state,
+    timers,
     advance: (ms: number) => void (clock += ms),
+    /** Let the next armed timer run (its delay passes on the clock too); false when none is. */
+    fire: () => {
+      const t = timers.find((x) => x.armed);
+      if (!t) return false;
+      t.armed = false;
+      clock += t.ms;
+      t.fn();
+      return true;
+    },
   };
 }
 
@@ -396,6 +418,101 @@ describe("a session the server no longer has (404)", () => {
   });
 });
 
+describe("a server that is restarting (a deploy)", () => {
+  // The gateway answers 502 while the app restarts, and nothing at all may answer (status 0): the
+  // stream is not broken, the server is away. The buffer keeps playing meanwhile.
+  it.each([502, 503, 504, 0, null])(
+    "is waited out on %s: no error, and the stream comes back where the viewer is",
+    async (status) => {
+      const { engine, asks, hlses, state, fire, media } = setup();
+      engine.start("A", 600);
+      await asks[0]!.resolve(hls("A", 600));
+      hlses[0]!.handlers.onManifestParsed();
+      state.position = 1299;
+      hlses[0]!.lost(status);
+      expect(engine.getSnapshot().error).toBeNull();
+      expect(asks).toHaveLength(1); // not at once: the server is still away
+      expect(fire()).toBe(true);
+      expect(asks[1]!.startAt).toBe(1299);
+      // The buffered picture is still playing while the server is asked: no "Loading…" over it.
+      expect(engine.getSnapshot().ready).toBe(true);
+      await asks[1]!.reject({ status: 502 }); // still away
+      expect(engine.getSnapshot().error).toBeNull();
+      state.position = 1310;
+      expect(fire()).toBe(true);
+      await asks[2]!.resolve(hls("A", 1310));
+      expect(hlses[1]!.loaded).toContain("?t=1310");
+      expect(engine.getSnapshot().ready).toBe(false); // the new stream is loading now
+      hlses[1]!.handlers.onManifestParsed();
+      expect(engine.getSnapshot()).toMatchObject({ ready: true, error: null });
+      expect(media.plays).toBeGreaterThan(0);
+    },
+  );
+
+  it("is waited out when the very first request finds it away", async () => {
+    const { engine, asks, fire } = setup();
+    engine.start("A", 0);
+    await asks[0]!.reject({ status: 0 });
+    expect(engine.getSnapshot()).toMatchObject({ ready: false, error: null });
+    expect(fire()).toBe(true);
+    await asks[1]!.resolve(hls("A"));
+    expect(engine.getSnapshot().error).toBeNull();
+  });
+
+  it("asks less and less often, and says so once it has been away too long", async () => {
+    const { engine, asks, hlses, fire, timers } = setup();
+    engine.start("A", 0);
+    await asks[0]!.resolve(hls("A"));
+    hlses[0]!.handlers.onManifestParsed();
+    hlses[0]!.lost(502);
+    let i = 1;
+    while (fire()) await asks[i++]!.reject({ status: 502 });
+    const delays = timers.map((t) => t.ms);
+    expect(delays[1]!).toBeGreaterThan(delays[0]!);
+    expect(delays.reduce((a, b) => a + b, 0)).toBeGreaterThan(AWAY_WINDOW_MS);
+    expect(engine.getSnapshot().error).toBe("generic");
+  });
+
+  it("starts the wait over after the server came back", async () => {
+    const { engine, asks, hlses, fire, advance } = setup();
+    engine.start("A", 0);
+    await asks[0]!.resolve(hls("A"));
+    hlses[0]!.handlers.onManifestParsed();
+    hlses[0]!.lost(502);
+    fire();
+    await asks[1]!.resolve(hls("A"));
+    hlses[1]!.handlers.onManifestParsed();
+    advance(AWAY_WINDOW_MS + 1);
+    hlses[1]!.lost(502); // a second deploy, long after the first
+    expect(engine.getSnapshot().error).toBeNull();
+  });
+
+  it("drops a pending retry once the viewer seeks or leaves the item", async () => {
+    const { engine, asks, hlses, fire } = setup();
+    engine.start("A", 0);
+    await asks[0]!.resolve(hls("A"));
+    hlses[0]!.handlers.onManifestParsed();
+    hlses[0]!.lost(502);
+    engine.seek("A", 2000); // the viewer's own restart goes out instead
+    expect(asks).toHaveLength(2);
+    fire();
+    expect(asks).toHaveLength(2);
+    await asks[1]!.reject({ status: 502 });
+    engine.leaveItem();
+    fire();
+    expect(asks).toHaveLength(2);
+  });
+
+  it("still says a real failure at once", async () => {
+    const { engine, asks, hlses } = setup();
+    engine.start("A", 0);
+    await asks[0]!.resolve(hls("A"));
+    hlses[0]!.handlers.onManifestParsed();
+    hlses[0]!.lost(500);
+    expect(engine.getSnapshot().error).toBe("generic");
+  });
+});
+
 describe("play intent while a session is asked for", () => {
   it("goes into the pending session, and never plays the outgoing media", async () => {
     const { engine, asks, media, hlses } = setup();