feat(overlay): shared layer-stack for topmost-only Escape across modals, popovers, players (R8 S3)

Replace the three mutually-unaware Escape systems — Modal's private modalStack,
the popovers' document listeners with a blanket stopPropagation, and the two
players' modalCount() checks — with one shared lib/layerStack.ts. Every open
dismissable layer registers in open order; a layer acts on Escape only when it
is topmost, and reads anyLayerAbove() to decide whether to defer. Coordination
is now a stack lookup, not event-propagation order, so it no longer depends on
which listener happens to fire first.

- Modal + useDismiss register via the stack; useDismiss drops the blanket
  stopPropagation (its documented side effect: it swallowed a player's own
  Escape while any popover was open).
- Both players register as a layer and defer via anyLayerAbove() instead of
  modalCount(); PlayerModal checks it before its maximise/fullscreen steps.
  A player MENU is a layer too, so it closes first and the next Escape reaches
  the auto-skip-cancel — fixing the documented PlexPlayer bug.
- AnchoredPopover routes non-hover Escape through the stack (floating-ui's own
  escapeKey off there) and registers as a layer, so a player/modal underneath
  defers once S4 migrates the in-player menus. FloatingTree still owns
  popover-vs-popover ordering; the stack adds the cross-type ordering.

frontend/src/lib/layerStack.ts

@@ -0,0 +1,76 @@
+import { useEffect, useRef } from "react";
+
+// R8 S3 — one shared registry of open "dismissable layers" (modals, popovers, players), ordered by
+// open time. It replaces the three mutually-unaware Escape systems that each guessed at ordering with
+// its own `stopPropagation`: Modal's private `modalStack`, the popovers' document listeners, and the
+// two players' `modalCount()` checks. Now EVERY layer keeps its own key handler but coordinates
+// through this one stack — a layer acts on Escape only when it is topmost, and reads `anyAbove()` to
+// know whether to defer. Because the decision is a stack lookup, not event-propagation order, the fix
+// is robust to which listener happens to run first (the old stopPropagation was not).
+//
+// Notably this fixes the documented PlexPlayer bug: its menus register here, so the player sees a menu
+// as a layer ABOVE it and defers, letting the menu's own Escape close it first — the player's
+// auto-skip-cancel Escape then lands on the next press, instead of being swallowed by a blanket
+// stopPropagation. (Popover-vs-popover ordering is still floating-ui's <FloatingTree>; this stack adds
+// the cross-TYPE ordering the tree can't see.)
+
+let stack: number[] = [];
+let nextId = 1;
+
+export function pushLayer(): number {
+  const id = nextId++;
+  stack.push(id);
+  return id;
+}
+
+export function removeLayer(id: number): void {
+  stack = stack.filter((x) => x !== id);
+}
+
+export function isTopmost(id: number): boolean {
+  return stack.length > 0 && stack[stack.length - 1] === id;
+}
+
+/** True when at least one layer sits above `id` — the "should I defer?" predicate for a layer with a
+ *  richer key handler than a plain close (the players). */
+export function anyLayerAbove(id: number): boolean {
+  const i = stack.indexOf(id);
+  return i !== -1 && i < stack.length - 1;
+}
+
+/** Register an open layer for as long as `active` is true, and get live predicates to read inside your
+ *  own key handler. Pass `onEscape` to also install a topmost-only Escape listener (the common case —
+ *  modals, popovers). Omit it for a layer that owns a bigger key handler and only needs `anyAbove()`
+ *  to decide whether to defer (the players). */
+export function useLayer(
+  active: boolean,
+  onEscape?: () => void,
+): { isTopmost: () => boolean; anyAbove: () => boolean } {
+  const idRef = useRef<number | null>(null);
+  const escRef = useRef(onEscape);
+  escRef.current = onEscape;
+
+  useEffect(() => {
+    if (!active) return;
+    const id = pushLayer();
+    idRef.current = id;
+    let onKey: ((e: KeyboardEvent) => void) | undefined;
+    if (escRef.current) {
+      onKey = (e: KeyboardEvent) => {
+        if (e.key === "Escape" && isTopmost(id)) escRef.current?.();
+      };
+      document.addEventListener("keydown", onKey);
+    }
+    return () => {
+      if (onKey) document.removeEventListener("keydown", onKey);
+      removeLayer(id);
+      idRef.current = null;
+    };
+    // Re-register only when open/closed toggles; onEscape is read fresh via escRef.
+  }, [active]);
+
+  return {
+    isTopmost: () => idRef.current != null && isTopmost(idRef.current),
+    anyAbove: () => idRef.current != null && anyLayerAbove(idRef.current),
+  };
+}

frontend/src/lib/useDismiss.ts

@@ -1,14 +1,25 @@
 import { useEffect, type RefObject } from "react";
+import { useLayer } from "./layerStack";
 
 /** While `active`, close (call `onClose`) on Escape or a mousedown outside ALL of the given
  * element(s). The reusable half of the popover/dropdown pattern that several components each
  * hand-rolled; positioning stays with the caller (it varies per popover). Pass the panel ref
- * (and optionally a trigger ref so clicking the trigger doesn't immediately re-close). */
+ * (and optionally a trigger ref so clicking the trigger doesn't immediately re-close).
+ *
+ * Escape is dispatched through the shared `layerStack` (R8 S3): the popover registers as a layer and
+ * closes only when it is topmost, so a Modal or another popover opened above it takes the Escape
+ * first, and a player below defers to it. This replaces the old blanket `stopPropagation` — which
+ * swallowed a player's own Escape (e.g. PlexPlayer's auto-skip-cancel) whenever any popover was open. */
 export function useDismiss(
   active: boolean,
   onClose: () => void,
   refs: RefObject<HTMLElement | null> | RefObject<HTMLElement | null>[],
 ): void {
+  // Registration + topmost-only Escape.
+  useLayer(active, onClose);
+
+  // Outside-press stays per-instance (the refs/positioning are the caller's): close on a mousedown
+  // that lands outside every registered element.
   useEffect(() => {
     if (!active) return;
     const list = Array.isArray(refs) ? refs : [refs];
@@ -16,29 +27,12 @@ export function useDismiss(
       const target = e.target as Node;
       if (!list.some((r) => r.current?.contains(target))) onClose();
     };
-    const onKey = (e: KeyboardEvent) => {
-      if (e.key === "Escape") {
-        // Stop the same Escape from also reaching a player's window-level listener underneath —
-        // this document listener bubbles first, so one Escape closes only this popover, not the
-        // popover AND the player behind it. (U-C quick-fix.)
-        //
-        // KNOWN LIMITATION (accepted, → R8 popover-platform): because this is a blanket
-        // stopPropagation, a PlexPlayer settings/tracks menu open DURING an auto-skip countdown
-        // swallows the Escape that would otherwise ALSO cancel the skip — Escape closes the menu,
-        // and the skip proceeds. Properly fixing this needs the shared layer-registry R8 builds
-        // (topmost-only dispatch), not a broad stopPropagation.
-        e.stopPropagation();
-        onClose();
-      }
-    };
     document.addEventListener("mousedown", onDown);
-    document.addEventListener("keydown", onKey);
     return () => {
       document.removeEventListener("mousedown", onDown);
-      document.removeEventListener("keydown", onKey);
     };
-    // Matches the original effects: (re)subscribe only when open/closed toggles. Refs are
-    // stable and onClose is read fresh from this run's closure.
+    // Matches the original effect: (re)subscribe only when open/closed toggles. Refs are stable and
+    // onClose is read fresh from this run's closure.
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [active]);
 }