siftlode/frontend/src/lib/layerStack.ts

import { useEffect, useEffectEvent, useRef } from "react";

// One shared registry of open "dismissable layers" (modals, popovers, players), ordered by
// open time. It replaces the three mutually-unaware Escape systems that each guessed at ordering with
// its own `stopPropagation`: Modal's private `modalStack`, the popovers' document listeners, and the
// two players' `modalCount()` checks. Now EVERY layer keeps its own key handler but coordinates
// through this one stack — a layer acts on Escape only when it is topmost, and reads `anyAbove()` to
// know whether to defer. Because the decision is a stack lookup, not event-propagation order, the fix
// is robust to which listener happens to run first (the old stopPropagation was not).
//
// Notably this fixes the documented PlexPlayer bug: its menus register here, so the player sees a menu
// as a layer ABOVE it and defers, letting the menu's own Escape close it first — the player's
// auto-skip-cancel Escape then lands on the next press, instead of being swallowed by a blanket
// stopPropagation. (Popover-vs-popover ordering is still floating-ui's <FloatingTree>; this stack adds
// the cross-TYPE ordering the tree can't see.)

let stack: number[] = [];
let nextId = 1;
// Layers that take Escape but do not own the keyboard: an open tooltip or hover card. They sit on the
// stack for Escape ordering like any other layer, but `anyBlockingLayer` looks past them — otherwise
// a tooltip that happens to be open under a resting mouse pointer would silence the page's `j`/`k`.
const passive = new Set<number>();
// Layers that DO own the keyboard but leave the Tab trap to the dialog below: a small inline prompt
// inside one (the recorder's step field). It must silence the page's keys — an autofocused field
// does — yet it is not a dialog of its own, so claiming the trap left the dialog around it with
// none, and Tab from anything outside the prompt walked into the page behind the scrim. The two
// questions are separate; a layer answers each for itself.
const defers = new Set<number>();
// The element a layer draws into, for the layers that hold content of their own to navigate (a
// dialog, the player): a list inside the TOPMOST such layer is walked by the keyboard although a
// layer is open — it is the one the user is looking at, not the page behind.
const roots = new Map<number, () => HTMLElement | null>();

export function pushLayer(opts?: {
  passive?: boolean;
  trapsTab?: boolean;
  root?: () => HTMLElement | null;
}): number {
  const id = nextId++;
  stack.push(id);
  if (opts?.passive) passive.add(id);
  if (opts?.trapsTab === false) defers.add(id);
  if (opts?.root) roots.set(id, opts.root);
  return id;
}

export function removeLayer(id: number): void {
  stack = stack.filter((x) => x !== id);
  passive.delete(id);
  defers.delete(id);
  roots.delete(id);
}

/** The element of the topmost layer that owns the keyboard — null while none is open, or while that
 *  layer gave none (a menu: its own keys move inside it). */
export function topBlockingLayerRoot(): HTMLElement | null {
  const blocking = stack.filter((id) => !passive.has(id));
  const top = blocking[blocking.length - 1];
  return top === undefined ? null : (roots.get(top)?.() ?? null);
}

/** True while a layer that owns the keyboard is open — a modal, a player, a click-opened menu. The
 *  page-level keys (`s`, the `g` chords, list navigation) stay quiet then: they act on the page
 *  underneath, which the user cannot see or reach while such a layer is up. A list INSIDE the
 *  topmost layer is the exception: it is walked (`topBlockingLayerRoot`, listNav). */
export function anyBlockingLayer(): boolean {
  return stack.some((id) => !passive.has(id));
}

export function isTopmost(id: number): boolean {
  return stack.length > 0 && stack[stack.length - 1] === id;
}

/** True when at least one layer sits above `id` — the "should I defer?" predicate for a layer with a
 *  richer key handler than a plain close (the players). */
export function anyLayerAbove(id: number): boolean {
  const i = stack.indexOf(id);
  return i !== -1 && i < stack.length - 1;
}

/** True when a layer that TRAPS THE TAB sits above `id` — the predicate for a dialog's Tab trap. A
 *  tooltip open under a resting pointer (passive) and an inline prompt inside the dialog itself
 *  (`trapsTab: false`) both take Escape first but neither takes the Tab: deferring the trap to one
 *  of those left Shift+Tab walking out of the dialog into the page behind its scrim. */
export function anyBlockingLayerAbove(id: number): boolean {
  const i = stack.indexOf(id);
  return i !== -1 && stack.slice(i + 1).some((x) => !passive.has(x) && !defers.has(x));
}

// UX-8 — a layer that ACTS on an Escape stamps the event here; a lower layer with its own key handler
// (the players, listening on `window`) reads the stamp instead of re-reading the stack. This closes
// the "one Escape closes BOTH the popover and the player" bug: the popover's Escape listener is on
// `document` and fires before the player's on `window` (bubble order), and closing the popover flushes
// its layer off the stack synchronously — so by the time the player's `anyLayerAbove()` runs, the stack
// already reads "nothing above" and the player ALSO closes. Tagging the one shared KeyboardEvent object
// (every listener in a single dispatch is handed the same instance) is immune to that mid-dispatch
// mutation; `anyLayerAbove` stays as the fallback for a layer above with no Escape handler of its own.
// WeakSet so a spent event is GC'd without a manual sweep.
const handledEscapes = new WeakSet<KeyboardEvent>();
export function markEscapeHandled(e: KeyboardEvent): void {
  handledEscapes.add(e);
}
export function escapeHandled(e: KeyboardEvent): boolean {
  return handledEscapes.has(e);
}

/** THE close-on-Escape test for layer `id`: an Escape no layer above has taken yet, while `id` is on
 *  top — and it claims the event for `id`. Both halves are needed: a list inside a dialog closes with a
 *  synchronous flush that pops its layer before the dialog's `window` listener runs, so topmost alone
 *  read "yes" and one Escape closed the list AND the dialog (the share dialog's user picker). */
export function takeEscape(e: KeyboardEvent, id: number): boolean {
  if (e.key !== "Escape" || escapeHandled(e) || !isTopmost(id)) return false;
  markEscapeHandled(e);
  return true;
}

/** Register an open layer for as long as `active` is true, and get live predicates to read inside your
 *  own key handler. Pass `onEscape` to also install a topmost-only Escape listener (the common case —
 *  modals, popovers). Omit it for a layer that owns a bigger key handler and only needs `anyAbove()`
 *  to decide whether to defer (the players). */
export function useLayer(
  active: boolean,
  onEscape?: () => void,
  opts?: { passive?: boolean; trapsTab?: boolean },
): { isTopmost: () => boolean; anyAbove: () => boolean; blockingAbove: () => boolean } {
  const idRef = useRef<number | null>(null);
  const escape = useEffectEvent(() => onEscape?.());
  const hasEscape = onEscape !== undefined;
  const passive = !!opts?.passive;
  const trapsTab = opts?.trapsTab;

  useEffect(() => {
    if (!active) return;
    const id = pushLayer({ passive, trapsTab });
    idRef.current = id;
    let onKey: ((e: KeyboardEvent) => void) | undefined;
    if (hasEscape) {
      onKey = (e: KeyboardEvent) => {
        // takeEscape stamps BEFORE closing, so a `window` listener below (fires after this `document`
        // one) sees the Escape as consumed even though closing us pops this layer off the stack.
        if (takeEscape(e, id)) escape();
      };
      document.addEventListener("keydown", onKey);
    }
    return () => {
      if (onKey) document.removeEventListener("keydown", onKey);
      removeLayer(id);
      idRef.current = null;
    };
    // Re-register when open/closed toggles (the shape flags are constant per caller); the Escape
    // handler is an effect event, so a new `onEscape` each render never re-registers the layer.
  }, [active, hasEscape, passive, trapsTab]);

  return {
    isTopmost: () => idRef.current != null && isTopmost(idRef.current),
    anyAbove: () => idRef.current != null && anyLayerAbove(idRef.current),
    blockingAbove: () => idRef.current != null && anyBlockingLayerAbove(idRef.current),
  };
}