fix(i18n): read every refusal from the backend's syntax tree, not by regex

The contract read Python with regexes, and each guard added for a missed shape had shapes of its own
it missed: a reason after another key, a status passed by name, a second `e.reason` handler with the
same text, a pin naming a type nothing checked.

backend/tests/refusal_sites.py (stdlib `ast`, run by the contract on the host's Python) reads every
`{"code": ...}` detail and follows each reason to its values: literals, both arms of a conditional,
module constants, the `Literal` on the parameter or attribute it travels in (on the class an `except`
catches), every call of a `str` helper with its defaults, a map's values with its keys held to the
index's type. What it cannot follow fails the contract, unless no screen shows that code; no pins.

The reason attributes are typed with their Literal, so a subclass or an assignment cannot widen
them; InvalidSetting takes a SettingReason, and the settings map is held to it (a reason the map
lacked was a 500). The per-family reverse checks are one check by prefix (the admin actions
included), and the leaf-key filter applies only to the suffixed sign-in family.

frontend/src/i18n/backendIds.contract.test.ts

@@ -1,4 +1,6 @@
+import { execFileSync } from "node:child_process";
 import { readdirSync, readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
 import { describe, expect, it } from "vitest";
 
 import { RUN_STATUSES } from "../lib/activityUrlState";
@@ -27,6 +29,25 @@ import { vocabKey } from "../lib/vocab";
 const backend = (file: string) =>
   readFileSync(new URL(`../../../backend/app/${file}`, import.meta.url), "utf-8");
 
+// Every refusal the backend can answer, read from its syntax tree by `backend/tests/refusal_sites.py`
+// (stdlib `ast`, on the host's Python, as the ruff lane is): each code with every reason it can be sent
+// with — a reason in a variable followed to its `Literal` type, its helper's every call, its map — and
+// what it could not follow, which fails below. It also hands back every `Literal` alias it read.
+type RefusalScan = {
+  codes: Record<string, { reasons: string[]; unread: string[]; reasonless: string[] }>;
+  unreadCodes: string[];
+  errors: string[];
+  notRefusals: string[];
+  excluded: Record<string, number>;
+  aliases: Record<string, Record<string, string[]>>;
+};
+const scan = JSON.parse(
+  execFileSync("python", ["-I", "backend/tests/refusal_sites.py"], {
+    cwd: fileURLToPath(new URL("../../../", import.meta.url)),
+    encoding: "utf-8",
+  }),
+) as RefusalScan;
+
 const mods = import.meta.glob("./locales/*/*.json", { eager: true }) as Record<
   string,
   { default: Record<string, unknown> }
@@ -54,12 +75,11 @@ const keysOf = (lang: string): Set<string> => {
 const specs = [
   ...backend("sysconfig.py").matchAll(/ConfigSpec\(\s*"(\w+)",\s*"(\w+)",\s*"(\w+)",\s*"(\w+)"/g),
 ].map((m) => ({ key: m[1]!, type: m[2]!, group: m[3]!, attr: m[4]! }));
-// A choice key's options, as config.py spells them: `<attr>: <Alias> = …` → `<Alias> = Literal[…]`.
+// A choice key's options, as config.py spells them: `<attr>: <Alias> = …`, the alias's values from the scan.
 const configPy = backend("config.py");
 const choicesOf = (attr: string): string[] => {
   const alias = configPy.match(new RegExp(`^\\s+${attr}: (\\w+) =`, "m"))?.[1];
-  const body = alias && configPy.match(new RegExp(`^${alias} = Literal\\[([^\\]]*)\\]`, "m"))?.[1];
-  return [...(body ?? "").matchAll(/"(\w+)"/g)].map((m) => m[1]!);
+  return (alias && scan.aliases["config.py"]?.[alias]) || [];
 };
 const choiceIds = specs
   .filter((s) => s.type === "choice")
@@ -90,16 +110,10 @@ const runEnum = (name: string) =>
 const runStatuses = runEnum("RunStatus");
 const runTriggers = runEnum("RunTrigger");
 const quotaActions = enumValues("quota.py");
-// The admin actions' refusal reasons → `errors.admin.<reason>`. The codes are the refusal records
-// under that prefix (lib/api/refusalCodes), so a code recorded there is scanned here too; a detail
-// dict ruff wraps over several lines still matches.
+// The admin actions' refusals → `errors.admin.<reason>`: the codes recorded under that prefix.
 const adminCodes = Object.entries(REFUSAL_FAMILIES)
   .filter(([, f]) => f.prefix === "errors.admin")
   .map(([code]) => code);
-const adminRefusals = [
-  ...backend("routes/admin.py").matchAll(/\{\s*"code":\s*"(\w+)",\s*"reason":\s*"(\w+)"/g),
-].filter((m) => adminCodes.includes(m[1]!));
-const adminReasons = [...new Set(adminRefusals.map((m) => m[2]!))];
 // The download failure catalog's host-specific entries → `downloads.errors.fail.catalog.<detail>`.
 // A code holds a dot (`facebook.login_wall`), so the reverse check strips the prefix and the part
 // rather than splitting on dots.
@@ -120,66 +134,6 @@ const failCategories = [
   ]),
 ];
 
-// What a structured 404 says is gone → `errors.notFound.<what>`: every `not_found("x")` and every
-// `owned_or_404(..., what="x")` / `get_or_404(..., what="x")` under routes/, read from the sources.
-const routeFiles = (dir = "routes"): string[] =>
-  readdirSync(new URL(`../../../backend/app/${dir}/`, import.meta.url), {
-    withFileTypes: true,
-  }).flatMap((e) =>
-    e.isDirectory()
-      ? routeFiles(`${dir}/${e.name}`)
-      : e.name.endsWith(".py")
-        ? [`${dir}/${e.name}`]
-        : [],
-  );
-const routeSources = (): string[] => routeFiles().map(backend);
-// The words a job's result gives for not doing its work → `scheduler.skip.<vocabKey(word)>` (the Scheduler
-// page, the run notice and the audit row name them; lib/jobResult). Read as the `"skipped": "…"` /
-// `"reason": "…"` literals of the non-HTTP backend: under routes/ (and main.py) a `reason` is a refusal
-// code, which `errors.*` names. A word from an exception's text is not a literal and is not covered.
-const appFiles = (dir: string): string[] =>
-  readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
-    withFileTypes: true,
-  }).flatMap((e) =>
-    e.isDirectory()
-      ? e.name === "routes" || e.name === "__pycache__"
-        ? []
-        : appFiles(`${dir}${e.name}/`)
-      : e.name.endsWith(".py") && `${dir}${e.name}` !== "main.py"
-        ? [`${dir}${e.name}`]
-        : [],
-  );
-const skipWords = [
-  ...new Set(
-    appFiles("").flatMap((f) =>
-      [...backend(f).matchAll(/"(?:skipped|reason)": "([^"]+)"/g)].map((m) => m[1]!),
-    ),
-  ),
-];
-// A target the scan cannot read — `what=SOME_CONSTANT` — is a 404 no locale is checked for (one hid
-// `debug_session`). Only the helpers' own pass-through in `_common.py` may name it by variable.
-const opaqueWhats = routeFiles()
-  .filter((f) => !f.endsWith("/_common.py"))
-  .flatMap((f) =>
-    [...backend(f).matchAll(/not_found\((?!"|\))[^)]*\)|what=(?!")\S+/g)].map(
-      (m) => `${f}: ${m[0]}`,
-    ),
-  );
-const goneWhats = [
-  ...new Set(
-    routeSources().flatMap((src) =>
-      [...src.matchAll(/not_found\("(\w+)"\)|what="(\w+)"/g)].map((m) => (m[1] ?? m[2])!),
-    ),
-  ),
-];
-// A backend `Name = Literal["a", "b"]` alias, by name: the reasons a typed parameter accepts, which
-// basedpyright then holds at every call site — a reason passed by variable included.
-const literalOf = (file: string, name: string) =>
-  [
-    ...(
-      backend(file).match(new RegExp(`^${name} = Literal\\[([^\\]]*)\\]`, "m"))?.[1] ?? ""
-    ).matchAll(/"(\w+)"/g),
-  ].map((m) => m[1]!);
 // Every backend source, by its path under app/.
 const appPaths = (dir = ""): string[] =>
   readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
@@ -193,114 +147,32 @@ const appPaths = (dir = ""): string[] =>
         ? [`${dir}${e.name}`]
         : [],
   );
-const appSources = (): string[] => appPaths().map(backend);
-// Every generic refusal → `errors.refused.<reason>`: each `refused(<status>, "<reason>")` anywhere in
-// the backend (not `_refused`, a router's own family), the values of the settings map, which
-// `setting_refused` picks from at runtime, and the folder rules' `FolderReason`, answered with
-// `refused(409, e.reason)`.
-const refusedReasons = [
-  ...new Set([
-    ...appSources().flatMap((src) =>
-      [...src.matchAll(/(?<![\w.])refused\(\s*\d+,\s*"(\w+)"/g)].map((m) => m[1]!),
-    ),
-    ...literalOf("downloads/folders.py", "FolderReason"),
-    ...[
-      ...(backend("refusal.py").split("_SETTING_REASONS = {")[1]?.split("}")[0] ?? "").matchAll(
-        /:\s*"(\w+)"/g,
-      ),
-    ].map((m) => m[1]!),
-  ]),
-];
-// A `refused(…)` whose reason the scan above cannot read. Only two may: a folder rule's `e.reason`
-// (the nearest `except` above it catches `FolderError`, typed `FolderReason`) and the settings map.
-const opaqueRefused = appPaths().flatMap((f) => {
-  const src = backend(f);
-  return [...src.matchAll(/(?<![\w.])refused\(\s*\d+,\s*(?![\s"])([^,)]+)/g)]
-    .filter((m) => {
-      const arg = m[1]!.trim();
-      if (f === "refusal.py" && arg.startsWith("_SETTING_REASONS[")) return false;
-      const caught = [...src.slice(0, m.index).matchAll(/except ([^\n:]*):/g)].pop()?.[1] ?? "";
-      return !(arg === "e.reason" && /FolderError as e$/.test(caught));
-    })
-    .map((m) => `${f}: ${m[0]}`);
-});
-const youtubeReasons = literalOf("youtube/client.py", "YouTubeRefusal");
-const clipReasons = literalOf("downloads/clip.py", "ClipReason");
-
-// Every `"code": …` a backend dict carries, with the `"reason": …` that follows it. Two modules use a
-// `code` key for something that is not a refusal; everything else is one.
-const NOT_REFUSALS: Record<string, string> = {
-  "plex/subtitles.py":
-    "the player's subtitle status vocabulary (`/status`), read by the player itself",
-  "plex/tmdb.py": "a production country's ISO code",
-};
-const detailSites = appPaths()
-  .filter((f) => !(f in NOT_REFUSALS))
-  .flatMap((f) =>
-    [
-      ...backend(f).matchAll(
-        /"code":\s*(?:"(\w+)"|([^\s,}]+))(?:,\s*"reason":\s*(?:"(\w+)"|([^\s,}]+)))?/g,
-      ),
-    ].map((m) => ({ f, code: m[1], codeExpr: m[2], reason: m[3], reasonExpr: m[4] })),
-  );
-const backendCodes = [...new Set(detailSites.flatMap((s) => (s.code ? [s.code] : [])))];
-// A code the scan cannot read (`"code": SOME_CONSTANT`) is a refusal no record is checked for.
-const opaqueCodes = detailSites.filter((s) => !s.code).map((s) => `${s.f}: "code": ${s.codeExpr}`);
-// A reason passed by variable, each pinned to what holds it, so a new one has to say how it is held.
-const OPAQUE_REASONS: Record<string, string> = {
-  "refusal.py: refused ← reason": "a literal at each refused() call, else opaqueRefused",
-  "routes/auth.py: login ← reason": "_login_refused's LoginRefusal",
-  "routes/downloads.py: url_blocked ← urlguard.REASON_MALFORMED": "UrlBlockedReason",
-  "routes/downloads.py: url_blocked ← e.reason": "BlockedUrl's UrlBlockedReason",
-  "routes/downloads.py: cookies ← e.reason": "InvalidCookies' CookieReason",
-  "routes/downloads.py: edit ← e.reason": "EditError's EditReason",
-  "routes/downloads.py: clip ← e.reason": "ClipError's ClipReason",
-  "routes/files.py: quota ← e.reason": "QuotaExceeded's QuotaReason",
-  "routes/playlists.py: playlist ← reason": "_refused's PlaylistRefusal",
-  "routes/playlists.py: playlist_move ← e.reason": "MoveRefused's MoveReason",
-  "routes/plex/playback.py: subtitle ← code": "REFUSAL_UNSHOWN: no screen reads it",
-  "routes/_common.py: not_found ← what": "opaqueWhats",
-  "routes/_common.py: youtube ← reason": "YouTubeRefusal",
-};
-const opaqueReasons = [
+// The words a job's result gives for not doing its work → `scheduler.skip.<vocabKey(word)>` (the Scheduler
+// page, the run notice and the audit row name them; lib/jobResult). Read as the `"skipped": "…"` /
+// `"reason": "…"` literals of the non-HTTP backend: under routes/ (and main.py) a `reason` is a refusal
+// code, which `errors.*` names. A word from an exception's text is not a literal and is not covered.
+const skipWords = [
   ...new Set(
-    detailSites
-      .filter((s) => s.code && s.reasonExpr)
-      .map((s) => `${s.f}: ${s.code} ← ${s.reasonExpr}`),
+    appPaths()
+      .filter((f) => !f.startsWith("routes/") && f !== "main.py")
+      .flatMap((f) =>
+        [...backend(f).matchAll(/"(?:skipped|reason)": "([^"]+)"/g)].map((m) => m[1]!),
+      ),
   ),
 ];
-// A reason written as a literal in a detail dict, wherever that dict is built.
-const literalReasons = (code: string) =>
-  detailSites.filter((s) => s.code === code && s.reason).map((s) => s.reason!);
-// What each refusal code can carry as its reason: the Literal its typed parameter accepts, or the
-// call's own literal — and every literal any detail dict anywhere gives it.
-const reasonsOf: Record<string, string[]> = Object.fromEntries(
-  Object.entries({
-    refused: refusedReasons,
-    youtube: youtubeReasons,
-    not_found: goneWhats,
-    clip: clipReasons,
-    ...Object.fromEntries(adminCodes.map((c) => [c, []])),
-    playlist: literalOf("routes/playlists.py", "PlaylistRefusal"),
-    playlist_move: literalOf("reader/service.py", "MoveReason"),
-    edit: literalOf("downloads/service.py", "EditReason"),
-    cookies: literalOf("downloads/credentials.py", "CookieReason"),
-    url_blocked: literalOf("downloads/urlguard.py", "UrlBlockedReason"),
-    login: literalOf("routes/auth.py", "LoginRefusal"),
-    quota: literalOf("downloads/quota.py", "QuotaReason"),
-    reset: [],
-    collection: [],
-  } as Record<string, string[]>).map(([code, rs]) => [
-    code,
-    [...new Set([...rs, ...literalReasons(code)])],
-  ]),
-);
+
+const backendCodes = Object.keys(scan.codes);
 const recordedCodes = [
   ...Object.keys(REFUSAL_FAMILIES),
   ...Object.keys(REFUSAL_BY_REASON),
   ...Object.keys(REFUSAL_FIXED),
   ...Object.keys(REFUSAL_UNSHOWN),
 ];
+// The codes whose reason picks the sentence, and what each can carry as its reason.
+const reasoned = [...Object.keys(REFUSAL_FAMILIES), ...Object.keys(REFUSAL_BY_REASON)];
+const reasonsOf: Record<string, string[]> = Object.fromEntries(
+  reasoned.map((code) => [code, scan.codes[code]?.reasons ?? []]),
+);
 
 // The closed English vocabularies the backend WRITES into data, named through `vocab.<group>`
 // (lib/vocab) by their letters-and-digits key. A word the backend adds shows in English until named —
@@ -363,10 +235,11 @@ describe("backend ids have their names in both languages", () => {
     expect(runTriggers).toEqual(expect.arrayContaining(["scheduled", "ad_hoc"]));
     expect(adminCodes).toEqual(expect.arrayContaining(["activate", "resend"]));
     expect(failDetails).toEqual(expect.arrayContaining(["facebook.login_wall"]));
-    expect(goneWhats).toEqual(expect.arrayContaining(["playlist", "download", "job"]));
-    expect(opaqueWhats).toEqual([]);
-    expect(youtubeReasons).toEqual(["quota", "reconnect", "unavailable", "gone"]);
-    expect(clipReasons).toEqual(expect.arrayContaining(["too_long", "past_end", "bad_crop"]));
+    expect(reasonsOf.not_found).toEqual(
+      expect.arrayContaining(["playlist", "download", "job", "user", "debug_session"]),
+    );
+    expect(reasonsOf.youtube).toEqual(["gone", "quota", "reconnect", "unavailable"]);
+    expect(reasonsOf.clip).toEqual(expect.arrayContaining(["too_long", "past_end", "bad_crop"]));
     expect(crewJobs).toEqual(
       expect.arrayContaining(["Director", "Creator", "Director of Photography"]),
     );
@@ -382,11 +255,14 @@ describe("backend ids have their names in both languages", () => {
         "no service user",
       ]),
     );
-    expect(refusedReasons).toEqual(
-      expect.arrayContaining(["weak_password", "setting_too_low", "plex_write_failed"]),
+    expect(reasonsOf.refused).toEqual(
+      expect.arrayContaining([
+        "weak_password",
+        "setting_too_low",
+        "plex_write_failed",
+        "file_folder_gone",
+      ]),
     );
-    // Every code the frontend localizes is raised somewhere the scan can see.
-    expect(adminCodes.filter((c) => !adminRefusals.some((m) => m[1] === c))).toEqual([]);
   });
 
   it("every refusal code the backend answers is said by exactly one record, and no record names another", () => {
@@ -397,12 +273,19 @@ describe("backend ids have their names in both languages", () => {
     expect(recordedCodes.filter((c) => !backendCodes.includes(c))).toEqual([]);
   });
 
-  it("every refusal code and reason is one the scans can read, or says what holds it", () => {
-    // The modules set aside as not refusals still exist and still use the key.
-    for (const f of Object.keys(NOT_REFUSALS)) expect(backend(f), f).toContain('"code":');
-    expect(opaqueCodes).toEqual([]);
-    expect(opaqueRefused).toEqual([]);
-    expect(opaqueReasons.sort()).toEqual(Object.keys(OPAQUE_REASONS).sort());
+  it("every refusal code and reason is one the scan can follow", () => {
+    // The modules set aside as not refusals still use the key.
+    expect(Object.keys(scan.excluded).sort()).toEqual(scan.notRefusals);
+    expect(scan.unreadCodes).toEqual([]);
+    expect(scan.errors).toEqual([]);
+    // A reason it could not follow is a sentence nobody is asked for, unless no screen says that code.
+    expect(
+      Object.entries(scan.codes)
+        .filter(([code]) => !(code in REFUSAL_UNSHOWN))
+        .flatMap(([, c]) => c.unread),
+    ).toEqual([]);
+    // A code whose reason picks the sentence, sent without one, says only its fallback.
+    expect(reasoned.flatMap((code) => scan.codes[code]?.reasonless ?? [])).toEqual([]);
   });
 
   it("read every refusal's reasons (a scan that matches nothing is green over nothing)", () => {
@@ -533,33 +416,6 @@ describe("backend ids have their names in both languages", () => {
       expect([...named].filter((d) => !failDetails.includes(d))).toEqual([]);
     });
 
-    it(`${lang}: every "that is gone" a 404 names, and every YouTube refusal`, () => {
-      expect(missing(goneWhats.map((w) => `errors.notFound.${w}`))).toEqual([]);
-      expect(
-        [...idsUnder(keys, "errors.notFound")].filter(
-          (w) => w !== "generic" && !goneWhats.includes(w),
-        ),
-      ).toEqual([]);
-      expect(missing(youtubeReasons.map((r) => `errors.youtube.${r}`))).toEqual([]);
-      expect(
-        [...idsUnder(keys, "errors.youtube")].filter((r) => !youtubeReasons.includes(r)),
-      ).toEqual([]);
-    });
-
-    it(`${lang}: every generic refusal the backend can answer, and none it cannot`, () => {
-      expect(missing(refusedReasons.map((r) => `errors.refused.${r}`))).toEqual([]);
-      expect(
-        [...idsUnder(keys, "errors.refused")].filter((r) => !refusedReasons.includes(r)),
-      ).toEqual([]);
-    });
-
-    it(`${lang}: every reason a clip is refused, and none it cannot be`, () => {
-      expect(missing(clipReasons.map((r) => `clip.errors.${r}`))).toEqual([]);
-      expect([...idsUnder(keys, "clip.errors")].filter((r) => !clipReasons.includes(r))).toEqual(
-        [],
-      );
-    });
-
     it(`${lang}: every crew job, auto-tag and built-in profile the backend writes, and no other`, () => {
       for (const [group, words] of Object.entries(VOCAB)) {
         const wanted = words.map((w) => `vocab.${group}.${vocabKey(w)}`);
@@ -588,32 +444,32 @@ describe("backend ids have their names in both languages", () => {
     });
 
     it(`${lang}: no sentence for a refusal reason the backend cannot send`, () => {
-      // A family whose prefix is its own (the shared `errors.admin` is checked below). A suffixed
-      // family shares its prefix with more than its refusals — the sign-in screens' `gate`,
-      // `addressLabel` — so only an id with the refusal's own key counts as a reason there.
+      // By prefix: the admin actions share `errors.admin`. A suffixed family shares its prefix with more
+      // than its refusals — the sign-in screens' `gate`, `addressLabel` — so only an id with the
+      // refusal's own key counts as a reason there.
+      const byPrefix = new Map<
+        string,
+        { suffix: string; reasons: Set<string>; spare: Set<string> }
+      >();
       for (const [code, f] of Object.entries(REFUSAL_FAMILIES)) {
-        if (f.prefix === "errors.admin") continue;
-        const suffix = "suffix" in f ? f.suffix : "";
-        const fallbackId = f.fallback.startsWith(`${f.prefix}.`)
-          ? f.fallback.slice(f.prefix.length + 1)
-          : null;
-        expect(
-          [...idsUnder(keys, f.prefix)].filter(
-            (r) =>
-              keys.has(`${f.prefix}.${r}${suffix}`) &&
-              r !== fallbackId &&
-              !reasonsOf[code]!.includes(r),
-          ),
-          code,
-        ).toEqual([]);
+        const p = byPrefix.get(f.prefix) ?? {
+          suffix: "suffix" in f ? f.suffix : "",
+          reasons: new Set<string>(),
+          spare: new Set<string>(),
+        };
+        for (const r of reasonsOf[code]!) p.reasons.add(r);
+        if (f.fallback.startsWith(`${f.prefix}.`))
+          p.spare.add(f.fallback.slice(f.prefix.length + 1));
+        byPrefix.set(f.prefix, p);
       }
-    });
-
-    it(`${lang}: every admin refusal reason`, () => {
-      expect(missing(adminReasons.map((r) => `errors.admin.${r}`))).toEqual([]);
-      expect([...idsUnder(keys, "errors.admin")].filter((r) => !adminReasons.includes(r))).toEqual(
-        [],
-      );
+      for (const [prefix, { suffix, reasons, spare }] of byPrefix)
+        expect(
+          [...idsUnder(keys, prefix)].filter(
+            (r) =>
+              (!suffix || keys.has(`${prefix}.${r}${suffix}`)) && !spare.has(r) && !reasons.has(r),
+          ),
+          prefix,
+        ).toEqual([]);
     });
   }
 });