Files
siftlode/frontend/src/i18n/backendIds.contract.test.ts
T
peter 9f580fd8c8 fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
2026-10-23 20:48:23 +02:00

467 lines
20 KiB
TypeScript

import { readdirSync, readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { RUN_STATUSES } from "../lib/activityUrlState";
import { vocabKey } from "../lib/vocab";
// Every id the BACKEND owns and the UI names must have its name in both languages. The UI looks these
// up with the raw id as the fallback — `t(\`config.fields.${key}.label\`, key)` — so a missing key never
// fails loudly: the admin Configuration page showed `maintenance`, `tmdb`, `tmdb_api_key`,
// `plex_overlay_map`… as bare identifiers, for months, and the i18n parity test stayed green because EN
// and HU were missing the SAME keys (owner report). The ids live in Python, so the check reads
// the registries straight from the backend sources:
// - `sysconfig.SPECS` → `config.groups.<group>`, `config.fields.<key>.label` + `.hint`, and for a
// "choice" key `config.choices.<key>.<option>` — the options read from its Literal in config.py
// - `sysconfig.JOB_INTERVAL_DEFAULTS` and `scheduler.AD_HOC_JOBS` → `scheduler.jobs.<id>` + `scheduler.jobDesc.<id>`
// - the `audit` action enum → `auditActions.<id>`; the `quota` action enum → `quotaActions.<id>`
// and the other way round: a name left behind for an id the backend dropped is dead weight that reads
// as a live setting to the next person (`plex_sync_interval_min` moved to the scheduler long ago).
// Progress phases are not covered: jobs pass most of them as variables, not literals a scan can see.
const backend = (file: string) =>
readFileSync(new URL(`../../../backend/app/${file}`, import.meta.url), "utf-8");
const mods = import.meta.glob("./locales/*/*.json", { eager: true }) as Record<
string,
{ default: Record<string, unknown> }
>;
function flatten(obj: Record<string, unknown>, prefix: string, out: Set<string>): Set<string> {
for (const [k, v] of Object.entries(obj)) {
const key = `${prefix}.${k}`;
if (v && typeof v === "object" && !Array.isArray(v))
flatten(v as Record<string, unknown>, key, out);
else out.add(key);
}
return out;
}
const keysOf = (lang: string): Set<string> => {
const out = new Set<string>();
for (const [path, mod] of Object.entries(mods)) {
const m = path.match(/\/locales\/([^/]+)\/([^/]+)\.json$/);
if (m?.[1] === lang) flatten(mod.default, m[2]!, out);
}
return out;
};
const specs = [
...backend("sysconfig.py").matchAll(/ConfigSpec\(\s*"(\w+)",\s*"(\w+)",\s*"(\w+)",\s*"(\w+)"/g),
].map((m) => ({ key: m[1]!, type: m[2]!, group: m[3]!, attr: m[4]! }));
// A choice key's options, as config.py spells them: `<attr>: <Alias> = …` → `<Alias> = Literal[…]`.
const configPy = backend("config.py");
const choicesOf = (attr: string): string[] => {
const alias = configPy.match(new RegExp(`^\\s+${attr}: (\\w+) =`, "m"))?.[1];
const body = alias && configPy.match(new RegExp(`^${alias} = Literal\\[([^\\]]*)\\]`, "m"))?.[1];
return [...(body ?? "").matchAll(/"(\w+)"/g)].map((m) => m[1]!);
};
const choiceIds = specs
.filter((s) => s.type === "choice")
.flatMap((s) => choicesOf(s.attr).map((c) => `config.choices.${s.key}.${c}`));
const schedulerPy = backend("scheduler.py");
// The interval jobs (sysconfig's `JOB_INTERVAL_DEFAULTS`: job → its default's setting), and the
// ad-hoc ones (`AD_HOC_JOBS` in scheduler.py, a tuple of constants named beside it).
const jobs = [
...[
...(backend("sysconfig.py").split("JOB_INTERVAL_DEFAULTS: dict[str, str] = {")[1] ?? "")
.split("}")[0]!
.matchAll(/"(\w+)": "\w+",/g),
].map((m) => m[1]!),
...[...(/^AD_HOC_JOBS = \(([^)]*)\)/m.exec(schedulerPy)?.[1] ?? "").matchAll(/\b[A-Z_]+\b/g)].map(
(m) => new RegExp(`^${m[0]} = "(\\w+)"$`, "m").exec(schedulerPy)?.[1] ?? m[0],
),
];
const enumValues = (file: string) =>
[...backend(file).matchAll(/^\s+[A-Z_]+ = "(\w+)"$/gm)].map((m) => m[1]!);
const auditActions = enumValues("audit.py");
// A background run's words (backend/app/runstatus.py): its status and what started it.
const runEnum = (name: string) =>
[
...(
backend("runstatus.py").split(`class ${name}(StrEnum):`)[1]?.split("\nclass ")[0] ?? ""
).matchAll(/^\s+[A-Z_]+ = "(\w+)"$/gm),
].map((m) => m[1]!);
const runStatuses = runEnum("RunStatus");
const runTriggers = runEnum("RunTrigger");
const quotaActions = enumValues("quota.py");
// The admin actions' refusal reasons → `errors.admin.<reason>`. The codes are read from
// `localizeDetail`'s own ADMIN_REFUSALS, so a code added there is scanned here too; a detail dict
// ruff wraps over several lines still matches.
const adminCodes = [
...(
readFileSync(new URL("../lib/api/core.ts", import.meta.url), "utf-8").match(
/ADMIN_REFUSALS = new Set\(\[([^\]]*)\]/,
)?.[1] ?? ""
).matchAll(/"(\w+)"/g),
].map((m) => m[1]!);
const adminRefusals = [
...backend("routes/admin.py").matchAll(/\{\s*"code":\s*"(\w+)",\s*"reason":\s*"(\w+)"/g),
].filter((m) => adminCodes.includes(m[1]!));
const adminReasons = [...new Set(adminRefusals.map((m) => m[2]!))];
// The download failure catalog's host-specific entries → `downloads.errors.fail.catalog.<detail>`.
// A code holds a dot (`facebook.login_wall`), so the reverse check strips the prefix and the part
// rather than splitting on dots.
const failDetails = [
...new Set([...backend("downloads/errors.py").matchAll(/detail="([\w.]+)"/g)].map((m) => m[1]!)),
];
const DETAIL = "downloads.errors.fail.catalog.";
// Every category `classify` / the catalog can give a failure → `downloads.errors.fail.cat.<code>`. A
// category without copy never fails loudly: the row falls back to the "unknown" sentence.
const failCategories = [
...new Set([
...[
...(backend("downloads/errors.py").split("_RULES: tuple")[1] ?? "")
.split("\n)\n")[0]!
.matchAll(/,\s*"(\w+)"\),/g),
].map((m) => m[1]!),
...[...backend("downloads/errors.py").matchAll(/code="(\w+)"/g)].map((m) => m[1]!),
]),
];
// What a structured 404 says is gone → `errors.notFound.<what>`: every `not_found("x")` and every
// `owned_or_404(..., what="x")` / `get_or_404(..., what="x")` under routes/, read from the sources.
const routeFiles = (dir = "routes"): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}/`, import.meta.url), {
withFileTypes: true,
}).flatMap((e) =>
e.isDirectory()
? routeFiles(`${dir}/${e.name}`)
: e.name.endsWith(".py")
? [`${dir}/${e.name}`]
: [],
);
const routeSources = (): string[] => routeFiles().map(backend);
// The words a job's result gives for not doing its work → `scheduler.skip.<vocabKey(word)>` (the Scheduler
// page, the run notice and the audit row name them; lib/jobResult). Read as the `"skipped": "…"` /
// `"reason": "…"` literals of the non-HTTP backend: under routes/ (and main.py) a `reason` is a refusal
// code, which `errors.*` names. A word from an exception's text is not a literal and is not covered.
const appFiles = (dir: string): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
withFileTypes: true,
}).flatMap((e) =>
e.isDirectory()
? e.name === "routes" || e.name === "__pycache__"
? []
: appFiles(`${dir}${e.name}/`)
: e.name.endsWith(".py") && `${dir}${e.name}` !== "main.py"
? [`${dir}${e.name}`]
: [],
);
const skipWords = [
...new Set(
appFiles("").flatMap((f) =>
[...backend(f).matchAll(/"(?:skipped|reason)": "([^"]+)"/g)].map((m) => m[1]!),
),
),
];
// A target the scan cannot read — `what=SOME_CONSTANT` — is a 404 no locale is checked for (one hid
// `debug_session`). Only the helpers' own pass-through in `_common.py` may name it by variable.
const opaqueWhats = routeFiles()
.filter((f) => !f.endsWith("/_common.py"))
.flatMap((f) =>
[...backend(f).matchAll(/not_found\((?!"|\))[^)]*\)|what=(?!")\S+/g)].map(
(m) => `${f}: ${m[0]}`,
),
);
const goneWhats = [
...new Set(
routeSources().flatMap((src) =>
[...src.matchAll(/not_found\("(\w+)"\)|what="(\w+)"/g)].map((m) => (m[1] ?? m[2])!),
),
),
];
// Every generic refusal → `errors.refused.<reason>`: each `refused(<status>, "<reason>")` anywhere in
// the backend (not `_refused`, a router's own family), plus the values of the settings map, which
// `setting_refused` picks from at runtime, and every `FolderError` reason.
const appSources = (dir = ""): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
withFileTypes: true,
}).flatMap((e) =>
e.isDirectory()
? appSources(`${dir}${e.name}/`)
: e.name.endsWith(".py")
? [backend(`${dir}${e.name}`)]
: [],
);
const refusedReasons = [
...new Set([
...appSources().flatMap((src) =>
[...src.matchAll(/(?<![\w.])refused\(\s*\d+,\s*"(\w+)"/g)].map((m) => m[1]!),
),
// A folder change the rules refuse travels as `FolderError("<reason>")` and is answered with
// `refused(409, e.reason)`, which the pattern above cannot read.
...appSources().flatMap((src) =>
[...src.matchAll(/FolderError\("(\w+)"\)/g)].map((m) => m[1]!),
),
...[
...(backend("refusal.py").split("_SETTING_REASONS = {")[1]?.split("}")[0] ?? "").matchAll(
/:\s*"(\w+)"/g,
),
].map((m) => m[1]!),
]),
];
// A refused YouTube action → `errors.youtube.<reason>`, the reasons read from their Literal.
const youtubeReasons = [
...(
backend("youtube/client.py").match(/^YouTubeRefusal = Literal\[([^\]]*)\]/m)?.[1] ?? ""
).matchAll(/"(\w+)"/g),
].map((m) => m[1]!);
// A clip the server will not cut → `clip.errors.<reason>`: every `ClipError("<reason>"…)` in the backend
// (a reason may carry its numbers after it: `ClipError("too_long", limit_s=…)`).
const clipReasons = [
...new Set(
appSources().flatMap((src) => [...src.matchAll(/ClipError\("(\w+)"[,)]/g)].map((m) => m[1]!)),
),
];
// The closed English vocabularies the backend WRITES into data, named through `vocab.<group>`
// (lib/vocab) by their letters-and-digits key. A word the backend adds shows in English until named —
// so each list is read from its source here: the TMDB crew jobs kept, every auto-tag the tagger can
// create, and the built-in download profiles of the latest migration that seeds them.
const pyStrings = (block: string | undefined, re: RegExp) =>
[...(block ?? "").matchAll(re)].map((m) => m[1]!);
const between = (src: string, start: string, end: string) => src.split(start)[1]?.split(end)[0];
const tmdbPy = backend("plex/tmdb.py");
const crewJobs = [
...new Set([
...pyStrings(between(tmdbPy, "_MOVIE_CREW_JOBS = (", ")"), /"([^"]+)"/g),
...pyStrings(between(tmdbPy, "_TV_CREW_JOBS = (", ")"), /"([^"]+)"/g),
]),
];
const autotagPy = backend("sync/autotag.py");
const autoTags = [
...new Set([
...pyStrings(between(autotagPy, "LANG_NAMES = {", "}"), /:\s*"([^"]+)"/g),
...pyStrings(between(autotagPy, "TOPIC_COLORS = {", "}"), /"([^"]+)":/g),
...pyStrings(between(autotagPy, "_TOPIC_SLUGS = {", "}"), /:\s*"([^"]+)"/g),
...pyStrings(between(autotagPy, "_CATEGORY_NAMES = {", "}"), /:\s*"([^"]+)"/g),
...pyStrings(between(autotagPy, "def map_topic_slug", "\n\n\n"), /return "([^"]+)"/g),
]),
];
const migrations = readdirSync(new URL("../../../backend/alembic/versions/", import.meta.url))
.filter((f) => f.endsWith(".py"))
.sort();
const seeding = migrations
.map((f) =>
readFileSync(new URL(`../../../backend/alembic/versions/${f}`, import.meta.url), "utf-8"),
)
.filter((src) => src.includes("_BUILTINS = ["))
.pop();
const builtinProfiles = pyStrings(
between(seeding ?? "", "_BUILTINS = [", "\n]"),
/^\s+\("([^"]+)",/gm,
);
const VOCAB = { crewJob: crewJobs, autoTag: autoTags, profile: builtinProfiles };
/** The ids under `prefix` a locale names, `prefix.<id>` or `prefix.<id>.<anything>`. */
const idsUnder = (keys: Set<string>, prefix: string) =>
new Set(
[...keys]
.filter((k) => k.startsWith(`${prefix}.`))
.map((k) => k.split(".")[prefix.split(".").length]!),
);
describe("backend ids have their names in both languages", () => {
it("read the registries (a scan that matches nothing is green over nothing)", () => {
expect(specs.map((s) => s.key)).toEqual(expect.arrayContaining(["smtp_host", "tmdb_language"]));
// Every choice key resolved its options (a Literal the scan can't read would check nothing).
for (const s of specs.filter((x) => x.type === "choice"))
expect(choicesOf(s.attr).length, s.key).toBeGreaterThan(1);
expect(choiceIds).toContain("config.choices.log_level.TRACE");
expect(jobs).toEqual(expect.arrayContaining(["rss_poll", "audit_gc"]));
expect(auditActions).toEqual(expect.arrayContaining(["config_set", "audit_clear"]));
expect(quotaActions.length).toBeGreaterThan(0);
expect(runStatuses).toEqual(expect.arrayContaining(["running", "interrupted"]));
expect(runTriggers).toEqual(expect.arrayContaining(["scheduled", "ad_hoc"]));
expect(adminCodes).toEqual(expect.arrayContaining(["activate", "resend"]));
expect(failDetails).toEqual(expect.arrayContaining(["facebook.login_wall"]));
expect(goneWhats).toEqual(expect.arrayContaining(["playlist", "download", "job"]));
expect(opaqueWhats).toEqual([]);
expect(youtubeReasons).toEqual(["quota", "reconnect", "unavailable", "gone"]);
expect(clipReasons).toEqual(expect.arrayContaining(["too_long", "past_end", "bad_crop"]));
expect(crewJobs).toEqual(
expect.arrayContaining(["Director", "Creator", "Director of Photography"]),
);
expect(autoTags).toEqual(
expect.arrayContaining(["English", "Gaming", "Health & Fitness", "Hobbies", "Sports"]),
);
expect(builtinProfiles).toEqual(expect.arrayContaining(["720p (MP4)", "Video only (MP4)"]));
expect(skipWords).toEqual(
expect.arrayContaining([
"disabled",
"sync paused",
"tmdb_api_key not set",
"no service user",
]),
);
expect(refusedReasons).toEqual(
expect.arrayContaining(["weak_password", "setting_too_low", "plex_write_failed"]),
);
// Every code the frontend localizes is raised somewhere the scan can see.
expect(adminCodes.filter((c) => !adminRefusals.some((m) => m[1] === c))).toEqual([]);
});
it("the run list's status chips are the server's run statuses", () => {
expect([...RUN_STATUSES].sort()).toEqual([...runStatuses].sort());
});
it("the Takeout import's notice is known by the server's job id", () => {
// Read as text: the slice pulls in the request layer, which needs a browser.
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
const ours = /^export const TAKEOUT_JOB = "(\w+)";$/m.exec(feedTs)?.[1];
expect(ours).toBeTruthy();
expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours);
});
it("the Takeout import's files and outcomes are the server's", () => {
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
const ours = (name: string) =>
[
...(
new RegExp(`^export const ${name} = \\[([^\\]]*)\\]`, "m").exec(feedTs)?.[1] ?? ""
).matchAll(/"(\w+)"/g),
].map((m) => m[1]);
const models = backend("reader/models.py");
const theirs = (cls: string) =>
[
...(
new RegExp(`^class ${cls}\\(StrEnum\\):\\n([\\s\\S]*?)(?=\\n\\S)`, "m").exec(
models,
)?.[1] ?? ""
).matchAll(/^\s+[A-Z_]+ = "(\w+)"/gm),
].map((m) => m[1]);
expect(theirs("TakeoutKind").length).toBeGreaterThan(0);
expect(ours("TAKEOUT_KINDS")).toEqual(theirs("TakeoutKind"));
expect(theirs("TakeoutOutcome").length).toBeGreaterThan(0);
expect(ours("TAKEOUT_OUTCOMES")).toEqual(theirs("TakeoutOutcome"));
});
for (const lang of ["en", "hu"]) {
const keys = keysOf(lang);
const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k));
it(`${lang}: every run status and trigger`, () => {
expect(
missing([
...runStatuses.flatMap((s) => [`activity.runs.status.${s}`, `scheduler.dot.${s}`]),
...runTriggers.map((s) => `scheduler.trigger.${s}`),
]),
).toEqual([]);
});
it(`${lang}: every config key and group`, () => {
expect(
missing([
...new Set(specs.map((s) => `config.groups.${s.group}`)),
...specs.flatMap((s) => [`config.fields.${s.key}.label`, `config.fields.${s.key}.hint`]),
]),
).toEqual([]);
const known = new Set(specs.map((s) => s.key));
expect([...idsUnder(keys, "config.fields")].filter((k) => !known.has(k))).toEqual([]);
const groups = new Set(specs.map((s) => s.group));
expect([...idsUnder(keys, "config.groups")].filter((g) => !groups.has(g))).toEqual([]);
});
it(`${lang}: every option of every choice key, and none for an option that is gone`, () => {
expect(missing(choiceIds)).toEqual([]);
expect(
[...keys].filter((k) => k.startsWith("config.choices.") && !choiceIds.includes(k)),
).toEqual([]);
});
it(`${lang}: every scheduler job`, () => {
expect(
missing(jobs.flatMap((j) => [`scheduler.jobs.${j}`, `scheduler.jobDesc.${j}`])),
).toEqual([]);
for (const prefix of ["scheduler.jobs", "scheduler.jobDesc"])
expect([...idsUnder(keys, prefix)].filter((j) => !jobs.includes(j))).toEqual([]);
});
it(`${lang}: every word a job's result gives for skipping its work`, () => {
expect(missing(skipWords.map((w) => `scheduler.skip.${vocabKey(w)}`))).toEqual([]);
});
it(`${lang}: every audit and quota action`, () => {
expect(auditActions.filter((a) => !idsUnder(keys, "auditActions").has(a))).toEqual([]);
expect(quotaActions.filter((a) => !idsUnder(keys, "quotaActions").has(a))).toEqual([]);
// …and no name for an action the backend no longer has (the unused-key contract leaves
// these two namespaces to this check).
expect([...idsUnder(keys, "auditActions")].filter((a) => !auditActions.includes(a))).toEqual(
[],
);
expect([...idsUnder(keys, "quotaActions")].filter((a) => !quotaActions.includes(a))).toEqual(
[],
);
});
it(`${lang}: every download failure category`, () => {
expect(failCategories.length).toBeGreaterThan(5);
expect(
missing(
failCategories.flatMap((c) => [
`downloads.errors.fail.cat.${c}.short`,
`downloads.errors.fail.cat.${c}.body`,
]),
),
).toEqual([]);
});
it(`${lang}: every download failure-catalog entry`, () => {
expect(
missing(failDetails.flatMap((d) => [`${DETAIL}${d}.short`, `${DETAIL}${d}.body`])),
).toEqual([]);
const named = new Set(
[...keys]
.filter((k) => k.startsWith(DETAIL))
.map((k) => k.slice(DETAIL.length).replace(/\.(short|body)$/, "")),
);
expect([...named].filter((d) => !failDetails.includes(d))).toEqual([]);
});
it(`${lang}: every "that is gone" a 404 names, and every YouTube refusal`, () => {
expect(missing(goneWhats.map((w) => `errors.notFound.${w}`))).toEqual([]);
expect(
[...idsUnder(keys, "errors.notFound")].filter(
(w) => w !== "generic" && !goneWhats.includes(w),
),
).toEqual([]);
expect(missing(youtubeReasons.map((r) => `errors.youtube.${r}`))).toEqual([]);
expect(
[...idsUnder(keys, "errors.youtube")].filter((r) => !youtubeReasons.includes(r)),
).toEqual([]);
});
it(`${lang}: every generic refusal the backend can answer, and none it cannot`, () => {
expect(missing(refusedReasons.map((r) => `errors.refused.${r}`))).toEqual([]);
expect(
[...idsUnder(keys, "errors.refused")].filter((r) => !refusedReasons.includes(r)),
).toEqual([]);
});
it(`${lang}: every reason a clip is refused, and none it cannot be`, () => {
expect(missing(clipReasons.map((r) => `clip.errors.${r}`))).toEqual([]);
expect([...idsUnder(keys, "clip.errors")].filter((r) => !clipReasons.includes(r))).toEqual(
[],
);
});
it(`${lang}: every crew job, auto-tag and built-in profile the backend writes, and no other`, () => {
for (const [group, words] of Object.entries(VOCAB)) {
const wanted = words.map((w) => `vocab.${group}.${vocabKey(w)}`);
expect(missing(wanted), group).toEqual([]);
expect(
[...keys].filter((k) => k.startsWith(`vocab.${group}.`) && !wanted.includes(k)),
group,
).toEqual([]);
}
});
it(`${lang}: every admin refusal reason`, () => {
expect(missing(adminReasons.map((r) => `errors.admin.${r}`))).toEqual([]);
expect([...idsUnder(keys, "errors.admin")].filter((r) => !adminReasons.includes(r))).toEqual(
[],
);
});
}
});