feat(scheduler): every background run is a durable row

What a job last did lived in the API process's memory and was gone at
every restart; the audit log kept a run only when it was manual, failed
or changed something. Now each run is a job_runs row written at its
start and its end in its own session: who started it, its outcome as
data, what it spent of the YouTube quota (quota_events.job_run_id) and
the lines it logged (log_events.run_id, filterable in the log console).

- a run a stopped process left open is marked interrupted at startup
- "run now" is a one-shot job on the scheduler, which now always starts
  (interval jobs only with SCHEDULER_ENABLED); the manual actor is reset
  after the run, since pool threads are reused
- the dashboard's last run comes from job_runs, so it survives a deploy
- runs leave the audit log: migration 0109 moves the old job_run rows
  over (and back on downgrade); the audit page's run folding goes
- job_run_retention_days (default 30) prunes old runs in audit_gc
This commit is contained in:
2026-10-17 23:08:55 +02:00
parent 4b1f6d7ec2
commit 45c0ee7d58
31 changed files with 870 additions and 489 deletions
@@ -1,48 +0,0 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import clsx from "clsx";
import { ChevronRight } from "lucide-react";
import type { AuditRow } from "../lib/api";
import { Button } from "./ui/Button";
import { RelativeTime } from "./ui/RelativeTime";
/** A folded run of one job's successful runs (lib/auditGroups): the row shows the newest, this
* opens the earlier ones, each with its moment and its result (`detailOf`, the row's own wording). */
export default function AuditEarlierRuns({
runs,
detailOf,
}: {
runs: AuditRow[];
detailOf: (run: AuditRow) => string | null;
}) {
const { t } = useTranslation();
const [open, setOpen] = useState(false);
const earlier = runs.slice(1);
return (
<div className="mt-1">
<Button
variant="link-muted"
aria-expanded={open}
onClick={() => setOpen(!open)}
icon={<ChevronRight className={clsx("transition-transform", open && "rotate-90")} />}
>
{t("audit.earlierRuns", { count: earlier.length })}
</Button>
{open && (
<ul className="mt-1 space-y-0.5 text-2xs text-muted">
{earlier.map((run) => (
<li key={run.id} className="flex gap-2">
<RelativeTime
at={run.created_at}
seconds
empty="—"
className="shrink-0 tabular-nums"
/>
<span className="min-w-0 wrap-break-word">{detailOf(run)}</span>
</li>
))}
</ul>
)}
</div>
);
}
+3 -5
View File
@@ -14,10 +14,9 @@ import { LoadingState, StateMessage } from "./ui/QueryState";
import { auditColumns } from "./auditColumns";
import { notifyActionError } from "../lib/actionErrors";
import { compareText } from "../lib/collate";
import { foldJobRuns } from "../lib/auditGroups";
// Admin-only audit trail: an append-only who/what/when log of admin actions + scheduler run
// summaries (never per-row). Read-only apart from Clear. Client-side sort/filter/paginate via
// Admin-only audit trail: an append-only who/what/when log of admin actions (never per-row; a
// background run is not one — runs have their own history). Read-only apart from Clear. Client-side sort/filter/paginate via
// DataTable; the API returns the most recent window (capped) — the log stays small by design.
export default function AuditLog() {
const { t, i18n } = useTranslation();
@@ -26,8 +25,7 @@ export default function AuditLog() {
const confirm = useConfirm();
const q = useQuery({ queryKey: qk.adminAudit(), queryFn: () => api.adminAudit() });
const data = q.data;
// Back-to-back successful runs of one job fold into one row (lib/auditGroups).
const rows = useMemo(() => foldJobRuns(data?.items ?? []), [data]);
const rows = useMemo(() => data?.items ?? [], [data]);
const actionOptions = useMemo(() => {
// By what the reader sees — the named action, in their alphabet — not by the code.
@@ -1,10 +1,10 @@
// @vitest-environment jsdom
//
// An audit row whose server summary is a fixed English line ("rss_poll: ok — new=5, failed=0",
// "log_level = DEBUG", "Background sync paused") reads in the reader's language, rebuilt from the row's
// An audit row whose server summary is a fixed English line ("log_level = DEBUG", "rss_poll → 15
// min/run", "Background sync paused") reads in the reader's language, rebuilt from the row's
// data through the REAL catalog. A row whose summary is data (an e-mail, a URL) keeps it.
import { afterEach, describe, expect, it } from "vitest";
import { cleanup, fireEvent, render } from "@testing-library/react";
import { cleanup, render } from "@testing-library/react";
import i18n, { setLanguage } from "../i18n";
import type { AuditRow } from "../lib/api";
import { auditColumns } from "./auditColumns";
@@ -20,7 +20,7 @@ const row = (over: Partial<AuditRow>): AuditRow => ({
id: 1,
created_at: null,
actor: null,
action: "job_run",
action: "sync_pause",
target_type: null,
target_id: null,
summary: null,
@@ -38,27 +38,6 @@ function details(r: AuditRow): string {
}
describe("an audit row's details", () => {
it("name a job run and its result in the reader's language", () => {
const r = row({
target_type: "job",
target_id: "rss_poll",
summary: "rss_poll: ok — new=5, failed=0",
after: { status: "ok", summary: "new=5, failed=0" },
});
expect(details(r)).toBe("RSS-lekérdezés (új feltöltések) kész | új videó: 5 · sikertelen: 0");
});
it("keep a failed run's error text, which is diagnostic", () => {
const r = row({
target_type: "job",
target_id: "enrich",
after: { status: "error", summary: "HTTP 403: quotaExceeded" },
});
expect(details(r)).toBe(
"Nem sikerült lefuttatni: Adatdúsítás + élő frissítés | HTTP 403: quotaExceeded",
);
});
it("name a config change's field and values", () => {
const set = row({
action: "config_set",
@@ -87,26 +66,6 @@ describe("an audit row's details", () => {
expect(details(secret)).toMatch(/ \| titok frissítve/);
});
it("name a nested result and a skipped run from the row's data", () => {
const nested = row({
target_type: "job",
target_id: "people_enrich",
after: {
status: "ok",
summary: "bios={'enriched': 12}, portraits={'cached': 40}",
result: { bios: { enriched: 12 }, portraits: { cached: 40 } },
},
});
expect(details(nested)).not.toContain("{");
expect(details(nested)).toMatch(/^Személyadatok és portrék \(Plex\) kész \| /);
const skipped = row({
target_type: "job",
target_id: "rss_poll",
after: { status: "skipped", summary: "sync paused" },
});
expect(details(skipped)).toBe("RSS-lekérdezés (új feltöltések) kihagyva | a szinkron szünetel");
});
it("tell a secret's reset from its update, and name a choice", () => {
const reset = row({
action: "config_reset",
@@ -157,49 +116,3 @@ describe("an audit row's details", () => {
);
});
});
// A folded run (lib/auditGroups) shows its newest run and opens to the earlier ones; a failed run
// is the line to see in a page of routine ones.
describe("a folded run of one job", () => {
const ok = (id: number, enriched: number) =>
row({
id,
target_type: "job",
target_id: "enrich",
after: { status: "ok", result: { enriched } },
});
it("names how many earlier runs it holds, and opens to each", () => {
setLanguage("hu");
const t = i18n.getFixedT("hu");
const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!;
const runs = [ok(3, 50), ok(2, 40), ok(1, 30)];
const { getByRole, container } = render(<>{col.render!({ ...runs[0]!, runs })}</>);
const toggle = getByRole("button", { name: "2 korábbi futás" });
expect(toggle.getAttribute("aria-expanded")).toBe("false");
expect(container.querySelector("li")).toBeNull();
fireEvent.click(toggle);
expect(toggle.getAttribute("aria-expanded")).toBe("true");
expect([...container.querySelectorAll("li")].map((li) => li.textContent)).toEqual([
"—dúsítva: 40",
"—dúsítva: 30",
]);
});
it("has no toggle for a lone run", () => {
const t = i18n.getFixedT("hu");
const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!;
const { container } = render(<>{col.render!(ok(1, 5))}</>);
expect(container.querySelector("button")).toBeNull();
});
it("paints a failed run's line in the danger colour", () => {
const t = i18n.getFixedT("hu");
const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!;
const failed = row({ target_type: "job", target_id: "enrich", after: { status: "error" } });
const { container } = render(<>{col.render!(failed)}</>);
expect(container.querySelector(".wrap-break-word")!.className).toContain("text-danger");
const { container: fine } = render(<>{col.render!(ok(2, 5))}</>);
expect(fine.querySelector(".wrap-break-word")!.className).not.toContain("text-danger");
});
});
+4 -22
View File
@@ -1,11 +1,8 @@
import type { TFunction } from "i18next";
import clsx from "clsx";
import type { AuditRow } from "../lib/api";
import type { AuditListRow } from "../lib/auditGroups";
import AuditEarlierRuns from "./AuditEarlierRuns";
import { formatNumber } from "../lib/format";
import type { JobResult } from "../lib/api";
import { jobResultText, jobRunText } from "../lib/jobResult";
import { jobResultText } from "../lib/jobResult";
import type { Column } from "./ui/DataTable";
import { RelativeTime } from "./ui/RelativeTime";
@@ -35,8 +32,7 @@ function counts(after: Record<string, unknown>): JobResult | null {
}
/** The details of the rows whose server `summary` is a fixed English line, rebuilt from the row's
* data in the reader's language: a job run names the job and its result (as the inbox does), a config
* change names the field and its values, an interval or batch change its new value, a count-bearing
* data in the reader's language: a config change names the field and its values, an interval or batch change its new value, a count-bearing
* action its counts; a pause/resume says nothing the action has not. `undefined` = not one of these (or
* an older row without the data); the generic summary + change rendering applies. */
function localDetails(
@@ -46,10 +42,6 @@ function localDetails(
): { head: string | null; sub: string | null } | undefined {
const after = row.after ?? {};
const before = row.before ?? {};
if (row.action === "job_run" && row.target_type === "job" && row.target_id) {
const run = jobRunText(t, { job_id: row.target_id, ...after }, lang);
return { head: run.title, sub: run.body };
}
if ((row.action === "config_set" || row.action === "config_reset") && row.target_id) {
const key = row.target_id;
const head = t(`config.fields.${key}.label`, key);
@@ -104,7 +96,7 @@ export function auditColumns(
t: TFunction,
lang: string,
actionOptions: { value: string; label: string }[],
): Column<AuditListRow>[] {
): Column<AuditRow>[] {
return [
{
key: "when",
@@ -146,23 +138,13 @@ export function auditColumns(
const local = localDetails(t, r, lang);
const summary = local ? local.head : r.summary;
const change = local ? local.sub : changeText(r);
// A failed run is the row to see in a page of routine ones: its line in the danger colour.
const failed = r.action === "job_run" && r.after?.status === "error";
return (
// wrap-break-word (wrap), NOT truncate: in the auto-layout table a `truncate` (nowrap) cell
// sizes to its longest row, so one long summary/change stretched the whole table past the
// card (the row rules bled into the page). Wrapping keeps every row inside the container.
<div className="min-w-0">
{summary && (
<div className={clsx("wrap-break-word", failed && "text-danger")}>{summary}</div>
)}
{summary && <div className="wrap-break-word">{summary}</div>}
{change && <div className="text-2xs text-muted wrap-break-word">{change}</div>}
{r.runs && r.runs.length > 1 && (
<AuditEarlierRuns
runs={r.runs}
detailOf={(run) => localDetails(t, run, lang)?.sub ?? run.summary}
/>
)}
</div>
);
},
+2 -4
View File
@@ -1,6 +1,6 @@
{
"title": "Audit log",
"intro": "An append-only record of admin actions and scheduler run summaries — who did what, and when. Logged at action granularity, never per item.",
"intro": "An append-only record of admin actions — who did what, and when. Logged at action granularity, never per item.",
"empty": "No audit entries yet.",
"system": "System",
"clear": "Clear audit log",
@@ -20,7 +20,5 @@
"actor": "Who",
"action": "Action",
"details": "Details"
},
"earlierRuns_one": "{{count}} earlier run",
"earlierRuns_other": "{{count}} earlier runs"
}
}
@@ -16,7 +16,6 @@
"maintenance_tune": "Maintenance tuned",
"sync_pause": "Sync paused",
"sync_resume": "Sync resumed",
"job_run": "Scheduler job run",
"audit_clear": "Audit log cleared",
"download_credential_set": "Download cookies set",
"download_credential_delete": "Download cookies deleted",
+4
View File
@@ -170,6 +170,10 @@
"label": "Retention (days)",
"hint": "How long admin audit-log entries are kept before the audit-log cleanup job prunes them. 0 = keep forever. Default 180."
},
"job_run_retention_days": {
"label": "Run history (days)",
"hint": "How long the history of background runs (scheduled, run now, a channel's full history) is kept before the cleanup job prunes it. 0 = keep forever. Default 30."
},
"log_level": {
"label": "Log level",
"hint": "How much the server writes to its log. INFO is the normal setting; DEBUG and TRACE are for reproducing a problem and are far more detailed. Takes effect immediately, no restart."
+2 -4
View File
@@ -1,6 +1,6 @@
{
"title": "Eseménynapló",
"intro": "Az admin-műveletek és az ütemező futás-összegzéseinek csak-hozzáfűző naplója — ki, mit, mikor. Művelet-szinten naplózva, soha nem elemenként.",
"intro": "Az admin-műveletek csak-hozzáfűző naplója — ki, mit, mikor. Művelet-szinten naplózva, soha nem elemenként.",
"empty": "Még nincs naplóbejegyzés.",
"system": "Rendszer",
"clear": "Eseménynapló ürítése",
@@ -20,7 +20,5 @@
"actor": "Ki",
"action": "Művelet",
"details": "Részletek"
},
"earlierRuns_one": "{{count}} korábbi futás",
"earlierRuns_other": "{{count}} korábbi futás"
}
}
@@ -16,7 +16,6 @@
"maintenance_tune": "Karbantartás hangolva",
"sync_pause": "Szinkron szüneteltetve",
"sync_resume": "Szinkron folytatva",
"job_run": "Ütemezett feladat lefutott",
"audit_clear": "Eseménynapló ürítve",
"download_credential_set": "Letöltési cookie-k beállítva",
"download_credential_delete": "Letöltési cookie-k törölve",
+4
View File
@@ -170,6 +170,10 @@
"label": "Megőrzés (nap)",
"hint": "Meddig maradnak meg az admin-naplóbejegyzések, mielőtt az eseménynapló-takarító feladat törli őket. 0 = örökre. Alapérték 180."
},
"job_run_retention_days": {
"label": "Futási előzmények (nap)",
"hint": "Meddig maradnak meg a háttérfutások (ütemezett, azonnali, egy csatorna teljes előzménye) adatai, mielőtt a takarító feladat törli őket. 0 = örökre. Alapérték 30."
},
"log_level": {
"label": "Naplózási szint",
"hint": "Mennyit naplózzon a szerver. Az INFO a szokásos; a DEBUG és a TRACE hibakereséshez való, sokkal részletesebb. Azonnal érvénybe lép, újraindítás nélkül."
-72
View File
@@ -1,72 +0,0 @@
// The audit log's first screen was one job's "done" ten times over. A job's successful runs fold
// into one row whatever ran between them (the owner: a fold broken by every other job read as the same
// row five times); only that job's own failure or skip breaks it.
import { describe, expect, it } from "vitest";
import type { AuditRow } from "./api";
import { foldJobRuns } from "./auditGroups";
let nextId = 1;
const run = (job: string, status = "ok"): AuditRow => ({
id: nextId++,
created_at: null,
actor: null,
action: "job_run",
target_type: "job",
target_id: job,
summary: null,
before: null,
after: { status },
});
const admin = (): AuditRow => ({ ...run("x"), action: "config_set", target_type: "config" });
const shape = (rows: ReturnType<typeof foldJobRuns>) =>
rows.map((r) => `${r.target_id}:${String(r.after?.status)}×${r.runs?.length ?? 1}`);
describe("folding the job runs", () => {
it("folds a job's successful runs into one row, newest first", () => {
const rows = [run("enrich"), run("enrich"), run("enrich"), run("people")];
const out = foldJobRuns(rows);
expect(shape(out)).toEqual(["enrich:ok×3", "people:ok×1"]);
expect(out[0]!.id).toBe(rows[0]!.id);
expect(out[0]!.runs!.map((r) => r.id)).toEqual(rows.slice(0, 3).map((r) => r.id));
});
it("never folds a failed or skipped run, and lets it break the run", () => {
const rows = [run("rss"), run("rss", "error"), run("rss", "error"), run("rss"), run("rss")];
expect(shape(foldJobRuns(rows))).toEqual([
"rss:ok×1",
"rss:error×1",
"rss:error×1",
"rss:ok×2",
]);
expect(shape(foldJobRuns([run("rss", "skipped"), run("rss", "skipped")]))).toEqual([
"rss:skipped×1",
"rss:skipped×1",
]);
});
it("folds across other jobs and an admin's own action, each at its own place", () => {
const rows = [run("enrich"), run("plex"), run("enrich"), admin(), run("shorts"), run("enrich")];
rows.push(run("plex"));
const out = foldJobRuns(rows);
expect(shape(out)).toEqual(["enrich:ok×3", "plex:ok×2", "x:ok×1", "shorts:ok×1"]);
expect(out[0]!.runs!.map((r) => r.id)).toEqual([rows[0]!.id, rows[2]!.id, rows[5]!.id]);
});
it("starts a new fold below the job's own failure, which another job's does not", () => {
const rows = [run("enrich"), run("enrich", "error"), run("enrich"), run("rss", "error")];
rows.push(run("enrich"));
expect(shape(foldJobRuns(rows))).toEqual([
"enrich:ok×1",
"enrich:error×1",
"enrich:ok×2",
"rss:error×1",
]);
});
it("leaves the rows it was given as they were", () => {
const rows = [run("enrich"), run("enrich")];
foldJobRuns(rows);
expect("runs" in rows[0]!).toBe(false);
});
});
-48
View File
@@ -1,48 +0,0 @@
import type { AuditRow } from "./api";
// The audit log's scheduler noise, folded (the owner's pick): the background jobs write a row per
// run that changed something, so an hour of "enrichment done" filled the first screen and pushed the
// admins' own actions down. A job's SUCCESSFUL runs become one row that names how many there were and
// opens to each — whatever ran between them (another job, an admin's action): the owner read a fold
// that broke at every other job as the same row five times. Only that job's own failed or skipped run
// breaks it: that row stays on its own, at its moment, where it is seen, and the job's older successes
// start a new fold below it. Rows arrive newest first; a fold stands at its newest run's place.
/** An audit row as the table lists it: a lone row, or the newest of a job's folded runs (`runs`, the
* whole fold newest first, the row itself included). */
export type AuditListRow = AuditRow & { runs?: AuditRow[] };
/** The job a row is a run of, or null for any other row. */
function jobOf(row: AuditRow): string | null {
return row.action === "job_run" && row.target_type === "job" && row.target_id
? row.target_id
: null;
}
export function foldJobRuns(rows: readonly AuditRow[]): AuditListRow[] {
const out: AuditListRow[] = [];
// Each job's open fold: the row its next older success joins.
const open = new Map<string, AuditListRow>();
for (const row of rows) {
const job = jobOf(row);
if (job === null) {
out.push(row);
continue;
}
if (row.after?.status !== "ok") {
// A failure or a skip stands alone and closes the job's fold.
open.delete(job);
out.push(row);
continue;
}
const head = open.get(job);
if (head) {
head.runs = [...(head.runs ?? [head]), row];
continue;
}
const fresh: AuditListRow = { ...row };
open.set(job, fresh);
out.push(fresh);
}
return out;
}