fix(activity): runs keep their true outcome and spend, the page reaches every kept row, a closing dialog leaves the server log alone

Backend
- A run's outcome write sits outside the job's own try: a database blip after a run that
  succeeded no longer records it as failed (the row waits for the next start's sweep).
- An interrupted run keeps what it spent (the sweep sums its quota rows, as finish does).
- The Scheduler's ad-hoc row says running while any backfill runs, not only the newest.
- recent_runs reads each job's last runs off its index instead of ranking the whole table.
- An interval change is stored with its audit row in one transaction, then rescheduled.
- One API instance per database is the stated contract (.env.example, jobruns).
- Turning full history on for a channel already complete records no run.
- Migration 0109 round-trips interrupted and ad-hoc runs; the run words' DB CHECKs are
  test-held to the enums; the account filter is one helper for the four activity reads.
- An audit row says how many of its request's log lines are still kept.

Frontend
- An "All" period reaches every row still kept (runs and admin actions outlive 90 days).
- Back/Forward is followed in place by the server log: a closing dialog no longer turns
  Live off; a pivot still opens a fresh log.
- Runs moved over from the audit log read their one-liner in the reader's language.
- Filter choices or accounts that failed to load say so beside the filters, with a retry.
- An old server-log link keeps its account, text and range; a remembered retired page
  opens its tab.
- A request whose lines are gone says so instead of linking to an empty list.
- The log lines and the history rows share one row frame, and the server log one list
  frame with the other tabs; an empty list is announced; each day's quota is readable.

backend/alembic/versions/0109_job_runs.py

@@ -12,7 +12,8 @@ log writer runs behind the event).
 
 Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows
 MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the
-audit log. The downgrade moves every run back into the audit log as such a row, then drops the table.
+audit log. The downgrade moves every run back into the audit log as such a row (one still running as
+interrupted, a run's trigger and target in its `after`), then drops the table.
 """
 
 import sqlalchemy as sa
@@ -70,15 +71,19 @@ def upgrade() -> None:
         postgresql_where=sa.text("run_id IS NOT NULL"),
     )
 
-    # The audit trail's runs move over. A status the run table does not know (none was ever
-    # written) would fail the CHECK, so the move is limited to the three the scheduler wrote.
+    # The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`,
+    # which a downgrade writes back (as it does a run's `trigger` and `target`, which the audit row
+    # has no column for), so a rollback and a redeploy keep every run. A status the run table does
+    # not know would fail the CHECK.
     op.execute(
         """
-        INSERT INTO job_runs (job, trigger, actor_id, status, started_at, finished_at, summary,
-                              result, error, failure)
+        INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at,
+                              summary, result, error, failure)
         SELECT target_id,
-               CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END,
+               coalesce(after::jsonb ->> 'trigger',
+                        CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END),
                actor_id,
+               after::jsonb ->> 'target',
                after::jsonb ->> 'status',
                created_at,
                created_at,
@@ -89,7 +94,7 @@ def upgrade() -> None:
           FROM audit_log
          WHERE action = 'job_run'
            AND target_id IS NOT NULL
-           AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped')
+           AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted')
          ORDER BY id
         """
     )
@@ -98,12 +103,15 @@ def upgrade() -> None:
         DELETE FROM audit_log
          WHERE action = 'job_run'
            AND target_id IS NOT NULL
-           AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped')
+           AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted')
         """
     )
 
 
 def downgrade() -> None:
+    # A run still `running` is one the rollback stopped: it goes back as interrupted. Its trigger and
+    # target ride in `after` when the audit row's own columns cannot say them (an ad-hoc run of one
+    # channel), so the upgrade restores them.
     op.execute(
         """
         INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at)
@@ -111,21 +119,26 @@ def downgrade() -> None:
                'job_run',
                'job',
                job,
-               left(job || ': ' || status
-                    || coalesce(' — ' || CASE WHEN status = 'error' THEN error ELSE summary END, ''),
+               left(job || ': ' || st
+                    || coalesce(' — ' || CASE WHEN st = 'error' THEN error ELSE summary END, ''),
                     255),
                (jsonb_build_object(
-                    'status', status,
-                    'summary', CASE WHEN status = 'error' THEN error ELSE summary END)
+                    'status', st,
+                    'summary', CASE WHEN st = 'error' THEN error ELSE summary END)
                 || CASE
-                     WHEN status = 'error' AND failure IS NOT NULL
+                     WHEN st = 'error' AND failure IS NOT NULL
                        THEN jsonb_build_object('result', failure)
-                     WHEN status <> 'error' AND result IS NOT NULL
+                     WHEN st <> 'error' AND result IS NOT NULL
                        THEN jsonb_build_object('result', result)
                      ELSE '{}'::jsonb
-                   END)::json,
+                   END
+                || CASE WHEN trigger = 'ad_hoc'
+                     THEN jsonb_build_object('trigger', trigger) ELSE '{}'::jsonb END
+                || CASE WHEN target IS NOT NULL
+                     THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json,
                coalesce(finished_at, started_at)
-          FROM job_runs
+          FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st
+                  FROM job_runs) AS r
          ORDER BY id
         """
     )

backend/app/jobruns.py

@@ -5,13 +5,14 @@ OWN short session, committed at once, for the reason `quota.record_usage` gives:
 may roll back (its `except: db.rollback()`), and the history of what happened must not go with it —
 nor may this bookkeeping commit the job's half-done session.
 
-Only the API process runs jobs (the worker never starts the scheduler), so at its startup nothing can
-still be running: `interrupt_leftovers` closes what a stopped process left open.
+Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a
+database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its
+startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open.
 """
 
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import delete, func, select, update
+from sqlalchemy import delete, func, select, union_all, update
 from sqlalchemy.engine import CursorResult
 from sqlalchemy.orm import Session
 
@@ -70,30 +71,46 @@ def finish(
 
 
 def interrupt_leftovers() -> int:
-    """Every row a previous process left `running` → `interrupted`. Called once at startup, before
-    the scheduler can open a new run. Returns how many."""
+    """Every row a previous process left `running` → `interrupted`, with what it spent until then
+    (its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler
+    can open a new run. Returns how many."""
+    spent = (
+        select(func.coalesce(func.sum(QuotaEvent.units), 0))
+        .where(QuotaEvent.job_run_id == JobRun.id)
+        .scalar_subquery()
+    )
     with SessionLocal() as s:
         res = s.execute(
             update(JobRun)
             .where(JobRun.status == RunStatus.RUNNING)
-            .values(status=RunStatus.INTERRUPTED, finished_at=datetime.now(timezone.utc))
+            .values(
+                status=RunStatus.INTERRUPTED,
+                finished_at=datetime.now(timezone.utc),
+                quota_units=spent,
+            )
         )
         s.commit()
         return res.rowcount if isinstance(res, CursorResult) else 0
 
 
-def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]:
-    """Each job's last `per_job` runs, newest first, by job id — one query (a window per job)."""
-    ranked = select(
-        JobRun.id,
-        func.row_number()
-        .over(partition_by=JobRun.job, order_by=(JobRun.started_at.desc(), JobRun.id.desc()))
-        .label("rank"),
+def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]:
+    """Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a
+    short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window
+    over the whole table grew with every run kept)."""
+    if not jobs:
+        return {}
+    newest = union_all(
+        *(
+            select(JobRun.id)
+            .where(JobRun.job == job)
+            .order_by(JobRun.started_at.desc(), JobRun.id.desc())
+            .limit(per_job)
+            for job in jobs
+        )
     ).subquery()
     rows = db.execute(
         select(JobRun)
-        .join(ranked, ranked.c.id == JobRun.id)
-        .where(ranked.c.rank <= per_job)
+        .where(JobRun.id.in_(select(newest.c.id)))
         .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc())
     ).scalars()
     out: dict[str, list[JobRun]] = {}
@@ -102,6 +119,16 @@ def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]:
     return out
 
 
+def running_one(db: Session, job: str) -> JobRun | None:
+    """The newest run of `job` still going, however many runs ended after it started."""
+    return db.scalars(
+        select(JobRun)
+        .where(JobRun.job == job, JobRun.status == RunStatus.RUNNING)
+        .order_by(JobRun.started_at.desc(), JobRun.id.desc())
+        .limit(1)
+    ).first()
+
+
 def gc(db: Session, retention_days: int) -> int:
     """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is
     never reaped. The caller commits."""

backend/app/models.py

@@ -720,7 +720,7 @@ class JobRun(Base):
         CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"),
         # Only a run still going has no end.
         CheckConstraint(
-            "(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished"
+            f"(status = '{RunStatus.RUNNING}') = (finished_at IS NULL)", name="ck_job_runs_finished"
         ),
     )
 

backend/app/routes/_account.py

@@ -0,0 +1,17 @@
+"""The account filter the activity page's lists share (runs, admin actions, server log, quota): one
+filter bar drives them all, so the value it sends means the same on every tab.
+
+The value is an account id, or `none` — the rows no account made (the background) — a value to filter
+on, not the absence of a filter.
+"""
+
+from sqlalchemy import ColumnElement
+
+NONE = "none"
+# The `user` query parameter's shape on every route that takes it.
+PATTERN = r"^(none|[0-9]{1,10})$"
+
+
+def condition(column, value: str) -> ColumnElement[bool]:
+    """`column` (an account id column) narrowed to `value`."""
+    return column.is_(None) if value == NONE else column == int(value)

backend/app/routes/admin_logs.py

@@ -25,6 +25,7 @@ from app import logsink
 from app.auth import admin_user
 from app.db import get_db
 from app.models import LogEvent, User
+from app.routes import _account as account
 from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
 from app.utils import like_escape
 
@@ -37,9 +38,9 @@ FACET_EVENTS_MAX = 100
 Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 # The filters a facet can leave out of its own count (`LogFilters.conditions`).
 Facet = Literal["levels", "sources", "user", "event"]
-# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
-# request) — a real value to filter on, not the absence of a filter.
-NONE = "none"
+# `none` selects the rows that have no event code, as the account filter's `none` (app/routes/_account)
+# selects the rows of no account — a real value to filter on, not the absence of a filter.
+NONE = account.NONE
 
 
 @dataclass(frozen=True)
@@ -67,11 +68,7 @@ class LogFilters:
         if self.sources and skip != "sources":
             out.append(LogEvent.source.in_(self.sources))
         if self.user is not None and skip != "user":
-            out.append(
-                LogEvent.user_id.is_(None)
-                if self.user == NONE
-                else LogEvent.user_id == int(self.user)
-            )
+            out.append(account.condition(LogEvent.user_id, self.user))
         if self.event is not None and skip != "event":
             out.append(
                 LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
@@ -103,7 +100,7 @@ class LogFilters:
 def log_filters(
     levels: list[Level] = Query(default=[]),
     sources: list[str] = Query(default=[], max_length=8),
-    user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
+    user: str | None = Query(default=None, pattern=account.PATTERN),
     event: str | None = Query(
         default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
     ),

backend/app/routes/audit.py

@@ -15,17 +15,15 @@ from app import audit
 from app.audit import AuditAction
 from app.auth import admin_user
 from app.db import get_db
-from app.models import AuditLog, User
+from app.models import AuditLog, LogEvent, User
+from app.routes import _account as account
 from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
 from app.utils import iso, like_escape
 
 router = APIRouter(prefix="/api/admin/audit", tags=["admin"])
 
-# `none` selects the rows no account made (the background) — a value to filter on, not "any".
-NONE = "none"
 
-
-def _serialize(row: AuditLog) -> dict:
+def _serialize(row: AuditLog, lines: dict[str, int]) -> dict:
     return {
         "id": row.id,
         "created_at": iso(row.created_at),
@@ -38,6 +36,9 @@ def _serialize(row: AuditLog) -> dict:
         "before": row.before,
         "after": row.after,
         "request_id": row.request_id,
+        # How many of that request's log lines are still kept: they go after days, the row after
+        # months, and a link to none would open an empty list.
+        "log_lines": lines.get(row.request_id, 0) if row.request_id else 0,
     }
 
 
@@ -56,11 +57,7 @@ class AuditFilters:
         if self.action is not None and skip != "action":
             out.append(AuditLog.action == self.action)
         if self.actor is not None:
-            out.append(
-                AuditLog.actor_id.is_(None)
-                if self.actor == NONE
-                else AuditLog.actor_id == int(self.actor)
-            )
+            out.append(account.condition(AuditLog.actor_id, self.actor))
         if self.since is not None:
             out.append(AuditLog.created_at >= utc(self.since))
         if self.until is not None:
@@ -82,7 +79,7 @@ class AuditFilters:
 
 def audit_filters(
     action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"),
-    user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
+    user: str | None = Query(default=None, pattern=account.PATTERN),
     q: str | None = Query(default=None, max_length=200),
     since: datetime | None = None,
     until: datetime | None = None,
@@ -109,8 +106,17 @@ def list_audit(
         limit,
         lambda r: r.created_at,
     )
+    requests = {r.request_id for r in rows if r.request_id}
+    lines: dict[str, int] = {}
+    if requests:
+        counted = db.execute(
+            select(LogEvent.request_id, func.count())
+            .where(LogEvent.request_id.in_(requests))
+            .group_by(LogEvent.request_id)
+        )
+        lines = {rid: n for rid, n in counted.tuples() if rid is not None}
     return {
-        "items": [_serialize(r) for r in rows],
+        "items": [_serialize(r, lines) for r in rows],
         "accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}),
         "next_cursor": cursor,
     }

backend/app/routes/channels.py

@@ -512,7 +512,9 @@ def update_channel(
     # so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks.
     # The request returns right away; the channel page's live poll shows the count climb. The
     # scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set).
-    if deep_turned_on:
+    # A channel whose history is all in gets no run: it would only say "nothing to do", one kept row
+    # per flip of the flag.
+    if deep_turned_on and not (channel is not None and channel.backfill_done):
         scheduler.run_ad_hoc(
             scheduler.CHANNEL_BACKFILL,
             partial(backfill_channel_now, channel_id, user.id),

backend/app/routes/quota.py

@@ -14,6 +14,7 @@ from app import quota
 from app.auth import admin_user, current_user
 from app.db import get_db
 from app.models import QuotaEvent, User
+from app.routes import _account as account
 from app.routes._keyset import utc
 
 router = APIRouter(prefix="/api/quota", tags=["quota"])
@@ -48,7 +49,7 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db))
 def admin_usage(
     since: datetime | None = None,
     until: datetime | None = None,
-    user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
+    user: str | None = Query(default=None, pattern=account.PATTERN),
     _: User = Depends(admin_user),
     db: Session = Depends(get_db),
 ) -> dict:
@@ -60,9 +61,7 @@ def admin_usage(
     if until is not None:
         window.append(QuotaEvent.created_at < utc(until))
     if user is not None:
-        window.append(
-            QuotaEvent.user_id.is_(None) if user == "none" else QuotaEvent.user_id == int(user)
-        )
+        window.append(account.condition(QuotaEvent.user_id, user))
 
     totals = db.execute(
         select(QuotaEvent.user_id, func.sum(QuotaEvent.units))

backend/app/routes/runs.py

@@ -19,14 +19,13 @@ from app import scheduler
 from app.auth import admin_user
 from app.db import get_db
 from app.models import Channel, JobRun, LogEvent, User
+from app.routes import _account as account
 from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
 from app.runstatus import RunStatus
 from app.utils import iso, like_escape
 
 router = APIRouter(prefix="/api/admin/runs", tags=["admin"])
 
-# `none` selects the runs no account started (the scheduled ones).
-NONE = "none"
 Facet = Literal["job", "status"]
 
 
@@ -68,11 +67,7 @@ class RunFilters:
         if self.status is not None and skip != "status":
             out.append(JobRun.status == self.status)
         if self.actor is not None:
-            out.append(
-                JobRun.actor_id.is_(None)
-                if self.actor == NONE
-                else JobRun.actor_id == int(self.actor)
-            )
+            out.append(account.condition(JobRun.actor_id, self.actor))
         if self.since is not None:
             out.append(JobRun.started_at >= utc(self.since))
         if self.until is not None:
@@ -93,7 +88,7 @@ class RunFilters:
 def run_filters(
     job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"),
     status: RunStatus | None = None,
-    user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
+    user: str | None = Query(default=None, pattern=account.PATTERN),
     q: str | None = Query(default=None, max_length=200),
     since: datetime | None = None,
     until: datetime | None = None,

backend/app/routes/scheduler.py

@@ -17,8 +17,9 @@ from app.scheduler import (
     JOB_INTERVALS,
     MAX_INTERVAL,
     MIN_INTERVAL,
-    apply_interval,
+    reschedule,
     scheduler_snapshot,
+    stage_interval,
     trigger_all,
     trigger_job,
 )
@@ -42,14 +43,14 @@ def set_job_interval(
     admin: User = Depends(admin_user),
     db: Session = Depends(get_db),
 ) -> dict:
-    """Change how often a scheduler job runs (minutes). Persisted as an override and applied
-    to the live scheduler immediately."""
+    """Change how often a scheduler job runs (minutes). Stored as an override together with its
+    audit row, then applied to the live scheduler."""
     if job_id not in JOB_INTERVALS:
         raise not_found("job")
     minutes = body.interval_minutes
     if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL):
         raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL)
-    applied = apply_interval(db, job_id, minutes)
+    applied = stage_interval(db, job_id, minutes)
     audit.record(
         db,
         admin.id,
@@ -60,6 +61,7 @@ def set_job_interval(
         after={"interval_minutes": applied},
     )
     db.commit()
+    reschedule(job_id, applied)
     return {"ok": True, "id": job_id, "interval_minutes": applied}
 
 

backend/app/runstatus.py

@@ -1,9 +1,11 @@
 """The words a background run's row is written with — the `job_runs` table's status and trigger.
 
 One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the
-model's CHECK constraints are built from these sets, and the dashboard reads them back. A run's
-status is never typed as a string anywhere else (the download pipeline's `cancelled`/`canceled`
-drift, `downloads/status.py`, is why).
+model's CHECK constraints are built from these sets, and the dashboard reads them back (the frontend's
+copies are held to them by `backendIds.contract.test.ts`). The migrated database's CHECKs are frozen
+in its migration: a test holds them equal to these sets, so a new word fails until a migration adds
+it (`test_runstatus.py`). The download pipeline's `cancelled`/`canceled` drift (`downloads/status.py`)
+is why a status is never typed as a bare string.
 """
 
 from enum import StrEnum

backend/app/scheduler.py

@@ -3,7 +3,7 @@ recent-first / deep backfill. Runs inside the API process (single worker)."""
 
 import contextvars
 import threading
-from datetime import datetime, timezone
+from datetime import timezone
 from typing import Callable
 
 from apscheduler.executors.pool import ThreadPoolExecutor
@@ -27,6 +27,7 @@ from app.sync.autotag import run_autotag_all
 from app.sync.explore import purge_ephemeral
 from app.sync.maintenance import run_maintenance
 from app.sync.playlists import sync_all_playlists
+from app.utils import iso
 from app.sync.runner import (
     run_deep_backfill,
     run_enrich,
@@ -67,18 +68,21 @@ def load_intervals(db) -> dict[str, int]:
     return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS}
 
 
-def apply_interval(db, job_id: str, minutes: int) -> int:
-    """Persist a job's interval override and reschedule the live job immediately (if the
-    scheduler runs in this process). Returns the applied value."""
+def stage_interval(db, job_id: str, minutes: int) -> int:
+    """Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it
+    with its audit row, then calls `reschedule` — the live job follows only what was stored."""
     if job_id not in JOB_INTERVALS:
         raise KeyError(job_id)
     minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes)))
     sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes)
-    db.commit()
+    return minutes
+
+
+def reschedule(job_id: str, minutes: int) -> None:
+    """Move the live interval job to `minutes` (if this process registered it)."""
     if _scheduler is not None and _scheduler.get_job(job_id) is not None:
         _scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
         log.info("rescheduled job %s to every %s min", job_id, minutes)
-    return minutes
 
 
 def _record(name: str, **fields) -> None:
@@ -87,7 +91,12 @@ def _record(name: str, **fields) -> None:
 
 
 def _notify_done(
-    db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None
+    db,
+    actor_id: int,
+    job_id: str,
+    status: RunStatus,
+    summary: str | None,
+    result: dict | None = None,
 ) -> None:
     """Post a completion notice to the triggering admin's inbox (manual runs only). Best
     effort — a notification failure must never mask the job's own outcome."""
@@ -139,24 +148,40 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None:
     def sink(current, total, phase):
         _record(name, progress={"current": current, "total": total, "phase": phase})
 
+    # Only the work is inside the try: writing its outcome is not the job, and a write that fails
+    # must not turn a run that succeeded into a failed one (see _close).
+    result = None
     try:
-        if is_sync_paused(db):
-            log.info("job %s skipped (sync paused)", name)
-            jobruns.finish(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
-            if actor_id is not None:
-                _notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
-            return
-        with progress.bind(sink):
-            result = fn(db)
-        summary, data = _finish_ok(name, run_id, result)
-        if actor_id is not None:
-            _notify_done(db, actor_id, name, "ok", summary, data)
+        paused = is_sync_paused(db)
+        if not paused:
+            with progress.bind(sink):
+                result = fn(db)
     except Exception as exc:
         db.rollback()
         log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
-        err, failure = _finish_error(run_id, exc)
+        err, failure = _failure(exc)
+        _close(run_id, RunStatus.ERROR, error=err, failure=failure)
         if actor_id is not None:
-            _notify_done(db, actor_id, name, "error", err, failure)
+            _notify_done(db, actor_id, name, RunStatus.ERROR, err, failure)
+        return
+    if paused:
+        log.info("job %s skipped (sync paused)", name)
+        _close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
+        if actor_id is not None:
+            _notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED)
+        return
+    summary, data = _finish_ok(name, run_id, result)
+    if actor_id is not None:
+        _notify_done(db, actor_id, name, RunStatus.OK, summary, data)
+
+
+def _close(run_id: int, status: RunStatus, **outcome) -> None:
+    """Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's:
+    the row then stays `running` until the next start's sweep closes it as interrupted."""
+    try:
+        jobruns.finish(run_id, status, **outcome)
+    except Exception:
+        log.exception("run %s: its outcome (%s) could not be recorded", run_id, status)
 
 
 def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]:
@@ -164,16 +189,15 @@ def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]
     summary = _summarize(result)
     data = _result_data(result)
     log.info("job %s -> %s", name, result)
-    jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data)
+    _close(run_id, RunStatus.OK, summary=summary, result=data)
     return summary, data
 
 
-def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]:
-    """Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as
-    a result does, so the client says it in the reader's language; the error stays the English line."""
+def _failure(exc: Exception) -> tuple[str, dict | None]:
+    """A raised run's English error line, and the failure the job named (progress.JobFailed) as
+    data, as a result is, so the client says it in the reader's language."""
     err = str(exc) or exc.__class__.__name__
     failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
-    jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure)
     return err, failure
 
 
@@ -201,10 +225,6 @@ def _summarize(result) -> str | None:
     return str(result)
 
 
-def _iso(ts: datetime | None) -> str | None:
-    return ts.astimezone(timezone.utc).isoformat() if ts else None
-
-
 def scheduler_snapshot(db) -> dict:
     """What the scheduler is doing, for the admin dashboard: each job's live state from this process
     and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how
@@ -223,7 +243,7 @@ def scheduler_snapshot(db) -> dict:
                 live_intervals[job.id] = round(iv.total_seconds() / 60)
     with _activity_lock:
         acts = {k: dict(v) for k, v in _activity.items()}
-    recent = jobruns.recent_runs(db, RECENT_RUNS)
+    recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
     jobs = []
     for job_id, default_interval in JOB_INTERVALS.items():
         a = acts.get(job_id, {})
@@ -239,17 +259,19 @@ def scheduler_snapshot(db) -> dict:
         )
     # Work a user starts on one target: no interval, no "run now" (it needs its target), but its
     # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
-    # target's own (sync/runner), so "running" is what its newest row says.
+    # target's own (sync/runner), so "running" is what its rows say: two channels can backfill at
+    # once, so a run still going counts even when a newer one already ended, and the row names it.
     backfills = recent.get(CHANNEL_BACKFILL, [])
-    newest = backfills[0] if backfills else None
+    going = jobruns.running_one(db, CHANNEL_BACKFILL)
+    shown = going or (backfills[0] if backfills else None)
     ad_hoc = [
         {
             "id": CHANNEL_BACKFILL,
-            "running": newest is not None and newest.status == RunStatus.RUNNING,
-            "target": newest.target if newest else None,
+            "running": going is not None,
+            "target": shown.target if shown else None,
             "target_title": (
-                db.scalar(select(Channel.title).where(Channel.id == newest.target))
-                if newest is not None and newest.target
+                db.scalar(select(Channel.title).where(Channel.id == shown.target))
+                if shown is not None and shown.target
                 else None
             ),
             **_last_runs(backfills),
@@ -275,8 +297,8 @@ def _last_runs(runs: list) -> dict:
     ended = run if run is not None and run.status != RunStatus.RUNNING else None
     return {
         "status": ended.status if ended else None,
-        "last_started": _iso(run.started_at) if run else None,
-        "last_finished": _iso(run.finished_at) if run else None,
+        "last_started": iso(run.started_at) if run else None,
+        "last_finished": iso(run.finished_at) if run else None,
         "last_result": ended.summary if ended else None,
         # The outcome as data: the result, or on a failed run the failure the job named.
         "result": _outcome_data(ended),
@@ -293,8 +315,8 @@ def _run_brief(run) -> dict:
         "id": run.id,
         "status": run.status,
         "trigger": run.trigger,
-        "started_at": _iso(run.started_at),
-        "finished_at": _iso(run.finished_at),
+        "started_at": iso(run.started_at),
+        "finished_at": iso(run.finished_at),
     }
 
 
@@ -575,11 +597,15 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non
         log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
         return
     with logs.bind(run_id=run_id):
+        # As in _run: only the work is inside the try.
         try:
-            _finish_ok(job, run_id, fn())
+            result = fn()
         except Exception as exc:
             log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
-            _finish_error(run_id, exc)
+            err, failure = _failure(exc)
+            _close(run_id, RunStatus.ERROR, error=err, failure=failure)
+            return
+        _finish_ok(job, run_id, result)
 
 
 # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
@@ -590,7 +616,8 @@ _POOL_SIZE = len(JOB_FUNCS) + 8
 def start_scheduler() -> None:
     """Start this process's scheduler. It always starts, so "run now" works on every install; only
     with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can
-    run, every run a previous process left open is closed as interrupted."""
+    run, every run a previous process left open is closed as interrupted — one API process serves a
+    database (app/jobruns.py), so an open run here is a dead process's."""
     global _scheduler
     if _scheduler is not None:
         return

backend/app/sysconfig.py

@@ -29,7 +29,9 @@ _coerce_warned: set[tuple[str, str]] = set()
 
 
 # When a changed value takes effect. "live": the next read obeys it — every consumer reads the key
-# where it uses it, never once at import or boot (test-held). "restart": the process has to start
+# where it uses it, never once at import (test-held: a read outside a function fails). A value read
+# once at startup is live only when its writer also applies it (the job intervals reschedule, the log
+# level is set as it is saved). "restart": the process has to start
 # again. No key needs a restart today, and the Configuration page has no way to say so yet, so a
 # "restart" key is refused until the page can (test-held).
 Applies = Literal["live", "restart"]

backend/tests/test_activity_reads.py

@@ -7,7 +7,10 @@ with rows that share a timestamp — the case an offset or a timestamp-only curs
 
 from datetime import datetime, timedelta, timezone
 
+from app.auth import admin_user
+from app.main import app
 from app.models import AuditLog, Channel, JobRun, LogEvent, QuotaEvent
+from tests.test_body_limit import _route_objects
 
 T0 = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc)
 
@@ -88,6 +91,32 @@ def test_an_admin_action_keeps_the_request_it_was_made_in(db, make_user, as_user
     assert row["request_id"] == r.headers["x-request-id"]
 
 
+def test_an_admin_action_says_how_many_of_its_requests_lines_are_still_kept(db, make_user, as_user):
+    """A request's lines are kept days, its audit row months: a link to lines gone would open an
+    empty list, so the row says how many are left, as a run does."""
+    admin = make_user(role="admin")
+    kept = _audit(db, "config_set", actor=admin.id, request_id="req-kept-0001")
+    gone = _audit(db, "config_set", actor=admin.id, request_id="req-gone-0001")
+    db.add_all(
+        [
+            LogEvent(
+                ts=T0,
+                level="INFO",
+                sev=20,
+                source="api",
+                logger="t",
+                msg="m",
+                request_id="req-kept-0001",
+            )
+            for _ in range(3)
+        ]
+    )
+    db.commit()
+    items = as_user(admin).get("/api/admin/audit").json()["items"]
+    lines = {r["id"]: r["log_lines"] for r in items}
+    assert (lines[kept], lines[gone]) == (3, 0)
+
+
 def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, make_user, as_user):
     admin = make_user(role="admin")
     db.add(Channel(id="UCchan", title="Aileen Wang"))
@@ -146,6 +175,8 @@ def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_
 
     def totals(**params) -> dict:
         body = client.get("/api/quota/admin", params=params).json()
+        # The chart is the same filter's: its days add up to the table's total.
+        assert sum(d["total"] for d in body["daily"]) == sum(r["total"] for r in body["rows"])
         return {r["user_id"]: r["total"] for r in body["rows"]}
 
     assert totals() == {admin.id: 150, None: 7}  # the last 30 days by default
@@ -153,3 +184,22 @@ def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_
     assert totals(until=(now - timedelta(days=1)).isoformat()) == {admin.id: 50}
     assert totals(user="none") == {None: 7}
     assert totals(user=str(admin.id)) == {admin.id: 150}
+
+
+def test_every_admin_read_and_write_is_admin_only():
+    """The route table, not a list kept by hand: every /api/admin route and the quota breakdown
+    resolve `admin_user`, so a dropped dependency cannot hand an account's runs, errors and e-mail
+    addresses to any signed-in user."""
+    routes = _route_objects(app)
+    admin = {k: r for k, r in routes.items() if k[1].startswith(("/api/admin", "/api/quota/admin"))}
+    assert ("GET", "/api/admin/runs") in admin and ("GET", "/api/quota/admin") in admin
+    open_ = []
+    for key, route in admin.items():
+        stack, calls = [route.dependant], set()
+        while stack:
+            d = stack.pop()
+            calls.add(d.call)
+            stack.extend(d.dependencies)
+        if admin_user not in calls:
+            open_.append(key)
+    assert open_ == []

backend/tests/test_audit_count_rows.py

@@ -1,6 +1,6 @@
 """An admin action that counts something keeps the count as data in its audit row's `after`, under
 the key the audit log names in the reader's language (`scheduler.result.<key>`, frontend
-auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word.
+lib/jobResult.ts, called from lib/auditText.ts) — not only in the English `summary`. A renamed key would fall back to the raw word.
 DB lane (Postgres).
 """
 

backend/tests/test_channel_backfill.py

@@ -136,6 +136,19 @@ def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel(
     assert calls == [(ch.id, me.id)]
 
 
+def test_turning_full_history_on_for_a_finished_channel_records_no_run(
+    db, monkeypatch, override_dep
+):
+    """Its whole history is in already: a run would only say "nothing to do", and switching the flag
+    off and on again would otherwise write one such row each time, kept for months."""
+    ch, me = _setup_channel_and_sub(db, email="done@example.com")
+    ch.backfill_done = True
+    db.commit()
+    resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
+    assert resp.status_code == 200, resp.text
+    assert (submitted, calls) == ([], [])
+
+
 class TestImmediateBackfillGuard:
     """backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per-
     channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a

backend/tests/test_migration_0109.py

@@ -96,3 +96,36 @@ def test_the_audit_logs_runs_move_into_the_run_table_and_back(db, make_user):
         ),
         (uid, "backfill", {"status": "skipped", "summary": "sync paused"}),
     ]
+
+
+def test_a_down_and_up_round_trip_keeps_every_run(db, make_user):
+    """A rollback then a redeploy: an interrupted run, a run caught mid-way and a user's run of one
+    channel come back as runs — not stranded in the audit log under an action no code knows."""
+    uid = make_user().id
+    with engine.begin() as conn:
+        conn.execute(
+            text(
+                "INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at,"
+                " finished_at) VALUES"
+                " ('backfill', 'scheduled', NULL, NULL, 'interrupted', now(), now()),"
+                " ('maintenance', 'manual', :u, NULL, 'running', now(), NULL),"
+                " ('channel_backfill', 'ad_hoc', :u, 'UC_x', 'ok', now(), now())"
+            ),
+            {"u": uid},
+        )
+    cfg = Config("alembic.ini")
+    command.downgrade(cfg, "0108_folder_links")
+    command.upgrade(cfg, "head")
+    with engine.connect() as conn:
+        runs = conn.execute(
+            text("SELECT job, trigger, actor_id, target, status FROM job_runs ORDER BY id")
+        ).all()
+        stranded = conn.execute(
+            text("SELECT count(*) FROM audit_log WHERE action = 'job_run'")
+        ).scalar()
+    assert [tuple(r) for r in runs] == [
+        ("backfill", "scheduled", None, None, "interrupted"),
+        ("maintenance", "manual", uid, None, "interrupted"),  # the rollback stopped it
+        ("channel_backfill", "ad_hoc", uid, "UC_x", "ok"),
+    ]
+    assert stranded == 0

backend/tests/test_runstatus.py

@@ -0,0 +1,27 @@
+"""The run words and the migrated database, DB-lane: the `job_runs` CHECKs are frozen in a migration,
+and the model's are built from the enums, so nothing else would notice the two drifting apart until a
+write of the new word failed in production."""
+
+import re
+
+from sqlalchemy import text
+
+from app.runstatus import RunStatus, RunTrigger
+
+
+def _words(definition: str) -> set[str]:
+    return set(re.findall(r"'([a-z_]+)'", definition))
+
+
+def test_the_databases_checks_allow_exactly_the_run_words(db):
+    defs = dict(
+        db.execute(
+            text(
+                "SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint"
+                " WHERE conname LIKE 'ck_job_runs_%'"
+            )
+        ).all()
+    )
+    assert _words(defs["ck_job_runs_status"]) == set(RunStatus)
+    assert _words(defs["ck_job_runs_trigger"]) == set(RunTrigger)
+    assert _words(defs["ck_job_runs_finished"]) == {RunStatus.RUNNING}

backend/tests/test_scheduler.py

@@ -14,6 +14,7 @@ from sqlalchemy import select
 
 from app import jobruns, logs, logsink, quota, scheduler, state, sysconfig
 from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent
+from app.routes import scheduler as scheduler_routes
 from app.runstatus import RunStatus, RunTrigger
 
 _JOB = "unit_test_claim"  # not a real job id — exercises the primitive without touching live jobs
@@ -256,6 +257,9 @@ def test_retention_reaps_finished_runs_only(db):
         ]
     )
     db.commit()
+    reaped = next(r for r in _runs(db) if r.status == "ok" and r.started_at < old + timedelta(1))
+    with logs.bind(run_id=reaped.id):
+        quota.record_usage(7)
     assert jobruns.gc(db, 0) == 0  # 0 = keep forever
     assert jobruns.gc(db, 30) == 1
     db.commit()
@@ -263,6 +267,8 @@ def test_retention_reaps_finished_runs_only(db):
         ("backfill", "running"),
         ("rss_poll", "ok"),
     ]
+    # Its spend stays in the budget's history (kept on its own retention), no longer anyone's run.
+    assert db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() == [(7, None)]
 
 
 # ---- "run now" through the scheduler --------------------------------------------------------------
@@ -413,6 +419,94 @@ def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch):
     assert run.status == "interrupted"
 
 
+def test_an_interrupted_run_keeps_what_it_spent(db):
+    """Its spend is in the quota history, tagged with its id: the sweep sums it as `finish` would."""
+    run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None)
+    with logs.bind(run_id=run_id):
+        quota.record_usage(40)
+        quota.record_usage(2)
+    assert jobruns.interrupt_leftovers() == 1
+    (run,) = _runs(db, "backfill")
+    assert (run.status, run.quota_units) == ("interrupted", 42)
+
+
+def _flaky_finish(monkeypatch) -> list:
+    """`jobruns.finish` whose first write fails (the database blinked); the statuses it was asked for."""
+    calls: list = []
+    real = jobruns.finish
+
+    def finish(run_id, status, **outcome):
+        calls.append(status)
+        if len(calls) == 1:
+            raise RuntimeError("db blip")
+        real(run_id, status, **outcome)
+
+    monkeypatch.setattr(jobruns, "finish", finish)
+    return calls
+
+
+def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_user, monkeypatch):
+    """The job succeeded; only its bookkeeping did not. Writing ERROR then would say the work failed."""
+    calls = _flaky_finish(monkeypatch)
+    scheduler._job("autotag", lambda _db: {"tagged": 1})
+    assert calls == ["ok"]
+    assert not scheduler._is_running("autotag")
+    (run,) = _runs(db, "autotag")
+    assert run.status == "running"  # left for the next start's sweep, not rewritten as a failure
+
+    user = make_user("blip@example.com")
+    calls.clear()
+    scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip")
+    assert calls == ["ok"]
+
+
+def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch):
+    def broken(*_a, **_k):
+        raise RuntimeError("db down")
+
+    monkeypatch.setattr(jobruns, "start", broken)
+    scheduler._job("autotag", lambda _db: pytest.fail("no row, no run"))
+    assert not scheduler._is_running("autotag")
+    monkeypatch.undo()
+    scheduler._job("autotag", lambda _db: {"tagged": 1})
+    assert [r.status for r in _runs(db, "autotag")] == ["ok"]
+
+
+def test_the_page_says_where_the_interval_jobs_do_not_run(db, live_scheduler, monkeypatch):
+    """The scheduler starts on every install now; only SCHEDULER_ENABLED says the jobs run here."""
+    assert scheduler.scheduler_snapshot(db)["running_here"] is False
+    monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True)
+    assert scheduler.scheduler_snapshot(db)["running_here"] is True
+
+
+def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_user):
+    """Two channels can backfill at once; a later run that ended at once must not hide the first."""
+    user = make_user("two@example.com")
+    db.add(Channel(id="UC_long", title="Long one"))
+    db.commit()
+    jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_long")
+    scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short")
+    (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
+    assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one")
+
+
+def test_an_interval_change_lands_with_its_audit_row_or_not_at_all(
+    db, make_user, as_user, live_scheduler, monkeypatch
+):
+    live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill")
+    admin = as_user(make_user(role="admin"))
+
+    def broken(*_a, **_k):
+        raise RuntimeError("audit write failed")
+
+    monkeypatch.setattr(scheduler_routes.audit, "record", broken)
+    with pytest.raises(RuntimeError):  # the handler raises: there is no response to check
+        _ = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45})
+    db.expire_all()
+    assert sysconfig.get_int(db, "job_interval_backfill") == scheduler.JOB_INTERVALS["backfill"]
+    assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=10)
+
+
 def test_every_job_has_an_interval_and_every_interval_a_job():
     assert set(scheduler.JOB_FUNCS) == set(scheduler.JOB_INTERVALS)