feat(scheduler): a channel's "get full history" backfill is a recorded run, and a row knows its last three runs

The one-channel deep backfill ran as a FastAPI BackgroundTask: what it fetched, what it spent and
how it failed reached only the log. It is now a one-shot run on the scheduler (`channel_backfill`,
trigger ad_hoc, actor = the user, target = the channel) with its quota tagged to the run. Its
per-channel exclusion with the scheduled backfill and the two-run cap are unchanged, and a run
they turn away says why in its result instead of returning nothing. A failure is no longer
swallowed: the run records it.

The scheduler snapshot carries each job's last three runs (one windowed query), so the page can
show an interrupted or skipped run instead of the latest outcome alone.

backend/app/scheduler.py

@@ -174,26 +174,35 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None:
             return
         with progress.bind(sink):
             result = fn(db)
-        summary = _summarize(result)
-        data = _result_data(result)
-        log.info("job %s -> %s", name, result)
-        jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data)
+        summary, data = _finish_ok(name, run_id, result)
         if actor_id is not None:
             _notify_done(db, actor_id, name, "ok", summary, data)
     except Exception as exc:
         db.rollback()
         log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
-        err = str(exc) or exc.__class__.__name__
-        # A failure the job named (progress.JobFailed) rides along as data, as a result does, so the
-        # client says it in the reader's language; `err` stays the English line.
-        failure = (
-            {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
-        )
-        jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure)
+        err, failure = _finish_error(run_id, exc)
         if actor_id is not None:
             _notify_done(db, actor_id, name, "error", err, failure)
 
 
+def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]:
+    """Close a run that returned: its result on the row, as data and as the English one-liner."""
+    summary = _summarize(result)
+    data = _result_data(result)
+    log.info("job %s -> %s", name, result)
+    jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data)
+    return summary, data
+
+
+def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]:
+    """Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as
+    a result does, so the client says it in the reader's language; the error stays the English line."""
+    err = str(exc) or exc.__class__.__name__
+    failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
+    jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure)
+    return err, failure
+
+
 # A skipped run's result, in the same shape as a job's own `{"skipped": …}`.
 _PAUSED = {"skipped": "sync paused"}
 
@@ -240,11 +249,12 @@ def scheduler_snapshot(db) -> dict:
                 live_intervals[job.id] = round(iv.total_seconds() / 60)
     with _activity_lock:
         acts = {k: dict(v) for k, v in _activity.items()}
-    last = jobruns.last_runs(db)
+    recent = jobruns.recent_runs(db, RECENT_RUNS)
     jobs = []
     for job_id, default_interval in JOB_INTERVALS.items():
         a = acts.get(job_id, {})
-        run = last.get(job_id)
+        runs = recent.get(job_id, [])
+        run = runs[0] if runs else None
         # A run still going has no outcome yet: only an ended run says how it went.
         ended = run if run is not None and run.status != RunStatus.RUNNING else None
         jobs.append(
@@ -264,11 +274,27 @@ def scheduler_snapshot(db) -> dict:
                 "last_run_id": run.id if run else None,
                 "quota_units": ended.quota_units if ended else None,
                 "progress": a.get("progress"),
+                # The last few runs, newest first, for the row's history dots.
+                "recent": [_run_brief(r) for r in runs],
             }
         )
     return {"running_here": running_here, "enabled": settings.scheduler_enabled, "jobs": jobs}
 
 
+# How many past runs a Scheduler row shows as dots; the full history is the run list.
+RECENT_RUNS = 3
+
+
+def _run_brief(run) -> dict:
+    return {
+        "id": run.id,
+        "status": run.status,
+        "trigger": run.trigger,
+        "started_at": _iso(run.started_at),
+        "finished_at": _iso(run.finished_at),
+    }
+
+
 def _outcome_data(run) -> dict | None:
     if run is None:
         return None
@@ -509,6 +535,48 @@ def trigger_all(actor_id: int | None = None) -> list[str]:
     return started
 
 
+# Work a user starts on one target outside the job list — today the "get full history" backfill of
+# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and
+# spent is in `job_runs` like any job's, not a background task only the log ever heard of.
+CHANNEL_BACKFILL = "channel_backfill"
+
+
+def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None:
+    """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its
+    own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so
+    a second request before the first has even started is that same run."""
+    sched = _scheduler
+    if sched is None:
+        # Only between shutdown and exit: start_scheduler runs at every API startup.
+        raise RuntimeError("the scheduler is not running")
+    try:
+        sched.add_job(
+            _ad_hoc,
+            "date",
+            args=(job, fn, actor_id, target),
+            id=f"{job}@{target}",
+            misfire_grace_time=None,
+        )
+    # silent-ok: not a failure — the run for this target is already queued, and it does this work.
+    except ConflictingIdError:
+        log.info("ad-hoc %s for %s already queued", job, target)
+
+
+def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None:
+    try:
+        run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target)
+    except Exception:
+        # No row to tell it in: the database is what failed. The log is the record.
+        log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
+        return
+    with logs.bind(run_id=run_id):
+        try:
+            _finish_ok(job, run_id, fn())
+        except Exception as exc:
+            log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
+            _finish_error(run_id, exc)
+
+
 # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
 # 10 would queue a scheduled fire behind long manual runs.
 _POOL_SIZE = len(JOB_FUNCS) + 8