fix(activity): runs keep their true outcome and spend, the page reaches every kept row, a closing dialog leaves the server log alone

Backend
- A run's outcome write sits outside the job's own try: a database blip after a run that
  succeeded no longer records it as failed (the row waits for the next start's sweep).
- An interrupted run keeps what it spent (the sweep sums its quota rows, as finish does).
- The Scheduler's ad-hoc row says running while any backfill runs, not only the newest.
- recent_runs reads each job's last runs off its index instead of ranking the whole table.
- An interval change is stored with its audit row in one transaction, then rescheduled.
- One API instance per database is the stated contract (.env.example, jobruns).
- Turning full history on for a channel already complete records no run.
- Migration 0109 round-trips interrupted and ad-hoc runs; the run words' DB CHECKs are
  test-held to the enums; the account filter is one helper for the four activity reads.
- An audit row says how many of its request's log lines are still kept.

Frontend
- An "All" period reaches every row still kept (runs and admin actions outlive 90 days).
- Back/Forward is followed in place by the server log: a closing dialog no longer turns
  Live off; a pivot still opens a fresh log.
- Runs moved over from the audit log read their one-liner in the reader's language.
- Filter choices or accounts that failed to load say so beside the filters, with a retry.
- An old server-log link keeps its account, text and range; a remembered retired page
  opens its tab.
- A request whose lines are gone says so instead of linking to an empty list.
- The log lines and the history rows share one row frame, and the server log one list
  frame with the other tabs; an empty list is announced; each day's quota is readable.

backend/tests/test_scheduler.py

@@ -14,6 +14,7 @@ from sqlalchemy import select
 
 from app import jobruns, logs, logsink, quota, scheduler, state, sysconfig
 from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent
+from app.routes import scheduler as scheduler_routes
 from app.runstatus import RunStatus, RunTrigger
 
 _JOB = "unit_test_claim"  # not a real job id — exercises the primitive without touching live jobs
@@ -256,6 +257,9 @@ def test_retention_reaps_finished_runs_only(db):
         ]
     )
     db.commit()
+    reaped = next(r for r in _runs(db) if r.status == "ok" and r.started_at < old + timedelta(1))
+    with logs.bind(run_id=reaped.id):
+        quota.record_usage(7)
     assert jobruns.gc(db, 0) == 0  # 0 = keep forever
     assert jobruns.gc(db, 30) == 1
     db.commit()
@@ -263,6 +267,8 @@ def test_retention_reaps_finished_runs_only(db):
         ("backfill", "running"),
         ("rss_poll", "ok"),
     ]
+    # Its spend stays in the budget's history (kept on its own retention), no longer anyone's run.
+    assert db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() == [(7, None)]
 
 
 # ---- "run now" through the scheduler --------------------------------------------------------------
@@ -413,6 +419,94 @@ def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch):
     assert run.status == "interrupted"
 
 
+def test_an_interrupted_run_keeps_what_it_spent(db):
+    """Its spend is in the quota history, tagged with its id: the sweep sums it as `finish` would."""
+    run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None)
+    with logs.bind(run_id=run_id):
+        quota.record_usage(40)
+        quota.record_usage(2)
+    assert jobruns.interrupt_leftovers() == 1
+    (run,) = _runs(db, "backfill")
+    assert (run.status, run.quota_units) == ("interrupted", 42)
+
+
+def _flaky_finish(monkeypatch) -> list:
+    """`jobruns.finish` whose first write fails (the database blinked); the statuses it was asked for."""
+    calls: list = []
+    real = jobruns.finish
+
+    def finish(run_id, status, **outcome):
+        calls.append(status)
+        if len(calls) == 1:
+            raise RuntimeError("db blip")
+        real(run_id, status, **outcome)
+
+    monkeypatch.setattr(jobruns, "finish", finish)
+    return calls
+
+
+def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_user, monkeypatch):
+    """The job succeeded; only its bookkeeping did not. Writing ERROR then would say the work failed."""
+    calls = _flaky_finish(monkeypatch)
+    scheduler._job("autotag", lambda _db: {"tagged": 1})
+    assert calls == ["ok"]
+    assert not scheduler._is_running("autotag")
+    (run,) = _runs(db, "autotag")
+    assert run.status == "running"  # left for the next start's sweep, not rewritten as a failure
+
+    user = make_user("blip@example.com")
+    calls.clear()
+    scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip")
+    assert calls == ["ok"]
+
+
+def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch):
+    def broken(*_a, **_k):
+        raise RuntimeError("db down")
+
+    monkeypatch.setattr(jobruns, "start", broken)
+    scheduler._job("autotag", lambda _db: pytest.fail("no row, no run"))
+    assert not scheduler._is_running("autotag")
+    monkeypatch.undo()
+    scheduler._job("autotag", lambda _db: {"tagged": 1})
+    assert [r.status for r in _runs(db, "autotag")] == ["ok"]
+
+
+def test_the_page_says_where_the_interval_jobs_do_not_run(db, live_scheduler, monkeypatch):
+    """The scheduler starts on every install now; only SCHEDULER_ENABLED says the jobs run here."""
+    assert scheduler.scheduler_snapshot(db)["running_here"] is False
+    monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True)
+    assert scheduler.scheduler_snapshot(db)["running_here"] is True
+
+
+def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_user):
+    """Two channels can backfill at once; a later run that ended at once must not hide the first."""
+    user = make_user("two@example.com")
+    db.add(Channel(id="UC_long", title="Long one"))
+    db.commit()
+    jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_long")
+    scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short")
+    (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
+    assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one")
+
+
+def test_an_interval_change_lands_with_its_audit_row_or_not_at_all(
+    db, make_user, as_user, live_scheduler, monkeypatch
+):
+    live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill")
+    admin = as_user(make_user(role="admin"))
+
+    def broken(*_a, **_k):
+        raise RuntimeError("audit write failed")
+
+    monkeypatch.setattr(scheduler_routes.audit, "record", broken)
+    with pytest.raises(RuntimeError):  # the handler raises: there is no response to check
+        _ = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45})
+    db.expire_all()
+    assert sysconfig.get_int(db, "job_interval_backfill") == scheduler.JOB_INTERVALS["backfill"]
+    assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=10)
+
+
 def test_every_job_has_an_interval_and_every_interval_a_job():
     assert set(scheduler.JOB_FUNCS) == set(scheduler.JOB_INTERVALS)