peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(activity): runs keep their true outcome and spend, the page reaches every kept row, a closing dialog leaves the server log alone

Browse Source 
Backend
- A run's outcome write sits outside the job's own try: a database blip after a run that
  succeeded no longer records it as failed (the row waits for the next start's sweep).
- An interrupted run keeps what it spent (the sweep sums its quota rows, as finish does).
- The Scheduler's ad-hoc row says running while any backfill runs, not only the newest.
- recent_runs reads each job's last runs off its index instead of ranking the whole table.
- An interval change is stored with its audit row in one transaction, then rescheduled.
- One API instance per database is the stated contract (.env.example, jobruns).
- Turning full history on for a channel already complete records no run.
- Migration 0109 round-trips interrupted and ad-hoc runs; the run words' DB CHECKs are
  test-held to the enums; the account filter is one helper for the four activity reads.
- An audit row says how many of its request's log lines are still kept.

Frontend
- An "All" period reaches every row still kept (runs and admin actions outlive 90 days).
- Back/Forward is followed in place by the server log: a closing dialog no longer turns
  Live off; a pivot still opens a fresh log.
- Runs moved over from the audit log read their one-liner in the reader's language.
- Filter choices or accounts that failed to load say so beside the filters, with a retry.
- An old server-log link keeps its account, text and range; a remembered retired page
  opens its tab.
- A request whose lines are gone says so instead of linking to an empty list.
- The log lines and the history rows share one row frame, and the server log one list
  frame with the other tabs; an empty list is announced; each day's quota is readable.
This commit is contained in:
peter
2026-10-18 15:38:32 +02:00
parent 3021be5166
commit fad656988a
44 changed files with 1101 additions and 426 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/scheduler.py
+70 -43
View File
@@ -3,7 +3,7 @@ recent-first / deep backfill. Runs inside the API process (single worker)."""
import contextvars
import threading
from datetime import datetime, timezone
from datetime import timezone
from typing import Callable
from apscheduler.executors.pool import ThreadPoolExecutor
@@ -27,6 +27,7 @@ from app.sync.autotag import run_autotag_all
from app.sync.explore import purge_ephemeral
from app.sync.maintenance import run_maintenance
from app.sync.playlists import sync_all_playlists
from app.utils import iso
from app.sync.runner import (
run_deep_backfill,
run_enrich,
@@ -67,18 +68,21 @@ def load_intervals(db) -> dict[str, int]:
return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS}
def apply_interval(db, job_id: str, minutes: int) -> int:
"""Persist a job's interval override and reschedule the live job immediately (if the
scheduler runs in this process). Returns the applied value."""
def stage_interval(db, job_id: str, minutes: int) -> int:
"""Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it
with its audit row, then calls `reschedule` — the live job follows only what was stored."""
if job_id not in JOB_INTERVALS:
raise KeyError(job_id)
minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes)))
sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes)
db.commit()
return minutes
def reschedule(job_id: str, minutes: int) -> None:
"""Move the live interval job to `minutes` (if this process registered it)."""
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
log.info("rescheduled job %s to every %s min", job_id, minutes)
return minutes
def _record(name: str, **fields) -> None:
@@ -87,7 +91,12 @@ def _record(name: str, **fields) -> None:
def _notify_done(
db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None
db,
actor_id: int,
job_id: str,
status: RunStatus,
summary: str | None,
result: dict | None = None,
) -> None:
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
effort — a notification failure must never mask the job's own outcome."""
@@ -139,24 +148,40 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None:
def sink(current, total, phase):
_record(name, progress={"current": current, "total": total, "phase": phase})
# Only the work is inside the try: writing its outcome is not the job, and a write that fails
# must not turn a run that succeeded into a failed one (see _close).
result = None
try:
if is_sync_paused(db):
log.info("job %s skipped (sync paused)", name)
jobruns.finish(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
if actor_id is not None:
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
return
with progress.bind(sink):
result = fn(db)
summary, data = _finish_ok(name, run_id, result)
if actor_id is not None:
_notify_done(db, actor_id, name, "ok", summary, data)
paused = is_sync_paused(db)
if not paused:
with progress.bind(sink):
result = fn(db)
except Exception as exc:
db.rollback()
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
err, failure = _finish_error(run_id, exc)
err, failure = _failure(exc)
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
if actor_id is not None:
_notify_done(db, actor_id, name, "error", err, failure)
_notify_done(db, actor_id, name, RunStatus.ERROR, err, failure)
return
if paused:
log.info("job %s skipped (sync paused)", name)
_close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
if actor_id is not None:
_notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED)
return
summary, data = _finish_ok(name, run_id, result)
if actor_id is not None:
_notify_done(db, actor_id, name, RunStatus.OK, summary, data)
def _close(run_id: int, status: RunStatus, **outcome) -> None:
"""Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's:
the row then stays `running` until the next start's sweep closes it as interrupted."""
try:
jobruns.finish(run_id, status, **outcome)
except Exception:
log.exception("run %s: its outcome (%s) could not be recorded", run_id, status)
def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]:
@@ -164,16 +189,15 @@ def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]
summary = _summarize(result)
data = _result_data(result)
log.info("job %s -> %s", name, result)
jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data)
_close(run_id, RunStatus.OK, summary=summary, result=data)
return summary, data
def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]:
"""Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as
a result does, so the client says it in the reader's language; the error stays the English line."""
def _failure(exc: Exception) -> tuple[str, dict | None]:
"""A raised run's English error line, and the failure the job named (progress.JobFailed) as
data, as a result is, so the client says it in the reader's language."""
err = str(exc) or exc.__class__.__name__
failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure)
return err, failure
@@ -201,10 +225,6 @@ def _summarize(result) -> str | None:
return str(result)
def _iso(ts: datetime | None) -> str | None:
return ts.astimezone(timezone.utc).isoformat() if ts else None
def scheduler_snapshot(db) -> dict:
"""What the scheduler is doing, for the admin dashboard: each job's live state from this process
and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how
@@ -223,7 +243,7 @@ def scheduler_snapshot(db) -> dict:
live_intervals[job.id] = round(iv.total_seconds() / 60)
with _activity_lock:
acts = {k: dict(v) for k, v in _activity.items()}
recent = jobruns.recent_runs(db, RECENT_RUNS)
recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
jobs = []
for job_id, default_interval in JOB_INTERVALS.items():
a = acts.get(job_id, {})
@@ -239,17 +259,19 @@ def scheduler_snapshot(db) -> dict:
)
# Work a user starts on one target: no interval, no "run now" (it needs its target), but its
# runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
# target's own (sync/runner), so "running" is what its newest row says.
# target's own (sync/runner), so "running" is what its rows say: two channels can backfill at
# once, so a run still going counts even when a newer one already ended, and the row names it.
backfills = recent.get(CHANNEL_BACKFILL, [])
newest = backfills[0] if backfills else None
going = jobruns.running_one(db, CHANNEL_BACKFILL)
shown = going or (backfills[0] if backfills else None)
ad_hoc = [
{
"id": CHANNEL_BACKFILL,
"running": newest is not None and newest.status == RunStatus.RUNNING,
"target": newest.target if newest else None,
"running": going is not None,
"target": shown.target if shown else None,
"target_title": (
db.scalar(select(Channel.title).where(Channel.id == newest.target))
if newest is not None and newest.target
db.scalar(select(Channel.title).where(Channel.id == shown.target))
if shown is not None and shown.target
else None
),
**_last_runs(backfills),
@@ -275,8 +297,8 @@ def _last_runs(runs: list) -> dict:
ended = run if run is not None and run.status != RunStatus.RUNNING else None
return {
"status": ended.status if ended else None,
"last_started": _iso(run.started_at) if run else None,
"last_finished": _iso(run.finished_at) if run else None,
"last_started": iso(run.started_at) if run else None,
"last_finished": iso(run.finished_at) if run else None,
"last_result": ended.summary if ended else None,
# The outcome as data: the result, or on a failed run the failure the job named.
"result": _outcome_data(ended),
@@ -293,8 +315,8 @@ def _run_brief(run) -> dict:
"id": run.id,
"status": run.status,
"trigger": run.trigger,
"started_at": _iso(run.started_at),
"finished_at": _iso(run.finished_at),
"started_at": iso(run.started_at),
"finished_at": iso(run.finished_at),
}
@@ -575,11 +597,15 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non
log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
return
with logs.bind(run_id=run_id):
# As in _run: only the work is inside the try.
try:
_finish_ok(job, run_id, fn())
result = fn()
except Exception as exc:
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
_finish_error(run_id, exc)
err, failure = _failure(exc)
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
return
_finish_ok(job, run_id, result)
# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
@@ -590,7 +616,8 @@ _POOL_SIZE = len(JOB_FUNCS) + 8
def start_scheduler() -> None:
"""Start this process's scheduler. It always starts, so "run now" works on every install; only
with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can
run, every run a previous process left open is closed as interrupted."""
run, every run a previous process left open is closed as interrupted — one API process serves a
database (app/jobruns.py), so an open run here is a dead process's."""
global _scheduler
if _scheduler is not None:
return