fix(activity): runs keep their true outcome and spend, the page reaches every kept row, a closing dialog leaves the server log alone
Backend - A run's outcome write sits outside the job's own try: a database blip after a run that succeeded no longer records it as failed (the row waits for the next start's sweep). - An interrupted run keeps what it spent (the sweep sums its quota rows, as finish does). - The Scheduler's ad-hoc row says running while any backfill runs, not only the newest. - recent_runs reads each job's last runs off its index instead of ranking the whole table. - An interval change is stored with its audit row in one transaction, then rescheduled. - One API instance per database is the stated contract (.env.example, jobruns). - Turning full history on for a channel already complete records no run. - Migration 0109 round-trips interrupted and ad-hoc runs; the run words' DB CHECKs are test-held to the enums; the account filter is one helper for the four activity reads. - An audit row says how many of its request's log lines are still kept. Frontend - An "All" period reaches every row still kept (runs and admin actions outlive 90 days). - Back/Forward is followed in place by the server log: a closing dialog no longer turns Live off; a pivot still opens a fresh log. - Runs moved over from the audit log read their one-liner in the reader's language. - Filter choices or accounts that failed to load say so beside the filters, with a retry. - An old server-log link keeps its account, text and range; a remembered retired page opens its tab. - A request whose lines are gone says so instead of linking to an empty list. - The log lines and the history rows share one row frame, and the server log one list frame with the other tabs; an empty list is announced; each day's quota is readable.
This commit is contained in:
+42
-15
@@ -5,13 +5,14 @@ OWN short session, committed at once, for the reason `quota.record_usage` gives:
|
||||
may roll back (its `except: db.rollback()`), and the history of what happened must not go with it —
|
||||
nor may this bookkeeping commit the job's half-done session.
|
||||
|
||||
Only the API process runs jobs (the worker never starts the scheduler), so at its startup nothing can
|
||||
still be running: `interrupt_leftovers` closes what a stopped process left open.
|
||||
Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a
|
||||
database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its
|
||||
startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy import delete, func, select, union_all, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -70,30 +71,46 @@ def finish(
|
||||
|
||||
|
||||
def interrupt_leftovers() -> int:
|
||||
"""Every row a previous process left `running` → `interrupted`. Called once at startup, before
|
||||
the scheduler can open a new run. Returns how many."""
|
||||
"""Every row a previous process left `running` → `interrupted`, with what it spent until then
|
||||
(its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler
|
||||
can open a new run. Returns how many."""
|
||||
spent = (
|
||||
select(func.coalesce(func.sum(QuotaEvent.units), 0))
|
||||
.where(QuotaEvent.job_run_id == JobRun.id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
with SessionLocal() as s:
|
||||
res = s.execute(
|
||||
update(JobRun)
|
||||
.where(JobRun.status == RunStatus.RUNNING)
|
||||
.values(status=RunStatus.INTERRUPTED, finished_at=datetime.now(timezone.utc))
|
||||
.values(
|
||||
status=RunStatus.INTERRUPTED,
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
quota_units=spent,
|
||||
)
|
||||
)
|
||||
s.commit()
|
||||
return res.rowcount if isinstance(res, CursorResult) else 0
|
||||
|
||||
|
||||
def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]:
|
||||
"""Each job's last `per_job` runs, newest first, by job id — one query (a window per job)."""
|
||||
ranked = select(
|
||||
JobRun.id,
|
||||
func.row_number()
|
||||
.over(partition_by=JobRun.job, order_by=(JobRun.started_at.desc(), JobRun.id.desc()))
|
||||
.label("rank"),
|
||||
def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]:
|
||||
"""Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a
|
||||
short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window
|
||||
over the whole table grew with every run kept)."""
|
||||
if not jobs:
|
||||
return {}
|
||||
newest = union_all(
|
||||
*(
|
||||
select(JobRun.id)
|
||||
.where(JobRun.job == job)
|
||||
.order_by(JobRun.started_at.desc(), JobRun.id.desc())
|
||||
.limit(per_job)
|
||||
for job in jobs
|
||||
)
|
||||
).subquery()
|
||||
rows = db.execute(
|
||||
select(JobRun)
|
||||
.join(ranked, ranked.c.id == JobRun.id)
|
||||
.where(ranked.c.rank <= per_job)
|
||||
.where(JobRun.id.in_(select(newest.c.id)))
|
||||
.order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc())
|
||||
).scalars()
|
||||
out: dict[str, list[JobRun]] = {}
|
||||
@@ -102,6 +119,16 @@ def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]:
|
||||
return out
|
||||
|
||||
|
||||
def running_one(db: Session, job: str) -> JobRun | None:
|
||||
"""The newest run of `job` still going, however many runs ended after it started."""
|
||||
return db.scalars(
|
||||
select(JobRun)
|
||||
.where(JobRun.job == job, JobRun.status == RunStatus.RUNNING)
|
||||
.order_by(JobRun.started_at.desc(), JobRun.id.desc())
|
||||
.limit(1)
|
||||
).first()
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Delete finished runs older than `retention_days` (0 = keep forever). A run still going is
|
||||
never reaped. The caller commits."""
|
||||
|
||||
@@ -720,7 +720,7 @@ class JobRun(Base):
|
||||
CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"),
|
||||
# Only a run still going has no end.
|
||||
CheckConstraint(
|
||||
"(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished"
|
||||
f"(status = '{RunStatus.RUNNING}') = (finished_at IS NULL)", name="ck_job_runs_finished"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""The account filter the activity page's lists share (runs, admin actions, server log, quota): one
|
||||
filter bar drives them all, so the value it sends means the same on every tab.
|
||||
|
||||
The value is an account id, or `none` — the rows no account made (the background) — a value to filter
|
||||
on, not the absence of a filter.
|
||||
"""
|
||||
|
||||
from sqlalchemy import ColumnElement
|
||||
|
||||
NONE = "none"
|
||||
# The `user` query parameter's shape on every route that takes it.
|
||||
PATTERN = r"^(none|[0-9]{1,10})$"
|
||||
|
||||
|
||||
def condition(column, value: str) -> ColumnElement[bool]:
|
||||
"""`column` (an account id column) narrowed to `value`."""
|
||||
return column.is_(None) if value == NONE else column == int(value)
|
||||
@@ -25,6 +25,7 @@ from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.utils import like_escape
|
||||
|
||||
@@ -37,9 +38,9 @@ FACET_EVENTS_MAX = 100
|
||||
Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
|
||||
Facet = Literal["levels", "sources", "user", "event"]
|
||||
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
|
||||
# request) — a real value to filter on, not the absence of a filter.
|
||||
NONE = "none"
|
||||
# `none` selects the rows that have no event code, as the account filter's `none` (app/routes/_account)
|
||||
# selects the rows of no account — a real value to filter on, not the absence of a filter.
|
||||
NONE = account.NONE
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -67,11 +68,7 @@ class LogFilters:
|
||||
if self.sources and skip != "sources":
|
||||
out.append(LogEvent.source.in_(self.sources))
|
||||
if self.user is not None and skip != "user":
|
||||
out.append(
|
||||
LogEvent.user_id.is_(None)
|
||||
if self.user == NONE
|
||||
else LogEvent.user_id == int(self.user)
|
||||
)
|
||||
out.append(account.condition(LogEvent.user_id, self.user))
|
||||
if self.event is not None and skip != "event":
|
||||
out.append(
|
||||
LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
|
||||
@@ -103,7 +100,7 @@ class LogFilters:
|
||||
def log_filters(
|
||||
levels: list[Level] = Query(default=[]),
|
||||
sources: list[str] = Query(default=[], max_length=8),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
event: str | None = Query(
|
||||
default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
|
||||
),
|
||||
|
||||
+18
-12
@@ -15,17 +15,15 @@ from app import audit
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import AuditLog, User
|
||||
from app.models import AuditLog, LogEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.utils import iso, like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/audit", tags=["admin"])
|
||||
|
||||
# `none` selects the rows no account made (the background) — a value to filter on, not "any".
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _serialize(row: AuditLog) -> dict:
|
||||
def _serialize(row: AuditLog, lines: dict[str, int]) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"created_at": iso(row.created_at),
|
||||
@@ -38,6 +36,9 @@ def _serialize(row: AuditLog) -> dict:
|
||||
"before": row.before,
|
||||
"after": row.after,
|
||||
"request_id": row.request_id,
|
||||
# How many of that request's log lines are still kept: they go after days, the row after
|
||||
# months, and a link to none would open an empty list.
|
||||
"log_lines": lines.get(row.request_id, 0) if row.request_id else 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -56,11 +57,7 @@ class AuditFilters:
|
||||
if self.action is not None and skip != "action":
|
||||
out.append(AuditLog.action == self.action)
|
||||
if self.actor is not None:
|
||||
out.append(
|
||||
AuditLog.actor_id.is_(None)
|
||||
if self.actor == NONE
|
||||
else AuditLog.actor_id == int(self.actor)
|
||||
)
|
||||
out.append(account.condition(AuditLog.actor_id, self.actor))
|
||||
if self.since is not None:
|
||||
out.append(AuditLog.created_at >= utc(self.since))
|
||||
if self.until is not None:
|
||||
@@ -82,7 +79,7 @@ class AuditFilters:
|
||||
|
||||
def audit_filters(
|
||||
action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
@@ -109,8 +106,17 @@ def list_audit(
|
||||
limit,
|
||||
lambda r: r.created_at,
|
||||
)
|
||||
requests = {r.request_id for r in rows if r.request_id}
|
||||
lines: dict[str, int] = {}
|
||||
if requests:
|
||||
counted = db.execute(
|
||||
select(LogEvent.request_id, func.count())
|
||||
.where(LogEvent.request_id.in_(requests))
|
||||
.group_by(LogEvent.request_id)
|
||||
)
|
||||
lines = {rid: n for rid, n in counted.tuples() if rid is not None}
|
||||
return {
|
||||
"items": [_serialize(r) for r in rows],
|
||||
"items": [_serialize(r, lines) for r in rows],
|
||||
"accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
@@ -512,7 +512,9 @@ def update_channel(
|
||||
# so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks.
|
||||
# The request returns right away; the channel page's live poll shows the count climb. The
|
||||
# scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set).
|
||||
if deep_turned_on:
|
||||
# A channel whose history is all in gets no run: it would only say "nothing to do", one kept row
|
||||
# per flip of the flag.
|
||||
if deep_turned_on and not (channel is not None and channel.backfill_done):
|
||||
scheduler.run_ad_hoc(
|
||||
scheduler.CHANNEL_BACKFILL,
|
||||
partial(backfill_channel_now, channel_id, user.id),
|
||||
|
||||
@@ -14,6 +14,7 @@ from app import quota
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import QuotaEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import utc
|
||||
|
||||
router = APIRouter(prefix="/api/quota", tags=["quota"])
|
||||
@@ -48,7 +49,7 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
def admin_usage(
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -60,9 +61,7 @@ def admin_usage(
|
||||
if until is not None:
|
||||
window.append(QuotaEvent.created_at < utc(until))
|
||||
if user is not None:
|
||||
window.append(
|
||||
QuotaEvent.user_id.is_(None) if user == "none" else QuotaEvent.user_id == int(user)
|
||||
)
|
||||
window.append(account.condition(QuotaEvent.user_id, user))
|
||||
|
||||
totals = db.execute(
|
||||
select(QuotaEvent.user_id, func.sum(QuotaEvent.units))
|
||||
|
||||
@@ -19,14 +19,13 @@ from app import scheduler
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import Channel, JobRun, LogEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.runstatus import RunStatus
|
||||
from app.utils import iso, like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/runs", tags=["admin"])
|
||||
|
||||
# `none` selects the runs no account started (the scheduled ones).
|
||||
NONE = "none"
|
||||
Facet = Literal["job", "status"]
|
||||
|
||||
|
||||
@@ -68,11 +67,7 @@ class RunFilters:
|
||||
if self.status is not None and skip != "status":
|
||||
out.append(JobRun.status == self.status)
|
||||
if self.actor is not None:
|
||||
out.append(
|
||||
JobRun.actor_id.is_(None)
|
||||
if self.actor == NONE
|
||||
else JobRun.actor_id == int(self.actor)
|
||||
)
|
||||
out.append(account.condition(JobRun.actor_id, self.actor))
|
||||
if self.since is not None:
|
||||
out.append(JobRun.started_at >= utc(self.since))
|
||||
if self.until is not None:
|
||||
@@ -93,7 +88,7 @@ class RunFilters:
|
||||
def run_filters(
|
||||
job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"),
|
||||
status: RunStatus | None = None,
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
|
||||
@@ -17,8 +17,9 @@ from app.scheduler import (
|
||||
JOB_INTERVALS,
|
||||
MAX_INTERVAL,
|
||||
MIN_INTERVAL,
|
||||
apply_interval,
|
||||
reschedule,
|
||||
scheduler_snapshot,
|
||||
stage_interval,
|
||||
trigger_all,
|
||||
trigger_job,
|
||||
)
|
||||
@@ -42,14 +43,14 @@ def set_job_interval(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Change how often a scheduler job runs (minutes). Persisted as an override and applied
|
||||
to the live scheduler immediately."""
|
||||
"""Change how often a scheduler job runs (minutes). Stored as an override together with its
|
||||
audit row, then applied to the live scheduler."""
|
||||
if job_id not in JOB_INTERVALS:
|
||||
raise not_found("job")
|
||||
minutes = body.interval_minutes
|
||||
if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL):
|
||||
raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL)
|
||||
applied = apply_interval(db, job_id, minutes)
|
||||
applied = stage_interval(db, job_id, minutes)
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
@@ -60,6 +61,7 @@ def set_job_interval(
|
||||
after={"interval_minutes": applied},
|
||||
)
|
||||
db.commit()
|
||||
reschedule(job_id, applied)
|
||||
return {"ok": True, "id": job_id, "interval_minutes": applied}
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""The words a background run's row is written with — the `job_runs` table's status and trigger.
|
||||
|
||||
One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the
|
||||
model's CHECK constraints are built from these sets, and the dashboard reads them back. A run's
|
||||
status is never typed as a string anywhere else (the download pipeline's `cancelled`/`canceled`
|
||||
drift, `downloads/status.py`, is why).
|
||||
model's CHECK constraints are built from these sets, and the dashboard reads them back (the frontend's
|
||||
copies are held to them by `backendIds.contract.test.ts`). The migrated database's CHECKs are frozen
|
||||
in its migration: a test holds them equal to these sets, so a new word fails until a migration adds
|
||||
it (`test_runstatus.py`). The download pipeline's `cancelled`/`canceled` drift (`downloads/status.py`)
|
||||
is why a status is never typed as a bare string.
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
+70
-43
@@ -3,7 +3,7 @@ recent-first / deep backfill. Runs inside the API process (single worker)."""
|
||||
|
||||
import contextvars
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from datetime import timezone
|
||||
from typing import Callable
|
||||
|
||||
from apscheduler.executors.pool import ThreadPoolExecutor
|
||||
@@ -27,6 +27,7 @@ from app.sync.autotag import run_autotag_all
|
||||
from app.sync.explore import purge_ephemeral
|
||||
from app.sync.maintenance import run_maintenance
|
||||
from app.sync.playlists import sync_all_playlists
|
||||
from app.utils import iso
|
||||
from app.sync.runner import (
|
||||
run_deep_backfill,
|
||||
run_enrich,
|
||||
@@ -67,18 +68,21 @@ def load_intervals(db) -> dict[str, int]:
|
||||
return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS}
|
||||
|
||||
|
||||
def apply_interval(db, job_id: str, minutes: int) -> int:
|
||||
"""Persist a job's interval override and reschedule the live job immediately (if the
|
||||
scheduler runs in this process). Returns the applied value."""
|
||||
def stage_interval(db, job_id: str, minutes: int) -> int:
|
||||
"""Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it
|
||||
with its audit row, then calls `reschedule` — the live job follows only what was stored."""
|
||||
if job_id not in JOB_INTERVALS:
|
||||
raise KeyError(job_id)
|
||||
minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes)))
|
||||
sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes)
|
||||
db.commit()
|
||||
return minutes
|
||||
|
||||
|
||||
def reschedule(job_id: str, minutes: int) -> None:
|
||||
"""Move the live interval job to `minutes` (if this process registered it)."""
|
||||
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
|
||||
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
|
||||
log.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||
return minutes
|
||||
|
||||
|
||||
def _record(name: str, **fields) -> None:
|
||||
@@ -87,7 +91,12 @@ def _record(name: str, **fields) -> None:
|
||||
|
||||
|
||||
def _notify_done(
|
||||
db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None
|
||||
db,
|
||||
actor_id: int,
|
||||
job_id: str,
|
||||
status: RunStatus,
|
||||
summary: str | None,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
||||
effort — a notification failure must never mask the job's own outcome."""
|
||||
@@ -139,24 +148,40 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None:
|
||||
def sink(current, total, phase):
|
||||
_record(name, progress={"current": current, "total": total, "phase": phase})
|
||||
|
||||
# Only the work is inside the try: writing its outcome is not the job, and a write that fails
|
||||
# must not turn a run that succeeded into a failed one (see _close).
|
||||
result = None
|
||||
try:
|
||||
if is_sync_paused(db):
|
||||
log.info("job %s skipped (sync paused)", name)
|
||||
jobruns.finish(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||
return
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
summary, data = _finish_ok(name, run_id, result)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "ok", summary, data)
|
||||
paused = is_sync_paused(db)
|
||||
if not paused:
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
||||
err, failure = _finish_error(run_id, exc)
|
||||
err, failure = _failure(exc)
|
||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "error", err, failure)
|
||||
_notify_done(db, actor_id, name, RunStatus.ERROR, err, failure)
|
||||
return
|
||||
if paused:
|
||||
log.info("job %s skipped (sync paused)", name)
|
||||
_close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED)
|
||||
return
|
||||
summary, data = _finish_ok(name, run_id, result)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, RunStatus.OK, summary, data)
|
||||
|
||||
|
||||
def _close(run_id: int, status: RunStatus, **outcome) -> None:
|
||||
"""Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's:
|
||||
the row then stays `running` until the next start's sweep closes it as interrupted."""
|
||||
try:
|
||||
jobruns.finish(run_id, status, **outcome)
|
||||
except Exception:
|
||||
log.exception("run %s: its outcome (%s) could not be recorded", run_id, status)
|
||||
|
||||
|
||||
def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]:
|
||||
@@ -164,16 +189,15 @@ def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]
|
||||
summary = _summarize(result)
|
||||
data = _result_data(result)
|
||||
log.info("job %s -> %s", name, result)
|
||||
jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data)
|
||||
_close(run_id, RunStatus.OK, summary=summary, result=data)
|
||||
return summary, data
|
||||
|
||||
|
||||
def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]:
|
||||
"""Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as
|
||||
a result does, so the client says it in the reader's language; the error stays the English line."""
|
||||
def _failure(exc: Exception) -> tuple[str, dict | None]:
|
||||
"""A raised run's English error line, and the failure the job named (progress.JobFailed) as
|
||||
data, as a result is, so the client says it in the reader's language."""
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
|
||||
jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
return err, failure
|
||||
|
||||
|
||||
@@ -201,10 +225,6 @@ def _summarize(result) -> str | None:
|
||||
return str(result)
|
||||
|
||||
|
||||
def _iso(ts: datetime | None) -> str | None:
|
||||
return ts.astimezone(timezone.utc).isoformat() if ts else None
|
||||
|
||||
|
||||
def scheduler_snapshot(db) -> dict:
|
||||
"""What the scheduler is doing, for the admin dashboard: each job's live state from this process
|
||||
and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how
|
||||
@@ -223,7 +243,7 @@ def scheduler_snapshot(db) -> dict:
|
||||
live_intervals[job.id] = round(iv.total_seconds() / 60)
|
||||
with _activity_lock:
|
||||
acts = {k: dict(v) for k, v in _activity.items()}
|
||||
recent = jobruns.recent_runs(db, RECENT_RUNS)
|
||||
recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
|
||||
jobs = []
|
||||
for job_id, default_interval in JOB_INTERVALS.items():
|
||||
a = acts.get(job_id, {})
|
||||
@@ -239,17 +259,19 @@ def scheduler_snapshot(db) -> dict:
|
||||
)
|
||||
# Work a user starts on one target: no interval, no "run now" (it needs its target), but its
|
||||
# runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
|
||||
# target's own (sync/runner), so "running" is what its newest row says.
|
||||
# target's own (sync/runner), so "running" is what its rows say: two channels can backfill at
|
||||
# once, so a run still going counts even when a newer one already ended, and the row names it.
|
||||
backfills = recent.get(CHANNEL_BACKFILL, [])
|
||||
newest = backfills[0] if backfills else None
|
||||
going = jobruns.running_one(db, CHANNEL_BACKFILL)
|
||||
shown = going or (backfills[0] if backfills else None)
|
||||
ad_hoc = [
|
||||
{
|
||||
"id": CHANNEL_BACKFILL,
|
||||
"running": newest is not None and newest.status == RunStatus.RUNNING,
|
||||
"target": newest.target if newest else None,
|
||||
"running": going is not None,
|
||||
"target": shown.target if shown else None,
|
||||
"target_title": (
|
||||
db.scalar(select(Channel.title).where(Channel.id == newest.target))
|
||||
if newest is not None and newest.target
|
||||
db.scalar(select(Channel.title).where(Channel.id == shown.target))
|
||||
if shown is not None and shown.target
|
||||
else None
|
||||
),
|
||||
**_last_runs(backfills),
|
||||
@@ -275,8 +297,8 @@ def _last_runs(runs: list) -> dict:
|
||||
ended = run if run is not None and run.status != RunStatus.RUNNING else None
|
||||
return {
|
||||
"status": ended.status if ended else None,
|
||||
"last_started": _iso(run.started_at) if run else None,
|
||||
"last_finished": _iso(run.finished_at) if run else None,
|
||||
"last_started": iso(run.started_at) if run else None,
|
||||
"last_finished": iso(run.finished_at) if run else None,
|
||||
"last_result": ended.summary if ended else None,
|
||||
# The outcome as data: the result, or on a failed run the failure the job named.
|
||||
"result": _outcome_data(ended),
|
||||
@@ -293,8 +315,8 @@ def _run_brief(run) -> dict:
|
||||
"id": run.id,
|
||||
"status": run.status,
|
||||
"trigger": run.trigger,
|
||||
"started_at": _iso(run.started_at),
|
||||
"finished_at": _iso(run.finished_at),
|
||||
"started_at": iso(run.started_at),
|
||||
"finished_at": iso(run.finished_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -575,11 +597,15 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non
|
||||
log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
|
||||
return
|
||||
with logs.bind(run_id=run_id):
|
||||
# As in _run: only the work is inside the try.
|
||||
try:
|
||||
_finish_ok(job, run_id, fn())
|
||||
result = fn()
|
||||
except Exception as exc:
|
||||
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
|
||||
_finish_error(run_id, exc)
|
||||
err, failure = _failure(exc)
|
||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
return
|
||||
_finish_ok(job, run_id, result)
|
||||
|
||||
|
||||
# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
|
||||
@@ -590,7 +616,8 @@ _POOL_SIZE = len(JOB_FUNCS) + 8
|
||||
def start_scheduler() -> None:
|
||||
"""Start this process's scheduler. It always starts, so "run now" works on every install; only
|
||||
with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can
|
||||
run, every run a previous process left open is closed as interrupted."""
|
||||
run, every run a previous process left open is closed as interrupted — one API process serves a
|
||||
database (app/jobruns.py), so an open run here is a dead process's."""
|
||||
global _scheduler
|
||||
if _scheduler is not None:
|
||||
return
|
||||
|
||||
@@ -29,7 +29,9 @@ _coerce_warned: set[tuple[str, str]] = set()
|
||||
|
||||
|
||||
# When a changed value takes effect. "live": the next read obeys it — every consumer reads the key
|
||||
# where it uses it, never once at import or boot (test-held). "restart": the process has to start
|
||||
# where it uses it, never once at import (test-held: a read outside a function fails). A value read
|
||||
# once at startup is live only when its writer also applies it (the job intervals reschedule, the log
|
||||
# level is set as it is saved). "restart": the process has to start
|
||||
# again. No key needs a restart today, and the Configuration page has no way to say so yet, so a
|
||||
# "restart" key is refused until the page can (test-held).
|
||||
Applies = Literal["live", "restart"]
|
||||
|
||||
Reference in New Issue
Block a user