peter
27c2fcbd2e
The one-channel deep backfill ran as a FastAPI BackgroundTask: what it fetched, what it spent and how it failed reached only the log. It is now a one-shot run on the scheduler (`channel_backfill`, trigger ad_hoc, actor = the user, target = the channel) with its quota tagged to the run. Its per-channel exclusion with the scheduled backfill and the two-run cap are unchanged, and a run they turn away says why in its result instead of returning nothing. A failure is no longer swallowed: the run records it. The scheduler snapshot carries each job's last three runs (one windowed query), so the page can show an interrupted or skipped run instead of the latest outcome alone.
115 lines
4.1 KiB
Python
115 lines
4.1 KiB
Python
"""The durable history of background runs — the `job_runs` table.
|
|
|
|
A run is written twice: a `running` row when it starts, its outcome when it ends. Each write is its
|
|
OWN short session, committed at once, for the reason `quota.record_usage` gives: the run's own work
|
|
may roll back (its `except: db.rollback()`), and the history of what happened must not go with it —
|
|
nor may this bookkeeping commit the job's half-done session.
|
|
|
|
Only the API process runs jobs (the worker never starts the scheduler), so at its startup nothing can
|
|
still be running: `interrupt_leftovers` closes what a stopped process left open.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import delete, func, select, update
|
|
from sqlalchemy.engine import CursorResult
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.db import SessionLocal
|
|
from app.models import JobRun, QuotaEvent
|
|
from app.runstatus import RunStatus, RunTrigger
|
|
|
|
|
|
def start(job: str, trigger: RunTrigger, actor_id: int | None, target: str | None = None) -> int:
|
|
"""Open a run's row; returns its id."""
|
|
with SessionLocal() as s:
|
|
row = JobRun(
|
|
job=job,
|
|
trigger=trigger,
|
|
actor_id=actor_id,
|
|
target=target,
|
|
status=RunStatus.RUNNING,
|
|
started_at=datetime.now(timezone.utc),
|
|
)
|
|
s.add(row)
|
|
s.commit()
|
|
return row.id
|
|
|
|
|
|
def finish(
|
|
run_id: int,
|
|
status: RunStatus,
|
|
*,
|
|
summary: str | None = None,
|
|
result: dict | None = None,
|
|
error: str | None = None,
|
|
failure: dict | None = None,
|
|
) -> None:
|
|
"""Close a run with its outcome and what it spent. Conditional on the row still running: a row
|
|
the startup sweep already called interrupted is not rewritten by a thread that outlived it."""
|
|
with SessionLocal() as s:
|
|
units = s.scalar(
|
|
select(func.coalesce(func.sum(QuotaEvent.units), 0)).where(
|
|
QuotaEvent.job_run_id == run_id
|
|
)
|
|
)
|
|
s.execute(
|
|
update(JobRun)
|
|
.where(JobRun.id == run_id, JobRun.status == RunStatus.RUNNING)
|
|
.values(
|
|
status=status,
|
|
finished_at=datetime.now(timezone.utc),
|
|
summary=summary,
|
|
result=result,
|
|
error=error,
|
|
failure=failure,
|
|
quota_units=units or 0,
|
|
)
|
|
)
|
|
s.commit()
|
|
|
|
|
|
def interrupt_leftovers() -> int:
|
|
"""Every row a previous process left `running` → `interrupted`. Called once at startup, before
|
|
the scheduler can open a new run. Returns how many."""
|
|
with SessionLocal() as s:
|
|
res = s.execute(
|
|
update(JobRun)
|
|
.where(JobRun.status == RunStatus.RUNNING)
|
|
.values(status=RunStatus.INTERRUPTED, finished_at=datetime.now(timezone.utc))
|
|
)
|
|
s.commit()
|
|
return res.rowcount if isinstance(res, CursorResult) else 0
|
|
|
|
|
|
def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]:
|
|
"""Each job's last `per_job` runs, newest first, by job id — one query (a window per job)."""
|
|
ranked = select(
|
|
JobRun.id,
|
|
func.row_number()
|
|
.over(partition_by=JobRun.job, order_by=(JobRun.started_at.desc(), JobRun.id.desc()))
|
|
.label("rank"),
|
|
).subquery()
|
|
rows = db.execute(
|
|
select(JobRun)
|
|
.join(ranked, ranked.c.id == JobRun.id)
|
|
.where(ranked.c.rank <= per_job)
|
|
.order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc())
|
|
).scalars()
|
|
out: dict[str, list[JobRun]] = {}
|
|
for r in rows:
|
|
out.setdefault(r.job, []).append(r)
|
|
return out
|
|
|
|
|
|
def gc(db: Session, retention_days: int) -> int:
|
|
"""Delete finished runs older than `retention_days` (0 = keep forever). A run still going is
|
|
never reaped. The caller commits."""
|
|
if retention_days <= 0:
|
|
return 0
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
|
res = db.execute(
|
|
delete(JobRun).where(JobRun.started_at < cutoff, JobRun.status != RunStatus.RUNNING)
|
|
)
|
|
return res.rowcount if isinstance(res, CursorResult) else 0
|