siftlode/backend/app/jobruns.py

"""The durable history of background runs — the `job_runs` table.

A run is written twice: a `running` row when it starts, its outcome when it ends. Each write is its
OWN short session, committed at once, for the reason `quota.record_usage` gives: the run's own work
may roll back (its `except: db.rollback()`), and the history of what happened must not go with it —
nor may this bookkeeping commit the job's half-done session.

Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a
database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its
startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open.
"""

from datetime import datetime, timedelta, timezone

from sqlalchemy import delete, func, select, union_all, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session

from app.db import SessionLocal
from app.models import JobRun, QuotaEvent
from app.runstatus import RunStatus, RunTrigger


def start(job: str, trigger: RunTrigger, actor_id: int | None, target: str | None = None) -> int:
    """Open a run's row; returns its id."""
    with SessionLocal() as s:
        row = JobRun(
            job=job,
            trigger=trigger,
            actor_id=actor_id,
            target=target,
            status=RunStatus.RUNNING,
            started_at=datetime.now(timezone.utc),
        )
        s.add(row)
        s.commit()
        return row.id


def finish(
    run_id: int,
    status: RunStatus,
    *,
    summary: str | None = None,
    result: dict | None = None,
    error: str | None = None,
    failure: dict | None = None,
) -> None:
    """Close a run with its outcome and what it spent. Conditional on the row still running: a row
    the startup sweep already called interrupted is not rewritten by a thread that outlived it."""
    with SessionLocal() as s:
        units = s.scalar(
            select(func.coalesce(func.sum(QuotaEvent.units), 0)).where(
                QuotaEvent.job_run_id == run_id
            )
        )
        s.execute(
            update(JobRun)
            .where(JobRun.id == run_id, JobRun.status == RunStatus.RUNNING)
            .values(
                status=status,
                finished_at=datetime.now(timezone.utc),
                summary=summary,
                result=result,
                error=error,
                failure=failure,
                quota_units=units or 0,
            )
        )
        s.commit()


def interrupt_leftovers() -> int:
    """Every row a previous process left `running` → `interrupted`, with what it spent until then
    (its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler
    can open a new run. Returns how many."""
    spent = (
        select(func.coalesce(func.sum(QuotaEvent.units), 0))
        .where(QuotaEvent.job_run_id == JobRun.id)
        .scalar_subquery()
    )
    with SessionLocal() as s:
        res = s.execute(
            update(JobRun)
            .where(JobRun.status == RunStatus.RUNNING)
            .values(
                status=RunStatus.INTERRUPTED,
                finished_at=datetime.now(timezone.utc),
                quota_units=spent,
            )
        )
        s.commit()
        return res.rowcount if isinstance(res, CursorResult) else 0


def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]:
    """Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a
    short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window
    over the whole table grew with every run kept)."""
    if not jobs:
        return {}
    newest = union_all(
        *(
            select(JobRun.id)
            .where(JobRun.job == job)
            .order_by(JobRun.started_at.desc(), JobRun.id.desc())
            .limit(per_job)
            for job in jobs
        )
    ).subquery()
    rows = db.execute(
        select(JobRun)
        .where(JobRun.id.in_(select(newest.c.id)))
        .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc())
    ).scalars()
    out: dict[str, list[JobRun]] = {}
    for r in rows:
        out.setdefault(r.job, []).append(r)
    return out


def running_one(db: Session, job: str) -> JobRun | None:
    """The newest run of `job` still going, however many runs ended after it started."""
    return db.scalars(
        select(JobRun)
        .where(JobRun.job == job, JobRun.status == RunStatus.RUNNING)
        .order_by(JobRun.started_at.desc(), JobRun.id.desc())
        .limit(1)
    ).first()


def gc(db: Session, retention_days: int) -> int:
    """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is
    never reaped. The caller commits."""
    if retention_days <= 0:
        return 0
    cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
    res = db.execute(
        delete(JobRun).where(JobRun.started_at < cutoff, JobRun.status != RunStatus.RUNNING)
    )
    return res.rowcount if isinstance(res, CursorResult) else 0