peter
5082636f15
- The scheduler keeps one registry of the runs this process has open, for the interval jobs and the ad-hoc runs alike. A row the table still says is `running` but no run here has open (its outcome failed to write) reads as interrupted, as the next start's sweep will close it: no pulsing dot beside "No run recorded yet". - Migration 0109's downgrade sums a still-running run's quota events into its spend before the link to them is dropped, as the startup sweep would. - A test holds that a dialog closing onto the same address leaves an open log line open.
161 lines
7.4 KiB
Python
161 lines
7.4 KiB
Python
"""Background runs get a durable history
|
|
|
|
Revision ID: 0109_job_runs
|
|
Revises: 0108_folder_links
|
|
Create Date: 2026-10-17
|
|
|
|
What a scheduler job last did lived in the API process's memory, gone at every restart; the audit
|
|
log kept a row for a run only when it was manual, failed, or changed something. Now every run is a
|
|
`job_runs` row, written when it starts and when it ends. A run's quota spend points at it
|
|
(`quota_events.job_run_id`), and so do the lines it logged (`log_events.run_id`, no foreign key: the
|
|
log writer runs behind the event).
|
|
|
|
Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows
|
|
MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the
|
|
audit log. The downgrade moves every run back into the audit log as such a row (one still running as
|
|
interrupted, what the audit row has no column for in its `after`), then drops the table.
|
|
"""
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
from sqlalchemy.dialects import postgresql
|
|
|
|
revision: str = "0109_job_runs"
|
|
down_revision: str | None = "0108_folder_links"
|
|
branch_labels: str | None = None
|
|
depends_on: str | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"job_runs",
|
|
sa.Column("id", sa.BigInteger(), primary_key=True),
|
|
sa.Column("job", sa.String(40), nullable=False),
|
|
sa.Column("trigger", sa.String(16), nullable=False),
|
|
sa.Column("actor_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="SET NULL")),
|
|
sa.Column("target", sa.String(128)),
|
|
sa.Column("status", sa.String(16), nullable=False),
|
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
|
sa.Column("summary", sa.Text()),
|
|
sa.Column("result", postgresql.JSONB()),
|
|
sa.Column("error", sa.Text()),
|
|
sa.Column("failure", postgresql.JSONB()),
|
|
sa.Column("quota_units", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.CheckConstraint(
|
|
"status IN ('running', 'ok', 'error', 'skipped', 'interrupted')",
|
|
name="ck_job_runs_status",
|
|
),
|
|
sa.CheckConstraint(
|
|
"trigger IN ('scheduled', 'manual', 'ad_hoc')", name="ck_job_runs_trigger"
|
|
),
|
|
sa.CheckConstraint(
|
|
"(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished"
|
|
),
|
|
)
|
|
op.create_index("ix_job_runs_job_started", "job_runs", ["job", "started_at"])
|
|
op.create_index("ix_job_runs_actor_id", "job_runs", ["actor_id"])
|
|
op.create_index("ix_job_runs_started_at", "job_runs", ["started_at"])
|
|
|
|
op.add_column(
|
|
"quota_events",
|
|
sa.Column("job_run_id", sa.BigInteger(), sa.ForeignKey("job_runs.id", ondelete="SET NULL")),
|
|
)
|
|
op.create_index("ix_quota_events_job_run_id", "quota_events", ["job_run_id"])
|
|
|
|
op.add_column("log_events", sa.Column("run_id", sa.BigInteger()))
|
|
op.create_index(
|
|
"ix_log_events_run",
|
|
"log_events",
|
|
["run_id"],
|
|
postgresql_where=sa.text("run_id IS NOT NULL"),
|
|
)
|
|
|
|
# The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`,
|
|
# which a downgrade writes back (as it does a run's trigger, target, spend and start, which the
|
|
# audit row has no column for: its actor alone cannot say "manual" once that admin is deleted),
|
|
# so a rollback and a redeploy keep every run as it was. A status the run table does
|
|
# not know would fail the CHECK.
|
|
op.execute(
|
|
"""
|
|
INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at,
|
|
summary, result, error, failure, quota_units)
|
|
SELECT target_id,
|
|
coalesce(after::jsonb ->> 'trigger',
|
|
CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END),
|
|
actor_id,
|
|
after::jsonb ->> 'target',
|
|
after::jsonb ->> 'status',
|
|
coalesce((after::jsonb ->> 'started_at')::timestamptz, created_at),
|
|
created_at,
|
|
CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb ->> 'summary' END,
|
|
CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb -> 'result' END,
|
|
CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb ->> 'summary' END,
|
|
CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END,
|
|
coalesce((after::jsonb ->> 'quota_units')::int, 0)
|
|
FROM audit_log
|
|
WHERE action = 'job_run'
|
|
AND target_id IS NOT NULL
|
|
AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted')
|
|
ORDER BY id
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
DELETE FROM audit_log
|
|
WHERE action = 'job_run'
|
|
AND target_id IS NOT NULL
|
|
AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted')
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
# A run still `running` is one the rollback stopped: it goes back as interrupted. What the audit
|
|
# row's own columns cannot say rides in `after` (its trigger, target, spend and start), so the
|
|
# upgrade restores them.
|
|
op.execute(
|
|
"""
|
|
INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at)
|
|
SELECT actor_id,
|
|
'job_run',
|
|
'job',
|
|
job,
|
|
left(job || ': ' || st
|
|
|| coalesce(' — ' || CASE WHEN st = 'error' THEN error ELSE summary END, ''),
|
|
255),
|
|
(jsonb_build_object(
|
|
'status', st,
|
|
'summary', CASE WHEN st = 'error' THEN error ELSE summary END)
|
|
|| CASE
|
|
WHEN st = 'error' AND failure IS NOT NULL
|
|
THEN jsonb_build_object('result', failure)
|
|
WHEN st <> 'error' AND result IS NOT NULL
|
|
THEN jsonb_build_object('result', result)
|
|
ELSE '{}'::jsonb
|
|
END
|
|
|| jsonb_build_object('trigger', trigger, 'quota_units', units,
|
|
'started_at', started_at)
|
|
|| CASE WHEN target IS NOT NULL
|
|
THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json,
|
|
coalesce(finished_at, started_at)
|
|
FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st,
|
|
-- A run stopped mid-way has not summed its spend yet: sum it as the startup
|
|
-- sweep would, before the link to it is dropped.
|
|
CASE WHEN status = 'running'
|
|
THEN (SELECT coalesce(sum(q.units), 0) FROM quota_events q
|
|
WHERE q.job_run_id = job_runs.id)
|
|
ELSE quota_units END AS units
|
|
FROM job_runs) AS r
|
|
ORDER BY id
|
|
"""
|
|
)
|
|
op.drop_index("ix_log_events_run", table_name="log_events")
|
|
op.drop_column("log_events", "run_id")
|
|
op.drop_index("ix_quota_events_job_run_id", table_name="quota_events")
|
|
op.drop_column("quota_events", "job_run_id")
|
|
op.drop_index("ix_job_runs_started_at", table_name="job_runs")
|
|
op.drop_index("ix_job_runs_actor_id", table_name="job_runs")
|
|
op.drop_index("ix_job_runs_job_started", table_name="job_runs")
|
|
op.drop_table("job_runs")
|