feat(scheduler): every background run is a durable row
What a job last did lived in the API process's memory and was gone at every restart; the audit log kept a run only when it was manual, failed or changed something. Now each run is a job_runs row written at its start and its end in its own session: who started it, its outcome as data, what it spent of the YouTube quota (quota_events.job_run_id) and the lines it logged (log_events.run_id, filterable in the log console). - a run a stopped process left open is marked interrupted at startup - "run now" is a one-shot job on the scheduler, which now always starts (interval jobs only with SCHEDULER_ENABLED); the manual actor is reset after the run, since pool threads are reused - the dashboard's last run comes from job_runs, so it survives a deploy - runs leave the audit log: migration 0109 moves the old job_run rows over (and back on downgrade); the audit page's run folding goes - job_run_retention_days (default 30) prunes old runs in audit_gc
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
"""Background runs get a durable history
|
||||
|
||||
Revision ID: 0109_job_runs
|
||||
Revises: 0108_folder_links
|
||||
Create Date: 2026-10-17
|
||||
|
||||
What a scheduler job last did lived in the API process's memory, gone at every restart; the audit
|
||||
log kept a row for a run only when it was manual, failed, or changed something. Now every run is a
|
||||
`job_runs` row, written when it starts and when it ends. A run's quota spend points at it
|
||||
(`quota_events.job_run_id`), and so do the lines it logged (`log_events.run_id`, no foreign key: the
|
||||
log writer runs behind the event).
|
||||
|
||||
Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows
|
||||
MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the
|
||||
audit log. The downgrade moves every run back into the audit log as such a row, then drops the table.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0109_job_runs"
|
||||
down_revision: str | None = "0108_folder_links"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"job_runs",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("job", sa.String(40), nullable=False),
|
||||
sa.Column("trigger", sa.String(16), nullable=False),
|
||||
sa.Column("actor_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="SET NULL")),
|
||||
sa.Column("target", sa.String(128)),
|
||||
sa.Column("status", sa.String(16), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("summary", sa.Text()),
|
||||
sa.Column("result", postgresql.JSONB()),
|
||||
sa.Column("error", sa.Text()),
|
||||
sa.Column("failure", postgresql.JSONB()),
|
||||
sa.Column("quota_units", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('running', 'ok', 'error', 'skipped', 'interrupted')",
|
||||
name="ck_job_runs_status",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"trigger IN ('scheduled', 'manual', 'ad_hoc')", name="ck_job_runs_trigger"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_job_runs_job_started", "job_runs", ["job", "started_at"])
|
||||
op.create_index("ix_job_runs_actor_id", "job_runs", ["actor_id"])
|
||||
op.create_index("ix_job_runs_started_at", "job_runs", ["started_at"])
|
||||
|
||||
op.add_column(
|
||||
"quota_events",
|
||||
sa.Column("job_run_id", sa.BigInteger(), sa.ForeignKey("job_runs.id", ondelete="SET NULL")),
|
||||
)
|
||||
op.create_index("ix_quota_events_job_run_id", "quota_events", ["job_run_id"])
|
||||
|
||||
op.add_column("log_events", sa.Column("run_id", sa.BigInteger()))
|
||||
op.create_index(
|
||||
"ix_log_events_run",
|
||||
"log_events",
|
||||
["run_id"],
|
||||
postgresql_where=sa.text("run_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
# The audit trail's runs move over. A status the run table does not know (none was ever
|
||||
# written) would fail the CHECK, so the move is limited to the three the scheduler wrote.
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO job_runs (job, trigger, actor_id, status, started_at, finished_at, summary,
|
||||
result, error, failure)
|
||||
SELECT target_id,
|
||||
CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END,
|
||||
actor_id,
|
||||
after::jsonb ->> 'status',
|
||||
created_at,
|
||||
created_at,
|
||||
CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb ->> 'summary' END,
|
||||
CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb -> 'result' END,
|
||||
CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb ->> 'summary' END,
|
||||
CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END
|
||||
FROM audit_log
|
||||
WHERE action = 'job_run'
|
||||
AND target_id IS NOT NULL
|
||||
AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped')
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM audit_log
|
||||
WHERE action = 'job_run'
|
||||
AND target_id IS NOT NULL
|
||||
AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped')
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at)
|
||||
SELECT actor_id,
|
||||
'job_run',
|
||||
'job',
|
||||
job,
|
||||
left(job || ': ' || status
|
||||
|| coalesce(' — ' || CASE WHEN status = 'error' THEN error ELSE summary END, ''),
|
||||
255),
|
||||
(jsonb_build_object(
|
||||
'status', status,
|
||||
'summary', CASE WHEN status = 'error' THEN error ELSE summary END)
|
||||
|| CASE
|
||||
WHEN status = 'error' AND failure IS NOT NULL
|
||||
THEN jsonb_build_object('result', failure)
|
||||
WHEN status <> 'error' AND result IS NOT NULL
|
||||
THEN jsonb_build_object('result', result)
|
||||
ELSE '{}'::jsonb
|
||||
END)::json,
|
||||
coalesce(finished_at, started_at)
|
||||
FROM job_runs
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
op.drop_index("ix_log_events_run", table_name="log_events")
|
||||
op.drop_column("log_events", "run_id")
|
||||
op.drop_index("ix_quota_events_job_run_id", table_name="quota_events")
|
||||
op.drop_column("quota_events", "job_run_id")
|
||||
op.drop_index("ix_job_runs_started_at", table_name="job_runs")
|
||||
op.drop_index("ix_job_runs_actor_id", table_name="job_runs")
|
||||
op.drop_index("ix_job_runs_job_started", table_name="job_runs")
|
||||
op.drop_table("job_runs")
|
||||
Reference in New Issue
Block a user