feat(activity): one Activity page — runs, admin actions, server log and quota under one filter bar
The audit log, the server log and the admin quota view were three pages (the quota one a Stats tab),
each with its own filters, and the background runs had no list at all. Now one page ("Napló" /
"Activity") reads all four, each still its own table on the server: a shared bar (text, a period of
1 h to 90 days defaulting to 7, an account) and each tab's own filters, the whole view in the URL.
The old page ids open their tab.
Server side: the audit log and the new run list (`/api/admin/runs`) filter and page like the server
log, through one keyset helper (`routes/_keyset`) the log now shares; an audit row keeps the request
it was made in (migration 0111), so its log lines are a pivot away, as a run's are (`lrun`). The
quota breakdown takes since/until and one account. The Scheduler rows get a history button to their
job's runs.
Also fixed on the way: re-saving the value in force on the Configuration page staged its override
without committing it (the commit sat inside the audit branch), so the save was lost.
This commit is contained in:
@@ -630,41 +630,7 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/admin_logs.py": [
|
||||
{
|
||||
"code": "reportReturnType",
|
||||
"range": {
|
||||
"startColumn": 15,
|
||||
"endColumn": 47,
|
||||
"lineCount": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/audit.py": [
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 17,
|
||||
"endColumn": 98,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportAssignmentType",
|
||||
"range": {
|
||||
"startColumn": 17,
|
||||
"endColumn": 98,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 22,
|
||||
"endColumn": 97,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportAttributeAccessIssue",
|
||||
"range": {
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""An audit row keeps the request that made it
|
||||
|
||||
Revision ID: 0111_audit_request_id
|
||||
Revises: 0110_intervals_in_sysconfig
|
||||
Create Date: 2026-10-18
|
||||
|
||||
The activity page opens an admin action's log lines from its row: `audit_log.request_id` is the
|
||||
request the change was made in, the same id `log_events.request_id` carries. Older rows have none —
|
||||
the request that made them was never recorded.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0111_audit_request_id"
|
||||
down_revision: str | None = "0110_intervals_in_sysconfig"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("audit_log", sa.Column("request_id", sa.String(64)))
|
||||
op.create_index("ix_audit_log_request_id", "audit_log", ["request_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_audit_log_request_id", table_name="audit_log")
|
||||
op.drop_column("audit_log", "request_id")
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Admin/scheduler audit trail: an append-only record of who changed what and when.
|
||||
"""Admin audit trail: an append-only record of who changed what and when.
|
||||
|
||||
Design (locked): log at ACTION / job-run granularity, NEVER per-row — a scheduler run that
|
||||
touches thousands of videos is ONE audit row summarising the outcome. Automatic job runs are
|
||||
only logged when they actually changed something (or errored); all admin actions and all manual
|
||||
runs are always logged. Secrets are never stored (config changes log the key name only).
|
||||
Design (locked): log at ACTION granularity, NEVER per-row — an action that touches thousands of
|
||||
rows is ONE audit row summarising the outcome. Background runs are not actions: they have their
|
||||
own history (app/jobruns.py). Secrets are never stored (config changes log the key name only).
|
||||
Each row keeps the request that made it, so the request's log lines are one click away.
|
||||
|
||||
`record()` only db.add()s the row — the caller commits it in the SAME transaction as the
|
||||
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
|
||||
@@ -15,6 +15,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import AuditLog
|
||||
|
||||
|
||||
@@ -102,6 +103,7 @@ def record(
|
||||
summary=(summary or None) and summary[:255],
|
||||
before=before,
|
||||
after=after,
|
||||
request_id=logs.current_request_id(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -93,6 +93,11 @@ _CONTEXT_VARS = {
|
||||
}
|
||||
|
||||
|
||||
def current_request_id() -> str | None:
|
||||
"""The HTTP request this context is serving, if any — the request middleware sets it."""
|
||||
return _request_id.get()
|
||||
|
||||
|
||||
def current_run_id() -> int | None:
|
||||
"""The background run this context is doing, if any — `bind(run_id=…)` sets it."""
|
||||
return _run_id.get()
|
||||
|
||||
@@ -52,6 +52,7 @@ from app.routes import ( # noqa: E402 — logging is configured first
|
||||
plex as plex_routes,
|
||||
public as public_routes,
|
||||
quota,
|
||||
runs as runs_routes,
|
||||
saved_views,
|
||||
scheduler as scheduler_routes,
|
||||
search as search_routes,
|
||||
@@ -345,6 +346,7 @@ app.include_router(config_routes.router)
|
||||
app.include_router(scheduler_routes.router)
|
||||
app.include_router(audit_routes.router)
|
||||
app.include_router(admin_logs_routes.router)
|
||||
app.include_router(runs_routes.router)
|
||||
app.include_router(quota.router)
|
||||
app.include_router(version.router)
|
||||
app.include_router(setup_routes.router)
|
||||
|
||||
@@ -572,11 +572,12 @@ class QuotaEvent(Base):
|
||||
|
||||
|
||||
class AuditLog(Base):
|
||||
"""Append-only admin/scheduler audit trail: who did what, when, and (where it applies)
|
||||
the before->after values. `actor_id` is the acting admin; NULL means the scheduler /
|
||||
background did it (SET NULL on user delete keeps the trail). Logged at ACTION / job-run
|
||||
granularity — never per-row — so a scheduler run that touches thousands of videos is ONE
|
||||
row summarising the outcome. `before`/`after` are small JSON snapshots (secrets excluded)."""
|
||||
"""Append-only admin audit trail: who did what, when, and (where it applies) the before->after
|
||||
values. `actor_id` is the acting admin; NULL means the background did it (SET NULL on user
|
||||
delete keeps the trail). Logged per ACTION, never per row; a background run is not an action
|
||||
and has its own history (`JobRun`). `before`/`after` are small JSON snapshots (secrets
|
||||
excluded). `request_id` is the request that made the change, so its log lines are one click
|
||||
away (NULL for rows written outside a request, and for rows older than the column)."""
|
||||
|
||||
__tablename__ = "audit_log"
|
||||
|
||||
@@ -593,6 +594,7 @@ class AuditLog(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), index=True
|
||||
)
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Keyset paging over `(timestamp desc, id desc)`, shared by the admin history lists (server log,
|
||||
audit log, background runs).
|
||||
|
||||
A timestamp alone is not unique (a browser log row's `ts` is clamped into its consent window; a run
|
||||
and its siblings can start in the same microsecond), and an offset would skip or repeat rows while
|
||||
writers add new ones at the top — so a page continues from the last row's `(timestamp, id)`.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import User
|
||||
|
||||
PAGE_MAX = 200
|
||||
|
||||
|
||||
def utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the columns are `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def encode_cursor(ts: datetime, row_id: int) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([ts.isoformat(), row_id]).encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
at = datetime.fromisoformat(ts)
|
||||
return (at if at.tzinfo is not None else at.replace(tzinfo=timezone.utc)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def older_than(ts_col, id_col, cursor: str):
|
||||
"""The rows after `cursor` in `(ts desc, id desc)` order. The `ts <=` conjunct is implied by the
|
||||
OR; it is there so the planner can bound the timestamp index scan instead of filtering every
|
||||
newer row first."""
|
||||
ts, row_id = decode_cursor(cursor)
|
||||
return and_(ts_col <= ts, or_(ts_col < ts, and_(ts_col == ts, id_col < row_id)))
|
||||
|
||||
|
||||
def page(rows: list, limit: int, ts_of) -> tuple[list, str | None]:
|
||||
"""`rows` was read with `limit + 1` (to know whether there is a next page without a count query):
|
||||
the page itself, and the cursor that continues after it (None on the last page)."""
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return rows, encode_cursor(ts_of(rows[-1]), rows[-1].id) if more else None
|
||||
|
||||
|
||||
def accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the row
|
||||
was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
@@ -8,33 +8,28 @@ What a row holds was redacted when it was written (`logs.record_fields`, and the
|
||||
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
|
||||
`[redacted]` / `[email]`, which is the point — the console shows that something was there.
|
||||
|
||||
Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent
|
||||
window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds
|
||||
new ones at the top. The live tail re-reads the first page rather than asking for "ids after N":
|
||||
Paging is a keyset over `(ts desc, id desc)` (`_keyset`). The live tail re-reads the first page rather than asking for "ids after N":
|
||||
several processes write this table, and a batch's ids are taken when it inserts, not when it
|
||||
commits, so a slower batch can land below an id the tail already passed — and would never show.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Text, and_, cast, func, or_, select
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import Text, cast, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.utils import like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/logs", tags=["admin"])
|
||||
|
||||
PAGE_MAX = 200
|
||||
# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
|
||||
# through the free-text search, and the response stays small whatever a self-host logs.
|
||||
FACET_EVENTS_MAX = 100
|
||||
@@ -47,35 +42,6 @@ Facet = Literal["levels", "sources", "user", "event"]
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _encode_cursor(row: LogEvent) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
return _utc(datetime.fromisoformat(ts)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the
|
||||
line was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogFilters:
|
||||
"""The console's filters, shared by the page and its facet counts."""
|
||||
@@ -117,9 +83,9 @@ class LogFilters:
|
||||
if self.run_id is not None:
|
||||
out.append(LogEvent.run_id == self.run_id)
|
||||
if self.since is not None:
|
||||
out.append(LogEvent.ts >= _utc(self.since))
|
||||
out.append(LogEvent.ts >= utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(LogEvent.ts < _utc(self.until))
|
||||
out.append(LogEvent.ts < utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
@@ -165,28 +131,20 @@ def list_logs(
|
||||
continues from a previous page's `next_cursor`."""
|
||||
query = select(LogEvent).where(*filters.conditions())
|
||||
if before is not None:
|
||||
ts, row_id = _decode_cursor(before)
|
||||
# The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the
|
||||
# `ts` index scan instead of filtering every newer row first.
|
||||
query = query.where(
|
||||
and_(
|
||||
LogEvent.ts <= ts,
|
||||
or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)),
|
||||
)
|
||||
)
|
||||
|
||||
# One more than asked, to know whether there is a next page without a second count query.
|
||||
rows = (
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
query = query.where(older_than(LogEvent.ts, LogEvent.id, before))
|
||||
rows, cursor = page(
|
||||
list(
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
),
|
||||
limit,
|
||||
lambda r: r.ts,
|
||||
)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"items": [logsink.serialize(r) for r in rows],
|
||||
"accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": _encode_cursor(rows[-1]) if more else None,
|
||||
"accounts": accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@@ -217,6 +175,6 @@ def log_facets(
|
||||
"sources": counts(LogEvent.source, "sources"),
|
||||
"events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
|
||||
"users": [{**u, "value": str(u["value"])} for u in users],
|
||||
"accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"accounts": accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
|
||||
+98
-27
@@ -1,11 +1,14 @@
|
||||
"""Admin audit log API: read the append-only who/what/when trail, and clear it.
|
||||
|
||||
Admin-only. The client renders these rows in a DataTable (client-side sort/filter/paginate), so
|
||||
this returns the most recent window of rows (capped) plus the true total, rather than paging
|
||||
server-side — the log stays small in practice (state-changing runs + admin actions only)."""
|
||||
Admin-only. The activity page reads it a page at a time, newest first, filtered on the server like
|
||||
the server log (`_keyset` paging), so the whole trail is reachable however long it grows."""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import delete, func, select
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import Text, cast, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
@@ -13,55 +16,123 @@ from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import AuditLog, User
|
||||
from app.utils import iso
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.utils import iso, like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/audit", tags=["admin"])
|
||||
|
||||
# Cap the client-side window. The audit log is small by design; if it ever outgrows this we'd
|
||||
# switch the DataTable to server-side paging.
|
||||
MAX_ROWS = 2000
|
||||
# `none` selects the rows no account made (the background) — a value to filter on, not "any".
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _serialize(row: AuditLog, actors: dict[int, str]) -> dict:
|
||||
def _serialize(row: AuditLog) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"created_at": iso(row.created_at),
|
||||
# actor_id NULL = the scheduler/background did it (client shows "System"); a set id that
|
||||
# can't be resolved would mean a deleted user, but SET NULL makes that NULL — so present.
|
||||
"actor": actors.get(row.actor_id) if row.actor_id is not None else None,
|
||||
# NULL = the background did it (the client shows "System"); the address is in `accounts`.
|
||||
"actor_id": row.actor_id,
|
||||
"action": row.action,
|
||||
"target_type": row.target_type,
|
||||
"target_id": row.target_id,
|
||||
"summary": row.summary,
|
||||
"before": row.before,
|
||||
"after": row.after,
|
||||
"request_id": row.request_id,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuditFilters:
|
||||
action: str | None
|
||||
actor: str | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Literal["action"] | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed. `skip="action"` leaves the action filter out, so the action
|
||||
menu counts every action under the other filters (the way back from a pick)."""
|
||||
out = []
|
||||
if self.action is not None and skip != "action":
|
||||
out.append(AuditLog.action == self.action)
|
||||
if self.actor is not None:
|
||||
out.append(
|
||||
AuditLog.actor_id.is_(None)
|
||||
if self.actor == NONE
|
||||
else AuditLog.actor_id == int(self.actor)
|
||||
)
|
||||
if self.since is not None:
|
||||
out.append(AuditLog.created_at >= utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(AuditLog.created_at < utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
AuditLog.summary.ilike(like),
|
||||
AuditLog.target_id.ilike(like),
|
||||
AuditLog.action.ilike(like),
|
||||
AuditLog.request_id.ilike(like),
|
||||
cast(AuditLog.before, Text).ilike(like),
|
||||
cast(AuditLog.after, Text).ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def audit_filters(
|
||||
action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> AuditFilters:
|
||||
return AuditFilters(action, user, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_audit(
|
||||
limit: int = 500,
|
||||
filters: AuditFilters = Depends(audit_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
limit = max(1, min(limit, MAX_ROWS))
|
||||
total = db.scalar(select(func.count()).select_from(AuditLog)) or 0
|
||||
rows = (
|
||||
db.execute(select(AuditLog).order_by(AuditLog.created_at.desc()).limit(limit))
|
||||
.scalars()
|
||||
.all()
|
||||
"""One page of rows, newest first, matching every filter given; `before` continues from a
|
||||
previous page's `next_cursor`."""
|
||||
query = select(AuditLog).where(*filters.conditions())
|
||||
if before is not None:
|
||||
query = query.where(older_than(AuditLog.created_at, AuditLog.id, before))
|
||||
order = (AuditLog.created_at.desc(), AuditLog.id.desc())
|
||||
rows, cursor = page(
|
||||
list(db.execute(query.order_by(*order).limit(limit + 1)).scalars().all()),
|
||||
limit,
|
||||
lambda r: r.created_at,
|
||||
)
|
||||
actor_ids = {r.actor_id for r in rows if r.actor_id is not None}
|
||||
actors: dict[int, str] = {}
|
||||
if actor_ids:
|
||||
actors = dict(db.execute(select(User.id, User.email).where(User.id.in_(actor_ids))).all())
|
||||
return {
|
||||
"items": [_serialize(r, actors) for r in rows],
|
||||
"total": total,
|
||||
"returned": len(rows),
|
||||
"items": [_serialize(r) for r in rows],
|
||||
"accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def audit_facets(
|
||||
filters: AuditFilters = Depends(audit_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The actions present under the other filters, with counts — what the action menu offers."""
|
||||
n = func.count().label("n")
|
||||
stmt = (
|
||||
select(AuditLog.action, n)
|
||||
.where(*filters.conditions("action"))
|
||||
.group_by(AuditLog.action)
|
||||
.order_by(n.desc(), AuditLog.action)
|
||||
)
|
||||
return {"actions": [{"value": a, "count": c} for a, c in db.execute(stmt).all()]}
|
||||
|
||||
|
||||
@router.delete("")
|
||||
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
||||
|
||||
@@ -89,7 +89,9 @@ def set_config(
|
||||
before=None if s.secret else {"value": _redact(old)},
|
||||
after=None if s.secret else {"value": _redact(new)},
|
||||
)
|
||||
db.commit()
|
||||
# The override is committed even when the trail skips it: re-saving the value in force still
|
||||
# pins it as an override, and an uncommitted write would be dropped with the session.
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ The aggregate daily budget lives in ApiQuotaUsage; here we read the per-spend au
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -14,6 +14,7 @@ from app import quota
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import QuotaEvent, User
|
||||
from app.routes._keyset import utc
|
||||
|
||||
router = APIRouter(prefix="/api/quota", tags=["quota"])
|
||||
|
||||
@@ -45,23 +46,34 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
|
||||
@router.get("/admin")
|
||||
def admin_usage(
|
||||
days: int = 30,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
days = max(1, min(days, 365))
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
"""Spend per account (and per action) and per day, between `since` (default: 30 days back) and
|
||||
`until` (default: now), optionally for one account (`none` = the background)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
lo = utc(since) or now - timedelta(days=30)
|
||||
window = [QuotaEvent.created_at >= lo]
|
||||
if until is not None:
|
||||
window.append(QuotaEvent.created_at < utc(until))
|
||||
if user is not None:
|
||||
window.append(
|
||||
QuotaEvent.user_id.is_(None) if user == "none" else QuotaEvent.user_id == int(user)
|
||||
)
|
||||
|
||||
totals = db.execute(
|
||||
select(QuotaEvent.user_id, func.sum(QuotaEvent.units))
|
||||
.where(QuotaEvent.created_at >= since)
|
||||
.where(*window)
|
||||
.group_by(QuotaEvent.user_id)
|
||||
).all()
|
||||
|
||||
per_action: dict[int | None, dict[str, int]] = {}
|
||||
for uid, action, units in db.execute(
|
||||
select(QuotaEvent.user_id, QuotaEvent.action, func.sum(QuotaEvent.units))
|
||||
.where(QuotaEvent.created_at >= since)
|
||||
.where(*window)
|
||||
.group_by(QuotaEvent.user_id, QuotaEvent.action)
|
||||
).all():
|
||||
per_action.setdefault(uid, {})[action] = int(units)
|
||||
@@ -84,16 +96,16 @@ def admin_usage(
|
||||
]
|
||||
rows.sort(key=lambda r: r["total"], reverse=True)
|
||||
|
||||
# Instance-wide daily series in Pacific days (matches the budget reset).
|
||||
# The daily series in Pacific days (matches the budget reset), of the one account when filtered.
|
||||
day_expr = func.date(func.timezone("America/Los_Angeles", QuotaEvent.created_at))
|
||||
daily = [
|
||||
{"day": str(day), "total": int(units)}
|
||||
for day, units in db.execute(
|
||||
select(day_expr, func.sum(QuotaEvent.units))
|
||||
.where(QuotaEvent.created_at >= since)
|
||||
.where(*window)
|
||||
.group_by(day_expr)
|
||||
.order_by(day_expr)
|
||||
).all()
|
||||
]
|
||||
|
||||
return {"range_days": days, "rows": rows, "daily": daily}
|
||||
return {"rows": rows, "daily": daily}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Admin background-run history: read `job_runs`, a page at a time.
|
||||
|
||||
Admin-only and read-only — rows leave the table through the retention job
|
||||
(`job_run_retention_days`). Filtered on the server and paged by `_keyset`, like the server log and
|
||||
the audit log beside it on the activity page. A row names what a reader needs without a second
|
||||
request: the channel a user's ad-hoc run worked on, and how many log lines the run left (only
|
||||
WARNING and up are stored, so most runs have none — the count says so before the click).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import scheduler
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import Channel, JobRun, LogEvent, User
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.runstatus import RunStatus
|
||||
from app.utils import iso, like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/runs", tags=["admin"])
|
||||
|
||||
# `none` selects the runs no account started (the scheduled ones).
|
||||
NONE = "none"
|
||||
Facet = Literal["job", "status"]
|
||||
|
||||
|
||||
def _serialize(run: JobRun, titles: dict[str, str], lines: dict[int, int]) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"job": run.job,
|
||||
"trigger": run.trigger,
|
||||
"actor_id": run.actor_id,
|
||||
"target": run.target,
|
||||
"target_title": titles.get(run.target) if run.target is not None else None,
|
||||
"status": run.status,
|
||||
"started_at": iso(run.started_at),
|
||||
"finished_at": iso(run.finished_at),
|
||||
"summary": run.summary,
|
||||
"result": run.result,
|
||||
"error": run.error,
|
||||
"failure": run.failure,
|
||||
"quota_units": run.quota_units,
|
||||
"log_lines": lines.get(run.id, 0),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunFilters:
|
||||
job: str | None
|
||||
status: RunStatus | None
|
||||
actor: str | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Facet | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed. `skip` leaves one menu's own filter out, so that menu counts
|
||||
its values under every other filter (the way back from a pick)."""
|
||||
out = []
|
||||
if self.job is not None and skip != "job":
|
||||
out.append(JobRun.job == self.job)
|
||||
if self.status is not None and skip != "status":
|
||||
out.append(JobRun.status == self.status)
|
||||
if self.actor is not None:
|
||||
out.append(
|
||||
JobRun.actor_id.is_(None)
|
||||
if self.actor == NONE
|
||||
else JobRun.actor_id == int(self.actor)
|
||||
)
|
||||
if self.since is not None:
|
||||
out.append(JobRun.started_at >= utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(JobRun.started_at < utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
JobRun.job.ilike(like),
|
||||
JobRun.target.ilike(like),
|
||||
JobRun.summary.ilike(like),
|
||||
JobRun.error.ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run_filters(
|
||||
job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"),
|
||||
status: RunStatus | None = None,
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> RunFilters:
|
||||
return RunFilters(job, status, user, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_runs(
|
||||
filters: RunFilters = Depends(run_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""One page of runs, newest first, matching every filter given; `before` continues from a
|
||||
previous page's `next_cursor`."""
|
||||
query = select(JobRun).where(*filters.conditions())
|
||||
if before is not None:
|
||||
query = query.where(older_than(JobRun.started_at, JobRun.id, before))
|
||||
order = (JobRun.started_at.desc(), JobRun.id.desc())
|
||||
runs, cursor = page(
|
||||
list(db.execute(query.order_by(*order).limit(limit + 1)).scalars().all()),
|
||||
limit,
|
||||
lambda r: r.started_at,
|
||||
)
|
||||
channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target}
|
||||
titles: dict[str, str] = {}
|
||||
if channels:
|
||||
rows = db.execute(select(Channel.id, Channel.title).where(Channel.id.in_(channels)))
|
||||
titles = {cid: title for cid, title in rows.tuples() if title is not None}
|
||||
lines: dict[int, int] = {}
|
||||
if runs:
|
||||
counted = db.execute(
|
||||
select(LogEvent.run_id, func.count())
|
||||
.where(LogEvent.run_id.in_([r.id for r in runs]))
|
||||
.group_by(LogEvent.run_id)
|
||||
)
|
||||
lines = {rid: n for rid, n in counted.tuples() if rid is not None}
|
||||
return {
|
||||
"items": [_serialize(r, titles, lines) for r in runs],
|
||||
"accounts": accounts(db, {r.actor_id for r in runs if r.actor_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def run_facets(
|
||||
filters: RunFilters = Depends(run_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""What the job menu and the status chips offer, each counted under every filter but its own."""
|
||||
|
||||
def counts(col, skip: Facet) -> list[dict]:
|
||||
n = func.count().label("n")
|
||||
stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
|
||||
return [{"value": v, "count": c} for v, c in db.execute(stmt).all()]
|
||||
|
||||
total = select(func.count()).select_from(JobRun).where(*filters.conditions("status"))
|
||||
return {
|
||||
"jobs": counts(JobRun.job, "job"),
|
||||
"statuses": counts(JobRun.status, "status"),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
"""The activity page's reads: the audit log and the background runs, filtered on the server and
|
||||
paged by a keyset, and the quota breakdown over a time range and one account.
|
||||
|
||||
Each list is asked for the rows a filter must keep AND must drop, and a page boundary is crossed
|
||||
with rows that share a timestamp — the case an offset or a timestamp-only cursor gets wrong.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from app.models import AuditLog, Channel, JobRun, LogEvent, QuotaEvent
|
||||
|
||||
T0 = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _audit(db, action, actor=None, at=T0, summary=None, request_id=None) -> int:
|
||||
row = AuditLog(
|
||||
actor_id=actor, action=action, summary=summary, created_at=at, request_id=request_id
|
||||
)
|
||||
db.add(row)
|
||||
db.flush()
|
||||
return row.id
|
||||
|
||||
|
||||
def _run(db, job, status="ok", actor=None, at=T0, target=None, trigger="scheduled") -> int:
|
||||
row = JobRun(
|
||||
job=job,
|
||||
trigger=trigger,
|
||||
actor_id=actor,
|
||||
target=target,
|
||||
status=status,
|
||||
started_at=at,
|
||||
finished_at=None if status == "running" else at + timedelta(seconds=4),
|
||||
quota_units=0,
|
||||
)
|
||||
db.add(row)
|
||||
db.flush()
|
||||
return row.id
|
||||
|
||||
|
||||
def _ids(body) -> list[int]:
|
||||
return [r["id"] for r in body["items"]]
|
||||
|
||||
|
||||
def test_the_audit_list_filters_on_the_server_and_pages_across_a_shared_timestamp(
|
||||
db, make_user, as_user
|
||||
):
|
||||
admin = make_user(role="admin")
|
||||
a = _audit(db, "config_set", admin.id, T0, "quota_daily_budget = 9000")
|
||||
b = _audit(db, "config_set", admin.id, T0, "log_level = DEBUG")
|
||||
c = _audit(db, "config_set", admin.id, T0, "smtp_host = mail")
|
||||
bg = _audit(db, "user_purged", None, T0 - timedelta(days=3))
|
||||
db.commit()
|
||||
client = as_user(admin)
|
||||
|
||||
first = client.get("/api/admin/audit", params={"limit": 2, "action": "config_set"}).json()
|
||||
assert _ids(first) == [c, b]
|
||||
second = client.get(
|
||||
"/api/admin/audit",
|
||||
params={"limit": 2, "action": "config_set", "before": first["next_cursor"]},
|
||||
).json()
|
||||
assert (_ids(second), second["next_cursor"]) == ([a], None)
|
||||
# A page that ends exactly at the last row says there is nothing older.
|
||||
whole = client.get("/api/admin/audit", params={"limit": 3, "action": "config_set"}).json()
|
||||
assert (_ids(whole), whole["next_cursor"]) == ([c, b, a], None)
|
||||
assert first["accounts"] == {str(admin.id): admin.email}
|
||||
|
||||
def ids(**params) -> list[int]:
|
||||
return _ids(client.get("/api/admin/audit", params=params).json())
|
||||
|
||||
assert ids(user="none") == [bg]
|
||||
assert ids(user=str(admin.id)) == [c, b, a]
|
||||
assert ids(q="DEBUG") == [b]
|
||||
assert ids(since=(T0 - timedelta(days=1)).isoformat()) == [c, b, a]
|
||||
assert ids(until=(T0 - timedelta(days=1)).isoformat()) == [bg]
|
||||
# The action menu counts every action under the other filters, its own pick included.
|
||||
facets = client.get("/api/admin/audit/facets", params={"action": "config_set"}).json()
|
||||
assert facets["actions"] == [
|
||||
{"value": "config_set", "count": 3},
|
||||
{"value": "user_purged", "count": 1},
|
||||
]
|
||||
|
||||
|
||||
def test_an_admin_action_keeps_the_request_it_was_made_in(db, make_user, as_user):
|
||||
client = as_user(make_user(role="admin"))
|
||||
r = client.patch("/api/admin/config/quota_daily_budget", json={"value": 9500})
|
||||
assert r.status_code == 200, r.text
|
||||
(row,) = client.get("/api/admin/audit", params={"action": "config_set"}).json()["items"]
|
||||
assert row["request_id"] == r.headers["x-request-id"]
|
||||
|
||||
|
||||
def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, make_user, as_user):
|
||||
admin = make_user(role="admin")
|
||||
db.add(Channel(id="UCchan", title="Aileen Wang"))
|
||||
ok1 = _run(db, "rss_poll", at=T0)
|
||||
ok2 = _run(db, "rss_poll", at=T0)
|
||||
bad = _run(db, "backfill", "error", admin.id, T0, trigger="manual")
|
||||
adhoc = _run(
|
||||
db, "channel_backfill", "ok", admin.id, T0 - timedelta(hours=1), "UCchan", "ad_hoc"
|
||||
)
|
||||
for _ in range(2):
|
||||
db.add(
|
||||
LogEvent(ts=T0, level="WARNING", sev=30, source="api", logger="t", msg="m", run_id=bad)
|
||||
)
|
||||
db.commit()
|
||||
client = as_user(admin)
|
||||
|
||||
first = client.get("/api/admin/runs", params={"limit": 2}).json()
|
||||
rest = client.get("/api/admin/runs", params={"limit": 2, "before": first["next_cursor"]}).json()
|
||||
assert _ids(first) + _ids(rest) == [bad, ok2, ok1, adhoc]
|
||||
assert rest["next_cursor"] is None
|
||||
rows = {r["id"]: r for r in first["items"] + rest["items"]}
|
||||
assert (rows[bad]["log_lines"], rows[ok2]["log_lines"]) == (2, 0)
|
||||
assert (rows[adhoc]["target_title"], rows[ok1]["target_title"]) == ("Aileen Wang", None)
|
||||
|
||||
def ids(**params) -> list[int]:
|
||||
return _ids(client.get("/api/admin/runs", params=params).json())
|
||||
|
||||
assert ids(status="error") == [bad]
|
||||
assert ids(job="rss_poll") == [ok2, ok1]
|
||||
assert ids(user="none") == [ok2, ok1]
|
||||
assert ids(user=str(admin.id)) == [bad, adhoc]
|
||||
assert ids(q="UCchan") == [adhoc]
|
||||
assert ids(since=(T0 - timedelta(minutes=30)).isoformat()) == [bad, ok2, ok1]
|
||||
assert client.get("/api/admin/runs", params={"status": "finished"}).status_code == 422
|
||||
|
||||
facets = client.get("/api/admin/runs/facets", params={"status": "error"}).json()
|
||||
assert facets["statuses"] == [{"value": "ok", "count": 3}, {"value": "error", "count": 1}]
|
||||
assert facets["jobs"] == [{"value": "backfill", "count": 1}]
|
||||
assert facets["total"] == 4
|
||||
|
||||
|
||||
def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_user):
|
||||
admin = make_user(role="admin")
|
||||
now = datetime.now(timezone.utc)
|
||||
db.add_all(
|
||||
[
|
||||
QuotaEvent(user_id=admin.id, action="search", units=100, created_at=now),
|
||||
QuotaEvent(user_id=None, action="backfill", units=7, created_at=now),
|
||||
QuotaEvent(
|
||||
user_id=admin.id, action="search", units=50, created_at=now - timedelta(days=10)
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
client = as_user(admin)
|
||||
|
||||
def totals(**params) -> dict:
|
||||
body = client.get("/api/quota/admin", params=params).json()
|
||||
return {r["user_id"]: r["total"] for r in body["rows"]}
|
||||
|
||||
assert totals() == {admin.id: 150, None: 7} # the last 30 days by default
|
||||
assert totals(since=(now - timedelta(days=1)).isoformat()) == {admin.id: 100, None: 7}
|
||||
assert totals(until=(now - timedelta(days=1)).isoformat()) == {admin.id: 50}
|
||||
assert totals(user="none") == {None: 7}
|
||||
assert totals(user=str(admin.id)) == {admin.id: 150}
|
||||
@@ -22,6 +22,7 @@ import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import SystemConfig
|
||||
from app.config import Settings, choices_of
|
||||
|
||||
CHOICE_KEYS = {s.key for s in sysconfig.SPECS if s.type == "choice"}
|
||||
@@ -180,3 +181,14 @@ def test_the_intervals_are_keys_of_the_scheduler_page_only():
|
||||
assert sysconfig.config_page_spec("job_interval_rss_poll") is None
|
||||
assert sysconfig.config_page_spec("quota_daily_budget") is not None
|
||||
assert "scheduler" not in sysconfig.describe(_FakeDb())["groups"]
|
||||
|
||||
|
||||
def test_saving_the_value_in_force_still_stores_the_override(db, make_user, as_user):
|
||||
client = as_user(make_user(role="admin"))
|
||||
default = sysconfig.get_int(db, "quota_daily_budget")
|
||||
r = client.patch("/api/admin/config/quota_daily_budget", json={"value": default})
|
||||
assert r.status_code == 200, r.text
|
||||
(item,) = [i for i in r.json()["groups"]["quota"] if i["key"] == "quota_daily_budget"]
|
||||
assert item["is_set"] is True
|
||||
db.expire_all()
|
||||
assert db.get(SystemConfig, "quota_daily_budget") is not None
|
||||
|
||||
Reference in New Issue
Block a user