siftlode/backend/app/routes/admin_logs.py

"""The admin diagnostics console's read side: query `log_events`, the server's stored log.

Admin-only and read-only — rows leave the table through the retention job (`log_retention_days`),
never through here. Reading is not audited: it changes nothing, and the console's live tail polls,
so a row per read would bury the audit log's real entries.

What a row holds was redacted when it was written (`logs.record_fields`, and the ingest route for a
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
`[redacted]` / `[email]`, which is the point — the console shows that something was there.

Paging is a keyset over `(ts desc, id desc)` (`_keyset`). The live tail re-reads the first page rather than asking for "ids after N":
several processes write this table, and a batch's ids are taken when it inserts, not when it
commits, so a slower batch can land below an id the tail already passed — and would never show.
"""

from dataclasses import dataclass
from datetime import datetime
from typing import Literal

from fastapi import APIRouter, Depends, Query
from sqlalchemy import Text, cast, func, or_, select
from sqlalchemy.orm import Session

from app import logsink
from app.auth import admin_user
from app.db import get_db
from app.models import LogEvent, User
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
from app.utils import like_escape

router = APIRouter(prefix="/api/admin/logs", tags=["admin"])

# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
# through the free-text search, and the response stays small whatever a self-host logs.
FACET_EVENTS_MAX = 100

Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
Facet = Literal["levels", "sources", "user", "event"]
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
# request) — a real value to filter on, not the absence of a filter.
NONE = "none"


@dataclass(frozen=True)
class LogFilters:
    """The console's filters, shared by the page and its facet counts."""

    levels: list[str]
    sources: list[str]
    user: str | None
    event: str | None
    request_id: str | None
    job_id: int | None
    run_id: int | None
    q: str | None
    since: datetime | None
    until: datetime | None

    def conditions(self, skip: Facet | None = None) -> list:
        """The WHERE conditions, ANDed by the caller. `skip` leaves one filter out: a facet counts
        its own values under every OTHER filter, so picking a level does not zero the other
        levels' counts (the way back), while a request pivot does narrow them (honest counts)."""
        out = []
        if self.levels and skip != "levels":
            out.append(LogEvent.level.in_(self.levels))
        if self.sources and skip != "sources":
            out.append(LogEvent.source.in_(self.sources))
        if self.user is not None and skip != "user":
            out.append(
                LogEvent.user_id.is_(None)
                if self.user == NONE
                else LogEvent.user_id == int(self.user)
            )
        if self.event is not None and skip != "event":
            out.append(
                LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
            )
        if self.request_id is not None:
            out.append(LogEvent.request_id == self.request_id)
        if self.job_id is not None:
            out.append(LogEvent.job_id == self.job_id)
        if self.run_id is not None:
            out.append(LogEvent.run_id == self.run_id)
        if self.since is not None:
            out.append(LogEvent.ts >= utc(self.since))
        if self.until is not None:
            out.append(LogEvent.ts < utc(self.until))
        if self.q and self.q.strip():
            like = f"%{like_escape(self.q.strip())}%"
            out.append(
                or_(
                    LogEvent.msg.ilike(like),
                    LogEvent.event.ilike(like),
                    LogEvent.request_id.ilike(like),
                    LogEvent.exc.ilike(like),
                    cast(LogEvent.attrs, Text).ilike(like),
                )
            )
        return out


def log_filters(
    levels: list[Level] = Query(default=[]),
    sources: list[str] = Query(default=[], max_length=8),
    user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
    event: str | None = Query(
        default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
    ),
    request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"),
    job_id: int | None = Query(default=None, ge=1),
    run_id: int | None = Query(default=None, ge=1),
    q: str | None = Query(default=None, max_length=200),
    since: datetime | None = None,
    until: datetime | None = None,
) -> LogFilters:
    return LogFilters(
        list(levels), list(sources), user, event, request_id, job_id, run_id, q, since, until
    )


@router.get("")
def list_logs(
    filters: LogFilters = Depends(log_filters),
    before: str | None = Query(default=None, max_length=200),
    limit: int = Query(default=100, ge=1, le=PAGE_MAX),
    _: User = Depends(admin_user),
    db: Session = Depends(get_db),
) -> dict:
    """One page of rows, newest first, matching every filter given (they AND together); `before`
    continues from a previous page's `next_cursor`."""
    query = select(LogEvent).where(*filters.conditions())
    if before is not None:
        query = query.where(older_than(LogEvent.ts, LogEvent.id, before))
    rows, cursor = page(
        list(
            db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
            .scalars()
            .all()
        ),
        limit,
        lambda r: r.ts,
    )
    return {
        "items": [logsink.serialize(r) for r in rows],
        "accounts": accounts(db, {r.user_id for r in rows if r.user_id is not None}),
        "next_cursor": cursor,
    }


@router.get("/facets")
def log_facets(
    filters: LogFilters = Depends(log_filters),
    _: User = Depends(admin_user),
    db: Session = Depends(get_db),
) -> dict:
    """What the filter chips offer, with counts: the levels, sources and accounts present, and the
    most frequent events — each counted under every filter but its own (`LogFilters.conditions`).
    `total` is the "all levels" count: every filter but the level one."""

    def counts(col, skip: Facet, cap: int | None = None) -> list[dict]:
        n = func.count().label("n")
        stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
        if cap is not None:
            stmt = stmt.limit(cap)
        return [
            {"value": NONE if value is None else value, "count": count}
            for value, count in db.execute(stmt).all()
        ]

    users = counts(LogEvent.user_id, "user")
    total = select(func.count()).select_from(LogEvent).where(*filters.conditions("levels"))
    return {
        "levels": counts(LogEvent.level, "levels"),
        "sources": counts(LogEvent.source, "sources"),
        "events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
        "users": [{**u, "value": str(u["value"])} for u in users],
        "accounts": accounts(db, {u["value"] for u in users if u["value"] != NONE}),
        "total": db.scalar(total) or 0,
    }