siftlode/backend/app/routes/admin_logs.py

"""The admin diagnostics console's read side: query `log_events`, the server's stored log.

Admin-only and read-only — rows leave the table through the retention job (`log_retention_days`),
never through here. Reading is not audited: it changes nothing, and the console's live tail polls,
so a row per read would bury the audit log's real entries.

What a row holds was redacted when it was written (`logs.record_fields`, and the ingest route for a
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
`[redacted]` / `[email]`, which is the point — the console shows that something was there.

Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent
window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds
new ones at the top. The live tail re-reads the first page rather than asking for "ids after N":
several processes write this table, and a batch's ids are taken when it inserts, not when it
commits, so a slower batch can land below an id the tail already passed — and would never show.
"""

import base64
import binascii
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Literal

from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import Text, and_, cast, func, or_, select
from sqlalchemy.orm import Session

from app import logsink
from app.auth import admin_user
from app.db import get_db
from app.models import LogEvent, User
from app.utils import like_escape

router = APIRouter(prefix="/api/admin/logs", tags=["admin"])

PAGE_MAX = 200
# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
# through the free-text search, and the response stays small whatever a self-host logs.
FACET_EVENTS_MAX = 100

Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
Facet = Literal["levels", "sources", "user", "event"]
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
# request) — a real value to filter on, not the absence of a filter.
NONE = "none"


def _utc(value: datetime | None) -> datetime | None:
    """A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read
    in the database session's zone instead."""
    if value is None or value.tzinfo is not None:
        return value
    return value.replace(tzinfo=timezone.utc)


def _encode_cursor(row: LogEvent) -> str:
    return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode()


def _decode_cursor(cursor: str) -> tuple[datetime, int]:
    try:
        ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
        return _utc(datetime.fromisoformat(ts)), int(row_id)
    except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
        raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc


def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
    """The accounts a page names, by id → address. An id with no account left (erased after the
    line was written) is simply absent, and the client shows the bare id."""
    if not user_ids:
        return {}
    rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
    return {str(uid): email for uid, email in rows}


@dataclass(frozen=True)
class LogFilters:
    """The console's filters, shared by the page and its facet counts."""

    levels: list[str]
    sources: list[str]
    user: str | None
    event: str | None
    request_id: str | None
    job_id: int | None
    run_id: int | None
    q: str | None
    since: datetime | None
    until: datetime | None

    def conditions(self, skip: Facet | None = None) -> list:
        """The WHERE conditions, ANDed by the caller. `skip` leaves one filter out: a facet counts
        its own values under every OTHER filter, so picking a level does not zero the other
        levels' counts (the way back), while a request pivot does narrow them (honest counts)."""
        out = []
        if self.levels and skip != "levels":
            out.append(LogEvent.level.in_(self.levels))
        if self.sources and skip != "sources":
            out.append(LogEvent.source.in_(self.sources))
        if self.user is not None and skip != "user":
            out.append(
                LogEvent.user_id.is_(None)
                if self.user == NONE
                else LogEvent.user_id == int(self.user)
            )
        if self.event is not None and skip != "event":
            out.append(
                LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
            )
        if self.request_id is not None:
            out.append(LogEvent.request_id == self.request_id)
        if self.job_id is not None:
            out.append(LogEvent.job_id == self.job_id)
        if self.run_id is not None:
            out.append(LogEvent.run_id == self.run_id)
        if self.since is not None:
            out.append(LogEvent.ts >= _utc(self.since))
        if self.until is not None:
            out.append(LogEvent.ts < _utc(self.until))
        if self.q and self.q.strip():
            like = f"%{like_escape(self.q.strip())}%"
            out.append(
                or_(
                    LogEvent.msg.ilike(like),
                    LogEvent.event.ilike(like),
                    LogEvent.request_id.ilike(like),
                    LogEvent.exc.ilike(like),
                    cast(LogEvent.attrs, Text).ilike(like),
                )
            )
        return out


def log_filters(
    levels: list[Level] = Query(default=[]),
    sources: list[str] = Query(default=[], max_length=8),
    user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
    event: str | None = Query(
        default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
    ),
    request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"),
    job_id: int | None = Query(default=None, ge=1),
    run_id: int | None = Query(default=None, ge=1),
    q: str | None = Query(default=None, max_length=200),
    since: datetime | None = None,
    until: datetime | None = None,
) -> LogFilters:
    return LogFilters(
        list(levels), list(sources), user, event, request_id, job_id, run_id, q, since, until
    )


@router.get("")
def list_logs(
    filters: LogFilters = Depends(log_filters),
    before: str | None = Query(default=None, max_length=200),
    limit: int = Query(default=100, ge=1, le=PAGE_MAX),
    _: User = Depends(admin_user),
    db: Session = Depends(get_db),
) -> dict:
    """One page of rows, newest first, matching every filter given (they AND together); `before`
    continues from a previous page's `next_cursor`."""
    query = select(LogEvent).where(*filters.conditions())
    if before is not None:
        ts, row_id = _decode_cursor(before)
        # The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the
        # `ts` index scan instead of filtering every newer row first.
        query = query.where(
            and_(
                LogEvent.ts <= ts,
                or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)),
            )
        )

    # One more than asked, to know whether there is a next page without a second count query.
    rows = (
        db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
        .scalars()
        .all()
    )
    more = len(rows) > limit
    rows = rows[:limit]
    return {
        "items": [logsink.serialize(r) for r in rows],
        "accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
        "next_cursor": _encode_cursor(rows[-1]) if more else None,
    }


@router.get("/facets")
def log_facets(
    filters: LogFilters = Depends(log_filters),
    _: User = Depends(admin_user),
    db: Session = Depends(get_db),
) -> dict:
    """What the filter chips offer, with counts: the levels, sources and accounts present, and the
    most frequent events — each counted under every filter but its own (`LogFilters.conditions`).
    `total` is the "all levels" count: every filter but the level one."""

    def counts(col, skip: Facet, cap: int | None = None) -> list[dict]:
        n = func.count().label("n")
        stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
        if cap is not None:
            stmt = stmt.limit(cap)
        return [
            {"value": NONE if value is None else value, "count": count}
            for value, count in db.execute(stmt).all()
        ]

    users = counts(LogEvent.user_id, "user")
    total = select(func.count()).select_from(LogEvent).where(*filters.conditions("levels"))
    return {
        "levels": counts(LogEvent.level, "levels"),
        "sources": counts(LogEvent.source, "sources"),
        "events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
        "users": [{**u, "value": str(u["value"])} for u in users],
        "accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}),
        "total": db.scalar(total) or 0,
    }