diff --git a/.env.example b/.env.example index 376f9315..3875dac1 100644 --- a/.env.example +++ b/.env.example @@ -68,9 +68,11 @@ SMTP_FROM= # --- Scheduler --- # The background scheduler (subscription sync, backfill, enrichment) runs inside the app. -# DATABASE_URL is set for you by docker-compose to the bundled `db` service. If you ever run -# more than one instance against the same database, keep SCHEDULER_ENABLED=true on exactly one -# of them and false on the rest, to avoid double quota use and write races. +# DATABASE_URL is set for you by docker-compose to the bundled `db` service. Run ONE app instance +# per database: a starting instance closes every run still open as interrupted (it assumes the +# previous process is gone), and "is this job already running" is held in the process. A second +# instance would cut the first one's runs short and could run the same job twice. +# SCHEDULER_ENABLED=false keeps the recurring jobs off; "run now" still works. SCHEDULER_ENABLED=true # --- Files library & downloads --- diff --git a/backend/alembic/versions/0109_job_runs.py b/backend/alembic/versions/0109_job_runs.py index c642a720..1def28b4 100644 --- a/backend/alembic/versions/0109_job_runs.py +++ b/backend/alembic/versions/0109_job_runs.py @@ -12,7 +12,8 @@ log writer runs behind the event). Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the -audit log. The downgrade moves every run back into the audit log as such a row, then drops the table. +audit log. The downgrade moves every run back into the audit log as such a row (one still running as +interrupted, a run's trigger and target in its `after`), then drops the table. """ import sqlalchemy as sa @@ -70,15 +71,19 @@ def upgrade() -> None: postgresql_where=sa.text("run_id IS NOT NULL"), ) - # The audit trail's runs move over. A status the run table does not know (none was ever - # written) would fail the CHECK, so the move is limited to the three the scheduler wrote. + # The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`, + # which a downgrade writes back (as it does a run's `trigger` and `target`, which the audit row + # has no column for), so a rollback and a redeploy keep every run. A status the run table does + # not know would fail the CHECK. op.execute( """ - INSERT INTO job_runs (job, trigger, actor_id, status, started_at, finished_at, summary, - result, error, failure) + INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at, + summary, result, error, failure) SELECT target_id, - CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END, + coalesce(after::jsonb ->> 'trigger', + CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END), actor_id, + after::jsonb ->> 'target', after::jsonb ->> 'status', created_at, created_at, @@ -89,7 +94,7 @@ def upgrade() -> None: FROM audit_log WHERE action = 'job_run' AND target_id IS NOT NULL - AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped') + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted') ORDER BY id """ ) @@ -98,12 +103,15 @@ def upgrade() -> None: DELETE FROM audit_log WHERE action = 'job_run' AND target_id IS NOT NULL - AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped') + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted') """ ) def downgrade() -> None: + # A run still `running` is one the rollback stopped: it goes back as interrupted. Its trigger and + # target ride in `after` when the audit row's own columns cannot say them (an ad-hoc run of one + # channel), so the upgrade restores them. op.execute( """ INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at) @@ -111,21 +119,26 @@ def downgrade() -> None: 'job_run', 'job', job, - left(job || ': ' || status - || coalesce(' — ' || CASE WHEN status = 'error' THEN error ELSE summary END, ''), + left(job || ': ' || st + || coalesce(' — ' || CASE WHEN st = 'error' THEN error ELSE summary END, ''), 255), (jsonb_build_object( - 'status', status, - 'summary', CASE WHEN status = 'error' THEN error ELSE summary END) + 'status', st, + 'summary', CASE WHEN st = 'error' THEN error ELSE summary END) || CASE - WHEN status = 'error' AND failure IS NOT NULL + WHEN st = 'error' AND failure IS NOT NULL THEN jsonb_build_object('result', failure) - WHEN status <> 'error' AND result IS NOT NULL + WHEN st <> 'error' AND result IS NOT NULL THEN jsonb_build_object('result', result) ELSE '{}'::jsonb - END)::json, + END + || CASE WHEN trigger = 'ad_hoc' + THEN jsonb_build_object('trigger', trigger) ELSE '{}'::jsonb END + || CASE WHEN target IS NOT NULL + THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json, coalesce(finished_at, started_at) - FROM job_runs + FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st + FROM job_runs) AS r ORDER BY id """ ) diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py index 18645366..a51249ae 100644 --- a/backend/app/jobruns.py +++ b/backend/app/jobruns.py @@ -5,13 +5,14 @@ OWN short session, committed at once, for the reason `quota.record_usage` gives: may roll back (its `except: db.rollback()`), and the history of what happened must not go with it — nor may this bookkeeping commit the job's half-done session. -Only the API process runs jobs (the worker never starts the scheduler), so at its startup nothing can -still be running: `interrupt_leftovers` closes what a stopped process left open. +Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a +database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its +startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open. """ from datetime import datetime, timedelta, timezone -from sqlalchemy import delete, func, select, update +from sqlalchemy import delete, func, select, union_all, update from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session @@ -70,30 +71,46 @@ def finish( def interrupt_leftovers() -> int: - """Every row a previous process left `running` → `interrupted`. Called once at startup, before - the scheduler can open a new run. Returns how many.""" + """Every row a previous process left `running` → `interrupted`, with what it spent until then + (its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler + can open a new run. Returns how many.""" + spent = ( + select(func.coalesce(func.sum(QuotaEvent.units), 0)) + .where(QuotaEvent.job_run_id == JobRun.id) + .scalar_subquery() + ) with SessionLocal() as s: res = s.execute( update(JobRun) .where(JobRun.status == RunStatus.RUNNING) - .values(status=RunStatus.INTERRUPTED, finished_at=datetime.now(timezone.utc)) + .values( + status=RunStatus.INTERRUPTED, + finished_at=datetime.now(timezone.utc), + quota_units=spent, + ) ) s.commit() return res.rowcount if isinstance(res, CursorResult) else 0 -def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]: - """Each job's last `per_job` runs, newest first, by job id — one query (a window per job).""" - ranked = select( - JobRun.id, - func.row_number() - .over(partition_by=JobRun.job, order_by=(JobRun.started_at.desc(), JobRun.id.desc())) - .label("rank"), +def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]: + """Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a + short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window + over the whole table grew with every run kept).""" + if not jobs: + return {} + newest = union_all( + *( + select(JobRun.id) + .where(JobRun.job == job) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(per_job) + for job in jobs + ) ).subquery() rows = db.execute( select(JobRun) - .join(ranked, ranked.c.id == JobRun.id) - .where(ranked.c.rank <= per_job) + .where(JobRun.id.in_(select(newest.c.id))) .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc()) ).scalars() out: dict[str, list[JobRun]] = {} @@ -102,6 +119,16 @@ def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]: return out +def running_one(db: Session, job: str) -> JobRun | None: + """The newest run of `job` still going, however many runs ended after it started.""" + return db.scalars( + select(JobRun) + .where(JobRun.job == job, JobRun.status == RunStatus.RUNNING) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(1) + ).first() + + def gc(db: Session, retention_days: int) -> int: """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is never reaped. The caller commits.""" diff --git a/backend/app/models.py b/backend/app/models.py index d9914a0f..9ebfbe74 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -720,7 +720,7 @@ class JobRun(Base): CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"), # Only a run still going has no end. CheckConstraint( - "(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished" + f"(status = '{RunStatus.RUNNING}') = (finished_at IS NULL)", name="ck_job_runs_finished" ), ) diff --git a/backend/app/routes/_account.py b/backend/app/routes/_account.py new file mode 100644 index 00000000..84976e86 --- /dev/null +++ b/backend/app/routes/_account.py @@ -0,0 +1,17 @@ +"""The account filter the activity page's lists share (runs, admin actions, server log, quota): one +filter bar drives them all, so the value it sends means the same on every tab. + +The value is an account id, or `none` — the rows no account made (the background) — a value to filter +on, not the absence of a filter. +""" + +from sqlalchemy import ColumnElement + +NONE = "none" +# The `user` query parameter's shape on every route that takes it. +PATTERN = r"^(none|[0-9]{1,10})$" + + +def condition(column, value: str) -> ColumnElement[bool]: + """`column` (an account id column) narrowed to `value`.""" + return column.is_(None) if value == NONE else column == int(value) diff --git a/backend/app/routes/admin_logs.py b/backend/app/routes/admin_logs.py index 6ec025cf..72a36631 100644 --- a/backend/app/routes/admin_logs.py +++ b/backend/app/routes/admin_logs.py @@ -25,6 +25,7 @@ from app import logsink from app.auth import admin_user from app.db import get_db from app.models import LogEvent, User +from app.routes import _account as account from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.utils import like_escape @@ -37,9 +38,9 @@ FACET_EVENTS_MAX = 100 Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # The filters a facet can leave out of its own count (`LogFilters.conditions`). Facet = Literal["levels", "sources", "user", "event"] -# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous -# request) — a real value to filter on, not the absence of a filter. -NONE = "none" +# `none` selects the rows that have no event code, as the account filter's `none` (app/routes/_account) +# selects the rows of no account — a real value to filter on, not the absence of a filter. +NONE = account.NONE @dataclass(frozen=True) @@ -67,11 +68,7 @@ class LogFilters: if self.sources and skip != "sources": out.append(LogEvent.source.in_(self.sources)) if self.user is not None and skip != "user": - out.append( - LogEvent.user_id.is_(None) - if self.user == NONE - else LogEvent.user_id == int(self.user) - ) + out.append(account.condition(LogEvent.user_id, self.user)) if self.event is not None and skip != "event": out.append( LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event @@ -103,7 +100,7 @@ class LogFilters: def log_filters( levels: list[Level] = Query(default=[]), sources: list[str] = Query(default=[], max_length=8), - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), event: str | None = Query( default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$" ), diff --git a/backend/app/routes/audit.py b/backend/app/routes/audit.py index bc0c8986..91e82c78 100644 --- a/backend/app/routes/audit.py +++ b/backend/app/routes/audit.py @@ -15,17 +15,15 @@ from app import audit from app.audit import AuditAction from app.auth import admin_user from app.db import get_db -from app.models import AuditLog, User +from app.models import AuditLog, LogEvent, User +from app.routes import _account as account from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.utils import iso, like_escape router = APIRouter(prefix="/api/admin/audit", tags=["admin"]) -# `none` selects the rows no account made (the background) — a value to filter on, not "any". -NONE = "none" - -def _serialize(row: AuditLog) -> dict: +def _serialize(row: AuditLog, lines: dict[str, int]) -> dict: return { "id": row.id, "created_at": iso(row.created_at), @@ -38,6 +36,9 @@ def _serialize(row: AuditLog) -> dict: "before": row.before, "after": row.after, "request_id": row.request_id, + # How many of that request's log lines are still kept: they go after days, the row after + # months, and a link to none would open an empty list. + "log_lines": lines.get(row.request_id, 0) if row.request_id else 0, } @@ -56,11 +57,7 @@ class AuditFilters: if self.action is not None and skip != "action": out.append(AuditLog.action == self.action) if self.actor is not None: - out.append( - AuditLog.actor_id.is_(None) - if self.actor == NONE - else AuditLog.actor_id == int(self.actor) - ) + out.append(account.condition(AuditLog.actor_id, self.actor)) if self.since is not None: out.append(AuditLog.created_at >= utc(self.since)) if self.until is not None: @@ -82,7 +79,7 @@ class AuditFilters: def audit_filters( action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"), - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), q: str | None = Query(default=None, max_length=200), since: datetime | None = None, until: datetime | None = None, @@ -109,8 +106,17 @@ def list_audit( limit, lambda r: r.created_at, ) + requests = {r.request_id for r in rows if r.request_id} + lines: dict[str, int] = {} + if requests: + counted = db.execute( + select(LogEvent.request_id, func.count()) + .where(LogEvent.request_id.in_(requests)) + .group_by(LogEvent.request_id) + ) + lines = {rid: n for rid, n in counted.tuples() if rid is not None} return { - "items": [_serialize(r) for r in rows], + "items": [_serialize(r, lines) for r in rows], "accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}), "next_cursor": cursor, } diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index d0856c94..3c1e753c 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -512,7 +512,9 @@ def update_channel( # so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. # The request returns right away; the channel page's live poll shows the count climb. The # scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set). - if deep_turned_on: + # A channel whose history is all in gets no run: it would only say "nothing to do", one kept row + # per flip of the flag. + if deep_turned_on and not (channel is not None and channel.backfill_done): scheduler.run_ad_hoc( scheduler.CHANNEL_BACKFILL, partial(backfill_channel_now, channel_id, user.id), diff --git a/backend/app/routes/quota.py b/backend/app/routes/quota.py index 1f4800c7..98df6d07 100644 --- a/backend/app/routes/quota.py +++ b/backend/app/routes/quota.py @@ -14,6 +14,7 @@ from app import quota from app.auth import admin_user, current_user from app.db import get_db from app.models import QuotaEvent, User +from app.routes import _account as account from app.routes._keyset import utc router = APIRouter(prefix="/api/quota", tags=["quota"]) @@ -48,7 +49,7 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) def admin_usage( since: datetime | None = None, until: datetime | None = None, - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), _: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: @@ -60,9 +61,7 @@ def admin_usage( if until is not None: window.append(QuotaEvent.created_at < utc(until)) if user is not None: - window.append( - QuotaEvent.user_id.is_(None) if user == "none" else QuotaEvent.user_id == int(user) - ) + window.append(account.condition(QuotaEvent.user_id, user)) totals = db.execute( select(QuotaEvent.user_id, func.sum(QuotaEvent.units)) diff --git a/backend/app/routes/runs.py b/backend/app/routes/runs.py index c39fd244..c1bb53d6 100644 --- a/backend/app/routes/runs.py +++ b/backend/app/routes/runs.py @@ -19,14 +19,13 @@ from app import scheduler from app.auth import admin_user from app.db import get_db from app.models import Channel, JobRun, LogEvent, User +from app.routes import _account as account from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.runstatus import RunStatus from app.utils import iso, like_escape router = APIRouter(prefix="/api/admin/runs", tags=["admin"]) -# `none` selects the runs no account started (the scheduled ones). -NONE = "none" Facet = Literal["job", "status"] @@ -68,11 +67,7 @@ class RunFilters: if self.status is not None and skip != "status": out.append(JobRun.status == self.status) if self.actor is not None: - out.append( - JobRun.actor_id.is_(None) - if self.actor == NONE - else JobRun.actor_id == int(self.actor) - ) + out.append(account.condition(JobRun.actor_id, self.actor)) if self.since is not None: out.append(JobRun.started_at >= utc(self.since)) if self.until is not None: @@ -93,7 +88,7 @@ class RunFilters: def run_filters( job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"), status: RunStatus | None = None, - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), q: str | None = Query(default=None, max_length=200), since: datetime | None = None, until: datetime | None = None, diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index c61f3305..db0f5308 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -17,8 +17,9 @@ from app.scheduler import ( JOB_INTERVALS, MAX_INTERVAL, MIN_INTERVAL, - apply_interval, + reschedule, scheduler_snapshot, + stage_interval, trigger_all, trigger_job, ) @@ -42,14 +43,14 @@ def set_job_interval( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - """Change how often a scheduler job runs (minutes). Persisted as an override and applied - to the live scheduler immediately.""" + """Change how often a scheduler job runs (minutes). Stored as an override together with its + audit row, then applied to the live scheduler.""" if job_id not in JOB_INTERVALS: raise not_found("job") minutes = body.interval_minutes if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL): raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL) - applied = apply_interval(db, job_id, minutes) + applied = stage_interval(db, job_id, minutes) audit.record( db, admin.id, @@ -60,6 +61,7 @@ def set_job_interval( after={"interval_minutes": applied}, ) db.commit() + reschedule(job_id, applied) return {"ok": True, "id": job_id, "interval_minutes": applied} diff --git a/backend/app/runstatus.py b/backend/app/runstatus.py index 497a22b1..cba6c896 100644 --- a/backend/app/runstatus.py +++ b/backend/app/runstatus.py @@ -1,9 +1,11 @@ """The words a background run's row is written with — the `job_runs` table's status and trigger. One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the -model's CHECK constraints are built from these sets, and the dashboard reads them back. A run's -status is never typed as a string anywhere else (the download pipeline's `cancelled`/`canceled` -drift, `downloads/status.py`, is why). +model's CHECK constraints are built from these sets, and the dashboard reads them back (the frontend's +copies are held to them by `backendIds.contract.test.ts`). The migrated database's CHECKs are frozen +in its migration: a test holds them equal to these sets, so a new word fails until a migration adds +it (`test_runstatus.py`). The download pipeline's `cancelled`/`canceled` drift (`downloads/status.py`) +is why a status is never typed as a bare string. """ from enum import StrEnum diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index e528ed44..29904095 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -3,7 +3,7 @@ recent-first / deep backfill. Runs inside the API process (single worker).""" import contextvars import threading -from datetime import datetime, timezone +from datetime import timezone from typing import Callable from apscheduler.executors.pool import ThreadPoolExecutor @@ -27,6 +27,7 @@ from app.sync.autotag import run_autotag_all from app.sync.explore import purge_ephemeral from app.sync.maintenance import run_maintenance from app.sync.playlists import sync_all_playlists +from app.utils import iso from app.sync.runner import ( run_deep_backfill, run_enrich, @@ -67,18 +68,21 @@ def load_intervals(db) -> dict[str, int]: return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS} -def apply_interval(db, job_id: str, minutes: int) -> int: - """Persist a job's interval override and reschedule the live job immediately (if the - scheduler runs in this process). Returns the applied value.""" +def stage_interval(db, job_id: str, minutes: int) -> int: + """Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it + with its audit row, then calls `reschedule` — the live job follows only what was stored.""" if job_id not in JOB_INTERVALS: raise KeyError(job_id) minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes))) sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes) - db.commit() + return minutes + + +def reschedule(job_id: str, minutes: int) -> None: + """Move the live interval job to `minutes` (if this process registered it).""" if _scheduler is not None and _scheduler.get_job(job_id) is not None: _scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes) log.info("rescheduled job %s to every %s min", job_id, minutes) - return minutes def _record(name: str, **fields) -> None: @@ -87,7 +91,12 @@ def _record(name: str, **fields) -> None: def _notify_done( - db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None + db, + actor_id: int, + job_id: str, + status: RunStatus, + summary: str | None, + result: dict | None = None, ) -> None: """Post a completion notice to the triggering admin's inbox (manual runs only). Best effort — a notification failure must never mask the job's own outcome.""" @@ -139,24 +148,40 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None: def sink(current, total, phase): _record(name, progress={"current": current, "total": total, "phase": phase}) + # Only the work is inside the try: writing its outcome is not the job, and a write that fails + # must not turn a run that succeeded into a failed one (see _close). + result = None try: - if is_sync_paused(db): - log.info("job %s skipped (sync paused)", name) - jobruns.finish(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED) - if actor_id is not None: - _notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED) - return - with progress.bind(sink): - result = fn(db) - summary, data = _finish_ok(name, run_id, result) - if actor_id is not None: - _notify_done(db, actor_id, name, "ok", summary, data) + paused = is_sync_paused(db) + if not paused: + with progress.bind(sink): + result = fn(db) except Exception as exc: db.rollback() log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) - err, failure = _finish_error(run_id, exc) + err, failure = _failure(exc) + _close(run_id, RunStatus.ERROR, error=err, failure=failure) if actor_id is not None: - _notify_done(db, actor_id, name, "error", err, failure) + _notify_done(db, actor_id, name, RunStatus.ERROR, err, failure) + return + if paused: + log.info("job %s skipped (sync paused)", name) + _close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED) + if actor_id is not None: + _notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED) + return + summary, data = _finish_ok(name, run_id, result) + if actor_id is not None: + _notify_done(db, actor_id, name, RunStatus.OK, summary, data) + + +def _close(run_id: int, status: RunStatus, **outcome) -> None: + """Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's: + the row then stays `running` until the next start's sweep closes it as interrupted.""" + try: + jobruns.finish(run_id, status, **outcome) + except Exception: + log.exception("run %s: its outcome (%s) could not be recorded", run_id, status) def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]: @@ -164,16 +189,15 @@ def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None] summary = _summarize(result) data = _result_data(result) log.info("job %s -> %s", name, result) - jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data) + _close(run_id, RunStatus.OK, summary=summary, result=data) return summary, data -def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]: - """Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as - a result does, so the client says it in the reader's language; the error stays the English line.""" +def _failure(exc: Exception) -> tuple[str, dict | None]: + """A raised run's English error line, and the failure the job named (progress.JobFailed) as + data, as a result is, so the client says it in the reader's language.""" err = str(exc) or exc.__class__.__name__ failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None - jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure) return err, failure @@ -201,10 +225,6 @@ def _summarize(result) -> str | None: return str(result) -def _iso(ts: datetime | None) -> str | None: - return ts.astimezone(timezone.utc).isoformat() if ts else None - - def scheduler_snapshot(db) -> dict: """What the scheduler is doing, for the admin dashboard: each job's live state from this process and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how @@ -223,7 +243,7 @@ def scheduler_snapshot(db) -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} - recent = jobruns.recent_runs(db, RECENT_RUNS) + recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) @@ -239,17 +259,19 @@ def scheduler_snapshot(db) -> dict: ) # Work a user starts on one target: no interval, no "run now" (it needs its target), but its # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the - # target's own (sync/runner), so "running" is what its newest row says. + # target's own (sync/runner), so "running" is what its rows say: two channels can backfill at + # once, so a run still going counts even when a newer one already ended, and the row names it. backfills = recent.get(CHANNEL_BACKFILL, []) - newest = backfills[0] if backfills else None + going = jobruns.running_one(db, CHANNEL_BACKFILL) + shown = going or (backfills[0] if backfills else None) ad_hoc = [ { "id": CHANNEL_BACKFILL, - "running": newest is not None and newest.status == RunStatus.RUNNING, - "target": newest.target if newest else None, + "running": going is not None, + "target": shown.target if shown else None, "target_title": ( - db.scalar(select(Channel.title).where(Channel.id == newest.target)) - if newest is not None and newest.target + db.scalar(select(Channel.title).where(Channel.id == shown.target)) + if shown is not None and shown.target else None ), **_last_runs(backfills), @@ -275,8 +297,8 @@ def _last_runs(runs: list) -> dict: ended = run if run is not None and run.status != RunStatus.RUNNING else None return { "status": ended.status if ended else None, - "last_started": _iso(run.started_at) if run else None, - "last_finished": _iso(run.finished_at) if run else None, + "last_started": iso(run.started_at) if run else None, + "last_finished": iso(run.finished_at) if run else None, "last_result": ended.summary if ended else None, # The outcome as data: the result, or on a failed run the failure the job named. "result": _outcome_data(ended), @@ -293,8 +315,8 @@ def _run_brief(run) -> dict: "id": run.id, "status": run.status, "trigger": run.trigger, - "started_at": _iso(run.started_at), - "finished_at": _iso(run.finished_at), + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), } @@ -575,11 +597,15 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return with logs.bind(run_id=run_id): + # As in _run: only the work is inside the try. try: - _finish_ok(job, run_id, fn()) + result = fn() except Exception as exc: log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) - _finish_error(run_id, exc) + err, failure = _failure(exc) + _close(run_id, RunStatus.ERROR, error=err, failure=failure) + return + _finish_ok(job, run_id, result) # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of @@ -590,7 +616,8 @@ _POOL_SIZE = len(JOB_FUNCS) + 8 def start_scheduler() -> None: """Start this process's scheduler. It always starts, so "run now" works on every install; only with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can - run, every run a previous process left open is closed as interrupted.""" + run, every run a previous process left open is closed as interrupted — one API process serves a + database (app/jobruns.py), so an open run here is a dead process's.""" global _scheduler if _scheduler is not None: return diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index bf6ef7f8..6ad61490 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -29,7 +29,9 @@ _coerce_warned: set[tuple[str, str]] = set() # When a changed value takes effect. "live": the next read obeys it — every consumer reads the key -# where it uses it, never once at import or boot (test-held). "restart": the process has to start +# where it uses it, never once at import (test-held: a read outside a function fails). A value read +# once at startup is live only when its writer also applies it (the job intervals reschedule, the log +# level is set as it is saved). "restart": the process has to start # again. No key needs a restart today, and the Configuration page has no way to say so yet, so a # "restart" key is refused until the page can (test-held). Applies = Literal["live", "restart"] diff --git a/backend/tests/test_activity_reads.py b/backend/tests/test_activity_reads.py index f5b3f69c..78e97ef7 100644 --- a/backend/tests/test_activity_reads.py +++ b/backend/tests/test_activity_reads.py @@ -7,7 +7,10 @@ with rows that share a timestamp — the case an offset or a timestamp-only curs from datetime import datetime, timedelta, timezone +from app.auth import admin_user +from app.main import app from app.models import AuditLog, Channel, JobRun, LogEvent, QuotaEvent +from tests.test_body_limit import _route_objects T0 = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc) @@ -88,6 +91,32 @@ def test_an_admin_action_keeps_the_request_it_was_made_in(db, make_user, as_user assert row["request_id"] == r.headers["x-request-id"] +def test_an_admin_action_says_how_many_of_its_requests_lines_are_still_kept(db, make_user, as_user): + """A request's lines are kept days, its audit row months: a link to lines gone would open an + empty list, so the row says how many are left, as a run does.""" + admin = make_user(role="admin") + kept = _audit(db, "config_set", actor=admin.id, request_id="req-kept-0001") + gone = _audit(db, "config_set", actor=admin.id, request_id="req-gone-0001") + db.add_all( + [ + LogEvent( + ts=T0, + level="INFO", + sev=20, + source="api", + logger="t", + msg="m", + request_id="req-kept-0001", + ) + for _ in range(3) + ] + ) + db.commit() + items = as_user(admin).get("/api/admin/audit").json()["items"] + lines = {r["id"]: r["log_lines"] for r in items} + assert (lines[kept], lines[gone]) == (3, 0) + + def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, make_user, as_user): admin = make_user(role="admin") db.add(Channel(id="UCchan", title="Aileen Wang")) @@ -146,6 +175,8 @@ def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_ def totals(**params) -> dict: body = client.get("/api/quota/admin", params=params).json() + # The chart is the same filter's: its days add up to the table's total. + assert sum(d["total"] for d in body["daily"]) == sum(r["total"] for r in body["rows"]) return {r["user_id"]: r["total"] for r in body["rows"]} assert totals() == {admin.id: 150, None: 7} # the last 30 days by default @@ -153,3 +184,22 @@ def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_ assert totals(until=(now - timedelta(days=1)).isoformat()) == {admin.id: 50} assert totals(user="none") == {None: 7} assert totals(user=str(admin.id)) == {admin.id: 150} + + +def test_every_admin_read_and_write_is_admin_only(): + """The route table, not a list kept by hand: every /api/admin route and the quota breakdown + resolve `admin_user`, so a dropped dependency cannot hand an account's runs, errors and e-mail + addresses to any signed-in user.""" + routes = _route_objects(app) + admin = {k: r for k, r in routes.items() if k[1].startswith(("/api/admin", "/api/quota/admin"))} + assert ("GET", "/api/admin/runs") in admin and ("GET", "/api/quota/admin") in admin + open_ = [] + for key, route in admin.items(): + stack, calls = [route.dependant], set() + while stack: + d = stack.pop() + calls.add(d.call) + stack.extend(d.dependencies) + if admin_user not in calls: + open_.append(key) + assert open_ == [] diff --git a/backend/tests/test_audit_count_rows.py b/backend/tests/test_audit_count_rows.py index cdf47d6b..a2a2daf2 100644 --- a/backend/tests/test_audit_count_rows.py +++ b/backend/tests/test_audit_count_rows.py @@ -1,6 +1,6 @@ """An admin action that counts something keeps the count as data in its audit row's `after`, under the key the audit log names in the reader's language (`scheduler.result.`, frontend -auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word. +lib/jobResult.ts, called from lib/auditText.ts) — not only in the English `summary`. A renamed key would fall back to the raw word. DB lane (Postgres). """ diff --git a/backend/tests/test_channel_backfill.py b/backend/tests/test_channel_backfill.py index 14dfd41d..6e00567c 100644 --- a/backend/tests/test_channel_backfill.py +++ b/backend/tests/test_channel_backfill.py @@ -136,6 +136,19 @@ def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel( assert calls == [(ch.id, me.id)] +def test_turning_full_history_on_for_a_finished_channel_records_no_run( + db, monkeypatch, override_dep +): + """Its whole history is in already: a run would only say "nothing to do", and switching the flag + off and on again would otherwise write one such row each time, kept for months.""" + ch, me = _setup_channel_and_sub(db, email="done@example.com") + ch.backfill_done = True + db.commit() + resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) + assert resp.status_code == 200, resp.text + assert (submitted, calls) == ([], []) + + class TestImmediateBackfillGuard: """backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per- channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a diff --git a/backend/tests/test_migration_0109.py b/backend/tests/test_migration_0109.py index 9b19b05c..a597e0d7 100644 --- a/backend/tests/test_migration_0109.py +++ b/backend/tests/test_migration_0109.py @@ -96,3 +96,36 @@ def test_the_audit_logs_runs_move_into_the_run_table_and_back(db, make_user): ), (uid, "backfill", {"status": "skipped", "summary": "sync paused"}), ] + + +def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): + """A rollback then a redeploy: an interrupted run, a run caught mid-way and a user's run of one + channel come back as runs — not stranded in the audit log under an action no code knows.""" + uid = make_user().id + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at," + " finished_at) VALUES" + " ('backfill', 'scheduled', NULL, NULL, 'interrupted', now(), now())," + " ('maintenance', 'manual', :u, NULL, 'running', now(), NULL)," + " ('channel_backfill', 'ad_hoc', :u, 'UC_x', 'ok', now(), now())" + ), + {"u": uid}, + ) + cfg = Config("alembic.ini") + command.downgrade(cfg, "0108_folder_links") + command.upgrade(cfg, "head") + with engine.connect() as conn: + runs = conn.execute( + text("SELECT job, trigger, actor_id, target, status FROM job_runs ORDER BY id") + ).all() + stranded = conn.execute( + text("SELECT count(*) FROM audit_log WHERE action = 'job_run'") + ).scalar() + assert [tuple(r) for r in runs] == [ + ("backfill", "scheduled", None, None, "interrupted"), + ("maintenance", "manual", uid, None, "interrupted"), # the rollback stopped it + ("channel_backfill", "ad_hoc", uid, "UC_x", "ok"), + ] + assert stranded == 0 diff --git a/backend/tests/test_runstatus.py b/backend/tests/test_runstatus.py new file mode 100644 index 00000000..3cc331fd --- /dev/null +++ b/backend/tests/test_runstatus.py @@ -0,0 +1,27 @@ +"""The run words and the migrated database, DB-lane: the `job_runs` CHECKs are frozen in a migration, +and the model's are built from the enums, so nothing else would notice the two drifting apart until a +write of the new word failed in production.""" + +import re + +from sqlalchemy import text + +from app.runstatus import RunStatus, RunTrigger + + +def _words(definition: str) -> set[str]: + return set(re.findall(r"'([a-z_]+)'", definition)) + + +def test_the_databases_checks_allow_exactly_the_run_words(db): + defs = dict( + db.execute( + text( + "SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE conname LIKE 'ck_job_runs_%'" + ) + ).all() + ) + assert _words(defs["ck_job_runs_status"]) == set(RunStatus) + assert _words(defs["ck_job_runs_trigger"]) == set(RunTrigger) + assert _words(defs["ck_job_runs_finished"]) == {RunStatus.RUNNING} diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 91c0c84b..767aec5d 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -14,6 +14,7 @@ from sqlalchemy import select from app import jobruns, logs, logsink, quota, scheduler, state, sysconfig from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent +from app.routes import scheduler as scheduler_routes from app.runstatus import RunStatus, RunTrigger _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs @@ -256,6 +257,9 @@ def test_retention_reaps_finished_runs_only(db): ] ) db.commit() + reaped = next(r for r in _runs(db) if r.status == "ok" and r.started_at < old + timedelta(1)) + with logs.bind(run_id=reaped.id): + quota.record_usage(7) assert jobruns.gc(db, 0) == 0 # 0 = keep forever assert jobruns.gc(db, 30) == 1 db.commit() @@ -263,6 +267,8 @@ def test_retention_reaps_finished_runs_only(db): ("backfill", "running"), ("rss_poll", "ok"), ] + # Its spend stays in the budget's history (kept on its own retention), no longer anyone's run. + assert db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() == [(7, None)] # ---- "run now" through the scheduler -------------------------------------------------------------- @@ -413,6 +419,94 @@ def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch): assert run.status == "interrupted" +def test_an_interrupted_run_keeps_what_it_spent(db): + """Its spend is in the quota history, tagged with its id: the sweep sums it as `finish` would.""" + run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None) + with logs.bind(run_id=run_id): + quota.record_usage(40) + quota.record_usage(2) + assert jobruns.interrupt_leftovers() == 1 + (run,) = _runs(db, "backfill") + assert (run.status, run.quota_units) == ("interrupted", 42) + + +def _flaky_finish(monkeypatch) -> list: + """`jobruns.finish` whose first write fails (the database blinked); the statuses it was asked for.""" + calls: list = [] + real = jobruns.finish + + def finish(run_id, status, **outcome): + calls.append(status) + if len(calls) == 1: + raise RuntimeError("db blip") + real(run_id, status, **outcome) + + monkeypatch.setattr(jobruns, "finish", finish) + return calls + + +def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_user, monkeypatch): + """The job succeeded; only its bookkeeping did not. Writing ERROR then would say the work failed.""" + calls = _flaky_finish(monkeypatch) + scheduler._job("autotag", lambda _db: {"tagged": 1}) + assert calls == ["ok"] + assert not scheduler._is_running("autotag") + (run,) = _runs(db, "autotag") + assert run.status == "running" # left for the next start's sweep, not rewritten as a failure + + user = make_user("blip@example.com") + calls.clear() + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") + assert calls == ["ok"] + + +def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch): + def broken(*_a, **_k): + raise RuntimeError("db down") + + monkeypatch.setattr(jobruns, "start", broken) + scheduler._job("autotag", lambda _db: pytest.fail("no row, no run")) + assert not scheduler._is_running("autotag") + monkeypatch.undo() + scheduler._job("autotag", lambda _db: {"tagged": 1}) + assert [r.status for r in _runs(db, "autotag")] == ["ok"] + + +def test_the_page_says_where_the_interval_jobs_do_not_run(db, live_scheduler, monkeypatch): + """The scheduler starts on every install now; only SCHEDULER_ENABLED says the jobs run here.""" + assert scheduler.scheduler_snapshot(db)["running_here"] is False + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True) + assert scheduler.scheduler_snapshot(db)["running_here"] is True + + +def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_user): + """Two channels can backfill at once; a later run that ended at once must not hide the first.""" + user = make_user("two@example.com") + db.add(Channel(id="UC_long", title="Long one")) + db.commit() + jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_long") + scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short") + (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one") + + +def test_an_interval_change_lands_with_its_audit_row_or_not_at_all( + db, make_user, as_user, live_scheduler, monkeypatch +): + live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill") + admin = as_user(make_user(role="admin")) + + def broken(*_a, **_k): + raise RuntimeError("audit write failed") + + monkeypatch.setattr(scheduler_routes.audit, "record", broken) + with pytest.raises(RuntimeError): # the handler raises: there is no response to check + _ = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45}) + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == scheduler.JOB_INTERVALS["backfill"] + assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=10) + + def test_every_job_has_an_interval_and_every_interval_a_job(): assert set(scheduler.JOB_FUNCS) == set(scheduler.JOB_INTERVALS) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index d9febfdf..8ea9f4c0 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -275,10 +275,9 @@ Also in force: - **Page chrome (B4):** a page's tabs and filters sit in ONE sticky band of the page colour, with the fade under it (below). Below ~600px of height the filter row folds into one "Filters · n" button. Every page has a visually hidden `h1`; the nav rail and the browser tab title are the visible identity. [WCAG 2.4.6] - Built as: the band is `PageShell`'s toolbar slot (`PageToolbar`), above the one scroller; a console - whose toolbar lives inside its frame (the server log) makes that toolbar `sticky` with `fade-below` - instead. The band holds controls only: an intro paragraph scrolls with the content (in the band it - ate three lines of a short window). The fold is `useFilterFold` + `FilterFoldButton`: the same row + Built as: the band is `PageShell`'s toolbar slot (`PageToolbar`), above the one scroller. The band + holds controls only: an intro paragraph scrolls with the content (in the band it ate three lines of a + short window). The fold is `useFilterFold` + `FilterFoldButton`: the same row shown or hidden in place, never copied into a popover. The hidden `h1` is `PageShell`'s `title`; a view with its own visible title (a channel, a Plex show, a playlist) renders it as `PageHeading` and the hidden one steps aside, so a page has exactly one `h1` (a hand `

` inside the shell is diff --git a/frontend/src/components/Activity.dom.test.tsx b/frontend/src/components/Activity.dom.test.tsx index d16478c4..e74f7fb2 100644 --- a/frontend/src/components/Activity.dom.test.tsx +++ b/frontend/src/components/Activity.dom.test.tsx @@ -6,7 +6,7 @@ // the account filter reaches the quota tab; a // failed read is an error with a retry, never an empty list. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import type { AuditQuery, AuditRow, HistoryPage, LogQuery, RunQuery, RunRow } from "../lib/api"; @@ -23,6 +23,8 @@ const runsMock = vi.fn<(q: RunQuery, before: string | null) => Promise Promise>>(); const logsMock = vi.fn<(q: LogQuery, before: string | null) => Promise>(); const quotaMock = vi.fn<(since: string, user: string | null) => Promise>(); +const runFacetsMock = vi.fn<() => Promise>(); +const usersMock = vi.fn<() => Promise>(); vi.mock("../lib/api", async (importOriginal) => { const actual = await importOriginal(); return { @@ -30,19 +32,21 @@ vi.mock("../lib/api", async (importOriginal) => { api: { ...actual.api, adminRuns: (q: RunQuery, before: string | null) => runsMock(q, before), - adminRunFacets: () => Promise.resolve({ jobs: [], statuses: [], total: 0 }), + adminRunFacets: () => runFacetsMock(), adminAudit: (q: AuditQuery, before: string | null) => auditMock(q, before), adminAuditFacets: () => Promise.resolve({ actions: [] }), adminLogs: (q: LogQuery, before: string | null) => logsMock(q, before), adminLogFacets: () => Promise.resolve({ levels: [], sources: [], events: [], users: [], accounts: {}, total: 0 }), - adminUsers: () => Promise.resolve([{ id: 1, email: "admin@example.com" }]), + adminUsers: () => usersMock(), adminQuota: (since: string, user: string | null) => quotaMock(since, user), + clearAudit: () => Promise.resolve({ ok: true, cleared: 1 }), }, }; }); import Activity from "./Activity"; +import { ChoiceContext } from "./confirmContext"; import { PageToolbarSlot } from "./ui/pageToolbarSlot"; // The filter bar lives in the toolbar, which portals into the shell's band: give it one. @@ -77,6 +81,7 @@ const ACTION: AuditRow = { before: null, after: { interval_minutes: 25 }, request_id: "0efdd960c9ad1977", + log_lines: 4, }; const page = (items: T[]): HistoryPage => ({ @@ -89,9 +94,12 @@ function renderPage() { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( - - - + {/* Every confirmation is answered yes. */} + Promise.resolve("confirm")}> + + + + , ); } @@ -107,6 +115,8 @@ beforeEach(() => { auditMock.mockResolvedValue(page([ACTION])); logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); quotaMock.mockResolvedValue({ rows: [], daily: [] }); + runFacetsMock.mockResolvedValue({ jobs: [], statuses: [], total: 0 }); + usersMock.mockResolvedValue([{ id: 1, email: "admin@example.com" }]); }); afterEach(() => { cleanup(); @@ -135,6 +145,52 @@ describe("Activity", () => { expect([url().get("rjob"), url().get("rst")]).toEqual(["backfill", "error"]); }); + it("Older reads the next page from its cursor, and offers none past the last", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockImplementation((_q, before) => + Promise.resolve( + before === null + ? { ...page([RUN]), next_cursor: "c1" } + : page([{ ...RUN, id: 40, error: "the older one" }]), + ), + ); + renderPage(); + fireEvent.click(await screen.findByRole("button", { name: "activity.older" })); + await waitFor(() => expect(screen.getByText("the older one")).toBeTruthy()); + expect(runsMock).toHaveBeenLastCalledWith(expect.anything(), "c1"); + expect(screen.getByText("quotaExceeded")).toBeTruthy(); + expect(screen.queryByRole("button", { name: "activity.older" })).toBeNull(); + }); + + it("an older page that fails says so and keeps what was loaded", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockImplementation((_q, before) => + before === null + ? Promise.resolve({ ...page([RUN]), next_cursor: "c1" }) + : Promise.reject(new Error("500")), + ); + renderPage(); + fireEvent.click(await screen.findByRole("button", { name: "activity.older" })); + const alert = await screen.findByRole("alert"); + expect(alert.textContent).toContain("activity.olderFailed"); + expect(screen.getByText("quotaExceeded")).toBeTruthy(); + }); + + it("offers no older page while a filter change is still loading", async () => { + window.history.replaceState(null, "", "/?page=activity"); + // The cursor on screen is the previous filter's: it would continue the wrong list. + runsMock.mockImplementation((q) => + q.status ? new Promise(() => {}) : Promise.resolve({ ...page([RUN]), next_cursor: "c1" }), + ); + renderPage(); + await screen.findByRole("button", { name: "activity.older" }); + fireEvent.click(screen.getByRole("button", { name: /activity\.runs\.status\.error/ })); + await waitFor(() => + expect(screen.queryByRole("button", { name: "activity.older" })).toBeNull(), + ); + expect(screen.getByText("quotaExceeded")).toBeTruthy(); // the old rows stay, dimmed + }); + it("a run opens to its log lines, and Back returns to the runs", async () => { window.history.replaceState({ sfPage: "activity" }, "", "/?page=activity&aw=24h&aq=quota"); renderPage(); @@ -168,6 +224,77 @@ describe("Activity", () => { expect(screen.getByText("quotaExceeded")).toBeTruthy(); }); + it("a run's outcome is its result in the reader's words, its failure in the danger colour", async () => { + runsMock.mockResolvedValue( + page([ + { ...RUN, id: 1, status: "ok", error: null, result: { new: 3 }, summary: "new=3" }, + // Moved over from the audit log before the result rode along as data: the one-liner only. + { ...RUN, id: 2, status: "ok", error: null, result: null, summary: "failed=2" }, + { ...RUN, id: 3, status: "interrupted", error: null }, + { ...RUN, id: 4 }, + ]), + ); + renderPage(); + await waitFor(() => expect(screen.getByText("scheduler.result.new: 3")).toBeTruthy()); + expect(screen.queryByText("new=3")).toBeNull(); + expect(screen.getByText("scheduler.result.failed: 2")).toBeTruthy(); + expect(screen.getByText("scheduler.interrupted").className).toContain("text-warn"); + expect(screen.getByText("quotaExceeded").className).toContain("text-danger"); + }); + + it("filter choices that failed to load say so beside the filters, with a retry", async () => { + runFacetsMock.mockRejectedValueOnce(new Error("500")); + usersMock.mockRejectedValueOnce(new Error("500")); + renderPage(); + await waitFor(() => + expect(screen.getAllByRole("alert").map((a) => a.textContent)).toEqual([ + expect.stringContaining("activity.accountsFailed"), + expect.stringContaining("activity.filtersFailed"), + ]), + ); + for (const retry of screen.getAllByRole("button", { name: "common.retry" })) + fireEvent.click(retry); + await waitFor(() => expect(screen.queryAllByRole("alert")).toEqual([])); + expect(runFacetsMock).toHaveBeenCalledTimes(2); + expect(usersMock).toHaveBeenCalledTimes(2); + }); + + it("clearing the admin actions reads the list again", async () => { + window.history.replaceState(null, "", "/?page=activity&at=admin"); + renderPage(); + await waitFor(() => expect(screen.getByText("auditActions.scheduler_interval")).toBeTruthy()); + auditMock.mockResolvedValue(page([])); + fireEvent.click(screen.getByRole("button", { name: "audit.clear" })); + await waitFor(() => expect(screen.getByText("audit.empty")).toBeTruthy()); + // An emptied list is announced, as the error and loading states are. + expect(screen.getByText("audit.empty").closest('[role="status"]')).toBeTruthy(); + }); + + it("closing a dialog over the server log leaves Live on", async () => { + window.history.replaceState(null, "", "/?page=activity&at=logs"); + renderPage(); + const live = await screen.findByRole("button", { name: "adminLogs.live" }); + fireEvent.click(live); + expect(live.getAttribute("aria-pressed")).toBe("true"); + // A closing dialog steps back over its own history entry: a popstate onto the same address. + act(() => { + window.dispatchEvent(new PopStateEvent("popstate")); + }); + const after = screen.getByRole("button", { name: "adminLogs.live" }); + expect(after.getAttribute("aria-pressed")).toBe("true"); + expect(after).toBe(live); // the same log, not a fresh one + }); + + it("an action whose request's lines are gone says so instead of linking to an empty list", async () => { + auditMock.mockResolvedValue(page([{ ...ACTION, log_lines: 0 }])); + window.history.replaceState(null, "", "/?page=activity&at=admin"); + renderPage(); + await waitFor(() => expect(screen.getByText("auditActions.scheduler_interval")).toBeTruthy()); + fireEvent.click(rowButton(/scheduler_interval/)); + expect(screen.getByText("activity.audit.noRequestLines")).toBeTruthy(); + expect(screen.queryByRole("button", { name: "activity.audit.requestLines" })).toBeNull(); + }); + it("the admin tab lists the actions, and an action opens its request's lines", async () => { window.history.replaceState(null, "", "/?page=activity&at=admin"); renderPage(); @@ -196,6 +323,24 @@ describe("Activity", () => { expect(Math.round(days)).toBe(30); }); + it("each day's spend is readable without a pointer", async () => { + window.history.replaceState(null, "", "/?page=activity&at=quota"); + quotaMock.mockResolvedValue({ + rows: [{ user_id: null, email: "System", total: 9, by_action: {} }], + daily: [ + { day: "2026-10-16", total: 4 }, + { day: "2026-10-17", total: 5 }, + ], + }); + renderPage(); + const chart = await screen.findByRole("list", { name: "activity.quota.daily" }); + expect( + within(chart) + .getAllByRole("listitem") + .map((d) => d.textContent), + ).toEqual(["stats.dailyTooltip", "stats.dailyTooltip"]); + }); + // The runs moved over from the audit log kept one instant for start and end: their length is // unknown, and a "1 sec" would be invented. it("a run whose start and end are one instant has no length", async () => { diff --git a/frontend/src/components/Activity.tsx b/frontend/src/components/Activity.tsx index 79a70cf1..d7ff5364 100644 --- a/frontend/src/components/Activity.tsx +++ b/frontend/src/components/Activity.tsx @@ -29,6 +29,7 @@ import FilterSelect from "./ui/FilterSelect"; import { InnerInput } from "./ui/Input"; import { fieldClass } from "./ui/fieldStyles"; import { PageToolbar } from "./ui/PageShell"; +import { StateMessage } from "./ui/QueryState"; import Tabs from "./ui/Tabs"; import { useTabPanel } from "./ui/tabPanel"; @@ -51,9 +52,12 @@ export default function Activity() { activityParamsToState(new URLSearchParams(window.location.search)), ); // Bumped when the view changes by a jump (a pivot, Back/Forward) rather than by typing: the search - // text then applies at once instead of after the typing debounce, and the server log re-reads its - // own filters from the address bar. + // text then applies at once instead of after the typing debounce. const [jumps, setJumps] = useState(0); + // Bumped by a pivot only: the server log, already open, takes the pivot's filters from the address + // bar when it mounts again. Back/Forward it follows in place (AdminLogs) — a popstate is also a + // dialog closing, onto the same address, and a fresh log would drop its Live and its focus. + const [pivots, setPivots] = useState(0); const q = useDebounced(state.q, 300, jumps); const panel = useTabPanel(); // `page` too: a link to a page this one replaced (audit, logs) is rewritten to this page's id. @@ -81,10 +85,13 @@ export default function Activity() { ); notifyNav(); setJumps((n) => n + 1); + setPivots((n) => n + 1); setState(next); }; const shared = { window: state.window, user: state.user, q }; + // The account menu's list (SharedBar reads the same query): without it the menu offers only "Any". + const users = useQuery({ queryKey: qk.adminUsers(), queryFn: api.adminUsers }); return (
@@ -99,6 +106,11 @@ export default function Activity() {
+ {users.isError && ( + void users.refetch()}> + {t("activity.accountsFailed")} + + )} {/* The quota tab says what its numbers are itself, beside them. */} {state.tab !== "quota" && (

@@ -121,9 +133,9 @@ export default function Activity() { onRequest={(requestId) => pivot({ requestId, jobId: null, runId: null })} /> ) : state.tab === "logs" ? ( - // Keyed by the jumps: a pivot or Back/Forward hands the log its filters through the address - // bar, which it reads when it mounts. - + // Keyed by the pivots: a pivot hands the log its filters through the address bar, which it + // reads when it mounts. + ) : ( )} diff --git a/frontend/src/components/ActivityAudit.tsx b/frontend/src/components/ActivityAudit.tsx index ff6d0ab9..dfdd46b0 100644 --- a/frontend/src/components/ActivityAudit.tsx +++ b/frontend/src/components/ActivityAudit.tsx @@ -13,6 +13,7 @@ import { qk } from "../lib/queryKeys"; import { useHistoryList } from "../lib/useHistoryList"; import type { Shared } from "./ActivityRuns"; import { useConfirm } from "./confirmContext"; +import { StateMessage } from "./ui/QueryState"; import HistoryList from "./HistoryList"; import HistoryRow from "./HistoryRow"; import { RelativeTime } from "./ui/RelativeTime"; @@ -67,8 +68,8 @@ export default function ActivityAudit({ const clear = useMutation({ mutationFn: () => api.clearAudit(), onSuccess: (r) => { - void qc.invalidateQueries({ queryKey: ["admin-audit"] }); - void qc.invalidateQueries({ queryKey: ["admin-audit-facets"] }); + void qc.invalidateQueries({ queryKey: qk.adminAudit() }); + void qc.invalidateQueries({ queryKey: qk.adminAuditFacets() }); notify({ level: "success", message: t("audit.cleared", { count: r.cleared }) }); }, onError: (e) => notifyActionError(e, t("audit.clearFailed")), @@ -115,6 +116,12 @@ export default function ActivityAudit({ {t("audit.clear")} + {/* The action menu above is built from these counts: without them it offers only "Any". */} + {facets.isError && ( + void facets.refetch()}> + {t("activity.filtersFailed")} + + )} - {facets.isError &&

{t("activity.countsFailed")}

} ); } @@ -203,17 +209,21 @@ function AuditDetail({ {JSON.stringify({ before: row.before, after: row.after }, null, 2)} )} - {row.request_id && ( -
- -
- )} + {/* A request's lines are kept days, its row months: a link to none would open an empty list. */} + {row.request_id && + (row.log_lines > 0 ? ( +
+ +
+ ) : ( +

{t("activity.audit.noRequestLines")}

+ ))} ); } diff --git a/frontend/src/components/ActivityQuota.tsx b/frontend/src/components/ActivityQuota.tsx index 12698596..7c601a8f 100644 --- a/frontend/src/components/ActivityQuota.tsx +++ b/frontend/src/components/ActivityQuota.tsx @@ -49,7 +49,8 @@ export default function ActivityQuota({ shared }: { shared: Shared }) { )}
- {t("activity.quota.total", { + {t(shared.window === "all" ? "activity.quota.totalAll" : "activity.quota.total", { + count: grandTotal, window: t(`activity.windows.${shared.window}`), units: formatNumber(grandTotal, lang), })} @@ -57,18 +58,33 @@ export default function ActivityQuota({ shared }: { shared: Shared }) { {data.daily.length === 0 ? (
{t("stats.noUsageInRange")}
) : ( -
+ // Each day's figure is in its bar's text too (read out, not shown): the tooltip only + // reaches a pointer. +
{data.daily.map((d) => (
+ + {t("stats.dailyTooltip", { + count: d.total, + day: d.day, + units: formatNumber(d.total, lang), + })} +
+ {/* The menus above are built from these counts: without them they offer only "Any". */} + {facets.isError && ( + void facets.refetch()}> + {t("activity.filtersFailed")} + + )} ))} - {facets.isError &&

{t("activity.countsFailed")}

}
); } @@ -160,7 +166,13 @@ function useOutcome(run: RunRow): { text: string; tone: string } { }; if (run.status === "interrupted") return { text: t("scheduler.interrupted"), tone: "text-warn" }; if (run.status === "running") return { text: t("scheduler.runningNow"), tone: "text-accent" }; - if (run.result) return { text: jobResultText(t, run.result, lang), tone: "text-muted" }; + // A run moved over from the audit log before its result rode along as data holds only the + // server's English one-liner: read back into a result, as the old audit rows were. + const result = + run.result ?? + parseSummary(run.summary) ?? + (run.status === "skipped" && run.summary ? { skipped: run.summary } : null); + if (result) return { text: jobResultText(t, result, lang), tone: "text-muted" }; return { text: run.summary ?? "", tone: "text-muted" }; } diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index e9932427..703dbc76 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -1,4 +1,4 @@ -import { useMemo, useRef, useState } from "react"; +import { useEffect, useEffectEvent, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { keepPreviousData, @@ -18,14 +18,13 @@ import { logsParamsToState, logsParamUpdates, type LogsUrlState } from "../lib/l import { notify } from "../lib/notifications"; import { qk } from "../lib/queryKeys"; import { useUrlMirror } from "../lib/useUrlMirror"; +import { HistoryFrame } from "./HistoryList"; import LogLine from "./LogLine"; -import NavGrid from "./ui/NavGrid"; import { Button } from "./ui/Button"; import FilterChipGroup from "./ui/FilterChipGroup"; import FilterSelect from "./ui/FilterSelect"; import { FilterFoldButton } from "./ui/FilterFold"; import { useFilterFold } from "../lib/useFilterFold"; -import { LoadingState, StateMessage } from "./ui/QueryState"; import { DismissibleChip } from "./ui/Chip"; import type { Shared } from "./ActivityRuns"; @@ -84,6 +83,19 @@ export default function AdminLogs({ }; const [open, setOpen] = useState(null); useUrlMirror(logsParamUpdates(state)); + // Back/Forward lands on another entry's filters; a dialog closing lands on these same ones, and + // changes nothing (Live and the open line stay). + const onLanded = useEffectEvent(() => { + const landed = logsParamsToState(new URLSearchParams(window.location.search)); + if (JSON.stringify(landed) === JSON.stringify(state)) return; + setOpen(null); + setState(landed); + }); + useEffect(() => { + const onPop = () => onLanded(); + window.addEventListener("popstate", onPop); + return () => window.removeEventListener("popstate", onPop); + }, []); const update = (patch: Partial) => { setOpen(null); setState((s) => ({ ...s, ...patch })); @@ -212,156 +224,130 @@ export default function AdminLogs({ }; return ( -
- {/* The page's own band carries the shared filters and stays in reach; this one scrolls. */} -
-
- update({ group })} - options={GROUPS.map((g) => ({ - value: g, - label: t(`settings.myLog.filters.${g}`), - count: groupCount(g), - }))} - /> -
- -
- - {!fold.folded && ( - <> - v)} - selected={[state.sources[0] ?? ""]} - onChange={([source]) => update({ sources: source ? [source] : [] })} - testId="logs-source" - /> - - v === "none" ? t("adminLogs.noEvent") : v, - )} - selected={[state.event ?? ""]} - onChange={([event]) => update({ event: event || null })} - testId="logs-event" - /> - - )} - - - - -
- - {(state.requestId || state.jobId !== null || state.runId !== null) && ( -
- {state.requestId && ( - update({ requestId: null })} - /> - )} - {state.jobId !== null && ( - update({ jobId: null })} - /> - )} - {state.runId !== null && ( - update({ runId: null })} - /> - )} + +
+ update({ group })} + options={GROUPS.map((g) => ({ + value: g, + label: t(`settings.myLog.filters.${g}`), + count: groupCount(g), + }))} + />
- )} -
- {failure && ( - - {t(`adminLogs.${failure}`)} - - )} - -
- {logs.isError && !logs.data ? ( - void logs.refetch()}> - {t("common.loadError")} - - ) : logs.isLoading ? ( - - ) : rows.length === 0 ? ( -

- {t("adminLogs.empty")} -

- ) : ( - - {rows.map((row, index) => ( - setOpen(open === row.id ? null : row.id)} - > - pivot({ requestId })} - onJob={(jobId) => pivot({ jobId })} - onRun={(runId) => pivot({ runId })} +
+ + {!fold.folded && ( + <> + v)} + selected={[state.sources[0] ?? ""]} + onChange={([source]) => update({ sources: source ? [source] : [] })} + testId="logs-source" /> - - ))} - - )} -
+ + v === "none" ? t("adminLogs.noEvent") : v, + )} + selected={[state.event ?? ""]} + onChange={([event]) => update({ event: event || null })} + testId="logs-event" + /> + + )} + + + + +
-
- {t("adminLogs.count", { count: rows.length })} - {olderCursor !== null && ( - - )} -
-
+ {(state.requestId || state.jobId !== null || state.runId !== null) && ( +
+ {state.requestId && ( + update({ requestId: null })} + /> + )} + {state.jobId !== null && ( + update({ jobId: null })} + /> + )} + {state.runId !== null && ( + update({ runId: null })} + /> + )} +
+ )} +
+ } + > + {rows.map((row, index) => ( + setOpen(open === row.id ? null : row.id)} + > + pivot({ requestId })} + onJob={(jobId) => pivot({ jobId })} + onRun={(runId) => pivot({ runId })} + /> + + ))} + ); } diff --git a/frontend/src/components/HistoryList.tsx b/frontend/src/components/HistoryList.tsx index 8862417f..a0e66b49 100644 --- a/frontend/src/components/HistoryList.tsx +++ b/frontend/src/components/HistoryList.tsx @@ -7,9 +7,99 @@ import NavGrid from "./ui/NavGrid"; import { Button } from "./ui/Button"; import { LoadingState, StateMessage } from "./ui/QueryState"; -// The frame of a history list on the activity page: the rows (or the honest state instead of them — -// a failed read is an error with a retry, never an empty list), the count, and "Older". +// The frame of a history list on the activity page — the runs, the admin actions, the server log: +// its own bar (if any), the rows or the honest state instead of them (a failed read is an error with a +// retry, never an empty list), a read that failed over rows still on screen, the count, and "Older". +// `HistoryFrame` draws it; each list says only what its failure and its "Older" are — the server log's +// count its filter facets and stop while Live (AdminLogs), the others not (HistoryList, below). +/** What a read that failed while rows stayed on screen means, and how to try it again. */ +export interface FrameFailure { + text: string; + onRetry: () => void; +} + +export function HistoryFrame({ + list, + rowCount, + header, + failure, + older, + count, + empty, + rowsClassName = "", + testId, + children, +}: { + /** The list's read. */ + list: { + data?: unknown; + isError: boolean; + isLoading: boolean; + isPlaceholderData: boolean; + refetch: () => unknown; + }; + rowCount: number; + /** The list's own bar above the rows. */ + header?: ReactNode; + failure: FrameFailure | null; + /** "Older", when there is an older page to ask for. */ + older: { label: string; onClick: () => void; disabled: boolean } | null; + /** "N runs shown", for the footer. */ + count: string; + /** Shown when the filters match nothing. */ + empty: string; + rowsClassName?: string; + testId?: string; + /** The rows. */ + children: ReactNode; +}) { + const { t } = useTranslation(); + return ( +
+ {header} + {failure && ( + + {failure.text} + + )} +
+ {list.isError && !list.data ? ( + void list.refetch()}> + {t("common.loadError")} + + ) : list.isLoading ? ( + + ) : rowCount === 0 ? ( + + {empty} + + ) : ( + {children} + )} +
+
+ {count} + {older && ( + + )} +
+
+ ); +} + +/** A history list read through `useHistoryList` (the runs, the admin actions). */ export default function HistoryList({ state, empty, @@ -26,59 +116,29 @@ export default function HistoryList({ }) { const { t } = useTranslation(); const { list, rows, hasOlder } = state; + const fetchOlder = () => void list.fetchNextPage(nextPageOnClick(list.isFetchingNextPage)); // A read that failed while rows were on screen keeps them, and says what the failure means. - const failure = !list.data + const failure: FrameFailure | null = !list.data ? null : list.isFetchNextPageError - ? "olderFailed" + ? { text: t("activity.olderFailed"), onRetry: fetchOlder } : list.isError - ? "stale" + ? { text: t("activity.stale"), onRetry: () => void list.refetch() } : null; return ( -
- {failure && ( - - failure === "olderFailed" - ? void list.fetchNextPage(nextPageOnClick(list.isFetchingNextPage)) - : void list.refetch() - } - className="border-b border-border" - > - {failure === "olderFailed" ? t("activity.olderFailed") : t("activity.stale")} - - )} -
- {list.isError && !list.data ? ( - void list.refetch()}> - {t("common.loadError")} - - ) : list.isLoading ? ( - - ) : rows.length === 0 ? ( -

{empty}

- ) : ( - {children} - )} -
-
- {count} - {hasOlder && ( - - )} -
-
+ + {children} + ); } diff --git a/frontend/src/components/HistoryRow.tsx b/frontend/src/components/HistoryRow.tsx index f48f57f2..bf19aa8b 100644 --- a/frontend/src/components/HistoryRow.tsx +++ b/frontend/src/components/HistoryRow.tsx @@ -3,9 +3,10 @@ import type { ReactNode } from "react"; import { navItemAttrs } from "../lib/listNav"; import { Pressable } from "./ui/Pressable"; -// One dense row of a history list on the activity page — a background run, an admin action: its -// cells on one line, and the row opens in place to its details, as a server log line does -// (./LogLine). Inside a NavGrid the keyboard walks the rows (`j`/`k`) and Enter opens one. +// One dense row of a history list — a background run, an admin action, a log line (./LogLine draws +// its lines through this): its cells on one line, and the row opens in place to its details. One frame +// for all of them, so a row opens, announces itself and takes the keyboard the same on every tab. +// Inside a NavGrid the keyboard walks the rows (`j`/`k`) and Enter opens one. export default function HistoryRow({ cells, @@ -15,6 +16,7 @@ export default function HistoryRow({ open, onToggle, title, + line = false, children, }: { /** The row's cells, laid out by `grid`. */ @@ -28,6 +30,8 @@ export default function HistoryRow({ onToggle: () => void; /** The row button's tooltip — the full date. */ title?: string; + /** A log line: tighter, its cells on one text baseline, its size the (monospace) list's own. */ + line?: boolean; /** The detail shown while the row is open. */ children: ReactNode; }) { @@ -43,13 +47,17 @@ export default function HistoryRow({ aria-expanded={open} onClick={onToggle} title={title} - className={`grid w-full items-center gap-x-2 px-3 py-1.5 text-left text-xs touch-target pointer-coarse:content-center ${grid} ${ - open ? "bg-accent/10" : "" - }`} + className={`grid w-full gap-x-2 px-3 text-left touch-target pointer-coarse:content-center ${ + line ? "items-baseline py-1" : "items-center py-1.5 text-xs" + } ${grid} ${open ? "bg-accent/10" : ""}`} > {cells} - {open &&
{children}
} + {open && ( +
+ {children} +
+ )}
); } diff --git a/frontend/src/components/LogLine.tsx b/frontend/src/components/LogLine.tsx index fca3dded..33b3c9ba 100644 --- a/frontend/src/components/LogLine.tsx +++ b/frontend/src/components/LogLine.tsx @@ -1,14 +1,14 @@ import type { ReactNode } from "react"; import type { Level } from "../lib/log"; -import { navItemAttrs } from "../lib/listNav"; +import HistoryRow from "./HistoryRow"; import { LEVEL_TAG, logClock } from "./logLineParts"; -import { Pressable } from "./ui/Pressable"; // One log line as both log viewers draw it — "Saját napló" (this tab's ring) and the admin // diagnostics console (the server's stored log): time, level, event code, message, and the line opens // in place to whatever detail its screen has. One component, so a level reads the same on both. -// Inside a NavGrid the keyboard walks the lines (`j`/`k`) and Enter opens one, as a click does. +// The row frame is the history lists' (./HistoryRow): inside a NavGrid the keyboard walks the lines +// (`j`/`k`) and Enter opens one, as a click does. export default function LogLine({ ts, @@ -36,29 +36,26 @@ export default function LogLine({ }) { const tag = LEVEL_TAG[level]; return ( -
+ {logClock(ts)} + {tag.text} + {event} + {/* A phone has no room for a fourth column: the message takes a line of its own under the + code, full width — sharing the code's line left it a few characters. */} + {msg} + + } > - - {logClock(ts)} - {tag.text} - {event} - {/* A phone has no room for a fourth column: the message takes a line of its own under the - code, full width — sharing the code's line left it a few characters. */} - {msg} - - {open &&
{children}
} -
+ {children} + ); } diff --git a/frontend/src/components/NavigationProvider.dom.test.tsx b/frontend/src/components/NavigationProvider.dom.test.tsx index eb2cc655..630c2a61 100644 --- a/frontend/src/components/NavigationProvider.dom.test.tsx +++ b/frontend/src/components/NavigationProvider.dom.test.tsx @@ -31,6 +31,8 @@ import { useBackToClose } from "../lib/history"; import { PageHeading } from "./ui/Headings"; import { qk } from "../lib/queryKeys"; +import { LS, setAccountRaw } from "../lib/storage"; +import { PAGE_KEY, START_ON_HOME } from "./navigationInit"; /** The Navigation API's view of the tab: `below` = whether each entry under the current one belongs * to this document, nearest first. jsdom has no Navigation API; Chrome's says this. */ @@ -239,6 +241,35 @@ describe("NavigationProvider · Back", () => { ]); }); + it("carries an old server-log link's account, text and range onto the page's own filters", () => { + window.history.replaceState(null, "", "/?page=logs&lu=3&lq=boom&lwin=1h"); + render(); + const url = new URLSearchParams(window.location.search); + expect([url.get("at"), url.get("au"), url.get("aq"), url.get("aw")]).toEqual([ + "logs", + "3", + "boom", + "1h", + ]); + // …and the dead keys leave the address bar instead of riding along to every page after. + expect(["lu", "lq", "lwin"].filter((k) => url.has(k))).toEqual([]); + }); + + it("an old server-log link without a range keeps the one it meant", () => { + window.history.replaceState(null, "", "/?page=logs"); + render(); + expect(new URLSearchParams(window.location.search).get("aw")).toBe("24h"); + }); + + it("a remembered retired page opens its tab", () => { + setAccountRaw(LS.startOnHome, START_ON_HOME.off); + setAccountRaw(PAGE_KEY, "logs"); + window.history.replaceState(null, "", "/"); + const { result } = render(); + expect(result.current.page).toBe("activity"); + expect(new URLSearchParams(window.location.search).get("at")).toBe("logs"); + }); + it("opens a retired page's tab when Back lands on its entry", () => { const { result } = render(); act(() => result.current.setPage("plex")); diff --git a/frontend/src/components/NavigationProvider.tsx b/frontend/src/components/NavigationProvider.tsx index 7ea53516..8e61101f 100644 --- a/frontend/src/components/NavigationProvider.tsx +++ b/frontend/src/components/NavigationProvider.tsx @@ -6,7 +6,7 @@ import { resolvePage, retiredParams, urlWithParams, type Page } from "../lib/url import { ACTIVITY_KEYS } from "../lib/activityUrlState"; import { pageTitleKey } from "../lib/pageMeta"; import { useDocumentObjectTitle } from "../lib/documentTitle"; -import { setAccountRaw } from "../lib/storage"; +import { getAccountRaw, setAccountRaw } from "../lib/storage"; import { backStaysInDocument, closeOverlaysThenNavigate, @@ -193,9 +193,12 @@ export function NavigationProvider({ children }: { children: ReactNode }) { // Mirror the resolved initial page to the address bar (feed implicit) so a page reached from the // stored landing pref — not just from the URL — is itself bookmarkable. urlWithParams preserves any // ?chan / ?v / ?pl / filter params the loaders already read. + // A retired page's link — or the remembered last page, when that is what opened — also opens its + // part of the page that replaced it (an `at` tab). + const linked = new URLSearchParams(window.location.search).get("page"); + const remembered = getAccountRaw(PAGE_KEY); const url = urlWithParams({ - // A retired page's link also opens its part of the page that replaced it (an `at` tab). - ...retiredParams(new URLSearchParams(window.location.search).get("page")), + ...retiredParams(linked ?? (resolvePage(remembered) === mountPage ? remembered : null)), page: mountPage === "feed" ? null : mountPage, }); if (st._yt && st._ytScrape) { diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts index 6e1c9b9d..4a23797e 100644 --- a/frontend/src/i18n/backendIds.contract.test.ts +++ b/frontend/src/i18n/backendIds.contract.test.ts @@ -1,5 +1,7 @@ import { readdirSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; + +import { RUN_STATUSES } from "../lib/activityUrlState"; import { vocabKey } from "../lib/vocab"; // Every id the BACKEND owns and the UI names must have its name in both languages. The UI looks these @@ -72,6 +74,15 @@ const jobs = [ const enumValues = (file: string) => [...backend(file).matchAll(/^\s+[A-Z_]+ = "(\w+)"$/gm)].map((m) => m[1]!); const auditActions = enumValues("audit.py"); +// A background run's words (backend/app/runstatus.py): its status and what started it. +const runEnum = (name: string) => + [ + ...( + backend("runstatus.py").split(`class ${name}(StrEnum):`)[1]?.split("\nclass ")[0] ?? "" + ).matchAll(/^\s+[A-Z_]+ = "(\w+)"$/gm), + ].map((m) => m[1]!); +const runStatuses = runEnum("RunStatus"); +const runTriggers = runEnum("RunTrigger"); const quotaActions = enumValues("quota.py"); // The admin actions' refusal reasons → `errors.admin.`. The codes are read from // `localizeDetail`'s own ADMIN_REFUSALS, so a code added there is scanned here too; a detail dict @@ -260,6 +271,8 @@ describe("backend ids have their names in both languages", () => { expect(jobs).toEqual(expect.arrayContaining(["rss_poll", "audit_gc"])); expect(auditActions).toEqual(expect.arrayContaining(["config_set", "audit_clear"])); expect(quotaActions.length).toBeGreaterThan(0); + expect(runStatuses).toEqual(expect.arrayContaining(["running", "interrupted"])); + expect(runTriggers).toEqual(expect.arrayContaining(["scheduled", "ad_hoc"])); expect(adminCodes).toEqual(expect.arrayContaining(["activate", "resend"])); expect(failDetails).toEqual(expect.arrayContaining(["facebook.login_wall"])); expect(goneWhats).toEqual(expect.arrayContaining(["playlist", "download", "job"])); @@ -288,10 +301,23 @@ describe("backend ids have their names in both languages", () => { expect(adminCodes.filter((c) => !adminRefusals.some((m) => m[1] === c))).toEqual([]); }); + it("the run list's status chips are the server's run statuses", () => { + expect([...RUN_STATUSES].sort()).toEqual([...runStatuses].sort()); + }); + for (const lang of ["en", "hu"]) { const keys = keysOf(lang); const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k)); + it(`${lang}: every run status and trigger`, () => { + expect( + missing([ + ...runStatuses.flatMap((s) => [`activity.runs.status.${s}`, `scheduler.dot.${s}`]), + ...runTriggers.map((s) => `scheduler.trigger.${s}`), + ]), + ).toEqual([]); + }); + it(`${lang}: every config key and group`, () => { expect( missing([ diff --git a/frontend/src/i18n/locales/en/activity.json b/frontend/src/i18n/locales/en/activity.json index dba2414c..d0e34b02 100644 --- a/frontend/src/i18n/locales/en/activity.json +++ b/frontend/src/i18n/locales/en/activity.json @@ -18,7 +18,8 @@ "24h": "24 hours", "7d": "7 days", "30d": "30 days", - "90d": "90 days" + "90d": "90 days", + "all": "All" }, "account": "Account", "any": "Any", @@ -26,7 +27,8 @@ "older": "Load older", "stale": "Couldn't refresh — what you see may be out of date", "olderFailed": "Couldn't load the older rows", - "countsFailed": "Couldn't load the filter counts", + "filtersFailed": "Couldn't load the filter choices: the menus may be missing some.", + "accountsFailed": "Couldn't load the accounts for the account filter.", "runs": { "job": "Job", "statusLabel": "Filter by status", @@ -57,10 +59,15 @@ "action": "Action", "target": "Target", "requestLines": "Its request's log lines →", + "noRequestLines": "Its request's log lines are no longer kept", "count_one": "{{count}} action shown · newest first", "count_other": "{{count}} actions shown · newest first" }, "quota": { - "total": "Spend over {{window}}: {{units}} units" + "daily": "Spend per day", + "total_one": "Spend over {{window}}: {{units}} unit", + "total_other": "Spend over {{window}}: {{units}} units", + "totalAll_one": "Spend over all kept history: {{units}} unit", + "totalAll_other": "Spend over all kept history: {{units}} units" } } diff --git a/frontend/src/i18n/locales/en/stats.json b/frontend/src/i18n/locales/en/stats.json index 8ab0cb00..d1d7bda6 100644 --- a/frontend/src/i18n/locales/en/stats.json +++ b/frontend/src/i18n/locales/en/stats.json @@ -4,7 +4,6 @@ "introAfter": " is shared background work (scheduled backfill, enrichment, resync) that isn't tied to one person.", "loading": "Loading…", "noUsageInRange": "No usage in this range.", - "dailyTooltip": "{{day}}: {{units}} units", "system": "System (background)", "sync": { "myStatus": "My sync status", @@ -41,5 +40,7 @@ "fullHistoryRequested_one": "Full history requested for {{count}} channel", "fullHistoryRequested_other": "Full history requested for {{count}} channels", "fullHistoryFailed": "Couldn't request full history" - } + }, + "dailyTooltip_one": "{{day}}: {{units}} unit", + "dailyTooltip_other": "{{day}}: {{units}} units" } diff --git a/frontend/src/i18n/locales/hu/activity.json b/frontend/src/i18n/locales/hu/activity.json index 33d26641..0b282a8d 100644 --- a/frontend/src/i18n/locales/hu/activity.json +++ b/frontend/src/i18n/locales/hu/activity.json @@ -18,7 +18,8 @@ "24h": "24 óra", "7d": "7 nap", "30d": "30 nap", - "90d": "90 nap" + "90d": "90 nap", + "all": "Mind" }, "account": "Fiók", "any": "Bármelyik", @@ -26,7 +27,8 @@ "older": "Régebbiek", "stale": "Nem sikerült frissíteni — amit látsz, elavult lehet", "olderFailed": "Nem sikerült betölteni a régebbi sorokat", - "countsFailed": "Nem sikerült betölteni a szűrők darabszámait", + "filtersFailed": "Nem sikerült betölteni a szűrők választékát: a menükből hiányozhatnak elemek.", + "accountsFailed": "Nem sikerült betölteni a fiókokat a fiókszűrőhöz.", "runs": { "job": "Feladat", "statusLabel": "Szűrés állapotra", @@ -57,10 +59,15 @@ "action": "Művelet", "target": "Cél", "requestLines": "A kérés naplósorai →", + "noRequestLines": "A kérés naplósorai már nincsenek meg", "count_one": "{{count}} művelet látszik · a legújabb elöl", "count_other": "{{count}} művelet látszik · a legújabb elöl" }, "quota": { - "total": "Felhasználás ({{window}}): {{units}} egység" + "daily": "Felhasználás naponta", + "total_one": "Felhasználás ({{window}}): {{units}} egység", + "total_other": "Felhasználás ({{window}}): {{units}} egység", + "totalAll_one": "Felhasználás (a teljes megőrzött idő): {{units}} egység", + "totalAll_other": "Felhasználás (a teljes megőrzött idő): {{units}} egység" } } diff --git a/frontend/src/i18n/locales/hu/stats.json b/frontend/src/i18n/locales/hu/stats.json index 94e73362..4c04918d 100644 --- a/frontend/src/i18n/locales/hu/stats.json +++ b/frontend/src/i18n/locales/hu/stats.json @@ -4,7 +4,6 @@ "introAfter": " a megosztott háttérmunka (ütemezett behúzás, dúsítás, újraszinkronizálás), amely nem köthető egy személyhez.", "loading": "Betöltés…", "noUsageInRange": "Nincs használat ebben az időszakban.", - "dailyTooltip": "{{day}}: {{units}} egység", "system": "Rendszer (háttér)", "sync": { "myStatus": "Saját szinkronizálási állapot", @@ -41,5 +40,7 @@ "fullHistoryRequested_one": "Teljes előzmény kérve {{count}} csatornához", "fullHistoryRequested_other": "Teljes előzmény kérve {{count}} csatornához", "fullHistoryFailed": "Nem sikerült teljes előzményt kérni" - } + }, + "dailyTooltip_one": "{{day}}: {{units}} egység", + "dailyTooltip_other": "{{day}}: {{units}} egység" } diff --git a/frontend/src/lib/activityUrlState.test.ts b/frontend/src/lib/activityUrlState.test.ts index b748d474..fd8a4290 100644 --- a/frontend/src/lib/activityUrlState.test.ts +++ b/frontend/src/lib/activityUrlState.test.ts @@ -9,6 +9,7 @@ import { activityParamUpdates, DEFAULT_ACTIVITY_STATE, sinceOf, + ACTIVITY_WINDOWS, type ActivityUrlState, } from "./activityUrlState"; @@ -54,4 +55,11 @@ describe("activityUrlState", () => { expect(sinceOf("24h", now)).toBe("2026-10-17T12:00:00.000Z"); expect(sinceOf("90d", now)).toBe("2026-07-20T12:00:00.000Z"); }); + + it("offers everything kept, past the longest range", () => { + // Runs and admin actions are kept 180 days by default, or for ever: a 90-day cap hid the rest. + expect(ACTIVITY_WINDOWS[ACTIVITY_WINDOWS.length - 1]).toBe("all"); + expect(activityParamsToState(new URLSearchParams("aw=all")).window).toBe("all"); + expect(sinceOf("all")).toBe("1970-01-01T00:00:00.000Z"); + }); }); diff --git a/frontend/src/lib/activityUrlState.ts b/frontend/src/lib/activityUrlState.ts index b1b495a4..4a754a28 100644 --- a/frontend/src/lib/activityUrlState.ts +++ b/frontend/src/lib/activityUrlState.ts @@ -3,12 +3,13 @@ // each tab's own (the run list's job and status, the audit list's action). The server log's own // filters keep their `l…` keys (./logsUrlState). setPage clears all of them (ACTIVITY_KEYS); the page // re-mirrors them while it is open. +import type { RunStatus } from "./api"; import { LOGS_KEYS } from "./logsUrlState"; export type ActivityTab = "runs" | "admin" | "logs" | "quota"; -export type ActivityWindow = "1h" | "24h" | "7d" | "30d" | "90d"; -/** The run statuses the list can narrow to (the API's `RunStatus`). */ -export type RunStatusFilter = "running" | "ok" | "error" | "skipped" | "interrupted"; +export type ActivityWindow = "1h" | "24h" | "7d" | "30d" | "90d" | "all"; +/** The run statuses the list can narrow to: the API's `RunStatus` (held by backendIds.contract.test). */ +export type RunStatusFilter = RunStatus; export interface ActivityUrlState { tab: ActivityTab; @@ -24,7 +25,9 @@ export interface ActivityUrlState { } export const ACTIVITY_TABS: readonly ActivityTab[] = ["runs", "admin", "logs", "quota"]; -export const ACTIVITY_WINDOWS: readonly ActivityWindow[] = ["1h", "24h", "7d", "30d", "90d"]; +// `all` = everything still kept: runs and admin actions are kept 180 days by default (0 = for ever), +// longer than any fixed range. +export const ACTIVITY_WINDOWS: readonly ActivityWindow[] = ["1h", "24h", "7d", "30d", "90d", "all"]; export const RUN_STATUSES: readonly RunStatusFilter[] = [ "error", "interrupted", @@ -33,12 +36,13 @@ export const RUN_STATUSES: readonly RunStatusFilter[] = [ "ok", ]; -const WINDOW_HOURS: Record = { +const WINDOW_HOURS: Record = { "1h": 1, "24h": 24, "7d": 24 * 7, "30d": 24 * 30, "90d": 24 * 90, + all: null, }; export const DEFAULT_ACTIVITY_STATE: ActivityUrlState = { @@ -92,7 +96,9 @@ export function activityParamUpdates(state: ActivityUrlState): Record | null; /** The request the change was made in — its log lines are one pivot away. Null on older rows. */ request_id: string | null; + /** How many of that request's log lines are still kept (they go after days, the row after months). */ + log_lines: number; } /** A page of a history list (audit log, runs), newest first; `next_cursor` continues it. */ @@ -142,7 +144,8 @@ export interface RunRow { summary: string | null; result: JobResult | null; error: string | null; - failure: { code: string; [key: string]: unknown } | null; + /** A failure the job named (`progress.JobFailed`): its reason, and the values it is said with. */ + failure: { failure: string; [key: string]: unknown } | null; quota_units: number; /** How many stored log lines the run left (WARNING and up only). */ log_lines: number; @@ -249,8 +252,10 @@ function queryString(values: object): string { export type JobResult = { [key: string]: number | string | JobResult }; /** How a run ended — the server's `RunStatus` (app/runstatus.py) minus `running`. `interrupted`: - * the process stopped mid-run (a deploy, a crash), and its next start closed the row. */ + * the process stopped mid-run (a deploy, a crash), and its next start closed the row. The values are + * held to the server's by backendIds.contract.test (through RUN_STATUSES, typed as `RunStatus`). */ export type RunOutcome = "ok" | "error" | "skipped" | "interrupted"; +export type RunStatus = RunOutcome | "running"; /** One past run, as the row's history dots show it. */ export interface RunBrief { diff --git a/frontend/src/lib/auditText.dom.test.ts b/frontend/src/lib/auditText.dom.test.ts index a1bee7e6..6683fa9e 100644 --- a/frontend/src/lib/auditText.dom.test.ts +++ b/frontend/src/lib/auditText.dom.test.ts @@ -25,6 +25,7 @@ const row = (over: Partial): AuditRow => ({ before: null, after: null, request_id: null, + log_lines: 0, ...over, }); diff --git a/frontend/src/lib/navRing.contract.test.ts b/frontend/src/lib/navRing.contract.test.ts index 822caa51..d143831e 100644 --- a/frontend/src/lib/navRing.contract.test.ts +++ b/frontend/src/lib/navRing.contract.test.ts @@ -37,9 +37,10 @@ it("index.css draws an inset item's ring inside, after the outside rule", () => // The items that sit flush, by their source: a new one of the kind should be added here. const src = (p: string) => readFileSync(new URL(p, import.meta.url), "utf8"); -it.each([["../components/LogLine.tsx", /navItemAttrs\(navIndex, [^)]*\{ inset: true \}\)/]])( - "%s asks for the inside ring", - (file, re) => { - expect(src(file)).toMatch(re); - }, -); +// The history rows (runs, admin actions) and the log lines share one frame, HistoryRow. +it.each([ + ["../components/HistoryRow.tsx", /navItemAttrs\(navIndex, [^)]*\{ inset: true \}\)/], + ["../components/LogLine.tsx", / { + expect(src(file)).toMatch(re); +}); diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index a2de239b..75b31923 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -87,8 +87,12 @@ export const qk = { adminInvites: () => ["admin-invites"] as const, adminPendingAccess: () => ["admin-pending-access"] as const, adminConfig: () => ["admin-config"] as const, - adminAudit: (filters: object) => ["admin-audit", filters] as const, - adminAuditFacets: (filters: object) => ["admin-audit-facets", filters] as const, + adminAudit: (filters?: object) => + filters !== undefined ? (["admin-audit", filters] as const) : (["admin-audit"] as const), + adminAuditFacets: (filters?: object) => + filters !== undefined + ? (["admin-audit-facets", filters] as const) + : (["admin-audit-facets"] as const), adminRuns: (filters: object) => ["admin-runs", filters] as const, adminRunFacets: (filters: object) => ["admin-run-facets", filters] as const, // The console's filters are part of the key; `since` is not (it moves with the clock) — the diff --git a/frontend/src/lib/urlState.ts b/frontend/src/lib/urlState.ts index cbb92217..c4afc707 100644 --- a/frontend/src/lib/urlState.ts +++ b/frontend/src/lib/urlState.ts @@ -142,15 +142,32 @@ function isPage(p: string | null | undefined): p is Page { } // A page this build no longer has, the page that took its place, and the params that open its part -// of that page. A bookmark, the remembered last page and an old history entry can still name one; all -// three resolve through `resolvePage`, so a retired page lands where its content went instead of on -// the feed, and the navigation provider writes `params` into the address bar as it rewrites the page. -const RETIRED: Readonly }>> = { +// of that page — from the old link's own params. A bookmark, the remembered last page and an old +// history entry can still name one; all three resolve through `resolvePage`, so a retired page lands +// where its content went instead of on the feed, and the navigation provider writes `params` into the +// address bar as it rewrites the page (a null clears a key the old page owned). +type Updates = Record; +const RETIRED: Readonly< + Record Updates }> +> = { // The download queue became the tray; what it downloaded lives in Files. downloads: { page: "files" }, // The audit log and the server log became tabs of the activity page (its `at` key). - audit: { page: "activity", params: { at: "admin" } }, - logs: { page: "activity", params: { at: "logs" } }, + audit: { page: "activity", params: () => ({ at: "admin" }) }, + // The server log's own account, text and range became the page's shared filters; a link without + // a range meant the log's default then, 24 hours. + logs: { + page: "activity", + params: (old) => ({ + at: "logs", + au: old.get("lu"), + aq: old.get("lq"), + aw: old.get("lwin") ?? "24h", + lu: null, + lq: null, + lwin: null, + }), + }, }; const retired = (p: string | null | undefined) => @@ -162,9 +179,13 @@ export function resolvePage(p: string | null | undefined): Page | null { return retired(p)?.page ?? null; } -/** The params that open a retired page's part of its successor (none for a live page). */ -export function retiredParams(p: string | null | undefined): Record { - return isPage(p) ? {} : (retired(p)?.params ?? {}); +/** The params that open a retired page's part of its successor, read from the old link's `params` + * (none for a live page). */ +export function retiredParams( + p: string | null | undefined, + params = new URLSearchParams(window.location.search), +): Updates { + return isPage(p) ? {} : (retired(p)?.params?.(params) ?? {}); } export function readPage(): Page {