From 45c0ee7d5884c99a1ff0ab32be023f55734f3676 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 17 Oct 2026 23:08:55 +0200 Subject: [PATCH 01/16] feat(scheduler): every background run is a durable row What a job last did lived in the API process's memory and was gone at every restart; the audit log kept a run only when it was manual, failed or changed something. Now each run is a job_runs row written at its start and its end in its own session: who started it, its outcome as data, what it spent of the YouTube quota (quota_events.job_run_id) and the lines it logged (log_events.run_id, filterable in the log console). - a run a stopped process left open is marked interrupted at startup - "run now" is a one-shot job on the scheduler, which now always starts (interval jobs only with SCHEDULER_ENABLED); the manual actor is reset after the run, since pool threads are reused - the dashboard's last run comes from job_runs, so it survives a deploy - runs leave the audit log: migration 0109 moves the old job_run rows over (and back on downgrade); the audit page's run folding goes - job_run_retention_days (default 30) prunes old runs in audit_gc --- CLAUDE.md | 4 +- backend/alembic/versions/0109_job_runs.py | 139 +++++++++ backend/app/README.md | 7 +- backend/app/audit.py | 2 - backend/app/config.py | 3 + backend/app/jobruns.py | 104 +++++++ backend/app/logs.py | 12 +- backend/app/logsink.py | 1 + backend/app/models.py | 52 ++++ backend/app/quota.py | 14 +- backend/app/routes/admin_logs.py | 8 +- backend/app/routes/scheduler.py | 2 +- backend/app/runstatus.py | 35 +++ backend/app/scheduler.py | 250 +++++++-------- backend/app/sysconfig.py | 2 + backend/tests/test_admin_logs.py | 11 +- backend/tests/test_migration_0109.py | 98 ++++++ backend/tests/test_plex_people_enrich.py | 7 - backend/tests/test_scheduler.py | 289 +++++++++++++++--- frontend/src/components/AuditEarlierRuns.tsx | 48 --- frontend/src/components/AuditLog.tsx | 8 +- .../src/components/auditColumns.dom.test.tsx | 95 +----- frontend/src/components/auditColumns.tsx | 26 +- frontend/src/i18n/locales/en/audit.json | 6 +- .../src/i18n/locales/en/auditActions.json | 1 - frontend/src/i18n/locales/en/config.json | 4 + frontend/src/i18n/locales/hu/audit.json | 6 +- .../src/i18n/locales/hu/auditActions.json | 1 - frontend/src/i18n/locales/hu/config.json | 4 + frontend/src/lib/auditGroups.test.ts | 72 ----- frontend/src/lib/auditGroups.ts | 48 --- 31 files changed, 870 insertions(+), 489 deletions(-) create mode 100644 backend/alembic/versions/0109_job_runs.py create mode 100644 backend/app/jobruns.py create mode 100644 backend/app/runstatus.py create mode 100644 backend/tests/test_migration_0109.py delete mode 100644 frontend/src/components/AuditEarlierRuns.tsx delete mode 100644 frontend/src/lib/auditGroups.test.ts delete mode 100644 frontend/src/lib/auditGroups.ts diff --git a/CLAUDE.md b/CLAUDE.md index 2acf4a0d..dba071cb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,7 +63,7 @@ how it gets built. Concretely: ## Architecture shape (don't fight it) `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, -`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, +`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`, `sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`, `security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own @@ -92,7 +92,7 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory | Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) | | User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI | | Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` | -| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) | +| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` and `jobruns` own their session) | | Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter | | Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals | | Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers | diff --git a/backend/alembic/versions/0109_job_runs.py b/backend/alembic/versions/0109_job_runs.py new file mode 100644 index 00000000..c642a720 --- /dev/null +++ b/backend/alembic/versions/0109_job_runs.py @@ -0,0 +1,139 @@ +"""Background runs get a durable history + +Revision ID: 0109_job_runs +Revises: 0108_folder_links +Create Date: 2026-10-17 + +What a scheduler job last did lived in the API process's memory, gone at every restart; the audit +log kept a row for a run only when it was manual, failed, or changed something. Now every run is a +`job_runs` row, written when it starts and when it ends. A run's quota spend points at it +(`quota_events.job_run_id`), and so do the lines it logged (`log_events.run_id`, no foreign key: the +log writer runs behind the event). + +Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows +MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the +audit log. The downgrade moves every run back into the audit log as such a row, then drops the table. +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0109_job_runs" +down_revision: str | None = "0108_folder_links" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "job_runs", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("job", sa.String(40), nullable=False), + sa.Column("trigger", sa.String(16), nullable=False), + sa.Column("actor_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="SET NULL")), + sa.Column("target", sa.String(128)), + sa.Column("status", sa.String(16), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True)), + sa.Column("summary", sa.Text()), + sa.Column("result", postgresql.JSONB()), + sa.Column("error", sa.Text()), + sa.Column("failure", postgresql.JSONB()), + sa.Column("quota_units", sa.Integer(), nullable=False, server_default="0"), + sa.CheckConstraint( + "status IN ('running', 'ok', 'error', 'skipped', 'interrupted')", + name="ck_job_runs_status", + ), + sa.CheckConstraint( + "trigger IN ('scheduled', 'manual', 'ad_hoc')", name="ck_job_runs_trigger" + ), + sa.CheckConstraint( + "(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished" + ), + ) + op.create_index("ix_job_runs_job_started", "job_runs", ["job", "started_at"]) + op.create_index("ix_job_runs_actor_id", "job_runs", ["actor_id"]) + op.create_index("ix_job_runs_started_at", "job_runs", ["started_at"]) + + op.add_column( + "quota_events", + sa.Column("job_run_id", sa.BigInteger(), sa.ForeignKey("job_runs.id", ondelete="SET NULL")), + ) + op.create_index("ix_quota_events_job_run_id", "quota_events", ["job_run_id"]) + + op.add_column("log_events", sa.Column("run_id", sa.BigInteger())) + op.create_index( + "ix_log_events_run", + "log_events", + ["run_id"], + postgresql_where=sa.text("run_id IS NOT NULL"), + ) + + # The audit trail's runs move over. A status the run table does not know (none was ever + # written) would fail the CHECK, so the move is limited to the three the scheduler wrote. + op.execute( + """ + INSERT INTO job_runs (job, trigger, actor_id, status, started_at, finished_at, summary, + result, error, failure) + SELECT target_id, + CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END, + actor_id, + after::jsonb ->> 'status', + created_at, + created_at, + CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb ->> 'summary' END, + CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb -> 'result' END, + CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb ->> 'summary' END, + CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END + FROM audit_log + WHERE action = 'job_run' + AND target_id IS NOT NULL + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped') + ORDER BY id + """ + ) + op.execute( + """ + DELETE FROM audit_log + WHERE action = 'job_run' + AND target_id IS NOT NULL + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped') + """ + ) + + +def downgrade() -> None: + op.execute( + """ + INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at) + SELECT actor_id, + 'job_run', + 'job', + job, + left(job || ': ' || status + || coalesce(' — ' || CASE WHEN status = 'error' THEN error ELSE summary END, ''), + 255), + (jsonb_build_object( + 'status', status, + 'summary', CASE WHEN status = 'error' THEN error ELSE summary END) + || CASE + WHEN status = 'error' AND failure IS NOT NULL + THEN jsonb_build_object('result', failure) + WHEN status <> 'error' AND result IS NOT NULL + THEN jsonb_build_object('result', result) + ELSE '{}'::jsonb + END)::json, + coalesce(finished_at, started_at) + FROM job_runs + ORDER BY id + """ + ) + op.drop_index("ix_log_events_run", table_name="log_events") + op.drop_column("log_events", "run_id") + op.drop_index("ix_quota_events_job_run_id", table_name="quota_events") + op.drop_column("quota_events", "job_run_id") + op.drop_index("ix_job_runs_started_at", table_name="job_runs") + op.drop_index("ix_job_runs_actor_id", table_name="job_runs") + op.drop_index("ix_job_runs_job_started", table_name="job_runs") + op.drop_table("job_runs") diff --git a/backend/app/README.md b/backend/app/README.md index 6ff33ab1..0dafee5a 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -21,7 +21,8 @@ Three layers, by *who* runs the code — not by feature: `issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker. 3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth` (with `apitokens`, a machine's - folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`, + folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`, `jobruns` (every + background run's durable row, its words in `runstatus`), `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the @@ -36,7 +37,7 @@ pre-abstract for a single caller; don't leave shared logic duplicated once there ## Transaction ownership This follows CLAUDE.md's canonical rule verbatim — **"the handler/job-wrapper commits; helpers flush -(only `quota` owns its session)"** — so read it that way, not as an exception to it. The commit belongs +(only `quota` and `jobruns` own their session)"** — so read it that way, not as an exception to it. The commit belongs to the **operation owner**; anything called *inside* another transaction only stages its change (`db.add` / `flush`) and lets the owner commit, so the whole operation is one transaction. `audit.record()` is the reference: it adds the audit row and the caller commits it together with the @@ -45,7 +46,7 @@ mutation it records, so an audit entry can never exist without its change (or vi - The **operation owner** is usually the route handler. A `service.*` operation the handler delegates a whole transition to (e.g. `service.cancel`) *is* that owner and commits its one transition — it isn't a mid-operation helper. A helper that owns its OWN short-lived session (the worker's `jobdb` writes, - the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes. + `jobruns`' run rows — the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes. - `SessionLocal` is `expire_on_commit=False`: after a raw `UPDATE` + commit, an ORM object you already loaded is **stale**. Read the fresh value from the row (RETURNING, or a re-`SELECT`), not the ORM attribute — see `service.decr_ref_count`. diff --git a/backend/app/audit.py b/backend/app/audit.py index d014d313..52349979 100644 --- a/backend/app/audit.py +++ b/backend/app/audit.py @@ -48,8 +48,6 @@ class AuditAction: MAINTENANCE_TUNE = "maintenance_tune" SYNC_PAUSE = "sync_pause" SYNC_RESUME = "sync_resume" - # Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic) - JOB_RUN = "job_run" # Per-user download cookie/auth credential set or removed (the value is NEVER recorded) DOWNLOAD_CREDENTIAL_SET = "download_credential_set" DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete" diff --git a/backend/app/config.py b/backend/app/config.py index fd477645..f5b66200 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -268,6 +268,9 @@ class Settings(BaseSettings): # YouTube quota history (quota_events): days kept (0 = forever). A year, so the admin usage view's # longest window (365 days) stays whole; the same daily cleanup job prunes it. quota_retention_days: int = 365 + # Background-run history (job_runs): days kept (0 = forever). Every run is a row (~1–1.5k a day + # at the default intervals), so a month; the same daily cleanup job prunes it. + job_run_retention_days: int = 30 # On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat". download_layout: DownloadLayout = "plex" # Default SponsorBlock (skip sponsor segments) for new custom profiles. diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py new file mode 100644 index 00000000..993af617 --- /dev/null +++ b/backend/app/jobruns.py @@ -0,0 +1,104 @@ +"""The durable history of background runs — the `job_runs` table. + +A run is written twice: a `running` row when it starts, its outcome when it ends. Each write is its +OWN short session, committed at once, for the reason `quota.record_usage` gives: the run's own work +may roll back (its `except: db.rollback()`), and the history of what happened must not go with it — +nor may this bookkeeping commit the job's half-done session. + +Only the API process runs jobs (the worker never starts the scheduler), so at its startup nothing can +still be running: `interrupt_leftovers` closes what a stopped process left open. +""" + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import delete, func, select, update +from sqlalchemy.engine import CursorResult +from sqlalchemy.orm import Session + +from app.db import SessionLocal +from app.models import JobRun, QuotaEvent +from app.runstatus import RunStatus, RunTrigger + + +def start(job: str, trigger: RunTrigger, actor_id: int | None, target: str | None = None) -> int: + """Open a run's row; returns its id.""" + with SessionLocal() as s: + row = JobRun( + job=job, + trigger=trigger, + actor_id=actor_id, + target=target, + status=RunStatus.RUNNING, + started_at=datetime.now(timezone.utc), + ) + s.add(row) + s.commit() + return row.id + + +def finish( + run_id: int, + status: RunStatus, + *, + summary: str | None = None, + result: dict | None = None, + error: str | None = None, + failure: dict | None = None, +) -> None: + """Close a run with its outcome and what it spent. Conditional on the row still running: a row + the startup sweep already called interrupted is not rewritten by a thread that outlived it.""" + with SessionLocal() as s: + units = s.scalar( + select(func.coalesce(func.sum(QuotaEvent.units), 0)).where( + QuotaEvent.job_run_id == run_id + ) + ) + s.execute( + update(JobRun) + .where(JobRun.id == run_id, JobRun.status == RunStatus.RUNNING) + .values( + status=status, + finished_at=datetime.now(timezone.utc), + summary=summary, + result=result, + error=error, + failure=failure, + quota_units=units or 0, + ) + ) + s.commit() + + +def interrupt_leftovers() -> int: + """Every row a previous process left `running` → `interrupted`. Called once at startup, before + the scheduler can open a new run. Returns how many.""" + with SessionLocal() as s: + res = s.execute( + update(JobRun) + .where(JobRun.status == RunStatus.RUNNING) + .values(status=RunStatus.INTERRUPTED, finished_at=datetime.now(timezone.utc)) + ) + s.commit() + return res.rowcount if isinstance(res, CursorResult) else 0 + + +def last_runs(db: Session) -> dict[str, JobRun]: + """Each job's most recent run (any trigger), by job id.""" + rows = db.execute( + select(JobRun) + .distinct(JobRun.job) + .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc()) + ).scalars() + return {r.job: r for r in rows} + + +def gc(db: Session, retention_days: int) -> int: + """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is + never reaped. The caller commits.""" + if retention_days <= 0: + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + res = db.execute( + delete(JobRun).where(JobRun.started_at < cutoff, JobRun.status != RunStatus.RUNNING) + ) + return res.rowcount if isinstance(res, CursorResult) else 0 diff --git a/backend/app/logs.py b/backend/app/logs.py index 62ff3609..fc6551bd 100644 --- a/backend/app/logs.py +++ b/backend/app/logs.py @@ -10,8 +10,8 @@ the logs were both unjoinable and leaky. `configure_logging()` replaces all four Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line: ts · level · sev · source · logger · event · msg · attrs plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`, -`job_id`) and the app version. That shape is what the admin log console filters on and what a Gitea -issue attachment will carry, so it is deliberately flat and stable. +`job_id`, `run_id`) and the app version. That shape is what the admin log console filters on and +what a Gitea issue attachment will carry, so it is deliberately flat and stable. Call sites stay ordinary stdlib logging: @@ -81,15 +81,23 @@ _debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar( "log_debug_session", default=None ) _job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None) +# The background run (app/jobruns.py) this context is doing; quota's spend is tagged with it too. +_run_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_run_id", default=None) _CONTEXT_VARS = { "request_id": _request_id, "user_id": _user_id, "debug_session_id": _debug_session, "job_id": _job_id, + "run_id": _run_id, } +def current_run_id() -> int | None: + """The background run this context is doing, if any — `bind(run_id=…)` sets it.""" + return _run_id.get() + + class _RequestScope: """What the auth dependency learns about a request, for every line logged after it. diff --git a/backend/app/logsink.py b/backend/app/logsink.py index c63cea8f..2310ebeb 100644 --- a/backend/app/logsink.py +++ b/backend/app/logsink.py @@ -362,6 +362,7 @@ def serialize(row: LogEvent) -> dict: "request_id": row.request_id, "user_id": row.user_id, "job_id": row.job_id, + "run_id": row.run_id, "debug_session_id": row.debug_session_id, "version": row.version, } diff --git a/backend/app/models.py b/backend/app/models.py index f7c4567c..880fa7de 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -24,6 +24,7 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from app.db import Base +from app.runstatus import RunStatus, RunTrigger, sql_in from app.utils import canonical_email # Every user-supplied display string in this schema shares one width, and it lives HERE so the @@ -563,6 +564,11 @@ class QuotaEvent(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), index=True ) + # The background run that spent it (app/jobruns.py); NULL = a request's own spend. SET NULL: a + # reaped run leaves the spend in the budget's history. + job_run_id: Mapped[int | None] = mapped_column( + ForeignKey("job_runs.id", ondelete="SET NULL"), index=True + ) class AuditLog(Base): @@ -609,6 +615,8 @@ class LogEvent(Base): "debug_session_id", postgresql_where="debug_session_id IS NOT NULL", ), + # A run's lines are `WHERE run_id = …`; request lines carry none. + Index("ix_log_events_run", "run_id", postgresql_where="run_id IS NOT NULL"), ) id: Mapped[int] = mapped_column(BigInteger, primary_key=True) @@ -624,7 +632,11 @@ class LogEvent(Base): request_id: Mapped[str | None] = mapped_column(String(64), index=True) user_id: Mapped[int | None] = mapped_column(Integer) debug_session_id: Mapped[str | None] = mapped_column(String(64)) + # A DOWNLOAD job's id (download_jobs) — not a background run, which is `run_id`. job_id: Mapped[int | None] = mapped_column(Integer) + # The background run (job_runs) that logged it. No foreign key, as `user_id`: the writer runs + # behind the event, and a run reaped meanwhile must not fail the batch its lines travel in. + run_id: Mapped[int | None] = mapped_column(BigInteger) version: Mapped[str | None] = mapped_column(String(64)) @@ -700,6 +712,46 @@ class SchedulerSetting(Base): interval_minutes: Mapped[int] = mapped_column(Integer) +class JobRun(Base): + """One run of a background job — scheduled, an admin's "run now", or a user's ad-hoc work on one + target — written when it starts and again when it ends (app/jobruns.py), each in its own + committed session, so the history survives the job's own rollback and a restart. A row left + `running` by a process that died is set `interrupted` at the next startup. + + `quota_units` is what the run spent of the YouTube budget: the `quota_events` rows carrying its + id, summed when it ends. `summary` is the English one-line fallback; `result` / `failure` are the + same outcome as data, which the client names in the reader's language.""" + + __tablename__ = "job_runs" + __table_args__ = ( + Index("ix_job_runs_job_started", "job", "started_at"), + CheckConstraint(f"status IN ({sql_in(RunStatus)})", name="ck_job_runs_status"), + CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"), + # Only a run still going has no end. + CheckConstraint( + "(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished" + ), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + job: Mapped[str] = mapped_column(String(40)) + trigger: Mapped[str] = mapped_column(String(16)) + # Who started it: the admin of a manual run, the user of an ad-hoc one; NULL = the schedule. + actor_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), index=True + ) + # What an ad-hoc run worked on (a channel id); NULL for the recurring jobs. + target: Mapped[str | None] = mapped_column(String(128)) + status: Mapped[str] = mapped_column(String(16)) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + summary: Mapped[str | None] = mapped_column(Text) + result: Mapped[dict | None] = mapped_column(JSONB) + error: Mapped[str | None] = mapped_column(Text) + failure: Mapped[dict | None] = mapped_column(JSONB) + quota_units: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + class SystemConfig(Base): """Admin-set overrides for operational config that otherwise comes from env/config defaults. Generic key/value store (one row per key); absent = use the env/config diff --git a/backend/app/quota.py b/backend/app/quota.py index c0ef1db5..6f2e5c81 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -16,7 +16,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session -from app import sysconfig +from app import logs, sysconfig from app.db import SessionLocal from app.models import ApiQuotaUsage, QuotaEvent @@ -204,6 +204,14 @@ def record_usage(units: int) -> None: ) with SessionLocal() as qs: qs.execute(stmt) - # Audit detail: who/what spent it (per-user attribution; NULL actor = system). - qs.add(QuotaEvent(user_id=_actor_id.get(), action=_action.get(), units=units)) + # Audit detail: who/what spent it (per-user attribution; NULL actor = system), and the + # background run it belongs to, so the run's row can say what it cost. + qs.add( + QuotaEvent( + user_id=_actor_id.get(), + action=_action.get(), + units=units, + job_run_id=logs.current_run_id(), + ) + ) qs.commit() diff --git a/backend/app/routes/admin_logs.py b/backend/app/routes/admin_logs.py index 8202cafd..60e2d50b 100644 --- a/backend/app/routes/admin_logs.py +++ b/backend/app/routes/admin_logs.py @@ -86,6 +86,7 @@ class LogFilters: event: str | None request_id: str | None job_id: int | None + run_id: int | None q: str | None since: datetime | None until: datetime | None @@ -113,6 +114,8 @@ class LogFilters: out.append(LogEvent.request_id == self.request_id) if self.job_id is not None: out.append(LogEvent.job_id == self.job_id) + if self.run_id is not None: + out.append(LogEvent.run_id == self.run_id) if self.since is not None: out.append(LogEvent.ts >= _utc(self.since)) if self.until is not None: @@ -140,11 +143,14 @@ def log_filters( ), request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"), job_id: int | None = Query(default=None, ge=1), + run_id: int | None = Query(default=None, ge=1), q: str | None = Query(default=None, max_length=200), since: datetime | None = None, until: datetime | None = None, ) -> LogFilters: - return LogFilters(list(levels), list(sources), user, event, request_id, job_id, q, since, until) + return LogFilters( + list(levels), list(sources), user, event, request_id, job_id, run_id, q, since, until + ) @router.get("") diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index 579a691f..c61f3305 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -117,7 +117,7 @@ def set_maintenance_settings( @router.get("") def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict: - snap = scheduler_snapshot() + snap = scheduler_snapshot(db) # Work still queued (shared catalog, so these are instance-wide, not per-user). deep_requested = ( diff --git a/backend/app/runstatus.py b/backend/app/runstatus.py new file mode 100644 index 00000000..497a22b1 --- /dev/null +++ b/backend/app/runstatus.py @@ -0,0 +1,35 @@ +"""The words a background run's row is written with — the `job_runs` table's status and trigger. + +One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the +model's CHECK constraints are built from these sets, and the dashboard reads them back. A run's +status is never typed as a string anywhere else (the download pipeline's `cancelled`/`canceled` +drift, `downloads/status.py`, is why). +""" + +from enum import StrEnum + + +class RunStatus(StrEnum): + RUNNING = "running" + OK = "ok" + ERROR = "error" + # Sync was paused: the run was due and deliberately did nothing. + SKIPPED = "skipped" + # The process stopped while the run was going (a deploy, a crash): set at the next startup, so a + # row never claims to be running in a process that no longer exists. + INTERRUPTED = "interrupted" + + +class RunTrigger(StrEnum): + # The run's interval came round. + SCHEDULED = "scheduled" + # An admin pressed "run now" (or "run all"). + MANUAL = "manual" + # A user's own action started work on one target (a channel's full history), outside the + # recurring jobs. + AD_HOC = "ad_hoc" + + +def sql_in(values: type[StrEnum]) -> str: + """`'a', 'b'` — the IN-list a CHECK constraint names these values with.""" + return ", ".join(f"'{v.value}'" for v in values) diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 8aa3f651..a007f02b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -6,11 +6,12 @@ import threading from datetime import datetime, timezone from typing import Callable +from apscheduler.executors.pool import ThreadPoolExecutor +from apscheduler.jobstores.base import ConflictingIdError from apscheduler.schedulers.background import BackgroundScheduler from sqlalchemy import select -from app import audit, debugsession, logs, logsink, progress, quota -from app.audit import AuditAction +from app import audit, debugsession, jobruns, logs, logsink, progress, quota from app.config import settings from app.db import SessionLocal from app.downloads.gc import run_download_gc @@ -20,6 +21,7 @@ from app.plex.tmdb import run_enrich as run_tmdb_enrich from app.plex.tmdb import run_people_enrich from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync from app.notifications import create_notification +from app.runstatus import RunStatus, RunTrigger from app.state import is_sync_paused from app.sync.autotag import run_autotag_all from app.sync.explore import purge_ephemeral @@ -37,10 +39,9 @@ from app.sync.runner import ( log = logs.get_logger(__name__) _scheduler: BackgroundScheduler | None = None -# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the -# scheduler lives in the same single-worker process as the API, so a request can read it -# directly). Ephemeral by design — it reflects "what the scheduler is doing right now and -# how the last runs went" since this process started; durable history isn't the goal here. +# What is running RIGHT NOW, per job: the single-run claim and the live progress. In memory on +# purpose — the scheduler lives in the API's one process, and progress changes every second. What a +# run did is durable instead: its `job_runs` row (app/jobruns.py), which outlives a restart. _activity: dict[str, dict] = {} _activity_lock = threading.Lock() @@ -130,49 +131,6 @@ def _notify_done( log.exception("failed to write completion notification for job %s", job_id) -def _run_changed(result) -> bool: - """Did an automatic job run actually do something worth an audit row? (No-op polls don't.) - A "change" = a truthy NUMERIC count; status-string dicts like {"skipped": "disabled"} (Plex - off — the default) are no-ops, not changes, so they don't flood the trail every interval.""" - if result is None: - return False - if isinstance(result, dict): - # A phased job (backfill's recent/deep, people_enrich's bios/portraits) nests a result per - # phase: a count inside a phase is a change too. - return any( - _run_changed(v) if isinstance(v, dict) else isinstance(v, (int, float)) and bool(v) - for v in result.values() - ) - if isinstance(result, (int, float)): - return result != 0 - return bool(result) - - -def _audit_run(db, actor_id, job_id, status, summary, result) -> None: - """Write ONE run-summary audit row per job run (never per-item). Manual runs (an admin - clicked "run now" → actor_id set) and errors are always logged; automatic runs only when - they changed something, so routine no-op polls don't spam the trail. Best-effort.""" - if actor_id is None and status == "ok" and not _run_changed(result): - return - # The result as data too: the audit log names it in the reader's language, as the inbox - # does; `summary` stays the English line of the row's own summary column. - data = _result_data(result) - try: - audit.record( - db, - actor_id, - AuditAction.JOB_RUN, - target_type="job", - target_id=job_id, - summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""), - after={"status": status, "summary": summary, **({"result": data} if data else {})}, - ) - db.commit() - except Exception: - db.rollback() - log.exception("failed to write audit row for job %s", job_id) - - def _job(name: str, fn) -> None: # The one place both run paths converge (manual trigger AND scheduled fire), so claim the run # HERE, atomically: whichever reaches this first wins, the other skips. This is the real @@ -180,11 +138,29 @@ def _job(name: str, fn) -> None: if not _claim_running(name): log.info("job %s already running — skipping this trigger", name) return - db = SessionLocal() actor_id = _manual_actor.get() # set only for a manual "run now" trigger - _record( - name, last_started=datetime.now(timezone.utc).isoformat(), last_error=None, progress=None - ) # running already claimed + trigger = RunTrigger.SCHEDULED if actor_id is None else RunTrigger.MANUAL + _record(name, progress=None) # running already claimed + db = SessionLocal() + try: + try: + run_id = jobruns.start(name, trigger, actor_id) + except Exception: + # No row to tell it in: the database is what failed. The log is the record. + log.exception("job %s not started: its run could not be recorded", name) + return + with logs.bind(run_id=run_id): + _run(db, name, fn, run_id, actor_id) + finally: + # ALWAYS release the claim, even if the body raised before its own status update — so a + # failure can never wedge the job as permanently "running". + _record(name, running=False, progress=None) + db.close() + + +def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None: + """The run itself, inside its `job_runs` row: the pause-skip, the work, and its outcome — on + the row, and in the triggering admin's inbox for a manual run.""" def sink(current, total, phase): _record(name, progress={"current": current, "total": total, "phase": phase}) @@ -192,34 +168,18 @@ def _job(name: str, fn) -> None: try: if is_sync_paused(db): log.info("job %s skipped (sync paused)", name) - _record( - name, - status="skipped", - last_finished=datetime.now(timezone.utc).isoformat(), - last_result="paused", - result=_PAUSED, - progress=None, - ) + jobruns.finish(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED) if actor_id is not None: _notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED) - _audit_run(db, actor_id, name, "skipped", "sync paused", _PAUSED) return with progress.bind(sink): result = fn(db) summary = _summarize(result) data = _result_data(result) log.info("job %s -> %s", name, result) - _record( - name, - status="ok", - last_finished=datetime.now(timezone.utc).isoformat(), - last_result=summary, - result=data, - progress=None, - ) + jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data) if actor_id is not None: _notify_done(db, actor_id, name, "ok", summary, data) - _audit_run(db, actor_id, name, "ok", summary, result) except Exception as exc: db.rollback() log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) @@ -229,22 +189,9 @@ def _job(name: str, fn) -> None: failure = ( {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None ) - _record( - name, - status="error", - last_finished=datetime.now(timezone.utc).isoformat(), - last_error=err, - result=failure, - progress=None, - ) + jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure) if actor_id is not None: _notify_done(db, actor_id, name, "error", err, failure) - _audit_run(db, actor_id, name, "error", err, failure) - finally: - # ALWAYS release the claim, even if the body raised before its own status update — so a - # failure can never wedge the job as permanently "running". - _record(name, running=False) - db.close() # A skipped run's result, in the same shape as a job's own `{"skipped": …}`. @@ -271,11 +218,17 @@ def _summarize(result) -> str | None: return str(result) -def scheduler_snapshot() -> dict: - """What the scheduler is doing, for the admin dashboard. `running_here` is False in - environments where the scheduler is disabled (e.g. local dev shares the prod-ish DB but - runs no scheduler), so the UI can say so while still showing DB-derived queue/quota.""" - running_here = _scheduler is not None +def _iso(ts: datetime | None) -> str | None: + return ts.astimezone(timezone.utc).isoformat() if ts else None + + +def scheduler_snapshot(db) -> dict: + """What the scheduler is doing, for the admin dashboard: each job's live state from this process + and its last run from `job_runs`, so a restart no longer blanks "when did it last run, and how + did it go". `running_here` is False where the scheduler runs no interval jobs + (SCHEDULER_ENABLED=false), so the UI can say so while still showing the DB-derived history, + queue and quota.""" + running_here = _scheduler is not None and settings.scheduler_enabled next_runs: dict[str, str | None] = {} live_intervals: dict[str, int] = {} if _scheduler is not None: @@ -287,27 +240,41 @@ def scheduler_snapshot() -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} + last = jobruns.last_runs(db) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) + run = last.get(job_id) + # A run still going has no outcome yet: only an ended run says how it went. + ended = run if run is not None and run.status != RunStatus.RUNNING else None jobs.append( { "id": job_id, "interval_minutes": live_intervals.get(job_id, default_interval), "next_run": next_runs.get(job_id), "running": a.get("running", False), - "status": a.get("status"), - "last_started": a.get("last_started"), - "last_finished": a.get("last_finished"), - "last_result": a.get("last_result"), - "result": a.get("result"), - "last_error": a.get("last_error"), + "status": ended.status if ended else None, + "last_started": _iso(run.started_at) if run else None, + "last_finished": _iso(run.finished_at) if run else None, + "last_result": ended.summary if ended else None, + # The outcome as data: the result, or on a failed run the failure the job named. + "result": _outcome_data(ended), + "last_error": ended.error if ended else None, + "last_trigger": run.trigger if run else None, + "last_run_id": run.id if run else None, + "quota_units": ended.quota_units if ended else None, "progress": a.get("progress"), } ) return {"running_here": running_here, "enabled": settings.scheduler_enabled, "jobs": jobs} +def _outcome_data(run) -> dict | None: + if run is None: + return None + return run.failure if run.status == RunStatus.ERROR else run.result + + def _rss_job() -> None: _job("rss_poll", run_rss_poll) @@ -408,15 +375,17 @@ def _people_enrich_job() -> None: def _audit_gc_job() -> None: - # The history cleanup: audit-log rows past audit_retention_days and YouTube quota history past - # quota_retention_days (0 = keep forever, each). Pure DB, no quota. + # The history cleanup: audit-log rows past audit_retention_days, YouTube quota history past + # quota_retention_days and finished runs past job_run_retention_days (0 = keep forever, each). + # Pure DB, no quota. def work(db): from app import sysconfig reaped = audit.gc(db, sysconfig.get_int(db, "audit_retention_days")) quota_reaped = quota.gc(db, sysconfig.get_int(db, "quota_retention_days")) + runs_reaped = jobruns.gc(db, sysconfig.get_int(db, "job_run_retention_days")) db.commit() - return {"reaped": reaped, "quota_reaped": quota_reaped} + return {"reaped": reaped, "quota_reaped": quota_reaped, "runs_reaped": runs_reaped} _job("audit_gc", work) @@ -485,15 +454,29 @@ def running_job_ids() -> set[str]: return {jid for jid, a in _activity.items() if a.get("running")} -def trigger_job(job_id: str, actor_id: int | None = None) -> str: - """Run a job immediately in a background thread, independent of its interval schedule. - The wrapper handles its own DB session, activity tracking and pause-skip, so the live - dashboard reflects the run. Returns "started", "already_running", or raises KeyError for - an unknown job. Refusing a concurrent run keeps a manual trigger from overlapping a - scheduled run (APScheduler enforces max_instances=1 for the scheduled side). +def _as_actor(fn: Callable[[], None], actor_id: int | None) -> Callable[[], None]: + """`fn` run as `actor_id`'s manual run. The actor is RESET afterwards: the executor's pool + threads are reused and keep their context from task to task, so an actor left set would make + the next scheduled run that thread picks up look like this admin's manual run.""" - `actor_id` (the admin who clicked "run now") is stashed in a contextvar inside the new - thread so the run posts a completion notification to that admin's inbox.""" + def run() -> None: + token = _manual_actor.set(actor_id) + try: + fn() + finally: + _manual_actor.reset(token) + + return run + + +def trigger_job(job_id: str, actor_id: int | None = None) -> str: + """Run a job now, independent of its interval schedule: a one-shot job on the same scheduler, + so a manual run shares the scheduled runs' executor and shutdown — no thread of its own. The + wrapper handles its own DB session, run row and pause-skip. Returns "started", + "already_running", or raises KeyError for an unknown job. + + `actor_id` (the admin who clicked "run now") is set in a contextvar inside the run's thread, so + the run is recorded as theirs and posts a completion notification to their inbox.""" fn = JOB_FUNCS.get(job_id) if fn is None: raise KeyError(job_id) @@ -502,13 +485,18 @@ def trigger_job(job_id: str, actor_id: int | None = None) -> str: # _job claims, sees it's taken, and skips — no double run. if _is_running(job_id): return "already_running" + sched = _scheduler + if sched is None: + # Only between shutdown and exit: start_scheduler runs at every API startup. + raise RuntimeError("the scheduler is not running") - def run() -> None: - if actor_id is not None: - _manual_actor.set(actor_id) - fn() - - threading.Thread(target=run, name=f"trigger-{job_id}", daemon=True).start() + try: + # One id per job: a second click before the first one-shot has even started is the same + # run, not a queued second one. misfire_grace_time=None: a busy pool delays it, never drops it. + sched.add_job(_as_actor(fn, actor_id), "date", id=f"{job_id}@now", misfire_grace_time=None) + # silent-ok: not a failure — the answer is the 409 the admin sees ("already running"). + except ConflictingIdError: + return "already_running" return "started" @@ -521,22 +509,36 @@ def trigger_all(actor_id: int | None = None) -> list[str]: return started +# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of +# 10 would queue a scheduled fire behind long manual runs. +_POOL_SIZE = len(JOB_FUNCS) + 8 + + def start_scheduler() -> None: + """Start this process's scheduler. It always starts, so "run now" works on every install; only + with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can + run, every run a previous process left open is closed as interrupted.""" global _scheduler - if not settings.scheduler_enabled or _scheduler is not None: + if _scheduler is not None: return - # Effective intervals = env defaults overlaid with any admin overrides from the DB. - db = SessionLocal() - try: - iv = load_intervals(db) - finally: - db.close() - scheduler = BackgroundScheduler(timezone="UTC") - for job_id, fn in JOB_FUNCS.items(): - scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id) + left = jobruns.interrupt_leftovers() + if left: + log.warning("%s run(s) left running by the previous process marked interrupted", left) + scheduler = BackgroundScheduler( + timezone="UTC", executors={"default": ThreadPoolExecutor(_POOL_SIZE)} + ) + if settings.scheduler_enabled: + # Effective intervals = env defaults overlaid with any admin overrides from the DB. + db = SessionLocal() + try: + iv = load_intervals(db) + finally: + db.close() + for job_id, fn in JOB_FUNCS.items(): + scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id) scheduler.start() _scheduler = scheduler - log.info("scheduler started") + log.info("scheduler started (interval jobs %s)", "on" if settings.scheduler_enabled else "off") def shutdown_scheduler() -> None: diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index daf8b1ce..4d91b865 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -181,6 +181,8 @@ SPECS: tuple[ConfigSpec, ...] = ( # --- Audit log --- # Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever). ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650), + # Days the job_runs table keeps a finished background run before the audit_gc job prunes it. + ConfigSpec("job_run_retention_days", "int", "audit", "job_run_retention_days", min=0, max=3650), # --- Diagnostics --- # Runtime log level for the API process. Applied without a restart — see routes/config.py and # main.py's lifespan. diff --git a/backend/tests/test_admin_logs.py b/backend/tests/test_admin_logs.py index 7277108c..6677d67d 100644 --- a/backend/tests/test_admin_logs.py +++ b/backend/tests/test_admin_logs.py @@ -24,6 +24,7 @@ def _row(**kw) -> dict: "request_id": None, "user_id": None, "job_id": None, + "run_id": None, "version": "0.90.0", } row.update(kw) @@ -80,6 +81,7 @@ def test_newest_first_and_every_row_once_across_pages_even_on_equal_timestamps(d ({"event": "none"}, {"warn"}), ({"request_id": "req-abcdef12"}, {"mine"}), ({"job_id": 42}, {"err"}), + ({"run_id": 7}, {"warn"}), # a background run's lines, not a download job's ({"q": "FORBIDDEN"}, {"err"}), # in attrs, any case ({"q": "KeyError"}, {"warn"}), # in the stack ({"q": "100%"}, {"plain"}), # a literal, not a wildcard @@ -102,7 +104,14 @@ def test_each_filter_selects_exactly_its_rows(db, admin, make_user, params, expe job_id=42, attrs={"code": "forbidden"}, ), - _row(msg="warn", level="WARNING", sev=13, event=None, exc="Traceback …\nKeyError: 'x'"), + _row( + msg="warn", + level="WARNING", + sev=13, + event=None, + exc="Traceback …\nKeyError: 'x'", + run_id=7, + ), _row(msg="plain", ts=T0 - timedelta(hours=2), attrs={"note": "100% done"}), # Matched by no filter above but `until`: another account, and `100%` read as a pattern. _row(msg="decoy", ts=T0 - timedelta(hours=3), attrs={"note": "1000 done"}, user_id=99), diff --git a/backend/tests/test_migration_0109.py b/backend/tests/test_migration_0109.py new file mode 100644 index 00000000..9b19b05c --- /dev/null +++ b/backend/tests/test_migration_0109.py @@ -0,0 +1,98 @@ +"""Migration 0109's data step, DB-lane: the audit log's run rows move into `job_runs` with their +outcome intact, and leave the audit log; an admin action stays where it was. The downgrade moves +them back.""" + +import json + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def _audit(conn, actor, action, target, after): + conn.execute( + text( + "INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after," + " created_at) VALUES (:a, :act, 'job', :t, 's', CAST(:after AS json)," + " CAST('2026-10-01T03:00:00+00:00' AS timestamptz))" + ), + {"a": actor, "act": action, "t": target, "after": json.dumps(after)}, + ) + + +def test_the_audit_logs_runs_move_into_the_run_table_and_back(db, make_user): + uid = make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0108_folder_links") + try: + with engine.begin() as conn: + _audit( + conn, + None, + "job_run", + "rss_poll", + {"status": "ok", "summary": "new=3", "result": {"new": 3}}, + ) + _audit( + conn, + uid, + "job_run", + "rss_poll", + {"status": "error", "summary": "boom", "result": {"failure": "rss_all_failed"}}, + ) + _audit( + conn, uid, "job_run", "backfill", {"status": "skipped", "summary": "sync paused"} + ) + _audit(conn, uid, "config_set", None, {"key": "x"}) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + runs = conn.execute( + text( + "SELECT job, trigger, actor_id, status, summary, result, error, failure," + " to_char(started_at AT TIME ZONE 'UTC', 'MM-DD HH24'), finished_at = started_at" + " FROM job_runs ORDER BY id" + ) + ).all() + left = conn.execute(text("SELECT action FROM audit_log")).scalars().all() + assert [tuple(r) for r in runs] == [ + ("rss_poll", "scheduled", None, "ok", "new=3", {"new": 3}, None, None, "10-01 03", True), + ( + "rss_poll", + "manual", + uid, + "error", + None, + None, + "boom", + {"failure": "rss_all_failed"}, + "10-01 03", + True, + ), + ("backfill", "manual", uid, "skipped", "sync paused", None, None, None, "10-01 03", True), + ] + assert left == ["config_set"] + + command.downgrade(cfg, "0108_folder_links") + try: + with engine.connect() as conn: + back = conn.execute( + text( + "SELECT actor_id, target_id, after::jsonb FROM audit_log" + " WHERE action = 'job_run' ORDER BY id" + ) + ).all() + finally: + command.upgrade(cfg, "head") + assert [tuple(r) for r in back] == [ + (None, "rss_poll", {"status": "ok", "summary": "new=3", "result": {"new": 3}}), + ( + uid, + "rss_poll", + {"status": "error", "summary": "boom", "result": {"failure": "rss_all_failed"}}, + ), + (uid, "backfill", {"status": "skipped", "summary": "sync paused"}), + ] diff --git a/backend/tests/test_plex_people_enrich.py b/backend/tests/test_plex_people_enrich.py index d10909d2..307f85ac 100644 --- a/backend/tests/test_plex_people_enrich.py +++ b/backend/tests/test_plex_people_enrich.py @@ -9,7 +9,6 @@ from datetime import datetime, timezone import httpx import pytest -from app import scheduler from app.models import PlexItem, PlexLibrary, PlexPerson, User from app.plex import imgcache from app.plex import people as plex_people @@ -340,12 +339,6 @@ def test_the_people_job_runs_its_halves_without_a_tmdb_key(db, cache_dir, plex_o } -def test_a_phase_count_is_a_change_worth_an_audit_row(): - assert scheduler._run_changed({"bios": {"enriched": 3}, "portraits": {"cached": 0}}) - assert scheduler._run_changed({"recent": {"videos_added": 2}, "deep": {}}) - assert not scheduler._run_changed({"bios": {"skipped": "x"}, "portraits": {"cached": 0}}) - - def test_the_person_page_says_a_bio_is_on_its_way(db, monkeypatch, override_dep): from fastapi.testclient import TestClient diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 9c47d3c9..acd52f76 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -1,11 +1,20 @@ -"""_claim_running is the single-run guard that both the scheduled fire and the manual trigger go +"""The scheduler's run path, DB-lane: the single-run guard, and every run as a durable `job_runs` row +— its outcome as data, what it spent, the lines it logged — that outlives the process. + +_claim_running is the single-run guard that both the scheduled fire and the manual trigger go through (at the top of _job): holding _activity_lock across the check-and-set means exactly one caller can hold a job's "running" claim at a time, and it's claimable again once released.""" +import logging +import time +from datetime import datetime, timedelta, timezone + +import pytest from sqlalchemy import select -from app import scheduler -from app.models import Notification +from app import jobruns, logs, quota, scheduler, state +from app.models import AuditLog, JobRun, Notification, QuotaEvent +from app.runstatus import RunStatus, RunTrigger _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs @@ -31,26 +40,38 @@ def test_claim_is_exclusive_until_released(): # ---- the result as data -------------------------------------------------------------------------- -def _snapshot_job(job_id): - return next(j for j in scheduler.scheduler_snapshot()["jobs"] if j["id"] == job_id) +def _snapshot_job(db, job_id): + return next(j for j in scheduler.scheduler_snapshot(db)["jobs"] if j["id"] == job_id) + + +def _runs(db, job=None) -> list[JobRun]: + db.expire_all() + q = select(JobRun).order_by(JobRun.id) + if job is not None: + q = q.where(JobRun.job == job) + return list(db.execute(q).scalars()) + + +def _manual(actor_id, name, fn): + token = scheduler._manual_actor.set(actor_id) + try: + scheduler._job(name, fn) + finally: + scheduler._manual_actor.reset(token) def test_a_run_keeps_its_result_as_data_for_the_readers_language(db, make_user): """The dashboard and the manual-run notice showed `_summarize`'s English one-liner ("new=3, failed=0", a Python repr for a nested phase). The result rides along as data, so the client names - it; the one-liner stays as the fallback and the audit text.""" + it; the one-liner stays as the fallback.""" admin = make_user("sched@example.com", role="admin") - token = scheduler._manual_actor.set(admin.id) - try: - scheduler._job("rss_poll", lambda _db: {"new": 3, "failed": 0, "fallback": 1}) - scheduler._job("enrich", lambda _db: 5) # enrich returns a bare count - finally: - scheduler._manual_actor.reset(token) + _manual(admin.id, "rss_poll", lambda _db: {"new": 3, "failed": 0, "fallback": 1}) + _manual(admin.id, "enrich", lambda _db: 5) # enrich returns a bare count - rss = _snapshot_job("rss_poll") + rss = _snapshot_job(db, "rss_poll") assert rss["result"] == {"new": 3, "failed": 0, "fallback": 1} assert rss["last_result"] == "new=3, failed=0, fallback=1" - assert _snapshot_job("enrich")["result"] == {"enriched": 5} + assert _snapshot_job(db, "enrich")["result"] == {"enriched": 5} notices = db.execute( select(Notification).where(Notification.user_id == admin.id).order_by(Notification.id) @@ -64,53 +85,227 @@ def test_a_run_keeps_its_result_as_data_for_the_readers_language(db, make_user): def test_a_failed_run_drops_the_previous_result(db): scheduler._job("rss_poll", lambda _db: {"new": 1}) scheduler._job("rss_poll", lambda _db: (_ for _ in ()).throw(RuntimeError("boom"))) - rss = _snapshot_job("rss_poll") + rss = _snapshot_job(db, "rss_poll") + assert rss["status"] == "error" assert rss["last_error"] == "boom" assert rss["result"] is None -def test_a_job_runs_audit_row_carries_its_result_as_data(db): - """The audit log names a run's result in the reader's language, as the inbox does — - from data, not from the English one-liner (a nested phase there was a Python dict repr).""" - from app.models import AuditLog, User - - admin = User(email="auditrun@example.com") - db.add(admin) - db.commit() - scheduler._audit_run(db, None, "people_enrich", "ok", "x", {"bios": {"enriched": 2}}) - scheduler._audit_run(db, admin.id, "rss_poll", "skipped", "sync paused", scheduler._PAUSED) - scheduler._audit_run(db, admin.id, "rss_poll", "error", "boom", None) - rows = db.execute(select(AuditLog).order_by(AuditLog.id)).scalars().all() - assert [r.after for r in rows] == [ - {"status": "ok", "summary": "x", "result": {"bios": {"enriched": 2}}}, - {"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}}, - {"status": "error", "summary": "boom"}, - ] - - def test_a_failure_the_job_names_rides_along_as_data(db, make_user): - """A failed RSS poll reached the Hungarian audit log as the server's English sentence. A - failure the job names (`progress.JobFailed`) is stored as data beside that sentence — on the - dashboard, the manual-run notice and the audit row — so the client says it in the reader's - language. Any other failure keeps only its own text (the test above).""" + """A failed RSS poll reached the Hungarian UI as the server's English sentence. A failure the job + names (`progress.JobFailed`) is stored as data beside that sentence — on the run's row, the + dashboard and the manual-run notice — so the client says it in the reader's language.""" from app import progress - from app.models import AuditLog def outage(_db): raise progress.JobFailed("rss_all_failed", "RSS poll failed", total=2, cause="transport") admin = make_user("failure@example.com", role="admin") - token = scheduler._manual_actor.set(admin.id) - try: - scheduler._job("rss_poll", outage) - finally: - scheduler._manual_actor.reset(token) + _manual(admin.id, "rss_poll", outage) named = {"failure": "rss_all_failed", "total": 2, "cause": "transport"} - rss = _snapshot_job("rss_poll") + rss = _snapshot_job(db, "rss_poll") assert rss["last_error"] == "RSS poll failed" assert rss["result"] == named notice = db.execute(select(Notification).where(Notification.user_id == admin.id)).scalar_one() assert notice.data["result"] == named - row = db.execute(select(AuditLog).order_by(AuditLog.id.desc())).scalars().first() - assert row is not None and row.after["result"] == named + (run,) = _runs(db, "rss_poll") + assert (run.status, run.error, run.failure) == ("error", "RSS poll failed", named) + + +# ---- the durable row ----------------------------------------------------------------------------- + + +def test_every_run_is_a_row_with_who_started_it_and_none_in_the_audit_log(db, make_user): + """A no-op scheduled poll was not written anywhere (the audit rule kept only changes), so "did + it run at 03:00?" had no answer. Every run is a row now; the audit log keeps admin actions only.""" + admin = make_user("runs@example.com", role="admin") + scheduler._job("rss_poll", lambda _db: {"new": 0}) + _manual(admin.id, "rss_poll", lambda _db: {"new": 2}) + + scheduled, manual = _runs(db, "rss_poll") + assert (scheduled.trigger, scheduled.actor_id, scheduled.status) == ("scheduled", None, "ok") + assert scheduled.result == {"new": 0} + assert (manual.trigger, manual.actor_id, manual.summary) == ("manual", admin.id, "new=2") + assert manual.finished_at is not None and manual.finished_at >= manual.started_at + assert db.execute(select(AuditLog)).scalars().all() == [] + + +def test_the_last_run_survives_a_restart(db): + """The dashboard's "last run" lived in the process's memory: a deploy blanked every job.""" + scheduler._job("plex_sync", lambda _db: {"items": 4}) + with scheduler._activity_lock: + scheduler._activity.clear() # what a new process starts with + + plex = _snapshot_job(db, "plex_sync") + assert (plex["status"], plex["result"], plex["last_trigger"]) == ( + "ok", + {"items": 4}, + "scheduled", + ) + assert plex["last_finished"] is not None + + +def test_a_paused_sync_is_a_skipped_run(db): + state.set_sync_paused(db, True) + db.commit() + scheduler._job("rss_poll", lambda _db: pytest.fail("a paused run must not do its work")) + (run,) = _runs(db, "rss_poll") + assert (run.status, run.result) == ("skipped", {"skipped": "sync paused"}) + + +def test_a_run_left_running_by_a_stopped_process_is_interrupted_and_stays_so(db): + """A deploy mid-run left nothing behind before; now the row would say `running` forever. The + next startup closes it, and the stale thread's own finish cannot rewrite it.""" + run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None) + assert jobruns.interrupt_leftovers() == 1 + + (run,) = _runs(db, "backfill") + assert run.status == "interrupted" and run.finished_at is not None + jobruns.finish(run_id, RunStatus.OK, summary="late") + (run,) = _runs(db, "backfill") + assert (run.status, run.summary) == ("interrupted", None) + + +def test_a_run_knows_what_it_spent_and_a_request_spend_is_nobodys_run(db): + def spends(_db): + quota.record_usage(5) + quota.record_usage(3) + return {"enriched": 2} + + scheduler._job("enrich", spends) + quota.record_usage(11) # a request's own spend, outside any run + + (run,) = _runs(db, "enrich") + assert run.quota_units == 8 + events = db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() + assert sorted(events) == [(3, run.id), (5, run.id), (11, None)] + assert _snapshot_job(db, "enrich")["quota_units"] == 8 + + +class _Collect(logging.Handler): + def __init__(self): + super().__init__() + self.addFilter(logs.ContextFilter()) + self.records: list[logging.LogRecord] = [] + + def emit(self, record): + self.records.append(record) + + +def test_a_runs_log_lines_carry_its_id(db): + """So a run's row can open exactly the lines it logged (the log table's `run_id`).""" + probe = logging.getLogger("siftlode.test.run") + handler = _Collect() + probe.addHandler(handler) + probe.setLevel(logging.INFO) + try: + scheduler._job("autotag", lambda _db: probe.info("tagging") or {"tagged": 1}) + probe.info("after the run") + finally: + probe.removeHandler(handler) + + (run,) = _runs(db, "autotag") + inside, after = (logs.record_fields(r).get("run_id") for r in handler.records) + assert (inside, after) == (run.id, None) + + +def test_retention_reaps_finished_runs_only(db): + old = datetime.now(timezone.utc) - timedelta(days=40) + db.add_all( + [ + JobRun( + job="rss_poll", + trigger="scheduled", + status="ok", + started_at=old, + finished_at=old, + ), + JobRun(job="backfill", trigger="scheduled", status="running", started_at=old), + JobRun( + job="rss_poll", + trigger="scheduled", + status="ok", + started_at=datetime.now(timezone.utc), + finished_at=datetime.now(timezone.utc), + ), + ] + ) + db.commit() + assert jobruns.gc(db, 0) == 0 # 0 = keep forever + assert jobruns.gc(db, 30) == 1 + db.commit() + assert sorted((r.job, r.status) for r in _runs(db)) == [ + ("backfill", "running"), + ("rss_poll", "ok"), + ] + + +# ---- "run now" through the scheduler -------------------------------------------------------------- + + +@pytest.fixture +def live_scheduler(db, monkeypatch): + """This process's scheduler, started as the API starts it (on the migrated schema `db` brings), + with interval jobs off.""" + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", False) + scheduler.start_scheduler() + try: + yield scheduler._scheduler + finally: + scheduler.shutdown_scheduler() + + +def _wait_finished(db, job, timeout=10.0) -> JobRun: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + runs = [r for r in _runs(db, job) if r.status != "running"] + if runs: + return runs[-1] + time.sleep(0.05) + raise AssertionError(f"{job} did not finish") + + +def test_run_now_goes_through_the_scheduler_even_with_interval_jobs_off( + db, make_user, live_scheduler, monkeypatch +): + admin = make_user("now@example.com", role="admin") + monkeypatch.setitem( + scheduler.JOB_FUNCS, "shorts", lambda: scheduler._job("shorts", lambda _db: {"probed": 3}) + ) + assert live_scheduler.get_jobs() == [] # SCHEDULER_ENABLED=false: no interval job + live_scheduler.pause() # hold the one-shot so the second click finds it pending + assert scheduler.trigger_job("shorts", admin.id) == "started" + assert scheduler.trigger_job("shorts", admin.id) == "already_running" + assert [j.id for j in live_scheduler.get_jobs()] == ["shorts@now"] + live_scheduler.resume() + + run = _wait_finished(db, "shorts") + assert (run.trigger, run.actor_id, run.result) == ("manual", admin.id, {"probed": 3}) + + +def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user): + """The scheduler's pool threads are reused: an actor a manual run left set made the next + scheduled run on that thread an admin's "manual" run, with a notice in their inbox.""" + admin = make_user("pool@example.com", role="admin") + seen = [] + scheduler._as_actor(lambda: seen.append(scheduler._manual_actor.get()), admin.id)() + assert seen == [admin.id] + assert scheduler._manual_actor.get() is None # this same thread's next task runs as nobody + + +def test_run_now_refuses_a_job_already_running(live_scheduler): + assert scheduler._claim_running("maintenance") is True + try: + assert scheduler.trigger_job("maintenance") == "already_running" + finally: + scheduler._record("maintenance", running=False) + + +def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch): + jobruns.start("plex_sync", RunTrigger.MANUAL, None) + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", False) + scheduler.start_scheduler() + scheduler.shutdown_scheduler() + (run,) = _runs(db, "plex_sync") + assert run.status == "interrupted" diff --git a/frontend/src/components/AuditEarlierRuns.tsx b/frontend/src/components/AuditEarlierRuns.tsx deleted file mode 100644 index a5433ac6..00000000 --- a/frontend/src/components/AuditEarlierRuns.tsx +++ /dev/null @@ -1,48 +0,0 @@ -import { useState } from "react"; -import { useTranslation } from "react-i18next"; -import clsx from "clsx"; -import { ChevronRight } from "lucide-react"; -import type { AuditRow } from "../lib/api"; -import { Button } from "./ui/Button"; -import { RelativeTime } from "./ui/RelativeTime"; - -/** A folded run of one job's successful runs (lib/auditGroups): the row shows the newest, this - * opens the earlier ones, each with its moment and its result (`detailOf`, the row's own wording). */ -export default function AuditEarlierRuns({ - runs, - detailOf, -}: { - runs: AuditRow[]; - detailOf: (run: AuditRow) => string | null; -}) { - const { t } = useTranslation(); - const [open, setOpen] = useState(false); - const earlier = runs.slice(1); - return ( -
- - {open && ( - - )} -
- ); -} diff --git a/frontend/src/components/AuditLog.tsx b/frontend/src/components/AuditLog.tsx index 95d402df..94bb5fc8 100644 --- a/frontend/src/components/AuditLog.tsx +++ b/frontend/src/components/AuditLog.tsx @@ -14,10 +14,9 @@ import { LoadingState, StateMessage } from "./ui/QueryState"; import { auditColumns } from "./auditColumns"; import { notifyActionError } from "../lib/actionErrors"; import { compareText } from "../lib/collate"; -import { foldJobRuns } from "../lib/auditGroups"; -// Admin-only audit trail: an append-only who/what/when log of admin actions + scheduler run -// summaries (never per-row). Read-only apart from Clear. Client-side sort/filter/paginate via +// Admin-only audit trail: an append-only who/what/when log of admin actions (never per-row; a +// background run is not one — runs have their own history). Read-only apart from Clear. Client-side sort/filter/paginate via // DataTable; the API returns the most recent window (capped) — the log stays small by design. export default function AuditLog() { const { t, i18n } = useTranslation(); @@ -26,8 +25,7 @@ export default function AuditLog() { const confirm = useConfirm(); const q = useQuery({ queryKey: qk.adminAudit(), queryFn: () => api.adminAudit() }); const data = q.data; - // Back-to-back successful runs of one job fold into one row (lib/auditGroups). - const rows = useMemo(() => foldJobRuns(data?.items ?? []), [data]); + const rows = useMemo(() => data?.items ?? [], [data]); const actionOptions = useMemo(() => { // By what the reader sees — the named action, in their alphabet — not by the code. diff --git a/frontend/src/components/auditColumns.dom.test.tsx b/frontend/src/components/auditColumns.dom.test.tsx index 43001585..34a8a0e8 100644 --- a/frontend/src/components/auditColumns.dom.test.tsx +++ b/frontend/src/components/auditColumns.dom.test.tsx @@ -1,10 +1,10 @@ // @vitest-environment jsdom // -// An audit row whose server summary is a fixed English line ("rss_poll: ok — new=5, failed=0", -// "log_level = DEBUG", "Background sync paused") reads in the reader's language, rebuilt from the row's +// An audit row whose server summary is a fixed English line ("log_level = DEBUG", "rss_poll → 15 +// min/run", "Background sync paused") reads in the reader's language, rebuilt from the row's // data through the REAL catalog. A row whose summary is data (an e-mail, a URL) keeps it. import { afterEach, describe, expect, it } from "vitest"; -import { cleanup, fireEvent, render } from "@testing-library/react"; +import { cleanup, render } from "@testing-library/react"; import i18n, { setLanguage } from "../i18n"; import type { AuditRow } from "../lib/api"; import { auditColumns } from "./auditColumns"; @@ -20,7 +20,7 @@ const row = (over: Partial): AuditRow => ({ id: 1, created_at: null, actor: null, - action: "job_run", + action: "sync_pause", target_type: null, target_id: null, summary: null, @@ -38,27 +38,6 @@ function details(r: AuditRow): string { } describe("an audit row's details", () => { - it("name a job run and its result in the reader's language", () => { - const r = row({ - target_type: "job", - target_id: "rss_poll", - summary: "rss_poll: ok — new=5, failed=0", - after: { status: "ok", summary: "new=5, failed=0" }, - }); - expect(details(r)).toBe("RSS-lekérdezés (új feltöltések) kész | új videó: 5 · sikertelen: 0"); - }); - - it("keep a failed run's error text, which is diagnostic", () => { - const r = row({ - target_type: "job", - target_id: "enrich", - after: { status: "error", summary: "HTTP 403: quotaExceeded" }, - }); - expect(details(r)).toBe( - "Nem sikerült lefuttatni: Adatdúsítás + élő frissítés | HTTP 403: quotaExceeded", - ); - }); - it("name a config change's field and values", () => { const set = row({ action: "config_set", @@ -87,26 +66,6 @@ describe("an audit row's details", () => { expect(details(secret)).toMatch(/ \| titok frissítve/); }); - it("name a nested result and a skipped run from the row's data", () => { - const nested = row({ - target_type: "job", - target_id: "people_enrich", - after: { - status: "ok", - summary: "bios={'enriched': 12}, portraits={'cached': 40}", - result: { bios: { enriched: 12 }, portraits: { cached: 40 } }, - }, - }); - expect(details(nested)).not.toContain("{"); - expect(details(nested)).toMatch(/^Személyadatok és portrék \(Plex\) kész \| /); - const skipped = row({ - target_type: "job", - target_id: "rss_poll", - after: { status: "skipped", summary: "sync paused" }, - }); - expect(details(skipped)).toBe("RSS-lekérdezés (új feltöltések) kihagyva | a szinkron szünetel"); - }); - it("tell a secret's reset from its update, and name a choice", () => { const reset = row({ action: "config_reset", @@ -157,49 +116,3 @@ describe("an audit row's details", () => { ); }); }); - -// A folded run (lib/auditGroups) shows its newest run and opens to the earlier ones; a failed run -// is the line to see in a page of routine ones. -describe("a folded run of one job", () => { - const ok = (id: number, enriched: number) => - row({ - id, - target_type: "job", - target_id: "enrich", - after: { status: "ok", result: { enriched } }, - }); - - it("names how many earlier runs it holds, and opens to each", () => { - setLanguage("hu"); - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const runs = [ok(3, 50), ok(2, 40), ok(1, 30)]; - const { getByRole, container } = render(<>{col.render!({ ...runs[0]!, runs })}); - const toggle = getByRole("button", { name: "2 korábbi futás" }); - expect(toggle.getAttribute("aria-expanded")).toBe("false"); - expect(container.querySelector("li")).toBeNull(); - fireEvent.click(toggle); - expect(toggle.getAttribute("aria-expanded")).toBe("true"); - expect([...container.querySelectorAll("li")].map((li) => li.textContent)).toEqual([ - "—dúsítva: 40", - "—dúsítva: 30", - ]); - }); - - it("has no toggle for a lone run", () => { - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const { container } = render(<>{col.render!(ok(1, 5))}); - expect(container.querySelector("button")).toBeNull(); - }); - - it("paints a failed run's line in the danger colour", () => { - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const failed = row({ target_type: "job", target_id: "enrich", after: { status: "error" } }); - const { container } = render(<>{col.render!(failed)}); - expect(container.querySelector(".wrap-break-word")!.className).toContain("text-danger"); - const { container: fine } = render(<>{col.render!(ok(2, 5))}); - expect(fine.querySelector(".wrap-break-word")!.className).not.toContain("text-danger"); - }); -}); diff --git a/frontend/src/components/auditColumns.tsx b/frontend/src/components/auditColumns.tsx index 782979f9..a11631a7 100644 --- a/frontend/src/components/auditColumns.tsx +++ b/frontend/src/components/auditColumns.tsx @@ -1,11 +1,8 @@ import type { TFunction } from "i18next"; -import clsx from "clsx"; import type { AuditRow } from "../lib/api"; -import type { AuditListRow } from "../lib/auditGroups"; -import AuditEarlierRuns from "./AuditEarlierRuns"; import { formatNumber } from "../lib/format"; import type { JobResult } from "../lib/api"; -import { jobResultText, jobRunText } from "../lib/jobResult"; +import { jobResultText } from "../lib/jobResult"; import type { Column } from "./ui/DataTable"; import { RelativeTime } from "./ui/RelativeTime"; @@ -35,8 +32,7 @@ function counts(after: Record): JobResult | null { } /** The details of the rows whose server `summary` is a fixed English line, rebuilt from the row's - * data in the reader's language: a job run names the job and its result (as the inbox does), a config - * change names the field and its values, an interval or batch change its new value, a count-bearing + * data in the reader's language: a config change names the field and its values, an interval or batch change its new value, a count-bearing * action its counts; a pause/resume says nothing the action has not. `undefined` = not one of these (or * an older row without the data); the generic summary + change rendering applies. */ function localDetails( @@ -46,10 +42,6 @@ function localDetails( ): { head: string | null; sub: string | null } | undefined { const after = row.after ?? {}; const before = row.before ?? {}; - if (row.action === "job_run" && row.target_type === "job" && row.target_id) { - const run = jobRunText(t, { job_id: row.target_id, ...after }, lang); - return { head: run.title, sub: run.body }; - } if ((row.action === "config_set" || row.action === "config_reset") && row.target_id) { const key = row.target_id; const head = t(`config.fields.${key}.label`, key); @@ -104,7 +96,7 @@ export function auditColumns( t: TFunction, lang: string, actionOptions: { value: string; label: string }[], -): Column[] { +): Column[] { return [ { key: "when", @@ -146,23 +138,13 @@ export function auditColumns( const local = localDetails(t, r, lang); const summary = local ? local.head : r.summary; const change = local ? local.sub : changeText(r); - // A failed run is the row to see in a page of routine ones: its line in the danger colour. - const failed = r.action === "job_run" && r.after?.status === "error"; return ( // wrap-break-word (wrap), NOT truncate: in the auto-layout table a `truncate` (nowrap) cell // sizes to its longest row, so one long summary/change stretched the whole table past the // card (the row rules bled into the page). Wrapping keeps every row inside the container.
- {summary && ( -
{summary}
- )} + {summary &&
{summary}
} {change &&
{change}
} - {r.runs && r.runs.length > 1 && ( - localDetails(t, run, lang)?.sub ?? run.summary} - /> - )}
); }, diff --git a/frontend/src/i18n/locales/en/audit.json b/frontend/src/i18n/locales/en/audit.json index 454f58ff..6e6eb7c6 100644 --- a/frontend/src/i18n/locales/en/audit.json +++ b/frontend/src/i18n/locales/en/audit.json @@ -1,6 +1,6 @@ { "title": "Audit log", - "intro": "An append-only record of admin actions and scheduler run summaries — who did what, and when. Logged at action granularity, never per item.", + "intro": "An append-only record of admin actions — who did what, and when. Logged at action granularity, never per item.", "empty": "No audit entries yet.", "system": "System", "clear": "Clear audit log", @@ -20,7 +20,5 @@ "actor": "Who", "action": "Action", "details": "Details" - }, - "earlierRuns_one": "{{count}} earlier run", - "earlierRuns_other": "{{count}} earlier runs" + } } diff --git a/frontend/src/i18n/locales/en/auditActions.json b/frontend/src/i18n/locales/en/auditActions.json index 1be63672..4c87fa0b 100644 --- a/frontend/src/i18n/locales/en/auditActions.json +++ b/frontend/src/i18n/locales/en/auditActions.json @@ -16,7 +16,6 @@ "maintenance_tune": "Maintenance tuned", "sync_pause": "Sync paused", "sync_resume": "Sync resumed", - "job_run": "Scheduler job run", "audit_clear": "Audit log cleared", "download_credential_set": "Download cookies set", "download_credential_delete": "Download cookies deleted", diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index eed76f55..2a73c870 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -170,6 +170,10 @@ "label": "Retention (days)", "hint": "How long admin audit-log entries are kept before the audit-log cleanup job prunes them. 0 = keep forever. Default 180." }, + "job_run_retention_days": { + "label": "Run history (days)", + "hint": "How long the history of background runs (scheduled, run now, a channel's full history) is kept before the cleanup job prunes it. 0 = keep forever. Default 30." + }, "log_level": { "label": "Log level", "hint": "How much the server writes to its log. INFO is the normal setting; DEBUG and TRACE are for reproducing a problem and are far more detailed. Takes effect immediately, no restart." diff --git a/frontend/src/i18n/locales/hu/audit.json b/frontend/src/i18n/locales/hu/audit.json index 60b56a83..0ec91b4d 100644 --- a/frontend/src/i18n/locales/hu/audit.json +++ b/frontend/src/i18n/locales/hu/audit.json @@ -1,6 +1,6 @@ { "title": "Eseménynapló", - "intro": "Az admin-műveletek és az ütemező futás-összegzéseinek csak-hozzáfűző naplója — ki, mit, mikor. Művelet-szinten naplózva, soha nem elemenként.", + "intro": "Az admin-műveletek csak-hozzáfűző naplója — ki, mit, mikor. Művelet-szinten naplózva, soha nem elemenként.", "empty": "Még nincs naplóbejegyzés.", "system": "Rendszer", "clear": "Eseménynapló ürítése", @@ -20,7 +20,5 @@ "actor": "Ki", "action": "Művelet", "details": "Részletek" - }, - "earlierRuns_one": "{{count}} korábbi futás", - "earlierRuns_other": "{{count}} korábbi futás" + } } diff --git a/frontend/src/i18n/locales/hu/auditActions.json b/frontend/src/i18n/locales/hu/auditActions.json index 6e9fff00..a7e4671b 100644 --- a/frontend/src/i18n/locales/hu/auditActions.json +++ b/frontend/src/i18n/locales/hu/auditActions.json @@ -16,7 +16,6 @@ "maintenance_tune": "Karbantartás hangolva", "sync_pause": "Szinkron szüneteltetve", "sync_resume": "Szinkron folytatva", - "job_run": "Ütemezett feladat lefutott", "audit_clear": "Eseménynapló ürítve", "download_credential_set": "Letöltési cookie-k beállítva", "download_credential_delete": "Letöltési cookie-k törölve", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index a3e3fc5c..f2db4b2a 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -170,6 +170,10 @@ "label": "Megőrzés (nap)", "hint": "Meddig maradnak meg az admin-naplóbejegyzések, mielőtt az eseménynapló-takarító feladat törli őket. 0 = örökre. Alapérték 180." }, + "job_run_retention_days": { + "label": "Futási előzmények (nap)", + "hint": "Meddig maradnak meg a háttérfutások (ütemezett, azonnali, egy csatorna teljes előzménye) adatai, mielőtt a takarító feladat törli őket. 0 = örökre. Alapérték 30." + }, "log_level": { "label": "Naplózási szint", "hint": "Mennyit naplózzon a szerver. Az INFO a szokásos; a DEBUG és a TRACE hibakereséshez való, sokkal részletesebb. Azonnal érvénybe lép, újraindítás nélkül." diff --git a/frontend/src/lib/auditGroups.test.ts b/frontend/src/lib/auditGroups.test.ts deleted file mode 100644 index 85ff1f18..00000000 --- a/frontend/src/lib/auditGroups.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -// The audit log's first screen was one job's "done" ten times over. A job's successful runs fold -// into one row whatever ran between them (the owner: a fold broken by every other job read as the same -// row five times); only that job's own failure or skip breaks it. -import { describe, expect, it } from "vitest"; -import type { AuditRow } from "./api"; -import { foldJobRuns } from "./auditGroups"; - -let nextId = 1; -const run = (job: string, status = "ok"): AuditRow => ({ - id: nextId++, - created_at: null, - actor: null, - action: "job_run", - target_type: "job", - target_id: job, - summary: null, - before: null, - after: { status }, -}); -const admin = (): AuditRow => ({ ...run("x"), action: "config_set", target_type: "config" }); - -const shape = (rows: ReturnType) => - rows.map((r) => `${r.target_id}:${String(r.after?.status)}×${r.runs?.length ?? 1}`); - -describe("folding the job runs", () => { - it("folds a job's successful runs into one row, newest first", () => { - const rows = [run("enrich"), run("enrich"), run("enrich"), run("people")]; - const out = foldJobRuns(rows); - expect(shape(out)).toEqual(["enrich:ok×3", "people:ok×1"]); - expect(out[0]!.id).toBe(rows[0]!.id); - expect(out[0]!.runs!.map((r) => r.id)).toEqual(rows.slice(0, 3).map((r) => r.id)); - }); - - it("never folds a failed or skipped run, and lets it break the run", () => { - const rows = [run("rss"), run("rss", "error"), run("rss", "error"), run("rss"), run("rss")]; - expect(shape(foldJobRuns(rows))).toEqual([ - "rss:ok×1", - "rss:error×1", - "rss:error×1", - "rss:ok×2", - ]); - expect(shape(foldJobRuns([run("rss", "skipped"), run("rss", "skipped")]))).toEqual([ - "rss:skipped×1", - "rss:skipped×1", - ]); - }); - - it("folds across other jobs and an admin's own action, each at its own place", () => { - const rows = [run("enrich"), run("plex"), run("enrich"), admin(), run("shorts"), run("enrich")]; - rows.push(run("plex")); - const out = foldJobRuns(rows); - expect(shape(out)).toEqual(["enrich:ok×3", "plex:ok×2", "x:ok×1", "shorts:ok×1"]); - expect(out[0]!.runs!.map((r) => r.id)).toEqual([rows[0]!.id, rows[2]!.id, rows[5]!.id]); - }); - - it("starts a new fold below the job's own failure, which another job's does not", () => { - const rows = [run("enrich"), run("enrich", "error"), run("enrich"), run("rss", "error")]; - rows.push(run("enrich")); - expect(shape(foldJobRuns(rows))).toEqual([ - "enrich:ok×1", - "enrich:error×1", - "enrich:ok×2", - "rss:error×1", - ]); - }); - - it("leaves the rows it was given as they were", () => { - const rows = [run("enrich"), run("enrich")]; - foldJobRuns(rows); - expect("runs" in rows[0]!).toBe(false); - }); -}); diff --git a/frontend/src/lib/auditGroups.ts b/frontend/src/lib/auditGroups.ts deleted file mode 100644 index 4cb380a7..00000000 --- a/frontend/src/lib/auditGroups.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { AuditRow } from "./api"; - -// The audit log's scheduler noise, folded (the owner's pick): the background jobs write a row per -// run that changed something, so an hour of "enrichment done" filled the first screen and pushed the -// admins' own actions down. A job's SUCCESSFUL runs become one row that names how many there were and -// opens to each — whatever ran between them (another job, an admin's action): the owner read a fold -// that broke at every other job as the same row five times. Only that job's own failed or skipped run -// breaks it: that row stays on its own, at its moment, where it is seen, and the job's older successes -// start a new fold below it. Rows arrive newest first; a fold stands at its newest run's place. - -/** An audit row as the table lists it: a lone row, or the newest of a job's folded runs (`runs`, the - * whole fold newest first, the row itself included). */ -export type AuditListRow = AuditRow & { runs?: AuditRow[] }; - -/** The job a row is a run of, or null for any other row. */ -function jobOf(row: AuditRow): string | null { - return row.action === "job_run" && row.target_type === "job" && row.target_id - ? row.target_id - : null; -} - -export function foldJobRuns(rows: readonly AuditRow[]): AuditListRow[] { - const out: AuditListRow[] = []; - // Each job's open fold: the row its next older success joins. - const open = new Map(); - for (const row of rows) { - const job = jobOf(row); - if (job === null) { - out.push(row); - continue; - } - if (row.after?.status !== "ok") { - // A failure or a skip stands alone and closes the job's fold. - open.delete(job); - out.push(row); - continue; - } - const head = open.get(job); - if (head) { - head.runs = [...(head.runs ?? [head]), row]; - continue; - } - const fresh: AuditListRow = { ...row }; - open.set(job, fresh); - out.push(fresh); - } - return out; -} From 8fd91b7b1561c95648f9e270e3adc30ab8cd63b2 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 17 Oct 2026 23:15:17 +0200 Subject: [PATCH 02/16] fix(scheduler): keep run history 180 days, as long as the audit log kept the runs it moved over --- backend/app/config.py | 7 ++++--- frontend/src/i18n/locales/en/config.json | 2 +- frontend/src/i18n/locales/hu/config.json | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index f5b66200..a77eb37c 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -268,9 +268,10 @@ class Settings(BaseSettings): # YouTube quota history (quota_events): days kept (0 = forever). A year, so the admin usage view's # longest window (365 days) stays whole; the same daily cleanup job prunes it. quota_retention_days: int = 365 - # Background-run history (job_runs): days kept (0 = forever). Every run is a row (~1–1.5k a day - # at the default intervals), so a month; the same daily cleanup job prunes it. - job_run_retention_days: int = 30 + # Background-run history (job_runs): days kept (0 = forever). The audit log's 180, which is how + # long a run lived there before it moved here: ~1.8k rows a day at the default intervals + # (measured 2026-10-17), ~320k at the limit. The same daily cleanup job prunes it. + job_run_retention_days: int = 180 # On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat". download_layout: DownloadLayout = "plex" # Default SponsorBlock (skip sponsor segments) for new custom profiles. diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index 2a73c870..0b516fbb 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -172,7 +172,7 @@ }, "job_run_retention_days": { "label": "Run history (days)", - "hint": "How long the history of background runs (scheduled, run now, a channel's full history) is kept before the cleanup job prunes it. 0 = keep forever. Default 30." + "hint": "How long the history of background runs (scheduled, run now, a channel's full history) is kept before the cleanup job prunes it. 0 = keep forever. Default 180." }, "log_level": { "label": "Log level", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index f2db4b2a..de9be7a9 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -172,7 +172,7 @@ }, "job_run_retention_days": { "label": "Futási előzmények (nap)", - "hint": "Meddig maradnak meg a háttérfutások (ütemezett, azonnali, egy csatorna teljes előzménye) adatai, mielőtt a takarító feladat törli őket. 0 = örökre. Alapérték 30." + "hint": "Meddig maradnak meg a háttérfutások (ütemezett, azonnali, egy csatorna teljes előzménye) adatai, mielőtt a takarító feladat törli őket. 0 = örökre. Alapérték 180." }, "log_level": { "label": "Naplózási szint", From 911527a548a0e38855d86ad4bea35be19d2d9555 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 17 Oct 2026 23:22:13 +0200 Subject: [PATCH 03/16] =?UTF-8?q?fix(scheduler):=20the=20dashboard=20no=20?= =?UTF-8?q?longer=20says=20a=20job=20has=20not=20run=20"this=20session"=20?= =?UTF-8?q?=E2=80=94=20runs=20are=20kept=20across=20restarts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/tests/test_scheduler.py | 4 +++- frontend/src/i18n/locales/en/scheduler.json | 4 ++-- frontend/src/i18n/locales/hu/scheduler.json | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index acd52f76..1afc149d 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -12,7 +12,7 @@ from datetime import datetime, timedelta, timezone import pytest from sqlalchemy import select -from app import jobruns, logs, quota, scheduler, state +from app import jobruns, logs, logsink, quota, scheduler, state from app.models import AuditLog, JobRun, Notification, QuotaEvent from app.runstatus import RunStatus, RunTrigger @@ -208,6 +208,8 @@ def test_a_runs_log_lines_carry_its_id(db): (run,) = _runs(db, "autotag") inside, after = (logs.record_fields(r).get("run_id") for r in handler.records) assert (inside, after) == (run.id, None) + # …and it reaches the stored row, whose columns are the model's. + assert logsink.to_row(logs.record_fields(handler.records[0]))["run_id"] == run.id def test_retention_reaps_finished_runs_only(db): diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index cc67eee1..148445a3 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -18,7 +18,7 @@ "everyMin_other": "every {{count}} min", "runningNow": "Running now…", "lastRun": "Last run {{time}}", - "neverRun": "Hasn't run yet this session", + "neverRun": "No run recorded yet", "active": "active", "nextRun": "next", "quotaUsed": "{{used}} / {{budget}} units used today", @@ -57,7 +57,7 @@ "dot": { "running": "running now", "ok": "last run OK", - "idle": "idle (not run yet this session)", + "idle": "idle (no run recorded yet)", "error": "last run failed", "skipped": "skipped (sync paused)" }, diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index d0e6e08b..238d1a79 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -18,7 +18,7 @@ "everyMin_other": "{{count}} percenként", "runningNow": "Most fut…", "lastRun": "Utolsó futás {{time}}", - "neverRun": "Ebben a munkamenetben még nem futott", + "neverRun": "Még nincs rögzített futás", "active": "aktív", "nextRun": "köv.", "quotaUsed": "{{used}} / {{budget}} egység ma felhasználva", @@ -57,7 +57,7 @@ "dot": { "running": "most fut", "ok": "utolsó futás OK", - "idle": "tétlen (ebben a munkamenetben még nem futott)", + "idle": "tétlen (még nincs rögzített futás)", "error": "utolsó futás hibázott", "skipped": "kihagyva (szinkron szüneteltetve)" }, From 27c2fcbd2e45ce5960e1d2ba50c5aa233387b110 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 01:05:24 +0200 Subject: [PATCH 04/16] feat(scheduler): a channel's "get full history" backfill is a recorded run, and a row knows its last three runs The one-channel deep backfill ran as a FastAPI BackgroundTask: what it fetched, what it spent and how it failed reached only the log. It is now a one-shot run on the scheduler (`channel_backfill`, trigger ad_hoc, actor = the user, target = the channel) with its quota tagged to the run. Its per-channel exclusion with the scheduled backfill and the two-run cap are unchanged, and a run they turn away says why in its result instead of returning nothing. A failure is no longer swallowed: the run records it. The scheduler snapshot carries each job's last three runs (one windowed query), so the page can show an interrupted or skipped run instead of the latest outcome alone. --- backend/app/jobruns.py | 18 +++- backend/app/routes/channels.py | 21 +++-- backend/app/scheduler.py | 94 ++++++++++++++++++--- backend/app/sync/runner.py | 43 +++++----- backend/tests/test_channel_backfill.py | 58 +++++++++++-- backend/tests/test_scheduler.py | 67 +++++++++++++++ frontend/src/i18n/locales/en/scheduler.json | 5 +- frontend/src/i18n/locales/hu/scheduler.json | 5 +- 8 files changed, 256 insertions(+), 55 deletions(-) diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py index 993af617..18645366 100644 --- a/backend/app/jobruns.py +++ b/backend/app/jobruns.py @@ -82,14 +82,24 @@ def interrupt_leftovers() -> int: return res.rowcount if isinstance(res, CursorResult) else 0 -def last_runs(db: Session) -> dict[str, JobRun]: - """Each job's most recent run (any trigger), by job id.""" +def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]: + """Each job's last `per_job` runs, newest first, by job id — one query (a window per job).""" + ranked = select( + JobRun.id, + func.row_number() + .over(partition_by=JobRun.job, order_by=(JobRun.started_at.desc(), JobRun.id.desc())) + .label("rank"), + ).subquery() rows = db.execute( select(JobRun) - .distinct(JobRun.job) + .join(ranked, ranked.c.id == JobRun.id) + .where(ranked.c.rank <= per_job) .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc()) ).scalars() - return {r.job: r for r in rows} + out: dict[str, list[JobRun]] = {} + for r in rows: + out.setdefault(r.job, []).append(r) + return out def gc(db: Session, retention_days: int) -> int: diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index e30ffcaa..d0856c94 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -3,15 +3,16 @@ hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side w (subscribe / unsubscribe) that require the optional write scope.""" from datetime import datetime, timezone +from functools import partial -from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi import APIRouter, Depends from pydantic import BaseModel from sqlalchemy import Select, func, select, update from sqlalchemy.orm import Session from app.collation import collated, sorted_words from app.mailtext import lang_for_user -from app import logs, quota +from app import logs, quota, scheduler from app.auth import admin_user, current_user, require_write_scope from app.db import get_db from app.models import ( @@ -481,7 +482,6 @@ class ChannelUpdateIn(BaseModel): def update_channel( channel_id: str, body: ChannelUpdateIn, - background: BackgroundTasks, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: @@ -508,12 +508,17 @@ def update_channel( with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT): run_recent_backfill(db, [channel], max_channels=1) - # …and kick the FULL deep backfill for this one channel immediately in the background, so the - # whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The - # request returns right away; the channel page's live poll shows the count climb. The scheduler - # still finishes a very large channel's tail on its normal runs (deep_requested stays set). + # …and kick the FULL deep backfill for this one channel immediately, as a recorded run of its own, + # so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. + # The request returns right away; the channel page's live poll shows the count climb. The + # scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set). if deep_turned_on: - background.add_task(backfill_channel_now, channel_id, user.id) + scheduler.run_ad_hoc( + scheduler.CHANNEL_BACKFILL, + partial(backfill_channel_now, channel_id, user.id), + actor_id=user.id, + target=channel_id, + ) return { "ok": True, diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index a007f02b..5280f98b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -174,26 +174,35 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None: return with progress.bind(sink): result = fn(db) - summary = _summarize(result) - data = _result_data(result) - log.info("job %s -> %s", name, result) - jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data) + summary, data = _finish_ok(name, run_id, result) if actor_id is not None: _notify_done(db, actor_id, name, "ok", summary, data) except Exception as exc: db.rollback() log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) - err = str(exc) or exc.__class__.__name__ - # A failure the job named (progress.JobFailed) rides along as data, as a result does, so the - # client says it in the reader's language; `err` stays the English line. - failure = ( - {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None - ) - jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure) + err, failure = _finish_error(run_id, exc) if actor_id is not None: _notify_done(db, actor_id, name, "error", err, failure) +def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]: + """Close a run that returned: its result on the row, as data and as the English one-liner.""" + summary = _summarize(result) + data = _result_data(result) + log.info("job %s -> %s", name, result) + jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data) + return summary, data + + +def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]: + """Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as + a result does, so the client says it in the reader's language; the error stays the English line.""" + err = str(exc) or exc.__class__.__name__ + failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None + jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure) + return err, failure + + # A skipped run's result, in the same shape as a job's own `{"skipped": …}`. _PAUSED = {"skipped": "sync paused"} @@ -240,11 +249,12 @@ def scheduler_snapshot(db) -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} - last = jobruns.last_runs(db) + recent = jobruns.recent_runs(db, RECENT_RUNS) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) - run = last.get(job_id) + runs = recent.get(job_id, []) + run = runs[0] if runs else None # A run still going has no outcome yet: only an ended run says how it went. ended = run if run is not None and run.status != RunStatus.RUNNING else None jobs.append( @@ -264,11 +274,27 @@ def scheduler_snapshot(db) -> dict: "last_run_id": run.id if run else None, "quota_units": ended.quota_units if ended else None, "progress": a.get("progress"), + # The last few runs, newest first, for the row's history dots. + "recent": [_run_brief(r) for r in runs], } ) return {"running_here": running_here, "enabled": settings.scheduler_enabled, "jobs": jobs} +# How many past runs a Scheduler row shows as dots; the full history is the run list. +RECENT_RUNS = 3 + + +def _run_brief(run) -> dict: + return { + "id": run.id, + "status": run.status, + "trigger": run.trigger, + "started_at": _iso(run.started_at), + "finished_at": _iso(run.finished_at), + } + + def _outcome_data(run) -> dict | None: if run is None: return None @@ -509,6 +535,48 @@ def trigger_all(actor_id: int | None = None) -> list[str]: return started +# Work a user starts on one target outside the job list — today the "get full history" backfill of +# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and +# spent is in `job_runs` like any job's, not a background task only the log ever heard of. +CHANNEL_BACKFILL = "channel_backfill" + + +def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None: + """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its + own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so + a second request before the first has even started is that same run.""" + sched = _scheduler + if sched is None: + # Only between shutdown and exit: start_scheduler runs at every API startup. + raise RuntimeError("the scheduler is not running") + try: + sched.add_job( + _ad_hoc, + "date", + args=(job, fn, actor_id, target), + id=f"{job}@{target}", + misfire_grace_time=None, + ) + # silent-ok: not a failure — the run for this target is already queued, and it does this work. + except ConflictingIdError: + log.info("ad-hoc %s for %s already queued", job, target) + + +def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None: + try: + run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) + except Exception: + # No row to tell it in: the database is what failed. The log is the record. + log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) + return + with logs.bind(run_id=run_id): + try: + _finish_ok(job, run_id, fn()) + except Exception as exc: + log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) + _finish_error(run_id, exc) + + # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of # 10 would queue a scheduled fire behind long manual runs. _POOL_SIZE = len(JOB_FUNCS) + 8 diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index c13c0875..d13e904b 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -437,7 +437,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - # Channels being deep-backfilled RIGHT NOW by EITHER path — the interactive backfill_channel_now (a -# FastAPI BackgroundTasks threadpool thread) OR the scheduler's run_deep_backfill (its own thread). +# one-shot run on the scheduler's pool, see scheduler.run_ad_hoc) OR the scheduler's run_deep_backfill. # BOTH register the channel they're paging (under the lock) so the other skips it: no two runs page # the same channel's `backfill_cursor` concurrently, whichever started first. `_immediate_running` # counts only the interactive path, capped so one user's burst can't drain the shared budget (the @@ -448,9 +448,9 @@ _backfill_lock = threading.Lock() _MAX_CONCURRENT_IMMEDIATE = 2 -def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None: - """User-initiated "get full history": deep-backfill ONE channel to completion right now, in a - background task with its OWN session, so the full catalogue lands in seconds instead of over the +def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> dict: + """User-initiated "get full history": deep-backfill ONE channel to completion right now, as its + own recorded run with its OWN session, so the full catalogue lands in seconds instead of over the paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69 pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in @@ -459,34 +459,35 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the shared backfill budget by opting many channels in at once (the overflow, and any channel too large to finish under the reserve, is finished by the scheduler since deep_requested stays set), and - mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and - swallows failures.""" + mutually excluded per-channel with the scheduler via _backfill_in_progress. + + Returns what the run did, as a job result: the counts, or `{"skipped": }` when it left the + channel to the scheduler. A failure raises — the run that wraps this records it.""" global _immediate_running with _backfill_lock: - if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE: - return # already backfilling this channel, or too many interactive runs — leave it to the scheduler + if channel_id in _backfill_in_progress: + return {"skipped": "channel busy"} + if _immediate_running >= _MAX_CONCURRENT_IMMEDIATE: + return {"skipped": "at capacity"} _backfill_in_progress.add(channel_id) _immediate_running += 1 try: with SessionLocal() as db: user = get_service_user(db) if user is None: - return + return {"skipped": "no service user"} channel = db.get(Channel, channel_id) if channel is None or channel.backfill_done or not channel.uploads_playlist_id: - return + return {"skipped": "nothing to do"} if quota.backfill_reserve_reached(): - return - try: - # Attribute the spend to the requesting user under the full-backfill action, so it - # shows up correctly in the usage breakdown (not the default OTHER/system bucket). - with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL): - with YouTubeClient(db, user) as yt: - backfill_channel_deep(db, yt, channel, max_pages) - reconcile_full_history(db) - except Exception: - db.rollback() - log.exception("Immediate backfill failed for channel %s", channel_id) + return {"skipped": "low quota"} + # Attribute the spend to the requesting user under the full-backfill action, so it + # shows up correctly in the usage breakdown (not the default OTHER/system bucket). + with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL): + with YouTubeClient(db, user) as yt: + added = backfill_channel_deep(db, yt, channel, max_pages) + reconciled = reconcile_full_history(db) + return {"videos_added": added, "reconciled_full": reconciled} finally: with _backfill_lock: _backfill_in_progress.discard(channel_id) diff --git a/backend/tests/test_channel_backfill.py b/backend/tests/test_channel_backfill.py index 7e651605..14dfd41d 100644 --- a/backend/tests/test_channel_backfill.py +++ b/backend/tests/test_channel_backfill.py @@ -107,22 +107,32 @@ def _setup_channel_and_sub(db, *, email): def _patch_deep_requested(db, monkeypatch, override_dep, user): + submitted: list[tuple] = [] calls: list[tuple] = [] - # background.add_task runs after the response in TestClient; record what would be scheduled. + # Record the run the route submits, and what its work would call, instead of running it. + monkeypatch.setattr( + channels_route.scheduler, + "run_ad_hoc", + lambda job, fn, **kw: submitted.append((job, kw)) or fn(), + ) monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a)) monkeypatch.setattr(state, "_configured_cache", True) override_dep(get_db, db.info["request"]) override_dep(current_user, lambda: user) resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True}) - return resp, calls + return resp, submitted, calls -def test_deep_request_schedules_immediate_backfill(db, monkeypatch, override_dep): +def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel( + db, monkeypatch, override_dep +): + """It was a BackgroundTask: what it did, spent or failed with reached only the log.""" ch, me = _setup_channel_and_sub(db, email="wire@example.com") - resp, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) + resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) assert resp.status_code == 200, resp.text assert resp.json()["deep_requested"] is True - # scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs. + assert submitted == [("channel_backfill", {"actor_id": me.id, "target": ch.id})] + # its work runs exactly once, with (channel_id, user_id) — the args backfill_channel_now needs. assert calls == [(ch.id, me.id)] @@ -148,15 +158,49 @@ class TestImmediateBackfillGuard: def test_a_second_call_for_an_in_flight_channel_is_a_noop(self, monkeypatch): runner._backfill_in_progress.add("UC_busy") monkeypatch.setattr(runner, "SessionLocal", self._boom_session) - runner.backfill_channel_now("UC_busy", 1) # returns immediately; _boom_session never runs + # returns at once — _boom_session never runs — and the run says why it did nothing. + assert runner.backfill_channel_now("UC_busy", 1) == {"skipped": "channel busy"} assert runner._backfill_in_progress == {"UC_busy"} # the in-flight entry is untouched def test_over_the_concurrency_cap_falls_back_to_the_scheduler(self, monkeypatch): runner._immediate_running = runner._MAX_CONCURRENT_IMMEDIATE monkeypatch.setattr(runner, "SessionLocal", self._boom_session) - runner.backfill_channel_now("UC_new", 1) # over the cap → no-op, scheduler will handle it + # over the cap → no-op, the scheduler will handle it + assert runner.backfill_channel_now("UC_new", 1) == {"skipped": "at capacity"} assert "UC_new" not in runner._backfill_in_progress + @staticmethod + def _one_channel(db, monkeypatch, cid): + ch = Channel(id=cid, title=cid, uploads_playlist_id="UU" + cid, backfill_done=False) + me = User(email=f"{cid}@example.com") + db.add_all([ch, me]) + db.commit() + monkeypatch.setattr(runner, "get_service_user", lambda _db: me) + monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False) + monkeypatch.setattr(runner, "YouTubeClient", lambda *a, **k: _NullYt()) + return ch, me + + def test_a_failure_reaches_its_run_and_still_frees_the_channel(self, db, monkeypatch): + """The failure was logged and swallowed, so the run that now wraps it would read "ok".""" + ch, me = self._one_channel(db, monkeypatch, "UC_fail") + + def boom(*_a): + raise RuntimeError("playlistItems 500") + + monkeypatch.setattr(runner, "backfill_channel_deep", boom) + with pytest.raises(RuntimeError, match="playlistItems 500"): + runner.backfill_channel_now(ch.id, me.id) + assert runner._backfill_in_progress == set() and runner._immediate_running == 0 + + def test_a_finished_backfill_says_what_it_added(self, db, monkeypatch): + ch, me = self._one_channel(db, monkeypatch, "UC_ok") + monkeypatch.setattr(runner, "backfill_channel_deep", lambda *_a: 7) + monkeypatch.setattr(runner, "reconcile_full_history", lambda _db: 1) + assert runner.backfill_channel_now(ch.id, me.id) == { + "videos_added": 7, + "reconciled_full": 1, + } + def test_scheduler_deep_pass_skips_an_in_flight_channel(self, db, monkeypatch): # A channel the scheduler's query WOULD pick (deep-requested, recent-synced, not done)… ch = Channel( diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 1afc149d..afcaac84 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -146,6 +146,28 @@ def test_the_last_run_survives_a_restart(db): assert plex["last_finished"] is not None +def test_a_row_shows_its_last_three_runs_newest_first_an_interrupted_one_included(db): + """The row said only how the LAST run went; an interrupted run read as "no run recorded".""" + oldest = jobruns.start("shorts", RunTrigger.SCHEDULED, None) + jobruns.interrupt_leftovers() + scheduler._job("shorts", lambda _db: {"probed": 1}) + scheduler._job("shorts", lambda _db: {"probed": 2}) + scheduler._job("shorts", lambda _db: 1 / 0) + scheduler._job("rss_poll", lambda _db: {"new": 0}) # another job's run stays on its own row + + recent = _snapshot_job(db, "shorts")["recent"] + assert [r["status"] for r in recent] == ["error", "ok", "ok"] + assert oldest not in [r["id"] for r in recent] # the fourth-newest is not shown + assert recent[0]["trigger"] == "scheduled" and recent[0]["finished_at"] is not None + assert [r["status"] for r in _snapshot_job(db, "rss_poll")["recent"]] == ["ok"] + + jobruns.start("shorts", RunTrigger.SCHEDULED, None) + jobruns.interrupt_leftovers() + shorts = _snapshot_job(db, "shorts") + assert shorts["status"] == "interrupted" + assert [r["status"] for r in shorts["recent"]] == ["interrupted", "error", "ok"] + + def test_a_paused_sync_is_a_skipped_run(db): state.set_sync_paused(db, True) db.commit() @@ -286,6 +308,51 @@ def test_run_now_goes_through_the_scheduler_even_with_interval_jobs_off( assert (run.trigger, run.actor_id, run.result) == ("manual", admin.id, {"probed": 3}) +def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure( + db, make_user, live_scheduler +): + """The one-channel deep backfill ran as a BackgroundTask nothing recorded: its spend counted as + nobody's run and a failure lived only in the log.""" + user = make_user("adhoc@example.com") + + def work(): + quota.record_usage(3) + return {"videos_added": 50} + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, work, actor_id=user.id, target="UC_a") + run = _wait_finished(db, "channel_backfill") + assert (run.trigger, run.actor_id, run.target) == ("ad_hoc", user.id, "UC_a") + assert (run.status, run.result, run.quota_units) == ("ok", {"videos_added": 50}, 3) + + def fail(): + raise RuntimeError("quota exceeded") + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, fail, actor_id=user.id, target="UC_b") + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and len(_finished(db, "channel_backfill")) < 2: + time.sleep(0.05) + failed = next(r for r in _runs(db, "channel_backfill") if r.target == "UC_b") + assert (failed.status, failed.error) == ("error", "quota exceeded") + + +def _finished(db, job) -> list[JobRun]: + return [r for r in _runs(db, job) if r.status != "running"] + + +def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( + db, make_user, live_scheduler +): + user = make_user("twice@example.com") + live_scheduler.pause() # hold the one-shots so the second request finds the first pending + for target in ("UC_x", "UC_x", "UC_y"): + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, dict, actor_id=user.id, target=target) + assert sorted(j.id for j in live_scheduler.get_jobs()) == [ + "channel_backfill@UC_x", + "channel_backfill@UC_y", + ] + live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next + + def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user): """The scheduler's pool threads are reused: an actor a manual run left set made the next scheduled run on that thread an admin's "manual" run, with a notice in their inbox.""" diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index 148445a3..d2140e87 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -185,7 +185,10 @@ "syncpaused": "sync paused", "lowquota": "low quota", "noreadscope": "no YouTube read access", - "busy": "another sync is running" + "busy": "another sync is running", + "channelbusy": "this channel is already being backfilled", + "atcapacity": "two full-history backfills are already running — the scheduler finishes this one", + "nothingtodo": "nothing left to fetch" }, "failure": { "rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.", diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 238d1a79..a49bb6bb 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -185,7 +185,10 @@ "syncpaused": "a szinkron szünetel", "lowquota": "kevés a kvóta", "noreadscope": "nincs YouTube-olvasási jogosultság", - "busy": "egy másik szinkron fut" + "busy": "egy másik szinkron fut", + "channelbusy": "ennek a csatornának a betöltése már fut", + "atcapacity": "már két teljes előzmény-betöltés fut — ezt az ütemező fejezi be", + "nothingtodo": "nincs több letöltendő" }, "failure": { "rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.", From 1b1e3cdcf2323250beb4fdb250a8df66125042ac Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 01:20:53 +0200 Subject: [PATCH 05/16] feat(scheduler): each row shows its last three runs, an interrupted run as such, and the ad-hoc runs beside the jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A row's one dot said how the LAST run went, and an interrupted run fell through to "idle" — a grey dot that read as "no run recorded". The row now leads with its last three runs, newest first, each named for a screen reader and in its tooltip (how it went, when, who started it); an interrupted run is a ring, set apart from a skipped run and a failure, and its line says the server restarted under it. A channel's full-history backfill gets its own "Ad-hoc runs" block: its runs, its channel and what it fetched, no interval and no run button — it needs its target. AD_HOC_JOBS names every such job; the backend-id contract holds their names in both languages, and run_ad_hoc refuses any other id. --- backend/app/scheduler.py | 69 +++++-- backend/tests/test_scheduler.py | 35 +++- .../src/components/Scheduler.dom.test.tsx | 86 +++++++- .../components/Scheduler.menu.dom.test.tsx | 5 + frontend/src/components/Scheduler.tsx | 185 ++++++++++++++---- frontend/src/i18n/backendIds.contract.test.ts | 17 +- frontend/src/i18n/locales/en/scheduler.json | 26 ++- frontend/src/i18n/locales/hu/scheduler.json | 26 ++- frontend/src/lib/api/admin.ts | 44 ++++- 9 files changed, 404 insertions(+), 89 deletions(-) diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 5280f98b..987443c9 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -15,7 +15,7 @@ from app import audit, debugsession, jobruns, logs, logsink, progress, quota from app.config import settings from app.db import SessionLocal from app.downloads.gc import run_download_gc -from app.models import SchedulerSetting +from app.models import Channel, SchedulerSetting from app.plex.sync import sync as run_plex_sync from app.plex.tmdb import run_enrich as run_tmdb_enrich from app.plex.tmdb import run_people_enrich @@ -253,38 +253,67 @@ def scheduler_snapshot(db) -> dict: jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) - runs = recent.get(job_id, []) - run = runs[0] if runs else None - # A run still going has no outcome yet: only an ended run says how it went. - ended = run if run is not None and run.status != RunStatus.RUNNING else None jobs.append( { "id": job_id, "interval_minutes": live_intervals.get(job_id, default_interval), "next_run": next_runs.get(job_id), "running": a.get("running", False), - "status": ended.status if ended else None, - "last_started": _iso(run.started_at) if run else None, - "last_finished": _iso(run.finished_at) if run else None, - "last_result": ended.summary if ended else None, - # The outcome as data: the result, or on a failed run the failure the job named. - "result": _outcome_data(ended), - "last_error": ended.error if ended else None, - "last_trigger": run.trigger if run else None, - "last_run_id": run.id if run else None, - "quota_units": ended.quota_units if ended else None, "progress": a.get("progress"), - # The last few runs, newest first, for the row's history dots. - "recent": [_run_brief(r) for r in runs], + **_last_runs(recent.get(job_id, [])), } ) - return {"running_here": running_here, "enabled": settings.scheduler_enabled, "jobs": jobs} + # Work a user starts on one target: no interval, no "run now" (it needs its target), but its + # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the + # target's own (sync/runner), so "running" is what its newest row says. + backfills = recent.get(CHANNEL_BACKFILL, []) + newest = backfills[0] if backfills else None + ad_hoc = [ + { + "id": CHANNEL_BACKFILL, + "running": newest is not None and newest.status == RunStatus.RUNNING, + "target": newest.target if newest else None, + "target_title": ( + db.scalar(select(Channel.title).where(Channel.id == newest.target)) + if newest is not None and newest.target + else None + ), + **_last_runs(backfills), + } + ] + return { + "running_here": running_here, + "enabled": settings.scheduler_enabled, + "jobs": jobs, + "ad_hoc": ad_hoc, + } # How many past runs a Scheduler row shows as dots; the full history is the run list. RECENT_RUNS = 3 +def _last_runs(runs: list) -> dict: + """A row's run fields from its newest runs (newest first): how the last one went, and the few + before it for the history dots.""" + run = runs[0] if runs else None + # A run still going has no outcome yet: only an ended run says how it went. + ended = run if run is not None and run.status != RunStatus.RUNNING else None + return { + "status": ended.status if ended else None, + "last_started": _iso(run.started_at) if run else None, + "last_finished": _iso(run.finished_at) if run else None, + "last_result": ended.summary if ended else None, + # The outcome as data: the result, or on a failed run the failure the job named. + "result": _outcome_data(ended), + "last_error": ended.error if ended else None, + "last_trigger": run.trigger if run else None, + "last_run_id": run.id if run else None, + "quota_units": ended.quota_units if ended else None, + "recent": [_run_brief(r) for r in runs], + } + + def _run_brief(run) -> dict: return { "id": run.id, @@ -539,12 +568,16 @@ def trigger_all(actor_id: int | None = None) -> list[str]: # one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and # spent is in `job_runs` like any job's, not a background task only the log ever heard of. CHANNEL_BACKFILL = "channel_backfill" +# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple). +AD_HOC_JOBS = (CHANNEL_BACKFILL,) def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None: """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so a second request before the first has even started is that same run.""" + if job not in AD_HOC_JOBS: + raise KeyError(job) # an unnamed job would show on the page as its bare id sched = _scheduler if sched is None: # Only between shutdown and exit: start_scheduler runs at every API startup. diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index afcaac84..9150a7a3 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -13,7 +13,7 @@ import pytest from sqlalchemy import select from app import jobruns, logs, logsink, quota, scheduler, state -from app.models import AuditLog, JobRun, Notification, QuotaEvent +from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent from app.runstatus import RunStatus, RunTrigger _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs @@ -335,10 +335,43 @@ def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure assert (failed.status, failed.error) == ("error", "quota exceeded") +def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user): + user = make_user("shown@example.com") + db.add(Channel(id="UC_shown", title="BBC Earth")) + db.commit() + scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none") + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown") + + (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert (row["id"], row["running"], row["target"], row["target_title"]) == ( + "channel_backfill", + False, + "UC_shown", + "BBC Earth", + ) + assert (row["status"], row["result"], row["last_trigger"]) == ( + "ok", + {"videos_added": 69}, + "ad_hoc", + ) + assert len(row["recent"]) == 2 + assert all(j["id"] != "channel_backfill" for j in scheduler.scheduler_snapshot(db)["jobs"]) + + jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_shown") + assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is True + + def _finished(db, job) -> list[JobRun]: return [r for r in _runs(db, job) if r.status != "running"] +def test_an_ad_hoc_job_outside_the_named_list_is_refused(live_scheduler): + """The page names every ad-hoc job from AD_HOC_JOBS; one outside it would show as a bare id.""" + with pytest.raises(KeyError): + scheduler.run_ad_hoc("unnamed_job", dict, actor_id=1, target="x") + assert live_scheduler.get_jobs() == [] + + def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( db, make_user, live_scheduler ): diff --git a/frontend/src/components/Scheduler.dom.test.tsx b/frontend/src/components/Scheduler.dom.test.tsx index 88301ba4..89477800 100644 --- a/frontend/src/components/Scheduler.dom.test.tsx +++ b/frontend/src/components/Scheduler.dom.test.tsx @@ -6,7 +6,7 @@ // because this page polls: every few seconds a fresh status arrives, and a mirror is exactly the // thing a poll can overwrite at the wrong moment. import { describe, it, expect, vi, afterEach, beforeEach } from "vitest"; -import { render, screen, fireEvent, cleanup, act, waitFor } from "@testing-library/react"; +import { render, screen, fireEvent, cleanup, act, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; vi.mock("react-i18next", async (importOriginal) => { @@ -15,7 +15,7 @@ vi.mock("react-i18next", async (importOriginal) => { }); import Scheduler from "./Scheduler"; -import { api, type SchedulerStatus } from "../lib/api"; +import { api, type RunBrief, type SchedulerAdHoc, type SchedulerStatus } from "../lib/api"; import { qk } from "../lib/queryKeys"; const status = (over: Partial = {}): SchedulerStatus => ({ @@ -34,9 +34,14 @@ const status = (over: Partial = {}): SchedulerStatus => ({ last_result: null, result: null, last_error: null, + last_trigger: null, + last_run_id: null, + quota_units: null, + recent: [], progress: null, }, ], + ad_hoc: [], queue: { channels_recent_pending: 0, channels_deep_pending: 0, @@ -281,3 +286,80 @@ describe("a job's last result", () => { expect(screen.queryByText(/new=3/)).toBeNull(); }); }); + +describe("a row's run history", () => { + const at = (minutesAgo: number) => new Date(Date.now() - minutesAgo * 60_000).toISOString(); + const run = (id: number, status: RunBrief["status"], minutesAgo: number): RunBrief => ({ + id, + status, + trigger: "scheduled", + started_at: at(minutesAgo), + finished_at: status === "running" ? null : at(minutesAgo), + }); + const dots = (root: HTMLElement = document.body) => + within(root) + .getAllByRole("img") + .map((d) => d.getAttribute("data-dot")); + + // An interrupted run fell through to "idle" — a grey dot that said no run was ever recorded. + it("shows the last runs newest first, an interrupted one as such", async () => { + const base = status(); + await mount({ + ...base, + jobs: [ + { + ...base.jobs[0]!, + status: "interrupted", + last_finished: at(2), + recent: [run(3, "interrupted", 2), run(2, "error", 12), run(1, "ok", 22)], + }, + ], + }); + expect(dots()).toEqual(["interrupted", "error", "ok"]); + expect(screen.getByText(/scheduler\.interrupted/)).toBeTruthy(); + expect(screen.getByText("scheduler.dot.interrupted")).toBeTruthy(); // the legend names it + }); + + it("shows one idle dot for a job that never ran", async () => { + await mount(); + expect(dots()).toEqual(["idle"]); + }); + + it("gives a run the job has claimed, but not yet recorded, the newest dot", async () => { + const base = status(); + await mount({ + ...base, + jobs: [ + { + ...base.jobs[0]!, + running: true, + recent: [run(3, "ok", 10), run(2, "ok", 20), run(1, "error", 30)], + }, + ], + }); + expect(dots()).toEqual(["running", "ok", "ok"]); + }); + + // The one-channel full-history backfill ran unseen: a background task only the log heard of. + it("lists a user's ad-hoc run beside the jobs, with its channel and no run button", async () => { + const base = status(); + const adHoc: SchedulerAdHoc = { + ...base.jobs[0]!, + id: "channel_backfill", + running: false, + target: "UC_bbc", + target_title: "BBC Earth", + last_finished: at(60), + result: { videos_added: 69 }, + recent: [run(9, "ok", 60), run(8, "ok", 120)], + }; + await mount({ ...base, ad_hoc: [adHoc] }); + expect(screen.getByText("scheduler.adHocTitle")).toBeTruthy(); + const row = screen.getByText("scheduler.jobs.channel_backfill").closest(".p-3")!; + expect(dots(row)).toEqual(["ok", "ok"]); + expect(within(row).getByText(/BBC Earth/)).toBeTruthy(); + expect(within(row).getByText(/scheduler\.result\.videos_added: 69/)).toBeTruthy(); + expect(within(row).getByText("scheduler.adHocStarted")).toBeTruthy(); + expect(within(row).queryByRole("button", { name: "scheduler.runNow" })).toBeNull(); + }); +}); diff --git a/frontend/src/components/Scheduler.menu.dom.test.tsx b/frontend/src/components/Scheduler.menu.dom.test.tsx index c9322a1f..fbf38e8f 100644 --- a/frontend/src/components/Scheduler.menu.dom.test.tsx +++ b/frontend/src/components/Scheduler.menu.dom.test.tsx @@ -32,9 +32,14 @@ const status = (over: Partial = {}): SchedulerStatus => ({ last_result: null, result: null, last_error: null, + last_trigger: null, + last_run_id: null, + quota_units: null, + recent: [], progress: null, }, ], + ad_hoc: [], queue: { channels_recent_pending: 0, channels_deep_pending: 0, diff --git a/frontend/src/components/Scheduler.tsx b/frontend/src/components/Scheduler.tsx index 29075f6b..91d8333f 100644 --- a/frontend/src/components/Scheduler.tsx +++ b/frontend/src/components/Scheduler.tsx @@ -17,7 +17,14 @@ import { Trash2, X, } from "lucide-react"; -import { api, type SchedulerJob, type SchedulerStatus } from "../lib/api"; +import { + api, + type RunBrief, + type RunFields, + type SchedulerAdHoc, + type SchedulerJob, + type SchedulerStatus, +} from "../lib/api"; import { useLiveQuery } from "../lib/useLiveQuery"; import { Button } from "./ui/Button"; import { LoadingState, StateMessage } from "./ui/QueryState"; @@ -57,41 +64,40 @@ function fmtCountdown(t: TFunction, s: number, lang: string): string { return `${formatUnit(h, "hour", lang)} ${formatUnit(m % 60, "minute", lang)}`; } -const DOT_CLASS: Record = { +// Interrupted is a ring, not a fill: the job did not fail, but its work did not finish either — set +// apart from a skipped run (the same colour, filled) and from a failure. +const DOT_CLASS = { running: "bg-accent animate-pulse", ok: "bg-success", error: "bg-danger", skipped: "bg-warn", + interrupted: "border-2 border-warn", idle: "bg-border", -}; +} as const; -function statusKey(job: SchedulerJob): keyof typeof DOT_CLASS { - if (job.running) return "running"; - if (job.status === "ok") return "ok"; - if (job.status === "error") return "error"; - if (job.status === "skipped") return "skipped"; - return "idle"; -} +type DotKey = keyof typeof DOT_CLASS; -function StatusDot({ - k, - withTooltip = true, -}: { - k: keyof typeof DOT_CLASS; - withTooltip?: boolean; -}) { - const { t } = useTranslation(); - const dot = ; - return withTooltip ? {dot} : dot; +/** A status dot. `label` names it for a screen reader (the tooltip shows only on hover, and only with + * hints on); a legend's dot has its words beside it and stays unnamed. */ +function StatusDot({ k, label }: { k: DotKey; label?: string }) { + return ( + + ); } function StatusLegend() { const { t } = useTranslation(); return (
- {(["running", "ok", "idle", "error", "skipped"] as const).map((k) => ( + {(["running", "ok", "idle", "error", "skipped", "interrupted"] as const).map((k) => ( - + {t(`scheduler.dot.${k}`)} ))} @@ -99,6 +105,106 @@ function StatusLegend() { ); } +// The server sends a row's newest runs, at most this many (RECENT_RUNS in app/scheduler.py). +const MAX_DOTS = 3; + +/** A row's last runs, newest first (leftmost), each saying in its tooltip how it went, when, and who + * started it. A row with no run yet shows one idle dot. While a run goes on the newest dot pulses: + * the job's own claim says so a moment before its run row exists, so that moment gets a dot too. */ +function RunDots({ runs, running }: { runs: RunBrief[]; running: boolean }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const claimed = running && runs[0]?.status !== "running"; + const shown: (RunBrief | null)[] = claimed ? [null, ...runs].slice(0, MAX_DOTS) : runs; + const dots = shown.length === 0 ? [undefined] : shown; + return ( + + {dots.map((run) => { + const hint = + run === undefined + ? t("scheduler.dot.idle") + : run === null + ? t("scheduler.dot.running") + : t("scheduler.dotRun", { + state: t(`scheduler.dot.${run.status}`), + time: relativeTime(run.started_at, lang, t), + trigger: t(`scheduler.trigger.${run.trigger}`), + }); + const k: DotKey = run === undefined ? "idle" : run === null ? "running" : run.status; + return ( + + + + ); + })} + + ); +} + +/** "Last run 2 min ago · ", the line under a row's name: how the newest ended run went + * — its failure, its result, or that the process stopped under it. */ +function LastRun({ row, running, target }: { row: RunFields; running: boolean; target?: string }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + return ( +
+ {running + ? t("scheduler.runningNow") + : row.last_finished + ? t("scheduler.lastRun", { time: relativeTime(row.last_finished, lang, t) }) + : t("scheduler.neverRun")} + {target && · {target}} + {row.last_error ? ( + + {" · "} + {jobFailureText(t, row.result, lang) ?? row.last_error} + + ) : row.status === "interrupted" ? ( + · {t("scheduler.interrupted")} + ) : row.result ? ( + · {jobResultText(t, row.result, lang)} + ) : row.last_result ? ( + · {row.last_result} + ) : null} +
+ ); +} + +/** A user's run on one target (a channel's full history): its runs as a job's are shown, but it has + * no interval and no "run now" — it needs its target, so it is started where that is chosen. */ +function AdHocRow({ row }: { row: SchedulerAdHoc }) { + const { t } = useTranslation(); + return ( +
+ +
+
+ + + {t(`scheduler.jobs.${row.id}`, row.id)} + + +
+ +
+
+ {row.running ? ( + {t("scheduler.active")} + ) : ( + t("scheduler.adHocStarted") + )} +
+
+ ); +} + // Shown for ANY running job. With reported counts it's a determinate bar; for a job that // runs but doesn't report progress (or hasn't yet) it falls back to a generic "working" // label and an indeterminate sliver — so every active run is visible, not just the ones @@ -186,7 +292,7 @@ function JobRow({ return (
- +
@@ -247,26 +353,7 @@ function JobRow({ )}
-
- {job.running - ? t("scheduler.runningNow") - : job.last_finished - ? t("scheduler.lastRun", { time: relativeTime(job.last_finished, lang, t) }) - : t("scheduler.neverRun")} - {job.last_error ? ( - - {" · "} - {jobFailureText(t, job.result, lang) ?? job.last_error} - - ) : job.result ? ( - · {jobResultText(t, job.result, lang)} - ) : job.last_result ? ( - · {job.last_result} - ) : null} -
+ {job.running && }
@@ -646,6 +733,18 @@ export default function Scheduler() {
+ {/* Ad-hoc runs: a user's work on one target, shown beside the jobs because it spends the same quota */} + {data.ad_hoc.length > 0 && ( +
+ {t("scheduler.adHocTitle")} +
+ {data.ad_hoc.map((row) => ( + + ))} +
+
+ )} + {/* Queue */}
{t("scheduler.queueTitle")} diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts index f2a3671d..7c6b8ba5 100644 --- a/frontend/src/i18n/backendIds.contract.test.ts +++ b/frontend/src/i18n/backendIds.contract.test.ts @@ -10,7 +10,7 @@ import { vocabKey } from "../lib/vocab"; // the registries straight from the backend sources: // - `sysconfig.SPECS` → `config.groups.`, `config.fields..label` + `.hint`, and for a // "choice" key `config.choices..
+ + {state.rows.map((row, index) => { + const { head, sub } = auditText(t, row, lang); + const who = actor(row); + return ( + setOpen(open === row.id ? null : row.id)} + title={formatDateTimeSec(row.created_at, lang)} + cells={ + <> + {auditActionLabel(t, row.action)} + + {formatDateTime(row.created_at, lang)} + + + {who ?? t("audit.system")} + + + {formatDateTime(row.created_at, lang)} + + + {[head, sub].filter(Boolean).join(" · ")} + + + } + > + + + ); + })} + + {facets.isError &&

{t("activity.countsFailed")}

} +
+ ); +} + +function AuditDetail({ + row, + who, + onRequest, +}: { + row: AuditRow; + who: string | null; + onRequest: (requestId: string) => void; +}) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const { head, sub } = auditText(t, row, lang); + return ( +
+
+
{t("audit.cols.actor")}
+
{who ?? t("audit.system")}
+
{t("audit.cols.when")}
+
{formatDateTimeSec(row.created_at, lang)}
+ {(head || sub) && ( + <> +
{t("audit.cols.details")}
+
{[head, sub].filter(Boolean).join(" · ")}
+ + )} + {row.target_id && ( + <> +
{t("activity.audit.target")}
+
{row.target_id}
+ + )} +
+ {(row.before || row.after) && ( + 
+          {JSON.stringify({ before: row.before, after: row.after }, null, 2)}
+
+ )} + {row.request_id && ( +
+ +
+ )} +
+ ); +} diff --git a/frontend/src/components/ActivityQuota.tsx b/frontend/src/components/ActivityQuota.tsx new file mode 100644 index 00000000..12698596 --- /dev/null +++ b/frontend/src/components/ActivityQuota.tsx @@ -0,0 +1,137 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { keepPreviousData, useQuery } from "@tanstack/react-query"; + +import { api, type AdminQuotaRow } from "../lib/api"; +import { sinceOf } from "../lib/activityUrlState"; +import { formatNumber, quotaActionLabel } from "../lib/format"; +import { qk } from "../lib/queryKeys"; +import type { Shared } from "./ActivityRuns"; +import { Pressable } from "./ui/Pressable"; +import { LoadingState, StateMessage } from "./ui/QueryState"; +import { SURFACE } from "./ui/surfaces"; + +// Activity → Quota: the shared YouTube API budget's spend over the page's time range — per day +// (Pacific days, as the budget resets), then per account, each opening to its actions. The +// account filter narrows both to one account; the text filter has nothing to match here. + +export default function ActivityQuota({ shared }: { shared: Shared }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const q = useQuery({ + queryKey: qk.adminQuota({ window: shared.window, user: shared.user }), + queryFn: () => api.adminQuota(sinceOf(shared.window), shared.user), + placeholderData: keepPreviousData, + }); + const data = q.data; + const maxDaily = Math.max(1, ...(data?.daily ?? []).map((d) => d.total)); + const grandTotal = (data?.rows ?? []).reduce((s, r) => s + r.total, 0); + + return ( +
+

+ {t("stats.introBefore")} + {t("stats.introSystem")} + {t("stats.introAfter")} +

+ {q.isError && !data ? ( + void q.refetch()}> + {t("common.loadError")} + + ) : q.isLoading || !data ? ( + + ) : ( + <> + {q.isError && ( + void q.refetch()} className="mb-3"> + {t("activity.stale")} + + )} +
+
+ {t("activity.quota.total", { + window: t(`activity.windows.${shared.window}`), + units: formatNumber(grandTotal, lang), + })} +
+ {data.daily.length === 0 ? ( +
{t("stats.noUsageInRange")}
+ ) : ( +
+ {data.daily.map((d) => ( +
+
+
+ ))} +
+ )} +
+ +
+ {data.rows.length === 0 ? ( +
{t("stats.noUsageInRange")}
+ ) : ( + data.rows.map((r) => ( + + )) + )} +
+ + )} +
+ ); +} + +function UserRow({ row, max }: { row: AdminQuotaRow; max: number }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const [open, setOpen] = useState(false); + const actions = Object.entries(row.by_action).sort((a, b) => b[1] - a[1]); + const isSystem = row.user_id === null; + return ( +
+ setOpen((o) => !o)} + aria-expanded={open} + className="w-full flex items-center gap-3" + > + + {isSystem ? t("stats.system") : row.email} + + + {formatNumber(row.total, lang)} + + + {/* Proportion bar */} +
+
+
+ {open && actions.length > 0 && ( +
+ {actions.map(([action, units]) => ( +
+ {quotaActionLabel(t, action)} + {formatNumber(units, lang)} +
+ ))} +
+ )} +
+ ); +} diff --git a/frontend/src/components/ActivityRuns.tsx b/frontend/src/components/ActivityRuns.tsx new file mode 100644 index 00000000..3d0ea4d4 --- /dev/null +++ b/frontend/src/components/ActivityRuns.tsx @@ -0,0 +1,257 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { keepPreviousData, useQuery } from "@tanstack/react-query"; + +import { api, type LogFacet, type RunQuery, type RunRow } from "../lib/api"; +import { + RUN_STATUSES, + sinceOf, + type ActivityUrlState, + type RunStatusFilter, +} from "../lib/activityUrlState"; +import { formatDateTime, formatDateTimeSec, formatNumber, formatRuntime } from "../lib/format"; +import { jobFailureText, jobResultText } from "../lib/jobResult"; +import { qk } from "../lib/queryKeys"; +import { useHistoryList } from "../lib/useHistoryList"; +import HistoryList from "./HistoryList"; +import HistoryRow from "./HistoryRow"; +import { StatusDot } from "./RunStatusDot"; +import { Button } from "./ui/Button"; +import FilterChipGroup from "./ui/FilterChipGroup"; +import FilterSelect from "./ui/FilterSelect"; + +// Activity → Runs: every background run, newest first — the scheduled ones, an admin's "run now", a +// user's work on one channel. A row opens to who started it, when, what it returned or why it +// failed, and two ways on: the lines the run logged, and the other runs of its job. + +const GRID = + "grid-cols-[0.75rem_minmax(0,1fr)_auto] sm:grid-cols-[0.75rem_minmax(0,12rem)_8rem_8rem_4rem_3.5rem_minmax(0,1fr)]"; + +/** The shared filters, as the page passes them down. */ +export interface Shared { + window: ActivityUrlState["window"]; + user: string | null; + q: string; +} + +export default function ActivityRuns({ + shared, + job, + status, + onFilter, + onLogs, +}: { + shared: Shared; + job: string | null; + status: RunStatusFilter | null; + onFilter: (patch: { job?: string | null; status?: RunStatusFilter | null }) => void; + /** Open the server log at this run's lines. */ + onLogs: (runId: number) => void; +}) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const [open, setOpen] = useState(null); + const query: RunQuery = { + user: shared.user ?? undefined, + q: shared.q.trim() || undefined, + job: job ?? undefined, + status: status ?? undefined, + }; + const key = { ...query, window: shared.window }; + const state = useHistoryList(qk.adminRuns(key), (before) => + api.adminRuns({ ...query, since: sinceOf(shared.window) }, before), + ); + const facets = useQuery({ + queryKey: qk.adminRunFacets(key), + queryFn: () => api.adminRunFacets({ ...query, since: sinceOf(shared.window) }), + placeholderData: keepPreviousData, + }); + const count = (list: LogFacet[] | undefined, value: string) => + list?.find((f) => f.value === value)?.count ?? (facets.data ? 0 : undefined); + const jobName = (id: string) => t(`scheduler.jobs.${id}`, id); + // The job in use stays offered at zero: under the other filters it may count nothing, and a menu + // without it would show no way back. + const jobs = [ + ...(facets.data?.jobs ?? []), + ...(job && !facets.data?.jobs.some((f) => f.value === job) ? [{ value: job, count: 0 }] : []), + ]; + + return ( +
+
+ ({ value: f.value, label: jobName(f.value), count: f.count })), + ]} + selected={[job ?? ""]} + onChange={([value]) => onFilter({ job: value || null })} + testId="runs-job" + /> + + onFilter({ status: value === "all" ? null : (value as RunStatusFilter) }) + } + options={[ + { value: "all", label: t("activity.any"), count: facets.data?.total }, + ...RUN_STATUSES.map((s) => ({ + value: s, + label: t(`activity.runs.status.${s}`), + count: count(facets.data?.statuses, s), + })), + ]} + /> +
+ + {state.rows.map((run, index) => ( + setOpen(open === run.id ? null : run.id)} + title={formatDateTimeSec(run.started_at, lang)} + cells={ + + } + > + onLogs(run.id)} + onJob={() => onFilter({ job: run.job })} + /> + + ))} + + {facets.isError &&

{t("activity.countsFailed")}

} +
+ ); +} + +function seconds(run: RunRow): number | null { + if (!run.finished_at) return null; + return (new Date(run.finished_at).getTime() - new Date(run.started_at).getTime()) / 1000; +} + +/** What the run came to, in the reader's language: its failure, that the process stopped under it, + * or its result. */ +function useOutcome(run: RunRow): { text: string; tone: string } { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + if (run.status === "error") + return { + text: jobFailureText(t, run.failure, lang) ?? run.error ?? t("scheduler.dot.error"), + tone: "text-danger", + }; + if (run.status === "interrupted") return { text: t("scheduler.interrupted"), tone: "text-warn" }; + if (run.status === "running") return { text: t("scheduler.runningNow"), tone: "text-accent" }; + if (run.result) return { text: jobResultText(t, run.result, lang), tone: "text-muted" }; + return { text: run.summary ?? "", tone: "text-muted" }; +} + +function RunCells({ run, actor }: { run: RunRow; actor: string | null | undefined }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const outcome = useOutcome(run); + const secs = seconds(run); + const name = t(`scheduler.jobs.${run.job}`, run.job); + return ( + <> + + + {name} + {run.target_title && · {run.target_title}} + + {/* A phone has room for the name and the time only; the row opens to the rest. */} + + {formatDateTime(run.started_at, lang)} + + + {t(`scheduler.trigger.${run.trigger}`)} + {actor && ` · ${actor}`} + + + {formatDateTime(run.started_at, lang)} + + + {secs === null ? "—" : formatRuntime(Math.max(1, secs), lang)} + + + {formatNumber(run.quota_units, lang)} + + {outcome.text} + + ); +} + +function RunDetail({ + run, + actor, + onLogs, + onJob, +}: { + run: RunRow; + actor: string | null | undefined; + onLogs: () => void; + onJob: () => void; +}) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const link = "underline decoration-dotted underline-offset-2 hover:decoration-solid"; + return ( +
+
+
{t("activity.runs.fields.started")}
+
+ {run.actor_id === null + ? t(`scheduler.trigger.${run.trigger}`) + : `${actor ?? `#${run.actor_id}`} · ${t(`scheduler.trigger.${run.trigger}`)}`} +
+
{t("activity.runs.fields.when")}
+
+ {formatDateTimeSec(run.started_at, lang)} + {run.finished_at && ` → ${formatDateTimeSec(run.finished_at, lang)}`} +
+ {run.target && ( + <> +
{t("activity.runs.fields.target")}
+
+ {run.target_title ? `${run.target_title} · ` : ""} + {run.target} +
+ + )} +
{t("activity.runs.fields.quota")}
+
{formatNumber(run.quota_units, lang)}
+
+ {run.error && ( + 
{run.error}
+ )} + {run.result && ( + 
+          {JSON.stringify(run.result, null, 2)}
+
+ )} +
+ + +
+
+ ); +} diff --git a/frontend/src/components/AdminLogs.dom.test.tsx b/frontend/src/components/AdminLogs.dom.test.tsx index 9b9c28de..cfbb7062 100644 --- a/frontend/src/components/AdminLogs.dom.test.tsx +++ b/frontend/src/components/AdminLogs.dom.test.tsx @@ -1,9 +1,11 @@ // @vitest-environment jsdom // -// The admin log console. A request id narrows the list to that request (and says so, in the -// filter row and the URL); a filter value the facets no longer count stays pickable; "Live" watches -// the newest page only, and stops on a failed read instead of re-raising it; a failed read is an -// error, not an empty log; Copy takes every loaded line, oldest first. +// The activity page's server-log tab, rendered through the page (which owns the shared filters and +// the pivots). A request or run id narrows the list to that request or run (and says so, in the +// filter row and the URL), and Back returns to the view the pivot was made from; an old link to the +// retired log page opens this tab; the account in use stays pickable; "Live" watches the newest page +// only, and stops on a failed read instead of re-raising it; a failed read is an error, not an empty +// log; Copy takes every loaded line, oldest first. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -32,6 +34,7 @@ vi.mock("../lib/api", async (importOriginal) => { ...actual.api, adminLogs: (query: LogQuery, before: string | null) => logsMock(query, before), adminLogFacets: (query: LogQuery) => facetsMock(query), + adminUsers: () => Promise.resolve([{ id: 1, email: "admin@example.com" }]), }, }; }); @@ -44,7 +47,11 @@ vi.mock("../lib/clipboard", () => ({ }, })); -import AdminLogs from "./AdminLogs"; +import Activity from "./Activity"; +import { PageToolbarSlot } from "./ui/pageToolbarSlot"; + +// The filter bar lives in the toolbar, which portals into the shell's band: give it one. +const slot = document.body.appendChild(document.createElement("div")); function row(id: number, extra: Partial = {}): ServerLogRow { return { @@ -60,6 +67,7 @@ function row(id: number, extra: Partial = {}): ServerLogRow { request_id: `req-${String(id).padStart(8, "0")}`, user_id: 1, job_id: null, + run_id: null, debug_session_id: null, version: "0.90.0", ...extra, @@ -79,7 +87,9 @@ function renderConsole() { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( - + + + , ); } @@ -93,7 +103,7 @@ const lines = () => .filter((b) => b.hasAttribute("aria-expanded")); beforeEach(() => { - window.history.replaceState(null, "", "/?page=logs"); + window.history.replaceState(null, "", "/?page=activity&at=logs"); facetsMock.mockResolvedValue(FACETS); }); afterEach(() => { @@ -105,7 +115,7 @@ afterEach(() => { describe("AdminLogs", () => { it("a request id narrows the list to that request, with every other filter off", async () => { - window.history.replaceState(null, "", "/?page=logs&lgrp=error&lq=boom"); + window.history.replaceState(null, "", "/?page=activity&at=logs&lgrp=error&aq=boom"); logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); renderConsole(); @@ -121,11 +131,15 @@ describe("AdminLogs", () => { ); expect(screen.getByText("adminLogs.pivotRequest:req-00000001")).toBeTruthy(); const url = new URLSearchParams(window.location.search); - expect([url.get("lreq"), url.get("lgrp"), url.get("lq")]).toEqual(["req-00000001", null, null]); + expect([url.get("lreq"), url.get("lgrp"), url.get("aq")]).toEqual(["req-00000001", null, null]); }); it("Back from a request's lines returns to the view the pivot was made from", async () => { - window.history.replaceState({ sfPage: "logs" }, "", "/?page=logs&lgrp=error&lq=boom"); + window.history.replaceState( + { sfPage: "activity" }, + "", + "/?page=activity&at=logs&lgrp=error&aq=boom", + ); const depth = window.history.length; logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); renderConsole(); @@ -157,23 +171,60 @@ describe("AdminLogs", () => { ); expect(screen.queryByText("adminLogs.pivotRequest:req-00000001")).toBeNull(); const url = new URLSearchParams(window.location.search); - expect([url.get("lreq"), url.get("lgrp"), url.get("lq")]).toEqual([null, "error", "boom"]); + expect([url.get("lreq"), url.get("lgrp"), url.get("aq")]).toEqual([null, "error", "boom"]); // Neither jump waited out the typing debounce: no read ever mixed one view's filters with the // other's search text (the request with "boom", or the restored view without it). const mixed = logsMock.mock.calls.filter(([q]) => (q.request_id != null) === (q.q === "boom")); expect(mixed).toEqual([]); }); - it("keeps the account in use offered when the facets no longer count it", async () => { - window.history.replaceState(null, "", "/?page=logs&lu=7"); + it("a run id narrows the list to that run's lines", async () => { + logsMock.mockResolvedValue({ + items: [row(1, { run_id: 77 })], + accounts: {}, + next_cursor: null, + }); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + fireEvent.click(lines()[0]!); + fireEvent.click(screen.getByRole("button", { name: "77" })); + + await waitFor(() => + expect(logsMock).toHaveBeenLastCalledWith( + expect.objectContaining({ run_id: 77, request_id: undefined }), + null, + ), + ); + expect(screen.getByText("adminLogs.pivotRun:77")).toBeTruthy(); + expect(new URLSearchParams(window.location.search).get("lrun")).toBe("77"); + }); + + it("an old link to the server log page opens this tab with its filter", async () => { + window.history.replaceState(null, "", "/?page=logs&lreq=req-00000009"); logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); renderConsole(); - await waitFor(() => expect(facetsMock).toHaveBeenCalled()); - fireEvent.click(screen.getByTestId("logs-user")); + await waitFor(() => + expect(logsMock).toHaveBeenCalledWith( + expect.objectContaining({ request_id: "req-00000009" }), + null, + ), + ); + expect(new URLSearchParams(window.location.search).get("page")).toBe("activity"); + }); - expect(await screen.findByTestId("logs-user-7")).toBeTruthy(); - expect(screen.getByTestId("logs-user-1")).toBeTruthy(); + it("keeps the account in use offered when the account list does not have it", async () => { + window.history.replaceState(null, "", "/?page=activity&at=logs&au=7"); + logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); + renderConsole(); + await waitFor(() => + expect(logsMock).toHaveBeenCalledWith(expect.objectContaining({ user: "7" }), null), + ); + + fireEvent.click(screen.getByTestId("activity-user")); + + expect(await screen.findByTestId("activity-user-7")).toBeTruthy(); + expect(screen.getByTestId("activity-user-1")).toBeTruthy(); }); it("Live watches the newest page only", async () => { diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index 0dad5d23..e9932427 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -1,5 +1,4 @@ -import clsx from "clsx"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { keepPreviousData, @@ -7,26 +6,17 @@ import { useQuery, useQueryClient, } from "@tanstack/react-query"; -import { Copy, Radio, Search } from "lucide-react"; +import { Copy, Radio } from "lucide-react"; import { api, type LogFacet, type LogQuery, type ServerLogRow } from "../lib/api"; import { copyText } from "../lib/clipboard"; import { formatDateTimeSec } from "../lib/format"; -import { nextNavId, notifyNav, rememberNavPosition } from "../lib/history"; import { nextPageOnClick } from "../lib/nextPage"; import { LEVEL_GROUPS, type LevelGroup } from "../lib/log"; -import { - DEFAULT_LOGS_STATE, - logsParamsToState, - logsParamUpdates, - WINDOW_HOURS, - type LogsUrlState, - type LogWindow, -} from "../lib/logsUrlState"; +import { sinceOf } from "../lib/activityUrlState"; +import { logsParamsToState, logsParamUpdates, type LogsUrlState } from "../lib/logsUrlState"; import { notify } from "../lib/notifications"; import { qk } from "../lib/queryKeys"; -import { useDebounced } from "../lib/useDebounced"; -import { urlWithParams } from "../lib/urlState"; import { useUrlMirror } from "../lib/useUrlMirror"; import LogLine from "./LogLine"; import NavGrid from "./ui/NavGrid"; @@ -35,46 +25,45 @@ import FilterChipGroup from "./ui/FilterChipGroup"; import FilterSelect from "./ui/FilterSelect"; import { FilterFoldButton } from "./ui/FilterFold"; import { useFilterFold } from "../lib/useFilterFold"; -import { useStickyTop } from "./ui/stickyTop"; -import { Section } from "./ui/form"; import { LoadingState, StateMessage } from "./ui/QueryState"; import { DismissibleChip } from "./ui/Chip"; -import { InnerInput } from "./ui/Input"; -import { fieldClass } from "./ui/fieldStyles"; +import type { Shared } from "./ActivityRuns"; -// Admin → Szervernapló: the server's stored log (`log_events`). One list, newest first; a line -// opens in place to its full record, and its request id / job id narrow the list to that request or -// job. Filters live in the URL, so a filtered view is a link. "Élő" re-reads the newest page every +// Activity → Server log: the server's stored log (`log_events`). One list, newest first; a line +// opens in place to its full record, and its request / job / run id narrows the list to that +// request, job or run (the page does that: a pivot is a new history entry with every other filter +// off). The time range, account and text come from the page's shared bar; the rest live in the URL +// (./logsUrlState), so a filtered view is a link. "Élő" re-reads the newest page every // few seconds while the tab is visible — the newest page only: live is watching the top, so paging // into older rows turns it off. A failed read stops the polling (the request already reported it // once) and marks the list as possibly stale, instead of re-raising the failure every few seconds. const LIVE_MS = 4000; const GROUPS: readonly LevelGroup[] = ["all", "error", "warning", "info", "detail"]; -const WINDOWS: readonly LogWindow[] = ["1h", "24h", "7d"]; -// A pivot shows everything about one request or job: the other filters would only hide part of it. -const PIVOT_RESET: Omit = { - ...DEFAULT_LOGS_STATE, - window: "7d", -}; - -function sinceOf(window: LogWindow): string { - return new Date(Date.now() - WINDOW_HOURS[window] * 3_600_000).toISOString(); -} - -function toQuery(state: LogsUrlState, q: string): LogQuery { +function toQuery(state: LogsUrlState, shared: Shared): LogQuery { return { levels: state.group === "all" ? undefined : LEVEL_GROUPS[state.group], sources: state.sources, - user: state.user ?? undefined, + user: shared.user ?? undefined, event: state.event ?? undefined, request_id: state.requestId ?? undefined, job_id: state.jobId ?? undefined, - q: q.trim() || undefined, + run_id: state.runId ?? undefined, + q: shared.q.trim() || undefined, }; } -export default function AdminLogs() { +/** The ids a line can narrow the list to. */ +export type LogPivot = Pick; + +export default function AdminLogs({ + shared, + onPivot, +}: { + shared: Shared; + /** Show only one request's, job's or run's lines (the page clears every other filter). */ + onPivot: (pivot: LogPivot) => void; +}) { const { t, i18n } = useTranslation(); const lang = i18n.language; const qc = useQueryClient(); @@ -82,14 +71,9 @@ export default function AdminLogs() { logsParamsToState(new URLSearchParams(window.location.search)), ); const [live, setLiveState] = useState(false); - // The toolbar sticks at the scroller's top (./ui/stickyTop), and folds its facet row when short. - useStickyTop(); + // The facet row folds away when the screen is short. const fold = useFilterFold(); - const facetFilters = - state.sources.length + - (state.user ? 1 : 0) + - (state.event ? 1 : 0) + - (state.window !== DEFAULT_LOGS_STATE.window ? 1 : 0); + const facetFilters = state.sources.length + (state.event ? 1 : 0); // Read by `getNextPageParam` when a page is fetched, not when it renders: while live, a refetch // stops after the newest page, however many pages the cache holds — a "Load older" that lands // after Live was switched on, or a filter set revisited with older pages still cached. @@ -99,41 +83,26 @@ export default function AdminLogs() { setLiveState(on); }; const [open, setOpen] = useState(null); - // Bumped when the view changes by a jump (a pivot, Back/Forward) rather than by typing: the - // search box's text then applies at once instead of after the typing debounce. - const [jumps, setJumps] = useState(0); - const q = useDebounced(state.q, 300, jumps); useUrlMirror(logsParamUpdates(state)); - // Back/Forward lands on another view of the console (a pivot pushed one, below): read it back. - useEffect(() => { - const onPop = () => { - setOpen(null); - setJumps((n) => n + 1); - setState(logsParamsToState(new URLSearchParams(window.location.search))); - }; - window.addEventListener("popstate", onPop); - return () => window.removeEventListener("popstate", onPop); - }, []); - const update = (patch: Partial) => { setOpen(null); setState((s) => ({ ...s, ...patch })); }; - const query = toQuery(state, q); - const key = qk.adminLogs({ ...query, window: state.window }); + const query = toQuery(state, shared); + const key = qk.adminLogs({ ...query, window: shared.window }); const logs = useInfiniteQuery({ queryKey: key, queryFn: ({ pageParam }) => - api.adminLogs({ ...query, since: sinceOf(state.window) }, pageParam), + api.adminLogs({ ...query, since: sinceOf(shared.window) }, pageParam), initialPageParam: null as string | null, getNextPageParam: (last) => (liveRef.current ? undefined : (last.next_cursor ?? undefined)), placeholderData: keepPreviousData, refetchInterval: (q) => (live && q.state.status !== "error" ? LIVE_MS : false), }); const facets = useQuery({ - queryKey: qk.adminLogFacets({ ...query, window: state.window }), - queryFn: () => api.adminLogFacets({ ...query, since: sinceOf(state.window) }), + queryKey: qk.adminLogFacets({ ...query, window: shared.window }), + queryFn: () => api.adminLogFacets({ ...query, since: sinceOf(shared.window) }), placeholderData: keepPreviousData, refetchInterval: (q) => (live && q.state.status !== "error" ? LIVE_MS * 4 : false), }); @@ -217,19 +186,9 @@ export default function AdminLogs() { return facets.data ? sum(levels.filter((f) => inGroup.includes(f.value))) : undefined; }; - // A pivot follows a link to another view, so it is its own history entry and Back returns to the - // view it was made from. The other filters are adjustments of the view on screen and replace it. - const pivot = (patch: Pick) => { + const pivot = (patch: Partial) => { setLive(false); - const next = { ...state, ...PIVOT_RESET, ...patch }; - window.history.pushState( - { ...rememberNavPosition(), sfScroll: nextNavId() }, - "", - urlWithParams(logsParamUpdates(next)), - ); - notifyNav(); - setJumps((n) => n + 1); - update(next); + onPivot({ requestId: null, jobId: null, runId: null, ...patch }); }; // "Any", then what the facet counted. The value in use stays offered even at zero — under the @@ -253,193 +212,155 @@ export default function AdminLogs() { }; return ( -
-
-

{t("adminLogs.intro")}

- -
+ +
+ + {!fold.folded && ( + <> + v)} + selected={[state.sources[0] ?? ""]} + onChange={([source]) => update({ sources: source ? [source] : [] })} + testId="logs-source" + /> + + v === "none" ? t("adminLogs.noEvent") : v, + )} + selected={[state.event ?? ""]} + onChange={([event]) => update({ event: event || null })} + testId="logs-event" + /> + + )} + + + + +
+ + {(state.requestId || state.jobId !== null || state.runId !== null) && ( +
+ {state.requestId && ( + update({ requestId: null })} + /> + )} + {state.jobId !== null && ( + update({ jobId: null })} + /> + )} + {state.runId !== null && ( + update({ runId: null })} + /> + )} +
+ )} +
+ + {failure && ( + + {t(`adminLogs.${failure}`)} + + )} + +
+ {logs.isError && !logs.data ? ( + void logs.refetch()}> + {t("common.loadError")} + + ) : logs.isLoading ? ( + + ) : rows.length === 0 ? ( +

+ {t("adminLogs.empty")} +

+ ) : ( + + {rows.map((row, index) => ( + setOpen(open === row.id ? null : row.id)} + > + pivot({ requestId })} + onJob={(jobId) => pivot({ jobId })} + onRun={(runId) => pivot({ runId })} + /> + + ))} + + )} +
+ +
+ {t("adminLogs.count", { count: rows.length })} + {olderCursor !== null && ( + + )} +
); } @@ -465,11 +386,13 @@ function RowDetail({ account, onRequest, onJob, + onRun, }: { row: ServerLogRow; account: string | null; onRequest: (id: string) => void; onJob: (id: number) => void; + onRun: (id: number) => void; }) { const { t, i18n } = useTranslation(); const lang = i18n.language; @@ -516,6 +439,21 @@ function RowDetail({ )} + {row.run_id !== null && ( + <> +
{t("adminLogs.fields.run")}
+
+ +
+ + )} {row.version && ( <>
{t("adminLogs.fields.version")}
diff --git a/frontend/src/components/AuditLog.tsx b/frontend/src/components/AuditLog.tsx deleted file mode 100644 index 94bb5fc8..00000000 --- a/frontend/src/components/AuditLog.tsx +++ /dev/null @@ -1,105 +0,0 @@ -import { useMemo } from "react"; -import { qk } from "../lib/queryKeys"; -import { useTranslation } from "react-i18next"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Trash2 } from "lucide-react"; -import { api } from "../lib/api"; -import { notify } from "../lib/notifications"; -import { LS } from "../lib/storage"; -import { Button } from "./ui/Button"; -import { Section } from "./ui/form"; -import { useConfirm } from "./confirmContext"; -import DataTable from "./ui/DataTable"; -import { LoadingState, StateMessage } from "./ui/QueryState"; -import { auditColumns } from "./auditColumns"; -import { notifyActionError } from "../lib/actionErrors"; -import { compareText } from "../lib/collate"; - -// Admin-only audit trail: an append-only who/what/when log of admin actions (never per-row; a -// background run is not one — runs have their own history). Read-only apart from Clear. Client-side sort/filter/paginate via -// DataTable; the API returns the most recent window (capped) — the log stays small by design. -export default function AuditLog() { - const { t, i18n } = useTranslation(); - const lang = i18n.language; - const qc = useQueryClient(); - const confirm = useConfirm(); - const q = useQuery({ queryKey: qk.adminAudit(), queryFn: () => api.adminAudit() }); - const data = q.data; - const rows = useMemo(() => data?.items ?? [], [data]); - - const actionOptions = useMemo(() => { - // By what the reader sees — the named action, in their alphabet — not by the code. - return Array.from(new Set(rows.map((r) => r.action))) - .map((a) => ({ value: a, label: t(`auditActions.${a}`, a) })) - .sort((a, b) => compareText(a.label, b.label, lang)); - }, [rows, t, lang]); - const columns = useMemo(() => auditColumns(t, lang, actionOptions), [t, lang, actionOptions]); - - const clear = useMutation({ - mutationFn: () => api.clearAudit(), - onSuccess: (r) => { - qc.invalidateQueries({ queryKey: qk.adminAudit() }); - notify({ level: "success", message: t("audit.cleared", { count: r.cleared }) }); - }, - onError: (e) => notifyActionError(e, t("audit.clearFailed")), - }); - const onClear = async () => { - if ( - await confirm({ - title: t("audit.clearTitle"), - message: t("audit.clearConfirm"), - confirmLabel: t("audit.clear"), - danger: true, - }) - ) - clear.mutate(); - }; - - const truncated = !!data && data.total > data.returned; - - return ( -
-
-

{t("audit.intro")}

- {q.isError && !q.data ? ( - q.refetch()}> - {t("common.loadError")} - - ) : q.isLoading ? ( - - ) : rows.length === 0 ? ( -

{t("audit.empty")}

- ) : ( - <> - String(r.id)} - persistKey={LS.auditTable} - controlsPosition="top" - controlsInBand - controlsBandClassName="px-4 max-w-5xl mx-auto" - controlsLeading={ - - } - emptyText={t("audit.empty")} - /> - {truncated && ( -

- {t("audit.truncated", { returned: data!.returned, total: data!.total })} -

- )} - - )} -
-
- ); -} diff --git a/frontend/src/components/HistoryList.tsx b/frontend/src/components/HistoryList.tsx new file mode 100644 index 00000000..8862417f --- /dev/null +++ b/frontend/src/components/HistoryList.tsx @@ -0,0 +1,84 @@ +import type { ReactNode } from "react"; +import { useTranslation } from "react-i18next"; + +import { nextPageOnClick } from "../lib/nextPage"; +import type { HistoryList as HistoryListState } from "../lib/useHistoryList"; +import NavGrid from "./ui/NavGrid"; +import { Button } from "./ui/Button"; +import { LoadingState, StateMessage } from "./ui/QueryState"; + +// The frame of a history list on the activity page: the rows (or the honest state instead of them — +// a failed read is an error with a retry, never an empty list), the count, and "Older". + +export default function HistoryList({ + state, + empty, + count, + children, +}: { + state: HistoryListState; + /** Shown when the filters match nothing. */ + empty: string; + /** "N runs shown", for the footer. */ + count: string; + /** The rows, drawn from `state.rows`. */ + children: ReactNode; +}) { + const { t } = useTranslation(); + const { list, rows, hasOlder } = state; + // A read that failed while rows were on screen keeps them, and says what the failure means. + const failure = !list.data + ? null + : list.isFetchNextPageError + ? "olderFailed" + : list.isError + ? "stale" + : null; + return ( +
+ {failure && ( + + failure === "olderFailed" + ? void list.fetchNextPage(nextPageOnClick(list.isFetchingNextPage)) + : void list.refetch() + } + className="border-b border-border" + > + {failure === "olderFailed" ? t("activity.olderFailed") : t("activity.stale")} + + )} +
+ {list.isError && !list.data ? ( + void list.refetch()}> + {t("common.loadError")} + + ) : list.isLoading ? ( + + ) : rows.length === 0 ? ( +

{empty}

+ ) : ( + {children} + )} +
+
+ {count} + {hasOlder && ( + + )} +
+
+ ); +} diff --git a/frontend/src/components/HistoryRow.tsx b/frontend/src/components/HistoryRow.tsx new file mode 100644 index 00000000..f48f57f2 --- /dev/null +++ b/frontend/src/components/HistoryRow.tsx @@ -0,0 +1,55 @@ +import type { ReactNode } from "react"; + +import { navItemAttrs } from "../lib/listNav"; +import { Pressable } from "./ui/Pressable"; + +// One dense row of a history list on the activity page — a background run, an admin action: its +// cells on one line, and the row opens in place to its details, as a server log line does +// (./LogLine). Inside a NavGrid the keyboard walks the rows (`j`/`k`) and Enter opens one. + +export default function HistoryRow({ + cells, + grid, + navIndex, + navText, + open, + onToggle, + title, + children, +}: { + /** The row's cells, laid out by `grid`. */ + cells: ReactNode; + /** The grid's column template classes (phone first, then wider screens). */ + grid: string; + navIndex: number; + /** What the keyboard's type-ahead matches. */ + navText: string; + open: boolean; + onToggle: () => void; + /** The row button's tooltip — the full date. */ + title?: string; + /** The detail shown while the row is open. */ + children: ReactNode; +}) { + return ( +
+ + {cells} + + {open &&
{children}
} +
+ ); +} diff --git a/frontend/src/components/IssueComposer.dom.test.tsx b/frontend/src/components/IssueComposer.dom.test.tsx index 32437c55..02d9fbfd 100644 --- a/frontend/src/components/IssueComposer.dom.test.tsx +++ b/frontend/src/components/IssueComposer.dom.test.tsx @@ -112,6 +112,7 @@ const row = (over: Partial): ServerLogRow => ({ request_id: null, user_id: 1, job_id: null, + run_id: null, debug_session_id: SID, version: null, ...over, diff --git a/frontend/src/components/NavigationProvider.dom.test.tsx b/frontend/src/components/NavigationProvider.dom.test.tsx index 5d020e2f..60ee5a80 100644 --- a/frontend/src/components/NavigationProvider.dom.test.tsx +++ b/frontend/src/components/NavigationProvider.dom.test.tsx @@ -151,11 +151,12 @@ describe("NavigationProvider · setPage's same-page guard", () => { expect(result.current.page).toBe("plex"); }); - it("leaves every page's own URL keys behind, the log console's included", () => { + it("leaves every page's own URL keys behind, the activity page's included", () => { window.history.replaceState( - { sfPage: "logs" }, + { sfPage: "activity" }, "", - "/?page=logs&lgrp=error&lreq=req-abcdef12&pscope=movie&ctab=hidden&keep=1", + "/?page=activity&at=logs&aw=24h&au=none&aq=x&rjob=backfill&rst=error&aact=config_set" + + "&lgrp=error&lreq=req-abcdef12&lrun=7&pscope=movie&ctab=hidden&keep=1", ); const { result } = render(); act(() => result.current.setPage("plex")); diff --git a/frontend/src/components/NavigationProvider.tsx b/frontend/src/components/NavigationProvider.tsx index a8d18c0f..d73a2dec 100644 --- a/frontend/src/components/NavigationProvider.tsx +++ b/frontend/src/components/NavigationProvider.tsx @@ -3,7 +3,7 @@ import { useLatest } from "../lib/useLatest"; import { useTranslation } from "react-i18next"; import { useMe } from "../lib/useMe"; import { resolvePage, urlWithParams, type Page } from "../lib/urlState"; -import { LOGS_KEYS } from "../lib/logsUrlState"; +import { ACTIVITY_KEYS } from "../lib/activityUrlState"; import { pageTitleKey } from "../lib/pageMeta"; import { useDocumentObjectTitle } from "../lib/documentTitle"; import { setAccountRaw } from "../lib/storage"; @@ -92,7 +92,7 @@ export function NavigationProvider({ children }: { children: ReactNode }) { ctab: null, csort: null, cf: null, - ...Object.fromEntries(LOGS_KEYS.map((k) => [k, null])), + ...Object.fromEntries(ACTIVITY_KEYS.map((k) => [k, null])), }), ); notifyNav(); diff --git a/frontend/src/components/RunStatusDot.tsx b/frontend/src/components/RunStatusDot.tsx new file mode 100644 index 00000000..cb3ba7f5 --- /dev/null +++ b/frontend/src/components/RunStatusDot.tsx @@ -0,0 +1,29 @@ +// A background run's status as a dot — the Scheduler's rows and the activity page's run list draw +// the same one, so a state reads the same on both. + +// Interrupted is a ring, not a fill: the job did not fail, but its work did not finish either — set +// apart from a skipped run (the same colour, filled) and from a failure. +const DOT_CLASS = { + running: "bg-accent animate-pulse", + ok: "bg-success", + error: "bg-danger", + skipped: "bg-warn", + interrupted: "border-2 border-warn", + idle: "bg-border", +} as const; + +export type DotKey = keyof typeof DOT_CLASS; + +/** A status dot. `label` names it for a screen reader (the tooltip shows only on hover, and only with + * hints on); a legend's dot has its words beside it and stays unnamed. */ +export function StatusDot({ k, label }: { k: DotKey; label?: string }) { + return ( + + ); +} diff --git a/frontend/src/components/Scheduler.dom.test.tsx b/frontend/src/components/Scheduler.dom.test.tsx index 89477800..6f875298 100644 --- a/frontend/src/components/Scheduler.dom.test.tsx +++ b/frontend/src/components/Scheduler.dom.test.tsx @@ -15,6 +15,7 @@ vi.mock("react-i18next", async (importOriginal) => { }); import Scheduler from "./Scheduler"; +import { NavigationActionsContext } from "./navigationContext"; import { api, type RunBrief, type SchedulerAdHoc, type SchedulerStatus } from "../lib/api"; import { qk } from "../lib/queryKeys"; @@ -363,3 +364,37 @@ describe("a row's run history", () => { expect(within(row).queryByRole("button", { name: "scheduler.runNow" })).toBeNull(); }); }); + +describe("a row's history", () => { + // The dots show three runs; the rest is the activity page's run list, narrowed to the row's job. + it("opens the activity page's runs of that job", async () => { + const setPage = vi.fn((_page: string, then?: () => void) => then?.()); + window.history.replaceState(null, "", "/?page=scheduler"); + current = status(); + qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + {}, + closeChannel: () => {}, + enterYtSearch: () => {}, + exitYtSearch: () => {}, + }} + > + + + + , + ); + await screen.findByText("scheduler.maintenance.batchLabel"); + const job = current.jobs[0]!.id; + const row = screen.getByText(`scheduler.jobs.${job}`).closest(".p-3")!; + + fireEvent.click(within(row).getByRole("button", { name: "scheduler.history" })); + + expect(setPage).toHaveBeenCalledWith("activity", expect.any(Function)); + const url = new URLSearchParams(window.location.search); + expect([url.get("page"), url.get("rjob")]).toEqual(["activity", job]); + }); +}); diff --git a/frontend/src/components/Scheduler.tsx b/frontend/src/components/Scheduler.tsx index 91d8333f..1b1cd714 100644 --- a/frontend/src/components/Scheduler.tsx +++ b/frontend/src/components/Scheduler.tsx @@ -10,6 +10,7 @@ import { Clock, Database, Gauge, + History, Pause, Pencil, Play, @@ -43,6 +44,9 @@ import { ContextMenu } from "./ui/ContextMenu"; import { MoreButton } from "./ui/MoreButton"; import { type ActionMenuItem } from "./ui/ActionMenu"; import { jobFailureText, jobResultText } from "../lib/jobResult"; +import { StatusDot, type DotKey } from "./RunStatusDot"; +import { useNavigationActions } from "./navigationContext"; +import { urlWithParams } from "../lib/urlState"; const POLL_MS = 4000; // While any job is running, poll faster so short scheduled runs aren't missed between ticks. @@ -64,33 +68,6 @@ function fmtCountdown(t: TFunction, s: number, lang: string): string { return `${formatUnit(h, "hour", lang)} ${formatUnit(m % 60, "minute", lang)}`; } -// Interrupted is a ring, not a fill: the job did not fail, but its work did not finish either — set -// apart from a skipped run (the same colour, filled) and from a failure. -const DOT_CLASS = { - running: "bg-accent animate-pulse", - ok: "bg-success", - error: "bg-danger", - skipped: "bg-warn", - interrupted: "border-2 border-warn", - idle: "bg-border", -} as const; - -type DotKey = keyof typeof DOT_CLASS; - -/** A status dot. `label` names it for a screen reader (the tooltip shows only on hover, and only with - * hints on); a legend's dot has its words beside it and stays unnamed. */ -function StatusDot({ k, label }: { k: DotKey; label?: string }) { - return ( - - ); -} - function StatusLegend() { const { t } = useTranslation(); return ( @@ -173,6 +150,31 @@ function LastRun({ row, running, target }: { row: RunFields; running: boolean; t ); } +/** A row's way to its whole history: the activity page's run list, narrowed to this job. */ +function HistoryButton({ job }: { job: string }) { + const { t } = useTranslation(); + const { setPage } = useNavigationActions(); + return ( + + + setPage("activity", () => + window.history.replaceState( + window.history.state, + "", + urlWithParams({ page: "activity", rjob: job }), + ), + ) + } + > + + + + ); +} + /** A user's run on one target (a channel's full history): its runs as a job's are shown, but it has * no interval and no "run now" — it needs its target, so it is started where that is chosen. */ function AdHocRow({ row }: { row: SchedulerAdHoc }) { @@ -201,6 +203,7 @@ function AdHocRow({ row }: { row: SchedulerAdHoc }) { t("scheduler.adHocStarted") )}
+ ); } @@ -382,6 +385,7 @@ function JobRow({ + ); } diff --git a/frontend/src/components/Stats.tsx b/frontend/src/components/Stats.tsx index 3c895cd2..79dd7daf 100644 --- a/frontend/src/components/Stats.tsx +++ b/frontend/src/components/Stats.tsx @@ -1,67 +1,27 @@ -import { useState } from "react"; import { useTranslation } from "react-i18next"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { qk } from "../lib/queryKeys"; -import { History, Pause, Play, RefreshCw } from "lucide-react"; -import { api, type AdminQuotaRow } from "../lib/api"; -import { formatEta, formatNumber, formatUnit, quotaActionLabel } from "../lib/format"; +import { History, RefreshCw } from "lucide-react"; +import { api } from "../lib/api"; +import { formatEta, formatNumber, quotaActionLabel } from "../lib/format"; import { notify } from "../lib/notifications"; import { notifyYouTubeActionError } from "../lib/youtubeErrors"; import { useWizard } from "./wizardContext"; -import { LS, useTabPersistedState } from "../lib/storage"; import Tooltip from "./ui/Tooltip"; import { LoadingState, StateMessage } from "./ui/QueryState"; -import FilterChipGroup from "./ui/FilterChipGroup"; -import Tabs from "./ui/Tabs"; -import { useTabPanel } from "./ui/tabPanel"; -import { PageToolbar } from "./ui/PageShell"; import { Section } from "./ui/form"; import { SectionLabel } from "./ui/Headings"; import { StatBars, StatTile, StatTiles } from "./ui/StatTiles"; -import { useMe } from "../lib/useMe"; import { notifyActionError } from "../lib/actionErrors"; import { Button } from "./ui/Button"; -import { Pressable } from "./ui/Pressable"; -import { SURFACE } from "./ui/surfaces"; -const RANGES = [7, 30, 90] as const; -type StatsTab = "overview" | "system"; - -// Usage & stats page. "Overview" is per-user (sync status + your own API usage + actions); -// "System" is the admin-only instance-wide quota dashboard + background-sync control. The -// per-user content used to live in Settings → Sync; it moved here so Settings stays config-only. +// Usage & stats page: your sync status, your own API usage and the manual sync actions (they used to +// live in Settings → Sync; they moved here so Settings stays config-only). The instance-wide quota +// view is the activity page's Quota tab. export default function Stats() { - const me = useMe(); - const { t } = useTranslation(); - const isAdmin = me.role === "admin"; - const [tabRaw, setTab] = useTabPersistedState(LS.statsTab, "overview"); - // Clamp at render: only admins may land on the System tab (covers a stale stored value). - const tab: StatsTab = tabRaw === "system" && isAdmin ? "system" : "overview"; - const panel = useTabPanel(); - return ( - // A tab panel only where there are tabs: everyone but an admin sees the Overview alone. -
- {/* Only admins have a second (System) tab; everyone else just sees the Overview. The tabs are - the page's band (B4), so they stay in reach on the long System tab. */} - {isAdmin && ( - -
- ({ - id, - label: t(`stats.tabs.${id}`), - }))} - className="flex flex-wrap gap-1 pb-2" - /> -
- - )} - - {tab === "system" && isAdmin ? : } +
+
); } @@ -230,165 +190,3 @@ function Overview() { ); } - -// Admin-only view: instance-wide quota dashboard + the global background-sync pause toggle. -function SystemStats() { - const { t, i18n } = useTranslation(); - const lang = i18n.language; - const qc = useQueryClient(); - const [days, setDays] = useState(30); - const q = useQuery({ queryKey: qk.adminQuota(days), queryFn: () => api.adminQuota(days) }); - const status = useQuery({ queryKey: qk.myStatus(), queryFn: api.myStatus }); - const pauseResume = useMutation({ - mutationFn: (paused: boolean) => (paused ? api.resumeSync() : api.pauseSync()), - onSuccess: () => qc.invalidateQueries({ queryKey: qk.myStatus() }), - onError: (e, paused) => - notifyActionError( - e, - t(paused ? "errors.writeFailed.syncResume" : "errors.writeFailed.syncPause"), - ), - }); - const data = q.data; - const maxDaily = Math.max(1, ...(data?.daily ?? []).map((d) => d.total)); - const grandTotal = (data?.rows ?? []).reduce((s, r) => s + r.total, 0); - const s = status.data; - - return ( - <> - {s && ( -
- - - -
- )} - -
-

{t("stats.title")}

- ({ - value: String(d), - label: t("stats.rangeDays", { d: formatUnit(d, "day", lang) }), - }))} - value={String(days)} - onChange={(v) => setDays(Number(v))} - /> -
- -

- {t("stats.introBefore")} - {t("stats.introSystem")} - {t("stats.introAfter")} -

- - {q.isError && !data ? ( - q.refetch()}> - {t("common.loadError")} - - ) : q.isLoading ? ( - - ) : !data ? ( -
{t("stats.noData")}
- ) : ( - <> - {/* Daily totals (instance-wide, Pacific days) */} -
-
- {t("stats.dailyTotal", { - d: formatUnit(data.range_days, "day", lang), - units: formatNumber(grandTotal, lang), - })} -
- {data.daily.length === 0 ? ( -
{t("stats.noUsageInRange")}
- ) : ( -
- {data.daily.map((d) => ( -
-
-
- ))} -
- )} -
- - {/* Per-user breakdown */} -
- {data.rows.length === 0 ? ( -
{t("stats.noUsageInRange")}
- ) : ( - data.rows.map((r) => ( - - )) - )} -
- - )} - - ); -} - -function UserRow({ row, max }: { row: AdminQuotaRow; max: number }) { - const { t, i18n } = useTranslation(); - const lang = i18n.language; - const [open, setOpen] = useState(false); - const actions = Object.entries(row.by_action).sort((a, b) => b[1] - a[1]); - const isSystem = row.user_id === null; - return ( -
- setOpen((o) => !o)} - aria-expanded={open} - className="w-full flex items-center gap-3" - > - - {isSystem ? t("stats.system") : row.email} - - - {formatNumber(row.total, lang)} - - - {/* Proportion bar */} -
-
-
- {open && actions.length > 0 && ( -
- {actions.map(([action, units]) => ( -
- {quotaActionLabel(t, action)} - {formatNumber(units, lang)} -
- ))} -
- )} -
- ); -} diff --git a/frontend/src/components/listNav.wiring.contract.test.ts b/frontend/src/components/listNav.wiring.contract.test.ts index 9d20ef98..a80d94d7 100644 --- a/frontend/src/components/listNav.wiring.contract.test.ts +++ b/frontend/src/components/listNav.wiring.contract.test.ts @@ -24,10 +24,7 @@ const CARDS = [ const LISTS = ["NavGrid", "Carousel", "VirtualGrid"]; /** `path#tag#n` → why it is not a keyboard list. An entry that no longer matches a tag fails the test. */ -const EXEMPT: Record = { - "components/AuditLog.tsx#DataTable#1": - "an admin log read in place: its rows open nothing, so stepping onto them would only move a ring", -}; +const EXEMPT: Record = {}; interface Tag { start: number; diff --git a/frontend/src/components/moduleRegistry.tsx b/frontend/src/components/moduleRegistry.tsx index 27945fcf..63d46ef5 100644 --- a/frontend/src/components/moduleRegistry.tsx +++ b/frontend/src/components/moduleRegistry.tsx @@ -32,8 +32,7 @@ const Stats = lazyWithPreload(() => import("./Stats")); const Scheduler = lazyWithPreload(() => import("./Scheduler")); const ConfigPanel = lazyWithPreload(() => import("./ConfigPanel")); const AdminUsers = lazyWithPreload(() => import("./AdminUsers")); -const AuditLog = lazyWithPreload(() => import("./AuditLog")); -const AdminLogs = lazyWithPreload(() => import("./AdminLogs")); +const Activity = lazyWithPreload(() => import("./Activity")); const SettingsPanel = lazyWithPreload(() => import("./SettingsPanel")); export type PageDef = { @@ -60,8 +59,7 @@ export const PAGE_CONTENT: Record = { scheduler: { Component: Scheduler }, config: { Component: ConfigPanel }, users: { Component: AdminUsers, maxWidth: "48rem" }, - audit: { Component: AuditLog }, - logs: { Component: AdminLogs }, + activity: { Component: Activity }, settings: { Component: SettingsPanel }, }; diff --git a/frontend/src/i18n/locales/en/activity.json b/frontend/src/i18n/locales/en/activity.json new file mode 100644 index 00000000..8aeb5efb --- /dev/null +++ b/frontend/src/i18n/locales/en/activity.json @@ -0,0 +1,63 @@ +{ + "title": "Activity", + "tabs": { + "runs": "Runs", + "admin": "Admin actions", + "logs": "Server log", + "quota": "Quota" + }, + "intro": { + "runs": "Every background run, newest first — scheduled, started by an admin, or a user's work on one channel. A row opens to what it did or why it failed, and to the lines it logged.", + "admin": "An append-only record of admin actions — who changed what, and when. A row opens to its before and after, and to the lines its request logged.", + "logs": "The server's stored log — lines from the API, the downloader and browsers, kept until the retention period ends. A line opens in place; its request, job or run id keeps only that one's lines." + }, + "search": "Search…", + "windowLabel": "Period", + "windows": { + "1h": "1 hour", + "24h": "24 hours", + "7d": "7 days", + "30d": "30 days", + "90d": "90 days" + }, + "account": "Account", + "any": "Any", + "noAccount": "No account (background)", + "older": "Load older", + "stale": "Couldn't refresh — what you see may be out of date", + "olderFailed": "Couldn't load the older rows", + "countsFailed": "Couldn't load the filter counts", + "runs": { + "job": "Job", + "statusLabel": "Filter by status", + "status": { + "error": "Failed", + "interrupted": "Interrupted", + "skipped": "Skipped", + "running": "Running", + "ok": "OK" + }, + "empty": "No runs match the filters.", + "count_one": "{{count}} run shown · newest first", + "count_other": "{{count}} runs shown · newest first", + "fields": { + "started": "Started by", + "when": "Start → end", + "target": "Target", + "quota": "Quota units" + }, + "logLines_one": "Its {{count}} log line →", + "logLines_other": "Its {{count}} log lines →", + "jobRuns": "This job's runs →" + }, + "audit": { + "action": "Action", + "target": "Target", + "requestLines": "Its request's log lines →", + "count_one": "{{count}} action shown · newest first", + "count_other": "{{count}} actions shown · newest first" + }, + "quota": { + "total": "Spend over {{window}}: {{units}} units" + } +} diff --git a/frontend/src/i18n/locales/en/adminLogs.json b/frontend/src/i18n/locales/en/adminLogs.json index 519ee9d8..96b3a827 100644 --- a/frontend/src/i18n/locales/en/adminLogs.json +++ b/frontend/src/i18n/locales/en/adminLogs.json @@ -1,17 +1,7 @@ { - "title": "Server log", - "intro": "The server's stored log — lines from the API, the downloader and browsers, kept until the retention period ends. A line opens in place; click its request or job id to keep only that request's or job's lines.", - "search": "Search the message, code, stack…", "levelLabel": "Filter by level", "source": "Source", - "account": "Account", "event": "Event", - "window": "Period", - "windows": { - "1h": "1 hour", - "24h": "24 hours", - "7d": "7 days" - }, "any": "Any", "noAccount": "No account", "noEvent": "No code", @@ -38,6 +28,9 @@ "account": "Account", "request": "Request", "job": "Job", - "version": "Version" - } + "version": "Version", + "run": "Run" + }, + "pivotRun": "run {{id}}", + "showRun": "Only this run's lines" } diff --git a/frontend/src/i18n/locales/en/audit.json b/frontend/src/i18n/locales/en/audit.json index 6e6eb7c6..62ce2a59 100644 --- a/frontend/src/i18n/locales/en/audit.json +++ b/frontend/src/i18n/locales/en/audit.json @@ -1,6 +1,4 @@ { - "title": "Audit log", - "intro": "An append-only record of admin actions — who did what, and when. Logged at action granularity, never per item.", "empty": "No audit entries yet.", "system": "System", "clear": "Clear audit log", @@ -9,7 +7,6 @@ "cleared_one": "Cleared {{count}} audit entry", "cleared_other": "Cleared {{count}} audit entries", "clearFailed": "Couldn't clear the audit log", - "truncated": "Showing the {{returned}} most recent of {{total}} entries.", "on": "On", "off": "Off", "default": "default", @@ -18,7 +15,6 @@ "cols": { "when": "When", "actor": "Who", - "action": "Action", "details": "Details" } } diff --git a/frontend/src/i18n/locales/en/header.json b/frontend/src/i18n/locales/en/header.json index ac80bdd2..442f9099 100644 --- a/frontend/src/i18n/locales/en/header.json +++ b/frontend/src/i18n/locales/en/header.json @@ -32,9 +32,8 @@ "signOut": "Sign out", "addAccount": "Add another account", "switchTo": "Switch to {{name}}", - "audit": "Audit log", - "logs": "Server log", - "googleAddress": "Google account: {{email}}" + "googleAddress": "Google account: {{email}}", + "activity": "Activity" }, "sync": { "yours": "yours", diff --git a/frontend/src/i18n/locales/en/nav.json b/frontend/src/i18n/locales/en/nav.json index f8cc9142..b2cf7533 100644 --- a/frontend/src/i18n/locales/en/nav.json +++ b/frontend/src/i18n/locales/en/nav.json @@ -18,9 +18,8 @@ "scheduler": "Scheduler", "config": "Config", "users": "Users", - "audit": "Audit", "admin": "Admin", - "logs": "Logs", - "settings": "Settings" + "settings": "Settings", + "activity": "Activity" } } diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index 808df3f3..1ed1eda7 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -211,5 +211,6 @@ }, "interrupted": "interrupted: the server restarted during the run", "adHocTitle": "Ad-hoc runs", - "adHocStarted": "started by a user" + "adHocStarted": "started by a user", + "history": "This job's runs" } diff --git a/frontend/src/i18n/locales/en/stats.json b/frontend/src/i18n/locales/en/stats.json index ae3935e9..8ab0cb00 100644 --- a/frontend/src/i18n/locales/en/stats.json +++ b/frontend/src/i18n/locales/en/stats.json @@ -1,17 +1,8 @@ { - "tabs": { - "overview": "Overview", - "system": "System" - }, - "title": "API quota usage", - "rangeDays": "{{d}}", - "rangeLabel": "Time range", "introBefore": "YouTube Data API units attributed by who triggered the spend. ", "introSystem": "System", "introAfter": " is shared background work (scheduled backfill, enrichment, resync) that isn't tied to one person.", "loading": "Loading…", - "noData": "No data.", - "dailyTotal": "Daily total ({{d}} · {{units}} units)", "noUsageInRange": "No usage in this range.", "dailyTooltip": "{{day}}: {{units}} units", "system": "System (background)", @@ -49,10 +40,6 @@ "syncFailed": "Couldn't sync subscriptions", "fullHistoryRequested_one": "Full history requested for {{count}} channel", "fullHistoryRequested_other": "Full history requested for {{count}} channels", - "fullHistoryFailed": "Couldn't request full history", - "admin": "Admin", - "adminPauseHint": "Pause or resume the background sync for the whole app (all users).", - "resumeBackgroundSync": "Resume background sync", - "pauseBackgroundSync": "Pause background sync" + "fullHistoryFailed": "Couldn't request full history" } } diff --git a/frontend/src/i18n/locales/hu/activity.json b/frontend/src/i18n/locales/hu/activity.json new file mode 100644 index 00000000..8ca6cad2 --- /dev/null +++ b/frontend/src/i18n/locales/hu/activity.json @@ -0,0 +1,63 @@ +{ + "title": "Napló", + "tabs": { + "runs": "Futások", + "admin": "Admin műveletek", + "logs": "Szervernapló", + "quota": "Kvóta" + }, + "intro": { + "runs": "Minden háttérfutás, a legújabbal kezdve — ütemezett, adminisztrátor indította, vagy egy felhasználó munkája egy csatornán. Egy sor kinyitva megmutatja, mit végzett vagy miért hiúsult meg, és a naplósorait.", + "admin": "Az adminisztrátori műveletek csak bővülő nyilvántartása — ki mit változtatott és mikor. Egy sor kinyitva megmutatja az előtte–utána állapotot és a kérése naplósorait.", + "logs": "A szerver tárolt naplója — az API, a letöltő és a böngészők sorai, amelyeket a megőrzési idő végéig tart meg. Egy sor a helyén nyílik ki; a kérés-, feladat- vagy futásazonosítójára kattintva csak az ahhoz tartozó sorok maradnak." + }, + "search": "Keresés…", + "windowLabel": "Időtartomány", + "windows": { + "1h": "1 óra", + "24h": "24 óra", + "7d": "7 nap", + "30d": "30 nap", + "90d": "90 nap" + }, + "account": "Fiók", + "any": "Bármelyik", + "noAccount": "Fiók nélkül (háttér)", + "older": "Régebbiek", + "stale": "Nem sikerült frissíteni — amit látsz, elavult lehet", + "olderFailed": "Nem sikerült betölteni a régebbi sorokat", + "countsFailed": "Nem sikerült betölteni a szűrők darabszámait", + "runs": { + "job": "Feladat", + "statusLabel": "Szűrés állapotra", + "status": { + "error": "Hibás", + "interrupted": "Megszakadt", + "skipped": "Kihagyva", + "running": "Fut", + "ok": "Rendben" + }, + "empty": "Nincs a szűrőknek megfelelő futás.", + "count_one": "{{count}} futás látszik · a legújabb elöl", + "count_other": "{{count}} futás látszik · a legújabb elöl", + "fields": { + "started": "Indította", + "when": "Kezdés → vége", + "target": "Cél", + "quota": "Kvótaegység" + }, + "logLines_one": "A futás {{count}} naplósora →", + "logLines_other": "A futás {{count}} naplósora →", + "jobRuns": "Ennek a feladatnak a futásai →" + }, + "audit": { + "action": "Művelet", + "target": "Cél", + "requestLines": "A kérés naplósorai →", + "count_one": "{{count}} művelet látszik · a legújabb elöl", + "count_other": "{{count}} művelet látszik · a legújabb elöl" + }, + "quota": { + "total": "Felhasználás ({{window}}): {{units}} egység" + } +} diff --git a/frontend/src/i18n/locales/hu/adminLogs.json b/frontend/src/i18n/locales/hu/adminLogs.json index 6a319a8a..e1ddefc5 100644 --- a/frontend/src/i18n/locales/hu/adminLogs.json +++ b/frontend/src/i18n/locales/hu/adminLogs.json @@ -1,17 +1,7 @@ { - "title": "Szervernapló", - "intro": "A szerver tárolt naplója — az API, a letöltő és a böngészők sorai, amelyeket a megőrzési idő végéig tart meg. Egy sor a helyén nyílik ki; a kérés- vagy feladat-azonosítójára kattintva csak az ahhoz tartozó sorok maradnak.", - "search": "Keresés az üzenetben, a kódban, a hívási veremben…", "levelLabel": "Szűrés szintre", "source": "Forrás", - "account": "Fiók", "event": "Esemény", - "window": "Időszak", - "windows": { - "1h": "1 óra", - "24h": "24 óra", - "7d": "7 nap" - }, "any": "Bármi", "noAccount": "Fiók nélkül", "noEvent": "Kód nélkül", @@ -38,6 +28,9 @@ "account": "Fiók", "request": "Kérés", "job": "Feladat", - "version": "Verzió" - } + "version": "Verzió", + "run": "Futás" + }, + "pivotRun": "futás: {{id}}", + "showRun": "Csak ennek a futásnak a sorai" } diff --git a/frontend/src/i18n/locales/hu/audit.json b/frontend/src/i18n/locales/hu/audit.json index 0ec91b4d..86f7ae76 100644 --- a/frontend/src/i18n/locales/hu/audit.json +++ b/frontend/src/i18n/locales/hu/audit.json @@ -1,6 +1,4 @@ { - "title": "Eseménynapló", - "intro": "Az admin-műveletek csak-hozzáfűző naplója — ki, mit, mikor. Művelet-szinten naplózva, soha nem elemenként.", "empty": "Még nincs naplóbejegyzés.", "system": "Rendszer", "clear": "Eseménynapló ürítése", @@ -9,7 +7,6 @@ "cleared_one": "{{count}} naplóbejegyzés törölve", "cleared_other": "{{count}} naplóbejegyzés törölve", "clearFailed": "Nem sikerült üríteni az eseménynaplót", - "truncated": "A legutóbbi {{returned}} bejegyzés látszik a(z) {{total}}-ból.", "on": "Be", "off": "Ki", "default": "alapérték", @@ -18,7 +15,6 @@ "cols": { "when": "Mikor", "actor": "Ki", - "action": "Művelet", "details": "Részletek" } } diff --git a/frontend/src/i18n/locales/hu/header.json b/frontend/src/i18n/locales/hu/header.json index f19bca06..972073b8 100644 --- a/frontend/src/i18n/locales/hu/header.json +++ b/frontend/src/i18n/locales/hu/header.json @@ -32,9 +32,8 @@ "signOut": "Kijelentkezés", "addAccount": "Másik fiók hozzáadása", "switchTo": "Váltás erre: {{name}}", - "audit": "Eseménynapló", - "logs": "Szervernapló", - "googleAddress": "Google-fiók: {{email}}" + "googleAddress": "Google-fiók: {{email}}", + "activity": "Napló" }, "sync": { "yours": "tiéd", diff --git a/frontend/src/i18n/locales/hu/nav.json b/frontend/src/i18n/locales/hu/nav.json index 1300d07a..c8088179 100644 --- a/frontend/src/i18n/locales/hu/nav.json +++ b/frontend/src/i18n/locales/hu/nav.json @@ -18,9 +18,8 @@ "scheduler": "Ütemező", "config": "Konfig", "users": "Felhasználók", - "audit": "Eseménynapló", "admin": "Admin", - "logs": "Napló", - "settings": "Beállítások" + "settings": "Beállítások", + "activity": "Napló" } } diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 154ade1b..7b9b511f 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -211,5 +211,6 @@ }, "interrupted": "megszakadt: a futás közben újraindult a szerver", "adHocTitle": "Alkalmi futások", - "adHocStarted": "felhasználó indítja" + "adHocStarted": "felhasználó indítja", + "history": "A feladat futásai" } diff --git a/frontend/src/i18n/locales/hu/stats.json b/frontend/src/i18n/locales/hu/stats.json index eee941bf..94e73362 100644 --- a/frontend/src/i18n/locales/hu/stats.json +++ b/frontend/src/i18n/locales/hu/stats.json @@ -1,17 +1,8 @@ { - "tabs": { - "overview": "Áttekintés", - "system": "Rendszer" - }, - "title": "API-kvóta használat", - "rangeDays": "{{d}}", - "rangeLabel": "Időtartomány", "introBefore": "A YouTube Data API-egységek aszerint, hogy ki váltotta ki a felhasználást. A ", "introSystem": "Rendszer", "introAfter": " a megosztott háttérmunka (ütemezett behúzás, dúsítás, újraszinkronizálás), amely nem köthető egy személyhez.", "loading": "Betöltés…", - "noData": "Nincs adat.", - "dailyTotal": "Napi összesen ({{d}} · {{units}} egység)", "noUsageInRange": "Nincs használat ebben az időszakban.", "dailyTooltip": "{{day}}: {{units}} egység", "system": "Rendszer (háttér)", @@ -49,10 +40,6 @@ "syncFailed": "Nem sikerült szinkronizálni a feliratkozásokat", "fullHistoryRequested_one": "Teljes előzmény kérve {{count}} csatornához", "fullHistoryRequested_other": "Teljes előzmény kérve {{count}} csatornához", - "fullHistoryFailed": "Nem sikerült teljes előzményt kérni", - "admin": "Adminisztrátor", - "adminPauseHint": "A háttér-szinkronizálás szüneteltetése vagy folytatása az egész alkalmazásra (minden felhasználóra).", - "resumeBackgroundSync": "Háttér-szinkronizálás folytatása", - "pauseBackgroundSync": "Háttér-szinkronizálás szüneteltetése" + "fullHistoryFailed": "Nem sikerült teljes előzményt kérni" } } diff --git a/frontend/src/lib/activityUrlState.test.ts b/frontend/src/lib/activityUrlState.test.ts new file mode 100644 index 00000000..9291d6e6 --- /dev/null +++ b/frontend/src/lib/activityUrlState.test.ts @@ -0,0 +1,64 @@ +// The activity page's view in the address bar: a link carries the tab and every filter, a hand-edited +// value the API would refuse degrades to "no filter", a filter at its default leaves no key behind, +// and a link to a page this one replaced opens that page's tab. +import { describe, expect, it } from "vitest"; + +import { + ACTIVITY_KEYS, + activityParamsToState, + activityParamUpdates, + DEFAULT_ACTIVITY_STATE, + sinceOf, + type ActivityUrlState, +} from "./activityUrlState"; + +const params = (updates: Record) => + new URLSearchParams(Object.entries(updates).filter((e): e is [string, string] => e[1] !== null)); + +describe("activityUrlState", () => { + it("round-trips the tab and every filter", () => { + const state: ActivityUrlState = { + tab: "admin", + window: "30d", + user: "none", + q: "100% done", + job: "channel_backfill", + status: "interrupted", + action: "config_set", + }; + expect(activityParamsToState(params(activityParamUpdates(state)))).toEqual(state); + }); + + it("leaves no key for a filter at its default, and opens on the runs of the last 7 days", () => { + expect( + Object.values(activityParamUpdates(DEFAULT_ACTIVITY_STATE)).every((v) => v === null), + ).toBe(true); + expect([DEFAULT_ACTIVITY_STATE.tab, DEFAULT_ACTIVITY_STATE.window]).toEqual(["runs", "7d"]); + }); + + it("drops a value the API would refuse instead of sending it", () => { + const state = activityParamsToState( + new URLSearchParams( + "at=everything&aw=1y&au=me&rjob=Not-A-Job&rst=finished&aact=DROP%20TABLE", + ), + ); + expect(state).toEqual(DEFAULT_ACTIVITY_STATE); + }); + + it("opens the tab of the page a link still names", () => { + expect(activityParamsToState(new URLSearchParams("page=audit")).tab).toBe("admin"); + expect(activityParamsToState(new URLSearchParams("page=logs")).tab).toBe("logs"); + // A tab the link names wins over the retired page's. + expect(activityParamsToState(new URLSearchParams("page=logs&at=quota")).tab).toBe("quota"); + }); + + it("clears its own keys and the server log's on a page change", () => { + expect(ACTIVITY_KEYS).toEqual(expect.arrayContaining(["at", "aw", "au", "aq", "rjob", "lreq"])); + }); + + it("starts the window that many hours back", () => { + const now = Date.UTC(2026, 9, 18, 12); + expect(sinceOf("24h", now)).toBe("2026-10-17T12:00:00.000Z"); + expect(sinceOf("90d", now)).toBe("2026-07-20T12:00:00.000Z"); + }); +}); diff --git a/frontend/src/lib/activityUrlState.ts b/frontend/src/lib/activityUrlState.ts new file mode 100644 index 00000000..f8c25ce8 --- /dev/null +++ b/frontend/src/lib/activityUrlState.ts @@ -0,0 +1,104 @@ +// The activity page's view in the address bar (`?page=activity&at=runs&aw=24h&rjob=backfill`), so a +// filtered view is a link: which tab, the filters every tab shares (time range, account, text), and +// each tab's own (the run list's job and status, the audit list's action). The server log's own +// filters keep their `l…` keys (./logsUrlState). setPage clears all of them (ACTIVITY_KEYS); the page +// re-mirrors them while it is open. +import { LOGS_KEYS } from "./logsUrlState"; + +export type ActivityTab = "runs" | "admin" | "logs" | "quota"; +export type ActivityWindow = "1h" | "24h" | "7d" | "30d" | "90d"; +/** The run statuses the list can narrow to (the API's `RunStatus`). */ +export type RunStatusFilter = "running" | "ok" | "error" | "skipped" | "interrupted"; + +export interface ActivityUrlState { + tab: ActivityTab; + window: ActivityWindow; + /** An account id, or `none` for what no account did (the background). */ + user: string | null; + q: string; + /** Runs: one job's runs. */ + job: string | null; + status: RunStatusFilter | null; + /** Admin actions: one action. */ + action: string | null; +} + +export const ACTIVITY_TABS: readonly ActivityTab[] = ["runs", "admin", "logs", "quota"]; +export const ACTIVITY_WINDOWS: readonly ActivityWindow[] = ["1h", "24h", "7d", "30d", "90d"]; +export const RUN_STATUSES: readonly RunStatusFilter[] = [ + "error", + "interrupted", + "skipped", + "running", + "ok", +]; + +export const WINDOW_HOURS: Record = { + "1h": 1, + "24h": 24, + "7d": 24 * 7, + "30d": 24 * 30, + "90d": 24 * 90, +}; + +export const DEFAULT_ACTIVITY_STATE: ActivityUrlState = { + tab: "runs", + window: "7d", + user: null, + q: "", + job: null, + status: null, + action: null, +}; + +const OWN_KEYS = ["at", "aw", "au", "aq", "rjob", "rst", "aact"] as const; +export const ACTIVITY_KEYS = [...OWN_KEYS, ...LOGS_KEYS] as const; + +// The pages this one replaced: a link or a bookmark to one opens its tab here (urlState's RETIRED +// sends the page id itself here). +const RETIRED_TAB: Readonly> = { audit: "admin", logs: "logs" }; + +// The same shapes the API validates: a hand-edited link degrades to "no filter" here instead of to +// a 422 on every request the page makes. +const USER_RE = /^(none|\d{1,10})$/; +const ID_RE = /^[a-z][a-z0-9_]*$/; +const Q_MAX = 200; + +const pick = (value: string | null, re: RegExp, max: number) => + value && value.length <= max && re.test(value) ? value : null; +const oneOf = (value: string | null, list: readonly T[]): T | null => + list.includes(value as T) ? (value as T) : null; + +export function activityParamsToState(params: URLSearchParams): ActivityUrlState { + return { + tab: + oneOf(params.get("at"), ACTIVITY_TABS) ?? + RETIRED_TAB[params.get("page") ?? ""] ?? + DEFAULT_ACTIVITY_STATE.tab, + window: oneOf(params.get("aw"), ACTIVITY_WINDOWS) ?? DEFAULT_ACTIVITY_STATE.window, + user: pick(params.get("au"), USER_RE, 10), + q: (params.get("aq") ?? "").slice(0, Q_MAX), + job: pick(params.get("rjob"), ID_RE, 40), + status: oneOf(params.get("rst"), RUN_STATUSES), + action: pick(params.get("aact"), ID_RE, 48), + }; +} + +/** The URL updates for a state: a filter at its default clears its key. */ +export function activityParamUpdates(state: ActivityUrlState): Record { + const d = DEFAULT_ACTIVITY_STATE; + return { + at: state.tab === d.tab ? null : state.tab, + aw: state.window === d.window ? null : state.window, + au: state.user, + aq: state.q.trim() ? state.q : null, + rjob: state.job, + rst: state.status, + aact: state.action, + }; +} + +/** Where the window starts, as the API takes it. */ +export function sinceOf(window: ActivityWindow, now = Date.now()): string { + return new Date(now - WINDOW_HOURS[window] * 3_600_000).toISOString(); +} diff --git a/frontend/src/lib/api/admin.ts b/frontend/src/lib/api/admin.ts index a844bd12..158de107 100644 --- a/frontend/src/lib/api/admin.ts +++ b/frontend/src/lib/api/admin.ts @@ -95,18 +95,69 @@ export interface AdminUserAccess extends AdminUserRow { export interface AuditRow { id: number; created_at: string | null; - actor: string | null; // email of the acting admin; null = System (scheduler/background) + /** The acting admin (address in the page's `accounts`); null = the background. */ + actor_id: number | null; action: string; // canonical key, localized on the client via auditActions. target_type: string | null; target_id: string | null; summary: string | null; before: Record | null; after: Record | null; + /** The request the change was made in — its log lines are one pivot away. Null on older rows. */ + request_id: string | null; } -export interface AuditListResult { - items: AuditRow[]; - total: number; // total rows in the log (may exceed `items` if capped) - returned: number; + +/** A page of a history list (audit log, runs), newest first; `next_cursor` continues it. */ +export interface HistoryPage { + items: T[]; + /** The accounts this page names, by id; an erased one is absent. */ + accounts: Record; + next_cursor: string | null; +} + +/** The filters every activity tab shares, as the APIs take them. */ +export interface SharedQuery { + /** An account id, or `none` for what no account did. */ + user?: string; + q?: string; + since?: string; +} + +export interface AuditQuery extends SharedQuery { + action?: string; +} + +/** One background run (`job_runs`), as the activity page lists it. */ +export interface RunRow { + id: number; + job: string; + trigger: RunBrief["trigger"]; + actor_id: number | null; + /** What a user's ad-hoc run worked on (a channel id), and that channel's title. */ + target: string | null; + target_title: string | null; + status: RunBrief["status"]; + started_at: string; + finished_at: string | null; + summary: string | null; + result: JobResult | null; + error: string | null; + failure: { code: string; [key: string]: unknown } | null; + quota_units: number; + /** How many stored log lines the run left (WARNING and up only). */ + log_lines: number; +} + +export interface RunQuery extends SharedQuery { + job?: string; + status?: string; +} + +/** The job menu and the status chips, each counted under every filter but its own. */ +export interface RunFacets { + jobs: LogFacet[]; + statuses: LogFacet[]; + total: number; } /** One stored log line (`log_events`), as the diagnostics console reads it. Values the server's @@ -124,6 +175,7 @@ export interface ServerLogRow { request_id: string | null; user_id: number | null; job_id: number | null; + run_id: number | null; debug_session_id: string | null; version: string | null; } @@ -136,6 +188,7 @@ export interface LogQuery { event?: string; request_id?: string; job_id?: number; + run_id?: number; q?: string; since?: string; } @@ -171,6 +224,7 @@ function logParams(query: LogQuery, extra: Record => req("/api/admin/config/test-issues", { method: "POST" }), // --- users & roles --- - adminAudit: (limit = 500): Promise => req(`/api/admin/audit?limit=${limit}`), + adminAudit: (query: AuditQuery, before: string | null): Promise> => + req(`/api/admin/audit${queryString({ ...query, before })}`), + adminAuditFacets: (query: AuditQuery): Promise<{ actions: LogFacet[] }> => + req(`/api/admin/audit/facets${queryString(query)}`), + adminRuns: (query: RunQuery, before: string | null): Promise> => + req(`/api/admin/runs${queryString({ ...query, before })}`), + adminRunFacets: (query: RunQuery): Promise => + req(`/api/admin/runs/facets${queryString(query)}`), clearAudit: (): Promise<{ cleared: number }> => req("/api/admin/audit", { method: "DELETE" }), // --- diagnostics console --- adminLogs: (query: LogQuery, before: string | null): Promise => diff --git a/frontend/src/lib/api/quota.ts b/frontend/src/lib/api/quota.ts index 964ef7f7..a0f2e132 100644 --- a/frontend/src/lib/api/quota.ts +++ b/frontend/src/lib/api/quota.ts @@ -18,12 +18,13 @@ export interface AdminQuotaRow { } export interface AdminQuota { - range_days: number; rows: AdminQuotaRow[]; daily: { day: string; total: number }[]; } export const quotaApi = { myUsage: (): Promise => req("/api/quota/my-usage"), - adminQuota: (days = 30): Promise => req(`/api/quota/admin?days=${days}`), + /** Spend from `since` to now, optionally for one account (`none` = the background). */ + adminQuota: (since: string, user: string | null): Promise => + req(`/api/quota/admin?${new URLSearchParams({ since, ...(user ? { user } : {}) }).toString()}`), }; diff --git a/frontend/src/components/auditColumns.dom.test.tsx b/frontend/src/lib/auditText.dom.test.ts similarity index 85% rename from frontend/src/components/auditColumns.dom.test.tsx rename to frontend/src/lib/auditText.dom.test.ts index 34a8a0e8..a0906ad5 100644 --- a/frontend/src/components/auditColumns.dom.test.tsx +++ b/frontend/src/lib/auditText.dom.test.ts @@ -4,37 +4,33 @@ // min/run", "Background sync paused") reads in the reader's language, rebuilt from the row's // data through the REAL catalog. A row whose summary is data (an e-mail, a URL) keeps it. import { afterEach, describe, expect, it } from "vitest"; -import { cleanup, render } from "@testing-library/react"; import i18n, { setLanguage } from "../i18n"; -import type { AuditRow } from "../lib/api"; -import { auditColumns } from "./auditColumns"; +import type { AuditRow } from "./api"; +import { auditText } from "./auditText"; -// Unmount BEFORE the language goes back: a mounted reader re-renders on the switch, and under a loaded -// run that render landed after the test environment was gone ("window is not defined"). afterEach(() => { - cleanup(); setLanguage("en"); }); const row = (over: Partial): AuditRow => ({ id: 1, created_at: null, - actor: null, + actor_id: null, action: "sync_pause", target_type: null, target_id: null, summary: null, before: null, after: null, + request_id: null, ...over, }); function details(r: AuditRow): string { setLanguage("hu"); const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const { container } = render(<>{col.render!(r)}); - return [...container.querySelectorAll(".wrap-break-word")].map((e) => e.textContent).join(" | "); + const { head, sub } = auditText(t, r, "hu"); + return [head, sub].filter(Boolean).join(" | "); } describe("an audit row's details", () => { diff --git a/frontend/src/components/auditColumns.tsx b/frontend/src/lib/auditText.ts similarity index 58% rename from frontend/src/components/auditColumns.tsx rename to frontend/src/lib/auditText.ts index a11631a7..dd31a20c 100644 --- a/frontend/src/components/auditColumns.tsx +++ b/frontend/src/lib/auditText.ts @@ -1,15 +1,7 @@ import type { TFunction } from "i18next"; -import type { AuditRow } from "../lib/api"; -import { formatNumber } from "../lib/format"; -import type { JobResult } from "../lib/api"; -import { jobResultText } from "../lib/jobResult"; -import type { Column } from "./ui/DataTable"; -import { RelativeTime } from "./ui/RelativeTime"; - -function actionLabel(t: TFunction, action: string): string { - // Canonical key → localized label; falls back to the raw key for any unmapped action. - return t(`auditActions.${action}`, action); -} +import type { AuditRow, JobResult } from "./api"; +import { formatNumber } from "./format"; +import { jobResultText } from "./jobResult"; function fmt(v: unknown): string { return v == null || v === "" ? "—" : String(v); @@ -90,64 +82,18 @@ function changeText(row: AuditRow): string | null { return null; } -/** Columns for the admin audit-log DataTable. `actionOptions` are the distinct actions present in - * the data (built by the page) so the Action column's select filter only offers real values. */ -export function auditColumns( +/** An audit row in the reader's language: `head` names what changed (a config field, a job; else + * the server's summary), `sub` the change itself (before → after, a count). Either may be null. */ +export function auditText( t: TFunction, + row: AuditRow, lang: string, - actionOptions: { value: string; label: string }[], -): Column[] { - return [ - { - key: "when", - header: t("audit.cols.when"), - nowrap: true, - sortable: true, - sortValue: (r) => r.created_at ?? "", - cardPrimary: true, - render: (r) => ( - - ), - }, - { - key: "actor", - header: t("audit.cols.actor"), - sortable: true, - sortValue: (r) => r.actor ?? "", - filter: { kind: "text", get: (r) => r.actor ?? t("audit.system") }, - render: (r) => - r.actor ? ( - {r.actor} - ) : ( - {t("audit.system")} - ), - }, - { - key: "action", - header: t("audit.cols.action"), - nowrap: true, - sortable: true, - sortValue: (r) => actionLabel(t, r.action), - filter: { kind: "select", options: actionOptions, test: (r, v) => r.action === v }, - render: (r) => {actionLabel(t, r.action)}, - }, - { - key: "details", - header: t("audit.cols.details"), - render: (r) => { - const local = localDetails(t, r, lang); - const summary = local ? local.head : r.summary; - const change = local ? local.sub : changeText(r); - return ( - // wrap-break-word (wrap), NOT truncate: in the auto-layout table a `truncate` (nowrap) cell - // sizes to its longest row, so one long summary/change stretched the whole table past the - // card (the row rules bled into the page). Wrapping keeps every row inside the container. -
- {summary &&
{summary}
} - {change &&
{change}
} -
- ); - }, - }, - ]; +): { head: string | null; sub: string | null } { + const local = localDetails(t, row, lang); + return local ?? { head: row.summary, sub: changeText(row) }; +} + +export function auditActionLabel(t: TFunction, action: string): string { + // Canonical key → localized label; falls back to the raw key for any unmapped action. + return t(`auditActions.${action}`, action); } diff --git a/frontend/src/lib/issueDraft.test.ts b/frontend/src/lib/issueDraft.test.ts index 495a8c3d..4843c7d7 100644 --- a/frontend/src/lib/issueDraft.test.ts +++ b/frontend/src/lib/issueDraft.test.ts @@ -19,6 +19,7 @@ const row = (over: Partial): ServerLogRow => ({ request_id: null, user_id: 1, job_id: null, + run_id: null, debug_session_id: "0123456789abcdef", version: null, ...over, diff --git a/frontend/src/lib/logsUrlState.test.ts b/frontend/src/lib/logsUrlState.test.ts index 67c80712..31f666ea 100644 --- a/frontend/src/lib/logsUrlState.test.ts +++ b/frontend/src/lib/logsUrlState.test.ts @@ -17,12 +17,10 @@ describe("logsUrlState", () => { const state: LogsUrlState = { group: "error", sources: ["worker"], - user: "none", event: "download.job.failed", requestId: "req-abcdef12", jobId: 42, - q: "100% done", - window: "7d", + runId: 7, }; expect(logsParamsToState(params(logsParamUpdates(state)))).toEqual(state); }); @@ -33,9 +31,7 @@ describe("logsUrlState", () => { it("drops a value the API would refuse instead of sending it", () => { const state = logsParamsToState( - new URLSearchParams( - "lgrp=loud&lsrc=api,WORKER,x y&lu=me&lev=Not.A.Code&lreq=short&ljob=0&lwin=1y", - ), + new URLSearchParams("lgrp=loud&lsrc=api,WORKER,x y&lev=Not.A.Code&lreq=short&ljob=0&lrun=-1"), ); expect(state).toEqual({ ...DEFAULT_LOGS_STATE, sources: ["api"] }); }); diff --git a/frontend/src/lib/logsUrlState.ts b/frontend/src/lib/logsUrlState.ts index 3105af84..57db7eb5 100644 --- a/frontend/src/lib/logsUrlState.ts +++ b/frontend/src/lib/logsUrlState.ts @@ -1,67 +1,56 @@ -// The admin diagnostics console's filters in the address bar (`?page=logs&lgrp=error&lreq=…`), so a -// filtered view — a request's lines, say — is a link. Keys are namespaced (`l…`) like the Plex and -// Channels ones; setPage clears them (LOGS_KEYS) and the console re-mirrors them while it is open. +// The server-log tab's own filters in the address bar (`?page=activity&at=logs&lgrp=error&lreq=…`), +// so a filtered view — a request's lines, say — is a link. The filters every activity tab shares +// (time range, account, text) live in ./activityUrlState. Keys are namespaced (`l…`) like the Plex +// and Channels ones; setPage clears them (via ACTIVITY_KEYS) and the tab re-mirrors them while open. import type { LevelGroup } from "./log"; -export type LogWindow = "1h" | "24h" | "7d"; - export interface LogsUrlState { group: LevelGroup; sources: string[]; - /** An account id, or `none` for the rows without one. */ - user: string | null; /** An event code, or `none` for the rows without one. */ event: string | null; requestId: string | null; jobId: number | null; - q: string; - window: LogWindow; + /** A background run's lines (`job_runs.id`). */ + runId: number | null; } export const DEFAULT_LOGS_STATE: LogsUrlState = { group: "all", sources: [], - user: null, event: null, requestId: null, jobId: null, - q: "", - window: "24h", + runId: null, }; -export const LOGS_KEYS = ["lgrp", "lsrc", "lu", "lev", "lreq", "ljob", "lq", "lwin"] as const; - -export const WINDOW_HOURS: Record = { "1h": 1, "24h": 24, "7d": 24 * 7 }; +export const LOGS_KEYS = ["lgrp", "lsrc", "lev", "lreq", "ljob", "lrun"] as const; const GROUPS: readonly LevelGroup[] = ["all", "error", "warning", "info", "detail"]; // The same shapes the API validates: a link someone edited by hand degrades to "no filter" here // instead of to a 422 on every request the console makes. -const USER_RE = /^(none|\d{1,10})$/; const EVENT_RE = /^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$/; const REQUEST_RE = /^[A-Za-z0-9._-]{8,64}$/; const SOURCE_RE = /^[a-z]{1,16}$/; const SOURCES_MAX = 8; -const Q_MAX = 200; const pick = (value: string | null, re: RegExp, max = 64) => value && value.length <= max && re.test(value) ? value : null; +const positive = (value: string | null) => + value && /^\d{1,10}$/.test(value) && Number(value) > 0 ? Number(value) : null; export function logsParamsToState(params: URLSearchParams): LogsUrlState { const group = params.get("lgrp"); - const window = params.get("lwin"); - const job = params.get("ljob"); return { group: GROUPS.includes(group as LevelGroup) ? (group as LevelGroup) : DEFAULT_LOGS_STATE.group, sources: (params.get("lsrc") ?? "") .split(",") .filter((s) => SOURCE_RE.test(s)) .slice(0, SOURCES_MAX), - user: pick(params.get("lu"), USER_RE), event: pick(params.get("lev"), EVENT_RE), requestId: pick(params.get("lreq"), REQUEST_RE), - jobId: job && /^\d{1,10}$/.test(job) && Number(job) > 0 ? Number(job) : null, - q: (params.get("lq") ?? "").slice(0, Q_MAX), - window: window && window in WINDOW_HOURS ? (window as LogWindow) : DEFAULT_LOGS_STATE.window, + jobId: positive(params.get("ljob")), + runId: positive(params.get("lrun")), }; } @@ -70,11 +59,9 @@ export function logsParamUpdates(state: LogsUrlState): Record = { scheduler: "header.scheduler", config: "header.configuration", users: "header.users", - audit: "audit.title", - logs: "adminLogs.title", + activity: "activity.title", settings: "settings.title", }; diff --git a/frontend/src/lib/queryKeys.test.ts b/frontend/src/lib/queryKeys.test.ts index f44d2303..c3b23b96 100644 --- a/frontend/src/lib/queryKeys.test.ts +++ b/frontend/src/lib/queryKeys.test.ts @@ -39,7 +39,7 @@ describe("qk — single-id and scalar keys", () => { expect(qk.downloadLinks(5)).toEqual(["download-links", 5]); expect(qk.dlQuota(3)).toEqual(["dl-quota", 3]); expect(qk.storyboard(5)).toEqual(["storyboard", 5]); - expect(qk.adminQuota(30)).toEqual(["admin-quota", 30]); + expect(qk.adminQuota({ window: "7d" })).toEqual(["admin-quota", { window: "7d" }]); expect(qk.plexPeopleCards("both", ["Actor"])).toEqual(["plex-people-cards", "both", ["Actor"]]); }); diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index 91aadcf5..a2de239b 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -87,7 +87,10 @@ export const qk = { adminInvites: () => ["admin-invites"] as const, adminPendingAccess: () => ["admin-pending-access"] as const, adminConfig: () => ["admin-config"] as const, - adminAudit: () => ["admin-audit"] as const, + adminAudit: (filters: object) => ["admin-audit", filters] as const, + adminAuditFacets: (filters: object) => ["admin-audit-facets", filters] as const, + adminRuns: (filters: object) => ["admin-runs", filters] as const, + adminRunFacets: (filters: object) => ["admin-run-facets", filters] as const, // The console's filters are part of the key; `since` is not (it moves with the clock) — the // window it came from is. adminLogs: (filters: object) => ["admin-logs", filters] as const, @@ -105,7 +108,7 @@ export const qk = { debugSessionPreview: (id: string, text: string) => ["debug-sessions", id, "preview", text] as const, adminDownloads: () => ["admin-downloads"] as const, - adminQuota: (days: number) => ["admin-quota", days] as const, + adminQuota: (filters: object) => ["admin-quota", filters] as const, // --- plex --- // The composite plex keys carry a heterogeneous arg list (a filters object, a ratingKeys diff --git a/frontend/src/lib/storage.ts b/frontend/src/lib/storage.ts index 9f4c4bed..43061a31 100644 --- a/frontend/src/lib/storage.ts +++ b/frontend/src/lib/storage.ts @@ -64,12 +64,10 @@ export const LS = { // before the subscription list has landed. A cache, rewritten from the list on every load. channelNames: "siftlode.channelNames", channelDiscoveryTable: "siftlode.channelDiscoveryTable", - // A page's sub-tab: sessionStorage, per browser tab (useTabPersistedState), like adminUsersTab and - // statsTab below. + // A page's sub-tab: sessionStorage, per browser tab (useTabPersistedState), like adminUsersTab + // below. settingsTab: "siftlode.settingsTab", configTab: "siftlode.configTab", - auditTable: "siftlode.auditTable", - statsTab: "siftlode.statsTab", adminUsersTab: "siftlode.adminUsersTab", filterCollapsed: "siftlode.filterCollapsed", playlist: "siftlode.playlist", diff --git a/frontend/src/lib/urlState.ts b/frontend/src/lib/urlState.ts index 09a62083..98c2f8c1 100644 --- a/frontend/src/lib/urlState.ts +++ b/frontend/src/lib/urlState.ts @@ -128,8 +128,7 @@ const PAGES = [ "scheduler", "config", "users", - "audit", - "logs", + "activity", "notifications", "files", "plex", @@ -148,6 +147,10 @@ function isPage(p: string | null | undefined): p is Page { const RETIRED: Readonly> = { // The download queue became the tray; what it downloaded lives in Files. downloads: "files", + // The audit log and the server log became tabs of the activity page (activityUrlState opens the + // right tab from the old page id). + audit: "activity", + logs: "activity", }; /** A page id as stored or linked: a live page, a retired page's successor, else null. */ diff --git a/frontend/src/lib/useHistoryList.ts b/frontend/src/lib/useHistoryList.ts new file mode 100644 index 00000000..d1e0e188 --- /dev/null +++ b/frontend/src/lib/useHistoryList.ts @@ -0,0 +1,33 @@ +import { useMemo } from "react"; +import { keepPreviousData, useInfiniteQuery } from "@tanstack/react-query"; + +import type { HistoryPage } from "./api"; + +/** A history list read a page at a time, newest first (the server's keyset paging): the rows loaded + * so far, the accounts they name, and whether there is an older page to ask for. While a filter + * change loads, the previous filter's rows stay on screen (dimmed by the caller) and no older page is + * offered — the cursor on screen belongs to the rows of the old filter. */ +export function useHistoryList( + queryKey: readonly unknown[], + fetchPage: (before: string | null) => Promise>, +) { + const list = useInfiniteQuery({ + queryKey, + queryFn: ({ pageParam }) => fetchPage(pageParam), + initialPageParam: null as string | null, + getNextPageParam: (last) => last.next_cursor ?? undefined, + placeholderData: keepPreviousData, + }); + const rows = useMemo(() => list.data?.pages.flatMap((p) => p.items) ?? [], [list.data]); + const accounts = useMemo( + () => + Object.assign({}, ...(list.data?.pages.map((p) => p.accounts) ?? [])) as Record< + string, + string + >, + [list.data], + ); + return { list, rows, accounts, hasOlder: !list.isPlaceholderData && list.hasNextPage }; +} + +export type HistoryList = ReturnType>; From fe6308d844085a5f0b5303cd8b1d29dcd040841c Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 03:48:30 +0200 Subject: [PATCH 09/16] refactor(activity): the window's hours stay private to its URL state --- frontend/src/lib/activityUrlState.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/activityUrlState.ts b/frontend/src/lib/activityUrlState.ts index f8c25ce8..b8297365 100644 --- a/frontend/src/lib/activityUrlState.ts +++ b/frontend/src/lib/activityUrlState.ts @@ -33,7 +33,7 @@ export const RUN_STATUSES: readonly RunStatusFilter[] = [ "ok", ]; -export const WINDOW_HOURS: Record = { +const WINDOW_HOURS: Record = { "1h": 1, "24h": 24, "7d": 24 * 7, From c288ba147bb2889a9106bff33ef70a73d175991b Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 04:00:38 +0200 Subject: [PATCH 10/16] fix(activity): an old audit/log link opens its tab, times read relative, one name for admin actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seen in the browser, not in the page's own tests: the navigation provider rewrites a retired page id to `activity` before the page's chunk reads the URL, so `?page=audit` landed on the Runs tab. The retired-page map now carries the params that open the successor's part (`at`), and the provider writes them as it rewrites the page — on load and on Back alike. A list time is relative with the exact second on hover (the DESIGN copy rule); a run moved over from the audit log, whose start and end are one instant, shows no length instead of an invented "1 sec". The audit log is called "admin actions" everywhere it is named (the clear button and its confirm, the Configuration group, the cleanup job, which also prunes the run history its description left out). --- docs/DESIGN.md | 2 +- frontend/src/components/Activity.dom.test.tsx | 25 ++++++++++++--- frontend/src/components/ActivityAudit.tsx | 19 +++++++----- frontend/src/components/ActivityRuns.tsx | 19 +++++++----- .../src/components/AdminLogs.dom.test.tsx | 22 +++---------- .../NavigationProvider.dom.test.tsx | 31 +++++++++++++++++++ .../src/components/NavigationProvider.tsx | 10 ++++-- frontend/src/i18n/locales/en/audit.json | 14 ++++----- .../src/i18n/locales/en/auditActions.json | 2 +- frontend/src/i18n/locales/en/config.json | 6 ++-- frontend/src/i18n/locales/en/scheduler.json | 4 +-- frontend/src/i18n/locales/hu/audit.json | 14 ++++----- .../src/i18n/locales/hu/auditActions.json | 2 +- frontend/src/i18n/locales/hu/config.json | 6 ++-- frontend/src/i18n/locales/hu/scheduler.json | 4 +-- frontend/src/lib/activityUrlState.test.ts | 7 ----- frontend/src/lib/activityUrlState.ts | 10 ++---- frontend/src/lib/urlState.ts | 28 +++++++++++------ 18 files changed, 133 insertions(+), 92 deletions(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index e417c297..d9febfdf 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -649,7 +649,7 @@ of a full-width panel and nothing on the right. `app/collation.py` on the server, so both sort alike. - **Hungarian words, not English ones** (E30 D3). Kept as Hungarian users say them: Feed, Shorts, Admin, Token, Port, and technical tokens (log levels, ffmpeg presets, remux). Everything else is Hungarian: - közvetítés (stream), Prioritás, Feladat (job), Eseménynapló (audit), Demó, Automatikus, Infó, + közvetítés (stream), Prioritás, Feladat (job), Napló (activity), Admin műveletek (audit), Demó, Automatikus, Infó, Felület / Szerver (frontend / backend), videók behúzása (backfill), átkódolás (transcode), e-mail. A longer Hungarian word gets its room — a column is sized to it, a pill truncates — never clipped. - A label says what happens ("Delete playlist"), not "OK"; a destructive confirm names the object. diff --git a/frontend/src/components/Activity.dom.test.tsx b/frontend/src/components/Activity.dom.test.tsx index c01c0ac0..4105bb05 100644 --- a/frontend/src/components/Activity.dom.test.tsx +++ b/frontend/src/components/Activity.dom.test.tsx @@ -3,7 +3,7 @@ // The activity page around its run and audit lists (the server-log tab has its own file): the view // is the URL — a link narrows the runs to one job, a status chip narrows them further; a run opens // to its log lines and an admin action to its request's, each a new history entry that Back undoes; -// a link to the retired audit page opens the admin tab; the account filter reaches the quota tab; a +// the account filter reaches the quota tab; a // failed read is an error with a retry, never an empty list. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; @@ -168,11 +168,10 @@ describe("Activity", () => { expect(screen.getByText("quotaExceeded")).toBeTruthy(); }); - it("a link to the retired audit page opens the admin tab, and an action opens its request's lines", async () => { - window.history.replaceState(null, "", "/?page=audit"); + it("the admin tab lists the actions, and an action opens its request's lines", async () => { + window.history.replaceState(null, "", "/?page=activity&at=admin"); renderPage(); await waitFor(() => expect(screen.getByText("auditActions.scheduler_interval")).toBeTruthy()); - expect(url().get("page")).toBe("activity"); expect(runsMock).not.toHaveBeenCalled(); fireEvent.click(rowButton(/auditActions\.scheduler_interval/)); @@ -197,6 +196,24 @@ describe("Activity", () => { expect(Math.round(days)).toBe(30); }); + // The runs moved over from the audit log kept one instant for start and end: their length is + // unknown, and a "1 sec" would be invented. + it("a run whose start and end are one instant has no length", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockResolvedValue( + page([ + { ...RUN, id: 1, status: "ok", error: null, finished_at: RUN.started_at, summary: "moved" }, + { ...RUN, id: 2, status: "ok", error: null, summary: "timed" }, + ]), + ); + renderPage(); + await waitFor(() => expect(screen.getByText("moved")).toBeTruthy()); + const cells = (text: string) => + [...rowButton(new RegExp(text)).children].map((c) => c.textContent); + expect(cells("moved")).toContain("—"); + expect(cells("timed")).not.toContain("—"); + }); + it("a failed read is an error with a retry, not an empty list", async () => { window.history.replaceState(null, "", "/?page=activity"); runsMock.mockRejectedValue(new Error("502")); diff --git a/frontend/src/components/ActivityAudit.tsx b/frontend/src/components/ActivityAudit.tsx index 020358d9..aaa2af01 100644 --- a/frontend/src/components/ActivityAudit.tsx +++ b/frontend/src/components/ActivityAudit.tsx @@ -7,7 +7,7 @@ import { api, type AuditQuery, type AuditRow } from "../lib/api"; import { notifyActionError } from "../lib/actionErrors"; import { sinceOf } from "../lib/activityUrlState"; import { auditActionLabel, auditText } from "../lib/auditText"; -import { formatDateTime, formatDateTimeSec } from "../lib/format"; +import { formatDateTimeSec } from "../lib/format"; import { notify } from "../lib/notifications"; import { qk } from "../lib/queryKeys"; import { useHistoryList } from "../lib/useHistoryList"; @@ -15,6 +15,7 @@ import type { Shared } from "./ActivityRuns"; import { useConfirm } from "./confirmContext"; import HistoryList from "./HistoryList"; import HistoryRow from "./HistoryRow"; +import { RelativeTime } from "./ui/RelativeTime"; import { Button } from "./ui/Button"; import FilterSelect from "./ui/FilterSelect"; @@ -134,17 +135,21 @@ export default function ActivityAudit({ cells={ <> {auditActionLabel(t, row.action)} - - {formatDateTime(row.created_at, lang)} - + {who ?? t("audit.system")} - - {formatDateTime(row.created_at, lang)} - + {[head, sub].filter(Boolean).join(" · ")} diff --git a/frontend/src/components/ActivityRuns.tsx b/frontend/src/components/ActivityRuns.tsx index 3d0ea4d4..ce4477bf 100644 --- a/frontend/src/components/ActivityRuns.tsx +++ b/frontend/src/components/ActivityRuns.tsx @@ -9,12 +9,13 @@ import { type ActivityUrlState, type RunStatusFilter, } from "../lib/activityUrlState"; -import { formatDateTime, formatDateTimeSec, formatNumber, formatRuntime } from "../lib/format"; +import { formatDateTimeSec, formatNumber, formatRuntime } from "../lib/format"; import { jobFailureText, jobResultText } from "../lib/jobResult"; import { qk } from "../lib/queryKeys"; import { useHistoryList } from "../lib/useHistoryList"; import HistoryList from "./HistoryList"; import HistoryRow from "./HistoryRow"; +import { RelativeTime } from "./ui/RelativeTime"; import { StatusDot } from "./RunStatusDot"; import { Button } from "./ui/Button"; import FilterChipGroup from "./ui/FilterChipGroup"; @@ -140,6 +141,8 @@ export default function ActivityRuns({ ); } +/** How long the run took; null while it runs, 0 when its start and end are one instant (the runs + * moved over from the audit log kept only when they were written — the length is unknown). */ function seconds(run: RunRow): number | null { if (!run.finished_at) return null; return (new Date(run.finished_at).getTime() - new Date(run.started_at).getTime()) / 1000; @@ -175,18 +178,18 @@ function RunCells({ run, actor }: { run: RunRow; actor: string | null | undefine {run.target_title && · {run.target_title}} {/* A phone has room for the name and the time only; the row opens to the rest. */} - - {formatDateTime(run.started_at, lang)} - + {t(`scheduler.trigger.${run.trigger}`)} {actor && ` · ${actor}`} + - {formatDateTime(run.started_at, lang)} - - - {secs === null ? "—" : formatRuntime(Math.max(1, secs), lang)} + {secs ? formatRuntime(Math.max(1, secs), lang) : "—"} {formatNumber(run.quota_units, lang)} diff --git a/frontend/src/components/AdminLogs.dom.test.tsx b/frontend/src/components/AdminLogs.dom.test.tsx index cfbb7062..fbf18d3b 100644 --- a/frontend/src/components/AdminLogs.dom.test.tsx +++ b/frontend/src/components/AdminLogs.dom.test.tsx @@ -2,10 +2,10 @@ // // The activity page's server-log tab, rendered through the page (which owns the shared filters and // the pivots). A request or run id narrows the list to that request or run (and says so, in the -// filter row and the URL), and Back returns to the view the pivot was made from; an old link to the -// retired log page opens this tab; the account in use stays pickable; "Live" watches the newest page -// only, and stops on a failed read instead of re-raising it; a failed read is an error, not an empty -// log; Copy takes every loaded line, oldest first. +// filter row and the URL), and Back returns to the view the pivot was made from; the account in use +// stays pickable; "Live" watches the newest page only, and stops on a failed read instead of +// re-raising it; a failed read is an error, not an empty log; Copy takes every loaded line, oldest +// first. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -199,20 +199,6 @@ describe("AdminLogs", () => { expect(new URLSearchParams(window.location.search).get("lrun")).toBe("77"); }); - it("an old link to the server log page opens this tab with its filter", async () => { - window.history.replaceState(null, "", "/?page=logs&lreq=req-00000009"); - logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); - renderConsole(); - - await waitFor(() => - expect(logsMock).toHaveBeenCalledWith( - expect.objectContaining({ request_id: "req-00000009" }), - null, - ), - ); - expect(new URLSearchParams(window.location.search).get("page")).toBe("activity"); - }); - it("keeps the account in use offered when the account list does not have it", async () => { window.history.replaceState(null, "", "/?page=activity&at=logs&au=7"); logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); diff --git a/frontend/src/components/NavigationProvider.dom.test.tsx b/frontend/src/components/NavigationProvider.dom.test.tsx index 60ee5a80..eb2cc655 100644 --- a/frontend/src/components/NavigationProvider.dom.test.tsx +++ b/frontend/src/components/NavigationProvider.dom.test.tsx @@ -219,6 +219,37 @@ describe("NavigationProvider · Back", () => { expect(window.location.search).toBe("?page=files&keep=1"); }); + // The audit log and the server log became the activity page's tabs: an old link, bookmark or entry + // opens that tab, with the filters it carried. + it("opens a retired page's tab on the page that replaced it", () => { + window.history.replaceState(null, "", "/?page=audit&keep=1"); + const { result } = render(); + expect(result.current.page).toBe("activity"); + expect(window.location.search).toBe("?page=activity&keep=1&at=admin"); + }); + + it("keeps a retired page's own filter as it opens its tab", () => { + window.history.replaceState(null, "", "/?page=logs&lreq=req-00000009"); + render(); + const url = new URLSearchParams(window.location.search); + expect([url.get("page"), url.get("at"), url.get("lreq")]).toEqual([ + "activity", + "logs", + "req-00000009", + ]); + }); + + it("opens a retired page's tab when Back lands on its entry", () => { + const { result } = render(); + act(() => result.current.setPage("plex")); + act(() => { + window.history.replaceState({ sfPage: "audit" }, "", "/?page=audit"); + window.dispatchEvent(new PopStateEvent("popstate", { state: { sfPage: "audit" } })); + }); + expect(result.current.page).toBe("activity"); + expect(window.location.search).toBe("?page=activity&at=admin"); + }); + // lib/history's own listener takes a spent focus stamp off the entry it lands on; the rewrite must // not put it back, or the next landing there (an overlay closing) would refocus a stale item. it("does not put back the focus stamp the landing spent", () => { diff --git a/frontend/src/components/NavigationProvider.tsx b/frontend/src/components/NavigationProvider.tsx index d73a2dec..7ea53516 100644 --- a/frontend/src/components/NavigationProvider.tsx +++ b/frontend/src/components/NavigationProvider.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react import { useLatest } from "../lib/useLatest"; import { useTranslation } from "react-i18next"; import { useMe } from "../lib/useMe"; -import { resolvePage, urlWithParams, type Page } from "../lib/urlState"; +import { resolvePage, retiredParams, urlWithParams, type Page } from "../lib/urlState"; import { ACTIVITY_KEYS } from "../lib/activityUrlState"; import { pageTitleKey } from "../lib/pageMeta"; import { useDocumentObjectTitle } from "../lib/documentTitle"; @@ -193,7 +193,11 @@ export function NavigationProvider({ children }: { children: ReactNode }) { // Mirror the resolved initial page to the address bar (feed implicit) so a page reached from the // stored landing pref — not just from the URL — is itself bookmarkable. urlWithParams preserves any // ?chan / ?v / ?pl / filter params the loaders already read. - const url = urlWithParams({ page: mountPage === "feed" ? null : mountPage }); + const url = urlWithParams({ + // A retired page's link also opens its part of the page that replaced it (an `at` tab). + ...retiredParams(new URLSearchParams(window.location.search).get("page")), + page: mountPage === "feed" ? null : mountPage, + }); if (st._yt && st._ytScrape) { window.history.replaceState({ ...st, sfPage: mountPage }, "", url); } else { @@ -213,7 +217,7 @@ export function NavigationProvider({ children }: { children: ReactNode }) { window.history.replaceState( { ...window.history.state, sfPage: p }, "", - urlWithParams({ page: p === "feed" ? null : p }), + urlWithParams({ ...retiredParams(sf), page: p === "feed" ? null : p }), ); setPageState(p); setAccountRaw(PAGE_KEY, p); diff --git a/frontend/src/i18n/locales/en/audit.json b/frontend/src/i18n/locales/en/audit.json index 62ce2a59..b24741d8 100644 --- a/frontend/src/i18n/locales/en/audit.json +++ b/frontend/src/i18n/locales/en/audit.json @@ -1,12 +1,12 @@ { - "empty": "No audit entries yet.", + "empty": "No admin actions match the filters.", "system": "System", - "clear": "Clear audit log", - "clearTitle": "Clear the audit log?", - "clearConfirm": "Permanently delete all audit entries? This can't be undone. (One entry recording that you cleared it is kept.)", - "cleared_one": "Cleared {{count}} audit entry", - "cleared_other": "Cleared {{count}} audit entries", - "clearFailed": "Couldn't clear the audit log", + "clear": "Clear admin actions", + "clearTitle": "Clear the admin actions?", + "clearConfirm": "Permanently delete every admin action entry? This can't be undone. (One entry recording that you cleared them is kept.)", + "cleared_one": "Cleared {{count}} admin action entry", + "cleared_other": "Cleared {{count}} admin action entries", + "clearFailed": "Couldn't clear the admin actions", "on": "On", "off": "Off", "default": "default", diff --git a/frontend/src/i18n/locales/en/auditActions.json b/frontend/src/i18n/locales/en/auditActions.json index 4c87fa0b..f11400ae 100644 --- a/frontend/src/i18n/locales/en/auditActions.json +++ b/frontend/src/i18n/locales/en/auditActions.json @@ -16,7 +16,7 @@ "maintenance_tune": "Maintenance tuned", "sync_pause": "Sync paused", "sync_resume": "Sync resumed", - "audit_clear": "Audit log cleared", + "audit_clear": "Admin actions cleared", "download_credential_set": "Download cookies set", "download_credential_delete": "Download cookies deleted", "invite_delete": "Whitelist entry deleted", diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index 69dab896..15510873 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -21,7 +21,7 @@ "downloads": "Downloads", "plex": "Plex", "tmdb": "TMDB", - "audit": "Audit log", + "audit": "Activity", "diagnostics": "Diagnostics", "issues": "Bug reports" }, @@ -163,8 +163,8 @@ "hint": "How many downloads the worker processes in parallel." }, "audit_retention_days": { - "label": "Retention (days)", - "hint": "How long admin audit-log entries are kept before the audit-log cleanup job prunes them. 0 = keep forever. Default 180." + "label": "Admin actions (days)", + "hint": "How long admin action entries are kept before the cleanup job prunes them. 0 = keep forever. Default 180." }, "job_run_retention_days": { "label": "Run history (days)", diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index 1ed1eda7..f3a40d0a 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -76,7 +76,7 @@ "plex_sync": "Mirrors the enabled Plex libraries into the app's catalogue — the metadata behind Plex browsing, search, filters and the info pages (genres, cast, ratings, artwork). If it stops, newly added or changed Plex titles don't appear until the next sync.", "plex_watch_sync": "Pulls recent Plex watch changes (finished episodes, resume positions) into Siftlode and re-pushes any local watch changes that didn't reach Plex. The cheap two-way keep-in-sync between full reconciles. If it stops, watched/resume made in the Plex app takes longer to show here.", "plex_watch_reconcile": "A full watch-state reconcile that rescans every library — catches what the incremental pass can't, notably an episode un-marked as watched on the Plex side. Heavier, so it runs about once a day. If it stops, un-watches done in Plex aren't mirrored back.", - "audit_gc": "Prunes admin audit-log entries and YouTube quota history older than their retention windows (Configuration → Audit log, Quota). If it stops, both just keep growing — harmless but unbounded.", + "audit_gc": "Prunes admin action entries, background-run history and YouTube quota history older than their retention windows (Configuration → Activity, Quota). If it stops, all three just keep growing — harmless but unbounded.", "log_gc": "Deletes stored log events older than the retention window (Configuration → Diagnostics). Runs hourly so nothing outlives the promised limit by more than an hour.", "tmdb_enrich": "Enriches the mirrored Plex catalogue from TMDB (keywords, crew, budget/revenue, TMDb score, recommendations, artwork), matched by each title's canonical id and served from the app's own database. Optional — a no-op until a TMDB API key is set. If it stops, new Plex titles just miss their TMDB extras until it runs again.", "people_enrich": "Makes the people in the Plex library local: fetches each person's biography from TMDB and caches their portrait, resized through Plex, the most-credited people first. A person page and a cast strip then open from the app's own data. The TMDB half does nothing without a TMDB API key, the portrait half nothing without Plex. If it stops, a person page fetches its own bio when opened, a moment later.", @@ -97,7 +97,7 @@ "plex_watch_sync": "Plex watch sync (incremental)", "plex_watch_reconcile": "Plex watch reconcile (full)", "tmdb_enrich": "TMDB enrichment (Plex)", - "audit_gc": "Audit-log and quota-history cleanup", + "audit_gc": "Activity and quota-history cleanup", "log_gc": "Log cleanup", "people_enrich": "People details and portraits (Plex)", "channel_backfill": "A channel's full history" diff --git a/frontend/src/i18n/locales/hu/audit.json b/frontend/src/i18n/locales/hu/audit.json index 86f7ae76..2c34e0d5 100644 --- a/frontend/src/i18n/locales/hu/audit.json +++ b/frontend/src/i18n/locales/hu/audit.json @@ -1,12 +1,12 @@ { - "empty": "Még nincs naplóbejegyzés.", + "empty": "Nincs a szűrőknek megfelelő admin művelet.", "system": "Rendszer", - "clear": "Eseménynapló ürítése", - "clearTitle": "Üríted az eseménynaplót?", - "clearConfirm": "Véglegesen törlöd az összes naplóbejegyzést? Ez nem vonható vissza. (Egy bejegyzés arról, hogy te ürítetted, megmarad.)", - "cleared_one": "{{count}} naplóbejegyzés törölve", - "cleared_other": "{{count}} naplóbejegyzés törölve", - "clearFailed": "Nem sikerült üríteni az eseménynaplót", + "clear": "Admin műveletek ürítése", + "clearTitle": "Üríted az admin műveleteket?", + "clearConfirm": "Véglegesen törlöd az összes admin művelet bejegyzését? Ez nem vonható vissza. (Egy bejegyzés arról, hogy te ürítetted, megmarad.)", + "cleared_one": "{{count}} bejegyzés törölve", + "cleared_other": "{{count}} bejegyzés törölve", + "clearFailed": "Nem sikerült üríteni az admin műveleteket", "on": "Be", "off": "Ki", "default": "alapérték", diff --git a/frontend/src/i18n/locales/hu/auditActions.json b/frontend/src/i18n/locales/hu/auditActions.json index a7e4671b..a9e08031 100644 --- a/frontend/src/i18n/locales/hu/auditActions.json +++ b/frontend/src/i18n/locales/hu/auditActions.json @@ -16,7 +16,7 @@ "maintenance_tune": "Karbantartás hangolva", "sync_pause": "Szinkron szüneteltetve", "sync_resume": "Szinkron folytatva", - "audit_clear": "Eseménynapló ürítve", + "audit_clear": "Admin műveletek ürítve", "download_credential_set": "Letöltési cookie-k beállítva", "download_credential_delete": "Letöltési cookie-k törölve", "invite_delete": "Fehérlista-bejegyzés törölve", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index 1a853bdc..fe41b1ef 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -21,7 +21,7 @@ "downloads": "Letöltések", "plex": "Plex", "tmdb": "TMDB", - "audit": "Eseménynapló", + "audit": "Napló", "diagnostics": "Diagnosztika", "issues": "Hibajegyek" }, @@ -163,8 +163,8 @@ "hint": "Hány letöltést dolgoz fel egyszerre a letöltő folyamat." }, "audit_retention_days": { - "label": "Megőrzés (nap)", - "hint": "Meddig maradnak meg az admin-naplóbejegyzések, mielőtt az eseménynapló-takarító feladat törli őket. 0 = örökre. Alapérték 180." + "label": "Admin műveletek (nap)", + "hint": "Meddig maradnak meg az admin műveletek bejegyzései, mielőtt a takarító feladat törli őket. 0 = örökre. Alapérték 180." }, "job_run_retention_days": { "label": "Futási előzmények (nap)", diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 7b9b511f..a6649a5f 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -76,7 +76,7 @@ "plex_sync": "Tükrözi az engedélyezett Plex-könyvtárakat az app katalógusába — ez adja a Plex böngészés, keresés, szűrők és infóoldalak metaadatait (műfaj, szereplők, pontszám, borítók). Ha leáll, az újonnan hozzáadott vagy módosított Plex-címek csak a következő szinkron után jelennek meg.", "plex_watch_sync": "Áthozza a Plex-oldali friss megtekintés-változásokat (befejezett epizódok, folytatás-pozíciók) a Siftlode-ba, és felnyomja a Plexet el nem érő helyi változásokat. Ez az olcsó kétirányú szinkron a teljes egyeztetések között. Ha leáll, a Plex-appban jelölt megtekintés/folytatás lassabban jelenik meg itt.", "plex_watch_reconcile": "A megtekintési állapot teljes egyeztetése, amely minden könyvtárat újraszken­nel — elkapja, amit az inkrementális menet nem, főleg ha egy epizódról a Plex oldalán levették a megtekintve jelet. Nehezebb, ezért kb. naponta fut. Ha leáll, a Plexben levett megtekintve jel nem tükröződik vissza.", - "audit_gc": "Törli a megőrzési időnél régebbi admin-naplóbejegyzéseket és YouTube-kvótaelőzményeket (Konfiguráció → Eseménynapló, Kvóta). Ha leáll, mindkettő csak nő — ártalmatlan, de korlátlanul.", + "audit_gc": "Törli a megőrzési időnél régebbi admin műveleteket, háttérfutás-előzményeket és YouTube-kvótaelőzményeket (Konfiguráció → Napló, Kvóta). Ha leáll, mindhárom csak nő — ártalmatlan, de korlátlanul.", "log_gc": "Törli a megőrzési időnél (Konfiguráció → Diagnosztika) régebbi tárolt naplóeseményeket. Óránként fut, így semmi sem marad meg egy óránál tovább az ígért határidőn túl.", "tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.", "people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.", @@ -97,7 +97,7 @@ "plex_watch_sync": "Plex megtekintés-szinkron (inkrementális)", "plex_watch_reconcile": "Plex megtekintés-egyeztetés (teljes)", "tmdb_enrich": "TMDB-dúsítás (Plex)", - "audit_gc": "Eseménynapló- és kvótaelőzmény-takarítás", + "audit_gc": "Napló- és kvótaelőzmény-takarítás", "log_gc": "Naplótakarítás", "people_enrich": "Személyadatok és portrék (Plex)", "channel_backfill": "Egy csatorna teljes előzménye" diff --git a/frontend/src/lib/activityUrlState.test.ts b/frontend/src/lib/activityUrlState.test.ts index 9291d6e6..b748d474 100644 --- a/frontend/src/lib/activityUrlState.test.ts +++ b/frontend/src/lib/activityUrlState.test.ts @@ -45,13 +45,6 @@ describe("activityUrlState", () => { expect(state).toEqual(DEFAULT_ACTIVITY_STATE); }); - it("opens the tab of the page a link still names", () => { - expect(activityParamsToState(new URLSearchParams("page=audit")).tab).toBe("admin"); - expect(activityParamsToState(new URLSearchParams("page=logs")).tab).toBe("logs"); - // A tab the link names wins over the retired page's. - expect(activityParamsToState(new URLSearchParams("page=logs&at=quota")).tab).toBe("quota"); - }); - it("clears its own keys and the server log's on a page change", () => { expect(ACTIVITY_KEYS).toEqual(expect.arrayContaining(["at", "aw", "au", "aq", "rjob", "lreq"])); }); diff --git a/frontend/src/lib/activityUrlState.ts b/frontend/src/lib/activityUrlState.ts index b8297365..b1b495a4 100644 --- a/frontend/src/lib/activityUrlState.ts +++ b/frontend/src/lib/activityUrlState.ts @@ -54,10 +54,6 @@ export const DEFAULT_ACTIVITY_STATE: ActivityUrlState = { const OWN_KEYS = ["at", "aw", "au", "aq", "rjob", "rst", "aact"] as const; export const ACTIVITY_KEYS = [...OWN_KEYS, ...LOGS_KEYS] as const; -// The pages this one replaced: a link or a bookmark to one opens its tab here (urlState's RETIRED -// sends the page id itself here). -const RETIRED_TAB: Readonly> = { audit: "admin", logs: "logs" }; - // The same shapes the API validates: a hand-edited link degrades to "no filter" here instead of to // a 422 on every request the page makes. const USER_RE = /^(none|\d{1,10})$/; @@ -71,10 +67,8 @@ const oneOf = (value: string | null, list: readonly T[]): T | export function activityParamsToState(params: URLSearchParams): ActivityUrlState { return { - tab: - oneOf(params.get("at"), ACTIVITY_TABS) ?? - RETIRED_TAB[params.get("page") ?? ""] ?? - DEFAULT_ACTIVITY_STATE.tab, + // A link to a page this one replaced arrives with its tab already set (urlState's RETIRED). + tab: oneOf(params.get("at"), ACTIVITY_TABS) ?? DEFAULT_ACTIVITY_STATE.tab, window: oneOf(params.get("aw"), ACTIVITY_WINDOWS) ?? DEFAULT_ACTIVITY_STATE.window, user: pick(params.get("au"), USER_RE, 10), q: (params.get("aq") ?? "").slice(0, Q_MAX), diff --git a/frontend/src/lib/urlState.ts b/frontend/src/lib/urlState.ts index 98c2f8c1..cbb92217 100644 --- a/frontend/src/lib/urlState.ts +++ b/frontend/src/lib/urlState.ts @@ -141,22 +141,30 @@ function isPage(p: string | null | undefined): p is Page { return !!p && (PAGES as readonly string[]).includes(p); } -// A page this build no longer has, and the page that took its place. A bookmark, the remembered last -// page and an old history entry can still name one; all three resolve through `resolvePage`, so a -// retired page lands where its content went instead of on the feed. -const RETIRED: Readonly> = { +// A page this build no longer has, the page that took its place, and the params that open its part +// of that page. A bookmark, the remembered last page and an old history entry can still name one; all +// three resolve through `resolvePage`, so a retired page lands where its content went instead of on +// the feed, and the navigation provider writes `params` into the address bar as it rewrites the page. +const RETIRED: Readonly }>> = { // The download queue became the tray; what it downloaded lives in Files. - downloads: "files", - // The audit log and the server log became tabs of the activity page (activityUrlState opens the - // right tab from the old page id). - audit: "activity", - logs: "activity", + downloads: { page: "files" }, + // The audit log and the server log became tabs of the activity page (its `at` key). + audit: { page: "activity", params: { at: "admin" } }, + logs: { page: "activity", params: { at: "logs" } }, }; +const retired = (p: string | null | undefined) => + p && Object.prototype.hasOwnProperty.call(RETIRED, p) ? RETIRED[p]! : null; + /** A page id as stored or linked: a live page, a retired page's successor, else null. */ export function resolvePage(p: string | null | undefined): Page | null { if (isPage(p)) return p; - return p && Object.prototype.hasOwnProperty.call(RETIRED, p) ? RETIRED[p]! : null; + return retired(p)?.page ?? null; +} + +/** The params that open a retired page's part of its successor (none for a live page). */ +export function retiredParams(p: string | null | undefined): Record { + return isPage(p) ? {} : (retired(p)?.params ?? {}); } export function readPage(): Page { From 3021be51663951952cd83a6c1b3a529c6add7fe5 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 04:22:52 +0200 Subject: [PATCH 11/16] fix(activity): a run's quota reads with its unit, its outcome keeps room at a large text size, a token's audit rows read in Hungarian The run list showed the quota spend as a bare figure and, at the largest text size on a 1024-wide screen, its fixed columns left the outcome column no width at all. The outcome now keeps 10rem and the trigger column gives way; the audit list's summary keeps the same floor. A run that stored no log line says so instead of linking to an empty list. An API token revoke and a token's file work rendered the server's English line under the Hungarian UI; both are rebuilt in the reader's language, and the test holds the server's wording. --- frontend/src/components/Activity.dom.test.tsx | 31 +++++++++++++++++++ frontend/src/components/ActivityAudit.tsx | 2 +- frontend/src/components/ActivityRuns.tsx | 22 +++++++++---- frontend/src/i18n/locales/en/activity.json | 3 ++ frontend/src/i18n/locales/en/audit.json | 2 ++ frontend/src/i18n/locales/hu/activity.json | 3 ++ frontend/src/i18n/locales/hu/audit.json | 2 ++ frontend/src/lib/auditText.dom.test.ts | 26 ++++++++++++++++ frontend/src/lib/auditText.ts | 7 +++++ 9 files changed, 91 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/Activity.dom.test.tsx b/frontend/src/components/Activity.dom.test.tsx index 4105bb05..d16478c4 100644 --- a/frontend/src/components/Activity.dom.test.tsx +++ b/frontend/src/components/Activity.dom.test.tsx @@ -214,6 +214,37 @@ describe("Activity", () => { expect(cells("timed")).not.toContain("—"); }); + // A bare "212" in a row reads as nothing in particular: the spend carries its unit, and a run that + // spent none says nothing rather than "0". + it("a run's quota spend reads with its unit, and none spent leaves the cell blank", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockResolvedValue( + page([ + { ...RUN, id: 1, status: "ok", error: null, summary: "spent" }, + { ...RUN, id: 2, status: "ok", error: null, summary: "free", quota_units: 0 }, + ]), + ); + renderPage(); + await waitFor(() => expect(screen.getByText("spent")).toBeTruthy()); + const cells = (text: string) => + [...rowButton(new RegExp(text)).children].map((c) => c.textContent); + expect(cells("spent")).toContain("activity.runs.units"); + expect(cells("spent")).not.toContain("212"); + expect(cells("free")).not.toContain("0"); + expect(cells("free")).not.toContain("activity.runs.units"); + }); + + // Most runs store no line at all: a link there would only lead to an empty list. + it("a run that logged nothing says so instead of linking to an empty list", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockResolvedValue(page([{ ...RUN, log_lines: 0 }])); + renderPage(); + await waitFor(() => expect(screen.getByText("quotaExceeded")).toBeTruthy()); + fireEvent.click(rowButton(/quotaExceeded/)); + expect(screen.getByText("activity.runs.noLogLines")).toBeTruthy(); + expect(screen.queryByRole("button", { name: "activity.runs.logLines" })).toBeNull(); + }); + it("a failed read is an error with a retry, not an empty list", async () => { window.history.replaceState(null, "", "/?page=activity"); runsMock.mockRejectedValue(new Error("502")); diff --git a/frontend/src/components/ActivityAudit.tsx b/frontend/src/components/ActivityAudit.tsx index aaa2af01..ff6d0ab9 100644 --- a/frontend/src/components/ActivityAudit.tsx +++ b/frontend/src/components/ActivityAudit.tsx @@ -24,7 +24,7 @@ import FilterSelect from "./ui/FilterSelect"; // leaving one row that says who cleared it. const GRID = - "grid-cols-[minmax(0,1fr)_auto] sm:grid-cols-[minmax(0,12rem)_minmax(0,12rem)_8rem_minmax(0,1fr)]"; + "grid-cols-[minmax(0,1fr)_auto] sm:grid-cols-[minmax(0,12rem)_minmax(0,12rem)_8rem_minmax(10rem,1fr)]"; export default function ActivityAudit({ shared, diff --git a/frontend/src/components/ActivityRuns.tsx b/frontend/src/components/ActivityRuns.tsx index ce4477bf..777c45f1 100644 --- a/frontend/src/components/ActivityRuns.tsx +++ b/frontend/src/components/ActivityRuns.tsx @@ -26,7 +26,7 @@ import FilterSelect from "./ui/FilterSelect"; // failed, and two ways on: the lines the run logged, and the other runs of its job. const GRID = - "grid-cols-[0.75rem_minmax(0,1fr)_auto] sm:grid-cols-[0.75rem_minmax(0,12rem)_8rem_8rem_4rem_3.5rem_minmax(0,1fr)]"; + "grid-cols-[0.75rem_minmax(0,1fr)_auto] sm:grid-cols-[0.75rem_minmax(8rem,12rem)_minmax(0,8rem)_5.5rem_4rem_5.5rem_minmax(10rem,1fr)]"; /** The shared filters, as the page passes them down. */ export interface Shared { @@ -191,8 +191,13 @@ function RunCells({ run, actor }: { run: RunRow; actor: string | null | undefine {secs ? formatRuntime(Math.max(1, secs), lang) : "—"} - - {formatNumber(run.quota_units, lang)} + {/* The spend carries its unit (a bare figure in a row has no meaning); none spent is blank. */} + + {run.quota_units > 0 && + t("activity.runs.units", { + count: run.quota_units, + units: formatNumber(run.quota_units, lang), + })} {outcome.text} @@ -248,9 +253,14 @@ function RunDetail({ )}
- + {/* Most runs store no line at all; a link there would only open an empty list. */} + {run.log_lines > 0 ? ( + + ) : ( + {t("activity.runs.noLogLines")} + )} diff --git a/frontend/src/i18n/locales/en/activity.json b/frontend/src/i18n/locales/en/activity.json index 8aeb5efb..dba2414c 100644 --- a/frontend/src/i18n/locales/en/activity.json +++ b/frontend/src/i18n/locales/en/activity.json @@ -40,6 +40,8 @@ "empty": "No runs match the filters.", "count_one": "{{count}} run shown · newest first", "count_other": "{{count}} runs shown · newest first", + "units_one": "{{units}} unit", + "units_other": "{{units}} units", "fields": { "started": "Started by", "when": "Start → end", @@ -48,6 +50,7 @@ }, "logLines_one": "Its {{count}} log line →", "logLines_other": "Its {{count}} log lines →", + "noLogLines": "It logged no lines", "jobRuns": "This job's runs →" }, "audit": { diff --git a/frontend/src/i18n/locales/en/audit.json b/frontend/src/i18n/locales/en/audit.json index b24741d8..7ef1f0e8 100644 --- a/frontend/src/i18n/locales/en/audit.json +++ b/frontend/src/i18n/locales/en/audit.json @@ -12,6 +12,8 @@ "default": "default", "secretUpdated": "secret updated (the value is never logged)", "secretReset": "secret reset to its default (the value is never logged)", + "tokenRevoked": "revoked, with every session of its account", + "viaToken": "By API token {{token}}", "cols": { "when": "When", "actor": "Who", diff --git a/frontend/src/i18n/locales/hu/activity.json b/frontend/src/i18n/locales/hu/activity.json index 8ca6cad2..33d26641 100644 --- a/frontend/src/i18n/locales/hu/activity.json +++ b/frontend/src/i18n/locales/hu/activity.json @@ -40,6 +40,8 @@ "empty": "Nincs a szűrőknek megfelelő futás.", "count_one": "{{count}} futás látszik · a legújabb elöl", "count_other": "{{count}} futás látszik · a legújabb elöl", + "units_one": "{{units}} egység", + "units_other": "{{units}} egység", "fields": { "started": "Indította", "when": "Kezdés → vége", @@ -48,6 +50,7 @@ }, "logLines_one": "A futás {{count}} naplósora →", "logLines_other": "A futás {{count}} naplósora →", + "noLogLines": "Nem naplózott sort", "jobRuns": "Ennek a feladatnak a futásai →" }, "audit": { diff --git a/frontend/src/i18n/locales/hu/audit.json b/frontend/src/i18n/locales/hu/audit.json index 2c34e0d5..80662345 100644 --- a/frontend/src/i18n/locales/hu/audit.json +++ b/frontend/src/i18n/locales/hu/audit.json @@ -12,6 +12,8 @@ "default": "alapérték", "secretUpdated": "titok frissítve (az értéke sosem kerül a naplóba)", "secretReset": "titok visszaállítva az alapértékre (az értéke sosem kerül a naplóba)", + "tokenRevoked": "visszavonva, a fiók minden munkamenetével együtt", + "viaToken": "API-tokennel: {{token}}", "cols": { "when": "Mikor", "actor": "Ki", diff --git a/frontend/src/lib/auditText.dom.test.ts b/frontend/src/lib/auditText.dom.test.ts index a0906ad5..a1bee7e6 100644 --- a/frontend/src/lib/auditText.dom.test.ts +++ b/frontend/src/lib/auditText.dom.test.ts @@ -3,6 +3,8 @@ // An audit row whose server summary is a fixed English line ("log_level = DEBUG", "rss_poll → 15 // min/run", "Background sync paused") reads in the reader's language, rebuilt from the row's // data through the REAL catalog. A row whose summary is data (an e-mail, a URL) keeps it. +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import i18n, { setLanguage } from "../i18n"; import type { AuditRow } from "./api"; @@ -106,6 +108,30 @@ describe("an audit row's details", () => { expect(details(row({ action: "sync_pause", summary: "Background sync paused" }))).toBe(""); }); + it("say a token revoke and a token's file work in the reader's language", () => { + const revoke = details( + row({ + action: "api_token_revoke", + target_type: "api_token", + target_id: "11", + summary: "revoked with every session of the account", + }), + ); + expect(revoke).not.toMatch(/revoked/); + expect(revoke).toMatch(/munkamenet/); + const upload = details( + row({ action: "file_upload", target_type: "file", summary: "api token sfl_Q3qXFfcv…" }), + ); + expect(upload).not.toMatch(/api token/); + expect(upload).toContain("sfl_Q3qXFfcv…"); + // The line read above is the one the server writes. + const apitokens = readFileSync( + resolve(__dirname, "../../../backend/app/apitokens.py"), + "utf-8", + ); + expect(apitokens).toContain('f"api token {self.token_prefix}…"'); + }); + it("keep a summary that is data", () => { expect(details(row({ action: "invite_add", summary: "someone@example.com" }))).toBe( "someone@example.com", diff --git a/frontend/src/lib/auditText.ts b/frontend/src/lib/auditText.ts index dd31a20c..7ebb10b5 100644 --- a/frontend/src/lib/auditText.ts +++ b/frontend/src/lib/auditText.ts @@ -3,6 +3,10 @@ import type { AuditRow, JobResult } from "./api"; import { formatNumber } from "./format"; import { jobResultText } from "./jobResult"; +/** How the server's summary says an API token did the work (`apitokens.FilesCaller.via`, held by + * the test): the token's prefix follows. */ +const VIA_TOKEN = "api token "; + function fmt(v: unknown): string { return v == null || v === "" ? "—" : String(v); } @@ -63,6 +67,9 @@ function localDetails( return result ? { head: null, sub: jobResultText(t, result, lang) } : undefined; } if (row.action === "sync_pause" || row.action === "sync_resume") return { head: null, sub: null }; + if (row.action === "api_token_revoke") return { head: null, sub: t("audit.tokenRevoked") }; + if (row.summary?.startsWith(VIA_TOKEN)) + return { head: t("audit.viaToken", { token: row.summary.slice(VIA_TOKEN.length) }), sub: null }; return undefined; } From fad656988ad7381e8554855fce1c1b4eef9cfbb3 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 15:38:32 +0200 Subject: [PATCH 12/16] fix(activity): runs keep their true outcome and spend, the page reaches every kept row, a closing dialog leaves the server log alone Backend - A run's outcome write sits outside the job's own try: a database blip after a run that succeeded no longer records it as failed (the row waits for the next start's sweep). - An interrupted run keeps what it spent (the sweep sums its quota rows, as finish does). - The Scheduler's ad-hoc row says running while any backfill runs, not only the newest. - recent_runs reads each job's last runs off its index instead of ranking the whole table. - An interval change is stored with its audit row in one transaction, then rescheduled. - One API instance per database is the stated contract (.env.example, jobruns). - Turning full history on for a channel already complete records no run. - Migration 0109 round-trips interrupted and ad-hoc runs; the run words' DB CHECKs are test-held to the enums; the account filter is one helper for the four activity reads. - An audit row says how many of its request's log lines are still kept. Frontend - An "All" period reaches every row still kept (runs and admin actions outlive 90 days). - Back/Forward is followed in place by the server log: a closing dialog no longer turns Live off; a pivot still opens a fresh log. - Runs moved over from the audit log read their one-liner in the reader's language. - Filter choices or accounts that failed to load say so beside the filters, with a retry. - An old server-log link keeps its account, text and range; a remembered retired page opens its tab. - A request whose lines are gone says so instead of linking to an empty list. - The log lines and the history rows share one row frame, and the server log one list frame with the other tabs; an empty list is announced; each day's quota is readable. --- .env.example | 8 +- backend/alembic/versions/0109_job_runs.py | 45 ++- backend/app/jobruns.py | 57 +++- backend/app/models.py | 2 +- backend/app/routes/_account.py | 17 ++ backend/app/routes/admin_logs.py | 15 +- backend/app/routes/audit.py | 30 +- backend/app/routes/channels.py | 4 +- backend/app/routes/quota.py | 7 +- backend/app/routes/runs.py | 11 +- backend/app/routes/scheduler.py | 10 +- backend/app/runstatus.py | 8 +- backend/app/scheduler.py | 113 ++++--- backend/app/sysconfig.py | 4 +- backend/tests/test_activity_reads.py | 50 +++ backend/tests/test_audit_count_rows.py | 2 +- backend/tests/test_channel_backfill.py | 13 + backend/tests/test_migration_0109.py | 33 ++ backend/tests/test_runstatus.py | 27 ++ backend/tests/test_scheduler.py | 94 ++++++ docs/DESIGN.md | 7 +- frontend/src/components/Activity.dom.test.tsx | 157 +++++++++- frontend/src/components/Activity.tsx | 22 +- frontend/src/components/ActivityAudit.tsx | 38 ++- frontend/src/components/ActivityQuota.tsx | 20 +- frontend/src/components/ActivityRuns.tsx | 18 +- frontend/src/components/AdminLogs.tsx | 284 +++++++++--------- frontend/src/components/HistoryList.tsx | 160 +++++++--- frontend/src/components/HistoryRow.tsx | 22 +- frontend/src/components/LogLine.tsx | 49 ++- .../NavigationProvider.dom.test.tsx | 31 ++ .../src/components/NavigationProvider.tsx | 9 +- frontend/src/i18n/backendIds.contract.test.ts | 26 ++ frontend/src/i18n/locales/en/activity.json | 13 +- frontend/src/i18n/locales/en/stats.json | 5 +- frontend/src/i18n/locales/hu/activity.json | 13 +- frontend/src/i18n/locales/hu/stats.json | 5 +- frontend/src/lib/activityUrlState.test.ts | 8 + frontend/src/lib/activityUrlState.ts | 20 +- frontend/src/lib/api/admin.ts | 9 +- frontend/src/lib/auditText.dom.test.ts | 1 + frontend/src/lib/navRing.contract.test.ts | 13 +- frontend/src/lib/queryKeys.ts | 8 +- frontend/src/lib/urlState.ts | 39 ++- 44 files changed, 1101 insertions(+), 426 deletions(-) create mode 100644 backend/app/routes/_account.py create mode 100644 backend/tests/test_runstatus.py diff --git a/.env.example b/.env.example index 376f9315..3875dac1 100644 --- a/.env.example +++ b/.env.example @@ -68,9 +68,11 @@ SMTP_FROM= # --- Scheduler --- # The background scheduler (subscription sync, backfill, enrichment) runs inside the app. -# DATABASE_URL is set for you by docker-compose to the bundled `db` service. If you ever run -# more than one instance against the same database, keep SCHEDULER_ENABLED=true on exactly one -# of them and false on the rest, to avoid double quota use and write races. +# DATABASE_URL is set for you by docker-compose to the bundled `db` service. Run ONE app instance +# per database: a starting instance closes every run still open as interrupted (it assumes the +# previous process is gone), and "is this job already running" is held in the process. A second +# instance would cut the first one's runs short and could run the same job twice. +# SCHEDULER_ENABLED=false keeps the recurring jobs off; "run now" still works. SCHEDULER_ENABLED=true # --- Files library & downloads --- diff --git a/backend/alembic/versions/0109_job_runs.py b/backend/alembic/versions/0109_job_runs.py index c642a720..1def28b4 100644 --- a/backend/alembic/versions/0109_job_runs.py +++ b/backend/alembic/versions/0109_job_runs.py @@ -12,7 +12,8 @@ log writer runs behind the event). Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the -audit log. The downgrade moves every run back into the audit log as such a row, then drops the table. +audit log. The downgrade moves every run back into the audit log as such a row (one still running as +interrupted, a run's trigger and target in its `after`), then drops the table. """ import sqlalchemy as sa @@ -70,15 +71,19 @@ def upgrade() -> None: postgresql_where=sa.text("run_id IS NOT NULL"), ) - # The audit trail's runs move over. A status the run table does not know (none was ever - # written) would fail the CHECK, so the move is limited to the three the scheduler wrote. + # The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`, + # which a downgrade writes back (as it does a run's `trigger` and `target`, which the audit row + # has no column for), so a rollback and a redeploy keep every run. A status the run table does + # not know would fail the CHECK. op.execute( """ - INSERT INTO job_runs (job, trigger, actor_id, status, started_at, finished_at, summary, - result, error, failure) + INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at, + summary, result, error, failure) SELECT target_id, - CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END, + coalesce(after::jsonb ->> 'trigger', + CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END), actor_id, + after::jsonb ->> 'target', after::jsonb ->> 'status', created_at, created_at, @@ -89,7 +94,7 @@ def upgrade() -> None: FROM audit_log WHERE action = 'job_run' AND target_id IS NOT NULL - AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped') + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted') ORDER BY id """ ) @@ -98,12 +103,15 @@ def upgrade() -> None: DELETE FROM audit_log WHERE action = 'job_run' AND target_id IS NOT NULL - AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped') + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted') """ ) def downgrade() -> None: + # A run still `running` is one the rollback stopped: it goes back as interrupted. Its trigger and + # target ride in `after` when the audit row's own columns cannot say them (an ad-hoc run of one + # channel), so the upgrade restores them. op.execute( """ INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at) @@ -111,21 +119,26 @@ def downgrade() -> None: 'job_run', 'job', job, - left(job || ': ' || status - || coalesce(' — ' || CASE WHEN status = 'error' THEN error ELSE summary END, ''), + left(job || ': ' || st + || coalesce(' — ' || CASE WHEN st = 'error' THEN error ELSE summary END, ''), 255), (jsonb_build_object( - 'status', status, - 'summary', CASE WHEN status = 'error' THEN error ELSE summary END) + 'status', st, + 'summary', CASE WHEN st = 'error' THEN error ELSE summary END) || CASE - WHEN status = 'error' AND failure IS NOT NULL + WHEN st = 'error' AND failure IS NOT NULL THEN jsonb_build_object('result', failure) - WHEN status <> 'error' AND result IS NOT NULL + WHEN st <> 'error' AND result IS NOT NULL THEN jsonb_build_object('result', result) ELSE '{}'::jsonb - END)::json, + END + || CASE WHEN trigger = 'ad_hoc' + THEN jsonb_build_object('trigger', trigger) ELSE '{}'::jsonb END + || CASE WHEN target IS NOT NULL + THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json, coalesce(finished_at, started_at) - FROM job_runs + FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st + FROM job_runs) AS r ORDER BY id """ ) diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py index 18645366..a51249ae 100644 --- a/backend/app/jobruns.py +++ b/backend/app/jobruns.py @@ -5,13 +5,14 @@ OWN short session, committed at once, for the reason `quota.record_usage` gives: may roll back (its `except: db.rollback()`), and the history of what happened must not go with it — nor may this bookkeeping commit the job's half-done session. -Only the API process runs jobs (the worker never starts the scheduler), so at its startup nothing can -still be running: `interrupt_leftovers` closes what a stopped process left open. +Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a +database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its +startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open. """ from datetime import datetime, timedelta, timezone -from sqlalchemy import delete, func, select, update +from sqlalchemy import delete, func, select, union_all, update from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session @@ -70,30 +71,46 @@ def finish( def interrupt_leftovers() -> int: - """Every row a previous process left `running` → `interrupted`. Called once at startup, before - the scheduler can open a new run. Returns how many.""" + """Every row a previous process left `running` → `interrupted`, with what it spent until then + (its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler + can open a new run. Returns how many.""" + spent = ( + select(func.coalesce(func.sum(QuotaEvent.units), 0)) + .where(QuotaEvent.job_run_id == JobRun.id) + .scalar_subquery() + ) with SessionLocal() as s: res = s.execute( update(JobRun) .where(JobRun.status == RunStatus.RUNNING) - .values(status=RunStatus.INTERRUPTED, finished_at=datetime.now(timezone.utc)) + .values( + status=RunStatus.INTERRUPTED, + finished_at=datetime.now(timezone.utc), + quota_units=spent, + ) ) s.commit() return res.rowcount if isinstance(res, CursorResult) else 0 -def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]: - """Each job's last `per_job` runs, newest first, by job id — one query (a window per job).""" - ranked = select( - JobRun.id, - func.row_number() - .over(partition_by=JobRun.job, order_by=(JobRun.started_at.desc(), JobRun.id.desc())) - .label("rank"), +def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]: + """Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a + short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window + over the whole table grew with every run kept).""" + if not jobs: + return {} + newest = union_all( + *( + select(JobRun.id) + .where(JobRun.job == job) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(per_job) + for job in jobs + ) ).subquery() rows = db.execute( select(JobRun) - .join(ranked, ranked.c.id == JobRun.id) - .where(ranked.c.rank <= per_job) + .where(JobRun.id.in_(select(newest.c.id))) .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc()) ).scalars() out: dict[str, list[JobRun]] = {} @@ -102,6 +119,16 @@ def recent_runs(db: Session, per_job: int) -> dict[str, list[JobRun]]: return out +def running_one(db: Session, job: str) -> JobRun | None: + """The newest run of `job` still going, however many runs ended after it started.""" + return db.scalars( + select(JobRun) + .where(JobRun.job == job, JobRun.status == RunStatus.RUNNING) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(1) + ).first() + + def gc(db: Session, retention_days: int) -> int: """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is never reaped. The caller commits.""" diff --git a/backend/app/models.py b/backend/app/models.py index d9914a0f..9ebfbe74 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -720,7 +720,7 @@ class JobRun(Base): CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"), # Only a run still going has no end. CheckConstraint( - "(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished" + f"(status = '{RunStatus.RUNNING}') = (finished_at IS NULL)", name="ck_job_runs_finished" ), ) diff --git a/backend/app/routes/_account.py b/backend/app/routes/_account.py new file mode 100644 index 00000000..84976e86 --- /dev/null +++ b/backend/app/routes/_account.py @@ -0,0 +1,17 @@ +"""The account filter the activity page's lists share (runs, admin actions, server log, quota): one +filter bar drives them all, so the value it sends means the same on every tab. + +The value is an account id, or `none` — the rows no account made (the background) — a value to filter +on, not the absence of a filter. +""" + +from sqlalchemy import ColumnElement + +NONE = "none" +# The `user` query parameter's shape on every route that takes it. +PATTERN = r"^(none|[0-9]{1,10})$" + + +def condition(column, value: str) -> ColumnElement[bool]: + """`column` (an account id column) narrowed to `value`.""" + return column.is_(None) if value == NONE else column == int(value) diff --git a/backend/app/routes/admin_logs.py b/backend/app/routes/admin_logs.py index 6ec025cf..72a36631 100644 --- a/backend/app/routes/admin_logs.py +++ b/backend/app/routes/admin_logs.py @@ -25,6 +25,7 @@ from app import logsink from app.auth import admin_user from app.db import get_db from app.models import LogEvent, User +from app.routes import _account as account from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.utils import like_escape @@ -37,9 +38,9 @@ FACET_EVENTS_MAX = 100 Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # The filters a facet can leave out of its own count (`LogFilters.conditions`). Facet = Literal["levels", "sources", "user", "event"] -# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous -# request) — a real value to filter on, not the absence of a filter. -NONE = "none" +# `none` selects the rows that have no event code, as the account filter's `none` (app/routes/_account) +# selects the rows of no account — a real value to filter on, not the absence of a filter. +NONE = account.NONE @dataclass(frozen=True) @@ -67,11 +68,7 @@ class LogFilters: if self.sources and skip != "sources": out.append(LogEvent.source.in_(self.sources)) if self.user is not None and skip != "user": - out.append( - LogEvent.user_id.is_(None) - if self.user == NONE - else LogEvent.user_id == int(self.user) - ) + out.append(account.condition(LogEvent.user_id, self.user)) if self.event is not None and skip != "event": out.append( LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event @@ -103,7 +100,7 @@ class LogFilters: def log_filters( levels: list[Level] = Query(default=[]), sources: list[str] = Query(default=[], max_length=8), - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), event: str | None = Query( default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$" ), diff --git a/backend/app/routes/audit.py b/backend/app/routes/audit.py index bc0c8986..91e82c78 100644 --- a/backend/app/routes/audit.py +++ b/backend/app/routes/audit.py @@ -15,17 +15,15 @@ from app import audit from app.audit import AuditAction from app.auth import admin_user from app.db import get_db -from app.models import AuditLog, User +from app.models import AuditLog, LogEvent, User +from app.routes import _account as account from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.utils import iso, like_escape router = APIRouter(prefix="/api/admin/audit", tags=["admin"]) -# `none` selects the rows no account made (the background) — a value to filter on, not "any". -NONE = "none" - -def _serialize(row: AuditLog) -> dict: +def _serialize(row: AuditLog, lines: dict[str, int]) -> dict: return { "id": row.id, "created_at": iso(row.created_at), @@ -38,6 +36,9 @@ def _serialize(row: AuditLog) -> dict: "before": row.before, "after": row.after, "request_id": row.request_id, + # How many of that request's log lines are still kept: they go after days, the row after + # months, and a link to none would open an empty list. + "log_lines": lines.get(row.request_id, 0) if row.request_id else 0, } @@ -56,11 +57,7 @@ class AuditFilters: if self.action is not None and skip != "action": out.append(AuditLog.action == self.action) if self.actor is not None: - out.append( - AuditLog.actor_id.is_(None) - if self.actor == NONE - else AuditLog.actor_id == int(self.actor) - ) + out.append(account.condition(AuditLog.actor_id, self.actor)) if self.since is not None: out.append(AuditLog.created_at >= utc(self.since)) if self.until is not None: @@ -82,7 +79,7 @@ class AuditFilters: def audit_filters( action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"), - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), q: str | None = Query(default=None, max_length=200), since: datetime | None = None, until: datetime | None = None, @@ -109,8 +106,17 @@ def list_audit( limit, lambda r: r.created_at, ) + requests = {r.request_id for r in rows if r.request_id} + lines: dict[str, int] = {} + if requests: + counted = db.execute( + select(LogEvent.request_id, func.count()) + .where(LogEvent.request_id.in_(requests)) + .group_by(LogEvent.request_id) + ) + lines = {rid: n for rid, n in counted.tuples() if rid is not None} return { - "items": [_serialize(r) for r in rows], + "items": [_serialize(r, lines) for r in rows], "accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}), "next_cursor": cursor, } diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index d0856c94..3c1e753c 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -512,7 +512,9 @@ def update_channel( # so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. # The request returns right away; the channel page's live poll shows the count climb. The # scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set). - if deep_turned_on: + # A channel whose history is all in gets no run: it would only say "nothing to do", one kept row + # per flip of the flag. + if deep_turned_on and not (channel is not None and channel.backfill_done): scheduler.run_ad_hoc( scheduler.CHANNEL_BACKFILL, partial(backfill_channel_now, channel_id, user.id), diff --git a/backend/app/routes/quota.py b/backend/app/routes/quota.py index 1f4800c7..98df6d07 100644 --- a/backend/app/routes/quota.py +++ b/backend/app/routes/quota.py @@ -14,6 +14,7 @@ from app import quota from app.auth import admin_user, current_user from app.db import get_db from app.models import QuotaEvent, User +from app.routes import _account as account from app.routes._keyset import utc router = APIRouter(prefix="/api/quota", tags=["quota"]) @@ -48,7 +49,7 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) def admin_usage( since: datetime | None = None, until: datetime | None = None, - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), _: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: @@ -60,9 +61,7 @@ def admin_usage( if until is not None: window.append(QuotaEvent.created_at < utc(until)) if user is not None: - window.append( - QuotaEvent.user_id.is_(None) if user == "none" else QuotaEvent.user_id == int(user) - ) + window.append(account.condition(QuotaEvent.user_id, user)) totals = db.execute( select(QuotaEvent.user_id, func.sum(QuotaEvent.units)) diff --git a/backend/app/routes/runs.py b/backend/app/routes/runs.py index c39fd244..c1bb53d6 100644 --- a/backend/app/routes/runs.py +++ b/backend/app/routes/runs.py @@ -19,14 +19,13 @@ from app import scheduler from app.auth import admin_user from app.db import get_db from app.models import Channel, JobRun, LogEvent, User +from app.routes import _account as account from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.runstatus import RunStatus from app.utils import iso, like_escape router = APIRouter(prefix="/api/admin/runs", tags=["admin"]) -# `none` selects the runs no account started (the scheduled ones). -NONE = "none" Facet = Literal["job", "status"] @@ -68,11 +67,7 @@ class RunFilters: if self.status is not None and skip != "status": out.append(JobRun.status == self.status) if self.actor is not None: - out.append( - JobRun.actor_id.is_(None) - if self.actor == NONE - else JobRun.actor_id == int(self.actor) - ) + out.append(account.condition(JobRun.actor_id, self.actor)) if self.since is not None: out.append(JobRun.started_at >= utc(self.since)) if self.until is not None: @@ -93,7 +88,7 @@ class RunFilters: def run_filters( job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"), status: RunStatus | None = None, - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), q: str | None = Query(default=None, max_length=200), since: datetime | None = None, until: datetime | None = None, diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index c61f3305..db0f5308 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -17,8 +17,9 @@ from app.scheduler import ( JOB_INTERVALS, MAX_INTERVAL, MIN_INTERVAL, - apply_interval, + reschedule, scheduler_snapshot, + stage_interval, trigger_all, trigger_job, ) @@ -42,14 +43,14 @@ def set_job_interval( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - """Change how often a scheduler job runs (minutes). Persisted as an override and applied - to the live scheduler immediately.""" + """Change how often a scheduler job runs (minutes). Stored as an override together with its + audit row, then applied to the live scheduler.""" if job_id not in JOB_INTERVALS: raise not_found("job") minutes = body.interval_minutes if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL): raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL) - applied = apply_interval(db, job_id, minutes) + applied = stage_interval(db, job_id, minutes) audit.record( db, admin.id, @@ -60,6 +61,7 @@ def set_job_interval( after={"interval_minutes": applied}, ) db.commit() + reschedule(job_id, applied) return {"ok": True, "id": job_id, "interval_minutes": applied} diff --git a/backend/app/runstatus.py b/backend/app/runstatus.py index 497a22b1..cba6c896 100644 --- a/backend/app/runstatus.py +++ b/backend/app/runstatus.py @@ -1,9 +1,11 @@ """The words a background run's row is written with — the `job_runs` table's status and trigger. One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the -model's CHECK constraints are built from these sets, and the dashboard reads them back. A run's -status is never typed as a string anywhere else (the download pipeline's `cancelled`/`canceled` -drift, `downloads/status.py`, is why). +model's CHECK constraints are built from these sets, and the dashboard reads them back (the frontend's +copies are held to them by `backendIds.contract.test.ts`). The migrated database's CHECKs are frozen +in its migration: a test holds them equal to these sets, so a new word fails until a migration adds +it (`test_runstatus.py`). The download pipeline's `cancelled`/`canceled` drift (`downloads/status.py`) +is why a status is never typed as a bare string. """ from enum import StrEnum diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index e528ed44..29904095 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -3,7 +3,7 @@ recent-first / deep backfill. Runs inside the API process (single worker).""" import contextvars import threading -from datetime import datetime, timezone +from datetime import timezone from typing import Callable from apscheduler.executors.pool import ThreadPoolExecutor @@ -27,6 +27,7 @@ from app.sync.autotag import run_autotag_all from app.sync.explore import purge_ephemeral from app.sync.maintenance import run_maintenance from app.sync.playlists import sync_all_playlists +from app.utils import iso from app.sync.runner import ( run_deep_backfill, run_enrich, @@ -67,18 +68,21 @@ def load_intervals(db) -> dict[str, int]: return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS} -def apply_interval(db, job_id: str, minutes: int) -> int: - """Persist a job's interval override and reschedule the live job immediately (if the - scheduler runs in this process). Returns the applied value.""" +def stage_interval(db, job_id: str, minutes: int) -> int: + """Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it + with its audit row, then calls `reschedule` — the live job follows only what was stored.""" if job_id not in JOB_INTERVALS: raise KeyError(job_id) minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes))) sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes) - db.commit() + return minutes + + +def reschedule(job_id: str, minutes: int) -> None: + """Move the live interval job to `minutes` (if this process registered it).""" if _scheduler is not None and _scheduler.get_job(job_id) is not None: _scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes) log.info("rescheduled job %s to every %s min", job_id, minutes) - return minutes def _record(name: str, **fields) -> None: @@ -87,7 +91,12 @@ def _record(name: str, **fields) -> None: def _notify_done( - db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None + db, + actor_id: int, + job_id: str, + status: RunStatus, + summary: str | None, + result: dict | None = None, ) -> None: """Post a completion notice to the triggering admin's inbox (manual runs only). Best effort — a notification failure must never mask the job's own outcome.""" @@ -139,24 +148,40 @@ def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None: def sink(current, total, phase): _record(name, progress={"current": current, "total": total, "phase": phase}) + # Only the work is inside the try: writing its outcome is not the job, and a write that fails + # must not turn a run that succeeded into a failed one (see _close). + result = None try: - if is_sync_paused(db): - log.info("job %s skipped (sync paused)", name) - jobruns.finish(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED) - if actor_id is not None: - _notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED) - return - with progress.bind(sink): - result = fn(db) - summary, data = _finish_ok(name, run_id, result) - if actor_id is not None: - _notify_done(db, actor_id, name, "ok", summary, data) + paused = is_sync_paused(db) + if not paused: + with progress.bind(sink): + result = fn(db) except Exception as exc: db.rollback() log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) - err, failure = _finish_error(run_id, exc) + err, failure = _failure(exc) + _close(run_id, RunStatus.ERROR, error=err, failure=failure) if actor_id is not None: - _notify_done(db, actor_id, name, "error", err, failure) + _notify_done(db, actor_id, name, RunStatus.ERROR, err, failure) + return + if paused: + log.info("job %s skipped (sync paused)", name) + _close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED) + if actor_id is not None: + _notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED) + return + summary, data = _finish_ok(name, run_id, result) + if actor_id is not None: + _notify_done(db, actor_id, name, RunStatus.OK, summary, data) + + +def _close(run_id: int, status: RunStatus, **outcome) -> None: + """Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's: + the row then stays `running` until the next start's sweep closes it as interrupted.""" + try: + jobruns.finish(run_id, status, **outcome) + except Exception: + log.exception("run %s: its outcome (%s) could not be recorded", run_id, status) def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]: @@ -164,16 +189,15 @@ def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None] summary = _summarize(result) data = _result_data(result) log.info("job %s -> %s", name, result) - jobruns.finish(run_id, RunStatus.OK, summary=summary, result=data) + _close(run_id, RunStatus.OK, summary=summary, result=data) return summary, data -def _finish_error(run_id: int, exc: Exception) -> tuple[str, dict | None]: - """Close a run that raised. A failure the job named (progress.JobFailed) rides along as data, as - a result does, so the client says it in the reader's language; the error stays the English line.""" +def _failure(exc: Exception) -> tuple[str, dict | None]: + """A raised run's English error line, and the failure the job named (progress.JobFailed) as + data, as a result is, so the client says it in the reader's language.""" err = str(exc) or exc.__class__.__name__ failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None - jobruns.finish(run_id, RunStatus.ERROR, error=err, failure=failure) return err, failure @@ -201,10 +225,6 @@ def _summarize(result) -> str | None: return str(result) -def _iso(ts: datetime | None) -> str | None: - return ts.astimezone(timezone.utc).isoformat() if ts else None - - def scheduler_snapshot(db) -> dict: """What the scheduler is doing, for the admin dashboard: each job's live state from this process and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how @@ -223,7 +243,7 @@ def scheduler_snapshot(db) -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} - recent = jobruns.recent_runs(db, RECENT_RUNS) + recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) @@ -239,17 +259,19 @@ def scheduler_snapshot(db) -> dict: ) # Work a user starts on one target: no interval, no "run now" (it needs its target), but its # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the - # target's own (sync/runner), so "running" is what its newest row says. + # target's own (sync/runner), so "running" is what its rows say: two channels can backfill at + # once, so a run still going counts even when a newer one already ended, and the row names it. backfills = recent.get(CHANNEL_BACKFILL, []) - newest = backfills[0] if backfills else None + going = jobruns.running_one(db, CHANNEL_BACKFILL) + shown = going or (backfills[0] if backfills else None) ad_hoc = [ { "id": CHANNEL_BACKFILL, - "running": newest is not None and newest.status == RunStatus.RUNNING, - "target": newest.target if newest else None, + "running": going is not None, + "target": shown.target if shown else None, "target_title": ( - db.scalar(select(Channel.title).where(Channel.id == newest.target)) - if newest is not None and newest.target + db.scalar(select(Channel.title).where(Channel.id == shown.target)) + if shown is not None and shown.target else None ), **_last_runs(backfills), @@ -275,8 +297,8 @@ def _last_runs(runs: list) -> dict: ended = run if run is not None and run.status != RunStatus.RUNNING else None return { "status": ended.status if ended else None, - "last_started": _iso(run.started_at) if run else None, - "last_finished": _iso(run.finished_at) if run else None, + "last_started": iso(run.started_at) if run else None, + "last_finished": iso(run.finished_at) if run else None, "last_result": ended.summary if ended else None, # The outcome as data: the result, or on a failed run the failure the job named. "result": _outcome_data(ended), @@ -293,8 +315,8 @@ def _run_brief(run) -> dict: "id": run.id, "status": run.status, "trigger": run.trigger, - "started_at": _iso(run.started_at), - "finished_at": _iso(run.finished_at), + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), } @@ -575,11 +597,15 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return with logs.bind(run_id=run_id): + # As in _run: only the work is inside the try. try: - _finish_ok(job, run_id, fn()) + result = fn() except Exception as exc: log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) - _finish_error(run_id, exc) + err, failure = _failure(exc) + _close(run_id, RunStatus.ERROR, error=err, failure=failure) + return + _finish_ok(job, run_id, result) # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of @@ -590,7 +616,8 @@ _POOL_SIZE = len(JOB_FUNCS) + 8 def start_scheduler() -> None: """Start this process's scheduler. It always starts, so "run now" works on every install; only with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can - run, every run a previous process left open is closed as interrupted.""" + run, every run a previous process left open is closed as interrupted — one API process serves a + database (app/jobruns.py), so an open run here is a dead process's.""" global _scheduler if _scheduler is not None: return diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index bf6ef7f8..6ad61490 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -29,7 +29,9 @@ _coerce_warned: set[tuple[str, str]] = set() # When a changed value takes effect. "live": the next read obeys it — every consumer reads the key -# where it uses it, never once at import or boot (test-held). "restart": the process has to start +# where it uses it, never once at import (test-held: a read outside a function fails). A value read +# once at startup is live only when its writer also applies it (the job intervals reschedule, the log +# level is set as it is saved). "restart": the process has to start # again. No key needs a restart today, and the Configuration page has no way to say so yet, so a # "restart" key is refused until the page can (test-held). Applies = Literal["live", "restart"] diff --git a/backend/tests/test_activity_reads.py b/backend/tests/test_activity_reads.py index f5b3f69c..78e97ef7 100644 --- a/backend/tests/test_activity_reads.py +++ b/backend/tests/test_activity_reads.py @@ -7,7 +7,10 @@ with rows that share a timestamp — the case an offset or a timestamp-only curs from datetime import datetime, timedelta, timezone +from app.auth import admin_user +from app.main import app from app.models import AuditLog, Channel, JobRun, LogEvent, QuotaEvent +from tests.test_body_limit import _route_objects T0 = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc) @@ -88,6 +91,32 @@ def test_an_admin_action_keeps_the_request_it_was_made_in(db, make_user, as_user assert row["request_id"] == r.headers["x-request-id"] +def test_an_admin_action_says_how_many_of_its_requests_lines_are_still_kept(db, make_user, as_user): + """A request's lines are kept days, its audit row months: a link to lines gone would open an + empty list, so the row says how many are left, as a run does.""" + admin = make_user(role="admin") + kept = _audit(db, "config_set", actor=admin.id, request_id="req-kept-0001") + gone = _audit(db, "config_set", actor=admin.id, request_id="req-gone-0001") + db.add_all( + [ + LogEvent( + ts=T0, + level="INFO", + sev=20, + source="api", + logger="t", + msg="m", + request_id="req-kept-0001", + ) + for _ in range(3) + ] + ) + db.commit() + items = as_user(admin).get("/api/admin/audit").json()["items"] + lines = {r["id"]: r["log_lines"] for r in items} + assert (lines[kept], lines[gone]) == (3, 0) + + def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, make_user, as_user): admin = make_user(role="admin") db.add(Channel(id="UCchan", title="Aileen Wang")) @@ -146,6 +175,8 @@ def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_ def totals(**params) -> dict: body = client.get("/api/quota/admin", params=params).json() + # The chart is the same filter's: its days add up to the table's total. + assert sum(d["total"] for d in body["daily"]) == sum(r["total"] for r in body["rows"]) return {r["user_id"]: r["total"] for r in body["rows"]} assert totals() == {admin.id: 150, None: 7} # the last 30 days by default @@ -153,3 +184,22 @@ def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_ assert totals(until=(now - timedelta(days=1)).isoformat()) == {admin.id: 50} assert totals(user="none") == {None: 7} assert totals(user=str(admin.id)) == {admin.id: 150} + + +def test_every_admin_read_and_write_is_admin_only(): + """The route table, not a list kept by hand: every /api/admin route and the quota breakdown + resolve `admin_user`, so a dropped dependency cannot hand an account's runs, errors and e-mail + addresses to any signed-in user.""" + routes = _route_objects(app) + admin = {k: r for k, r in routes.items() if k[1].startswith(("/api/admin", "/api/quota/admin"))} + assert ("GET", "/api/admin/runs") in admin and ("GET", "/api/quota/admin") in admin + open_ = [] + for key, route in admin.items(): + stack, calls = [route.dependant], set() + while stack: + d = stack.pop() + calls.add(d.call) + stack.extend(d.dependencies) + if admin_user not in calls: + open_.append(key) + assert open_ == [] diff --git a/backend/tests/test_audit_count_rows.py b/backend/tests/test_audit_count_rows.py index cdf47d6b..a2a2daf2 100644 --- a/backend/tests/test_audit_count_rows.py +++ b/backend/tests/test_audit_count_rows.py @@ -1,6 +1,6 @@ """An admin action that counts something keeps the count as data in its audit row's `after`, under the key the audit log names in the reader's language (`scheduler.result.`, frontend -auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word. +lib/jobResult.ts, called from lib/auditText.ts) — not only in the English `summary`. A renamed key would fall back to the raw word. DB lane (Postgres). """ diff --git a/backend/tests/test_channel_backfill.py b/backend/tests/test_channel_backfill.py index 14dfd41d..6e00567c 100644 --- a/backend/tests/test_channel_backfill.py +++ b/backend/tests/test_channel_backfill.py @@ -136,6 +136,19 @@ def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel( assert calls == [(ch.id, me.id)] +def test_turning_full_history_on_for_a_finished_channel_records_no_run( + db, monkeypatch, override_dep +): + """Its whole history is in already: a run would only say "nothing to do", and switching the flag + off and on again would otherwise write one such row each time, kept for months.""" + ch, me = _setup_channel_and_sub(db, email="done@example.com") + ch.backfill_done = True + db.commit() + resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) + assert resp.status_code == 200, resp.text + assert (submitted, calls) == ([], []) + + class TestImmediateBackfillGuard: """backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per- channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a diff --git a/backend/tests/test_migration_0109.py b/backend/tests/test_migration_0109.py index 9b19b05c..a597e0d7 100644 --- a/backend/tests/test_migration_0109.py +++ b/backend/tests/test_migration_0109.py @@ -96,3 +96,36 @@ def test_the_audit_logs_runs_move_into_the_run_table_and_back(db, make_user): ), (uid, "backfill", {"status": "skipped", "summary": "sync paused"}), ] + + +def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): + """A rollback then a redeploy: an interrupted run, a run caught mid-way and a user's run of one + channel come back as runs — not stranded in the audit log under an action no code knows.""" + uid = make_user().id + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at," + " finished_at) VALUES" + " ('backfill', 'scheduled', NULL, NULL, 'interrupted', now(), now())," + " ('maintenance', 'manual', :u, NULL, 'running', now(), NULL)," + " ('channel_backfill', 'ad_hoc', :u, 'UC_x', 'ok', now(), now())" + ), + {"u": uid}, + ) + cfg = Config("alembic.ini") + command.downgrade(cfg, "0108_folder_links") + command.upgrade(cfg, "head") + with engine.connect() as conn: + runs = conn.execute( + text("SELECT job, trigger, actor_id, target, status FROM job_runs ORDER BY id") + ).all() + stranded = conn.execute( + text("SELECT count(*) FROM audit_log WHERE action = 'job_run'") + ).scalar() + assert [tuple(r) for r in runs] == [ + ("backfill", "scheduled", None, None, "interrupted"), + ("maintenance", "manual", uid, None, "interrupted"), # the rollback stopped it + ("channel_backfill", "ad_hoc", uid, "UC_x", "ok"), + ] + assert stranded == 0 diff --git a/backend/tests/test_runstatus.py b/backend/tests/test_runstatus.py new file mode 100644 index 00000000..3cc331fd --- /dev/null +++ b/backend/tests/test_runstatus.py @@ -0,0 +1,27 @@ +"""The run words and the migrated database, DB-lane: the `job_runs` CHECKs are frozen in a migration, +and the model's are built from the enums, so nothing else would notice the two drifting apart until a +write of the new word failed in production.""" + +import re + +from sqlalchemy import text + +from app.runstatus import RunStatus, RunTrigger + + +def _words(definition: str) -> set[str]: + return set(re.findall(r"'([a-z_]+)'", definition)) + + +def test_the_databases_checks_allow_exactly_the_run_words(db): + defs = dict( + db.execute( + text( + "SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE conname LIKE 'ck_job_runs_%'" + ) + ).all() + ) + assert _words(defs["ck_job_runs_status"]) == set(RunStatus) + assert _words(defs["ck_job_runs_trigger"]) == set(RunTrigger) + assert _words(defs["ck_job_runs_finished"]) == {RunStatus.RUNNING} diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 91c0c84b..767aec5d 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -14,6 +14,7 @@ from sqlalchemy import select from app import jobruns, logs, logsink, quota, scheduler, state, sysconfig from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent +from app.routes import scheduler as scheduler_routes from app.runstatus import RunStatus, RunTrigger _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs @@ -256,6 +257,9 @@ def test_retention_reaps_finished_runs_only(db): ] ) db.commit() + reaped = next(r for r in _runs(db) if r.status == "ok" and r.started_at < old + timedelta(1)) + with logs.bind(run_id=reaped.id): + quota.record_usage(7) assert jobruns.gc(db, 0) == 0 # 0 = keep forever assert jobruns.gc(db, 30) == 1 db.commit() @@ -263,6 +267,8 @@ def test_retention_reaps_finished_runs_only(db): ("backfill", "running"), ("rss_poll", "ok"), ] + # Its spend stays in the budget's history (kept on its own retention), no longer anyone's run. + assert db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() == [(7, None)] # ---- "run now" through the scheduler -------------------------------------------------------------- @@ -413,6 +419,94 @@ def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch): assert run.status == "interrupted" +def test_an_interrupted_run_keeps_what_it_spent(db): + """Its spend is in the quota history, tagged with its id: the sweep sums it as `finish` would.""" + run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None) + with logs.bind(run_id=run_id): + quota.record_usage(40) + quota.record_usage(2) + assert jobruns.interrupt_leftovers() == 1 + (run,) = _runs(db, "backfill") + assert (run.status, run.quota_units) == ("interrupted", 42) + + +def _flaky_finish(monkeypatch) -> list: + """`jobruns.finish` whose first write fails (the database blinked); the statuses it was asked for.""" + calls: list = [] + real = jobruns.finish + + def finish(run_id, status, **outcome): + calls.append(status) + if len(calls) == 1: + raise RuntimeError("db blip") + real(run_id, status, **outcome) + + monkeypatch.setattr(jobruns, "finish", finish) + return calls + + +def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_user, monkeypatch): + """The job succeeded; only its bookkeeping did not. Writing ERROR then would say the work failed.""" + calls = _flaky_finish(monkeypatch) + scheduler._job("autotag", lambda _db: {"tagged": 1}) + assert calls == ["ok"] + assert not scheduler._is_running("autotag") + (run,) = _runs(db, "autotag") + assert run.status == "running" # left for the next start's sweep, not rewritten as a failure + + user = make_user("blip@example.com") + calls.clear() + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") + assert calls == ["ok"] + + +def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch): + def broken(*_a, **_k): + raise RuntimeError("db down") + + monkeypatch.setattr(jobruns, "start", broken) + scheduler._job("autotag", lambda _db: pytest.fail("no row, no run")) + assert not scheduler._is_running("autotag") + monkeypatch.undo() + scheduler._job("autotag", lambda _db: {"tagged": 1}) + assert [r.status for r in _runs(db, "autotag")] == ["ok"] + + +def test_the_page_says_where_the_interval_jobs_do_not_run(db, live_scheduler, monkeypatch): + """The scheduler starts on every install now; only SCHEDULER_ENABLED says the jobs run here.""" + assert scheduler.scheduler_snapshot(db)["running_here"] is False + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True) + assert scheduler.scheduler_snapshot(db)["running_here"] is True + + +def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_user): + """Two channels can backfill at once; a later run that ended at once must not hide the first.""" + user = make_user("two@example.com") + db.add(Channel(id="UC_long", title="Long one")) + db.commit() + jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_long") + scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short") + (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one") + + +def test_an_interval_change_lands_with_its_audit_row_or_not_at_all( + db, make_user, as_user, live_scheduler, monkeypatch +): + live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill") + admin = as_user(make_user(role="admin")) + + def broken(*_a, **_k): + raise RuntimeError("audit write failed") + + monkeypatch.setattr(scheduler_routes.audit, "record", broken) + with pytest.raises(RuntimeError): # the handler raises: there is no response to check + _ = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45}) + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == scheduler.JOB_INTERVALS["backfill"] + assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=10) + + def test_every_job_has_an_interval_and_every_interval_a_job(): assert set(scheduler.JOB_FUNCS) == set(scheduler.JOB_INTERVALS) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index d9febfdf..8ea9f4c0 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -275,10 +275,9 @@ Also in force: - **Page chrome (B4):** a page's tabs and filters sit in ONE sticky band of the page colour, with the fade under it (below). Below ~600px of height the filter row folds into one "Filters · n" button. Every page has a visually hidden `h1`; the nav rail and the browser tab title are the visible identity. [WCAG 2.4.6] - Built as: the band is `PageShell`'s toolbar slot (`PageToolbar`), above the one scroller; a console - whose toolbar lives inside its frame (the server log) makes that toolbar `sticky` with `fade-below` - instead. The band holds controls only: an intro paragraph scrolls with the content (in the band it - ate three lines of a short window). The fold is `useFilterFold` + `FilterFoldButton`: the same row + Built as: the band is `PageShell`'s toolbar slot (`PageToolbar`), above the one scroller. The band + holds controls only: an intro paragraph scrolls with the content (in the band it ate three lines of a + short window). The fold is `useFilterFold` + `FilterFoldButton`: the same row shown or hidden in place, never copied into a popover. The hidden `h1` is `PageShell`'s `title`; a view with its own visible title (a channel, a Plex show, a playlist) renders it as `PageHeading` and the hidden one steps aside, so a page has exactly one `h1` (a hand `

` inside the shell is diff --git a/frontend/src/components/Activity.dom.test.tsx b/frontend/src/components/Activity.dom.test.tsx index d16478c4..e74f7fb2 100644 --- a/frontend/src/components/Activity.dom.test.tsx +++ b/frontend/src/components/Activity.dom.test.tsx @@ -6,7 +6,7 @@ // the account filter reaches the quota tab; a // failed read is an error with a retry, never an empty list. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import type { AuditQuery, AuditRow, HistoryPage, LogQuery, RunQuery, RunRow } from "../lib/api"; @@ -23,6 +23,8 @@ const runsMock = vi.fn<(q: RunQuery, before: string | null) => Promise Promise>>(); const logsMock = vi.fn<(q: LogQuery, before: string | null) => Promise>(); const quotaMock = vi.fn<(since: string, user: string | null) => Promise>(); +const runFacetsMock = vi.fn<() => Promise>(); +const usersMock = vi.fn<() => Promise>(); vi.mock("../lib/api", async (importOriginal) => { const actual = await importOriginal(); return { @@ -30,19 +32,21 @@ vi.mock("../lib/api", async (importOriginal) => { api: { ...actual.api, adminRuns: (q: RunQuery, before: string | null) => runsMock(q, before), - adminRunFacets: () => Promise.resolve({ jobs: [], statuses: [], total: 0 }), + adminRunFacets: () => runFacetsMock(), adminAudit: (q: AuditQuery, before: string | null) => auditMock(q, before), adminAuditFacets: () => Promise.resolve({ actions: [] }), adminLogs: (q: LogQuery, before: string | null) => logsMock(q, before), adminLogFacets: () => Promise.resolve({ levels: [], sources: [], events: [], users: [], accounts: {}, total: 0 }), - adminUsers: () => Promise.resolve([{ id: 1, email: "admin@example.com" }]), + adminUsers: () => usersMock(), adminQuota: (since: string, user: string | null) => quotaMock(since, user), + clearAudit: () => Promise.resolve({ ok: true, cleared: 1 }), }, }; }); import Activity from "./Activity"; +import { ChoiceContext } from "./confirmContext"; import { PageToolbarSlot } from "./ui/pageToolbarSlot"; // The filter bar lives in the toolbar, which portals into the shell's band: give it one. @@ -77,6 +81,7 @@ const ACTION: AuditRow = { before: null, after: { interval_minutes: 25 }, request_id: "0efdd960c9ad1977", + log_lines: 4, }; const page = (items: T[]): HistoryPage => ({ @@ -89,9 +94,12 @@ function renderPage() { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( - - - + {/* Every confirmation is answered yes. */} + Promise.resolve("confirm")}> + + + + , ); } @@ -107,6 +115,8 @@ beforeEach(() => { auditMock.mockResolvedValue(page([ACTION])); logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); quotaMock.mockResolvedValue({ rows: [], daily: [] }); + runFacetsMock.mockResolvedValue({ jobs: [], statuses: [], total: 0 }); + usersMock.mockResolvedValue([{ id: 1, email: "admin@example.com" }]); }); afterEach(() => { cleanup(); @@ -135,6 +145,52 @@ describe("Activity", () => { expect([url().get("rjob"), url().get("rst")]).toEqual(["backfill", "error"]); }); + it("Older reads the next page from its cursor, and offers none past the last", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockImplementation((_q, before) => + Promise.resolve( + before === null + ? { ...page([RUN]), next_cursor: "c1" } + : page([{ ...RUN, id: 40, error: "the older one" }]), + ), + ); + renderPage(); + fireEvent.click(await screen.findByRole("button", { name: "activity.older" })); + await waitFor(() => expect(screen.getByText("the older one")).toBeTruthy()); + expect(runsMock).toHaveBeenLastCalledWith(expect.anything(), "c1"); + expect(screen.getByText("quotaExceeded")).toBeTruthy(); + expect(screen.queryByRole("button", { name: "activity.older" })).toBeNull(); + }); + + it("an older page that fails says so and keeps what was loaded", async () => { + window.history.replaceState(null, "", "/?page=activity"); + runsMock.mockImplementation((_q, before) => + before === null + ? Promise.resolve({ ...page([RUN]), next_cursor: "c1" }) + : Promise.reject(new Error("500")), + ); + renderPage(); + fireEvent.click(await screen.findByRole("button", { name: "activity.older" })); + const alert = await screen.findByRole("alert"); + expect(alert.textContent).toContain("activity.olderFailed"); + expect(screen.getByText("quotaExceeded")).toBeTruthy(); + }); + + it("offers no older page while a filter change is still loading", async () => { + window.history.replaceState(null, "", "/?page=activity"); + // The cursor on screen is the previous filter's: it would continue the wrong list. + runsMock.mockImplementation((q) => + q.status ? new Promise(() => {}) : Promise.resolve({ ...page([RUN]), next_cursor: "c1" }), + ); + renderPage(); + await screen.findByRole("button", { name: "activity.older" }); + fireEvent.click(screen.getByRole("button", { name: /activity\.runs\.status\.error/ })); + await waitFor(() => + expect(screen.queryByRole("button", { name: "activity.older" })).toBeNull(), + ); + expect(screen.getByText("quotaExceeded")).toBeTruthy(); // the old rows stay, dimmed + }); + it("a run opens to its log lines, and Back returns to the runs", async () => { window.history.replaceState({ sfPage: "activity" }, "", "/?page=activity&aw=24h&aq=quota"); renderPage(); @@ -168,6 +224,77 @@ describe("Activity", () => { expect(screen.getByText("quotaExceeded")).toBeTruthy(); }); + it("a run's outcome is its result in the reader's words, its failure in the danger colour", async () => { + runsMock.mockResolvedValue( + page([ + { ...RUN, id: 1, status: "ok", error: null, result: { new: 3 }, summary: "new=3" }, + // Moved over from the audit log before the result rode along as data: the one-liner only. + { ...RUN, id: 2, status: "ok", error: null, result: null, summary: "failed=2" }, + { ...RUN, id: 3, status: "interrupted", error: null }, + { ...RUN, id: 4 }, + ]), + ); + renderPage(); + await waitFor(() => expect(screen.getByText("scheduler.result.new: 3")).toBeTruthy()); + expect(screen.queryByText("new=3")).toBeNull(); + expect(screen.getByText("scheduler.result.failed: 2")).toBeTruthy(); + expect(screen.getByText("scheduler.interrupted").className).toContain("text-warn"); + expect(screen.getByText("quotaExceeded").className).toContain("text-danger"); + }); + + it("filter choices that failed to load say so beside the filters, with a retry", async () => { + runFacetsMock.mockRejectedValueOnce(new Error("500")); + usersMock.mockRejectedValueOnce(new Error("500")); + renderPage(); + await waitFor(() => + expect(screen.getAllByRole("alert").map((a) => a.textContent)).toEqual([ + expect.stringContaining("activity.accountsFailed"), + expect.stringContaining("activity.filtersFailed"), + ]), + ); + for (const retry of screen.getAllByRole("button", { name: "common.retry" })) + fireEvent.click(retry); + await waitFor(() => expect(screen.queryAllByRole("alert")).toEqual([])); + expect(runFacetsMock).toHaveBeenCalledTimes(2); + expect(usersMock).toHaveBeenCalledTimes(2); + }); + + it("clearing the admin actions reads the list again", async () => { + window.history.replaceState(null, "", "/?page=activity&at=admin"); + renderPage(); + await waitFor(() => expect(screen.getByText("auditActions.scheduler_interval")).toBeTruthy()); + auditMock.mockResolvedValue(page([])); + fireEvent.click(screen.getByRole("button", { name: "audit.clear" })); + await waitFor(() => expect(screen.getByText("audit.empty")).toBeTruthy()); + // An emptied list is announced, as the error and loading states are. + expect(screen.getByText("audit.empty").closest('[role="status"]')).toBeTruthy(); + }); + + it("closing a dialog over the server log leaves Live on", async () => { + window.history.replaceState(null, "", "/?page=activity&at=logs"); + renderPage(); + const live = await screen.findByRole("button", { name: "adminLogs.live" }); + fireEvent.click(live); + expect(live.getAttribute("aria-pressed")).toBe("true"); + // A closing dialog steps back over its own history entry: a popstate onto the same address. + act(() => { + window.dispatchEvent(new PopStateEvent("popstate")); + }); + const after = screen.getByRole("button", { name: "adminLogs.live" }); + expect(after.getAttribute("aria-pressed")).toBe("true"); + expect(after).toBe(live); // the same log, not a fresh one + }); + + it("an action whose request's lines are gone says so instead of linking to an empty list", async () => { + auditMock.mockResolvedValue(page([{ ...ACTION, log_lines: 0 }])); + window.history.replaceState(null, "", "/?page=activity&at=admin"); + renderPage(); + await waitFor(() => expect(screen.getByText("auditActions.scheduler_interval")).toBeTruthy()); + fireEvent.click(rowButton(/scheduler_interval/)); + expect(screen.getByText("activity.audit.noRequestLines")).toBeTruthy(); + expect(screen.queryByRole("button", { name: "activity.audit.requestLines" })).toBeNull(); + }); + it("the admin tab lists the actions, and an action opens its request's lines", async () => { window.history.replaceState(null, "", "/?page=activity&at=admin"); renderPage(); @@ -196,6 +323,24 @@ describe("Activity", () => { expect(Math.round(days)).toBe(30); }); + it("each day's spend is readable without a pointer", async () => { + window.history.replaceState(null, "", "/?page=activity&at=quota"); + quotaMock.mockResolvedValue({ + rows: [{ user_id: null, email: "System", total: 9, by_action: {} }], + daily: [ + { day: "2026-10-16", total: 4 }, + { day: "2026-10-17", total: 5 }, + ], + }); + renderPage(); + const chart = await screen.findByRole("list", { name: "activity.quota.daily" }); + expect( + within(chart) + .getAllByRole("listitem") + .map((d) => d.textContent), + ).toEqual(["stats.dailyTooltip", "stats.dailyTooltip"]); + }); + // The runs moved over from the audit log kept one instant for start and end: their length is // unknown, and a "1 sec" would be invented. it("a run whose start and end are one instant has no length", async () => { diff --git a/frontend/src/components/Activity.tsx b/frontend/src/components/Activity.tsx index 79a70cf1..d7ff5364 100644 --- a/frontend/src/components/Activity.tsx +++ b/frontend/src/components/Activity.tsx @@ -29,6 +29,7 @@ import FilterSelect from "./ui/FilterSelect"; import { InnerInput } from "./ui/Input"; import { fieldClass } from "./ui/fieldStyles"; import { PageToolbar } from "./ui/PageShell"; +import { StateMessage } from "./ui/QueryState"; import Tabs from "./ui/Tabs"; import { useTabPanel } from "./ui/tabPanel"; @@ -51,9 +52,12 @@ export default function Activity() { activityParamsToState(new URLSearchParams(window.location.search)), ); // Bumped when the view changes by a jump (a pivot, Back/Forward) rather than by typing: the search - // text then applies at once instead of after the typing debounce, and the server log re-reads its - // own filters from the address bar. + // text then applies at once instead of after the typing debounce. const [jumps, setJumps] = useState(0); + // Bumped by a pivot only: the server log, already open, takes the pivot's filters from the address + // bar when it mounts again. Back/Forward it follows in place (AdminLogs) — a popstate is also a + // dialog closing, onto the same address, and a fresh log would drop its Live and its focus. + const [pivots, setPivots] = useState(0); const q = useDebounced(state.q, 300, jumps); const panel = useTabPanel(); // `page` too: a link to a page this one replaced (audit, logs) is rewritten to this page's id. @@ -81,10 +85,13 @@ export default function Activity() { ); notifyNav(); setJumps((n) => n + 1); + setPivots((n) => n + 1); setState(next); }; const shared = { window: state.window, user: state.user, q }; + // The account menu's list (SharedBar reads the same query): without it the menu offers only "Any". + const users = useQuery({ queryKey: qk.adminUsers(), queryFn: api.adminUsers }); return (
@@ -99,6 +106,11 @@ export default function Activity() {
+ {users.isError && ( + void users.refetch()}> + {t("activity.accountsFailed")} + + )} {/* The quota tab says what its numbers are itself, beside them. */} {state.tab !== "quota" && (

@@ -121,9 +133,9 @@ export default function Activity() { onRequest={(requestId) => pivot({ requestId, jobId: null, runId: null })} /> ) : state.tab === "logs" ? ( - // Keyed by the jumps: a pivot or Back/Forward hands the log its filters through the address - // bar, which it reads when it mounts. - + // Keyed by the pivots: a pivot hands the log its filters through the address bar, which it + // reads when it mounts. + ) : ( )} diff --git a/frontend/src/components/ActivityAudit.tsx b/frontend/src/components/ActivityAudit.tsx index ff6d0ab9..dfdd46b0 100644 --- a/frontend/src/components/ActivityAudit.tsx +++ b/frontend/src/components/ActivityAudit.tsx @@ -13,6 +13,7 @@ import { qk } from "../lib/queryKeys"; import { useHistoryList } from "../lib/useHistoryList"; import type { Shared } from "./ActivityRuns"; import { useConfirm } from "./confirmContext"; +import { StateMessage } from "./ui/QueryState"; import HistoryList from "./HistoryList"; import HistoryRow from "./HistoryRow"; import { RelativeTime } from "./ui/RelativeTime"; @@ -67,8 +68,8 @@ export default function ActivityAudit({ const clear = useMutation({ mutationFn: () => api.clearAudit(), onSuccess: (r) => { - void qc.invalidateQueries({ queryKey: ["admin-audit"] }); - void qc.invalidateQueries({ queryKey: ["admin-audit-facets"] }); + void qc.invalidateQueries({ queryKey: qk.adminAudit() }); + void qc.invalidateQueries({ queryKey: qk.adminAuditFacets() }); notify({ level: "success", message: t("audit.cleared", { count: r.cleared }) }); }, onError: (e) => notifyActionError(e, t("audit.clearFailed")), @@ -115,6 +116,12 @@ export default function ActivityAudit({ {t("audit.clear")}

+ {/* The action menu above is built from these counts: without them it offers only "Any". */} + {facets.isError && ( + void facets.refetch()}> + {t("activity.filtersFailed")} + + )} - {facets.isError &&

{t("activity.countsFailed")}

}
); } @@ -203,17 +209,21 @@ function AuditDetail({ {JSON.stringify({ before: row.before, after: row.after }, null, 2)} )} - {row.request_id && ( -
- -
- )} + {/* A request's lines are kept days, its row months: a link to none would open an empty list. */} + {row.request_id && + (row.log_lines > 0 ? ( +
+ +
+ ) : ( +

{t("activity.audit.noRequestLines")}

+ ))}
); } diff --git a/frontend/src/components/ActivityQuota.tsx b/frontend/src/components/ActivityQuota.tsx index 12698596..7c601a8f 100644 --- a/frontend/src/components/ActivityQuota.tsx +++ b/frontend/src/components/ActivityQuota.tsx @@ -49,7 +49,8 @@ export default function ActivityQuota({ shared }: { shared: Shared }) { )}
- {t("activity.quota.total", { + {t(shared.window === "all" ? "activity.quota.totalAll" : "activity.quota.total", { + count: grandTotal, window: t(`activity.windows.${shared.window}`), units: formatNumber(grandTotal, lang), })} @@ -57,18 +58,33 @@ export default function ActivityQuota({ shared }: { shared: Shared }) { {data.daily.length === 0 ? (
{t("stats.noUsageInRange")}
) : ( -
+ // Each day's figure is in its bar's text too (read out, not shown): the tooltip only + // reaches a pointer. +
{data.daily.map((d) => (
+ + {t("stats.dailyTooltip", { + count: d.total, + day: d.day, + units: formatNumber(d.total, lang), + })} +
+ {/* The menus above are built from these counts: without them they offer only "Any". */} + {facets.isError && ( + void facets.refetch()}> + {t("activity.filtersFailed")} + + )} ))} - {facets.isError &&

{t("activity.countsFailed")}

}
); } @@ -160,7 +166,13 @@ function useOutcome(run: RunRow): { text: string; tone: string } { }; if (run.status === "interrupted") return { text: t("scheduler.interrupted"), tone: "text-warn" }; if (run.status === "running") return { text: t("scheduler.runningNow"), tone: "text-accent" }; - if (run.result) return { text: jobResultText(t, run.result, lang), tone: "text-muted" }; + // A run moved over from the audit log before its result rode along as data holds only the + // server's English one-liner: read back into a result, as the old audit rows were. + const result = + run.result ?? + parseSummary(run.summary) ?? + (run.status === "skipped" && run.summary ? { skipped: run.summary } : null); + if (result) return { text: jobResultText(t, result, lang), tone: "text-muted" }; return { text: run.summary ?? "", tone: "text-muted" }; } diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index e9932427..703dbc76 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -1,4 +1,4 @@ -import { useMemo, useRef, useState } from "react"; +import { useEffect, useEffectEvent, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { keepPreviousData, @@ -18,14 +18,13 @@ import { logsParamsToState, logsParamUpdates, type LogsUrlState } from "../lib/l import { notify } from "../lib/notifications"; import { qk } from "../lib/queryKeys"; import { useUrlMirror } from "../lib/useUrlMirror"; +import { HistoryFrame } from "./HistoryList"; import LogLine from "./LogLine"; -import NavGrid from "./ui/NavGrid"; import { Button } from "./ui/Button"; import FilterChipGroup from "./ui/FilterChipGroup"; import FilterSelect from "./ui/FilterSelect"; import { FilterFoldButton } from "./ui/FilterFold"; import { useFilterFold } from "../lib/useFilterFold"; -import { LoadingState, StateMessage } from "./ui/QueryState"; import { DismissibleChip } from "./ui/Chip"; import type { Shared } from "./ActivityRuns"; @@ -84,6 +83,19 @@ export default function AdminLogs({ }; const [open, setOpen] = useState(null); useUrlMirror(logsParamUpdates(state)); + // Back/Forward lands on another entry's filters; a dialog closing lands on these same ones, and + // changes nothing (Live and the open line stay). + const onLanded = useEffectEvent(() => { + const landed = logsParamsToState(new URLSearchParams(window.location.search)); + if (JSON.stringify(landed) === JSON.stringify(state)) return; + setOpen(null); + setState(landed); + }); + useEffect(() => { + const onPop = () => onLanded(); + window.addEventListener("popstate", onPop); + return () => window.removeEventListener("popstate", onPop); + }, []); const update = (patch: Partial) => { setOpen(null); setState((s) => ({ ...s, ...patch })); @@ -212,156 +224,130 @@ export default function AdminLogs({ }; return ( -
- {/* The page's own band carries the shared filters and stays in reach; this one scrolls. */} -
-
- update({ group })} - options={GROUPS.map((g) => ({ - value: g, - label: t(`settings.myLog.filters.${g}`), - count: groupCount(g), - }))} - /> -
- -
- - {!fold.folded && ( - <> - v)} - selected={[state.sources[0] ?? ""]} - onChange={([source]) => update({ sources: source ? [source] : [] })} - testId="logs-source" - /> - - v === "none" ? t("adminLogs.noEvent") : v, - )} - selected={[state.event ?? ""]} - onChange={([event]) => update({ event: event || null })} - testId="logs-event" - /> - - )} - - - - -
- - {(state.requestId || state.jobId !== null || state.runId !== null) && ( -
- {state.requestId && ( - update({ requestId: null })} - /> - )} - {state.jobId !== null && ( - update({ jobId: null })} - /> - )} - {state.runId !== null && ( - update({ runId: null })} - /> - )} + +
+ update({ group })} + options={GROUPS.map((g) => ({ + value: g, + label: t(`settings.myLog.filters.${g}`), + count: groupCount(g), + }))} + />
- )} -
- {failure && ( - - {t(`adminLogs.${failure}`)} - - )} - -
- {logs.isError && !logs.data ? ( - void logs.refetch()}> - {t("common.loadError")} - - ) : logs.isLoading ? ( - - ) : rows.length === 0 ? ( -

- {t("adminLogs.empty")} -

- ) : ( - - {rows.map((row, index) => ( - setOpen(open === row.id ? null : row.id)} - > - pivot({ requestId })} - onJob={(jobId) => pivot({ jobId })} - onRun={(runId) => pivot({ runId })} +
+ + {!fold.folded && ( + <> + v)} + selected={[state.sources[0] ?? ""]} + onChange={([source]) => update({ sources: source ? [source] : [] })} + testId="logs-source" /> - - ))} - - )} -
+ + v === "none" ? t("adminLogs.noEvent") : v, + )} + selected={[state.event ?? ""]} + onChange={([event]) => update({ event: event || null })} + testId="logs-event" + /> + + )} + + + + +
-
- {t("adminLogs.count", { count: rows.length })} - {olderCursor !== null && ( - - )} -
-
+ {(state.requestId || state.jobId !== null || state.runId !== null) && ( +
+ {state.requestId && ( + update({ requestId: null })} + /> + )} + {state.jobId !== null && ( + update({ jobId: null })} + /> + )} + {state.runId !== null && ( + update({ runId: null })} + /> + )} +
+ )} +
+ } + > + {rows.map((row, index) => ( + setOpen(open === row.id ? null : row.id)} + > + pivot({ requestId })} + onJob={(jobId) => pivot({ jobId })} + onRun={(runId) => pivot({ runId })} + /> + + ))} + ); } diff --git a/frontend/src/components/HistoryList.tsx b/frontend/src/components/HistoryList.tsx index 8862417f..a0e66b49 100644 --- a/frontend/src/components/HistoryList.tsx +++ b/frontend/src/components/HistoryList.tsx @@ -7,9 +7,99 @@ import NavGrid from "./ui/NavGrid"; import { Button } from "./ui/Button"; import { LoadingState, StateMessage } from "./ui/QueryState"; -// The frame of a history list on the activity page: the rows (or the honest state instead of them — -// a failed read is an error with a retry, never an empty list), the count, and "Older". +// The frame of a history list on the activity page — the runs, the admin actions, the server log: +// its own bar (if any), the rows or the honest state instead of them (a failed read is an error with a +// retry, never an empty list), a read that failed over rows still on screen, the count, and "Older". +// `HistoryFrame` draws it; each list says only what its failure and its "Older" are — the server log's +// count its filter facets and stop while Live (AdminLogs), the others not (HistoryList, below). +/** What a read that failed while rows stayed on screen means, and how to try it again. */ +export interface FrameFailure { + text: string; + onRetry: () => void; +} + +export function HistoryFrame({ + list, + rowCount, + header, + failure, + older, + count, + empty, + rowsClassName = "", + testId, + children, +}: { + /** The list's read. */ + list: { + data?: unknown; + isError: boolean; + isLoading: boolean; + isPlaceholderData: boolean; + refetch: () => unknown; + }; + rowCount: number; + /** The list's own bar above the rows. */ + header?: ReactNode; + failure: FrameFailure | null; + /** "Older", when there is an older page to ask for. */ + older: { label: string; onClick: () => void; disabled: boolean } | null; + /** "N runs shown", for the footer. */ + count: string; + /** Shown when the filters match nothing. */ + empty: string; + rowsClassName?: string; + testId?: string; + /** The rows. */ + children: ReactNode; +}) { + const { t } = useTranslation(); + return ( +
+ {header} + {failure && ( + + {failure.text} + + )} +
+ {list.isError && !list.data ? ( + void list.refetch()}> + {t("common.loadError")} + + ) : list.isLoading ? ( + + ) : rowCount === 0 ? ( + + {empty} + + ) : ( + {children} + )} +
+
+ {count} + {older && ( + + )} +
+
+ ); +} + +/** A history list read through `useHistoryList` (the runs, the admin actions). */ export default function HistoryList({ state, empty, @@ -26,59 +116,29 @@ export default function HistoryList({ }) { const { t } = useTranslation(); const { list, rows, hasOlder } = state; + const fetchOlder = () => void list.fetchNextPage(nextPageOnClick(list.isFetchingNextPage)); // A read that failed while rows were on screen keeps them, and says what the failure means. - const failure = !list.data + const failure: FrameFailure | null = !list.data ? null : list.isFetchNextPageError - ? "olderFailed" + ? { text: t("activity.olderFailed"), onRetry: fetchOlder } : list.isError - ? "stale" + ? { text: t("activity.stale"), onRetry: () => void list.refetch() } : null; return ( -
- {failure && ( - - failure === "olderFailed" - ? void list.fetchNextPage(nextPageOnClick(list.isFetchingNextPage)) - : void list.refetch() - } - className="border-b border-border" - > - {failure === "olderFailed" ? t("activity.olderFailed") : t("activity.stale")} - - )} -
- {list.isError && !list.data ? ( - void list.refetch()}> - {t("common.loadError")} - - ) : list.isLoading ? ( - - ) : rows.length === 0 ? ( -

{empty}

- ) : ( - {children} - )} -
-
- {count} - {hasOlder && ( - - )} -
-
+ + {children} + ); } diff --git a/frontend/src/components/HistoryRow.tsx b/frontend/src/components/HistoryRow.tsx index f48f57f2..bf19aa8b 100644 --- a/frontend/src/components/HistoryRow.tsx +++ b/frontend/src/components/HistoryRow.tsx @@ -3,9 +3,10 @@ import type { ReactNode } from "react"; import { navItemAttrs } from "../lib/listNav"; import { Pressable } from "./ui/Pressable"; -// One dense row of a history list on the activity page — a background run, an admin action: its -// cells on one line, and the row opens in place to its details, as a server log line does -// (./LogLine). Inside a NavGrid the keyboard walks the rows (`j`/`k`) and Enter opens one. +// One dense row of a history list — a background run, an admin action, a log line (./LogLine draws +// its lines through this): its cells on one line, and the row opens in place to its details. One frame +// for all of them, so a row opens, announces itself and takes the keyboard the same on every tab. +// Inside a NavGrid the keyboard walks the rows (`j`/`k`) and Enter opens one. export default function HistoryRow({ cells, @@ -15,6 +16,7 @@ export default function HistoryRow({ open, onToggle, title, + line = false, children, }: { /** The row's cells, laid out by `grid`. */ @@ -28,6 +30,8 @@ export default function HistoryRow({ onToggle: () => void; /** The row button's tooltip — the full date. */ title?: string; + /** A log line: tighter, its cells on one text baseline, its size the (monospace) list's own. */ + line?: boolean; /** The detail shown while the row is open. */ children: ReactNode; }) { @@ -43,13 +47,17 @@ export default function HistoryRow({ aria-expanded={open} onClick={onToggle} title={title} - className={`grid w-full items-center gap-x-2 px-3 py-1.5 text-left text-xs touch-target pointer-coarse:content-center ${grid} ${ - open ? "bg-accent/10" : "" - }`} + className={`grid w-full gap-x-2 px-3 text-left touch-target pointer-coarse:content-center ${ + line ? "items-baseline py-1" : "items-center py-1.5 text-xs" + } ${grid} ${open ? "bg-accent/10" : ""}`} > {cells} - {open &&
{children}
} + {open && ( +
+ {children} +
+ )}
); } diff --git a/frontend/src/components/LogLine.tsx b/frontend/src/components/LogLine.tsx index fca3dded..33b3c9ba 100644 --- a/frontend/src/components/LogLine.tsx +++ b/frontend/src/components/LogLine.tsx @@ -1,14 +1,14 @@ import type { ReactNode } from "react"; import type { Level } from "../lib/log"; -import { navItemAttrs } from "../lib/listNav"; +import HistoryRow from "./HistoryRow"; import { LEVEL_TAG, logClock } from "./logLineParts"; -import { Pressable } from "./ui/Pressable"; // One log line as both log viewers draw it — "Saját napló" (this tab's ring) and the admin // diagnostics console (the server's stored log): time, level, event code, message, and the line opens // in place to whatever detail its screen has. One component, so a level reads the same on both. -// Inside a NavGrid the keyboard walks the lines (`j`/`k`) and Enter opens one, as a click does. +// The row frame is the history lists' (./HistoryRow): inside a NavGrid the keyboard walks the lines +// (`j`/`k`) and Enter opens one, as a click does. export default function LogLine({ ts, @@ -36,29 +36,26 @@ export default function LogLine({ }) { const tag = LEVEL_TAG[level]; return ( -
+ {logClock(ts)} + {tag.text} + {event} + {/* A phone has no room for a fourth column: the message takes a line of its own under the + code, full width — sharing the code's line left it a few characters. */} + {msg} + + } > - - {logClock(ts)} - {tag.text} - {event} - {/* A phone has no room for a fourth column: the message takes a line of its own under the - code, full width — sharing the code's line left it a few characters. */} - {msg} - - {open &&
{children}
} -
+ {children} + ); } diff --git a/frontend/src/components/NavigationProvider.dom.test.tsx b/frontend/src/components/NavigationProvider.dom.test.tsx index eb2cc655..630c2a61 100644 --- a/frontend/src/components/NavigationProvider.dom.test.tsx +++ b/frontend/src/components/NavigationProvider.dom.test.tsx @@ -31,6 +31,8 @@ import { useBackToClose } from "../lib/history"; import { PageHeading } from "./ui/Headings"; import { qk } from "../lib/queryKeys"; +import { LS, setAccountRaw } from "../lib/storage"; +import { PAGE_KEY, START_ON_HOME } from "./navigationInit"; /** The Navigation API's view of the tab: `below` = whether each entry under the current one belongs * to this document, nearest first. jsdom has no Navigation API; Chrome's says this. */ @@ -239,6 +241,35 @@ describe("NavigationProvider · Back", () => { ]); }); + it("carries an old server-log link's account, text and range onto the page's own filters", () => { + window.history.replaceState(null, "", "/?page=logs&lu=3&lq=boom&lwin=1h"); + render(); + const url = new URLSearchParams(window.location.search); + expect([url.get("at"), url.get("au"), url.get("aq"), url.get("aw")]).toEqual([ + "logs", + "3", + "boom", + "1h", + ]); + // …and the dead keys leave the address bar instead of riding along to every page after. + expect(["lu", "lq", "lwin"].filter((k) => url.has(k))).toEqual([]); + }); + + it("an old server-log link without a range keeps the one it meant", () => { + window.history.replaceState(null, "", "/?page=logs"); + render(); + expect(new URLSearchParams(window.location.search).get("aw")).toBe("24h"); + }); + + it("a remembered retired page opens its tab", () => { + setAccountRaw(LS.startOnHome, START_ON_HOME.off); + setAccountRaw(PAGE_KEY, "logs"); + window.history.replaceState(null, "", "/"); + const { result } = render(); + expect(result.current.page).toBe("activity"); + expect(new URLSearchParams(window.location.search).get("at")).toBe("logs"); + }); + it("opens a retired page's tab when Back lands on its entry", () => { const { result } = render(); act(() => result.current.setPage("plex")); diff --git a/frontend/src/components/NavigationProvider.tsx b/frontend/src/components/NavigationProvider.tsx index 7ea53516..8e61101f 100644 --- a/frontend/src/components/NavigationProvider.tsx +++ b/frontend/src/components/NavigationProvider.tsx @@ -6,7 +6,7 @@ import { resolvePage, retiredParams, urlWithParams, type Page } from "../lib/url import { ACTIVITY_KEYS } from "../lib/activityUrlState"; import { pageTitleKey } from "../lib/pageMeta"; import { useDocumentObjectTitle } from "../lib/documentTitle"; -import { setAccountRaw } from "../lib/storage"; +import { getAccountRaw, setAccountRaw } from "../lib/storage"; import { backStaysInDocument, closeOverlaysThenNavigate, @@ -193,9 +193,12 @@ export function NavigationProvider({ children }: { children: ReactNode }) { // Mirror the resolved initial page to the address bar (feed implicit) so a page reached from the // stored landing pref — not just from the URL — is itself bookmarkable. urlWithParams preserves any // ?chan / ?v / ?pl / filter params the loaders already read. + // A retired page's link — or the remembered last page, when that is what opened — also opens its + // part of the page that replaced it (an `at` tab). + const linked = new URLSearchParams(window.location.search).get("page"); + const remembered = getAccountRaw(PAGE_KEY); const url = urlWithParams({ - // A retired page's link also opens its part of the page that replaced it (an `at` tab). - ...retiredParams(new URLSearchParams(window.location.search).get("page")), + ...retiredParams(linked ?? (resolvePage(remembered) === mountPage ? remembered : null)), page: mountPage === "feed" ? null : mountPage, }); if (st._yt && st._ytScrape) { diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts index 6e1c9b9d..4a23797e 100644 --- a/frontend/src/i18n/backendIds.contract.test.ts +++ b/frontend/src/i18n/backendIds.contract.test.ts @@ -1,5 +1,7 @@ import { readdirSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; + +import { RUN_STATUSES } from "../lib/activityUrlState"; import { vocabKey } from "../lib/vocab"; // Every id the BACKEND owns and the UI names must have its name in both languages. The UI looks these @@ -72,6 +74,15 @@ const jobs = [ const enumValues = (file: string) => [...backend(file).matchAll(/^\s+[A-Z_]+ = "(\w+)"$/gm)].map((m) => m[1]!); const auditActions = enumValues("audit.py"); +// A background run's words (backend/app/runstatus.py): its status and what started it. +const runEnum = (name: string) => + [ + ...( + backend("runstatus.py").split(`class ${name}(StrEnum):`)[1]?.split("\nclass ")[0] ?? "" + ).matchAll(/^\s+[A-Z_]+ = "(\w+)"$/gm), + ].map((m) => m[1]!); +const runStatuses = runEnum("RunStatus"); +const runTriggers = runEnum("RunTrigger"); const quotaActions = enumValues("quota.py"); // The admin actions' refusal reasons → `errors.admin.`. The codes are read from // `localizeDetail`'s own ADMIN_REFUSALS, so a code added there is scanned here too; a detail dict @@ -260,6 +271,8 @@ describe("backend ids have their names in both languages", () => { expect(jobs).toEqual(expect.arrayContaining(["rss_poll", "audit_gc"])); expect(auditActions).toEqual(expect.arrayContaining(["config_set", "audit_clear"])); expect(quotaActions.length).toBeGreaterThan(0); + expect(runStatuses).toEqual(expect.arrayContaining(["running", "interrupted"])); + expect(runTriggers).toEqual(expect.arrayContaining(["scheduled", "ad_hoc"])); expect(adminCodes).toEqual(expect.arrayContaining(["activate", "resend"])); expect(failDetails).toEqual(expect.arrayContaining(["facebook.login_wall"])); expect(goneWhats).toEqual(expect.arrayContaining(["playlist", "download", "job"])); @@ -288,10 +301,23 @@ describe("backend ids have their names in both languages", () => { expect(adminCodes.filter((c) => !adminRefusals.some((m) => m[1] === c))).toEqual([]); }); + it("the run list's status chips are the server's run statuses", () => { + expect([...RUN_STATUSES].sort()).toEqual([...runStatuses].sort()); + }); + for (const lang of ["en", "hu"]) { const keys = keysOf(lang); const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k)); + it(`${lang}: every run status and trigger`, () => { + expect( + missing([ + ...runStatuses.flatMap((s) => [`activity.runs.status.${s}`, `scheduler.dot.${s}`]), + ...runTriggers.map((s) => `scheduler.trigger.${s}`), + ]), + ).toEqual([]); + }); + it(`${lang}: every config key and group`, () => { expect( missing([ diff --git a/frontend/src/i18n/locales/en/activity.json b/frontend/src/i18n/locales/en/activity.json index dba2414c..d0e34b02 100644 --- a/frontend/src/i18n/locales/en/activity.json +++ b/frontend/src/i18n/locales/en/activity.json @@ -18,7 +18,8 @@ "24h": "24 hours", "7d": "7 days", "30d": "30 days", - "90d": "90 days" + "90d": "90 days", + "all": "All" }, "account": "Account", "any": "Any", @@ -26,7 +27,8 @@ "older": "Load older", "stale": "Couldn't refresh — what you see may be out of date", "olderFailed": "Couldn't load the older rows", - "countsFailed": "Couldn't load the filter counts", + "filtersFailed": "Couldn't load the filter choices: the menus may be missing some.", + "accountsFailed": "Couldn't load the accounts for the account filter.", "runs": { "job": "Job", "statusLabel": "Filter by status", @@ -57,10 +59,15 @@ "action": "Action", "target": "Target", "requestLines": "Its request's log lines →", + "noRequestLines": "Its request's log lines are no longer kept", "count_one": "{{count}} action shown · newest first", "count_other": "{{count}} actions shown · newest first" }, "quota": { - "total": "Spend over {{window}}: {{units}} units" + "daily": "Spend per day", + "total_one": "Spend over {{window}}: {{units}} unit", + "total_other": "Spend over {{window}}: {{units}} units", + "totalAll_one": "Spend over all kept history: {{units}} unit", + "totalAll_other": "Spend over all kept history: {{units}} units" } } diff --git a/frontend/src/i18n/locales/en/stats.json b/frontend/src/i18n/locales/en/stats.json index 8ab0cb00..d1d7bda6 100644 --- a/frontend/src/i18n/locales/en/stats.json +++ b/frontend/src/i18n/locales/en/stats.json @@ -4,7 +4,6 @@ "introAfter": " is shared background work (scheduled backfill, enrichment, resync) that isn't tied to one person.", "loading": "Loading…", "noUsageInRange": "No usage in this range.", - "dailyTooltip": "{{day}}: {{units}} units", "system": "System (background)", "sync": { "myStatus": "My sync status", @@ -41,5 +40,7 @@ "fullHistoryRequested_one": "Full history requested for {{count}} channel", "fullHistoryRequested_other": "Full history requested for {{count}} channels", "fullHistoryFailed": "Couldn't request full history" - } + }, + "dailyTooltip_one": "{{day}}: {{units}} unit", + "dailyTooltip_other": "{{day}}: {{units}} units" } diff --git a/frontend/src/i18n/locales/hu/activity.json b/frontend/src/i18n/locales/hu/activity.json index 33d26641..0b282a8d 100644 --- a/frontend/src/i18n/locales/hu/activity.json +++ b/frontend/src/i18n/locales/hu/activity.json @@ -18,7 +18,8 @@ "24h": "24 óra", "7d": "7 nap", "30d": "30 nap", - "90d": "90 nap" + "90d": "90 nap", + "all": "Mind" }, "account": "Fiók", "any": "Bármelyik", @@ -26,7 +27,8 @@ "older": "Régebbiek", "stale": "Nem sikerült frissíteni — amit látsz, elavult lehet", "olderFailed": "Nem sikerült betölteni a régebbi sorokat", - "countsFailed": "Nem sikerült betölteni a szűrők darabszámait", + "filtersFailed": "Nem sikerült betölteni a szűrők választékát: a menükből hiányozhatnak elemek.", + "accountsFailed": "Nem sikerült betölteni a fiókokat a fiókszűrőhöz.", "runs": { "job": "Feladat", "statusLabel": "Szűrés állapotra", @@ -57,10 +59,15 @@ "action": "Művelet", "target": "Cél", "requestLines": "A kérés naplósorai →", + "noRequestLines": "A kérés naplósorai már nincsenek meg", "count_one": "{{count}} művelet látszik · a legújabb elöl", "count_other": "{{count}} művelet látszik · a legújabb elöl" }, "quota": { - "total": "Felhasználás ({{window}}): {{units}} egység" + "daily": "Felhasználás naponta", + "total_one": "Felhasználás ({{window}}): {{units}} egység", + "total_other": "Felhasználás ({{window}}): {{units}} egység", + "totalAll_one": "Felhasználás (a teljes megőrzött idő): {{units}} egység", + "totalAll_other": "Felhasználás (a teljes megőrzött idő): {{units}} egység" } } diff --git a/frontend/src/i18n/locales/hu/stats.json b/frontend/src/i18n/locales/hu/stats.json index 94e73362..4c04918d 100644 --- a/frontend/src/i18n/locales/hu/stats.json +++ b/frontend/src/i18n/locales/hu/stats.json @@ -4,7 +4,6 @@ "introAfter": " a megosztott háttérmunka (ütemezett behúzás, dúsítás, újraszinkronizálás), amely nem köthető egy személyhez.", "loading": "Betöltés…", "noUsageInRange": "Nincs használat ebben az időszakban.", - "dailyTooltip": "{{day}}: {{units}} egység", "system": "Rendszer (háttér)", "sync": { "myStatus": "Saját szinkronizálási állapot", @@ -41,5 +40,7 @@ "fullHistoryRequested_one": "Teljes előzmény kérve {{count}} csatornához", "fullHistoryRequested_other": "Teljes előzmény kérve {{count}} csatornához", "fullHistoryFailed": "Nem sikerült teljes előzményt kérni" - } + }, + "dailyTooltip_one": "{{day}}: {{units}} egység", + "dailyTooltip_other": "{{day}}: {{units}} egység" } diff --git a/frontend/src/lib/activityUrlState.test.ts b/frontend/src/lib/activityUrlState.test.ts index b748d474..fd8a4290 100644 --- a/frontend/src/lib/activityUrlState.test.ts +++ b/frontend/src/lib/activityUrlState.test.ts @@ -9,6 +9,7 @@ import { activityParamUpdates, DEFAULT_ACTIVITY_STATE, sinceOf, + ACTIVITY_WINDOWS, type ActivityUrlState, } from "./activityUrlState"; @@ -54,4 +55,11 @@ describe("activityUrlState", () => { expect(sinceOf("24h", now)).toBe("2026-10-17T12:00:00.000Z"); expect(sinceOf("90d", now)).toBe("2026-07-20T12:00:00.000Z"); }); + + it("offers everything kept, past the longest range", () => { + // Runs and admin actions are kept 180 days by default, or for ever: a 90-day cap hid the rest. + expect(ACTIVITY_WINDOWS[ACTIVITY_WINDOWS.length - 1]).toBe("all"); + expect(activityParamsToState(new URLSearchParams("aw=all")).window).toBe("all"); + expect(sinceOf("all")).toBe("1970-01-01T00:00:00.000Z"); + }); }); diff --git a/frontend/src/lib/activityUrlState.ts b/frontend/src/lib/activityUrlState.ts index b1b495a4..4a754a28 100644 --- a/frontend/src/lib/activityUrlState.ts +++ b/frontend/src/lib/activityUrlState.ts @@ -3,12 +3,13 @@ // each tab's own (the run list's job and status, the audit list's action). The server log's own // filters keep their `l…` keys (./logsUrlState). setPage clears all of them (ACTIVITY_KEYS); the page // re-mirrors them while it is open. +import type { RunStatus } from "./api"; import { LOGS_KEYS } from "./logsUrlState"; export type ActivityTab = "runs" | "admin" | "logs" | "quota"; -export type ActivityWindow = "1h" | "24h" | "7d" | "30d" | "90d"; -/** The run statuses the list can narrow to (the API's `RunStatus`). */ -export type RunStatusFilter = "running" | "ok" | "error" | "skipped" | "interrupted"; +export type ActivityWindow = "1h" | "24h" | "7d" | "30d" | "90d" | "all"; +/** The run statuses the list can narrow to: the API's `RunStatus` (held by backendIds.contract.test). */ +export type RunStatusFilter = RunStatus; export interface ActivityUrlState { tab: ActivityTab; @@ -24,7 +25,9 @@ export interface ActivityUrlState { } export const ACTIVITY_TABS: readonly ActivityTab[] = ["runs", "admin", "logs", "quota"]; -export const ACTIVITY_WINDOWS: readonly ActivityWindow[] = ["1h", "24h", "7d", "30d", "90d"]; +// `all` = everything still kept: runs and admin actions are kept 180 days by default (0 = for ever), +// longer than any fixed range. +export const ACTIVITY_WINDOWS: readonly ActivityWindow[] = ["1h", "24h", "7d", "30d", "90d", "all"]; export const RUN_STATUSES: readonly RunStatusFilter[] = [ "error", "interrupted", @@ -33,12 +36,13 @@ export const RUN_STATUSES: readonly RunStatusFilter[] = [ "ok", ]; -const WINDOW_HOURS: Record = { +const WINDOW_HOURS: Record = { "1h": 1, "24h": 24, "7d": 24 * 7, "30d": 24 * 30, "90d": 24 * 90, + all: null, }; export const DEFAULT_ACTIVITY_STATE: ActivityUrlState = { @@ -92,7 +96,9 @@ export function activityParamUpdates(state: ActivityUrlState): Record | null; /** The request the change was made in — its log lines are one pivot away. Null on older rows. */ request_id: string | null; + /** How many of that request's log lines are still kept (they go after days, the row after months). */ + log_lines: number; } /** A page of a history list (audit log, runs), newest first; `next_cursor` continues it. */ @@ -142,7 +144,8 @@ export interface RunRow { summary: string | null; result: JobResult | null; error: string | null; - failure: { code: string; [key: string]: unknown } | null; + /** A failure the job named (`progress.JobFailed`): its reason, and the values it is said with. */ + failure: { failure: string; [key: string]: unknown } | null; quota_units: number; /** How many stored log lines the run left (WARNING and up only). */ log_lines: number; @@ -249,8 +252,10 @@ function queryString(values: object): string { export type JobResult = { [key: string]: number | string | JobResult }; /** How a run ended — the server's `RunStatus` (app/runstatus.py) minus `running`. `interrupted`: - * the process stopped mid-run (a deploy, a crash), and its next start closed the row. */ + * the process stopped mid-run (a deploy, a crash), and its next start closed the row. The values are + * held to the server's by backendIds.contract.test (through RUN_STATUSES, typed as `RunStatus`). */ export type RunOutcome = "ok" | "error" | "skipped" | "interrupted"; +export type RunStatus = RunOutcome | "running"; /** One past run, as the row's history dots show it. */ export interface RunBrief { diff --git a/frontend/src/lib/auditText.dom.test.ts b/frontend/src/lib/auditText.dom.test.ts index a1bee7e6..6683fa9e 100644 --- a/frontend/src/lib/auditText.dom.test.ts +++ b/frontend/src/lib/auditText.dom.test.ts @@ -25,6 +25,7 @@ const row = (over: Partial): AuditRow => ({ before: null, after: null, request_id: null, + log_lines: 0, ...over, }); diff --git a/frontend/src/lib/navRing.contract.test.ts b/frontend/src/lib/navRing.contract.test.ts index 822caa51..d143831e 100644 --- a/frontend/src/lib/navRing.contract.test.ts +++ b/frontend/src/lib/navRing.contract.test.ts @@ -37,9 +37,10 @@ it("index.css draws an inset item's ring inside, after the outside rule", () => // The items that sit flush, by their source: a new one of the kind should be added here. const src = (p: string) => readFileSync(new URL(p, import.meta.url), "utf8"); -it.each([["../components/LogLine.tsx", /navItemAttrs\(navIndex, [^)]*\{ inset: true \}\)/]])( - "%s asks for the inside ring", - (file, re) => { - expect(src(file)).toMatch(re); - }, -); +// The history rows (runs, admin actions) and the log lines share one frame, HistoryRow. +it.each([ + ["../components/HistoryRow.tsx", /navItemAttrs\(navIndex, [^)]*\{ inset: true \}\)/], + ["../components/LogLine.tsx", / { + expect(src(file)).toMatch(re); +}); diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index a2de239b..75b31923 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -87,8 +87,12 @@ export const qk = { adminInvites: () => ["admin-invites"] as const, adminPendingAccess: () => ["admin-pending-access"] as const, adminConfig: () => ["admin-config"] as const, - adminAudit: (filters: object) => ["admin-audit", filters] as const, - adminAuditFacets: (filters: object) => ["admin-audit-facets", filters] as const, + adminAudit: (filters?: object) => + filters !== undefined ? (["admin-audit", filters] as const) : (["admin-audit"] as const), + adminAuditFacets: (filters?: object) => + filters !== undefined + ? (["admin-audit-facets", filters] as const) + : (["admin-audit-facets"] as const), adminRuns: (filters: object) => ["admin-runs", filters] as const, adminRunFacets: (filters: object) => ["admin-run-facets", filters] as const, // The console's filters are part of the key; `since` is not (it moves with the clock) — the diff --git a/frontend/src/lib/urlState.ts b/frontend/src/lib/urlState.ts index cbb92217..c4afc707 100644 --- a/frontend/src/lib/urlState.ts +++ b/frontend/src/lib/urlState.ts @@ -142,15 +142,32 @@ function isPage(p: string | null | undefined): p is Page { } // A page this build no longer has, the page that took its place, and the params that open its part -// of that page. A bookmark, the remembered last page and an old history entry can still name one; all -// three resolve through `resolvePage`, so a retired page lands where its content went instead of on -// the feed, and the navigation provider writes `params` into the address bar as it rewrites the page. -const RETIRED: Readonly }>> = { +// of that page — from the old link's own params. A bookmark, the remembered last page and an old +// history entry can still name one; all three resolve through `resolvePage`, so a retired page lands +// where its content went instead of on the feed, and the navigation provider writes `params` into the +// address bar as it rewrites the page (a null clears a key the old page owned). +type Updates = Record; +const RETIRED: Readonly< + Record Updates }> +> = { // The download queue became the tray; what it downloaded lives in Files. downloads: { page: "files" }, // The audit log and the server log became tabs of the activity page (its `at` key). - audit: { page: "activity", params: { at: "admin" } }, - logs: { page: "activity", params: { at: "logs" } }, + audit: { page: "activity", params: () => ({ at: "admin" }) }, + // The server log's own account, text and range became the page's shared filters; a link without + // a range meant the log's default then, 24 hours. + logs: { + page: "activity", + params: (old) => ({ + at: "logs", + au: old.get("lu"), + aq: old.get("lq"), + aw: old.get("lwin") ?? "24h", + lu: null, + lq: null, + lwin: null, + }), + }, }; const retired = (p: string | null | undefined) => @@ -162,9 +179,13 @@ export function resolvePage(p: string | null | undefined): Page | null { return retired(p)?.page ?? null; } -/** The params that open a retired page's part of its successor (none for a live page). */ -export function retiredParams(p: string | null | undefined): Record { - return isPage(p) ? {} : (retired(p)?.params ?? {}); +/** The params that open a retired page's part of its successor, read from the old link's `params` + * (none for a live page). */ +export function retiredParams( + p: string | null | undefined, + params = new URLSearchParams(window.location.search), +): Updates { + return isPage(p) ? {} : (retired(p)?.params?.(params) ?? {}); } export function readPage(): Page { From 13b7577d9f01c97c05ac22a5a8939ce2b57557c4 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 17:32:22 +0200 Subject: [PATCH 13/16] fix(activity): an ad-hoc run's row is one run's, a rollback keeps a run whole, Back reads one entry's filters - The channel-backfill row says "running" from the runs this process has open, not from the table: a row whose outcome failed to write no longer reads as running until a restart. The run still going leads the row, so a newer run's outcome never sits under the older one's channel. - Migration 0109's downgrade carries every run's trigger, spend and start in the audit row, and the upgrade restores them: a manual run whose admin was deleted stays manual. - Back/Forward is read by the page's one popstate listener, both halves in one render; the server log adopts its half during that render, so no read goes out with one entry's filters and the other's range. - The server log's failed facets use the other tabs' words (the menus may be short); the accounts message shows only when the menu really lacks them. --- backend/alembic/versions/0109_job_runs.py | 24 ++++----- backend/app/jobruns.py | 10 ---- backend/app/scheduler.py | 31 ++++++++++-- backend/tests/test_migration_0109.py | 37 +++++++++++--- backend/tests/test_scheduler.py | 30 ++++++++++-- frontend/src/components/Activity.dom.test.tsx | 49 ++++++++++++++++++- frontend/src/components/Activity.tsx | 24 ++++++--- .../src/components/AdminLogs.dom.test.tsx | 8 +++ frontend/src/components/AdminLogs.tsx | 38 +++++++------- frontend/src/i18n/locales/en/adminLogs.json | 1 - frontend/src/i18n/locales/hu/adminLogs.json | 1 - frontend/src/i18n/unusedKeys.contract.test.ts | 7 +-- 12 files changed, 188 insertions(+), 72 deletions(-) diff --git a/backend/alembic/versions/0109_job_runs.py b/backend/alembic/versions/0109_job_runs.py index 1def28b4..f75f2c5e 100644 --- a/backend/alembic/versions/0109_job_runs.py +++ b/backend/alembic/versions/0109_job_runs.py @@ -13,7 +13,7 @@ log writer runs behind the event). Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the audit log. The downgrade moves every run back into the audit log as such a row (one still running as -interrupted, a run's trigger and target in its `after`), then drops the table. +interrupted, what the audit row has no column for in its `after`), then drops the table. """ import sqlalchemy as sa @@ -72,25 +72,27 @@ def upgrade() -> None: ) # The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`, - # which a downgrade writes back (as it does a run's `trigger` and `target`, which the audit row - # has no column for), so a rollback and a redeploy keep every run. A status the run table does + # which a downgrade writes back (as it does a run's trigger, target, spend and start, which the + # audit row has no column for: its actor alone cannot say "manual" once that admin is deleted), + # so a rollback and a redeploy keep every run as it was. A status the run table does # not know would fail the CHECK. op.execute( """ INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at, - summary, result, error, failure) + summary, result, error, failure, quota_units) SELECT target_id, coalesce(after::jsonb ->> 'trigger', CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END), actor_id, after::jsonb ->> 'target', after::jsonb ->> 'status', - created_at, + coalesce((after::jsonb ->> 'started_at')::timestamptz, created_at), created_at, CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb ->> 'summary' END, CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb -> 'result' END, CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb ->> 'summary' END, - CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END + CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END, + coalesce((after::jsonb ->> 'quota_units')::int, 0) FROM audit_log WHERE action = 'job_run' AND target_id IS NOT NULL @@ -109,9 +111,9 @@ def upgrade() -> None: def downgrade() -> None: - # A run still `running` is one the rollback stopped: it goes back as interrupted. Its trigger and - # target ride in `after` when the audit row's own columns cannot say them (an ad-hoc run of one - # channel), so the upgrade restores them. + # A run still `running` is one the rollback stopped: it goes back as interrupted. What the audit + # row's own columns cannot say rides in `after` (its trigger, target, spend and start), so the + # upgrade restores them. op.execute( """ INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at) @@ -132,8 +134,8 @@ def downgrade() -> None: THEN jsonb_build_object('result', result) ELSE '{}'::jsonb END - || CASE WHEN trigger = 'ad_hoc' - THEN jsonb_build_object('trigger', trigger) ELSE '{}'::jsonb END + || jsonb_build_object('trigger', trigger, 'quota_units', quota_units, + 'started_at', started_at) || CASE WHEN target IS NOT NULL THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json, coalesce(finished_at, started_at) diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py index a51249ae..7beb7572 100644 --- a/backend/app/jobruns.py +++ b/backend/app/jobruns.py @@ -119,16 +119,6 @@ def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[Jo return out -def running_one(db: Session, job: str) -> JobRun | None: - """The newest run of `job` still going, however many runs ended after it started.""" - return db.scalars( - select(JobRun) - .where(JobRun.job == job, JobRun.status == RunStatus.RUNNING) - .order_by(JobRun.started_at.desc(), JobRun.id.desc()) - .limit(1) - ).first() - - def gc(db: Session, retention_days: int) -> int: """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is never reaped. The caller commits.""" diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 29904095..4e112a56 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -15,7 +15,7 @@ from app import audit, debugsession, jobruns, logs, logsink, progress, quota, sy from app.config import settings from app.db import SessionLocal from app.downloads.gc import run_download_gc -from app.models import Channel +from app.models import Channel, JobRun from app.plex.sync import sync as run_plex_sync from app.plex.tmdb import run_enrich as run_tmdb_enrich from app.plex.tmdb import run_people_enrich @@ -45,6 +45,10 @@ _scheduler: BackgroundScheduler | None = None # run did is durable instead: its `job_runs` row (app/jobruns.py), which outlives a restart. _activity: dict[str, dict] = {} _activity_lock = threading.Lock() +# The ad-hoc runs this process has open (run id -> job), under the same lock. A row can stay +# `running` in the table after its outcome failed to write (_close), so the table cannot say what is +# going; this can, as the one process per database (app/jobruns.py) is the one running them. +_open_ad_hoc: dict[int, str] = {} # Set (per-thread) by a manual "run now" trigger to the id of the admin who clicked, so that # run — and only that run, not the recurring scheduled ones — posts a completion notification @@ -243,6 +247,7 @@ def scheduler_snapshot(db) -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} + going_id = max((r for r, j in _open_ad_hoc.items() if j == CHANNEL_BACKFILL), default=None) recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): @@ -259,11 +264,17 @@ def scheduler_snapshot(db) -> dict: ) # Work a user starts on one target: no interval, no "run now" (it needs its target), but its # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the - # target's own (sync/runner), so "running" is what its rows say: two channels can backfill at - # once, so a run still going counts even when a newer one already ended, and the row names it. + # target's own (sync/runner), so "running" is the runs this process has open: two channels can + # backfill at once, so a run still going counts even when a newer one already ended. That run then + # leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the + # newer run's outcome under the older one's channel. backfills = recent.get(CHANNEL_BACKFILL, []) - going = jobruns.running_one(db, CHANNEL_BACKFILL) - shown = going or (backfills[0] if backfills else None) + going = db.get(JobRun, going_id) if going_id is not None else None + if going is not None and going.status != RunStatus.RUNNING: + going = None # its outcome is written; only the bookkeeping has not caught up + if going is not None: + backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS] + shown = backfills[0] if backfills else None ad_hoc = [ { "id": CHANNEL_BACKFILL, @@ -596,6 +607,16 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non # No row to tell it in: the database is what failed. The log is the record. log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return + with _activity_lock: + _open_ad_hoc[run_id] = job + try: + _ad_hoc_work(job, fn, run_id) + finally: + with _activity_lock: + _open_ad_hoc.pop(run_id, None) + + +def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: with logs.bind(run_id=run_id): # As in _run: only the work is inside the try. try: diff --git a/backend/tests/test_migration_0109.py b/backend/tests/test_migration_0109.py index a597e0d7..9fd08814 100644 --- a/backend/tests/test_migration_0109.py +++ b/backend/tests/test_migration_0109.py @@ -3,6 +3,7 @@ outcome intact, and leave the audit log; an admin action stays where it was. The them back.""" import json +from datetime import timedelta from alembic import command from alembic.config import Config @@ -87,14 +88,28 @@ def test_the_audit_logs_runs_move_into_the_run_table_and_back(db, make_user): ).all() finally: command.upgrade(cfg, "head") + + # What the audit row has no column for rides along, so the next upgrade restores it. + def run(trigger: str) -> dict: + return {"trigger": trigger, "quota_units": 0, "started_at": "2026-10-01T03:00:00+00:00"} + assert [tuple(r) for r in back] == [ - (None, "rss_poll", {"status": "ok", "summary": "new=3", "result": {"new": 3}}), + ( + None, + "rss_poll", + {"status": "ok", "summary": "new=3", "result": {"new": 3}, **run("scheduled")}, + ), ( uid, "rss_poll", - {"status": "error", "summary": "boom", "result": {"failure": "rss_all_failed"}}, + { + "status": "error", + "summary": "boom", + "result": {"failure": "rss_all_failed"}, + **run("manual"), + }, ), - (uid, "backfill", {"status": "skipped", "summary": "sync paused"}), + (uid, "backfill", {"status": "skipped", "summary": "sync paused", **run("manual")}), ] @@ -106,10 +121,13 @@ def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): conn.execute( text( "INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at," - " finished_at) VALUES" - " ('backfill', 'scheduled', NULL, NULL, 'interrupted', now(), now())," - " ('maintenance', 'manual', :u, NULL, 'running', now(), NULL)," - " ('channel_backfill', 'ad_hoc', :u, 'UC_x', 'ok', now(), now())" + " finished_at, quota_units) VALUES" + " ('backfill', 'scheduled', NULL, NULL, 'interrupted', now(), now(), 0)," + " ('maintenance', 'manual', :u, NULL, 'running', now(), NULL, 0)," + " ('channel_backfill', 'ad_hoc', :u, 'UC_x', 'ok', now(), now(), 0)," + # Its admin is gone (SET NULL): only the row itself still says it was manual. + " ('enrich', 'manual', NULL, NULL, 'ok'," + " '2026-10-01 10:00+00', '2026-10-01 10:30+00', 42)" ), {"u": uid}, ) @@ -120,6 +138,9 @@ def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): runs = conn.execute( text("SELECT job, trigger, actor_id, target, status FROM job_runs ORDER BY id") ).all() + kept = conn.execute( + text("SELECT quota_units, finished_at - started_at FROM job_runs WHERE job = 'enrich'") + ).one() stranded = conn.execute( text("SELECT count(*) FROM audit_log WHERE action = 'job_run'") ).scalar() @@ -127,5 +148,7 @@ def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): ("backfill", "scheduled", None, None, "interrupted"), ("maintenance", "manual", uid, None, "interrupted"), # the rollback stopped it ("channel_backfill", "ad_hoc", uid, "UC_x", "ok"), + ("enrich", "manual", None, None, "ok"), ] + assert tuple(kept) == (42, timedelta(minutes=30)) assert stranded == 0 diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 767aec5d..cb6f1ef7 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -363,8 +363,15 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m assert len(row["recent"]) == 2 assert all(j["id"] != "channel_backfill" for j in scheduler.scheduler_snapshot(db)["jobs"]) - jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_shown") - assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is True + seen = [] + scheduler._ad_hoc( + "channel_backfill", + lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {}, + user.id, + "UC_shown", + ) + assert seen == [True] + assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False def _finished(db, job) -> list[JobRun]: @@ -458,6 +465,8 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use calls.clear() scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") assert calls == ["ok"] + # Its row is still `running`, but nothing here is: the page does not call it going. + assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch): @@ -484,10 +493,21 @@ def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_use user = make_user("two@example.com") db.add(Channel(id="UC_long", title="Long one")) db.commit() - jobruns.start("channel_backfill", RunTrigger.AD_HOC, user.id, "UC_long") - scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short") - (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + rows = [] + + def long_one(): + scheduler._ad_hoc( + "channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short" + ) + rows.extend(scheduler.scheduler_snapshot(db)["ad_hoc"]) + return {} + + scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long") + (row,) = rows assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one") + # The outcome beside it is the long one's (none yet), never the short one's skip. + assert (row["status"], row["result"], row["last_error"]) == (None, None, None) + assert [r["status"] for r in row["recent"]] == ["running", "ok"] def test_an_interval_change_lands_with_its_audit_row_or_not_at_all( diff --git a/frontend/src/components/Activity.dom.test.tsx b/frontend/src/components/Activity.dom.test.tsx index e74f7fb2..53c2d1f3 100644 --- a/frontend/src/components/Activity.dom.test.tsx +++ b/frontend/src/components/Activity.dom.test.tsx @@ -45,6 +45,7 @@ vi.mock("../lib/api", async (importOriginal) => { }; }); +import { qk } from "../lib/queryKeys"; import Activity from "./Activity"; import { ChoiceContext } from "./confirmContext"; import { PageToolbarSlot } from "./ui/pageToolbarSlot"; @@ -90,8 +91,7 @@ const page = (items: T[]): HistoryPage => ({ next_cursor: null, }); -function renderPage() { - const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); +function renderPage(qc = new QueryClient({ defaultOptions: { queries: { retry: false } } })) { return render( {/* Every confirmation is answered yes. */} @@ -259,6 +259,16 @@ describe("Activity", () => { expect(usersMock).toHaveBeenCalledTimes(2); }); + it("accounts that could not be refreshed, still listed, are not called unloaded", async () => { + const qc = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 0 } } }); + qc.setQueryData(qk.adminUsers(), [{ id: 1, email: "admin@example.com" }]); + usersMock.mockRejectedValue(new Error("502")); + renderPage(qc); + await waitFor(() => expect(usersMock).toHaveBeenCalled()); + await waitFor(() => expect(qc.getQueryState(qk.adminUsers())?.status).toBe("error")); + expect(screen.queryByText("activity.accountsFailed")).toBeNull(); + }); + it("clearing the admin actions reads the list again", async () => { window.history.replaceState(null, "", "/?page=activity&at=admin"); renderPage(); @@ -285,6 +295,41 @@ describe("Activity", () => { expect(after).toBe(live); // the same log, not a fresh one }); + it("Back/Forward on the server log reads only the filters of the entry it lands on", async () => { + // A browser runs the microtasks between two popstate listeners, so each listener's update + // renders on its own; a test dispatch does not. Each listener runs in its own act here. + const listeners: EventListener[] = []; + const addOther = window.addEventListener.bind(window); + const removeOther = window.removeEventListener.bind(window); + const add = vi.spyOn(window, "addEventListener").mockImplementation((type, l, o) => { + if (type === "popstate") listeners.push(l as EventListener); + else addOther(type, l, o); + }); + const remove = vi.spyOn(window, "removeEventListener").mockImplementation((type, l, o) => { + if (type === "popstate") listeners.splice(listeners.indexOf(l as EventListener), 1); + else removeOther(type, l, o); + }); + window.history.replaceState(null, "", "/?page=activity&at=logs&aw=24h"); + renderPage(); + await waitFor(() => expect(logsMock).toHaveBeenCalled()); + logsMock.mockClear(); + // Forward onto a pivot's entry: one run's lines over the log's whole retention. + window.history.pushState(null, "", "/?page=activity&at=logs&aw=90d&lrun=41"); + for (const l of [...listeners]) { + await act(async () => l(new PopStateEvent("popstate"))); + } + await waitFor(() => + expect(logsMock).toHaveBeenLastCalledWith(expect.objectContaining({ run_id: 41 }), null), + ); + const days = (since?: string) => (Date.now() - Date.parse(since ?? "")) / 86_400_000; + // Never one run's lines over a day, nor the whole log over 90 days. + expect(logsMock.mock.calls.map(([q]) => [q.run_id ?? null, days(q.since) > 30])).toEqual( + logsMock.mock.calls.map(() => [41, true]), + ); + add.mockRestore(); + remove.mockRestore(); + }); + it("an action whose request's lines are gone says so instead of linking to an empty list", async () => { auditMock.mockResolvedValue(page([{ ...ACTION, log_lines: 0 }])); window.history.replaceState(null, "", "/?page=activity&at=admin"); diff --git a/frontend/src/components/Activity.tsx b/frontend/src/components/Activity.tsx index d7ff5364..8db4fa7f 100644 --- a/frontend/src/components/Activity.tsx +++ b/frontend/src/components/Activity.tsx @@ -15,7 +15,12 @@ import { type ActivityWindow, } from "../lib/activityUrlState"; import { nextNavId, notifyNav, rememberNavPosition } from "../lib/history"; -import { DEFAULT_LOGS_STATE, logsParamUpdates } from "../lib/logsUrlState"; +import { + DEFAULT_LOGS_STATE, + logsParamsToState, + logsParamUpdates, + type LogsUrlState, +} from "../lib/logsUrlState"; import { qk } from "../lib/queryKeys"; import { useDebounced } from "../lib/useDebounced"; import { urlWithParams } from "../lib/urlState"; @@ -55,17 +60,23 @@ export default function Activity() { // text then applies at once instead of after the typing debounce. const [jumps, setJumps] = useState(0); // Bumped by a pivot only: the server log, already open, takes the pivot's filters from the address - // bar when it mounts again. Back/Forward it follows in place (AdminLogs) — a popstate is also a - // dialog closing, onto the same address, and a fresh log would drop its Live and its focus. + // bar when it mounts again. Back/Forward it follows in place — a popstate is also a dialog closing, + // onto the same address, and a fresh log would drop its Live and its focus. const [pivots, setPivots] = useState(0); + // The log's half of the entry Back/Forward landed on, read here with this page's half: one listener, + // so both change in one render. Two listeners rendered in between (a browser runs the microtasks + // after each), and that render read the log with one entry's filters and the other's range. + const [landed, setLanded] = useState<{ logs: LogsUrlState } | null>(null); const q = useDebounced(state.q, 300, jumps); const panel = useTabPanel(); // `page` too: a link to a page this one replaced (audit, logs) is rewritten to this page's id. useUrlMirror({ page: "activity", ...activityParamUpdates(state) }); useEffect(() => { const onPop = () => { + const params = new URLSearchParams(window.location.search); setJumps((n) => n + 1); - setState(activityParamsToState(new URLSearchParams(window.location.search))); + setState(activityParamsToState(params)); + setLanded({ logs: logsParamsToState(params) }); }; window.addEventListener("popstate", onPop); return () => window.removeEventListener("popstate", onPop); @@ -106,7 +117,8 @@ export default function Activity() {
- {users.isError && ( + {/* Only when the menu really lacks them: a refresh that failed keeps the list it had. */} + {users.isError && !users.data && ( void users.refetch()}> {t("activity.accountsFailed")} @@ -135,7 +147,7 @@ export default function Activity() { ) : state.tab === "logs" ? ( // Keyed by the pivots: a pivot hands the log its filters through the address bar, which it // reads when it mounts. - + ) : ( )} diff --git a/frontend/src/components/AdminLogs.dom.test.tsx b/frontend/src/components/AdminLogs.dom.test.tsx index fbf18d3b..fde06401 100644 --- a/frontend/src/components/AdminLogs.dom.test.tsx +++ b/frontend/src/components/AdminLogs.dom.test.tsx @@ -295,6 +295,14 @@ describe("AdminLogs", () => { expect((await screen.findByRole("alert")).textContent).toContain("adminLogs.olderFailed"); }); + it("facets that failed say the filter menus may be short, in the other tabs' words", async () => { + logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); + facetsMock.mockRejectedValue(new Error("500")); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + expect((await screen.findByRole("alert")).textContent).toContain("activity.filtersFailed"); + }); + it("offers no older page while a filter change loads", async () => { let landErrors!: (page: LogPage) => void; logsMock.mockImplementation((q) => diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index 703dbc76..763212ba 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -1,4 +1,4 @@ -import { useEffect, useEffectEvent, useMemo, useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { keepPreviousData, @@ -57,9 +57,12 @@ export type LogPivot = Pick; export default function AdminLogs({ shared, + landed, onPivot, }: { shared: Shared; + /** The filters of the entry Back/Forward landed on (the page reads the address bar, once). */ + landed: { logs: LogsUrlState } | null; /** Show only one request's, job's or run's lines (the page clears every other filter). */ onPivot: (pivot: LogPivot) => void; }) { @@ -84,18 +87,16 @@ export default function AdminLogs({ const [open, setOpen] = useState(null); useUrlMirror(logsParamUpdates(state)); // Back/Forward lands on another entry's filters; a dialog closing lands on these same ones, and - // changes nothing (Live and the open line stay). - const onLanded = useEffectEvent(() => { - const landed = logsParamsToState(new URLSearchParams(window.location.search)); - if (JSON.stringify(landed) === JSON.stringify(state)) return; - setOpen(null); - setState(landed); - }); - useEffect(() => { - const onPop = () => onLanded(); - window.addEventListener("popstate", onPop); - return () => window.removeEventListener("popstate", onPop); - }, []); + // changes nothing (Live and the open line stay). Adopted during the render that brings it, with + // the page's half. + const [seen, setSeen] = useState(landed); + if (landed !== seen) { + setSeen(landed); + if (landed && JSON.stringify(landed.logs) !== JSON.stringify(state)) { + setOpen(null); + setState(landed.logs); + } + } const update = (patch: Partial) => { setOpen(null); setState((s) => ({ ...s, ...patch })); @@ -127,15 +128,16 @@ export default function AdminLogs({ ? null : (logs.data?.pages[logs.data.pages.length - 1]?.next_cursor ?? null); // A read that failed while lines were on screen: they are kept, and the message says what the - // failure means for them. (With no lines, the list's own error state says it.) + // failure means for them. (With no lines, the list's own error state says it.) Without the facets + // the Source and Event menus offer only "Any", as on the other tabs, so the same words say it. const failure = !logs.data ? null : logs.isFetchNextPageError - ? "olderFailed" + ? "adminLogs.olderFailed" : logs.isError || (facets.isError && facets.data) - ? "stale" + ? "adminLogs.stale" : facets.isError - ? "countsFailed" + ? "activity.filtersFailed" : null; const retryFailed = () => { if (logs.isFetchNextPageError) @@ -229,7 +231,7 @@ export default function AdminLogs({ list={logs} rowCount={rows.length} rowsClassName="font-mono text-2xs" - failure={failure && { text: t(`adminLogs.${failure}`), onRetry: retryFailed }} + failure={failure && { text: t(failure), onRetry: retryFailed }} older={ olderCursor !== null ? { label: t("adminLogs.older"), onClick: loadOlder, disabled: logs.isFetchingNextPage } diff --git a/frontend/src/i18n/locales/en/adminLogs.json b/frontend/src/i18n/locales/en/adminLogs.json index 96b3a827..8071aa3f 100644 --- a/frontend/src/i18n/locales/en/adminLogs.json +++ b/frontend/src/i18n/locales/en/adminLogs.json @@ -18,7 +18,6 @@ "empty": "No lines match the filters.", "stale": "Couldn't refresh — what you see may be out of date", "olderFailed": "Couldn't load the older lines", - "countsFailed": "Couldn't load the filter counts", "count_one": "{{count}} line loaded · newest first", "count_other": "{{count}} lines loaded · newest first", "older": "Load older", diff --git a/frontend/src/i18n/locales/hu/adminLogs.json b/frontend/src/i18n/locales/hu/adminLogs.json index e1ddefc5..c0d97fb6 100644 --- a/frontend/src/i18n/locales/hu/adminLogs.json +++ b/frontend/src/i18n/locales/hu/adminLogs.json @@ -18,7 +18,6 @@ "empty": "Nincs a szűrésnek megfelelő sor.", "stale": "Nem sikerült frissíteni — amit látsz, elavult lehet", "olderFailed": "Nem sikerült betölteni a régebbi sorokat", - "countsFailed": "Nem sikerült betölteni a szűrők számait", "count_one": "{{count}} sor betöltve · a legújabb felül", "count_other": "{{count}} sor betöltve · a legújabb felül", "older": "Régebbiek betöltése", diff --git a/frontend/src/i18n/unusedKeys.contract.test.ts b/frontend/src/i18n/unusedKeys.contract.test.ts index 2a89b2d4..c7905892 100644 --- a/frontend/src/i18n/unusedKeys.contract.test.ts +++ b/frontend/src/i18n/unusedKeys.contract.test.ts @@ -16,12 +16,7 @@ import { describe, expect, it } from "vitest"; // keys — so it covers nothing; a key it reads that nothing else names is listed below. Plural forms // count through their base key. A key read in a way this cannot see goes in ALLOWED with the reason, // rather than the check being loosened for everyone. -const LOG_FAILURE = "AdminLogs reads `adminLogs.${failure}`, a word its query hook returns"; -const ALLOWED: Record = { - "adminLogs.olderFailed": LOG_FAILURE, - "adminLogs.stale": LOG_FAILURE, - "adminLogs.countsFailed": LOG_FAILURE, -}; +const ALLOWED: Record = {}; // Namespaces that are one backend enum each, and held key-for-value to it by backendIds.contract. const BACKEND_ENUMS = ["auditActions.", "quotaActions."]; From 5082636f151a0bc13dd5847ba9a7c16698fa577d Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 21:21:19 +0200 Subject: [PATCH 14/16] fix(activity): a run nothing here has open reads as interrupted, a rollback sums a stopped run's spend - The scheduler keeps one registry of the runs this process has open, for the interval jobs and the ad-hoc runs alike. A row the table still says is `running` but no run here has open (its outcome failed to write) reads as interrupted, as the next start's sweep will close it: no pulsing dot beside "No run recorded yet". - Migration 0109's downgrade sums a still-running run's quota events into its spend before the link to them is dropped, as the startup sweep would. - A test holds that a dialog closing onto the same address leaves an open log line open. --- backend/alembic/versions/0109_job_runs.py | 10 ++- backend/app/scheduler.py | 69 ++++++++++++++----- backend/tests/test_migration_0109.py | 11 +++ backend/tests/test_scheduler.py | 27 +++++++- .../src/components/AdminLogs.dom.test.tsx | 13 ++++ 5 files changed, 109 insertions(+), 21 deletions(-) diff --git a/backend/alembic/versions/0109_job_runs.py b/backend/alembic/versions/0109_job_runs.py index f75f2c5e..310be464 100644 --- a/backend/alembic/versions/0109_job_runs.py +++ b/backend/alembic/versions/0109_job_runs.py @@ -134,12 +134,18 @@ def downgrade() -> None: THEN jsonb_build_object('result', result) ELSE '{}'::jsonb END - || jsonb_build_object('trigger', trigger, 'quota_units', quota_units, + || jsonb_build_object('trigger', trigger, 'quota_units', units, 'started_at', started_at) || CASE WHEN target IS NOT NULL THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json, coalesce(finished_at, started_at) - FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st + FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st, + -- A run stopped mid-way has not summed its spend yet: sum it as the startup + -- sweep would, before the link to it is dropped. + CASE WHEN status = 'running' + THEN (SELECT coalesce(sum(q.units), 0) FROM quota_events q + WHERE q.job_run_id = job_runs.id) + ELSE quota_units END AS units FROM job_runs) AS r ORDER BY id """ diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 4e112a56..d7e4c82b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -45,10 +45,26 @@ _scheduler: BackgroundScheduler | None = None # run did is durable instead: its `job_runs` row (app/jobruns.py), which outlives a restart. _activity: dict[str, dict] = {} _activity_lock = threading.Lock() -# The ad-hoc runs this process has open (run id -> job), under the same lock. A row can stay -# `running` in the table after its outcome failed to write (_close), so the table cannot say what is -# going; this can, as the one process per database (app/jobruns.py) is the one running them. -_open_ad_hoc: dict[int, str] = {} +# The runs this process has open (run id -> job), under the same lock. A row can stay `running` in +# the table after its outcome failed to write (_close), so the table cannot say what is going; this +# can, as the one process per database (app/jobruns.py) is the one running them. +_open_runs: dict[int, str] = {} + + +def _opened(run_id: int, job: str) -> None: + with _activity_lock: + _open_runs[run_id] = job + + +def _closed(run_id: int) -> None: + with _activity_lock: + _open_runs.pop(run_id, None) + + +def _open_run_ids() -> dict[int, str]: + with _activity_lock: + return dict(_open_runs) + # Set (per-thread) by a manual "run now" trigger to the id of the admin who clicked, so that # run — and only that run, not the recurring scheduled ones — posts a completion notification @@ -129,6 +145,7 @@ def _job(name: str, fn) -> None: trigger = RunTrigger.SCHEDULED if actor_id is None else RunTrigger.MANUAL _record(name, progress=None) # running already claimed db = SessionLocal() + run_id = None try: try: run_id = jobruns.start(name, trigger, actor_id) @@ -136,11 +153,14 @@ def _job(name: str, fn) -> None: # No row to tell it in: the database is what failed. The log is the record. log.exception("job %s not started: its run could not be recorded", name) return + _opened(run_id, name) with logs.bind(run_id=run_id): _run(db, name, fn, run_id, actor_id) finally: # ALWAYS release the claim, even if the body raised before its own status update — so a # failure can never wedge the job as permanently "running". + if run_id is not None: + _closed(run_id) _record(name, running=False, progress=None) db.close() @@ -247,8 +267,12 @@ def scheduler_snapshot(db) -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} - going_id = max((r for r, j in _open_ad_hoc.items() if j == CHANNEL_BACKFILL), default=None) + # Open before the read or after it: a run that started or ended while the rows were read is + # still counted as the open run it was, never as a stray one. + open_runs = _open_run_ids() recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) + open_runs |= _open_run_ids() + going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) @@ -259,7 +283,7 @@ def scheduler_snapshot(db) -> dict: "next_run": next_runs.get(job_id), "running": a.get("running", False), "progress": a.get("progress"), - **_last_runs(recent.get(job_id, [])), + **_last_runs(recent.get(job_id, []), open_runs), } ) # Work a user starts on one target: no interval, no "run now" (it needs its target), but its @@ -285,7 +309,7 @@ def scheduler_snapshot(db) -> dict: if shown is not None and shown.target else None ), - **_last_runs(backfills), + **_last_runs(backfills, open_runs), } ] return { @@ -300,16 +324,27 @@ def scheduler_snapshot(db) -> dict: RECENT_RUNS = 3 -def _last_runs(runs: list) -> dict: +def _seen(run, open_runs: dict[int, str]) -> RunStatus: + """A run's status as the page tells it. A row `running` that no run here has open is one whose + outcome failed to write: the next start's sweep closes it as interrupted, so it reads so now — + not as a run still going beside a row that says nothing runs.""" + if run.status == RunStatus.RUNNING and run.id not in open_runs: + return RunStatus.INTERRUPTED + return RunStatus(run.status) + + +def _last_runs(runs: list, open_runs: dict[int, str]) -> dict: """A row's run fields from its newest runs (newest first): how the last one went, and the few before it for the history dots.""" run = runs[0] if runs else None + status = _seen(run, open_runs) if run is not None else None # A run still going has no outcome yet: only an ended run says how it went. - ended = run if run is not None and run.status != RunStatus.RUNNING else None + ended = run if status is not None and status != RunStatus.RUNNING else None return { - "status": ended.status if ended else None, + "status": status if ended else None, "last_started": iso(run.started_at) if run else None, - "last_finished": iso(run.finished_at) if run else None, + # A stray row has no end of its own: it ended, as far as anyone can tell, where it began. + "last_finished": iso(ended.finished_at or ended.started_at) if ended else None, "last_result": ended.summary if ended else None, # The outcome as data: the result, or on a failed run the failure the job named. "result": _outcome_data(ended), @@ -317,14 +352,14 @@ def _last_runs(runs: list) -> dict: "last_trigger": run.trigger if run else None, "last_run_id": run.id if run else None, "quota_units": ended.quota_units if ended else None, - "recent": [_run_brief(r) for r in runs], + "recent": [_run_brief(r, open_runs) for r in runs], } -def _run_brief(run) -> dict: +def _run_brief(run, open_runs: dict[int, str]) -> dict: return { "id": run.id, - "status": run.status, + "status": _seen(run, open_runs), "trigger": run.trigger, "started_at": iso(run.started_at), "finished_at": iso(run.finished_at), @@ -607,13 +642,11 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non # No row to tell it in: the database is what failed. The log is the record. log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return - with _activity_lock: - _open_ad_hoc[run_id] = job + _opened(run_id, job) try: _ad_hoc_work(job, fn, run_id) finally: - with _activity_lock: - _open_ad_hoc.pop(run_id, None) + _closed(run_id) def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: diff --git a/backend/tests/test_migration_0109.py b/backend/tests/test_migration_0109.py index 9fd08814..495d7af7 100644 --- a/backend/tests/test_migration_0109.py +++ b/backend/tests/test_migration_0109.py @@ -131,6 +131,13 @@ def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): ), {"u": uid}, ) + # What the run caught mid-way had spent: only its quota rows know it yet. + conn.execute( + text( + "INSERT INTO quota_events (action, units, job_run_id)" + " SELECT 'videos.list', 7, id FROM job_runs WHERE job = 'maintenance'" + ) + ) cfg = Config("alembic.ini") command.downgrade(cfg, "0108_folder_links") command.upgrade(cfg, "head") @@ -141,6 +148,9 @@ def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): kept = conn.execute( text("SELECT quota_units, finished_at - started_at FROM job_runs WHERE job = 'enrich'") ).one() + stopped = conn.execute( + text("SELECT quota_units FROM job_runs WHERE job = 'maintenance'") + ).scalar() stranded = conn.execute( text("SELECT count(*) FROM audit_log WHERE action = 'job_run'") ).scalar() @@ -151,4 +161,5 @@ def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): ("enrich", "manual", None, None, "ok"), ] assert tuple(kept) == (42, timedelta(minutes=30)) + assert stopped == 7 assert stranded == 0 diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index cb6f1ef7..32707328 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -460,13 +460,38 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use assert not scheduler._is_running("autotag") (run,) = _runs(db, "autotag") assert run.status == "running" # left for the next start's sweep, not rewritten as a failure + # Nothing here runs it: the page reads it as the sweep will close it, not as a run going on. + row = _row(db, "autotag") + assert (row["running"], row["status"], row["recent"][0]["status"]) == ( + False, + "interrupted", + "interrupted", + ) + assert row["last_finished"] is not None user = make_user("blip@example.com") calls.clear() scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") assert calls == ["ok"] # Its row is still `running`, but nothing here is: the page does not call it going. - assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False + (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert (row["running"], row["status"], row["recent"][0]["status"]) == ( + False, + "interrupted", + "interrupted", + ) + + +def test_a_run_going_on_reads_as_running_on_its_row(db): + seen = [] + scheduler._job("autotag", lambda _db: seen.append(_row(db, "autotag")) or {"tagged": 1}) + (row,) = seen + assert (row["running"], row["status"], row["recent"][0]["status"]) == (True, None, "running") + assert _row(db, "autotag")["recent"][0]["status"] == "ok" + + +def _row(db, job_id: str) -> dict: + return next(j for j in scheduler.scheduler_snapshot(db)["jobs"] if j["id"] == job_id) def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch): diff --git a/frontend/src/components/AdminLogs.dom.test.tsx b/frontend/src/components/AdminLogs.dom.test.tsx index fde06401..01580322 100644 --- a/frontend/src/components/AdminLogs.dom.test.tsx +++ b/frontend/src/components/AdminLogs.dom.test.tsx @@ -114,6 +114,19 @@ afterEach(() => { }); describe("AdminLogs", () => { + it("a dialog closing onto the same address leaves the open line open", async () => { + logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + fireEvent.click(lines()[0]!); + expect(lines()[0]!.getAttribute("aria-expanded")).toBe("true"); + // A closing dialog steps back over its own history entry: a popstate onto the same address. + act(() => { + window.dispatchEvent(new PopStateEvent("popstate")); + }); + expect(lines()[0]!.getAttribute("aria-expanded")).toBe("true"); + }); + it("a request id narrows the list to that request, with every other filter off", async () => { window.history.replaceState(null, "", "/?page=activity&at=logs&lgrp=error&aq=boom"); logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); From b842e4e871785d4bda5acaf6a74a2fb9e98e5add Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 21:40:52 +0200 Subject: [PATCH 15/16] fix(ui): a looping pulse stops for reduced motion; the e2e waits for skeleton bones, not for any pulse - Seven looping pulses (a running job's dot, the scheduler's progress bar, a queued download, the download tray, a playlist upload, a channel's full-history action, the setup wizard's test mail) ran regardless of prefers-reduced-motion, which docs/DESIGN.md promises to respect: each is motion-safe now, held by a contract test. - The a11y suite's "settled" wait took every aria-hidden pulse for a loading skeleton, so the Scheduler legend's running dot, which pulses for good, kept it waiting until it timed out. A skeleton bone now says it is one (data-skeleton) and the wait looks for that. --- frontend/e2e/a11y.spec.ts | 7 ++-- frontend/src/components/Channels.tsx | 4 ++- frontend/src/components/DownloadButton.tsx | 5 ++- frontend/src/components/DownloadTray.tsx | 2 +- frontend/src/components/RunStatusDot.tsx | 2 +- frontend/src/components/Scheduler.tsx | 2 +- frontend/src/components/SetupWizard.tsx | 4 ++- frontend/src/components/ui/Skeleton.tsx | 1 + .../src/components/usePlaylistActions.tsx | 2 +- .../src/lib/reducedMotion.contract.test.ts | 34 +++++++++++++++++++ 10 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 frontend/src/lib/reducedMotion.contract.test.ts diff --git a/frontend/e2e/a11y.spec.ts b/frontend/e2e/a11y.spec.ts index d62b31e1..42830509 100644 --- a/frontend/e2e/a11y.spec.ts +++ b/frontend/e2e/a11y.spec.ts @@ -13,11 +13,10 @@ import { gotoPage, openApp, openFirstVideo } from "./helpers"; // Chromium only (playwright.config.ts leaves it out of the other projects). const TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22aa"]; -/** Wait until nothing is loading: no QueryState spinner and no skeleton bone left. */ +/** Wait until nothing is loading: no QueryState spinner and no skeleton bone left. A bone says it is + * one (`data-skeleton`): a pulse alone is not loading — a running job's dot pulses as long as it runs. */ async function settled(page: Page): Promise { - await expect( - page.locator('[role="status"] .animate-spin, [aria-hidden="true"].animate-pulse'), - ).toHaveCount(0); + await expect(page.locator('[role="status"] .animate-spin, [data-skeleton]')).toHaveCount(0); } async function violations(page: Page): Promise { diff --git a/frontend/src/components/Channels.tsx b/frontend/src/components/Channels.tsx index 125709fe..fc1e6c85 100644 --- a/frontend/src/components/Channels.tsx +++ b/frontend/src/components/Channels.tsx @@ -515,7 +515,9 @@ export default function Channels() { id: "backfill", label: t("channels.backfillEverything"), title: t("channels.backfillEverythingHint"), - icon: , + icon: ( + + ), onSelect: () => deepAll.mutate(), disabled: deepAll.isPending, }, diff --git a/frontend/src/components/DownloadButton.tsx b/frontend/src/components/DownloadButton.tsx index e12c2f84..f68d5f38 100644 --- a/frontend/src/components/DownloadButton.tsx +++ b/frontend/src/components/DownloadButton.tsx @@ -54,7 +54,10 @@ export default function DownloadButton({ > {/* The state is in the glyph: downloaded or queued is the accent, a running one pulses. */} {open && ( diff --git a/frontend/src/components/DownloadTray.tsx b/frontend/src/components/DownloadTray.tsx index 9937cfca..bee55acc 100644 --- a/frontend/src/components/DownloadTray.tsx +++ b/frontend/src/components/DownloadTray.tsx @@ -98,7 +98,7 @@ function Progress({ job }: { job: QueueJob }) { style={{ width: `${job.progress}%` }} /> ) : ( - + )} {bytes && {job.progress}%} diff --git a/frontend/src/components/RunStatusDot.tsx b/frontend/src/components/RunStatusDot.tsx index cb3ba7f5..a1657c38 100644 --- a/frontend/src/components/RunStatusDot.tsx +++ b/frontend/src/components/RunStatusDot.tsx @@ -4,7 +4,7 @@ // Interrupted is a ring, not a fill: the job did not fail, but its work did not finish either — set // apart from a skipped run (the same colour, filled) and from a failure. const DOT_CLASS = { - running: "bg-accent animate-pulse", + running: "bg-accent motion-safe:animate-pulse", ok: "bg-success", error: "bg-danger", skipped: "bg-warn", diff --git a/frontend/src/components/Scheduler.tsx b/frontend/src/components/Scheduler.tsx index 1b1cd714..e4d501cf 100644 --- a/frontend/src/components/Scheduler.tsx +++ b/frontend/src/components/Scheduler.tsx @@ -238,7 +238,7 @@ function JobProgress({ p }: { p: SchedulerJob["progress"] }) { /> ) : ( // Indeterminate: total unknown, so a slim moving sliver instead of a fill. -
+
)}
diff --git a/frontend/src/components/SetupWizard.tsx b/frontend/src/components/SetupWizard.tsx index 6989b7f2..03f01ba8 100644 --- a/frontend/src/components/SetupWizard.tsx +++ b/frontend/src/components/SetupWizard.tsx @@ -259,7 +259,9 @@ export default function SetupWizard() { onClick={sendTest} disabled={testState === "sending" || !testTo.trim()} > - + {t("setup.smtp.test")}
diff --git a/frontend/src/components/ui/Skeleton.tsx b/frontend/src/components/ui/Skeleton.tsx index 5c1576ff..115fef2c 100644 --- a/frontend/src/components/ui/Skeleton.tsx +++ b/frontend/src/components/ui/Skeleton.tsx @@ -13,6 +13,7 @@ function Skeleton({ className, inline = false }: { className?: string; inline?: return (