diff --git a/.env.example b/.env.example index 376f9315..3875dac1 100644 --- a/.env.example +++ b/.env.example @@ -68,9 +68,11 @@ SMTP_FROM= # --- Scheduler --- # The background scheduler (subscription sync, backfill, enrichment) runs inside the app. -# DATABASE_URL is set for you by docker-compose to the bundled `db` service. If you ever run -# more than one instance against the same database, keep SCHEDULER_ENABLED=true on exactly one -# of them and false on the rest, to avoid double quota use and write races. +# DATABASE_URL is set for you by docker-compose to the bundled `db` service. Run ONE app instance +# per database: a starting instance closes every run still open as interrupted (it assumes the +# previous process is gone), and "is this job already running" is held in the process. A second +# instance would cut the first one's runs short and could run the same job twice. +# SCHEDULER_ENABLED=false keeps the recurring jobs off; "run now" still works. SCHEDULER_ENABLED=true # --- Files library & downloads --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 8092db61..8ff67e53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.108.0 — 2026-10-18 + +### New + +- Activity: one admin page for background runs, admin actions, the server log and quota, with one time range, account and search. A run opens to its outcome and its log lines, an admin action to its request's lines; old audit and log links open their tab. +- Scheduler: each job shows its last three runs (a run cut short by a restart is marked) and a button opens all of them; turning on a channel's full history is now a run of its own, listed under Ad-hoc runs. +- Configuration: how long the run history is kept (Activity › Run history, 180 days by default). Stats keeps your own numbers; the system quota view moved to Activity › Quota. + +### Fixed + +- Configuration: saving a setting with the value already in force now keeps it, instead of dropping the change. +- Configuration: the unused SponsorBlock default setting is gone. + ## 0.107.3 — 2026-10-17 ### New diff --git a/CLAUDE.md b/CLAUDE.md index 2acf4a0d..b57a4030 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,7 +63,7 @@ how it gets built. Concretely: ## Architecture shape (don't fight it) `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, -`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, +`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`, `sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`, `security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own @@ -92,8 +92,8 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory | Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) | | User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI | | Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` | -| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) | -| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter | +| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` and `jobruns` own their session) | +| Admin knob storage | `sysconfig` (values) | env / `app_state` / a table of its own (the job intervals had one) | | Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals | | Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers | | FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks | diff --git a/VERSION b/VERSION index 775915d4..3f667dbc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.107.3 +0.108.0 diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json index 824be795..af17ec70 100644 --- a/backend/.basedpyright/baseline.json +++ b/backend/.basedpyright/baseline.json @@ -630,41 +630,7 @@ } } ], - "./app/routes/admin_logs.py": [ - { - "code": "reportReturnType", - "range": { - "startColumn": 15, - "endColumn": 47, - "lineCount": 1 - } - } - ], "./app/routes/audit.py": [ - { - "code": "reportCallIssue", - "range": { - "startColumn": 17, - "endColumn": 98, - "lineCount": 1 - } - }, - { - "code": "reportAssignmentType", - "range": { - "startColumn": 17, - "endColumn": 98, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 22, - "endColumn": 97, - "lineCount": 1 - } - }, { "code": "reportAttributeAccessIssue", "range": { diff --git a/backend/alembic/versions/0109_job_runs.py b/backend/alembic/versions/0109_job_runs.py new file mode 100644 index 00000000..310be464 --- /dev/null +++ b/backend/alembic/versions/0109_job_runs.py @@ -0,0 +1,160 @@ +"""Background runs get a durable history + +Revision ID: 0109_job_runs +Revises: 0108_folder_links +Create Date: 2026-10-17 + +What a scheduler job last did lived in the API process's memory, gone at every restart; the audit +log kept a row for a run only when it was manual, failed, or changed something. Now every run is a +`job_runs` row, written when it starts and when it ends. A run's quota spend points at it +(`quota_events.job_run_id`), and so do the lines it logged (`log_events.run_id`, no foreign key: the +log writer runs behind the event). + +Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows +MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the +audit log. The downgrade moves every run back into the audit log as such a row (one still running as +interrupted, what the audit row has no column for in its `after`), then drops the table. +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0109_job_runs" +down_revision: str | None = "0108_folder_links" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "job_runs", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("job", sa.String(40), nullable=False), + sa.Column("trigger", sa.String(16), nullable=False), + sa.Column("actor_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="SET NULL")), + sa.Column("target", sa.String(128)), + sa.Column("status", sa.String(16), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True)), + sa.Column("summary", sa.Text()), + sa.Column("result", postgresql.JSONB()), + sa.Column("error", sa.Text()), + sa.Column("failure", postgresql.JSONB()), + sa.Column("quota_units", sa.Integer(), nullable=False, server_default="0"), + sa.CheckConstraint( + "status IN ('running', 'ok', 'error', 'skipped', 'interrupted')", + name="ck_job_runs_status", + ), + sa.CheckConstraint( + "trigger IN ('scheduled', 'manual', 'ad_hoc')", name="ck_job_runs_trigger" + ), + sa.CheckConstraint( + "(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished" + ), + ) + op.create_index("ix_job_runs_job_started", "job_runs", ["job", "started_at"]) + op.create_index("ix_job_runs_actor_id", "job_runs", ["actor_id"]) + op.create_index("ix_job_runs_started_at", "job_runs", ["started_at"]) + + op.add_column( + "quota_events", + sa.Column("job_run_id", sa.BigInteger(), sa.ForeignKey("job_runs.id", ondelete="SET NULL")), + ) + op.create_index("ix_quota_events_job_run_id", "quota_events", ["job_run_id"]) + + op.add_column("log_events", sa.Column("run_id", sa.BigInteger())) + op.create_index( + "ix_log_events_run", + "log_events", + ["run_id"], + postgresql_where=sa.text("run_id IS NOT NULL"), + ) + + # The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`, + # which a downgrade writes back (as it does a run's trigger, target, spend and start, which the + # audit row has no column for: its actor alone cannot say "manual" once that admin is deleted), + # so a rollback and a redeploy keep every run as it was. A status the run table does + # not know would fail the CHECK. + op.execute( + """ + INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at, + summary, result, error, failure, quota_units) + SELECT target_id, + coalesce(after::jsonb ->> 'trigger', + CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END), + actor_id, + after::jsonb ->> 'target', + after::jsonb ->> 'status', + coalesce((after::jsonb ->> 'started_at')::timestamptz, created_at), + created_at, + CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb ->> 'summary' END, + CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb -> 'result' END, + CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb ->> 'summary' END, + CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END, + coalesce((after::jsonb ->> 'quota_units')::int, 0) + FROM audit_log + WHERE action = 'job_run' + AND target_id IS NOT NULL + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted') + ORDER BY id + """ + ) + op.execute( + """ + DELETE FROM audit_log + WHERE action = 'job_run' + AND target_id IS NOT NULL + AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted') + """ + ) + + +def downgrade() -> None: + # A run still `running` is one the rollback stopped: it goes back as interrupted. What the audit + # row's own columns cannot say rides in `after` (its trigger, target, spend and start), so the + # upgrade restores them. + op.execute( + """ + INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at) + SELECT actor_id, + 'job_run', + 'job', + job, + left(job || ': ' || st + || coalesce(' — ' || CASE WHEN st = 'error' THEN error ELSE summary END, ''), + 255), + (jsonb_build_object( + 'status', st, + 'summary', CASE WHEN st = 'error' THEN error ELSE summary END) + || CASE + WHEN st = 'error' AND failure IS NOT NULL + THEN jsonb_build_object('result', failure) + WHEN st <> 'error' AND result IS NOT NULL + THEN jsonb_build_object('result', result) + ELSE '{}'::jsonb + END + || jsonb_build_object('trigger', trigger, 'quota_units', units, + 'started_at', started_at) + || CASE WHEN target IS NOT NULL + THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json, + coalesce(finished_at, started_at) + FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st, + -- A run stopped mid-way has not summed its spend yet: sum it as the startup + -- sweep would, before the link to it is dropped. + CASE WHEN status = 'running' + THEN (SELECT coalesce(sum(q.units), 0) FROM quota_events q + WHERE q.job_run_id = job_runs.id) + ELSE quota_units END AS units + FROM job_runs) AS r + ORDER BY id + """ + ) + op.drop_index("ix_log_events_run", table_name="log_events") + op.drop_column("log_events", "run_id") + op.drop_index("ix_quota_events_job_run_id", table_name="quota_events") + op.drop_column("quota_events", "job_run_id") + op.drop_index("ix_job_runs_started_at", table_name="job_runs") + op.drop_index("ix_job_runs_actor_id", table_name="job_runs") + op.drop_index("ix_job_runs_job_started", table_name="job_runs") + op.drop_table("job_runs") diff --git a/backend/alembic/versions/0110_intervals_in_sysconfig.py b/backend/alembic/versions/0110_intervals_in_sysconfig.py new file mode 100644 index 00000000..4730f333 --- /dev/null +++ b/backend/alembic/versions/0110_intervals_in_sysconfig.py @@ -0,0 +1,79 @@ +"""Job intervals move into sysconfig; the unread SponsorBlock default goes + +Revision ID: 0110_intervals_in_sysconfig +Revises: 0109_job_runs +Create Date: 2026-10-18 + +An admin's interval override lived in its own table, `scheduler_settings`, beside the one store every +other admin knob uses. Each override becomes the `job_interval_` sysconfig key and the table +goes. Only the jobs that still exist are copied: a row for any other id never did anything. + +The `sponsorblock_default` setting was offered on the Configuration page but nothing ever read it, so +its key is gone and its stored value with it — the downgrade cannot bring that value back. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0110_intervals_in_sysconfig" +down_revision: str | None = "0109_job_runs" +branch_labels: str | None = None +depends_on: str | None = None + +# The interval jobs as of this revision (a migration does not read the live code's list). +_JOBS = ( + "rss_poll", + "enrich", + "backfill", + "autotag", + "shorts", + "subscriptions", + "playlist_sync", + "maintenance", + "explore_cleanup", + "download_gc", + "plex_sync", + "plex_watch_sync", + "plex_watch_reconcile", + "tmdb_enrich", + "people_enrich", + "audit_gc", + "log_gc", +) +_PREFIX = "job_interval_" + + +def upgrade() -> None: + op.execute( + sa.text( + "INSERT INTO system_config (key, value, encrypted)" + " SELECT :prefix || job_id, interval_minutes::text, false FROM scheduler_settings" + " WHERE interval_minutes IS NOT NULL AND job_id IN :jobs" + " ON CONFLICT (key) DO NOTHING" + ).bindparams(sa.bindparam("jobs", expanding=True), prefix=_PREFIX, jobs=list(_JOBS)) + ) + op.drop_table("scheduler_settings") + op.execute("DELETE FROM system_config WHERE key = 'sponsorblock_default'") + + +def downgrade() -> None: + op.create_table( + "scheduler_settings", + sa.Column("job_id", sa.String(40), primary_key=True), + sa.Column("interval_minutes", sa.Integer(), nullable=False), + ) + op.execute( + sa.text( + "INSERT INTO scheduler_settings (job_id, interval_minutes)" + " SELECT substr(key, :start), value::int FROM system_config WHERE key IN :keys" + ).bindparams( + sa.bindparam("keys", expanding=True), + start=len(_PREFIX) + 1, + keys=[_PREFIX + j for j in _JOBS], + ) + ) + op.execute( + sa.text("DELETE FROM system_config WHERE key IN :keys").bindparams( + sa.bindparam("keys", expanding=True), keys=[_PREFIX + j for j in _JOBS] + ) + ) diff --git a/backend/alembic/versions/0111_audit_request_id.py b/backend/alembic/versions/0111_audit_request_id.py new file mode 100644 index 00000000..cbf2832d --- /dev/null +++ b/backend/alembic/versions/0111_audit_request_id.py @@ -0,0 +1,28 @@ +"""An audit row keeps the request that made it + +Revision ID: 0111_audit_request_id +Revises: 0110_intervals_in_sysconfig +Create Date: 2026-10-18 + +The activity page opens an admin action's log lines from its row: `audit_log.request_id` is the +request the change was made in, the same id `log_events.request_id` carries. Older rows have none — +the request that made them was never recorded. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0111_audit_request_id" +down_revision: str | None = "0110_intervals_in_sysconfig" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column("audit_log", sa.Column("request_id", sa.String(64))) + op.create_index("ix_audit_log_request_id", "audit_log", ["request_id"]) + + +def downgrade() -> None: + op.drop_index("ix_audit_log_request_id", table_name="audit_log") + op.drop_column("audit_log", "request_id") diff --git a/backend/app/README.md b/backend/app/README.md index 6ff33ab1..7a9ac54f 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -21,7 +21,8 @@ Three layers, by *who* runs the code — not by feature: `issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker. 3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth` (with `apitokens`, a machine's - folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`, + folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`, `jobruns` (every + background run's durable row, its words in `runstatus`), `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the @@ -36,7 +37,7 @@ pre-abstract for a single caller; don't leave shared logic duplicated once there ## Transaction ownership This follows CLAUDE.md's canonical rule verbatim — **"the handler/job-wrapper commits; helpers flush -(only `quota` owns its session)"** — so read it that way, not as an exception to it. The commit belongs +(only `quota` and `jobruns` own their session)"** — so read it that way, not as an exception to it. The commit belongs to the **operation owner**; anything called *inside* another transaction only stages its change (`db.add` / `flush`) and lets the owner commit, so the whole operation is one transaction. `audit.record()` is the reference: it adds the audit row and the caller commits it together with the @@ -45,7 +46,7 @@ mutation it records, so an audit entry can never exist without its change (or vi - The **operation owner** is usually the route handler. A `service.*` operation the handler delegates a whole transition to (e.g. `service.cancel`) *is* that owner and commits its one transition — it isn't a mid-operation helper. A helper that owns its OWN short-lived session (the worker's `jobdb` writes, - the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes. + `jobruns`' run rows — the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes. - `SessionLocal` is `expire_on_commit=False`: after a raw `UPDATE` + commit, an ORM object you already loaded is **stale**. Read the fresh value from the row (RETURNING, or a re-`SELECT`), not the ORM attribute — see `service.decr_ref_count`. @@ -67,8 +68,9 @@ mutation it records, so an audit entry can never exist without its change (or vi - **`app.config.settings`** — boot/env config (DB URL, download root, worker concurrency): read once at startup, not user-tunable at runtime. - **`sysconfig.SPECS`** — the runtime admin knobs (DB-override-or-default). A new admin-tunable value - goes in `SPECS`; don't invent a fourth store. (`state.py`'s `maintenance_batch` predates this and is - slated to fold into `sysconfig` — do not add more like it.) + goes in `SPECS`; don't invent a fourth store. Each spec states when a change takes effect + (`applies`, required): `live` keys are read where they are used, never at import (test-held). The + scheduler's job intervals are `SPECS` keys too, edited on the Scheduler page, not the Configuration page. ## The gate diff --git a/backend/app/audit.py b/backend/app/audit.py index d014d313..0123a173 100644 --- a/backend/app/audit.py +++ b/backend/app/audit.py @@ -1,9 +1,9 @@ -"""Admin/scheduler audit trail: an append-only record of who changed what and when. +"""Admin audit trail: an append-only record of who changed what and when. -Design (locked): log at ACTION / job-run granularity, NEVER per-row — a scheduler run that -touches thousands of videos is ONE audit row summarising the outcome. Automatic job runs are -only logged when they actually changed something (or errored); all admin actions and all manual -runs are always logged. Secrets are never stored (config changes log the key name only). +Design (locked): log at ACTION granularity, NEVER per-row — an action that touches thousands of +rows is ONE audit row summarising the outcome. Background runs are not actions: they have their +own history (app/jobruns.py). Secrets are never stored (config changes log the key name only). +Each row keeps the request that made it, so the request's log lines are one click away. `record()` only db.add()s the row — the caller commits it in the SAME transaction as the mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops @@ -15,6 +15,7 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import delete from sqlalchemy.orm import Session +from app import logs from app.models import AuditLog @@ -48,8 +49,6 @@ class AuditAction: MAINTENANCE_TUNE = "maintenance_tune" SYNC_PAUSE = "sync_pause" SYNC_RESUME = "sync_resume" - # Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic) - JOB_RUN = "job_run" # Per-user download cookie/auth credential set or removed (the value is NEVER recorded) DOWNLOAD_CREDENTIAL_SET = "download_credential_set" DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete" @@ -104,6 +103,7 @@ def record( summary=(summary or None) and summary[:255], before=before, after=after, + request_id=logs.current_request_id(), ) ) diff --git a/backend/app/config.py b/backend/app/config.py index fd477645..b1279f0c 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -268,10 +268,12 @@ class Settings(BaseSettings): # YouTube quota history (quota_events): days kept (0 = forever). A year, so the admin usage view's # longest window (365 days) stays whole; the same daily cleanup job prunes it. quota_retention_days: int = 365 + # Background-run history (job_runs): days kept (0 = forever). The audit log's 180, which is how + # long a run lived there before it moved here: ~1.8k rows a day at the default intervals + # (measured 2026-10-17), ~320k at the limit. The same daily cleanup job prunes it. + job_run_retention_days: int = 180 # On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat". download_layout: DownloadLayout = "plex" - # Default SponsorBlock (skip sponsor segments) for new custom profiles. - sponsorblock_default: bool = False # --- Diagnostics / logging --- # BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py new file mode 100644 index 00000000..7beb7572 --- /dev/null +++ b/backend/app/jobruns.py @@ -0,0 +1,131 @@ +"""The durable history of background runs — the `job_runs` table. + +A run is written twice: a `running` row when it starts, its outcome when it ends. Each write is its +OWN short session, committed at once, for the reason `quota.record_usage` gives: the run's own work +may roll back (its `except: db.rollback()`), and the history of what happened must not go with it — +nor may this bookkeeping commit the job's half-done session. + +Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a +database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its +startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open. +""" + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import delete, func, select, union_all, update +from sqlalchemy.engine import CursorResult +from sqlalchemy.orm import Session + +from app.db import SessionLocal +from app.models import JobRun, QuotaEvent +from app.runstatus import RunStatus, RunTrigger + + +def start(job: str, trigger: RunTrigger, actor_id: int | None, target: str | None = None) -> int: + """Open a run's row; returns its id.""" + with SessionLocal() as s: + row = JobRun( + job=job, + trigger=trigger, + actor_id=actor_id, + target=target, + status=RunStatus.RUNNING, + started_at=datetime.now(timezone.utc), + ) + s.add(row) + s.commit() + return row.id + + +def finish( + run_id: int, + status: RunStatus, + *, + summary: str | None = None, + result: dict | None = None, + error: str | None = None, + failure: dict | None = None, +) -> None: + """Close a run with its outcome and what it spent. Conditional on the row still running: a row + the startup sweep already called interrupted is not rewritten by a thread that outlived it.""" + with SessionLocal() as s: + units = s.scalar( + select(func.coalesce(func.sum(QuotaEvent.units), 0)).where( + QuotaEvent.job_run_id == run_id + ) + ) + s.execute( + update(JobRun) + .where(JobRun.id == run_id, JobRun.status == RunStatus.RUNNING) + .values( + status=status, + finished_at=datetime.now(timezone.utc), + summary=summary, + result=result, + error=error, + failure=failure, + quota_units=units or 0, + ) + ) + s.commit() + + +def interrupt_leftovers() -> int: + """Every row a previous process left `running` → `interrupted`, with what it spent until then + (its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler + can open a new run. Returns how many.""" + spent = ( + select(func.coalesce(func.sum(QuotaEvent.units), 0)) + .where(QuotaEvent.job_run_id == JobRun.id) + .scalar_subquery() + ) + with SessionLocal() as s: + res = s.execute( + update(JobRun) + .where(JobRun.status == RunStatus.RUNNING) + .values( + status=RunStatus.INTERRUPTED, + finished_at=datetime.now(timezone.utc), + quota_units=spent, + ) + ) + s.commit() + return res.rowcount if isinstance(res, CursorResult) else 0 + + +def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]: + """Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a + short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window + over the whole table grew with every run kept).""" + if not jobs: + return {} + newest = union_all( + *( + select(JobRun.id) + .where(JobRun.job == job) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(per_job) + for job in jobs + ) + ).subquery() + rows = db.execute( + select(JobRun) + .where(JobRun.id.in_(select(newest.c.id))) + .order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc()) + ).scalars() + out: dict[str, list[JobRun]] = {} + for r in rows: + out.setdefault(r.job, []).append(r) + return out + + +def gc(db: Session, retention_days: int) -> int: + """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is + never reaped. The caller commits.""" + if retention_days <= 0: + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + res = db.execute( + delete(JobRun).where(JobRun.started_at < cutoff, JobRun.status != RunStatus.RUNNING) + ) + return res.rowcount if isinstance(res, CursorResult) else 0 diff --git a/backend/app/logs.py b/backend/app/logs.py index 62ff3609..3cd315fa 100644 --- a/backend/app/logs.py +++ b/backend/app/logs.py @@ -10,8 +10,8 @@ the logs were both unjoinable and leaky. `configure_logging()` replaces all four Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line: ts · level · sev · source · logger · event · msg · attrs plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`, -`job_id`) and the app version. That shape is what the admin log console filters on and what a Gitea -issue attachment will carry, so it is deliberately flat and stable. +`job_id`, `run_id`) and the app version. That shape is what the admin log console filters on and +what a Gitea issue attachment will carry, so it is deliberately flat and stable. Call sites stay ordinary stdlib logging: @@ -81,15 +81,28 @@ _debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar( "log_debug_session", default=None ) _job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None) +# The background run (app/jobruns.py) this context is doing; quota's spend is tagged with it too. +_run_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_run_id", default=None) _CONTEXT_VARS = { "request_id": _request_id, "user_id": _user_id, "debug_session_id": _debug_session, "job_id": _job_id, + "run_id": _run_id, } +def current_request_id() -> str | None: + """The HTTP request this context is serving, if any — the request middleware sets it.""" + return _request_id.get() + + +def current_run_id() -> int | None: + """The background run this context is doing, if any — `bind(run_id=…)` sets it.""" + return _run_id.get() + + class _RequestScope: """What the auth dependency learns about a request, for every line logged after it. diff --git a/backend/app/logsink.py b/backend/app/logsink.py index c63cea8f..2310ebeb 100644 --- a/backend/app/logsink.py +++ b/backend/app/logsink.py @@ -362,6 +362,7 @@ def serialize(row: LogEvent) -> dict: "request_id": row.request_id, "user_id": row.user_id, "job_id": row.job_id, + "run_id": row.run_id, "debug_session_id": row.debug_session_id, "version": row.version, } diff --git a/backend/app/main.py b/backend/app/main.py index 9fd2f132..1c5c00c6 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -52,6 +52,7 @@ from app.routes import ( # noqa: E402 — logging is configured first plex as plex_routes, public as public_routes, quota, + runs as runs_routes, saved_views, scheduler as scheduler_routes, search as search_routes, @@ -345,6 +346,7 @@ app.include_router(config_routes.router) app.include_router(scheduler_routes.router) app.include_router(audit_routes.router) app.include_router(admin_logs_routes.router) +app.include_router(runs_routes.router) app.include_router(quota.router) app.include_router(version.router) app.include_router(setup_routes.router) diff --git a/backend/app/models.py b/backend/app/models.py index f7c4567c..9ebfbe74 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -24,6 +24,7 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from app.db import Base +from app.runstatus import RunStatus, RunTrigger, sql_in from app.utils import canonical_email # Every user-supplied display string in this schema shares one width, and it lives HERE so the @@ -563,14 +564,20 @@ class QuotaEvent(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), index=True ) + # The background run that spent it (app/jobruns.py); NULL = a request's own spend. SET NULL: a + # reaped run leaves the spend in the budget's history. + job_run_id: Mapped[int | None] = mapped_column( + ForeignKey("job_runs.id", ondelete="SET NULL"), index=True + ) class AuditLog(Base): - """Append-only admin/scheduler audit trail: who did what, when, and (where it applies) - the before->after values. `actor_id` is the acting admin; NULL means the scheduler / - background did it (SET NULL on user delete keeps the trail). Logged at ACTION / job-run - granularity — never per-row — so a scheduler run that touches thousands of videos is ONE - row summarising the outcome. `before`/`after` are small JSON snapshots (secrets excluded).""" + """Append-only admin audit trail: who did what, when, and (where it applies) the before->after + values. `actor_id` is the acting admin; NULL means the background did it (SET NULL on user + delete keeps the trail). Logged per ACTION, never per row; a background run is not an action + and has its own history (`JobRun`). `before`/`after` are small JSON snapshots (secrets + excluded). `request_id` is the request that made the change, so its log lines are one click + away (NULL for rows written outside a request, and for rows older than the column).""" __tablename__ = "audit_log" @@ -587,6 +594,7 @@ class AuditLog(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), index=True ) + request_id: Mapped[str | None] = mapped_column(String(64), index=True) class LogEvent(Base): @@ -609,6 +617,8 @@ class LogEvent(Base): "debug_session_id", postgresql_where="debug_session_id IS NOT NULL", ), + # A run's lines are `WHERE run_id = …`; request lines carry none. + Index("ix_log_events_run", "run_id", postgresql_where="run_id IS NOT NULL"), ) id: Mapped[int] = mapped_column(BigInteger, primary_key=True) @@ -624,7 +634,11 @@ class LogEvent(Base): request_id: Mapped[str | None] = mapped_column(String(64), index=True) user_id: Mapped[int | None] = mapped_column(Integer) debug_session_id: Mapped[str | None] = mapped_column(String(64)) + # A DOWNLOAD job's id (download_jobs) — not a background run, which is `run_id`. job_id: Mapped[int | None] = mapped_column(Integer) + # The background run (job_runs) that logged it. No foreign key, as `user_id`: the writer runs + # behind the event, and a run reaped meanwhile must not fail the batch its lines travel in. + run_id: Mapped[int | None] = mapped_column(BigInteger) version: Mapped[str | None] = mapped_column(String(64)) @@ -689,15 +703,44 @@ class AppState(Base): setup_token_hash: Mapped[str | None] = mapped_column(String(64)) -class SchedulerSetting(Base): - """Admin override for a scheduler job's run interval (minutes). One row per job id; - absent = use the env/config default. DB-backed so it's editable from the admin UI at - runtime without a redeploy (env stays a fallback).""" +class JobRun(Base): + """One run of a background job — scheduled, an admin's "run now", or a user's ad-hoc work on one + target — written when it starts and again when it ends (app/jobruns.py), each in its own + committed session, so the history survives the job's own rollback and a restart. A row left + `running` by a process that died is set `interrupted` at the next startup. - __tablename__ = "scheduler_settings" + `quota_units` is what the run spent of the YouTube budget: the `quota_events` rows carrying its + id, summed when it ends. `summary` is the English one-line fallback; `result` / `failure` are the + same outcome as data, which the client names in the reader's language.""" - job_id: Mapped[str] = mapped_column(String(40), primary_key=True) - interval_minutes: Mapped[int] = mapped_column(Integer) + __tablename__ = "job_runs" + __table_args__ = ( + Index("ix_job_runs_job_started", "job", "started_at"), + CheckConstraint(f"status IN ({sql_in(RunStatus)})", name="ck_job_runs_status"), + CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"), + # Only a run still going has no end. + CheckConstraint( + f"(status = '{RunStatus.RUNNING}') = (finished_at IS NULL)", name="ck_job_runs_finished" + ), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + job: Mapped[str] = mapped_column(String(40)) + trigger: Mapped[str] = mapped_column(String(16)) + # Who started it: the admin of a manual run, the user of an ad-hoc one; NULL = the schedule. + actor_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), index=True + ) + # What an ad-hoc run worked on (a channel id); NULL for the recurring jobs. + target: Mapped[str | None] = mapped_column(String(128)) + status: Mapped[str] = mapped_column(String(16)) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + summary: Mapped[str | None] = mapped_column(Text) + result: Mapped[dict | None] = mapped_column(JSONB) + error: Mapped[str | None] = mapped_column(Text) + failure: Mapped[dict | None] = mapped_column(JSONB) + quota_units: Mapped[int] = mapped_column(Integer, default=0, server_default="0") class SystemConfig(Base): diff --git a/backend/app/quota.py b/backend/app/quota.py index c0ef1db5..6f2e5c81 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -16,7 +16,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session -from app import sysconfig +from app import logs, sysconfig from app.db import SessionLocal from app.models import ApiQuotaUsage, QuotaEvent @@ -204,6 +204,14 @@ def record_usage(units: int) -> None: ) with SessionLocal() as qs: qs.execute(stmt) - # Audit detail: who/what spent it (per-user attribution; NULL actor = system). - qs.add(QuotaEvent(user_id=_actor_id.get(), action=_action.get(), units=units)) + # Audit detail: who/what spent it (per-user attribution; NULL actor = system), and the + # background run it belongs to, so the run's row can say what it cost. + qs.add( + QuotaEvent( + user_id=_actor_id.get(), + action=_action.get(), + units=units, + job_run_id=logs.current_run_id(), + ) + ) qs.commit() diff --git a/backend/app/routes/_account.py b/backend/app/routes/_account.py new file mode 100644 index 00000000..84976e86 --- /dev/null +++ b/backend/app/routes/_account.py @@ -0,0 +1,17 @@ +"""The account filter the activity page's lists share (runs, admin actions, server log, quota): one +filter bar drives them all, so the value it sends means the same on every tab. + +The value is an account id, or `none` — the rows no account made (the background) — a value to filter +on, not the absence of a filter. +""" + +from sqlalchemy import ColumnElement + +NONE = "none" +# The `user` query parameter's shape on every route that takes it. +PATTERN = r"^(none|[0-9]{1,10})$" + + +def condition(column, value: str) -> ColumnElement[bool]: + """`column` (an account id column) narrowed to `value`.""" + return column.is_(None) if value == NONE else column == int(value) diff --git a/backend/app/routes/_keyset.py b/backend/app/routes/_keyset.py new file mode 100644 index 00000000..f57368a4 --- /dev/null +++ b/backend/app/routes/_keyset.py @@ -0,0 +1,66 @@ +"""Keyset paging over `(timestamp desc, id desc)`, shared by the admin history lists (server log, +audit log, background runs). + +A timestamp alone is not unique (a browser log row's `ts` is clamped into its consent window; a run +and its siblings can start in the same microsecond), and an offset would skip or repeat rows while +writers add new ones at the top — so a page continues from the last row's `(timestamp, id)`. +""" + +import base64 +import binascii +import json +from datetime import datetime, timezone + +from fastapi import HTTPException +from sqlalchemy import and_, or_, select +from sqlalchemy.orm import Session + +from app.models import User + +PAGE_MAX = 200 + + +def utc(value: datetime | None) -> datetime | None: + """A bound without a zone is UTC — the columns are `timestamptz`, and a naive value would be read + in the database session's zone instead.""" + if value is None or value.tzinfo is not None: + return value + return value.replace(tzinfo=timezone.utc) + + +def encode_cursor(ts: datetime, row_id: int) -> str: + return base64.urlsafe_b64encode(json.dumps([ts.isoformat(), row_id]).encode()).decode() + + +def decode_cursor(cursor: str) -> tuple[datetime, int]: + try: + ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode())) + at = datetime.fromisoformat(ts) + return (at if at.tzinfo is not None else at.replace(tzinfo=timezone.utc)), int(row_id) + except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc: + raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc + + +def older_than(ts_col, id_col, cursor: str): + """The rows after `cursor` in `(ts desc, id desc)` order. The `ts <=` conjunct is implied by the + OR; it is there so the planner can bound the timestamp index scan instead of filtering every + newer row first.""" + ts, row_id = decode_cursor(cursor) + return and_(ts_col <= ts, or_(ts_col < ts, and_(ts_col == ts, id_col < row_id))) + + +def page(rows: list, limit: int, ts_of) -> tuple[list, str | None]: + """`rows` was read with `limit + 1` (to know whether there is a next page without a count query): + the page itself, and the cursor that continues after it (None on the last page).""" + more = len(rows) > limit + rows = rows[:limit] + return rows, encode_cursor(ts_of(rows[-1]), rows[-1].id) if more else None + + +def accounts(db: Session, user_ids: set[int]) -> dict[str, str]: + """The accounts a page names, by id → address. An id with no account left (erased after the row + was written) is simply absent, and the client shows the bare id.""" + if not user_ids: + return {} + rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all() + return {str(uid): email for uid, email in rows} diff --git a/backend/app/routes/admin_logs.py b/backend/app/routes/admin_logs.py index 8202cafd..72a36631 100644 --- a/backend/app/routes/admin_logs.py +++ b/backend/app/routes/admin_logs.py @@ -8,33 +8,29 @@ What a row holds was redacted when it was written (`logs.record_fields`, and the browser's rows), so this serves the columns as they are; a value the redactor took out reads as `[redacted]` / `[email]`, which is the point — the console shows that something was there. -Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent -window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds -new ones at the top. The live tail re-reads the first page rather than asking for "ids after N": +Paging is a keyset over `(ts desc, id desc)` (`_keyset`). The live tail re-reads the first page rather than asking for "ids after N": several processes write this table, and a batch's ids are taken when it inserts, not when it commits, so a slower batch can land below an id the tail already passed — and would never show. """ -import base64 -import binascii -import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime from typing import Literal -from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import Text, and_, cast, func, or_, select +from fastapi import APIRouter, Depends, Query +from sqlalchemy import Text, cast, func, or_, select from sqlalchemy.orm import Session from app import logsink from app.auth import admin_user from app.db import get_db from app.models import LogEvent, User +from app.routes import _account as account +from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc from app.utils import like_escape router = APIRouter(prefix="/api/admin/logs", tags=["admin"]) -PAGE_MAX = 200 # A facet list is a chip row, not a report: past this many distinct events the rest are reachable # through the free-text search, and the response stays small whatever a self-host logs. FACET_EVENTS_MAX = 100 @@ -42,38 +38,9 @@ FACET_EVENTS_MAX = 100 Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # The filters a facet can leave out of its own count (`LogFilters.conditions`). Facet = Literal["levels", "sources", "user", "event"] -# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous -# request) — a real value to filter on, not the absence of a filter. -NONE = "none" - - -def _utc(value: datetime | None) -> datetime | None: - """A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read - in the database session's zone instead.""" - if value is None or value.tzinfo is not None: - return value - return value.replace(tzinfo=timezone.utc) - - -def _encode_cursor(row: LogEvent) -> str: - return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode() - - -def _decode_cursor(cursor: str) -> tuple[datetime, int]: - try: - ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode())) - return _utc(datetime.fromisoformat(ts)), int(row_id) - except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc: - raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc - - -def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]: - """The accounts a page names, by id → address. An id with no account left (erased after the - line was written) is simply absent, and the client shows the bare id.""" - if not user_ids: - return {} - rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all() - return {str(uid): email for uid, email in rows} +# `none` selects the rows that have no event code, as the account filter's `none` (app/routes/_account) +# selects the rows of no account — a real value to filter on, not the absence of a filter. +NONE = account.NONE @dataclass(frozen=True) @@ -86,6 +53,7 @@ class LogFilters: event: str | None request_id: str | None job_id: int | None + run_id: int | None q: str | None since: datetime | None until: datetime | None @@ -100,11 +68,7 @@ class LogFilters: if self.sources and skip != "sources": out.append(LogEvent.source.in_(self.sources)) if self.user is not None and skip != "user": - out.append( - LogEvent.user_id.is_(None) - if self.user == NONE - else LogEvent.user_id == int(self.user) - ) + out.append(account.condition(LogEvent.user_id, self.user)) if self.event is not None and skip != "event": out.append( LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event @@ -113,10 +77,12 @@ class LogFilters: out.append(LogEvent.request_id == self.request_id) if self.job_id is not None: out.append(LogEvent.job_id == self.job_id) + if self.run_id is not None: + out.append(LogEvent.run_id == self.run_id) if self.since is not None: - out.append(LogEvent.ts >= _utc(self.since)) + out.append(LogEvent.ts >= utc(self.since)) if self.until is not None: - out.append(LogEvent.ts < _utc(self.until)) + out.append(LogEvent.ts < utc(self.until)) if self.q and self.q.strip(): like = f"%{like_escape(self.q.strip())}%" out.append( @@ -134,17 +100,20 @@ class LogFilters: def log_filters( levels: list[Level] = Query(default=[]), sources: list[str] = Query(default=[], max_length=8), - user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"), + user: str | None = Query(default=None, pattern=account.PATTERN), event: str | None = Query( default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$" ), request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"), job_id: int | None = Query(default=None, ge=1), + run_id: int | None = Query(default=None, ge=1), q: str | None = Query(default=None, max_length=200), since: datetime | None = None, until: datetime | None = None, ) -> LogFilters: - return LogFilters(list(levels), list(sources), user, event, request_id, job_id, q, since, until) + return LogFilters( + list(levels), list(sources), user, event, request_id, job_id, run_id, q, since, until + ) @router.get("") @@ -159,28 +128,20 @@ def list_logs( continues from a previous page's `next_cursor`.""" query = select(LogEvent).where(*filters.conditions()) if before is not None: - ts, row_id = _decode_cursor(before) - # The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the - # `ts` index scan instead of filtering every newer row first. - query = query.where( - and_( - LogEvent.ts <= ts, - or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)), - ) - ) - - # One more than asked, to know whether there is a next page without a second count query. - rows = ( - db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1)) - .scalars() - .all() + query = query.where(older_than(LogEvent.ts, LogEvent.id, before)) + rows, cursor = page( + list( + db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1)) + .scalars() + .all() + ), + limit, + lambda r: r.ts, ) - more = len(rows) > limit - rows = rows[:limit] return { "items": [logsink.serialize(r) for r in rows], - "accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}), - "next_cursor": _encode_cursor(rows[-1]) if more else None, + "accounts": accounts(db, {r.user_id for r in rows if r.user_id is not None}), + "next_cursor": cursor, } @@ -211,6 +172,6 @@ def log_facets( "sources": counts(LogEvent.source, "sources"), "events": counts(LogEvent.event, "event", FACET_EVENTS_MAX), "users": [{**u, "value": str(u["value"])} for u in users], - "accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}), + "accounts": accounts(db, {u["value"] for u in users if u["value"] != NONE}), "total": db.scalar(total) or 0, } diff --git a/backend/app/routes/audit.py b/backend/app/routes/audit.py index 1c007231..91e82c78 100644 --- a/backend/app/routes/audit.py +++ b/backend/app/routes/audit.py @@ -1,67 +1,144 @@ """Admin audit log API: read the append-only who/what/when trail, and clear it. -Admin-only. The client renders these rows in a DataTable (client-side sort/filter/paginate), so -this returns the most recent window of rows (capped) plus the true total, rather than paging -server-side — the log stays small in practice (state-changing runs + admin actions only).""" +Admin-only. The activity page reads it a page at a time, newest first, filtered on the server like +the server log (`_keyset` paging), so the whole trail is reachable however long it grows.""" -from fastapi import APIRouter, Depends -from sqlalchemy import delete, func, select +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from fastapi import APIRouter, Depends, Query +from sqlalchemy import Text, cast, delete, func, or_, select from sqlalchemy.orm import Session from app import audit from app.audit import AuditAction from app.auth import admin_user from app.db import get_db -from app.models import AuditLog, User -from app.utils import iso +from app.models import AuditLog, LogEvent, User +from app.routes import _account as account +from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc +from app.utils import iso, like_escape router = APIRouter(prefix="/api/admin/audit", tags=["admin"]) -# Cap the client-side window. The audit log is small by design; if it ever outgrows this we'd -# switch the DataTable to server-side paging. -MAX_ROWS = 2000 - -def _serialize(row: AuditLog, actors: dict[int, str]) -> dict: +def _serialize(row: AuditLog, lines: dict[str, int]) -> dict: return { "id": row.id, "created_at": iso(row.created_at), - # actor_id NULL = the scheduler/background did it (client shows "System"); a set id that - # can't be resolved would mean a deleted user, but SET NULL makes that NULL — so present. - "actor": actors.get(row.actor_id) if row.actor_id is not None else None, + # NULL = the background did it (the client shows "System"); the address is in `accounts`. + "actor_id": row.actor_id, "action": row.action, "target_type": row.target_type, "target_id": row.target_id, "summary": row.summary, "before": row.before, "after": row.after, + "request_id": row.request_id, + # How many of that request's log lines are still kept: they go after days, the row after + # months, and a link to none would open an empty list. + "log_lines": lines.get(row.request_id, 0) if row.request_id else 0, } +@dataclass(frozen=True) +class AuditFilters: + action: str | None + actor: str | None + q: str | None + since: datetime | None + until: datetime | None + + def conditions(self, skip: Literal["action"] | None = None) -> list: + """The WHERE conditions, ANDed. `skip="action"` leaves the action filter out, so the action + menu counts every action under the other filters (the way back from a pick).""" + out = [] + if self.action is not None and skip != "action": + out.append(AuditLog.action == self.action) + if self.actor is not None: + out.append(account.condition(AuditLog.actor_id, self.actor)) + if self.since is not None: + out.append(AuditLog.created_at >= utc(self.since)) + if self.until is not None: + out.append(AuditLog.created_at < utc(self.until)) + if self.q and self.q.strip(): + like = f"%{like_escape(self.q.strip())}%" + out.append( + or_( + AuditLog.summary.ilike(like), + AuditLog.target_id.ilike(like), + AuditLog.action.ilike(like), + AuditLog.request_id.ilike(like), + cast(AuditLog.before, Text).ilike(like), + cast(AuditLog.after, Text).ilike(like), + ) + ) + return out + + +def audit_filters( + action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"), + user: str | None = Query(default=None, pattern=account.PATTERN), + q: str | None = Query(default=None, max_length=200), + since: datetime | None = None, + until: datetime | None = None, +) -> AuditFilters: + return AuditFilters(action, user, q, since, until) + + @router.get("") def list_audit( - limit: int = 500, + filters: AuditFilters = Depends(audit_filters), + before: str | None = Query(default=None, max_length=200), + limit: int = Query(default=100, ge=1, le=PAGE_MAX), _: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - limit = max(1, min(limit, MAX_ROWS)) - total = db.scalar(select(func.count()).select_from(AuditLog)) or 0 - rows = ( - db.execute(select(AuditLog).order_by(AuditLog.created_at.desc()).limit(limit)) - .scalars() - .all() + """One page of rows, newest first, matching every filter given; `before` continues from a + previous page's `next_cursor`.""" + query = select(AuditLog).where(*filters.conditions()) + if before is not None: + query = query.where(older_than(AuditLog.created_at, AuditLog.id, before)) + order = (AuditLog.created_at.desc(), AuditLog.id.desc()) + rows, cursor = page( + list(db.execute(query.order_by(*order).limit(limit + 1)).scalars().all()), + limit, + lambda r: r.created_at, ) - actor_ids = {r.actor_id for r in rows if r.actor_id is not None} - actors: dict[int, str] = {} - if actor_ids: - actors = dict(db.execute(select(User.id, User.email).where(User.id.in_(actor_ids))).all()) + requests = {r.request_id for r in rows if r.request_id} + lines: dict[str, int] = {} + if requests: + counted = db.execute( + select(LogEvent.request_id, func.count()) + .where(LogEvent.request_id.in_(requests)) + .group_by(LogEvent.request_id) + ) + lines = {rid: n for rid, n in counted.tuples() if rid is not None} return { - "items": [_serialize(r, actors) for r in rows], - "total": total, - "returned": len(rows), + "items": [_serialize(r, lines) for r in rows], + "accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}), + "next_cursor": cursor, } +@router.get("/facets") +def audit_facets( + filters: AuditFilters = Depends(audit_filters), + _: User = Depends(admin_user), + db: Session = Depends(get_db), +) -> dict: + """The actions present under the other filters, with counts — what the action menu offers.""" + n = func.count().label("n") + stmt = ( + select(AuditLog.action, n) + .where(*filters.conditions("action")) + .group_by(AuditLog.action) + .order_by(n.desc(), AuditLog.action) + ) + return {"actions": [{"value": a, "count": c} for a, c in db.execute(stmt).all()]} + + @router.delete("") def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict: """Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence).""" diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index e30ffcaa..3c1e753c 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -3,15 +3,16 @@ hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side w (subscribe / unsubscribe) that require the optional write scope.""" from datetime import datetime, timezone +from functools import partial -from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi import APIRouter, Depends from pydantic import BaseModel from sqlalchemy import Select, func, select, update from sqlalchemy.orm import Session from app.collation import collated, sorted_words from app.mailtext import lang_for_user -from app import logs, quota +from app import logs, quota, scheduler from app.auth import admin_user, current_user, require_write_scope from app.db import get_db from app.models import ( @@ -481,7 +482,6 @@ class ChannelUpdateIn(BaseModel): def update_channel( channel_id: str, body: ChannelUpdateIn, - background: BackgroundTasks, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: @@ -508,12 +508,19 @@ def update_channel( with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT): run_recent_backfill(db, [channel], max_channels=1) - # …and kick the FULL deep backfill for this one channel immediately in the background, so the - # whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The - # request returns right away; the channel page's live poll shows the count climb. The scheduler - # still finishes a very large channel's tail on its normal runs (deep_requested stays set). - if deep_turned_on: - background.add_task(backfill_channel_now, channel_id, user.id) + # …and kick the FULL deep backfill for this one channel immediately, as a recorded run of its own, + # so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. + # The request returns right away; the channel page's live poll shows the count climb. The + # scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set). + # A channel whose history is all in gets no run: it would only say "nothing to do", one kept row + # per flip of the flag. + if deep_turned_on and not (channel is not None and channel.backfill_done): + scheduler.run_ad_hoc( + scheduler.CHANNEL_BACKFILL, + partial(backfill_channel_now, channel_id, user.id), + actor_id=user.id, + target=channel_id, + ) return { "ok": True, diff --git a/backend/app/routes/config.py b/backend/app/routes/config.py index e4425ebb..343bcb56 100644 --- a/backend/app/routes/config.py +++ b/backend/app/routes/config.py @@ -61,7 +61,7 @@ def set_config( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - s = sysconfig.spec(key) + s = sysconfig.config_page_spec(key) if s is None: raise not_found("config_key") if "value" not in body.model_fields_set: @@ -89,7 +89,9 @@ def set_config( before=None if s.secret else {"value": _redact(old)}, after=None if s.secret else {"value": _redact(new)}, ) - db.commit() + # The override is committed even when the trail skips it: re-saving the value in force still + # pins it as an override, and an uncommitted write would be dropped with the session. + db.commit() _apply_runtime(db, key) return sysconfig.describe(db) @@ -100,7 +102,7 @@ def reset_config( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - s = sysconfig.spec(key) + s = sysconfig.config_page_spec(key) if s is None: raise not_found("config_key") old = None if s.secret else sysconfig.get(db, key) diff --git a/backend/app/routes/quota.py b/backend/app/routes/quota.py index 4db9508d..98df6d07 100644 --- a/backend/app/routes/quota.py +++ b/backend/app/routes/quota.py @@ -6,7 +6,7 @@ The aggregate daily budget lives in ApiQuotaUsage; here we read the per-spend au from datetime import datetime, timedelta, timezone -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from sqlalchemy import func, select from sqlalchemy.orm import Session @@ -14,6 +14,8 @@ from app import quota from app.auth import admin_user, current_user from app.db import get_db from app.models import QuotaEvent, User +from app.routes import _account as account +from app.routes._keyset import utc router = APIRouter(prefix="/api/quota", tags=["quota"]) @@ -45,23 +47,32 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) @router.get("/admin") def admin_usage( - days: int = 30, + since: datetime | None = None, + until: datetime | None = None, + user: str | None = Query(default=None, pattern=account.PATTERN), _: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - days = max(1, min(days, 365)) - since = datetime.now(timezone.utc) - timedelta(days=days) + """Spend per account (and per action) and per day, between `since` (default: 30 days back) and + `until` (default: now), optionally for one account (`none` = the background).""" + now = datetime.now(timezone.utc) + lo = utc(since) or now - timedelta(days=30) + window = [QuotaEvent.created_at >= lo] + if until is not None: + window.append(QuotaEvent.created_at < utc(until)) + if user is not None: + window.append(account.condition(QuotaEvent.user_id, user)) totals = db.execute( select(QuotaEvent.user_id, func.sum(QuotaEvent.units)) - .where(QuotaEvent.created_at >= since) + .where(*window) .group_by(QuotaEvent.user_id) ).all() per_action: dict[int | None, dict[str, int]] = {} for uid, action, units in db.execute( select(QuotaEvent.user_id, QuotaEvent.action, func.sum(QuotaEvent.units)) - .where(QuotaEvent.created_at >= since) + .where(*window) .group_by(QuotaEvent.user_id, QuotaEvent.action) ).all(): per_action.setdefault(uid, {})[action] = int(units) @@ -84,16 +95,16 @@ def admin_usage( ] rows.sort(key=lambda r: r["total"], reverse=True) - # Instance-wide daily series in Pacific days (matches the budget reset). + # The daily series in Pacific days (matches the budget reset), of the one account when filtered. day_expr = func.date(func.timezone("America/Los_Angeles", QuotaEvent.created_at)) daily = [ {"day": str(day), "total": int(units)} for day, units in db.execute( select(day_expr, func.sum(QuotaEvent.units)) - .where(QuotaEvent.created_at >= since) + .where(*window) .group_by(day_expr) .order_by(day_expr) ).all() ] - return {"range_days": days, "rows": rows, "daily": daily} + return {"rows": rows, "daily": daily} diff --git a/backend/app/routes/runs.py b/backend/app/routes/runs.py new file mode 100644 index 00000000..c1bb53d6 --- /dev/null +++ b/backend/app/routes/runs.py @@ -0,0 +1,156 @@ +"""Admin background-run history: read `job_runs`, a page at a time. + +Admin-only and read-only — rows leave the table through the retention job +(`job_run_retention_days`). Filtered on the server and paged by `_keyset`, like the server log and +the audit log beside it on the activity page. A row names what a reader needs without a second +request: the channel a user's ad-hoc run worked on, and how many log lines the run left (only +WARNING and up are stored, so most runs have none — the count says so before the click). +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from fastapi import APIRouter, Depends, Query +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +from app import scheduler +from app.auth import admin_user +from app.db import get_db +from app.models import Channel, JobRun, LogEvent, User +from app.routes import _account as account +from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc +from app.runstatus import RunStatus +from app.utils import iso, like_escape + +router = APIRouter(prefix="/api/admin/runs", tags=["admin"]) + +Facet = Literal["job", "status"] + + +def _serialize(run: JobRun, titles: dict[str, str], lines: dict[int, int]) -> dict: + return { + "id": run.id, + "job": run.job, + "trigger": run.trigger, + "actor_id": run.actor_id, + "target": run.target, + "target_title": titles.get(run.target) if run.target is not None else None, + "status": run.status, + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), + "summary": run.summary, + "result": run.result, + "error": run.error, + "failure": run.failure, + "quota_units": run.quota_units, + "log_lines": lines.get(run.id, 0), + } + + +@dataclass(frozen=True) +class RunFilters: + job: str | None + status: RunStatus | None + actor: str | None + q: str | None + since: datetime | None + until: datetime | None + + def conditions(self, skip: Facet | None = None) -> list: + """The WHERE conditions, ANDed. `skip` leaves one menu's own filter out, so that menu counts + its values under every other filter (the way back from a pick).""" + out = [] + if self.job is not None and skip != "job": + out.append(JobRun.job == self.job) + if self.status is not None and skip != "status": + out.append(JobRun.status == self.status) + if self.actor is not None: + out.append(account.condition(JobRun.actor_id, self.actor)) + if self.since is not None: + out.append(JobRun.started_at >= utc(self.since)) + if self.until is not None: + out.append(JobRun.started_at < utc(self.until)) + if self.q and self.q.strip(): + like = f"%{like_escape(self.q.strip())}%" + out.append( + or_( + JobRun.job.ilike(like), + JobRun.target.ilike(like), + JobRun.summary.ilike(like), + JobRun.error.ilike(like), + ) + ) + return out + + +def run_filters( + job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"), + status: RunStatus | None = None, + user: str | None = Query(default=None, pattern=account.PATTERN), + q: str | None = Query(default=None, max_length=200), + since: datetime | None = None, + until: datetime | None = None, +) -> RunFilters: + return RunFilters(job, status, user, q, since, until) + + +@router.get("") +def list_runs( + filters: RunFilters = Depends(run_filters), + before: str | None = Query(default=None, max_length=200), + limit: int = Query(default=100, ge=1, le=PAGE_MAX), + _: User = Depends(admin_user), + db: Session = Depends(get_db), +) -> dict: + """One page of runs, newest first, matching every filter given; `before` continues from a + previous page's `next_cursor`.""" + query = select(JobRun).where(*filters.conditions()) + if before is not None: + query = query.where(older_than(JobRun.started_at, JobRun.id, before)) + order = (JobRun.started_at.desc(), JobRun.id.desc()) + runs, cursor = page( + list(db.execute(query.order_by(*order).limit(limit + 1)).scalars().all()), + limit, + lambda r: r.started_at, + ) + channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target} + titles: dict[str, str] = {} + if channels: + rows = db.execute(select(Channel.id, Channel.title).where(Channel.id.in_(channels))) + titles = {cid: title for cid, title in rows.tuples() if title is not None} + lines: dict[int, int] = {} + if runs: + counted = db.execute( + select(LogEvent.run_id, func.count()) + .where(LogEvent.run_id.in_([r.id for r in runs])) + .group_by(LogEvent.run_id) + ) + lines = {rid: n for rid, n in counted.tuples() if rid is not None} + return { + "items": [_serialize(r, titles, lines) for r in runs], + "accounts": accounts(db, {r.actor_id for r in runs if r.actor_id is not None}), + "next_cursor": cursor, + } + + +@router.get("/facets") +def run_facets( + filters: RunFilters = Depends(run_filters), + _: User = Depends(admin_user), + db: Session = Depends(get_db), +) -> dict: + """What the job menu and the status chips offer, each counted under every filter but its own.""" + + def counts(col, skip: Facet) -> list[dict]: + n = func.count().label("n") + stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col) + return [{"value": v, "count": c} for v, c in db.execute(stmt).all()] + + total = select(func.count()).select_from(JobRun).where(*filters.conditions("status")) + return { + "jobs": counts(JobRun.job, "job"), + "statuses": counts(JobRun.status, "status"), + "total": db.scalar(total) or 0, + } diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index 579a691f..db0f5308 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -17,8 +17,9 @@ from app.scheduler import ( JOB_INTERVALS, MAX_INTERVAL, MIN_INTERVAL, - apply_interval, + reschedule, scheduler_snapshot, + stage_interval, trigger_all, trigger_job, ) @@ -42,14 +43,14 @@ def set_job_interval( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - """Change how often a scheduler job runs (minutes). Persisted as an override and applied - to the live scheduler immediately.""" + """Change how often a scheduler job runs (minutes). Stored as an override together with its + audit row, then applied to the live scheduler.""" if job_id not in JOB_INTERVALS: raise not_found("job") minutes = body.interval_minutes if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL): raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL) - applied = apply_interval(db, job_id, minutes) + applied = stage_interval(db, job_id, minutes) audit.record( db, admin.id, @@ -60,6 +61,7 @@ def set_job_interval( after={"interval_minutes": applied}, ) db.commit() + reschedule(job_id, applied) return {"ok": True, "id": job_id, "interval_minutes": applied} @@ -117,7 +119,7 @@ def set_maintenance_settings( @router.get("") def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict: - snap = scheduler_snapshot() + snap = scheduler_snapshot(db) # Work still queued (shared catalog, so these are instance-wide, not per-user). deep_requested = ( diff --git a/backend/app/routes/setup.py b/backend/app/routes/setup.py index cddbfcec..7ad48324 100644 --- a/backend/app/routes/setup.py +++ b/backend/app/routes/setup.py @@ -91,12 +91,12 @@ def setup_admin(body: SetupAdminIn, db: Session = Depends(require_setup)) -> dic @router.post("/config") def setup_config(settings_map: dict[str, Any], db: Session = Depends(require_setup)) -> dict: - """Persist provided DB-backed settings (Google creds, SMTP, quota…). Keys must be in the - sysconfig registry; empty values are skipped (left at the default). Genuinely dynamic + """Persist provided DB-backed settings (Google creds, SMTP, quota…). Keys must be the + Configuration page's; empty values are skipped (left at the default). Genuinely dynamic key→value map, so it stays a typed dict rather than a fixed model.""" saved: list[str] = [] for key, value in settings_map.items(): - if sysconfig.spec(key) is None: + if sysconfig.config_page_spec(key) is None: raise refused(400, "unknown_setting", key=key) if value is None or value == "": continue diff --git a/backend/app/runstatus.py b/backend/app/runstatus.py new file mode 100644 index 00000000..cba6c896 --- /dev/null +++ b/backend/app/runstatus.py @@ -0,0 +1,37 @@ +"""The words a background run's row is written with — the `job_runs` table's status and trigger. + +One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the +model's CHECK constraints are built from these sets, and the dashboard reads them back (the frontend's +copies are held to them by `backendIds.contract.test.ts`). The migrated database's CHECKs are frozen +in its migration: a test holds them equal to these sets, so a new word fails until a migration adds +it (`test_runstatus.py`). The download pipeline's `cancelled`/`canceled` drift (`downloads/status.py`) +is why a status is never typed as a bare string. +""" + +from enum import StrEnum + + +class RunStatus(StrEnum): + RUNNING = "running" + OK = "ok" + ERROR = "error" + # Sync was paused: the run was due and deliberately did nothing. + SKIPPED = "skipped" + # The process stopped while the run was going (a deploy, a crash): set at the next startup, so a + # row never claims to be running in a process that no longer exists. + INTERRUPTED = "interrupted" + + +class RunTrigger(StrEnum): + # The run's interval came round. + SCHEDULED = "scheduled" + # An admin pressed "run now" (or "run all"). + MANUAL = "manual" + # A user's own action started work on one target (a channel's full history), outside the + # recurring jobs. + AD_HOC = "ad_hoc" + + +def sql_in(values: type[StrEnum]) -> str: + """`'a', 'b'` — the IN-list a CHECK constraint names these values with.""" + return ", ".join(f"'{v.value}'" for v in values) diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 8aa3f651..d7e4c82b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -3,28 +3,31 @@ recent-first / deep backfill. Runs inside the API process (single worker).""" import contextvars import threading -from datetime import datetime, timezone +from datetime import timezone from typing import Callable +from apscheduler.executors.pool import ThreadPoolExecutor +from apscheduler.jobstores.base import ConflictingIdError from apscheduler.schedulers.background import BackgroundScheduler from sqlalchemy import select -from app import audit, debugsession, logs, logsink, progress, quota -from app.audit import AuditAction +from app import audit, debugsession, jobruns, logs, logsink, progress, quota, sysconfig from app.config import settings from app.db import SessionLocal from app.downloads.gc import run_download_gc -from app.models import SchedulerSetting +from app.models import Channel, JobRun from app.plex.sync import sync as run_plex_sync from app.plex.tmdb import run_enrich as run_tmdb_enrich from app.plex.tmdb import run_people_enrich from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync from app.notifications import create_notification +from app.runstatus import RunStatus, RunTrigger from app.state import is_sync_paused from app.sync.autotag import run_autotag_all from app.sync.explore import purge_ephemeral from app.sync.maintenance import run_maintenance from app.sync.playlists import sync_all_playlists +from app.utils import iso from app.sync.runner import ( run_deep_backfill, run_enrich, @@ -37,12 +40,31 @@ from app.sync.runner import ( log = logs.get_logger(__name__) _scheduler: BackgroundScheduler | None = None -# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the -# scheduler lives in the same single-worker process as the API, so a request can read it -# directly). Ephemeral by design — it reflects "what the scheduler is doing right now and -# how the last runs went" since this process started; durable history isn't the goal here. +# What is running RIGHT NOW, per job: the single-run claim and the live progress. In memory on +# purpose — the scheduler lives in the API's one process, and progress changes every second. What a +# run did is durable instead: its `job_runs` row (app/jobruns.py), which outlives a restart. _activity: dict[str, dict] = {} _activity_lock = threading.Lock() +# The runs this process has open (run id -> job), under the same lock. A row can stay `running` in +# the table after its outcome failed to write (_close), so the table cannot say what is going; this +# can, as the one process per database (app/jobruns.py) is the one running them. +_open_runs: dict[int, str] = {} + + +def _opened(run_id: int, job: str) -> None: + with _activity_lock: + _open_runs[run_id] = job + + +def _closed(run_id: int) -> None: + with _activity_lock: + _open_runs.pop(run_id, None) + + +def _open_run_ids() -> dict[int, str]: + with _activity_lock: + return dict(_open_runs) + # Set (per-thread) by a manual "run now" trigger to the id of the admin who clicked, so that # run — and only that run, not the recurring scheduled ones — posts a completion notification @@ -51,59 +73,36 @@ _manual_actor: contextvars.ContextVar[int | None] = contextvars.ContextVar( "manual_actor", default=None ) -# Each interval job's env/config default period (minutes). The admin can override any of -# these at runtime (stored in scheduler_settings, applied live); see load_intervals. +# Each interval job's env/config default period (minutes). The admin can override any of these at +# runtime from the Scheduler page — a sysconfig key per job (`sysconfig.interval_key`), applied +# live; see load_intervals. JOB_INTERVALS: dict[str, int] = { - "rss_poll": settings.rss_poll_minutes, - "enrich": settings.enrich_interval_minutes, - "backfill": settings.backfill_interval_minutes, - "autotag": settings.autotag_interval_minutes, - "shorts": settings.shorts_probe_interval_minutes, - "subscriptions": settings.subscriptions_resync_minutes, - "playlist_sync": settings.playlist_sync_minutes, - "maintenance": settings.maintenance_interval_minutes, - "explore_cleanup": settings.explore_cleanup_minutes, - "download_gc": settings.download_gc_minutes, - "plex_sync": settings.plex_sync_interval_min, - "plex_watch_sync": settings.plex_watch_sync_interval_min, - "plex_watch_reconcile": settings.plex_watch_reconcile_interval_min, - "tmdb_enrich": settings.tmdb_enrich_interval_min, - "people_enrich": settings.people_enrich_interval_min, - "audit_gc": settings.audit_gc_minutes, - "log_gc": settings.log_gc_minutes, + job: getattr(settings, attr) for job, attr in sysconfig.JOB_INTERVAL_DEFAULTS.items() } - -# Sane bounds for an admin-set interval (minutes). -MIN_INTERVAL = 1 -MAX_INTERVAL = 1440 # one day +MIN_INTERVAL = sysconfig.MIN_INTERVAL +MAX_INTERVAL = sysconfig.MAX_INTERVAL def load_intervals(db) -> dict[str, int]: - """Effective per-job intervals: the env/config defaults overlaid with any admin overrides - saved in scheduler_settings.""" - iv = dict(JOB_INTERVALS) - for row in db.execute(select(SchedulerSetting)).scalars(): - if row.job_id in iv and row.interval_minutes: - iv[row.job_id] = row.interval_minutes - return iv + """Effective per-job intervals: each job's sysconfig override, else its env/config default.""" + return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS} -def apply_interval(db, job_id: str, minutes: int) -> int: - """Persist a job's interval override and reschedule the live job immediately (if the - scheduler runs in this process). Returns the applied value.""" +def stage_interval(db, job_id: str, minutes: int) -> int: + """Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it + with its audit row, then calls `reschedule` — the live job follows only what was stored.""" if job_id not in JOB_INTERVALS: raise KeyError(job_id) minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes))) - row = db.get(SchedulerSetting, job_id) - if row is None: - db.add(SchedulerSetting(job_id=job_id, interval_minutes=minutes)) - else: - row.interval_minutes = minutes - db.commit() + sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes) + return minutes + + +def reschedule(job_id: str, minutes: int) -> None: + """Move the live interval job to `minutes` (if this process registered it).""" if _scheduler is not None and _scheduler.get_job(job_id) is not None: _scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes) log.info("rescheduled job %s to every %s min", job_id, minutes) - return minutes def _record(name: str, **fields) -> None: @@ -112,7 +111,12 @@ def _record(name: str, **fields) -> None: def _notify_done( - db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None + db, + actor_id: int, + job_id: str, + status: RunStatus, + summary: str | None, + result: dict | None = None, ) -> None: """Post a completion notice to the triggering admin's inbox (manual runs only). Best effort — a notification failure must never mask the job's own outcome.""" @@ -130,49 +134,6 @@ def _notify_done( log.exception("failed to write completion notification for job %s", job_id) -def _run_changed(result) -> bool: - """Did an automatic job run actually do something worth an audit row? (No-op polls don't.) - A "change" = a truthy NUMERIC count; status-string dicts like {"skipped": "disabled"} (Plex - off — the default) are no-ops, not changes, so they don't flood the trail every interval.""" - if result is None: - return False - if isinstance(result, dict): - # A phased job (backfill's recent/deep, people_enrich's bios/portraits) nests a result per - # phase: a count inside a phase is a change too. - return any( - _run_changed(v) if isinstance(v, dict) else isinstance(v, (int, float)) and bool(v) - for v in result.values() - ) - if isinstance(result, (int, float)): - return result != 0 - return bool(result) - - -def _audit_run(db, actor_id, job_id, status, summary, result) -> None: - """Write ONE run-summary audit row per job run (never per-item). Manual runs (an admin - clicked "run now" → actor_id set) and errors are always logged; automatic runs only when - they changed something, so routine no-op polls don't spam the trail. Best-effort.""" - if actor_id is None and status == "ok" and not _run_changed(result): - return - # The result as data too: the audit log names it in the reader's language, as the inbox - # does; `summary` stays the English line of the row's own summary column. - data = _result_data(result) - try: - audit.record( - db, - actor_id, - AuditAction.JOB_RUN, - target_type="job", - target_id=job_id, - summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""), - after={"status": status, "summary": summary, **({"result": data} if data else {})}, - ) - db.commit() - except Exception: - db.rollback() - log.exception("failed to write audit row for job %s", job_id) - - def _job(name: str, fn) -> None: # The one place both run paths converge (manual trigger AND scheduled fire), so claim the run # HERE, atomically: whichever reaches this first wins, the other skips. This is the real @@ -180,71 +141,88 @@ def _job(name: str, fn) -> None: if not _claim_running(name): log.info("job %s already running — skipping this trigger", name) return - db = SessionLocal() actor_id = _manual_actor.get() # set only for a manual "run now" trigger - _record( - name, last_started=datetime.now(timezone.utc).isoformat(), last_error=None, progress=None - ) # running already claimed + trigger = RunTrigger.SCHEDULED if actor_id is None else RunTrigger.MANUAL + _record(name, progress=None) # running already claimed + db = SessionLocal() + run_id = None + try: + try: + run_id = jobruns.start(name, trigger, actor_id) + except Exception: + # No row to tell it in: the database is what failed. The log is the record. + log.exception("job %s not started: its run could not be recorded", name) + return + _opened(run_id, name) + with logs.bind(run_id=run_id): + _run(db, name, fn, run_id, actor_id) + finally: + # ALWAYS release the claim, even if the body raised before its own status update — so a + # failure can never wedge the job as permanently "running". + if run_id is not None: + _closed(run_id) + _record(name, running=False, progress=None) + db.close() + + +def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None: + """The run itself, inside its `job_runs` row: the pause-skip, the work, and its outcome — on + the row, and in the triggering admin's inbox for a manual run.""" def sink(current, total, phase): _record(name, progress={"current": current, "total": total, "phase": phase}) + # Only the work is inside the try: writing its outcome is not the job, and a write that fails + # must not turn a run that succeeded into a failed one (see _close). + result = None try: - if is_sync_paused(db): - log.info("job %s skipped (sync paused)", name) - _record( - name, - status="skipped", - last_finished=datetime.now(timezone.utc).isoformat(), - last_result="paused", - result=_PAUSED, - progress=None, - ) - if actor_id is not None: - _notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED) - _audit_run(db, actor_id, name, "skipped", "sync paused", _PAUSED) - return - with progress.bind(sink): - result = fn(db) - summary = _summarize(result) - data = _result_data(result) - log.info("job %s -> %s", name, result) - _record( - name, - status="ok", - last_finished=datetime.now(timezone.utc).isoformat(), - last_result=summary, - result=data, - progress=None, - ) - if actor_id is not None: - _notify_done(db, actor_id, name, "ok", summary, data) - _audit_run(db, actor_id, name, "ok", summary, result) + paused = is_sync_paused(db) + if not paused: + with progress.bind(sink): + result = fn(db) except Exception as exc: db.rollback() log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) - err = str(exc) or exc.__class__.__name__ - # A failure the job named (progress.JobFailed) rides along as data, as a result does, so the - # client says it in the reader's language; `err` stays the English line. - failure = ( - {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None - ) - _record( - name, - status="error", - last_finished=datetime.now(timezone.utc).isoformat(), - last_error=err, - result=failure, - progress=None, - ) + err, failure = _failure(exc) + _close(run_id, RunStatus.ERROR, error=err, failure=failure) if actor_id is not None: - _notify_done(db, actor_id, name, "error", err, failure) - _audit_run(db, actor_id, name, "error", err, failure) - finally: - # ALWAYS release the claim, even if the body raised before its own status update — so a - # failure can never wedge the job as permanently "running". - _record(name, running=False) - db.close() + _notify_done(db, actor_id, name, RunStatus.ERROR, err, failure) + return + if paused: + log.info("job %s skipped (sync paused)", name) + _close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED) + if actor_id is not None: + _notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED) + return + summary, data = _finish_ok(name, run_id, result) + if actor_id is not None: + _notify_done(db, actor_id, name, RunStatus.OK, summary, data) + + +def _close(run_id: int, status: RunStatus, **outcome) -> None: + """Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's: + the row then stays `running` until the next start's sweep closes it as interrupted.""" + try: + jobruns.finish(run_id, status, **outcome) + except Exception: + log.exception("run %s: its outcome (%s) could not be recorded", run_id, status) + + +def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]: + """Close a run that returned: its result on the row, as data and as the English one-liner.""" + summary = _summarize(result) + data = _result_data(result) + log.info("job %s -> %s", name, result) + _close(run_id, RunStatus.OK, summary=summary, result=data) + return summary, data + + +def _failure(exc: Exception) -> tuple[str, dict | None]: + """A raised run's English error line, and the failure the job named (progress.JobFailed) as + data, as a result is, so the client says it in the reader's language.""" + err = str(exc) or exc.__class__.__name__ + failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None + return err, failure # A skipped run's result, in the same shape as a job's own `{"skipped": …}`. @@ -271,11 +249,13 @@ def _summarize(result) -> str | None: return str(result) -def scheduler_snapshot() -> dict: - """What the scheduler is doing, for the admin dashboard. `running_here` is False in - environments where the scheduler is disabled (e.g. local dev shares the prod-ish DB but - runs no scheduler), so the UI can say so while still showing DB-derived queue/quota.""" - running_here = _scheduler is not None +def scheduler_snapshot(db) -> dict: + """What the scheduler is doing, for the admin dashboard: each job's live state from this process + and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how + did it go"; and the same for the ad-hoc runs users start on one target. `running_here` is False where the scheduler runs no interval jobs + (SCHEDULER_ENABLED=false), so the UI can say so while still showing the DB-derived history, + queue and quota.""" + running_here = _scheduler is not None and settings.scheduler_enabled next_runs: dict[str, str | None] = {} live_intervals: dict[str, int] = {} if _scheduler is not None: @@ -287,6 +267,12 @@ def scheduler_snapshot() -> dict: live_intervals[job.id] = round(iv.total_seconds() / 60) with _activity_lock: acts = {k: dict(v) for k, v in _activity.items()} + # Open before the read or after it: a run that started or ended while the rows were read is + # still counted as the open run it was, never as a stray one. + open_runs = _open_run_ids() + recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) + open_runs |= _open_run_ids() + going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) @@ -296,16 +282,94 @@ def scheduler_snapshot() -> dict: "interval_minutes": live_intervals.get(job_id, default_interval), "next_run": next_runs.get(job_id), "running": a.get("running", False), - "status": a.get("status"), - "last_started": a.get("last_started"), - "last_finished": a.get("last_finished"), - "last_result": a.get("last_result"), - "result": a.get("result"), - "last_error": a.get("last_error"), "progress": a.get("progress"), + **_last_runs(recent.get(job_id, []), open_runs), } ) - return {"running_here": running_here, "enabled": settings.scheduler_enabled, "jobs": jobs} + # Work a user starts on one target: no interval, no "run now" (it needs its target), but its + # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the + # target's own (sync/runner), so "running" is the runs this process has open: two channels can + # backfill at once, so a run still going counts even when a newer one already ended. That run then + # leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the + # newer run's outcome under the older one's channel. + backfills = recent.get(CHANNEL_BACKFILL, []) + going = db.get(JobRun, going_id) if going_id is not None else None + if going is not None and going.status != RunStatus.RUNNING: + going = None # its outcome is written; only the bookkeeping has not caught up + if going is not None: + backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS] + shown = backfills[0] if backfills else None + ad_hoc = [ + { + "id": CHANNEL_BACKFILL, + "running": going is not None, + "target": shown.target if shown else None, + "target_title": ( + db.scalar(select(Channel.title).where(Channel.id == shown.target)) + if shown is not None and shown.target + else None + ), + **_last_runs(backfills, open_runs), + } + ] + return { + "running_here": running_here, + "enabled": settings.scheduler_enabled, + "jobs": jobs, + "ad_hoc": ad_hoc, + } + + +# How many past runs a Scheduler row shows as dots; the full history is the run list. +RECENT_RUNS = 3 + + +def _seen(run, open_runs: dict[int, str]) -> RunStatus: + """A run's status as the page tells it. A row `running` that no run here has open is one whose + outcome failed to write: the next start's sweep closes it as interrupted, so it reads so now — + not as a run still going beside a row that says nothing runs.""" + if run.status == RunStatus.RUNNING and run.id not in open_runs: + return RunStatus.INTERRUPTED + return RunStatus(run.status) + + +def _last_runs(runs: list, open_runs: dict[int, str]) -> dict: + """A row's run fields from its newest runs (newest first): how the last one went, and the few + before it for the history dots.""" + run = runs[0] if runs else None + status = _seen(run, open_runs) if run is not None else None + # A run still going has no outcome yet: only an ended run says how it went. + ended = run if status is not None and status != RunStatus.RUNNING else None + return { + "status": status if ended else None, + "last_started": iso(run.started_at) if run else None, + # A stray row has no end of its own: it ended, as far as anyone can tell, where it began. + "last_finished": iso(ended.finished_at or ended.started_at) if ended else None, + "last_result": ended.summary if ended else None, + # The outcome as data: the result, or on a failed run the failure the job named. + "result": _outcome_data(ended), + "last_error": ended.error if ended else None, + "last_trigger": run.trigger if run else None, + "last_run_id": run.id if run else None, + "quota_units": ended.quota_units if ended else None, + "recent": [_run_brief(r, open_runs) for r in runs], + } + + +def _run_brief(run, open_runs: dict[int, str]) -> dict: + return { + "id": run.id, + "status": _seen(run, open_runs), + "trigger": run.trigger, + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), + } + + +def _outcome_data(run) -> dict | None: + if run is None: + return None + return run.failure if run.status == RunStatus.ERROR else run.result def _rss_job() -> None: @@ -408,15 +472,16 @@ def _people_enrich_job() -> None: def _audit_gc_job() -> None: - # The history cleanup: audit-log rows past audit_retention_days and YouTube quota history past - # quota_retention_days (0 = keep forever, each). Pure DB, no quota. + # The history cleanup: audit-log rows past audit_retention_days, YouTube quota history past + # quota_retention_days and finished runs past job_run_retention_days (0 = keep forever, each). + # Pure DB, no quota. def work(db): - from app import sysconfig reaped = audit.gc(db, sysconfig.get_int(db, "audit_retention_days")) quota_reaped = quota.gc(db, sysconfig.get_int(db, "quota_retention_days")) + runs_reaped = jobruns.gc(db, sysconfig.get_int(db, "job_run_retention_days")) db.commit() - return {"reaped": reaped, "quota_reaped": quota_reaped} + return {"reaped": reaped, "quota_reaped": quota_reaped, "runs_reaped": runs_reaped} _job("audit_gc", work) @@ -425,7 +490,6 @@ def _log_gc_job() -> None: # Prune log_events rows older than log_retention_days — the limit the privacy policy states, so # this runs hourly rather than daily. Pure DB, no quota. def work(db): - from app import sysconfig days = sysconfig.get_int(db, "log_retention_days") reaped = logsink.gc(db, days) @@ -485,15 +549,29 @@ def running_job_ids() -> set[str]: return {jid for jid, a in _activity.items() if a.get("running")} -def trigger_job(job_id: str, actor_id: int | None = None) -> str: - """Run a job immediately in a background thread, independent of its interval schedule. - The wrapper handles its own DB session, activity tracking and pause-skip, so the live - dashboard reflects the run. Returns "started", "already_running", or raises KeyError for - an unknown job. Refusing a concurrent run keeps a manual trigger from overlapping a - scheduled run (APScheduler enforces max_instances=1 for the scheduled side). +def _as_actor(fn: Callable[[], None], actor_id: int | None) -> Callable[[], None]: + """`fn` run as `actor_id`'s manual run. The actor is RESET afterwards: the executor's pool + threads are reused and keep their context from task to task, so an actor left set would make + the next scheduled run that thread picks up look like this admin's manual run.""" - `actor_id` (the admin who clicked "run now") is stashed in a contextvar inside the new - thread so the run posts a completion notification to that admin's inbox.""" + def run() -> None: + token = _manual_actor.set(actor_id) + try: + fn() + finally: + _manual_actor.reset(token) + + return run + + +def trigger_job(job_id: str, actor_id: int | None = None) -> str: + """Run a job now, independent of its interval schedule: a one-shot job on the same scheduler, + so a manual run shares the scheduled runs' executor and shutdown — no thread of its own. The + wrapper handles its own DB session, run row and pause-skip. Returns "started", + "already_running", or raises KeyError for an unknown job. + + `actor_id` (the admin who clicked "run now") is set in a contextvar inside the run's thread, so + the run is recorded as theirs and posts a completion notification to their inbox.""" fn = JOB_FUNCS.get(job_id) if fn is None: raise KeyError(job_id) @@ -502,13 +580,18 @@ def trigger_job(job_id: str, actor_id: int | None = None) -> str: # _job claims, sees it's taken, and skips — no double run. if _is_running(job_id): return "already_running" + sched = _scheduler + if sched is None: + # Only between shutdown and exit: start_scheduler runs at every API startup. + raise RuntimeError("the scheduler is not running") - def run() -> None: - if actor_id is not None: - _manual_actor.set(actor_id) - fn() - - threading.Thread(target=run, name=f"trigger-{job_id}", daemon=True).start() + try: + # One id per job: a second click before the first one-shot has even started is the same + # run, not a queued second one. misfire_grace_time=None: a busy pool delays it, never drops it. + sched.add_job(_as_actor(fn, actor_id), "date", id=f"{job_id}@now", misfire_grace_time=None) + # silent-ok: not a failure — the answer is the 409 the admin sees ("already running"). + except ConflictingIdError: + return "already_running" return "started" @@ -521,22 +604,95 @@ def trigger_all(actor_id: int | None = None) -> list[str]: return started -def start_scheduler() -> None: - global _scheduler - if not settings.scheduler_enabled or _scheduler is not None: - return - # Effective intervals = env defaults overlaid with any admin overrides from the DB. - db = SessionLocal() +# Work a user starts on one target outside the job list — today the "get full history" backfill of +# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and +# spent is in `job_runs` like any job's, not a background task only the log ever heard of. +CHANNEL_BACKFILL = "channel_backfill" +# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple). +AD_HOC_JOBS = (CHANNEL_BACKFILL,) + + +def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None: + """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its + own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so + a second request before the first has even started is that same run.""" + if job not in AD_HOC_JOBS: + raise KeyError(job) # an unnamed job would show on the page as its bare id + sched = _scheduler + if sched is None: + # Only between shutdown and exit: start_scheduler runs at every API startup. + raise RuntimeError("the scheduler is not running") try: - iv = load_intervals(db) + sched.add_job( + _ad_hoc, + "date", + args=(job, fn, actor_id, target), + id=f"{job}@{target}", + misfire_grace_time=None, + ) + # silent-ok: not a failure — the run for this target is already queued, and it does this work. + except ConflictingIdError: + log.info("ad-hoc %s for %s already queued", job, target) + + +def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None: + try: + run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) + except Exception: + # No row to tell it in: the database is what failed. The log is the record. + log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) + return + _opened(run_id, job) + try: + _ad_hoc_work(job, fn, run_id) finally: - db.close() - scheduler = BackgroundScheduler(timezone="UTC") - for job_id, fn in JOB_FUNCS.items(): - scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id) + _closed(run_id) + + +def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: + with logs.bind(run_id=run_id): + # As in _run: only the work is inside the try. + try: + result = fn() + except Exception as exc: + log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) + err, failure = _failure(exc) + _close(run_id, RunStatus.ERROR, error=err, failure=failure) + return + _finish_ok(job, run_id, result) + + +# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of +# 10 would queue a scheduled fire behind long manual runs. +_POOL_SIZE = len(JOB_FUNCS) + 8 + + +def start_scheduler() -> None: + """Start this process's scheduler. It always starts, so "run now" works on every install; only + with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can + run, every run a previous process left open is closed as interrupted — one API process serves a + database (app/jobruns.py), so an open run here is a dead process's.""" + global _scheduler + if _scheduler is not None: + return + left = jobruns.interrupt_leftovers() + if left: + log.warning("%s run(s) left running by the previous process marked interrupted", left) + scheduler = BackgroundScheduler( + timezone="UTC", executors={"default": ThreadPoolExecutor(_POOL_SIZE)} + ) + if settings.scheduler_enabled: + # Effective intervals = env defaults overlaid with any admin overrides from the DB. + db = SessionLocal() + try: + iv = load_intervals(db) + finally: + db.close() + for job_id, fn in JOB_FUNCS.items(): + scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id) scheduler.start() _scheduler = scheduler - log.info("scheduler started") + log.info("scheduler started (interval jobs %s)", "on" if settings.scheduler_enabled else "off") def shutdown_scheduler() -> None: diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index c13c0875..d13e904b 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -437,7 +437,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - # Channels being deep-backfilled RIGHT NOW by EITHER path — the interactive backfill_channel_now (a -# FastAPI BackgroundTasks threadpool thread) OR the scheduler's run_deep_backfill (its own thread). +# one-shot run on the scheduler's pool, see scheduler.run_ad_hoc) OR the scheduler's run_deep_backfill. # BOTH register the channel they're paging (under the lock) so the other skips it: no two runs page # the same channel's `backfill_cursor` concurrently, whichever started first. `_immediate_running` # counts only the interactive path, capped so one user's burst can't drain the shared budget (the @@ -448,9 +448,9 @@ _backfill_lock = threading.Lock() _MAX_CONCURRENT_IMMEDIATE = 2 -def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None: - """User-initiated "get full history": deep-backfill ONE channel to completion right now, in a - background task with its OWN session, so the full catalogue lands in seconds instead of over the +def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> dict: + """User-initiated "get full history": deep-backfill ONE channel to completion right now, as its + own recorded run with its OWN session, so the full catalogue lands in seconds instead of over the paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69 pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in @@ -459,34 +459,35 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the shared backfill budget by opting many channels in at once (the overflow, and any channel too large to finish under the reserve, is finished by the scheduler since deep_requested stays set), and - mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and - swallows failures.""" + mutually excluded per-channel with the scheduler via _backfill_in_progress. + + Returns what the run did, as a job result: the counts, or `{"skipped": }` when it left the + channel to the scheduler. A failure raises — the run that wraps this records it.""" global _immediate_running with _backfill_lock: - if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE: - return # already backfilling this channel, or too many interactive runs — leave it to the scheduler + if channel_id in _backfill_in_progress: + return {"skipped": "channel busy"} + if _immediate_running >= _MAX_CONCURRENT_IMMEDIATE: + return {"skipped": "at capacity"} _backfill_in_progress.add(channel_id) _immediate_running += 1 try: with SessionLocal() as db: user = get_service_user(db) if user is None: - return + return {"skipped": "no service user"} channel = db.get(Channel, channel_id) if channel is None or channel.backfill_done or not channel.uploads_playlist_id: - return + return {"skipped": "nothing to do"} if quota.backfill_reserve_reached(): - return - try: - # Attribute the spend to the requesting user under the full-backfill action, so it - # shows up correctly in the usage breakdown (not the default OTHER/system bucket). - with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL): - with YouTubeClient(db, user) as yt: - backfill_channel_deep(db, yt, channel, max_pages) - reconcile_full_history(db) - except Exception: - db.rollback() - log.exception("Immediate backfill failed for channel %s", channel_id) + return {"skipped": "low quota"} + # Attribute the spend to the requesting user under the full-backfill action, so it + # shows up correctly in the usage breakdown (not the default OTHER/system bucket). + with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL): + with YouTubeClient(db, user) as yt: + added = backfill_channel_deep(db, yt, channel, max_pages) + reconciled = reconcile_full_history(db) + return {"videos_added": added, "reconciled_full": reconciled} finally: with _backfill_lock: _backfill_in_progress.discard(channel_id) diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index daf8b1ce..6ad61490 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -8,7 +8,8 @@ stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secr stored in the DB and stay env-only). """ -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Literal from sqlalchemy import select from sqlalchemy.orm import Session @@ -27,6 +28,16 @@ _decrypt_warned: set[str] = set() _coerce_warned: set[tuple[str, str]] = set() +# When a changed value takes effect. "live": the next read obeys it — every consumer reads the key +# where it uses it, never once at import (test-held: a read outside a function fails). A value read +# once at startup is live only when its writer also applies it (the job intervals reschedule, the log +# level is set as it is saved). "restart": the process has to start +# again. No key needs a restart today, and the Configuration page has no way to say so yet, so a +# "restart" key is refused until the page can (test-held). +Applies = Literal["live", "restart"] +LIVE: Applies = "live" + + @dataclass(frozen=True) class ConfigSpec: key: str @@ -37,6 +48,8 @@ class ConfigSpec: min: int | None = None max: int | None = None allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset" + # Required, with no default: each key states it, so a new one cannot be live by omission. + applies: Applies = field(kw_only=True) @property def choices(self) -> tuple[str, ...]: @@ -45,30 +58,78 @@ class ConfigSpec: return choices_of(Settings.model_fields[self.default_attr].annotation) +# Each interval job of the scheduler and the `settings` attribute holding its default period +# (minutes). Its override is the `job_interval_` key, set from the Scheduler page. +JOB_INTERVAL_DEFAULTS: dict[str, str] = { + "rss_poll": "rss_poll_minutes", + "enrich": "enrich_interval_minutes", + "backfill": "backfill_interval_minutes", + "autotag": "autotag_interval_minutes", + "shorts": "shorts_probe_interval_minutes", + "subscriptions": "subscriptions_resync_minutes", + "playlist_sync": "playlist_sync_minutes", + "maintenance": "maintenance_interval_minutes", + "explore_cleanup": "explore_cleanup_minutes", + "download_gc": "download_gc_minutes", + "plex_sync": "plex_sync_interval_min", + "plex_watch_sync": "plex_watch_sync_interval_min", + "plex_watch_reconcile": "plex_watch_reconcile_interval_min", + "tmdb_enrich": "tmdb_enrich_interval_min", + "people_enrich": "people_enrich_interval_min", + "audit_gc": "audit_gc_minutes", + "log_gc": "log_gc_minutes", +} +# Sane bounds for an admin-set interval (minutes). +MIN_INTERVAL = 1 +MAX_INTERVAL = 1440 # one day + + +def interval_key(job: str) -> str: + return f"job_interval_{job}" + + # Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it # off env into the admin UI; the admin page renders it automatically from this list. A group's keys # stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS. SPECS: tuple[ConfigSpec, ...] = ( # --- Access / registration --- - ConfigSpec("allow_registration", "bool", "access", "allow_registration"), + ConfigSpec("allow_registration", "bool", "access", "allow_registration", applies=LIVE), # --- Email / SMTP (one logical group; the password is the only secret) --- - ConfigSpec("smtp_host", "str", "email", "smtp_host"), - ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535), - ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True), - ConfigSpec("smtp_from", "str", "email", "smtp_from"), - ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), + ConfigSpec("smtp_host", "str", "email", "smtp_host", applies=LIVE), + ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535, applies=LIVE), + ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True, applies=LIVE), + ConfigSpec("smtp_from", "str", "email", "smtp_from", applies=LIVE), + ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True, applies=LIVE), # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- - ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), + ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True, applies=LIVE), # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). - ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True), + ConfigSpec( + "youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True, applies=LIVE + ), # --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the # install wizard / Configuration page so no restart is needed when the creds change. --- - ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True), - ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True), - # --- Quota (shared YouTube Data API daily budget) --- - ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), + ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True, applies=LIVE), ConfigSpec( - "backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000 + "google_client_secret", "str", "google", "google_client_secret", secret=True, applies=LIVE + ), + # --- Quota (shared YouTube Data API daily budget) --- + ConfigSpec( + "quota_daily_budget", + "int", + "quota", + "quota_daily_budget", + min=1, + max=1_000_000, + applies=LIVE, + ), + ConfigSpec( + "backfill_quota_reserve", + "int", + "quota", + "backfill_quota_reserve", + min=0, + max=1_000_000, + applies=LIVE, ), ConfigSpec( "search_daily_limit_per_user", @@ -77,10 +138,19 @@ SPECS: tuple[ConfigSpec, ...] = ( "search_daily_limit_per_user", min=0, max=100_000, + applies=LIVE, ), # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). - ConfigSpec("search_source", "choice", "quota", "search_source"), - ConfigSpec("quota_retention_days", "int", "quota", "quota_retention_days", min=0, max=3650), + ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE), + ConfigSpec( + "quota_retention_days", + "int", + "quota", + "quota_retention_days", + min=0, + max=3650, + applies=LIVE, + ), # --- Backfill (recent-first first pass per channel) --- ConfigSpec( "backfill_recent_max_videos", @@ -89,19 +159,46 @@ SPECS: tuple[ConfigSpec, ...] = ( "backfill_recent_max_videos", min=1, max=100_000, + applies=LIVE, ), ConfigSpec( - "backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500 + "backfill_recent_max_days", + "int", + "backfill", + "backfill_recent_max_days", + min=1, + max=36_500, + applies=LIVE, + ), + ConfigSpec( + "rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168, applies=LIVE ), - ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168), # --- Shorts probe --- ConfigSpec( - "shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600 + "shorts_probe_max_seconds", + "int", + "shorts", + "shorts_probe_max_seconds", + min=1, + max=3_600, + applies=LIVE, + ), + ConfigSpec( + "shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000, applies=LIVE ), - ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000), # --- Batch sizes --- - ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), - ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), + ConfigSpec( + "enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50, applies=LIVE + ), + ConfigSpec( + "autotag_title_sample", + "int", + "batch", + "autotag_title_sample", + min=1, + max=1_000, + applies=LIVE, + ), # --- Maintenance (rolling re-validation of already-known videos) --- # Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin # knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard. @@ -112,16 +209,33 @@ SPECS: tuple[ConfigSpec, ...] = ( "maintenance_revalidate_batch", min=100, max=100_000, + applies=LIVE, ), # --- Discovery cleanup (search + explore ephemeral content) --- # Days an explored-but-unsubscribed channel is kept before the cleanup job purges it. - ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650), - # Days an un-kept live-search result video is kept before the cleanup job removes it. - ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650), - # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- - ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0), ConfigSpec( - "download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0 + "explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650, applies=LIVE + ), + # Days an un-kept live-search result video is kept before the cleanup job removes it. + ConfigSpec( + "search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650, applies=LIVE + ), + # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- + ConfigSpec( + "download_total_max_bytes", + "int", + "downloads", + "download_total_max_bytes", + min=0, + applies=LIVE, + ), + ConfigSpec( + "download_default_max_bytes", + "int", + "downloads", + "download_default_max_bytes", + min=0, + applies=LIVE, ), ConfigSpec( "download_default_max_concurrent", @@ -130,6 +244,7 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_default_max_concurrent", min=1, max=20, + applies=LIVE, ), ConfigSpec( "download_default_max_jobs", @@ -138,6 +253,7 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_default_max_jobs", min=1, max=100_000, + applies=LIVE, ), # min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as # `if cap and size > cap`, so a falsy value disabled it). Now the route passes this number @@ -161,14 +277,27 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_upload_max_bytes", min=1, max=settings.download_upload_max_bytes, + applies=LIVE, ), ConfigSpec( - "download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650 + "download_retention_days", + "int", + "downloads", + "download_retention_days", + min=0, + max=3_650, + applies=LIVE, ), ConfigSpec( - "download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650 + "download_gc_grace_days", + "int", + "downloads", + "download_gc_grace_days", + min=0, + max=3_650, + applies=LIVE, ), - ConfigSpec("download_layout", "choice", "downloads", "download_layout"), + ConfigSpec("download_layout", "choice", "downloads", "download_layout", applies=LIVE), ConfigSpec( "download_worker_concurrency", "int", @@ -176,60 +305,112 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_worker_concurrency", min=1, max=16, + applies=LIVE, ), - ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"), # --- Audit log --- # Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever). - ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650), + ConfigSpec( + "audit_retention_days", + "int", + "audit", + "audit_retention_days", + min=0, + max=3650, + applies=LIVE, + ), + # Days the job_runs table keeps a finished background run before the audit_gc job prunes it. + ConfigSpec( + "job_run_retention_days", + "int", + "audit", + "job_run_retention_days", + min=0, + max=3650, + applies=LIVE, + ), # --- Diagnostics --- # Runtime log level for the API process. Applied without a restart — see routes/config.py and # main.py's lifespan. - ConfigSpec("log_level", "choice", "diagnostics", "log_level"), + ConfigSpec("log_level", "choice", "diagnostics", "log_level", applies=LIVE), # Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs # carry account-linked request history, and the privacy policy names a limit. - ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90), + ConfigSpec( + "log_retention_days", + "int", + "diagnostics", + "log_retention_days", + min=1, + max=90, + applies=LIVE, + ), # --- Plex integration (optional; local-file playback + Plex metadata) --- - ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"), - ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"), - ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True), - ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"), - ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"), - ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"), - # (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.) - ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16), - ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64), + ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled", applies=LIVE), + ConfigSpec("plex_server_url", "str", "plex", "plex_server_url", applies=LIVE), + ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True, applies=LIVE), + ConfigSpec("plex_path_map", "str", "plex", "plex_path_map", applies=LIVE), + ConfigSpec("plex_libraries", "str", "plex", "plex_libraries", applies=LIVE), + ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map", applies=LIVE), + ConfigSpec( + "plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16, applies=LIVE + ), + ConfigSpec( + "plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64, applies=LIVE + ), # Transcode re-encode tuning. The libx264 knobs apply only to the software fallback. # crf 0–51 (x264 range); max_height 0 = source. - ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"), - ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"), - ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51), + ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw", applies=LIVE), + ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset", applies=LIVE), ConfigSpec( - "plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320 + "plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51, applies=LIVE + ), + ConfigSpec( + "plex_transcode_max_height", + "int", + "plex", + "plex_transcode_max_height", + min=0, + max=4320, + applies=LIVE, ), # Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play). - ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"), - # NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job - # (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS. + ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm", applies=LIVE), # --- TMDB enrichment (optional) — the API key is the only secret --- - ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True), - ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"), + ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True, applies=LIVE), + ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language", applies=LIVE), # --- Bug reports: the Gitea issue tracker a debug session's issue goes to --- - ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"), - ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"), - ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True), - # (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch - # is a config.py default — both stay out of SPECS, like plex_sync_interval_min.) + ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url", applies=LIVE), + ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo", applies=LIVE), + ConfigSpec( + "issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True, applies=LIVE + ), + # --- Scheduler: how often each interval job runs (minutes), one key per job --- + *( + ConfigSpec( + interval_key(job), + "int", + "scheduler", + attr, + min=MIN_INTERVAL, + max=MAX_INTERVAL, + applies=LIVE, + ) + for job, attr in JOB_INTERVAL_DEFAULTS.items() + ), ) # The admin page's sections (the owner's pick): sixteen groups as tabs wrapped to three rows at 133% # zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in -# exactly one section (test-held), so a new group cannot fall off the page. +# exactly one section, or is edited on its own page (test-held), so a new group cannot fall off. SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = ( ("access", ("access", "google", "email")), ("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")), ("media", ("downloads", "plex", "tmdb")), ("system", ("maintenance", "audit", "diagnostics", "issues")), ) +# Groups another admin page edits, so the Configuration page neither lists nor writes them: an +# interval is changed on the Scheduler page, which also reschedules the running job — a write that +# skipped it would leave the job on its old period while the page showed the new one. +OWN_PAGE_GROUPS: tuple[str, ...] = ("scheduler",) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} @@ -238,6 +419,12 @@ def spec(key: str) -> ConfigSpec | None: return _BY_KEY.get(key) +def config_page_spec(key: str) -> ConfigSpec | None: + """`spec`, for the Configuration page: None for a key edited on its own page as well.""" + s = _BY_KEY.get(key) + return None if s is None or s.group in OWN_PAGE_GROUPS else s + + def secrets_manageable() -> bool: """Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY).""" return bool(settings.token_encryption_key) @@ -426,13 +613,16 @@ def reset(db: Session, key: str) -> None: def describe(db: Session) -> dict: - """Admin-UI metadata + current values, grouped. Secret values are never echoed — - only whether an override is set and whether an env default exists.""" + """Admin-UI metadata + current values, grouped — the keys of the Configuration page, not those + edited on their own page. Secret values are never echoed — only whether an override is set and + whether an env default exists.""" groups: dict[str, list[dict]] = {} # Every override in one query: a db.get per key was one round trip each (two for a key with no # row, which the session cannot cache), ~40 ms of the page's ~55 on prod. rows = {r.key: r for r in db.scalars(select(SystemConfig))} for s in SPECS: + if s.group in OWN_PAGE_GROUPS: + continue row = rows.get(s.key) is_set = row is not None item: dict = { diff --git a/backend/tests/test_activity_reads.py b/backend/tests/test_activity_reads.py new file mode 100644 index 00000000..78e97ef7 --- /dev/null +++ b/backend/tests/test_activity_reads.py @@ -0,0 +1,205 @@ +"""The activity page's reads: the audit log and the background runs, filtered on the server and +paged by a keyset, and the quota breakdown over a time range and one account. + +Each list is asked for the rows a filter must keep AND must drop, and a page boundary is crossed +with rows that share a timestamp — the case an offset or a timestamp-only cursor gets wrong. +""" + +from datetime import datetime, timedelta, timezone + +from app.auth import admin_user +from app.main import app +from app.models import AuditLog, Channel, JobRun, LogEvent, QuotaEvent +from tests.test_body_limit import _route_objects + +T0 = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc) + + +def _audit(db, action, actor=None, at=T0, summary=None, request_id=None) -> int: + row = AuditLog( + actor_id=actor, action=action, summary=summary, created_at=at, request_id=request_id + ) + db.add(row) + db.flush() + return row.id + + +def _run(db, job, status="ok", actor=None, at=T0, target=None, trigger="scheduled") -> int: + row = JobRun( + job=job, + trigger=trigger, + actor_id=actor, + target=target, + status=status, + started_at=at, + finished_at=None if status == "running" else at + timedelta(seconds=4), + quota_units=0, + ) + db.add(row) + db.flush() + return row.id + + +def _ids(body) -> list[int]: + return [r["id"] for r in body["items"]] + + +def test_the_audit_list_filters_on_the_server_and_pages_across_a_shared_timestamp( + db, make_user, as_user +): + admin = make_user(role="admin") + a = _audit(db, "config_set", admin.id, T0, "quota_daily_budget = 9000") + b = _audit(db, "config_set", admin.id, T0, "log_level = DEBUG") + c = _audit(db, "config_set", admin.id, T0, "smtp_host = mail") + bg = _audit(db, "user_purged", None, T0 - timedelta(days=3)) + db.commit() + client = as_user(admin) + + first = client.get("/api/admin/audit", params={"limit": 2, "action": "config_set"}).json() + assert _ids(first) == [c, b] + second = client.get( + "/api/admin/audit", + params={"limit": 2, "action": "config_set", "before": first["next_cursor"]}, + ).json() + assert (_ids(second), second["next_cursor"]) == ([a], None) + # A page that ends exactly at the last row says there is nothing older. + whole = client.get("/api/admin/audit", params={"limit": 3, "action": "config_set"}).json() + assert (_ids(whole), whole["next_cursor"]) == ([c, b, a], None) + assert first["accounts"] == {str(admin.id): admin.email} + + def ids(**params) -> list[int]: + return _ids(client.get("/api/admin/audit", params=params).json()) + + assert ids(user="none") == [bg] + assert ids(user=str(admin.id)) == [c, b, a] + assert ids(q="DEBUG") == [b] + assert ids(since=(T0 - timedelta(days=1)).isoformat()) == [c, b, a] + assert ids(until=(T0 - timedelta(days=1)).isoformat()) == [bg] + # The action menu counts every action under the other filters, its own pick included. + facets = client.get("/api/admin/audit/facets", params={"action": "config_set"}).json() + assert facets["actions"] == [ + {"value": "config_set", "count": 3}, + {"value": "user_purged", "count": 1}, + ] + + +def test_an_admin_action_keeps_the_request_it_was_made_in(db, make_user, as_user): + client = as_user(make_user(role="admin")) + r = client.patch("/api/admin/config/quota_daily_budget", json={"value": 9500}) + assert r.status_code == 200, r.text + (row,) = client.get("/api/admin/audit", params={"action": "config_set"}).json()["items"] + assert row["request_id"] == r.headers["x-request-id"] + + +def test_an_admin_action_says_how_many_of_its_requests_lines_are_still_kept(db, make_user, as_user): + """A request's lines are kept days, its audit row months: a link to lines gone would open an + empty list, so the row says how many are left, as a run does.""" + admin = make_user(role="admin") + kept = _audit(db, "config_set", actor=admin.id, request_id="req-kept-0001") + gone = _audit(db, "config_set", actor=admin.id, request_id="req-gone-0001") + db.add_all( + [ + LogEvent( + ts=T0, + level="INFO", + sev=20, + source="api", + logger="t", + msg="m", + request_id="req-kept-0001", + ) + for _ in range(3) + ] + ) + db.commit() + items = as_user(admin).get("/api/admin/audit").json()["items"] + lines = {r["id"]: r["log_lines"] for r in items} + assert (lines[kept], lines[gone]) == (3, 0) + + +def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, make_user, as_user): + admin = make_user(role="admin") + db.add(Channel(id="UCchan", title="Aileen Wang")) + ok1 = _run(db, "rss_poll", at=T0) + ok2 = _run(db, "rss_poll", at=T0) + bad = _run(db, "backfill", "error", admin.id, T0, trigger="manual") + adhoc = _run( + db, "channel_backfill", "ok", admin.id, T0 - timedelta(hours=1), "UCchan", "ad_hoc" + ) + for _ in range(2): + db.add( + LogEvent(ts=T0, level="WARNING", sev=30, source="api", logger="t", msg="m", run_id=bad) + ) + db.commit() + client = as_user(admin) + + first = client.get("/api/admin/runs", params={"limit": 2}).json() + rest = client.get("/api/admin/runs", params={"limit": 2, "before": first["next_cursor"]}).json() + assert _ids(first) + _ids(rest) == [bad, ok2, ok1, adhoc] + assert rest["next_cursor"] is None + rows = {r["id"]: r for r in first["items"] + rest["items"]} + assert (rows[bad]["log_lines"], rows[ok2]["log_lines"]) == (2, 0) + assert (rows[adhoc]["target_title"], rows[ok1]["target_title"]) == ("Aileen Wang", None) + + def ids(**params) -> list[int]: + return _ids(client.get("/api/admin/runs", params=params).json()) + + assert ids(status="error") == [bad] + assert ids(job="rss_poll") == [ok2, ok1] + assert ids(user="none") == [ok2, ok1] + assert ids(user=str(admin.id)) == [bad, adhoc] + assert ids(q="UCchan") == [adhoc] + assert ids(since=(T0 - timedelta(minutes=30)).isoformat()) == [bad, ok2, ok1] + assert client.get("/api/admin/runs", params={"status": "finished"}).status_code == 422 + + facets = client.get("/api/admin/runs/facets", params={"status": "error"}).json() + assert facets["statuses"] == [{"value": "ok", "count": 3}, {"value": "error", "count": 1}] + assert facets["jobs"] == [{"value": "backfill", "count": 1}] + assert facets["total"] == 4 + + +def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_user): + admin = make_user(role="admin") + now = datetime.now(timezone.utc) + db.add_all( + [ + QuotaEvent(user_id=admin.id, action="search", units=100, created_at=now), + QuotaEvent(user_id=None, action="backfill", units=7, created_at=now), + QuotaEvent( + user_id=admin.id, action="search", units=50, created_at=now - timedelta(days=10) + ), + ] + ) + db.commit() + client = as_user(admin) + + def totals(**params) -> dict: + body = client.get("/api/quota/admin", params=params).json() + # The chart is the same filter's: its days add up to the table's total. + assert sum(d["total"] for d in body["daily"]) == sum(r["total"] for r in body["rows"]) + return {r["user_id"]: r["total"] for r in body["rows"]} + + assert totals() == {admin.id: 150, None: 7} # the last 30 days by default + assert totals(since=(now - timedelta(days=1)).isoformat()) == {admin.id: 100, None: 7} + assert totals(until=(now - timedelta(days=1)).isoformat()) == {admin.id: 50} + assert totals(user="none") == {None: 7} + assert totals(user=str(admin.id)) == {admin.id: 150} + + +def test_every_admin_read_and_write_is_admin_only(): + """The route table, not a list kept by hand: every /api/admin route and the quota breakdown + resolve `admin_user`, so a dropped dependency cannot hand an account's runs, errors and e-mail + addresses to any signed-in user.""" + routes = _route_objects(app) + admin = {k: r for k, r in routes.items() if k[1].startswith(("/api/admin", "/api/quota/admin"))} + assert ("GET", "/api/admin/runs") in admin and ("GET", "/api/quota/admin") in admin + open_ = [] + for key, route in admin.items(): + stack, calls = [route.dependant], set() + while stack: + d = stack.pop() + calls.add(d.call) + stack.extend(d.dependencies) + if admin_user not in calls: + open_.append(key) + assert open_ == [] diff --git a/backend/tests/test_admin_logs.py b/backend/tests/test_admin_logs.py index 7277108c..6677d67d 100644 --- a/backend/tests/test_admin_logs.py +++ b/backend/tests/test_admin_logs.py @@ -24,6 +24,7 @@ def _row(**kw) -> dict: "request_id": None, "user_id": None, "job_id": None, + "run_id": None, "version": "0.90.0", } row.update(kw) @@ -80,6 +81,7 @@ def test_newest_first_and_every_row_once_across_pages_even_on_equal_timestamps(d ({"event": "none"}, {"warn"}), ({"request_id": "req-abcdef12"}, {"mine"}), ({"job_id": 42}, {"err"}), + ({"run_id": 7}, {"warn"}), # a background run's lines, not a download job's ({"q": "FORBIDDEN"}, {"err"}), # in attrs, any case ({"q": "KeyError"}, {"warn"}), # in the stack ({"q": "100%"}, {"plain"}), # a literal, not a wildcard @@ -102,7 +104,14 @@ def test_each_filter_selects_exactly_its_rows(db, admin, make_user, params, expe job_id=42, attrs={"code": "forbidden"}, ), - _row(msg="warn", level="WARNING", sev=13, event=None, exc="Traceback …\nKeyError: 'x'"), + _row( + msg="warn", + level="WARNING", + sev=13, + event=None, + exc="Traceback …\nKeyError: 'x'", + run_id=7, + ), _row(msg="plain", ts=T0 - timedelta(hours=2), attrs={"note": "100% done"}), # Matched by no filter above but `until`: another account, and `100%` read as a pattern. _row(msg="decoy", ts=T0 - timedelta(hours=3), attrs={"note": "1000 done"}, user_id=99), diff --git a/backend/tests/test_audit_count_rows.py b/backend/tests/test_audit_count_rows.py index cdf47d6b..a2a2daf2 100644 --- a/backend/tests/test_audit_count_rows.py +++ b/backend/tests/test_audit_count_rows.py @@ -1,6 +1,6 @@ """An admin action that counts something keeps the count as data in its audit row's `after`, under the key the audit log names in the reader's language (`scheduler.result.`, frontend -auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word. +lib/jobResult.ts, called from lib/auditText.ts) — not only in the English `summary`. A renamed key would fall back to the raw word. DB lane (Postgres). """ diff --git a/backend/tests/test_channel_backfill.py b/backend/tests/test_channel_backfill.py index 7e651605..6e00567c 100644 --- a/backend/tests/test_channel_backfill.py +++ b/backend/tests/test_channel_backfill.py @@ -107,25 +107,48 @@ def _setup_channel_and_sub(db, *, email): def _patch_deep_requested(db, monkeypatch, override_dep, user): + submitted: list[tuple] = [] calls: list[tuple] = [] - # background.add_task runs after the response in TestClient; record what would be scheduled. + # Record the run the route submits, and what its work would call, instead of running it. + monkeypatch.setattr( + channels_route.scheduler, + "run_ad_hoc", + lambda job, fn, **kw: submitted.append((job, kw)) or fn(), + ) monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a)) monkeypatch.setattr(state, "_configured_cache", True) override_dep(get_db, db.info["request"]) override_dep(current_user, lambda: user) resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True}) - return resp, calls + return resp, submitted, calls -def test_deep_request_schedules_immediate_backfill(db, monkeypatch, override_dep): +def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel( + db, monkeypatch, override_dep +): + """It was a BackgroundTask: what it did, spent or failed with reached only the log.""" ch, me = _setup_channel_and_sub(db, email="wire@example.com") - resp, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) + resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) assert resp.status_code == 200, resp.text assert resp.json()["deep_requested"] is True - # scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs. + assert submitted == [("channel_backfill", {"actor_id": me.id, "target": ch.id})] + # its work runs exactly once, with (channel_id, user_id) — the args backfill_channel_now needs. assert calls == [(ch.id, me.id)] +def test_turning_full_history_on_for_a_finished_channel_records_no_run( + db, monkeypatch, override_dep +): + """Its whole history is in already: a run would only say "nothing to do", and switching the flag + off and on again would otherwise write one such row each time, kept for months.""" + ch, me = _setup_channel_and_sub(db, email="done@example.com") + ch.backfill_done = True + db.commit() + resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me) + assert resp.status_code == 200, resp.text + assert (submitted, calls) == ([], []) + + class TestImmediateBackfillGuard: """backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per- channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a @@ -148,15 +171,49 @@ class TestImmediateBackfillGuard: def test_a_second_call_for_an_in_flight_channel_is_a_noop(self, monkeypatch): runner._backfill_in_progress.add("UC_busy") monkeypatch.setattr(runner, "SessionLocal", self._boom_session) - runner.backfill_channel_now("UC_busy", 1) # returns immediately; _boom_session never runs + # returns at once — _boom_session never runs — and the run says why it did nothing. + assert runner.backfill_channel_now("UC_busy", 1) == {"skipped": "channel busy"} assert runner._backfill_in_progress == {"UC_busy"} # the in-flight entry is untouched def test_over_the_concurrency_cap_falls_back_to_the_scheduler(self, monkeypatch): runner._immediate_running = runner._MAX_CONCURRENT_IMMEDIATE monkeypatch.setattr(runner, "SessionLocal", self._boom_session) - runner.backfill_channel_now("UC_new", 1) # over the cap → no-op, scheduler will handle it + # over the cap → no-op, the scheduler will handle it + assert runner.backfill_channel_now("UC_new", 1) == {"skipped": "at capacity"} assert "UC_new" not in runner._backfill_in_progress + @staticmethod + def _one_channel(db, monkeypatch, cid): + ch = Channel(id=cid, title=cid, uploads_playlist_id="UU" + cid, backfill_done=False) + me = User(email=f"{cid}@example.com") + db.add_all([ch, me]) + db.commit() + monkeypatch.setattr(runner, "get_service_user", lambda _db: me) + monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False) + monkeypatch.setattr(runner, "YouTubeClient", lambda *a, **k: _NullYt()) + return ch, me + + def test_a_failure_reaches_its_run_and_still_frees_the_channel(self, db, monkeypatch): + """The failure was logged and swallowed, so the run that now wraps it would read "ok".""" + ch, me = self._one_channel(db, monkeypatch, "UC_fail") + + def boom(*_a): + raise RuntimeError("playlistItems 500") + + monkeypatch.setattr(runner, "backfill_channel_deep", boom) + with pytest.raises(RuntimeError, match="playlistItems 500"): + runner.backfill_channel_now(ch.id, me.id) + assert runner._backfill_in_progress == set() and runner._immediate_running == 0 + + def test_a_finished_backfill_says_what_it_added(self, db, monkeypatch): + ch, me = self._one_channel(db, monkeypatch, "UC_ok") + monkeypatch.setattr(runner, "backfill_channel_deep", lambda *_a: 7) + monkeypatch.setattr(runner, "reconcile_full_history", lambda _db: 1) + assert runner.backfill_channel_now(ch.id, me.id) == { + "videos_added": 7, + "reconciled_full": 1, + } + def test_scheduler_deep_pass_skips_an_in_flight_channel(self, db, monkeypatch): # A channel the scheduler's query WOULD pick (deep-requested, recent-synced, not done)… ch = Channel( diff --git a/backend/tests/test_migration_0109.py b/backend/tests/test_migration_0109.py new file mode 100644 index 00000000..495d7af7 --- /dev/null +++ b/backend/tests/test_migration_0109.py @@ -0,0 +1,165 @@ +"""Migration 0109's data step, DB-lane: the audit log's run rows move into `job_runs` with their +outcome intact, and leave the audit log; an admin action stays where it was. The downgrade moves +them back.""" + +import json +from datetime import timedelta + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def _audit(conn, actor, action, target, after): + conn.execute( + text( + "INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after," + " created_at) VALUES (:a, :act, 'job', :t, 's', CAST(:after AS json)," + " CAST('2026-10-01T03:00:00+00:00' AS timestamptz))" + ), + {"a": actor, "act": action, "t": target, "after": json.dumps(after)}, + ) + + +def test_the_audit_logs_runs_move_into_the_run_table_and_back(db, make_user): + uid = make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0108_folder_links") + try: + with engine.begin() as conn: + _audit( + conn, + None, + "job_run", + "rss_poll", + {"status": "ok", "summary": "new=3", "result": {"new": 3}}, + ) + _audit( + conn, + uid, + "job_run", + "rss_poll", + {"status": "error", "summary": "boom", "result": {"failure": "rss_all_failed"}}, + ) + _audit( + conn, uid, "job_run", "backfill", {"status": "skipped", "summary": "sync paused"} + ) + _audit(conn, uid, "config_set", None, {"key": "x"}) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + runs = conn.execute( + text( + "SELECT job, trigger, actor_id, status, summary, result, error, failure," + " to_char(started_at AT TIME ZONE 'UTC', 'MM-DD HH24'), finished_at = started_at" + " FROM job_runs ORDER BY id" + ) + ).all() + left = conn.execute(text("SELECT action FROM audit_log")).scalars().all() + assert [tuple(r) for r in runs] == [ + ("rss_poll", "scheduled", None, "ok", "new=3", {"new": 3}, None, None, "10-01 03", True), + ( + "rss_poll", + "manual", + uid, + "error", + None, + None, + "boom", + {"failure": "rss_all_failed"}, + "10-01 03", + True, + ), + ("backfill", "manual", uid, "skipped", "sync paused", None, None, None, "10-01 03", True), + ] + assert left == ["config_set"] + + command.downgrade(cfg, "0108_folder_links") + try: + with engine.connect() as conn: + back = conn.execute( + text( + "SELECT actor_id, target_id, after::jsonb FROM audit_log" + " WHERE action = 'job_run' ORDER BY id" + ) + ).all() + finally: + command.upgrade(cfg, "head") + + # What the audit row has no column for rides along, so the next upgrade restores it. + def run(trigger: str) -> dict: + return {"trigger": trigger, "quota_units": 0, "started_at": "2026-10-01T03:00:00+00:00"} + + assert [tuple(r) for r in back] == [ + ( + None, + "rss_poll", + {"status": "ok", "summary": "new=3", "result": {"new": 3}, **run("scheduled")}, + ), + ( + uid, + "rss_poll", + { + "status": "error", + "summary": "boom", + "result": {"failure": "rss_all_failed"}, + **run("manual"), + }, + ), + (uid, "backfill", {"status": "skipped", "summary": "sync paused", **run("manual")}), + ] + + +def test_a_down_and_up_round_trip_keeps_every_run(db, make_user): + """A rollback then a redeploy: an interrupted run, a run caught mid-way and a user's run of one + channel come back as runs — not stranded in the audit log under an action no code knows.""" + uid = make_user().id + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at," + " finished_at, quota_units) VALUES" + " ('backfill', 'scheduled', NULL, NULL, 'interrupted', now(), now(), 0)," + " ('maintenance', 'manual', :u, NULL, 'running', now(), NULL, 0)," + " ('channel_backfill', 'ad_hoc', :u, 'UC_x', 'ok', now(), now(), 0)," + # Its admin is gone (SET NULL): only the row itself still says it was manual. + " ('enrich', 'manual', NULL, NULL, 'ok'," + " '2026-10-01 10:00+00', '2026-10-01 10:30+00', 42)" + ), + {"u": uid}, + ) + # What the run caught mid-way had spent: only its quota rows know it yet. + conn.execute( + text( + "INSERT INTO quota_events (action, units, job_run_id)" + " SELECT 'videos.list', 7, id FROM job_runs WHERE job = 'maintenance'" + ) + ) + cfg = Config("alembic.ini") + command.downgrade(cfg, "0108_folder_links") + command.upgrade(cfg, "head") + with engine.connect() as conn: + runs = conn.execute( + text("SELECT job, trigger, actor_id, target, status FROM job_runs ORDER BY id") + ).all() + kept = conn.execute( + text("SELECT quota_units, finished_at - started_at FROM job_runs WHERE job = 'enrich'") + ).one() + stopped = conn.execute( + text("SELECT quota_units FROM job_runs WHERE job = 'maintenance'") + ).scalar() + stranded = conn.execute( + text("SELECT count(*) FROM audit_log WHERE action = 'job_run'") + ).scalar() + assert [tuple(r) for r in runs] == [ + ("backfill", "scheduled", None, None, "interrupted"), + ("maintenance", "manual", uid, None, "interrupted"), # the rollback stopped it + ("channel_backfill", "ad_hoc", uid, "UC_x", "ok"), + ("enrich", "manual", None, None, "ok"), + ] + assert tuple(kept) == (42, timedelta(minutes=30)) + assert stopped == 7 + assert stranded == 0 diff --git a/backend/tests/test_migration_0110.py b/backend/tests/test_migration_0110.py new file mode 100644 index 00000000..ba3677b8 --- /dev/null +++ b/backend/tests/test_migration_0110.py @@ -0,0 +1,56 @@ +"""Migration 0110's data step, DB-lane: an admin's interval override moves from `scheduler_settings` +into sysconfig and back, a row for a job that no longer exists is not carried over, and the unread +SponsorBlock default leaves the store. + +The schema half is checked by test_migrations.py; a value lost on the way is invisible there. +""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app import scheduler +from app.db import engine + + +def _config(conn) -> dict[str, str]: + return dict(conn.execute(text("SELECT key, value FROM system_config")).tuples().all()) + + +def test_an_interval_override_moves_into_sysconfig_and_back(db): + cfg = Config("alembic.ini") + command.downgrade(cfg, "0109_job_runs") + try: + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO scheduler_settings (job_id, interval_minutes)" + " VALUES ('rss_poll', 25), ('log_gc', 90), ('retired_job', 5)" + ) + ) + conn.execute( + text( + "INSERT INTO system_config (key, value, encrypted) VALUES" + " ('sponsorblock_default', 'true', false), ('quota_daily_budget', '5000', false)" + ) + ) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + assert _config(conn) == { + "job_interval_rss_poll": "25", + "job_interval_log_gc": "90", + "quota_daily_budget": "5000", + } + assert conn.execute(text("SELECT to_regclass('scheduler_settings')")).scalar() is None + assert scheduler.load_intervals(db)["rss_poll"] == 25 + + command.downgrade(cfg, "0109_job_runs") + try: + with engine.connect() as conn: + back = conn.execute(text("SELECT job_id, interval_minutes FROM scheduler_settings")) + assert dict(back.tuples().all()) == {"rss_poll": 25, "log_gc": 90} + assert _config(conn) == {"quota_daily_budget": "5000"} + finally: + command.upgrade(cfg, "head") diff --git a/backend/tests/test_plex_people_enrich.py b/backend/tests/test_plex_people_enrich.py index d10909d2..307f85ac 100644 --- a/backend/tests/test_plex_people_enrich.py +++ b/backend/tests/test_plex_people_enrich.py @@ -9,7 +9,6 @@ from datetime import datetime, timezone import httpx import pytest -from app import scheduler from app.models import PlexItem, PlexLibrary, PlexPerson, User from app.plex import imgcache from app.plex import people as plex_people @@ -340,12 +339,6 @@ def test_the_people_job_runs_its_halves_without_a_tmdb_key(db, cache_dir, plex_o } -def test_a_phase_count_is_a_change_worth_an_audit_row(): - assert scheduler._run_changed({"bios": {"enriched": 3}, "portraits": {"cached": 0}}) - assert scheduler._run_changed({"recent": {"videos_added": 2}, "deep": {}}) - assert not scheduler._run_changed({"bios": {"skipped": "x"}, "portraits": {"cached": 0}}) - - def test_the_person_page_says_a_bio_is_on_its_way(db, monkeypatch, override_dep): from fastapi.testclient import TestClient diff --git a/backend/tests/test_runstatus.py b/backend/tests/test_runstatus.py new file mode 100644 index 00000000..3cc331fd --- /dev/null +++ b/backend/tests/test_runstatus.py @@ -0,0 +1,27 @@ +"""The run words and the migrated database, DB-lane: the `job_runs` CHECKs are frozen in a migration, +and the model's are built from the enums, so nothing else would notice the two drifting apart until a +write of the new word failed in production.""" + +import re + +from sqlalchemy import text + +from app.runstatus import RunStatus, RunTrigger + + +def _words(definition: str) -> set[str]: + return set(re.findall(r"'([a-z_]+)'", definition)) + + +def test_the_databases_checks_allow_exactly_the_run_words(db): + defs = dict( + db.execute( + text( + "SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE conname LIKE 'ck_job_runs_%'" + ) + ).all() + ) + assert _words(defs["ck_job_runs_status"]) == set(RunStatus) + assert _words(defs["ck_job_runs_trigger"]) == set(RunTrigger) + assert _words(defs["ck_job_runs_finished"]) == {RunStatus.RUNNING} diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 9c47d3c9..32707328 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -1,11 +1,21 @@ -"""_claim_running is the single-run guard that both the scheduled fire and the manual trigger go +"""The scheduler's run path, DB-lane: the single-run guard, and every run as a durable `job_runs` row +— its outcome as data, what it spent, the lines it logged — that outlives the process. + +_claim_running is the single-run guard that both the scheduled fire and the manual trigger go through (at the top of _job): holding _activity_lock across the check-and-set means exactly one caller can hold a job's "running" claim at a time, and it's claimable again once released.""" +import logging +import time +from datetime import datetime, timedelta, timezone + +import pytest from sqlalchemy import select -from app import scheduler -from app.models import Notification +from app import jobruns, logs, logsink, quota, scheduler, state, sysconfig +from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent +from app.routes import scheduler as scheduler_routes +from app.runstatus import RunStatus, RunTrigger _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs @@ -31,26 +41,38 @@ def test_claim_is_exclusive_until_released(): # ---- the result as data -------------------------------------------------------------------------- -def _snapshot_job(job_id): - return next(j for j in scheduler.scheduler_snapshot()["jobs"] if j["id"] == job_id) +def _snapshot_job(db, job_id): + return next(j for j in scheduler.scheduler_snapshot(db)["jobs"] if j["id"] == job_id) + + +def _runs(db, job=None) -> list[JobRun]: + db.expire_all() + q = select(JobRun).order_by(JobRun.id) + if job is not None: + q = q.where(JobRun.job == job) + return list(db.execute(q).scalars()) + + +def _manual(actor_id, name, fn): + token = scheduler._manual_actor.set(actor_id) + try: + scheduler._job(name, fn) + finally: + scheduler._manual_actor.reset(token) def test_a_run_keeps_its_result_as_data_for_the_readers_language(db, make_user): """The dashboard and the manual-run notice showed `_summarize`'s English one-liner ("new=3, failed=0", a Python repr for a nested phase). The result rides along as data, so the client names - it; the one-liner stays as the fallback and the audit text.""" + it; the one-liner stays as the fallback.""" admin = make_user("sched@example.com", role="admin") - token = scheduler._manual_actor.set(admin.id) - try: - scheduler._job("rss_poll", lambda _db: {"new": 3, "failed": 0, "fallback": 1}) - scheduler._job("enrich", lambda _db: 5) # enrich returns a bare count - finally: - scheduler._manual_actor.reset(token) + _manual(admin.id, "rss_poll", lambda _db: {"new": 3, "failed": 0, "fallback": 1}) + _manual(admin.id, "enrich", lambda _db: 5) # enrich returns a bare count - rss = _snapshot_job("rss_poll") + rss = _snapshot_job(db, "rss_poll") assert rss["result"] == {"new": 3, "failed": 0, "fallback": 1} assert rss["last_result"] == "new=3, failed=0, fallback=1" - assert _snapshot_job("enrich")["result"] == {"enriched": 5} + assert _snapshot_job(db, "enrich")["result"] == {"enriched": 5} notices = db.execute( select(Notification).where(Notification.user_id == admin.id).order_by(Notification.id) @@ -64,53 +86,518 @@ def test_a_run_keeps_its_result_as_data_for_the_readers_language(db, make_user): def test_a_failed_run_drops_the_previous_result(db): scheduler._job("rss_poll", lambda _db: {"new": 1}) scheduler._job("rss_poll", lambda _db: (_ for _ in ()).throw(RuntimeError("boom"))) - rss = _snapshot_job("rss_poll") + rss = _snapshot_job(db, "rss_poll") + assert rss["status"] == "error" assert rss["last_error"] == "boom" assert rss["result"] is None -def test_a_job_runs_audit_row_carries_its_result_as_data(db): - """The audit log names a run's result in the reader's language, as the inbox does — - from data, not from the English one-liner (a nested phase there was a Python dict repr).""" - from app.models import AuditLog, User - - admin = User(email="auditrun@example.com") - db.add(admin) - db.commit() - scheduler._audit_run(db, None, "people_enrich", "ok", "x", {"bios": {"enriched": 2}}) - scheduler._audit_run(db, admin.id, "rss_poll", "skipped", "sync paused", scheduler._PAUSED) - scheduler._audit_run(db, admin.id, "rss_poll", "error", "boom", None) - rows = db.execute(select(AuditLog).order_by(AuditLog.id)).scalars().all() - assert [r.after for r in rows] == [ - {"status": "ok", "summary": "x", "result": {"bios": {"enriched": 2}}}, - {"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}}, - {"status": "error", "summary": "boom"}, - ] - - def test_a_failure_the_job_names_rides_along_as_data(db, make_user): - """A failed RSS poll reached the Hungarian audit log as the server's English sentence. A - failure the job names (`progress.JobFailed`) is stored as data beside that sentence — on the - dashboard, the manual-run notice and the audit row — so the client says it in the reader's - language. Any other failure keeps only its own text (the test above).""" + """A failed RSS poll reached the Hungarian UI as the server's English sentence. A failure the job + names (`progress.JobFailed`) is stored as data beside that sentence — on the run's row, the + dashboard and the manual-run notice — so the client says it in the reader's language.""" from app import progress - from app.models import AuditLog def outage(_db): raise progress.JobFailed("rss_all_failed", "RSS poll failed", total=2, cause="transport") admin = make_user("failure@example.com", role="admin") - token = scheduler._manual_actor.set(admin.id) - try: - scheduler._job("rss_poll", outage) - finally: - scheduler._manual_actor.reset(token) + _manual(admin.id, "rss_poll", outage) named = {"failure": "rss_all_failed", "total": 2, "cause": "transport"} - rss = _snapshot_job("rss_poll") + rss = _snapshot_job(db, "rss_poll") assert rss["last_error"] == "RSS poll failed" assert rss["result"] == named notice = db.execute(select(Notification).where(Notification.user_id == admin.id)).scalar_one() assert notice.data["result"] == named - row = db.execute(select(AuditLog).order_by(AuditLog.id.desc())).scalars().first() - assert row is not None and row.after["result"] == named + (run,) = _runs(db, "rss_poll") + assert (run.status, run.error, run.failure) == ("error", "RSS poll failed", named) + + +# ---- the durable row ----------------------------------------------------------------------------- + + +def test_every_run_is_a_row_with_who_started_it_and_none_in_the_audit_log(db, make_user): + """A no-op scheduled poll was not written anywhere (the audit rule kept only changes), so "did + it run at 03:00?" had no answer. Every run is a row now; the audit log keeps admin actions only.""" + admin = make_user("runs@example.com", role="admin") + scheduler._job("rss_poll", lambda _db: {"new": 0}) + _manual(admin.id, "rss_poll", lambda _db: {"new": 2}) + + scheduled, manual = _runs(db, "rss_poll") + assert (scheduled.trigger, scheduled.actor_id, scheduled.status) == ("scheduled", None, "ok") + assert scheduled.result == {"new": 0} + assert (manual.trigger, manual.actor_id, manual.summary) == ("manual", admin.id, "new=2") + assert manual.finished_at is not None and manual.finished_at >= manual.started_at + assert db.execute(select(AuditLog)).scalars().all() == [] + + +def test_the_last_run_survives_a_restart(db): + """The dashboard's "last run" lived in the process's memory: a deploy blanked every job.""" + scheduler._job("plex_sync", lambda _db: {"items": 4}) + with scheduler._activity_lock: + scheduler._activity.clear() # what a new process starts with + + plex = _snapshot_job(db, "plex_sync") + assert (plex["status"], plex["result"], plex["last_trigger"]) == ( + "ok", + {"items": 4}, + "scheduled", + ) + assert plex["last_finished"] is not None + + +def test_a_row_shows_its_last_three_runs_newest_first_an_interrupted_one_included(db): + """The row said only how the LAST run went; an interrupted run read as "no run recorded".""" + oldest = jobruns.start("shorts", RunTrigger.SCHEDULED, None) + jobruns.interrupt_leftovers() + scheduler._job("shorts", lambda _db: {"probed": 1}) + scheduler._job("shorts", lambda _db: {"probed": 2}) + scheduler._job("shorts", lambda _db: 1 / 0) + scheduler._job("rss_poll", lambda _db: {"new": 0}) # another job's run stays on its own row + + recent = _snapshot_job(db, "shorts")["recent"] + assert [r["status"] for r in recent] == ["error", "ok", "ok"] + assert oldest not in [r["id"] for r in recent] # the fourth-newest is not shown + assert recent[0]["trigger"] == "scheduled" and recent[0]["finished_at"] is not None + assert [r["status"] for r in _snapshot_job(db, "rss_poll")["recent"]] == ["ok"] + + jobruns.start("shorts", RunTrigger.SCHEDULED, None) + jobruns.interrupt_leftovers() + shorts = _snapshot_job(db, "shorts") + assert shorts["status"] == "interrupted" + assert [r["status"] for r in shorts["recent"]] == ["interrupted", "error", "ok"] + + +def test_a_paused_sync_is_a_skipped_run(db): + state.set_sync_paused(db, True) + db.commit() + scheduler._job("rss_poll", lambda _db: pytest.fail("a paused run must not do its work")) + (run,) = _runs(db, "rss_poll") + assert (run.status, run.result) == ("skipped", {"skipped": "sync paused"}) + + +def test_a_run_left_running_by_a_stopped_process_is_interrupted_and_stays_so(db): + """A deploy mid-run left nothing behind before; now the row would say `running` forever. The + next startup closes it, and the stale thread's own finish cannot rewrite it.""" + run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None) + assert jobruns.interrupt_leftovers() == 1 + + (run,) = _runs(db, "backfill") + assert run.status == "interrupted" and run.finished_at is not None + jobruns.finish(run_id, RunStatus.OK, summary="late") + (run,) = _runs(db, "backfill") + assert (run.status, run.summary) == ("interrupted", None) + + +def test_a_run_knows_what_it_spent_and_a_request_spend_is_nobodys_run(db): + def spends(_db): + quota.record_usage(5) + quota.record_usage(3) + return {"enriched": 2} + + scheduler._job("enrich", spends) + quota.record_usage(11) # a request's own spend, outside any run + + (run,) = _runs(db, "enrich") + assert run.quota_units == 8 + events = db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() + assert sorted(events) == [(3, run.id), (5, run.id), (11, None)] + assert _snapshot_job(db, "enrich")["quota_units"] == 8 + + +class _Collect(logging.Handler): + def __init__(self): + super().__init__() + self.addFilter(logs.ContextFilter()) + self.records: list[logging.LogRecord] = [] + + def emit(self, record): + self.records.append(record) + + +def test_a_runs_log_lines_carry_its_id(db): + """So a run's row can open exactly the lines it logged (the log table's `run_id`).""" + probe = logging.getLogger("siftlode.test.run") + handler = _Collect() + probe.addHandler(handler) + probe.setLevel(logging.INFO) + try: + scheduler._job("autotag", lambda _db: probe.info("tagging") or {"tagged": 1}) + probe.info("after the run") + finally: + probe.removeHandler(handler) + + (run,) = _runs(db, "autotag") + inside, after = (logs.record_fields(r).get("run_id") for r in handler.records) + assert (inside, after) == (run.id, None) + # …and it reaches the stored row, whose columns are the model's. + assert logsink.to_row(logs.record_fields(handler.records[0]))["run_id"] == run.id + + +def test_retention_reaps_finished_runs_only(db): + old = datetime.now(timezone.utc) - timedelta(days=40) + db.add_all( + [ + JobRun( + job="rss_poll", + trigger="scheduled", + status="ok", + started_at=old, + finished_at=old, + ), + JobRun(job="backfill", trigger="scheduled", status="running", started_at=old), + JobRun( + job="rss_poll", + trigger="scheduled", + status="ok", + started_at=datetime.now(timezone.utc), + finished_at=datetime.now(timezone.utc), + ), + ] + ) + db.commit() + reaped = next(r for r in _runs(db) if r.status == "ok" and r.started_at < old + timedelta(1)) + with logs.bind(run_id=reaped.id): + quota.record_usage(7) + assert jobruns.gc(db, 0) == 0 # 0 = keep forever + assert jobruns.gc(db, 30) == 1 + db.commit() + assert sorted((r.job, r.status) for r in _runs(db)) == [ + ("backfill", "running"), + ("rss_poll", "ok"), + ] + # Its spend stays in the budget's history (kept on its own retention), no longer anyone's run. + assert db.execute(select(QuotaEvent.units, QuotaEvent.job_run_id)).all() == [(7, None)] + + +# ---- "run now" through the scheduler -------------------------------------------------------------- + + +@pytest.fixture +def live_scheduler(db, monkeypatch): + """This process's scheduler, started as the API starts it (on the migrated schema `db` brings), + with interval jobs off.""" + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", False) + scheduler.start_scheduler() + try: + yield scheduler._scheduler + finally: + scheduler.shutdown_scheduler() + + +def _wait_finished(db, job, timeout=10.0) -> JobRun: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + runs = [r for r in _runs(db, job) if r.status != "running"] + if runs: + return runs[-1] + time.sleep(0.05) + raise AssertionError(f"{job} did not finish") + + +def test_run_now_goes_through_the_scheduler_even_with_interval_jobs_off( + db, make_user, live_scheduler, monkeypatch +): + admin = make_user("now@example.com", role="admin") + monkeypatch.setitem( + scheduler.JOB_FUNCS, "shorts", lambda: scheduler._job("shorts", lambda _db: {"probed": 3}) + ) + assert live_scheduler.get_jobs() == [] # SCHEDULER_ENABLED=false: no interval job + live_scheduler.pause() # hold the one-shot so the second click finds it pending + assert scheduler.trigger_job("shorts", admin.id) == "started" + assert scheduler.trigger_job("shorts", admin.id) == "already_running" + assert [j.id for j in live_scheduler.get_jobs()] == ["shorts@now"] + live_scheduler.resume() + + run = _wait_finished(db, "shorts") + assert (run.trigger, run.actor_id, run.result) == ("manual", admin.id, {"probed": 3}) + + +def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure( + db, make_user, live_scheduler +): + """The one-channel deep backfill ran as a BackgroundTask nothing recorded: its spend counted as + nobody's run and a failure lived only in the log.""" + user = make_user("adhoc@example.com") + + def work(): + quota.record_usage(3) + return {"videos_added": 50} + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, work, actor_id=user.id, target="UC_a") + run = _wait_finished(db, "channel_backfill") + assert (run.trigger, run.actor_id, run.target) == ("ad_hoc", user.id, "UC_a") + assert (run.status, run.result, run.quota_units) == ("ok", {"videos_added": 50}, 3) + + def fail(): + raise RuntimeError("quota exceeded") + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, fail, actor_id=user.id, target="UC_b") + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and len(_finished(db, "channel_backfill")) < 2: + time.sleep(0.05) + failed = next(r for r in _runs(db, "channel_backfill") if r.target == "UC_b") + assert (failed.status, failed.error) == ("error", "quota exceeded") + + +def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user): + user = make_user("shown@example.com") + db.add(Channel(id="UC_shown", title="BBC Earth")) + db.commit() + scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none") + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown") + + (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert (row["id"], row["running"], row["target"], row["target_title"]) == ( + "channel_backfill", + False, + "UC_shown", + "BBC Earth", + ) + assert (row["status"], row["result"], row["last_trigger"]) == ( + "ok", + {"videos_added": 69}, + "ad_hoc", + ) + assert len(row["recent"]) == 2 + assert all(j["id"] != "channel_backfill" for j in scheduler.scheduler_snapshot(db)["jobs"]) + + seen = [] + scheduler._ad_hoc( + "channel_backfill", + lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {}, + user.id, + "UC_shown", + ) + assert seen == [True] + assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False + + +def _finished(db, job) -> list[JobRun]: + return [r for r in _runs(db, job) if r.status != "running"] + + +def test_an_ad_hoc_job_outside_the_named_list_is_refused(live_scheduler): + """The page names every ad-hoc job from AD_HOC_JOBS; one outside it would show as a bare id.""" + with pytest.raises(KeyError): + scheduler.run_ad_hoc("unnamed_job", dict, actor_id=1, target="x") + assert live_scheduler.get_jobs() == [] + + +def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( + db, make_user, live_scheduler +): + user = make_user("twice@example.com") + live_scheduler.pause() # hold the one-shots so the second request finds the first pending + for target in ("UC_x", "UC_x", "UC_y"): + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, dict, actor_id=user.id, target=target) + assert sorted(j.id for j in live_scheduler.get_jobs()) == [ + "channel_backfill@UC_x", + "channel_backfill@UC_y", + ] + live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next + + +def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user): + """The scheduler's pool threads are reused: an actor a manual run left set made the next + scheduled run on that thread an admin's "manual" run, with a notice in their inbox.""" + admin = make_user("pool@example.com", role="admin") + seen = [] + scheduler._as_actor(lambda: seen.append(scheduler._manual_actor.get()), admin.id)() + assert seen == [admin.id] + assert scheduler._manual_actor.get() is None # this same thread's next task runs as nobody + + +def test_run_now_refuses_a_job_already_running(live_scheduler): + assert scheduler._claim_running("maintenance") is True + try: + assert scheduler.trigger_job("maintenance") == "already_running" + finally: + scheduler._record("maintenance", running=False) + + +def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch): + jobruns.start("plex_sync", RunTrigger.MANUAL, None) + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", False) + scheduler.start_scheduler() + scheduler.shutdown_scheduler() + (run,) = _runs(db, "plex_sync") + assert run.status == "interrupted" + + +def test_an_interrupted_run_keeps_what_it_spent(db): + """Its spend is in the quota history, tagged with its id: the sweep sums it as `finish` would.""" + run_id = jobruns.start("backfill", RunTrigger.SCHEDULED, None) + with logs.bind(run_id=run_id): + quota.record_usage(40) + quota.record_usage(2) + assert jobruns.interrupt_leftovers() == 1 + (run,) = _runs(db, "backfill") + assert (run.status, run.quota_units) == ("interrupted", 42) + + +def _flaky_finish(monkeypatch) -> list: + """`jobruns.finish` whose first write fails (the database blinked); the statuses it was asked for.""" + calls: list = [] + real = jobruns.finish + + def finish(run_id, status, **outcome): + calls.append(status) + if len(calls) == 1: + raise RuntimeError("db blip") + real(run_id, status, **outcome) + + monkeypatch.setattr(jobruns, "finish", finish) + return calls + + +def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_user, monkeypatch): + """The job succeeded; only its bookkeeping did not. Writing ERROR then would say the work failed.""" + calls = _flaky_finish(monkeypatch) + scheduler._job("autotag", lambda _db: {"tagged": 1}) + assert calls == ["ok"] + assert not scheduler._is_running("autotag") + (run,) = _runs(db, "autotag") + assert run.status == "running" # left for the next start's sweep, not rewritten as a failure + # Nothing here runs it: the page reads it as the sweep will close it, not as a run going on. + row = _row(db, "autotag") + assert (row["running"], row["status"], row["recent"][0]["status"]) == ( + False, + "interrupted", + "interrupted", + ) + assert row["last_finished"] is not None + + user = make_user("blip@example.com") + calls.clear() + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") + assert calls == ["ok"] + # Its row is still `running`, but nothing here is: the page does not call it going. + (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert (row["running"], row["status"], row["recent"][0]["status"]) == ( + False, + "interrupted", + "interrupted", + ) + + +def test_a_run_going_on_reads_as_running_on_its_row(db): + seen = [] + scheduler._job("autotag", lambda _db: seen.append(_row(db, "autotag")) or {"tagged": 1}) + (row,) = seen + assert (row["running"], row["status"], row["recent"][0]["status"]) == (True, None, "running") + assert _row(db, "autotag")["recent"][0]["status"] == "ok" + + +def _row(db, job_id: str) -> dict: + return next(j for j in scheduler.scheduler_snapshot(db)["jobs"] if j["id"] == job_id) + + +def test_a_run_that_cannot_be_recorded_does_not_wedge_its_job(db, monkeypatch): + def broken(*_a, **_k): + raise RuntimeError("db down") + + monkeypatch.setattr(jobruns, "start", broken) + scheduler._job("autotag", lambda _db: pytest.fail("no row, no run")) + assert not scheduler._is_running("autotag") + monkeypatch.undo() + scheduler._job("autotag", lambda _db: {"tagged": 1}) + assert [r.status for r in _runs(db, "autotag")] == ["ok"] + + +def test_the_page_says_where_the_interval_jobs_do_not_run(db, live_scheduler, monkeypatch): + """The scheduler starts on every install now; only SCHEDULER_ENABLED says the jobs run here.""" + assert scheduler.scheduler_snapshot(db)["running_here"] is False + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True) + assert scheduler.scheduler_snapshot(db)["running_here"] is True + + +def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_user): + """Two channels can backfill at once; a later run that ended at once must not hide the first.""" + user = make_user("two@example.com") + db.add(Channel(id="UC_long", title="Long one")) + db.commit() + rows = [] + + def long_one(): + scheduler._ad_hoc( + "channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_short" + ) + rows.extend(scheduler.scheduler_snapshot(db)["ad_hoc"]) + return {} + + scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long") + (row,) = rows + assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one") + # The outcome beside it is the long one's (none yet), never the short one's skip. + assert (row["status"], row["result"], row["last_error"]) == (None, None, None) + assert [r["status"] for r in row["recent"]] == ["running", "ok"] + + +def test_an_interval_change_lands_with_its_audit_row_or_not_at_all( + db, make_user, as_user, live_scheduler, monkeypatch +): + live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill") + admin = as_user(make_user(role="admin")) + + def broken(*_a, **_k): + raise RuntimeError("audit write failed") + + monkeypatch.setattr(scheduler_routes.audit, "record", broken) + with pytest.raises(RuntimeError): # the handler raises: there is no response to check + _ = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45}) + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == scheduler.JOB_INTERVALS["backfill"] + assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=10) + + +def test_every_job_has_an_interval_and_every_interval_a_job(): + assert set(scheduler.JOB_FUNCS) == set(scheduler.JOB_INTERVALS) + + +def test_a_stored_interval_is_what_the_job_starts_with(db, monkeypatch): + sysconfig.set_value(db, "job_interval_rss_poll", 25) + db.commit() + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True) + scheduler.start_scheduler() + try: + sched = scheduler._scheduler + assert sched is not None + assert sched.get_job("rss_poll").trigger.interval == timedelta(minutes=25) + default = scheduler.JOB_INTERVALS["enrich"] + assert sched.get_job("enrich").trigger.interval == timedelta(minutes=default) + finally: + scheduler.shutdown_scheduler() + + +def test_an_interval_set_on_the_scheduler_page_is_stored_and_reschedules_the_job( + db, make_user, as_user, live_scheduler +): + live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill") + admin = as_user(make_user(role="admin")) + r = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45}) + assert r.status_code == 200, r.text + assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=45) + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == 45 + assert scheduler.load_intervals(db)["backfill"] == 45 + # The Configuration page neither lists an interval nor writes one past the reschedule. + assert ( + admin.patch("/api/admin/config/job_interval_backfill", json={"value": 5}).status_code == 404 + ) + assert admin.delete("/api/admin/config/job_interval_backfill").status_code == 404 + assert "scheduler" not in admin.get("/api/admin/config").json()["groups"] + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == 45 + + +def test_the_setup_wizard_does_not_write_an_interval(db): + from fastapi import HTTPException + + from app.routes.setup import setup_config + + with pytest.raises(HTTPException) as refused: + setup_config({"job_interval_rss_poll": 5}, db) + assert refused.value.status_code == 400 + assert setup_config({"quota_daily_budget": 5000}, db)["saved"] == ["quota_daily_budget"] diff --git a/backend/tests/test_sysconfig_registry.py b/backend/tests/test_sysconfig_registry.py index 8a2f5d9b..fbcc6667 100644 --- a/backend/tests/test_sysconfig_registry.py +++ b/backend/tests/test_sysconfig_registry.py @@ -15,10 +15,14 @@ boot, sysconfig holds a stored override to it, and the admin page renders a pick The DB lane (`db` fixture) covers the store; the rest is pure logic. """ +import ast +from pathlib import Path + import pytest from pydantic import ValidationError from app import sysconfig +from app.models import SystemConfig from app.config import Settings, choices_of CHOICE_KEYS = {s.key for s in sysconfig.SPECS if s.type == "choice"} @@ -37,7 +41,8 @@ def test_each_group_is_one_run_of_keys(): def test_every_group_sits_in_exactly_one_section(): listed = [g for _, members in sysconfig.SECTIONS for g in members] assert len(listed) == len(set(listed)), listed - assert set(listed) == {s.group for s in sysconfig.SPECS} + assert set(listed) | set(sysconfig.OWN_PAGE_GROUPS) == {s.group for s in sysconfig.SPECS} + assert not set(listed) & set(sysconfig.OWN_PAGE_GROUPS) assert [sid for sid, _ in sysconfig.SECTIONS] == ["access", "youtube", "media", "system"] @@ -112,3 +117,78 @@ def test_set_value_stores_the_canonical_spelling_and_refuses_anything_else(db): with pytest.raises(ValueError, match="scrape, api"): sysconfig.set_value(db, "search_source", "bing") db.rollback() + + +# --- when a changed value takes effect -------------------------------------------------------------- + +_APP = Path(__file__).resolve().parent.parent / "app" +_READERS = {"get", "get_str", "get_int", "get_bool"} + + +def _key_reads() -> list[tuple[str, str, bool]]: + """Every `sysconfig.get*(db, "")` in the app outside the registry: (key, where, inside a + function).""" + found: list[tuple[str, str, bool]] = [] + + def walk(node: ast.AST, path: Path, in_fn: bool) -> None: + for child in ast.iter_child_nodes(node): + fn = in_fn or isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + if ( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Attribute) + and isinstance(child.func.value, ast.Name) + and child.func.value.id == "sysconfig" + and child.func.attr in _READERS + and len(child.args) >= 2 + and isinstance(child.args[1], ast.Constant) + and isinstance(child.args[1].value, str) + ): + found.append((child.args[1].value, f"{path.name}:{child.lineno}", in_fn)) + walk(child, path, fn) + + for path in _APP.rglob("*.py"): + if path.name != "sysconfig.py": + walk(ast.parse(path.read_text(encoding="utf-8")), path, False) + return found + + +def test_every_key_states_when_it_applies_and_none_needs_a_restart_yet(): + with pytest.raises(TypeError): + sysconfig.ConfigSpec("k", "int", "quota", "quota_daily_budget") # pyright: ignore[reportCallIssue] + # The Configuration page cannot tell the admin that a setting waits for a restart; a key that + # does needs that first (a badge on it and a banner once it changed). + assert {s.applies for s in sysconfig.SPECS} == {"live"} + + +def test_a_live_key_is_read_where_it_is_used_never_at_import(): + reads = _key_reads() + assert len(reads) > 50, len( + reads + ) # the scan sees the reads (it would pass vacuously otherwise) + assert [(k, at) for k, at, in_fn in reads if not in_fn] == [] + + +def test_every_key_on_the_configuration_page_has_a_reader(): + read = {k for k, _, _ in _key_reads()} + page = {s.key for s in sysconfig.SPECS if s.group not in sysconfig.OWN_PAGE_GROUPS} + assert sorted(page - read) == [] + + +def test_the_intervals_are_keys_of_the_scheduler_page_only(): + keys = {s.key for s in sysconfig.SPECS if s.group == "scheduler"} + assert keys == {sysconfig.interval_key(j) for j in sysconfig.JOB_INTERVAL_DEFAULTS} + assert all(len(k) <= 64 for k in keys) # system_config.key + assert sysconfig.config_page_spec("job_interval_rss_poll") is None + assert sysconfig.config_page_spec("quota_daily_budget") is not None + assert "scheduler" not in sysconfig.describe(_FakeDb())["groups"] + + +def test_saving_the_value_in_force_still_stores_the_override(db, make_user, as_user): + client = as_user(make_user(role="admin")) + default = sysconfig.get_int(db, "quota_daily_budget") + r = client.patch("/api/admin/config/quota_daily_budget", json={"value": default}) + assert r.status_code == 200, r.text + (item,) = [i for i in r.json()["groups"]["quota"] if i["key"] == "quota_daily_budget"] + assert item["is_set"] is True + db.expire_all() + assert db.get(SystemConfig, "quota_daily_budget") is not None diff --git a/docs/DESIGN.md b/docs/DESIGN.md index f8774b57..8ea9f4c0 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -275,10 +275,9 @@ Also in force: - **Page chrome (B4):** a page's tabs and filters sit in ONE sticky band of the page colour, with the fade under it (below). Below ~600px of height the filter row folds into one "Filters · n" button. Every page has a visually hidden `h1`; the nav rail and the browser tab title are the visible identity. [WCAG 2.4.6] - Built as: the band is `PageShell`'s toolbar slot (`PageToolbar`), above the one scroller; a console - whose toolbar lives inside its frame (the server log) makes that toolbar `sticky` with `fade-below` - instead. The band holds controls only: an intro paragraph scrolls with the content (in the band it - ate three lines of a short window). The fold is `useFilterFold` + `FilterFoldButton`: the same row + Built as: the band is `PageShell`'s toolbar slot (`PageToolbar`), above the one scroller. The band + holds controls only: an intro paragraph scrolls with the content (in the band it ate three lines of a + short window). The fold is `useFilterFold` + `FilterFoldButton`: the same row shown or hidden in place, never copied into a popover. The hidden `h1` is `PageShell`'s `title`; a view with its own visible title (a channel, a Plex show, a playlist) renders it as `PageHeading` and the hidden one steps aside, so a page has exactly one `h1` (a hand `

` inside the shell is @@ -649,7 +648,7 @@ of a full-width panel and nothing on the right. `app/collation.py` on the server, so both sort alike. - **Hungarian words, not English ones** (E30 D3). Kept as Hungarian users say them: Feed, Shorts, Admin, Token, Port, and technical tokens (log levels, ffmpeg presets, remux). Everything else is Hungarian: - közvetítés (stream), Prioritás, Feladat (job), Eseménynapló (audit), Demó, Automatikus, Infó, + közvetítés (stream), Prioritás, Feladat (job), Napló (activity), Admin műveletek (audit), Demó, Automatikus, Infó, Felület / Szerver (frontend / backend), videók behúzása (backfill), átkódolás (transcode), e-mail. A longer Hungarian word gets its room — a column is sized to it, a pill truncates — never clipped. - A label says what happens ("Delete playlist"), not "OK"; a destructive confirm names the object. @@ -668,127 +667,129 @@ of a full-width panel and nothing on the right. ## Decision log -| Date | Decision | Chosen | Over | Why | -| ---------- | ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2026-09-24 | Floating button | Back to top: a round 48 glass FAB; "round" = floats over something (media or the page) | A Material 3 rounded square; a labelled pill; the old 64 circle | The common web convention for back to top; round tells it apart from every toolbar button; 64 was oversized | -| 2026-09-24 | Device contract | Tablet portrait + landscape and up; phones out | A 1024px floor with a "too small" message | Portrait tablet use is real, and the app already renders at 820 portrait | -| 2026-09-24 | Card actions on touch | Fewer visible, the rest in a "…" menu | 44px buttons that no longer fit the card | Six 44px targets need more width than a card has | -| 2026-09-24 | Action placement | One menu with several triggers, never gesture-only, a desktop background menu that mirrors the toolbar | Visible buttons for everything | Fewer non-critical buttons on screen, without hiding any function | -| 2026-09-24 | Text selection | Text stays selectable; a click that ends a selection does nothing | Click-to-play on titles | The user must be able to copy any text the app shows | -| 2026-09-24 | A1 Button shape | r8 + a rung per context; glass is a Button variant | r12; a pill | Nested corners (r8 in r12 in r16); the pill already means "a filter or a choice" | -| 2026-09-24 | A2 Icon buttons | Two sizes, r8; a round button only on media | One size; all round | A lone toolbar icon at the row size looks lost; round rows read as tags | -| 2026-09-24 | A3 Touch size | 44 for every control primitive | 36 + gaps; Button only | One rule (HIG 44) instead of two sizes nobody remembers | -| 2026-09-24 | A4 Input | Bordered, a 55% fg-into-bg edge (≥ 3:1) | Filled; today's look as a primitive | The only option that clears WCAG 1.4.11 without a heavy fill on glass | -| 2026-09-24 | A5 Dropdown trigger | Pill in a toolbar, r8 in a form | Pill everywhere; r8 everywhere | Matches the neighbours in each context | -| 2026-09-24 | A6 Checkbox, switch, slider | Native, accent-tinted, the row is the target | Drawn primitives | Free keyboard / screen-reader / forced-colours support; only 3 sites | -| 2026-09-24 | B1 Density | Compact 32 / 28 / 24 by explicit height | Standard 38 / 32 / 26 | The owner's read of the lab; touch is covered separately (A3), and one toolbar row lines up at 28 | -| 2026-09-24 | B2 Scheme set | Midnight, Forest→cyan, Slate→fuchsia, Matrix→lime | Today's six | The accent collided with danger (YouTube), warn (Slate) and success (Forest, Matrix); Midnight and Starship were one scheme twice; 4 of 6 light accents failed 4.5:1 | -| 2026-09-24 | B3 State layers | Text colour at 10% / 16% over anything | Material's 8 / 12%; today's opacity + bg-card | Visible in every scheme and mode (≥ 1.2:1); today's row hover was 1.00:1 on a card | -| 2026-09-24 | B4 Page chrome | One sticky band + fade; filters fold when short | A floating glass bar; everything scrolls | Tabs and filters always in reach, in one place; a keyboard-up tablet keeps its content | -| 2026-09-24 | B5 Surfaces | One class + radius per role | Today's per-site mix | Nested corners and no per-site alpha drift | -| 2026-09-24 | B6 /watch | The app's surfaces, native video controls | Flat and minimal | The page strangers see should look like the product; native controls work on any device | -| 2026-09-24 | B7 Nav rail | Soft tint + edge bar, optional labels (a user setting), slim | Solid accent (recommended); a Material rail | The owner's combination: calmer than a solid fill, labels for who wants them, no width cost | -| 2026-09-24 | B8 Loading | Skeleton by shape, spinner in a control | A spinner everywhere | Nothing jumps when data lands; a long wait doesn't look stuck | -| 2026-09-24 | B9 Fade | Toolbars / headers, panel tabs, dialog edges, chip rows | Thumbnails; buttons | The north-star depth where content scrolls under chrome; not on content or small controls | -| 2026-09-24 | C1 Action classification | The table (lab C1): state + primary at rest, everyday toggles on hover, "…" on touch, destructive into the menu | Visible buttons per site, as today | One action list per object, nothing gesture-only, fewer buttons without losing a function | -| 2026-09-24 | C2 Side panels | A tap opens an overlay drawer; pin is a separate button ≥ 1024 | A tap pins and the content reflows | Below 768 the filters were unreachable; on touch a tap pinned and pushed the grid | -| 2026-09-24 | C3 Players on touch | Tap = show the controls; 44px seek hit area; 44 buttons; the YT description behind ⓘ | Today (tap also pauses; 7.5px seek bar) | Measured live: you couldn't reach the seek bar without pausing, and the hover-only description never opened on a tap | -| 2026-09-24 | C4 Dialogs | Focus by purpose; a pinned footer | Focus on Close; the footer scrolls | A form starts in its first field, and Save stays on screen with the keyboard up | -| 2026-09-24 | C5 Plex player controls | The owner's lean set: no Stop / Restart / ±10 s buttons, Back by position, Next only with a next item, Resume + From the start on the card, touch double-tap ±10 s | Every control in one row (it ran off-screen below 1024); Back by a press counter; ±10 s keyboard-only | The row fits a portrait tablet; the decision to restart is made on the card; a tablet keeps the 10 s skip | -| 2026-09-24 | Glass tuner | Retired; the schemes are the only colour choice | Keep it with ranges clamped to the rules | Free tuning is the drift the constitution exists to stop (owner) | -| 2026-09-24 | Matrix toned down | Moss #86a05a at Midnight's lightness, near-neutral greys; every white-on-accent part → `--accent-fg` | The B2 lime #a3e635 | The owner found the lime neon and tiring; it was the loudest accent (L 0.85, C 0.21), and fixed white parts sat on it at 1.5:1 | -| 2026-09-24 | Field primitives | `Input` / `Textarea` / `InnerInput` + `fieldClass`, `SelectTrigger`, the native controls; the lint rule "no raw ``" | `inputCls` and ~40 hand-styled boxes | Six edges, four radii and two focus looks for one control; the ≥ 3:1 edge held nowhere | -| 2026-09-25 | Surface roles | `SURFACE` roles in `ui/surfaces.ts`; the lint rule "no bare glass class"; menus p-1 with r8 rows | Three tiers paired ad hoc with five radii and three menu paddings | ~140 sites drifted; nested corners only hold if the radius comes with the role | -| 2026-09-25 | Static labels | Flat tonal `Badge`, no edge; a filtering tag is an edged chip | A glass pill with an edge, like the chips | Edged = pressable at a glance; a static tag that looks like a chip gets clicked and does nothing | -| 2026-09-25 | Header search | A 40 glass pill with the field edge (`lg`); what sits inside it is round | 44, r16, a glass box with a faint edge | The one rounded box among pills, and its edge under 3:1; the hero search is the YouTube/Gmail pill | -| 2026-09-25 | Switch row (F12) | The whole settings row toggles (`SwitchRow`; a hand-laid row points a label at the switch) | Only the 36×24 switch toggles | The iOS / Material settings row; the text was a dead click beside the control it names | -| 2026-09-25 | Edges and hovers (F14) | The ≥ 3:1 edge on the field family (fields, the form trigger); labelled pills keep the faint edge and the accent-edge hover | The ≥ 3:1 edge and a state-layer hover on every control | 1.4.11 asks an edge only where nothing else identifies the control; a heavy edge on every chip and tab weighs the whole UI down | -| 2026-09-25 | Rating and live (F16) | Own tokens: `--rating` (theme-scoped gold), `--live`/`--live-fg`, `--rating-media`, `--media-chrome`; palette classes lint-refused | Map them to `--warn` / `--danger` | A gold star and a red LIVE are what the eye expects; as warn/danger they would read as a status. The old amber score was 2:1 on light | -| 2026-09-25 | Fjord trial (B2 exception) | A fifth scheme, temporary: Midnight's neutrals, Playnite's steel accent (#337599 lifted to #5aa1cb) and steel rules; exit = fold into Midnight or remove | Retune Midnight straight away; borrow only the qualities | The owner wants to live with the calmer palette app-wide before it replaces the default; it breaks "own hue family" on purpose, and only until that call | -| 2026-09-25 | Whole-card row (owner) | A card row's B3 layer is the card's `::before` under its content; the open button's hit layer spans the card, the actions sit above it | The name button's own hover, ending at the first badge or button | A row that half-answers the pointer reads as broken; a badge click did nothing | -| 2026-09-25 | Badge tones (F29) | `Badge tone` neutral / accent / warn at 6%; light `--warn` #b45309 → #a14a07; edged label spans lint-refused | Keep edged status pills (a private Badge in Users, Settings "Granted", the suggestions count) | Edged = pressable at a glance; amber-700 was 4.58:1 as text and fell under 4.5 on its own tint | -| 2026-09-26 | Page band built (B4) | Controls only in the band (intros scroll); the fold shows the same row in place; one `h1` per page (`PageHeading` steps the hidden one aside); `SectionLabel` headings; Skip to content | A popover of the folded filters; the page title kept visible | A folded row is still the same controls, nothing duplicated; the rail and the tab title already name the page | -| 2026-09-26 | Fade built (B9) | `fade-below` on sticky toolbars and table headers; the scroller mask where the band sits on the backdrop | A painted band over the backdrop | The backdrop is the colour under the band, so fading the content into it is the same edge | -| 2026-09-26 | Viewport units | `dvh` everywhere (the Plex player stage with S3f) | `vh` / `h-screen` | `100vh` hides the last rows under a tablet browser's toolbar | -| 2026-09-26 | Video actions built (C1) | One list per video (menu, right-click, long-press, "…"); the card keeps watched / save / add to playlist on its picture on hover; download, details, copy, hide, reset in the menu | The six-button row under the card and the hover ⓘ | The menu is the one door every device has; the card loses a line of height and the touch card its row of 28px buttons | -| 2026-09-26 | Plex actions built (C1) | One list per Plex title (play/open, watched, add to playlist, media info, copy title, pin); the poster keeps watched / add / "…" on its picture on hover; seasons get open / watched / add / copy | The hover ⓘ, the pin-only menu, an episode's add button at rest, a collection's "Remove" X on the card | The same door as a video's; the collection grid goes to two columns on a phone so the touch "…" leaves the title its line | -| 2026-09-26 | Playlist actions built (C1) | A playlist row's "…" (play, pin, copy title / link, move, order, remove), Delete + an undo toast; one list per playlist (rename, pin, YouTube open / push / revert, delete) from its rail row and its header's "…" | The row's X column and its confirm; the header's pencil, YouTube link, push, revert and trash buttons | Remove is reversible, so it needs no confirm; the header keeps Play all; a narrow table drops "#" and Channel so the title keeps a width | -| 2026-09-26 | Channel + Plex list actions (C1) | A channel row's "…" (open, pin, YouTube, copy, full history, hide + undo, unsubscribe); a Plex playlist's title / show / season "…", Delete + undo; its header's "…" (rename, delete); an action toast lasts ≥ 8 s | The channel row's full-history, hide and unsubscribe buttons; the Plex list's red X on every row, show and season; the header's pencil and trash | Hiding and removing are reversible, so they need no confirm and the undo needs time; unsubscribe and deleting a playlist keep theirs | -| 2026-09-26 | Download / notification / Scheduler actions (C1) | A download row's "…" (play, share, save, edit, details, re-download, manage sharing, copies, cancel / delete / remove); a library row's Share stays in sight, ahead of save; a notification row's "…" and hover dismiss; the inbox band's and the Scheduler strip's "…" | The download row's five rare buttons and its cancel / delete / remove; the source link's copy button; an activity notice's Find / Undo links; the band's Clear everything; the strip's Purge discovery | The destructive rows keep their confirm, and Clear everything gains one; a row link keeps the browser's own menu; Find in feed drops the tag / search / age filters that could hide its video | -| 2026-09-26 | Channel menu: Tags + Priority (C1) | Tags A–Z as in-place checks; Priority as fixed steps 10 · 5 · 3 · 2 · 1 · 0, the current one ticked; a pick re-sorts at once and the page + focus follow the row; one drawn `MenuCheck` for every checkable row that cannot be an `` | Relative steps (+1 / +5); straight to the top; the stepper only; seven hand-drawn check marks (three box sizes, a tinted box, a × for "on", a trailing tick) | Bubbling a channel up is one pick; fixed levels keep the scale the reader already uses; the list moving where the reader can see it is the feedback | -| 2026-09-26 | Reorder without a drag (WCAG 2.5.7) | Row menus: Top / Up / Down / Bottom (YouTube + Plex playlists); saved views: four buttons with the place beside the selected view (owner: A); side panels in edit mode: an Up / Down pair per group (owner: A), the name wrapping there | A "…" on every saved-view row; a "…" on every panel group; drag only | A visible, one-click path where the list is edited; the edge moves shown off so the buttons never shift | -| 2026-09-26 | Titles in click surfaces | The surface stays a button, flagged `selectable`: its text selects, and the click that ends a selection does nothing; every plain `Pressable` pinned with its reason | Rebuilding each card and row so the title sits outside the button; exceptions for master-detail lists | Chrome selects inside a button once it is `user-select: text` (measured), so one flag on the primitive keeps the keyboard, focus and list walking every surface already has | -| 2026-09-26 | Right-click on a selection | The browser's own menu opens on selected words; beside them the object's menu | A "Copy selection" row in the app menu | The reader selected the words to do something with them; the native menu copies, searches and translates, and changes nothing else | -| 2026-09-26 | Keyboard ring on a flush row | Inside the row (a table row by its tag, a log line, a show header in its clipping card, with the card's corners) | The ring outside, as on a card | Outside it ran under the table head and past the table, and a clipping card cut it away entirely (invisible focus) | -| 2026-09-26 | Fade at rest | The sticky header's fade appears with the first 1rem of scroll (a scroll-driven animation; no fade where unsupported) | A fade that is always drawn | At rest it dimmed the first row and hid its ring's top edge; nothing is under the header yet | -| 2026-09-26 | Narrow playlist table | "#" and Channel hide by the table's own width (under 42rem), not the window's | Hiding below the `sm` window width | Beside the playlist rail a desktop window left the table 432 px and the title 0 px | -| 2026-09-26 | Background menu built | Five list pages (feed, Plex grid, channels, playlist, notifications); the toolbar's sort / view / filters and the band's actions; no destructive row | Every page with a toolbar; the "…" menus copied whole, delete included | The tab-only pages have nothing a menu would add; a blank-area right-click must not be one click from delete | -| 2026-09-26 | Media card anatomy (C) | One frame; the meta line pinned to its bottom; the length a badge on the picture; the Plex card on the same primitive as the video card | A: the year as an overline over the title; B: the year inline after it; a fixed two-line title slot | A one-line title left a hole the eye reads as a missing line; C puts nothing under a variable-height title | -| 2026-09-26 | One Play | `MediaPlayGlyph` (the media button variant) and only where the click plays | The Plex card's hand-painted accent circle; a playable dot on every card | One Play for every card; a dot that only a colour and a tooltip explain says nothing | -| 2026-09-26 | Filtering values on the info page | A value that filters (year, certificate, studio, director) is an edged chip; a fact is flat text | Everything flat, the clickable ones only hover-tinted | The S3d rule: edged = pressable, so the reader sees which values do something | -| 2026-09-26 | Shelf density | On any strip: one title line over one meta line | The grid's two lines and full meta on a strip | The tallest card set the strip's height and left holes in the cards with less to say | -| 2026-09-26 | Compact list row (B) | The title leads; a faint "…" at the end; the toggles float over the date on hover | A leading 128px action column | That column sat empty at rest, a hole at the start of every row | -| 2026-09-26 | Media-info layout (B) | A main column (collections first, crew, cast) beside a facts sidebar; an empty column is not drawn | Every section full width, stacked | The collection is the most used; the keywords' column left a void beside a short details box | -| 2026-09-26 | Mixed Home shelf (A) | Plex titles as wide art beside videos | Posters, with the videos not stretched | Stretching a video card to a poster's height left an ~80px gap | -| 2026-09-27 | Home hero | A card (hover row + Play on the picture, the whole menu, a touch "…"; owner: A); three quarters wide and centred on a wide window; one fact line; the summary at the measure | The hero's pin row alone; full width; an 18rem picture column | The one answer stands in focus, and it offers what the same card offers anywhere | -| 2026-09-27 | Home shelf tiles | A channel / playlist tile: Play, ↗ open, "…" with Open; a show acts on its next episode, ⓘ opens it; the shelf slot 232px | The tile's pin row alone; 200px | A tile without its object's actions read as broken; at 200 the Play touched the hover row | -| 2026-09-27 | Synopsis preview (C) | A ~0.9 s mouse rest opens a passive panel beside the card | The synopsis as the picture's `title` tooltip; an ⓘ to hover; none | One long tooltip line fired anywhere on the art and clashed with the cut title's own tooltip | -| 2026-09-27 | Reading measure | Prose capped at ~65 characters a line (`measure`) | The container's full width | At 110–130 characters a line the eye loses the next line's start | -| 2026-09-24 | Chip family | One `Chip` (filter / token / tag / suggest / ghost) + `DismissibleChip`; nothing under 24; the lint rule "no raw ` + + {/* The action menu above is built from these counts: without them it offers only "Any". */} + {facets.isError && ( + void facets.refetch()}> + {t("activity.filtersFailed")} + + )} + + {state.rows.map((row, index) => { + const { head, sub } = auditText(t, row, lang); + const who = actor(row); + return ( + setOpen(open === row.id ? null : row.id)} + title={formatDateTimeSec(row.created_at, lang)} + cells={ + <> + {auditActionLabel(t, row.action)} + + + {who ?? t("audit.system")} + + + + {[head, sub].filter(Boolean).join(" · ")} + + + } + > + + + ); + })} + + + ); +} + +function AuditDetail({ + row, + who, + onRequest, +}: { + row: AuditRow; + who: string | null; + onRequest: (requestId: string) => void; +}) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const { head, sub } = auditText(t, row, lang); + return ( +
+
+
{t("audit.cols.actor")}
+
{who ?? t("audit.system")}
+
{t("audit.cols.when")}
+
{formatDateTimeSec(row.created_at, lang)}
+ {(head || sub) && ( + <> +
{t("audit.cols.details")}
+
{[head, sub].filter(Boolean).join(" · ")}
+ + )} + {row.target_id && ( + <> +
{t("activity.audit.target")}
+
{row.target_id}
+ + )} +
+ {(row.before || row.after) && ( + 
+          {JSON.stringify({ before: row.before, after: row.after }, null, 2)}
+
+ )} + {/* A request's lines are kept days, its row months: a link to none would open an empty list. */} + {row.request_id && + (row.log_lines > 0 ? ( +
+ +
+ ) : ( +

{t("activity.audit.noRequestLines")}

+ ))} +
+ ); +} diff --git a/frontend/src/components/ActivityQuota.tsx b/frontend/src/components/ActivityQuota.tsx new file mode 100644 index 00000000..7c601a8f --- /dev/null +++ b/frontend/src/components/ActivityQuota.tsx @@ -0,0 +1,153 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { keepPreviousData, useQuery } from "@tanstack/react-query"; + +import { api, type AdminQuotaRow } from "../lib/api"; +import { sinceOf } from "../lib/activityUrlState"; +import { formatNumber, quotaActionLabel } from "../lib/format"; +import { qk } from "../lib/queryKeys"; +import type { Shared } from "./ActivityRuns"; +import { Pressable } from "./ui/Pressable"; +import { LoadingState, StateMessage } from "./ui/QueryState"; +import { SURFACE } from "./ui/surfaces"; + +// Activity → Quota: the shared YouTube API budget's spend over the page's time range — per day +// (Pacific days, as the budget resets), then per account, each opening to its actions. The +// account filter narrows both to one account; the text filter has nothing to match here. + +export default function ActivityQuota({ shared }: { shared: Shared }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const q = useQuery({ + queryKey: qk.adminQuota({ window: shared.window, user: shared.user }), + queryFn: () => api.adminQuota(sinceOf(shared.window), shared.user), + placeholderData: keepPreviousData, + }); + const data = q.data; + const maxDaily = Math.max(1, ...(data?.daily ?? []).map((d) => d.total)); + const grandTotal = (data?.rows ?? []).reduce((s, r) => s + r.total, 0); + + return ( +
+

+ {t("stats.introBefore")} + {t("stats.introSystem")} + {t("stats.introAfter")} +

+ {q.isError && !data ? ( + void q.refetch()}> + {t("common.loadError")} + + ) : q.isLoading || !data ? ( + + ) : ( + <> + {q.isError && ( + void q.refetch()} className="mb-3"> + {t("activity.stale")} + + )} +
+
+ {t(shared.window === "all" ? "activity.quota.totalAll" : "activity.quota.total", { + count: grandTotal, + window: t(`activity.windows.${shared.window}`), + units: formatNumber(grandTotal, lang), + })} +
+ {data.daily.length === 0 ? ( +
{t("stats.noUsageInRange")}
+ ) : ( + // Each day's figure is in its bar's text too (read out, not shown): the tooltip only + // reaches a pointer. +
+ {data.daily.map((d) => ( +
+ + {t("stats.dailyTooltip", { + count: d.total, + day: d.day, + units: formatNumber(d.total, lang), + })} + +
+
+ ))} +
+ )} +
+ +
+ {data.rows.length === 0 ? ( +
{t("stats.noUsageInRange")}
+ ) : ( + data.rows.map((r) => ( + + )) + )} +
+ + )} +
+ ); +} + +function UserRow({ row, max }: { row: AdminQuotaRow; max: number }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const [open, setOpen] = useState(false); + const actions = Object.entries(row.by_action).sort((a, b) => b[1] - a[1]); + const isSystem = row.user_id === null; + return ( +
+ setOpen((o) => !o)} + aria-expanded={open} + className="w-full flex items-center gap-3" + > + + {isSystem ? t("stats.system") : row.email} + + + {formatNumber(row.total, lang)} + + + {/* Proportion bar */} +
+
+
+ {open && actions.length > 0 && ( +
+ {actions.map(([action, units]) => ( +
+ {quotaActionLabel(t, action)} + {formatNumber(units, lang)} +
+ ))} +
+ )} +
+ ); +} diff --git a/frontend/src/components/ActivityRuns.tsx b/frontend/src/components/ActivityRuns.tsx new file mode 100644 index 00000000..8407c8ff --- /dev/null +++ b/frontend/src/components/ActivityRuns.tsx @@ -0,0 +1,282 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { keepPreviousData, useQuery } from "@tanstack/react-query"; + +import { api, type LogFacet, type RunQuery, type RunRow } from "../lib/api"; +import { + RUN_STATUSES, + sinceOf, + type ActivityUrlState, + type RunStatusFilter, +} from "../lib/activityUrlState"; +import { formatDateTimeSec, formatNumber, formatRuntime } from "../lib/format"; +import { jobFailureText, jobResultText, parseSummary } from "../lib/jobResult"; +import { qk } from "../lib/queryKeys"; +import { useHistoryList } from "../lib/useHistoryList"; +import HistoryList from "./HistoryList"; +import HistoryRow from "./HistoryRow"; +import { StateMessage } from "./ui/QueryState"; +import { RelativeTime } from "./ui/RelativeTime"; +import { StatusDot } from "./RunStatusDot"; +import { Button } from "./ui/Button"; +import FilterChipGroup from "./ui/FilterChipGroup"; +import FilterSelect from "./ui/FilterSelect"; + +// Activity → Runs: every background run, newest first — the scheduled ones, an admin's "run now", a +// user's work on one channel. A row opens to who started it, when, what it returned or why it +// failed, and two ways on: the lines the run logged, and the other runs of its job. + +const GRID = + "grid-cols-[0.75rem_minmax(0,1fr)_auto] sm:grid-cols-[0.75rem_minmax(8rem,12rem)_minmax(0,8rem)_5.5rem_4rem_5.5rem_minmax(10rem,1fr)]"; + +/** The shared filters, as the page passes them down. */ +export interface Shared { + window: ActivityUrlState["window"]; + user: string | null; + q: string; +} + +export default function ActivityRuns({ + shared, + job, + status, + onFilter, + onLogs, +}: { + shared: Shared; + job: string | null; + status: RunStatusFilter | null; + onFilter: (patch: { job?: string | null; status?: RunStatusFilter | null }) => void; + /** Open the server log at this run's lines. */ + onLogs: (runId: number) => void; +}) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const [open, setOpen] = useState(null); + const query: RunQuery = { + user: shared.user ?? undefined, + q: shared.q.trim() || undefined, + job: job ?? undefined, + status: status ?? undefined, + }; + const key = { ...query, window: shared.window }; + const state = useHistoryList(qk.adminRuns(key), (before) => + api.adminRuns({ ...query, since: sinceOf(shared.window) }, before), + ); + const facets = useQuery({ + queryKey: qk.adminRunFacets(key), + queryFn: () => api.adminRunFacets({ ...query, since: sinceOf(shared.window) }), + placeholderData: keepPreviousData, + }); + const count = (list: LogFacet[] | undefined, value: string) => + list?.find((f) => f.value === value)?.count ?? (facets.data ? 0 : undefined); + const jobName = (id: string) => t(`scheduler.jobs.${id}`, id); + // The job in use stays offered at zero: under the other filters it may count nothing, and a menu + // without it would show no way back. + const jobs = [ + ...(facets.data?.jobs ?? []), + ...(job && !facets.data?.jobs.some((f) => f.value === job) ? [{ value: job, count: 0 }] : []), + ]; + + return ( +
+
+ ({ value: f.value, label: jobName(f.value), count: f.count })), + ]} + selected={[job ?? ""]} + onChange={([value]) => onFilter({ job: value || null })} + testId="runs-job" + /> + + onFilter({ status: value === "all" ? null : (value as RunStatusFilter) }) + } + options={[ + { value: "all", label: t("activity.any"), count: facets.data?.total }, + ...RUN_STATUSES.map((s) => ({ + value: s, + label: t(`activity.runs.status.${s}`), + count: count(facets.data?.statuses, s), + })), + ]} + /> +
+ {/* The menus above are built from these counts: without them they offer only "Any". */} + {facets.isError && ( + void facets.refetch()}> + {t("activity.filtersFailed")} + + )} + + {state.rows.map((run, index) => ( + setOpen(open === run.id ? null : run.id)} + title={formatDateTimeSec(run.started_at, lang)} + cells={ + + } + > + onLogs(run.id)} + onJob={() => onFilter({ job: run.job })} + /> + + ))} + +
+ ); +} + +/** How long the run took; null while it runs, 0 when its start and end are one instant (the runs + * moved over from the audit log kept only when they were written — the length is unknown). */ +function seconds(run: RunRow): number | null { + if (!run.finished_at) return null; + return (new Date(run.finished_at).getTime() - new Date(run.started_at).getTime()) / 1000; +} + +/** What the run came to, in the reader's language: its failure, that the process stopped under it, + * or its result. */ +function useOutcome(run: RunRow): { text: string; tone: string } { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + if (run.status === "error") + return { + text: jobFailureText(t, run.failure, lang) ?? run.error ?? t("scheduler.dot.error"), + tone: "text-danger", + }; + if (run.status === "interrupted") return { text: t("scheduler.interrupted"), tone: "text-warn" }; + if (run.status === "running") return { text: t("scheduler.runningNow"), tone: "text-accent" }; + // A run moved over from the audit log before its result rode along as data holds only the + // server's English one-liner: read back into a result, as the old audit rows were. + const result = + run.result ?? + parseSummary(run.summary) ?? + (run.status === "skipped" && run.summary ? { skipped: run.summary } : null); + if (result) return { text: jobResultText(t, result, lang), tone: "text-muted" }; + return { text: run.summary ?? "", tone: "text-muted" }; +} + +function RunCells({ run, actor }: { run: RunRow; actor: string | null | undefined }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const outcome = useOutcome(run); + const secs = seconds(run); + const name = t(`scheduler.jobs.${run.job}`, run.job); + return ( + <> + + + {name} + {run.target_title && · {run.target_title}} + + {/* A phone has room for the name and the time only; the row opens to the rest. */} + + + {t(`scheduler.trigger.${run.trigger}`)} + {actor && ` · ${actor}`} + + + + {secs ? formatRuntime(Math.max(1, secs), lang) : "—"} + + {/* The spend carries its unit (a bare figure in a row has no meaning); none spent is blank. */} + + {run.quota_units > 0 && + t("activity.runs.units", { + count: run.quota_units, + units: formatNumber(run.quota_units, lang), + })} + + {outcome.text} + + ); +} + +function RunDetail({ + run, + actor, + onLogs, + onJob, +}: { + run: RunRow; + actor: string | null | undefined; + onLogs: () => void; + onJob: () => void; +}) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const link = "underline decoration-dotted underline-offset-2 hover:decoration-solid"; + return ( +
+
+
{t("activity.runs.fields.started")}
+
+ {run.actor_id === null + ? t(`scheduler.trigger.${run.trigger}`) + : `${actor ?? `#${run.actor_id}`} · ${t(`scheduler.trigger.${run.trigger}`)}`} +
+
{t("activity.runs.fields.when")}
+
+ {formatDateTimeSec(run.started_at, lang)} + {run.finished_at && ` → ${formatDateTimeSec(run.finished_at, lang)}`} +
+ {run.target && ( + <> +
{t("activity.runs.fields.target")}
+
+ {run.target_title ? `${run.target_title} · ` : ""} + {run.target} +
+ + )} +
{t("activity.runs.fields.quota")}
+
{formatNumber(run.quota_units, lang)}
+
+ {run.error && ( + 
{run.error}
+ )} + {run.result && ( + 
+          {JSON.stringify(run.result, null, 2)}
+
+ )} +
+ {/* Most runs store no line at all; a link there would only open an empty list. */} + {run.log_lines > 0 ? ( + + ) : ( + {t("activity.runs.noLogLines")} + )} + +
+
+ ); +} diff --git a/frontend/src/components/AdminLogs.dom.test.tsx b/frontend/src/components/AdminLogs.dom.test.tsx index 9b9c28de..01580322 100644 --- a/frontend/src/components/AdminLogs.dom.test.tsx +++ b/frontend/src/components/AdminLogs.dom.test.tsx @@ -1,9 +1,11 @@ // @vitest-environment jsdom // -// The admin log console. A request id narrows the list to that request (and says so, in the -// filter row and the URL); a filter value the facets no longer count stays pickable; "Live" watches -// the newest page only, and stops on a failed read instead of re-raising it; a failed read is an -// error, not an empty log; Copy takes every loaded line, oldest first. +// The activity page's server-log tab, rendered through the page (which owns the shared filters and +// the pivots). A request or run id narrows the list to that request or run (and says so, in the +// filter row and the URL), and Back returns to the view the pivot was made from; the account in use +// stays pickable; "Live" watches the newest page only, and stops on a failed read instead of +// re-raising it; a failed read is an error, not an empty log; Copy takes every loaded line, oldest +// first. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -32,6 +34,7 @@ vi.mock("../lib/api", async (importOriginal) => { ...actual.api, adminLogs: (query: LogQuery, before: string | null) => logsMock(query, before), adminLogFacets: (query: LogQuery) => facetsMock(query), + adminUsers: () => Promise.resolve([{ id: 1, email: "admin@example.com" }]), }, }; }); @@ -44,7 +47,11 @@ vi.mock("../lib/clipboard", () => ({ }, })); -import AdminLogs from "./AdminLogs"; +import Activity from "./Activity"; +import { PageToolbarSlot } from "./ui/pageToolbarSlot"; + +// The filter bar lives in the toolbar, which portals into the shell's band: give it one. +const slot = document.body.appendChild(document.createElement("div")); function row(id: number, extra: Partial = {}): ServerLogRow { return { @@ -60,6 +67,7 @@ function row(id: number, extra: Partial = {}): ServerLogRow { request_id: `req-${String(id).padStart(8, "0")}`, user_id: 1, job_id: null, + run_id: null, debug_session_id: null, version: "0.90.0", ...extra, @@ -79,7 +87,9 @@ function renderConsole() { const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( - + + + , ); } @@ -93,7 +103,7 @@ const lines = () => .filter((b) => b.hasAttribute("aria-expanded")); beforeEach(() => { - window.history.replaceState(null, "", "/?page=logs"); + window.history.replaceState(null, "", "/?page=activity&at=logs"); facetsMock.mockResolvedValue(FACETS); }); afterEach(() => { @@ -104,8 +114,21 @@ afterEach(() => { }); describe("AdminLogs", () => { + it("a dialog closing onto the same address leaves the open line open", async () => { + logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + fireEvent.click(lines()[0]!); + expect(lines()[0]!.getAttribute("aria-expanded")).toBe("true"); + // A closing dialog steps back over its own history entry: a popstate onto the same address. + act(() => { + window.dispatchEvent(new PopStateEvent("popstate")); + }); + expect(lines()[0]!.getAttribute("aria-expanded")).toBe("true"); + }); + it("a request id narrows the list to that request, with every other filter off", async () => { - window.history.replaceState(null, "", "/?page=logs&lgrp=error&lq=boom"); + window.history.replaceState(null, "", "/?page=activity&at=logs&lgrp=error&aq=boom"); logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); renderConsole(); @@ -121,11 +144,15 @@ describe("AdminLogs", () => { ); expect(screen.getByText("adminLogs.pivotRequest:req-00000001")).toBeTruthy(); const url = new URLSearchParams(window.location.search); - expect([url.get("lreq"), url.get("lgrp"), url.get("lq")]).toEqual(["req-00000001", null, null]); + expect([url.get("lreq"), url.get("lgrp"), url.get("aq")]).toEqual(["req-00000001", null, null]); }); it("Back from a request's lines returns to the view the pivot was made from", async () => { - window.history.replaceState({ sfPage: "logs" }, "", "/?page=logs&lgrp=error&lq=boom"); + window.history.replaceState( + { sfPage: "activity" }, + "", + "/?page=activity&at=logs&lgrp=error&aq=boom", + ); const depth = window.history.length; logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); renderConsole(); @@ -157,23 +184,46 @@ describe("AdminLogs", () => { ); expect(screen.queryByText("adminLogs.pivotRequest:req-00000001")).toBeNull(); const url = new URLSearchParams(window.location.search); - expect([url.get("lreq"), url.get("lgrp"), url.get("lq")]).toEqual([null, "error", "boom"]); + expect([url.get("lreq"), url.get("lgrp"), url.get("aq")]).toEqual([null, "error", "boom"]); // Neither jump waited out the typing debounce: no read ever mixed one view's filters with the // other's search text (the request with "boom", or the restored view without it). const mixed = logsMock.mock.calls.filter(([q]) => (q.request_id != null) === (q.q === "boom")); expect(mixed).toEqual([]); }); - it("keeps the account in use offered when the facets no longer count it", async () => { - window.history.replaceState(null, "", "/?page=logs&lu=7"); + it("a run id narrows the list to that run's lines", async () => { + logsMock.mockResolvedValue({ + items: [row(1, { run_id: 77 })], + accounts: {}, + next_cursor: null, + }); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + fireEvent.click(lines()[0]!); + fireEvent.click(screen.getByRole("button", { name: "77" })); + + await waitFor(() => + expect(logsMock).toHaveBeenLastCalledWith( + expect.objectContaining({ run_id: 77, request_id: undefined }), + null, + ), + ); + expect(screen.getByText("adminLogs.pivotRun:77")).toBeTruthy(); + expect(new URLSearchParams(window.location.search).get("lrun")).toBe("77"); + }); + + it("keeps the account in use offered when the account list does not have it", async () => { + window.history.replaceState(null, "", "/?page=activity&at=logs&au=7"); logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); renderConsole(); - await waitFor(() => expect(facetsMock).toHaveBeenCalled()); + await waitFor(() => + expect(logsMock).toHaveBeenCalledWith(expect.objectContaining({ user: "7" }), null), + ); - fireEvent.click(screen.getByTestId("logs-user")); + fireEvent.click(screen.getByTestId("activity-user")); - expect(await screen.findByTestId("logs-user-7")).toBeTruthy(); - expect(screen.getByTestId("logs-user-1")).toBeTruthy(); + expect(await screen.findByTestId("activity-user-7")).toBeTruthy(); + expect(screen.getByTestId("activity-user-1")).toBeTruthy(); }); it("Live watches the newest page only", async () => { @@ -258,6 +308,14 @@ describe("AdminLogs", () => { expect((await screen.findByRole("alert")).textContent).toContain("adminLogs.olderFailed"); }); + it("facets that failed say the filter menus may be short, in the other tabs' words", async () => { + logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); + facetsMock.mockRejectedValue(new Error("500")); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + expect((await screen.findByRole("alert")).textContent).toContain("activity.filtersFailed"); + }); + it("offers no older page while a filter change loads", async () => { let landErrors!: (page: LogPage) => void; logsMock.mockImplementation((q) => diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index 0dad5d23..763212ba 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -1,5 +1,4 @@ -import clsx from "clsx"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { keepPreviousData, @@ -7,74 +6,66 @@ import { useQuery, useQueryClient, } from "@tanstack/react-query"; -import { Copy, Radio, Search } from "lucide-react"; +import { Copy, Radio } from "lucide-react"; import { api, type LogFacet, type LogQuery, type ServerLogRow } from "../lib/api"; import { copyText } from "../lib/clipboard"; import { formatDateTimeSec } from "../lib/format"; -import { nextNavId, notifyNav, rememberNavPosition } from "../lib/history"; import { nextPageOnClick } from "../lib/nextPage"; import { LEVEL_GROUPS, type LevelGroup } from "../lib/log"; -import { - DEFAULT_LOGS_STATE, - logsParamsToState, - logsParamUpdates, - WINDOW_HOURS, - type LogsUrlState, - type LogWindow, -} from "../lib/logsUrlState"; +import { sinceOf } from "../lib/activityUrlState"; +import { logsParamsToState, logsParamUpdates, type LogsUrlState } from "../lib/logsUrlState"; import { notify } from "../lib/notifications"; import { qk } from "../lib/queryKeys"; -import { useDebounced } from "../lib/useDebounced"; -import { urlWithParams } from "../lib/urlState"; import { useUrlMirror } from "../lib/useUrlMirror"; +import { HistoryFrame } from "./HistoryList"; import LogLine from "./LogLine"; -import NavGrid from "./ui/NavGrid"; import { Button } from "./ui/Button"; import FilterChipGroup from "./ui/FilterChipGroup"; import FilterSelect from "./ui/FilterSelect"; import { FilterFoldButton } from "./ui/FilterFold"; import { useFilterFold } from "../lib/useFilterFold"; -import { useStickyTop } from "./ui/stickyTop"; -import { Section } from "./ui/form"; -import { LoadingState, StateMessage } from "./ui/QueryState"; import { DismissibleChip } from "./ui/Chip"; -import { InnerInput } from "./ui/Input"; -import { fieldClass } from "./ui/fieldStyles"; +import type { Shared } from "./ActivityRuns"; -// Admin → Szervernapló: the server's stored log (`log_events`). One list, newest first; a line -// opens in place to its full record, and its request id / job id narrow the list to that request or -// job. Filters live in the URL, so a filtered view is a link. "Élő" re-reads the newest page every +// Activity → Server log: the server's stored log (`log_events`). One list, newest first; a line +// opens in place to its full record, and its request / job / run id narrows the list to that +// request, job or run (the page does that: a pivot is a new history entry with every other filter +// off). The time range, account and text come from the page's shared bar; the rest live in the URL +// (./logsUrlState), so a filtered view is a link. "Élő" re-reads the newest page every // few seconds while the tab is visible — the newest page only: live is watching the top, so paging // into older rows turns it off. A failed read stops the polling (the request already reported it // once) and marks the list as possibly stale, instead of re-raising the failure every few seconds. const LIVE_MS = 4000; const GROUPS: readonly LevelGroup[] = ["all", "error", "warning", "info", "detail"]; -const WINDOWS: readonly LogWindow[] = ["1h", "24h", "7d"]; -// A pivot shows everything about one request or job: the other filters would only hide part of it. -const PIVOT_RESET: Omit = { - ...DEFAULT_LOGS_STATE, - window: "7d", -}; - -function sinceOf(window: LogWindow): string { - return new Date(Date.now() - WINDOW_HOURS[window] * 3_600_000).toISOString(); -} - -function toQuery(state: LogsUrlState, q: string): LogQuery { +function toQuery(state: LogsUrlState, shared: Shared): LogQuery { return { levels: state.group === "all" ? undefined : LEVEL_GROUPS[state.group], sources: state.sources, - user: state.user ?? undefined, + user: shared.user ?? undefined, event: state.event ?? undefined, request_id: state.requestId ?? undefined, job_id: state.jobId ?? undefined, - q: q.trim() || undefined, + run_id: state.runId ?? undefined, + q: shared.q.trim() || undefined, }; } -export default function AdminLogs() { +/** The ids a line can narrow the list to. */ +export type LogPivot = Pick; + +export default function AdminLogs({ + shared, + landed, + onPivot, +}: { + shared: Shared; + /** The filters of the entry Back/Forward landed on (the page reads the address bar, once). */ + landed: { logs: LogsUrlState } | null; + /** Show only one request's, job's or run's lines (the page clears every other filter). */ + onPivot: (pivot: LogPivot) => void; +}) { const { t, i18n } = useTranslation(); const lang = i18n.language; const qc = useQueryClient(); @@ -82,14 +73,9 @@ export default function AdminLogs() { logsParamsToState(new URLSearchParams(window.location.search)), ); const [live, setLiveState] = useState(false); - // The toolbar sticks at the scroller's top (./ui/stickyTop), and folds its facet row when short. - useStickyTop(); + // The facet row folds away when the screen is short. const fold = useFilterFold(); - const facetFilters = - state.sources.length + - (state.user ? 1 : 0) + - (state.event ? 1 : 0) + - (state.window !== DEFAULT_LOGS_STATE.window ? 1 : 0); + const facetFilters = state.sources.length + (state.event ? 1 : 0); // Read by `getNextPageParam` when a page is fetched, not when it renders: while live, a refetch // stops after the newest page, however many pages the cache holds — a "Load older" that lands // after Live was switched on, or a filter set revisited with older pages still cached. @@ -99,41 +85,37 @@ export default function AdminLogs() { setLiveState(on); }; const [open, setOpen] = useState(null); - // Bumped when the view changes by a jump (a pivot, Back/Forward) rather than by typing: the - // search box's text then applies at once instead of after the typing debounce. - const [jumps, setJumps] = useState(0); - const q = useDebounced(state.q, 300, jumps); useUrlMirror(logsParamUpdates(state)); - // Back/Forward lands on another view of the console (a pivot pushed one, below): read it back. - useEffect(() => { - const onPop = () => { + // Back/Forward lands on another entry's filters; a dialog closing lands on these same ones, and + // changes nothing (Live and the open line stay). Adopted during the render that brings it, with + // the page's half. + const [seen, setSeen] = useState(landed); + if (landed !== seen) { + setSeen(landed); + if (landed && JSON.stringify(landed.logs) !== JSON.stringify(state)) { setOpen(null); - setJumps((n) => n + 1); - setState(logsParamsToState(new URLSearchParams(window.location.search))); - }; - window.addEventListener("popstate", onPop); - return () => window.removeEventListener("popstate", onPop); - }, []); - + setState(landed.logs); + } + } const update = (patch: Partial) => { setOpen(null); setState((s) => ({ ...s, ...patch })); }; - const query = toQuery(state, q); - const key = qk.adminLogs({ ...query, window: state.window }); + const query = toQuery(state, shared); + const key = qk.adminLogs({ ...query, window: shared.window }); const logs = useInfiniteQuery({ queryKey: key, queryFn: ({ pageParam }) => - api.adminLogs({ ...query, since: sinceOf(state.window) }, pageParam), + api.adminLogs({ ...query, since: sinceOf(shared.window) }, pageParam), initialPageParam: null as string | null, getNextPageParam: (last) => (liveRef.current ? undefined : (last.next_cursor ?? undefined)), placeholderData: keepPreviousData, refetchInterval: (q) => (live && q.state.status !== "error" ? LIVE_MS : false), }); const facets = useQuery({ - queryKey: qk.adminLogFacets({ ...query, window: state.window }), - queryFn: () => api.adminLogFacets({ ...query, since: sinceOf(state.window) }), + queryKey: qk.adminLogFacets({ ...query, window: shared.window }), + queryFn: () => api.adminLogFacets({ ...query, since: sinceOf(shared.window) }), placeholderData: keepPreviousData, refetchInterval: (q) => (live && q.state.status !== "error" ? LIVE_MS * 4 : false), }); @@ -146,15 +128,16 @@ export default function AdminLogs() { ? null : (logs.data?.pages[logs.data.pages.length - 1]?.next_cursor ?? null); // A read that failed while lines were on screen: they are kept, and the message says what the - // failure means for them. (With no lines, the list's own error state says it.) + // failure means for them. (With no lines, the list's own error state says it.) Without the facets + // the Source and Event menus offer only "Any", as on the other tabs, so the same words say it. const failure = !logs.data ? null : logs.isFetchNextPageError - ? "olderFailed" + ? "adminLogs.olderFailed" : logs.isError || (facets.isError && facets.data) - ? "stale" + ? "adminLogs.stale" : facets.isError - ? "countsFailed" + ? "activity.filtersFailed" : null; const retryFailed = () => { if (logs.isFetchNextPageError) @@ -217,19 +200,9 @@ export default function AdminLogs() { return facets.data ? sum(levels.filter((f) => inGroup.includes(f.value))) : undefined; }; - // A pivot follows a link to another view, so it is its own history entry and Back returns to the - // view it was made from. The other filters are adjustments of the view on screen and replace it. - const pivot = (patch: Pick) => { + const pivot = (patch: Partial) => { setLive(false); - const next = { ...state, ...PIVOT_RESET, ...patch }; - window.history.pushState( - { ...rememberNavPosition(), sfScroll: nextNavId() }, - "", - urlWithParams(logsParamUpdates(next)), - ); - notifyNav(); - setJumps((n) => n + 1); - update(next); + onPivot({ requestId: null, jobId: null, runId: null, ...patch }); }; // "Any", then what the facet counted. The value in use stays offered even at zero — under the @@ -253,194 +226,130 @@ export default function AdminLogs() { }; return ( -
-
-

{t("adminLogs.intro")}

+ +
+ update({ group })} + options={GROUPS.map((g) => ({ + value: g, + label: t(`settings.myLog.filters.${g}`), + count: groupCount(g), + }))} + /> +
-
-
+ } + > + {rows.map((row, index) => ( + setOpen(open === row.id ? null : row.id)} + > + pivot({ requestId })} + onJob={(jobId) => pivot({ jobId })} + onRun={(runId) => pivot({ runId })} + /> + + ))} + ); } @@ -465,11 +374,13 @@ function RowDetail({ account, onRequest, onJob, + onRun, }: { row: ServerLogRow; account: string | null; onRequest: (id: string) => void; onJob: (id: number) => void; + onRun: (id: number) => void; }) { const { t, i18n } = useTranslation(); const lang = i18n.language; @@ -516,6 +427,21 @@ function RowDetail({ )} + {row.run_id !== null && ( + <> +
{t("adminLogs.fields.run")}
+
+ +
+ + )} {row.version && ( <>
{t("adminLogs.fields.version")}
diff --git a/frontend/src/components/AuditEarlierRuns.tsx b/frontend/src/components/AuditEarlierRuns.tsx deleted file mode 100644 index a5433ac6..00000000 --- a/frontend/src/components/AuditEarlierRuns.tsx +++ /dev/null @@ -1,48 +0,0 @@ -import { useState } from "react"; -import { useTranslation } from "react-i18next"; -import clsx from "clsx"; -import { ChevronRight } from "lucide-react"; -import type { AuditRow } from "../lib/api"; -import { Button } from "./ui/Button"; -import { RelativeTime } from "./ui/RelativeTime"; - -/** A folded run of one job's successful runs (lib/auditGroups): the row shows the newest, this - * opens the earlier ones, each with its moment and its result (`detailOf`, the row's own wording). */ -export default function AuditEarlierRuns({ - runs, - detailOf, -}: { - runs: AuditRow[]; - detailOf: (run: AuditRow) => string | null; -}) { - const { t } = useTranslation(); - const [open, setOpen] = useState(false); - const earlier = runs.slice(1); - return ( -
- - {open && ( -
    - {earlier.map((run) => ( -
  • - - {detailOf(run)} -
    • - ))} -
- )} -
- ); -} diff --git a/frontend/src/components/AuditLog.tsx b/frontend/src/components/AuditLog.tsx deleted file mode 100644 index 95d402df..00000000 --- a/frontend/src/components/AuditLog.tsx +++ /dev/null @@ -1,107 +0,0 @@ -import { useMemo } from "react"; -import { qk } from "../lib/queryKeys"; -import { useTranslation } from "react-i18next"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Trash2 } from "lucide-react"; -import { api } from "../lib/api"; -import { notify } from "../lib/notifications"; -import { LS } from "../lib/storage"; -import { Button } from "./ui/Button"; -import { Section } from "./ui/form"; -import { useConfirm } from "./confirmContext"; -import DataTable from "./ui/DataTable"; -import { LoadingState, StateMessage } from "./ui/QueryState"; -import { auditColumns } from "./auditColumns"; -import { notifyActionError } from "../lib/actionErrors"; -import { compareText } from "../lib/collate"; -import { foldJobRuns } from "../lib/auditGroups"; - -// Admin-only audit trail: an append-only who/what/when log of admin actions + scheduler run -// summaries (never per-row). Read-only apart from Clear. Client-side sort/filter/paginate via -// DataTable; the API returns the most recent window (capped) — the log stays small by design. -export default function AuditLog() { - const { t, i18n } = useTranslation(); - const lang = i18n.language; - const qc = useQueryClient(); - const confirm = useConfirm(); - const q = useQuery({ queryKey: qk.adminAudit(), queryFn: () => api.adminAudit() }); - const data = q.data; - // Back-to-back successful runs of one job fold into one row (lib/auditGroups). - const rows = useMemo(() => foldJobRuns(data?.items ?? []), [data]); - - const actionOptions = useMemo(() => { - // By what the reader sees — the named action, in their alphabet — not by the code. - return Array.from(new Set(rows.map((r) => r.action))) - .map((a) => ({ value: a, label: t(`auditActions.${a}`, a) })) - .sort((a, b) => compareText(a.label, b.label, lang)); - }, [rows, t, lang]); - const columns = useMemo(() => auditColumns(t, lang, actionOptions), [t, lang, actionOptions]); - - const clear = useMutation({ - mutationFn: () => api.clearAudit(), - onSuccess: (r) => { - qc.invalidateQueries({ queryKey: qk.adminAudit() }); - notify({ level: "success", message: t("audit.cleared", { count: r.cleared }) }); - }, - onError: (e) => notifyActionError(e, t("audit.clearFailed")), - }); - const onClear = async () => { - if ( - await confirm({ - title: t("audit.clearTitle"), - message: t("audit.clearConfirm"), - confirmLabel: t("audit.clear"), - danger: true, - }) - ) - clear.mutate(); - }; - - const truncated = !!data && data.total > data.returned; - - return ( -
-
-

{t("audit.intro")}

- {q.isError && !q.data ? ( - q.refetch()}> - {t("common.loadError")} - - ) : q.isLoading ? ( - - ) : rows.length === 0 ? ( -

{t("audit.empty")}

- ) : ( - <> - String(r.id)} - persistKey={LS.auditTable} - controlsPosition="top" - controlsInBand - controlsBandClassName="px-4 max-w-5xl mx-auto" - controlsLeading={ - - } - emptyText={t("audit.empty")} - /> - {truncated && ( -

- {t("audit.truncated", { returned: data!.returned, total: data!.total })} -

- )} - - )} -
-
- ); -} diff --git a/frontend/src/components/Channels.tsx b/frontend/src/components/Channels.tsx index 125709fe..fc1e6c85 100644 --- a/frontend/src/components/Channels.tsx +++ b/frontend/src/components/Channels.tsx @@ -515,7 +515,9 @@ export default function Channels() { id: "backfill", label: t("channels.backfillEverything"), title: t("channels.backfillEverythingHint"), - icon: , + icon: ( + + ), onSelect: () => deepAll.mutate(), disabled: deepAll.isPending, }, diff --git a/frontend/src/components/DownloadButton.tsx b/frontend/src/components/DownloadButton.tsx index e12c2f84..f68d5f38 100644 --- a/frontend/src/components/DownloadButton.tsx +++ b/frontend/src/components/DownloadButton.tsx @@ -54,7 +54,10 @@ export default function DownloadButton({ > {/* The state is in the glyph: downloaded or queued is the accent, a running one pulses. */} {open && ( diff --git a/frontend/src/components/DownloadTray.tsx b/frontend/src/components/DownloadTray.tsx index 9937cfca..bee55acc 100644 --- a/frontend/src/components/DownloadTray.tsx +++ b/frontend/src/components/DownloadTray.tsx @@ -98,7 +98,7 @@ function Progress({ job }: { job: QueueJob }) { style={{ width: `${job.progress}%` }} /> ) : ( - + )} {bytes && {job.progress}%} diff --git a/frontend/src/components/HistoryList.tsx b/frontend/src/components/HistoryList.tsx new file mode 100644 index 00000000..a0e66b49 --- /dev/null +++ b/frontend/src/components/HistoryList.tsx @@ -0,0 +1,144 @@ +import type { ReactNode } from "react"; +import { useTranslation } from "react-i18next"; + +import { nextPageOnClick } from "../lib/nextPage"; +import type { HistoryList as HistoryListState } from "../lib/useHistoryList"; +import NavGrid from "./ui/NavGrid"; +import { Button } from "./ui/Button"; +import { LoadingState, StateMessage } from "./ui/QueryState"; + +// The frame of a history list on the activity page — the runs, the admin actions, the server log: +// its own bar (if any), the rows or the honest state instead of them (a failed read is an error with a +// retry, never an empty list), a read that failed over rows still on screen, the count, and "Older". +// `HistoryFrame` draws it; each list says only what its failure and its "Older" are — the server log's +// count its filter facets and stop while Live (AdminLogs), the others not (HistoryList, below). + +/** What a read that failed while rows stayed on screen means, and how to try it again. */ +export interface FrameFailure { + text: string; + onRetry: () => void; +} + +export function HistoryFrame({ + list, + rowCount, + header, + failure, + older, + count, + empty, + rowsClassName = "", + testId, + children, +}: { + /** The list's read. */ + list: { + data?: unknown; + isError: boolean; + isLoading: boolean; + isPlaceholderData: boolean; + refetch: () => unknown; + }; + rowCount: number; + /** The list's own bar above the rows. */ + header?: ReactNode; + failure: FrameFailure | null; + /** "Older", when there is an older page to ask for. */ + older: { label: string; onClick: () => void; disabled: boolean } | null; + /** "N runs shown", for the footer. */ + count: string; + /** Shown when the filters match nothing. */ + empty: string; + rowsClassName?: string; + testId?: string; + /** The rows. */ + children: ReactNode; +}) { + const { t } = useTranslation(); + return ( +
+ {header} + {failure && ( + + {failure.text} + + )} +
+ {list.isError && !list.data ? ( + void list.refetch()}> + {t("common.loadError")} + + ) : list.isLoading ? ( + + ) : rowCount === 0 ? ( + + {empty} + + ) : ( + {children} + )} +
+
+ {count} + {older && ( + + )} +
+
+ ); +} + +/** A history list read through `useHistoryList` (the runs, the admin actions). */ +export default function HistoryList({ + state, + empty, + count, + children, +}: { + state: HistoryListState; + /** Shown when the filters match nothing. */ + empty: string; + /** "N runs shown", for the footer. */ + count: string; + /** The rows, drawn from `state.rows`. */ + children: ReactNode; +}) { + const { t } = useTranslation(); + const { list, rows, hasOlder } = state; + const fetchOlder = () => void list.fetchNextPage(nextPageOnClick(list.isFetchingNextPage)); + // A read that failed while rows were on screen keeps them, and says what the failure means. + const failure: FrameFailure | null = !list.data + ? null + : list.isFetchNextPageError + ? { text: t("activity.olderFailed"), onRetry: fetchOlder } + : list.isError + ? { text: t("activity.stale"), onRetry: () => void list.refetch() } + : null; + return ( + + {children} + + ); +} diff --git a/frontend/src/components/HistoryRow.tsx b/frontend/src/components/HistoryRow.tsx new file mode 100644 index 00000000..bf19aa8b --- /dev/null +++ b/frontend/src/components/HistoryRow.tsx @@ -0,0 +1,63 @@ +import type { ReactNode } from "react"; + +import { navItemAttrs } from "../lib/listNav"; +import { Pressable } from "./ui/Pressable"; + +// One dense row of a history list — a background run, an admin action, a log line (./LogLine draws +// its lines through this): its cells on one line, and the row opens in place to its details. One frame +// for all of them, so a row opens, announces itself and takes the keyboard the same on every tab. +// Inside a NavGrid the keyboard walks the rows (`j`/`k`) and Enter opens one. + +export default function HistoryRow({ + cells, + grid, + navIndex, + navText, + open, + onToggle, + title, + line = false, + children, +}: { + /** The row's cells, laid out by `grid`. */ + cells: ReactNode; + /** The grid's column template classes (phone first, then wider screens). */ + grid: string; + navIndex: number; + /** What the keyboard's type-ahead matches. */ + navText: string; + open: boolean; + onToggle: () => void; + /** The row button's tooltip — the full date. */ + title?: string; + /** A log line: tighter, its cells on one text baseline, its size the (monospace) list's own. */ + line?: boolean; + /** The detail shown while the row is open. */ + children: ReactNode; +}) { + return ( +
+ + {cells} + + {open && ( +
+ {children} +
+ )} +
+ ); +} diff --git a/frontend/src/components/IssueComposer.dom.test.tsx b/frontend/src/components/IssueComposer.dom.test.tsx index 32437c55..02d9fbfd 100644 --- a/frontend/src/components/IssueComposer.dom.test.tsx +++ b/frontend/src/components/IssueComposer.dom.test.tsx @@ -112,6 +112,7 @@ const row = (over: Partial): ServerLogRow => ({ request_id: null, user_id: 1, job_id: null, + run_id: null, debug_session_id: SID, version: null, ...over, diff --git a/frontend/src/components/LogLine.tsx b/frontend/src/components/LogLine.tsx index fca3dded..33b3c9ba 100644 --- a/frontend/src/components/LogLine.tsx +++ b/frontend/src/components/LogLine.tsx @@ -1,14 +1,14 @@ import type { ReactNode } from "react"; import type { Level } from "../lib/log"; -import { navItemAttrs } from "../lib/listNav"; +import HistoryRow from "./HistoryRow"; import { LEVEL_TAG, logClock } from "./logLineParts"; -import { Pressable } from "./ui/Pressable"; // One log line as both log viewers draw it — "Saját napló" (this tab's ring) and the admin // diagnostics console (the server's stored log): time, level, event code, message, and the line opens // in place to whatever detail its screen has. One component, so a level reads the same on both. -// Inside a NavGrid the keyboard walks the lines (`j`/`k`) and Enter opens one, as a click does. +// The row frame is the history lists' (./HistoryRow): inside a NavGrid the keyboard walks the lines +// (`j`/`k`) and Enter opens one, as a click does. export default function LogLine({ ts, @@ -36,29 +36,26 @@ export default function LogLine({ }) { const tag = LEVEL_TAG[level]; return ( -
+ {logClock(ts)} + {tag.text} + {event} + {/* A phone has no room for a fourth column: the message takes a line of its own under the + code, full width — sharing the code's line left it a few characters. */} + {msg} + + } > - - {logClock(ts)} - {tag.text} - {event} - {/* A phone has no room for a fourth column: the message takes a line of its own under the - code, full width — sharing the code's line left it a few characters. */} - {msg} - - {open &&
{children}
} -
+ {children} + ); } diff --git a/frontend/src/components/NavigationProvider.dom.test.tsx b/frontend/src/components/NavigationProvider.dom.test.tsx index 5d020e2f..630c2a61 100644 --- a/frontend/src/components/NavigationProvider.dom.test.tsx +++ b/frontend/src/components/NavigationProvider.dom.test.tsx @@ -31,6 +31,8 @@ import { useBackToClose } from "../lib/history"; import { PageHeading } from "./ui/Headings"; import { qk } from "../lib/queryKeys"; +import { LS, setAccountRaw } from "../lib/storage"; +import { PAGE_KEY, START_ON_HOME } from "./navigationInit"; /** The Navigation API's view of the tab: `below` = whether each entry under the current one belongs * to this document, nearest first. jsdom has no Navigation API; Chrome's says this. */ @@ -151,11 +153,12 @@ describe("NavigationProvider · setPage's same-page guard", () => { expect(result.current.page).toBe("plex"); }); - it("leaves every page's own URL keys behind, the log console's included", () => { + it("leaves every page's own URL keys behind, the activity page's included", () => { window.history.replaceState( - { sfPage: "logs" }, + { sfPage: "activity" }, "", - "/?page=logs&lgrp=error&lreq=req-abcdef12&pscope=movie&ctab=hidden&keep=1", + "/?page=activity&at=logs&aw=24h&au=none&aq=x&rjob=backfill&rst=error&aact=config_set" + + "&lgrp=error&lreq=req-abcdef12&lrun=7&pscope=movie&ctab=hidden&keep=1", ); const { result } = render(); act(() => result.current.setPage("plex")); @@ -218,6 +221,66 @@ describe("NavigationProvider · Back", () => { expect(window.location.search).toBe("?page=files&keep=1"); }); + // The audit log and the server log became the activity page's tabs: an old link, bookmark or entry + // opens that tab, with the filters it carried. + it("opens a retired page's tab on the page that replaced it", () => { + window.history.replaceState(null, "", "/?page=audit&keep=1"); + const { result } = render(); + expect(result.current.page).toBe("activity"); + expect(window.location.search).toBe("?page=activity&keep=1&at=admin"); + }); + + it("keeps a retired page's own filter as it opens its tab", () => { + window.history.replaceState(null, "", "/?page=logs&lreq=req-00000009"); + render(); + const url = new URLSearchParams(window.location.search); + expect([url.get("page"), url.get("at"), url.get("lreq")]).toEqual([ + "activity", + "logs", + "req-00000009", + ]); + }); + + it("carries an old server-log link's account, text and range onto the page's own filters", () => { + window.history.replaceState(null, "", "/?page=logs&lu=3&lq=boom&lwin=1h"); + render(); + const url = new URLSearchParams(window.location.search); + expect([url.get("at"), url.get("au"), url.get("aq"), url.get("aw")]).toEqual([ + "logs", + "3", + "boom", + "1h", + ]); + // …and the dead keys leave the address bar instead of riding along to every page after. + expect(["lu", "lq", "lwin"].filter((k) => url.has(k))).toEqual([]); + }); + + it("an old server-log link without a range keeps the one it meant", () => { + window.history.replaceState(null, "", "/?page=logs"); + render(); + expect(new URLSearchParams(window.location.search).get("aw")).toBe("24h"); + }); + + it("a remembered retired page opens its tab", () => { + setAccountRaw(LS.startOnHome, START_ON_HOME.off); + setAccountRaw(PAGE_KEY, "logs"); + window.history.replaceState(null, "", "/"); + const { result } = render(); + expect(result.current.page).toBe("activity"); + expect(new URLSearchParams(window.location.search).get("at")).toBe("logs"); + }); + + it("opens a retired page's tab when Back lands on its entry", () => { + const { result } = render(); + act(() => result.current.setPage("plex")); + act(() => { + window.history.replaceState({ sfPage: "audit" }, "", "/?page=audit"); + window.dispatchEvent(new PopStateEvent("popstate", { state: { sfPage: "audit" } })); + }); + expect(result.current.page).toBe("activity"); + expect(window.location.search).toBe("?page=activity&at=admin"); + }); + // lib/history's own listener takes a spent focus stamp off the entry it lands on; the rewrite must // not put it back, or the next landing there (an overlay closing) would refocus a stale item. it("does not put back the focus stamp the landing spent", () => { diff --git a/frontend/src/components/NavigationProvider.tsx b/frontend/src/components/NavigationProvider.tsx index a8d18c0f..8e61101f 100644 --- a/frontend/src/components/NavigationProvider.tsx +++ b/frontend/src/components/NavigationProvider.tsx @@ -2,11 +2,11 @@ import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react import { useLatest } from "../lib/useLatest"; import { useTranslation } from "react-i18next"; import { useMe } from "../lib/useMe"; -import { resolvePage, urlWithParams, type Page } from "../lib/urlState"; -import { LOGS_KEYS } from "../lib/logsUrlState"; +import { resolvePage, retiredParams, urlWithParams, type Page } from "../lib/urlState"; +import { ACTIVITY_KEYS } from "../lib/activityUrlState"; import { pageTitleKey } from "../lib/pageMeta"; import { useDocumentObjectTitle } from "../lib/documentTitle"; -import { setAccountRaw } from "../lib/storage"; +import { getAccountRaw, setAccountRaw } from "../lib/storage"; import { backStaysInDocument, closeOverlaysThenNavigate, @@ -92,7 +92,7 @@ export function NavigationProvider({ children }: { children: ReactNode }) { ctab: null, csort: null, cf: null, - ...Object.fromEntries(LOGS_KEYS.map((k) => [k, null])), + ...Object.fromEntries(ACTIVITY_KEYS.map((k) => [k, null])), }), ); notifyNav(); @@ -193,7 +193,14 @@ export function NavigationProvider({ children }: { children: ReactNode }) { // Mirror the resolved initial page to the address bar (feed implicit) so a page reached from the // stored landing pref — not just from the URL — is itself bookmarkable. urlWithParams preserves any // ?chan / ?v / ?pl / filter params the loaders already read. - const url = urlWithParams({ page: mountPage === "feed" ? null : mountPage }); + // A retired page's link — or the remembered last page, when that is what opened — also opens its + // part of the page that replaced it (an `at` tab). + const linked = new URLSearchParams(window.location.search).get("page"); + const remembered = getAccountRaw(PAGE_KEY); + const url = urlWithParams({ + ...retiredParams(linked ?? (resolvePage(remembered) === mountPage ? remembered : null)), + page: mountPage === "feed" ? null : mountPage, + }); if (st._yt && st._ytScrape) { window.history.replaceState({ ...st, sfPage: mountPage }, "", url); } else { @@ -213,7 +220,7 @@ export function NavigationProvider({ children }: { children: ReactNode }) { window.history.replaceState( { ...window.history.state, sfPage: p }, "", - urlWithParams({ page: p === "feed" ? null : p }), + urlWithParams({ ...retiredParams(sf), page: p === "feed" ? null : p }), ); setPageState(p); setAccountRaw(PAGE_KEY, p); diff --git a/frontend/src/components/RunStatusDot.tsx b/frontend/src/components/RunStatusDot.tsx new file mode 100644 index 00000000..a1657c38 --- /dev/null +++ b/frontend/src/components/RunStatusDot.tsx @@ -0,0 +1,29 @@ +// A background run's status as a dot — the Scheduler's rows and the activity page's run list draw +// the same one, so a state reads the same on both. + +// Interrupted is a ring, not a fill: the job did not fail, but its work did not finish either — set +// apart from a skipped run (the same colour, filled) and from a failure. +const DOT_CLASS = { + running: "bg-accent motion-safe:animate-pulse", + ok: "bg-success", + error: "bg-danger", + skipped: "bg-warn", + interrupted: "border-2 border-warn", + idle: "bg-border", +} as const; + +export type DotKey = keyof typeof DOT_CLASS; + +/** A status dot. `label` names it for a screen reader (the tooltip shows only on hover, and only with + * hints on); a legend's dot has its words beside it and stays unnamed. */ +export function StatusDot({ k, label }: { k: DotKey; label?: string }) { + return ( + + ); +} diff --git a/frontend/src/components/Scheduler.dom.test.tsx b/frontend/src/components/Scheduler.dom.test.tsx index 88301ba4..6f875298 100644 --- a/frontend/src/components/Scheduler.dom.test.tsx +++ b/frontend/src/components/Scheduler.dom.test.tsx @@ -6,7 +6,7 @@ // because this page polls: every few seconds a fresh status arrives, and a mirror is exactly the // thing a poll can overwrite at the wrong moment. import { describe, it, expect, vi, afterEach, beforeEach } from "vitest"; -import { render, screen, fireEvent, cleanup, act, waitFor } from "@testing-library/react"; +import { render, screen, fireEvent, cleanup, act, waitFor, within } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; vi.mock("react-i18next", async (importOriginal) => { @@ -15,7 +15,8 @@ vi.mock("react-i18next", async (importOriginal) => { }); import Scheduler from "./Scheduler"; -import { api, type SchedulerStatus } from "../lib/api"; +import { NavigationActionsContext } from "./navigationContext"; +import { api, type RunBrief, type SchedulerAdHoc, type SchedulerStatus } from "../lib/api"; import { qk } from "../lib/queryKeys"; const status = (over: Partial = {}): SchedulerStatus => ({ @@ -34,9 +35,14 @@ const status = (over: Partial = {}): SchedulerStatus => ({ last_result: null, result: null, last_error: null, + last_trigger: null, + last_run_id: null, + quota_units: null, + recent: [], progress: null, }, ], + ad_hoc: [], queue: { channels_recent_pending: 0, channels_deep_pending: 0, @@ -281,3 +287,114 @@ describe("a job's last result", () => { expect(screen.queryByText(/new=3/)).toBeNull(); }); }); + +describe("a row's run history", () => { + const at = (minutesAgo: number) => new Date(Date.now() - minutesAgo * 60_000).toISOString(); + const run = (id: number, status: RunBrief["status"], minutesAgo: number): RunBrief => ({ + id, + status, + trigger: "scheduled", + started_at: at(minutesAgo), + finished_at: status === "running" ? null : at(minutesAgo), + }); + const dots = (root: HTMLElement = document.body) => + within(root) + .getAllByRole("img") + .map((d) => d.getAttribute("data-dot")); + + // An interrupted run fell through to "idle" — a grey dot that said no run was ever recorded. + it("shows the last runs newest first, an interrupted one as such", async () => { + const base = status(); + await mount({ + ...base, + jobs: [ + { + ...base.jobs[0]!, + status: "interrupted", + last_finished: at(2), + recent: [run(3, "interrupted", 2), run(2, "error", 12), run(1, "ok", 22)], + }, + ], + }); + expect(dots()).toEqual(["interrupted", "error", "ok"]); + expect(screen.getByText(/scheduler\.interrupted/)).toBeTruthy(); + expect(screen.getByText("scheduler.dot.interrupted")).toBeTruthy(); // the legend names it + }); + + it("shows one idle dot for a job that never ran", async () => { + await mount(); + expect(dots()).toEqual(["idle"]); + }); + + it("gives a run the job has claimed, but not yet recorded, the newest dot", async () => { + const base = status(); + await mount({ + ...base, + jobs: [ + { + ...base.jobs[0]!, + running: true, + recent: [run(3, "ok", 10), run(2, "ok", 20), run(1, "error", 30)], + }, + ], + }); + expect(dots()).toEqual(["running", "ok", "ok"]); + }); + + // The one-channel full-history backfill ran unseen: a background task only the log heard of. + it("lists a user's ad-hoc run beside the jobs, with its channel and no run button", async () => { + const base = status(); + const adHoc: SchedulerAdHoc = { + ...base.jobs[0]!, + id: "channel_backfill", + running: false, + target: "UC_bbc", + target_title: "BBC Earth", + last_finished: at(60), + result: { videos_added: 69 }, + recent: [run(9, "ok", 60), run(8, "ok", 120)], + }; + await mount({ ...base, ad_hoc: [adHoc] }); + expect(screen.getByText("scheduler.adHocTitle")).toBeTruthy(); + const row = screen.getByText("scheduler.jobs.channel_backfill").closest(".p-3")!; + expect(dots(row)).toEqual(["ok", "ok"]); + expect(within(row).getByText(/BBC Earth/)).toBeTruthy(); + expect(within(row).getByText(/scheduler\.result\.videos_added: 69/)).toBeTruthy(); + expect(within(row).getByText("scheduler.adHocStarted")).toBeTruthy(); + expect(within(row).queryByRole("button", { name: "scheduler.runNow" })).toBeNull(); + }); +}); + +describe("a row's history", () => { + // The dots show three runs; the rest is the activity page's run list, narrowed to the row's job. + it("opens the activity page's runs of that job", async () => { + const setPage = vi.fn((_page: string, then?: () => void) => then?.()); + window.history.replaceState(null, "", "/?page=scheduler"); + current = status(); + qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + {}, + closeChannel: () => {}, + enterYtSearch: () => {}, + exitYtSearch: () => {}, + }} + > + + + + , + ); + await screen.findByText("scheduler.maintenance.batchLabel"); + const job = current.jobs[0]!.id; + const row = screen.getByText(`scheduler.jobs.${job}`).closest(".p-3")!; + + fireEvent.click(within(row).getByRole("button", { name: "scheduler.history" })); + + expect(setPage).toHaveBeenCalledWith("activity", expect.any(Function)); + const url = new URLSearchParams(window.location.search); + expect([url.get("page"), url.get("rjob")]).toEqual(["activity", job]); + }); +}); diff --git a/frontend/src/components/Scheduler.menu.dom.test.tsx b/frontend/src/components/Scheduler.menu.dom.test.tsx index c9322a1f..fbf38e8f 100644 --- a/frontend/src/components/Scheduler.menu.dom.test.tsx +++ b/frontend/src/components/Scheduler.menu.dom.test.tsx @@ -32,9 +32,14 @@ const status = (over: Partial = {}): SchedulerStatus => ({ last_result: null, result: null, last_error: null, + last_trigger: null, + last_run_id: null, + quota_units: null, + recent: [], progress: null, }, ], + ad_hoc: [], queue: { channels_recent_pending: 0, channels_deep_pending: 0, diff --git a/frontend/src/components/Scheduler.tsx b/frontend/src/components/Scheduler.tsx index 29075f6b..e4d501cf 100644 --- a/frontend/src/components/Scheduler.tsx +++ b/frontend/src/components/Scheduler.tsx @@ -10,6 +10,7 @@ import { Clock, Database, Gauge, + History, Pause, Pencil, Play, @@ -17,7 +18,14 @@ import { Trash2, X, } from "lucide-react"; -import { api, type SchedulerJob, type SchedulerStatus } from "../lib/api"; +import { + api, + type RunBrief, + type RunFields, + type SchedulerAdHoc, + type SchedulerJob, + type SchedulerStatus, +} from "../lib/api"; import { useLiveQuery } from "../lib/useLiveQuery"; import { Button } from "./ui/Button"; import { LoadingState, StateMessage } from "./ui/QueryState"; @@ -36,6 +44,9 @@ import { ContextMenu } from "./ui/ContextMenu"; import { MoreButton } from "./ui/MoreButton"; import { type ActionMenuItem } from "./ui/ActionMenu"; import { jobFailureText, jobResultText } from "../lib/jobResult"; +import { StatusDot, type DotKey } from "./RunStatusDot"; +import { useNavigationActions } from "./navigationContext"; +import { urlWithParams } from "../lib/urlState"; const POLL_MS = 4000; // While any job is running, poll faster so short scheduled runs aren't missed between ticks. @@ -57,41 +68,13 @@ function fmtCountdown(t: TFunction, s: number, lang: string): string { return `${formatUnit(h, "hour", lang)} ${formatUnit(m % 60, "minute", lang)}`; } -const DOT_CLASS: Record = { - running: "bg-accent animate-pulse", - ok: "bg-success", - error: "bg-danger", - skipped: "bg-warn", - idle: "bg-border", -}; - -function statusKey(job: SchedulerJob): keyof typeof DOT_CLASS { - if (job.running) return "running"; - if (job.status === "ok") return "ok"; - if (job.status === "error") return "error"; - if (job.status === "skipped") return "skipped"; - return "idle"; -} - -function StatusDot({ - k, - withTooltip = true, -}: { - k: keyof typeof DOT_CLASS; - withTooltip?: boolean; -}) { - const { t } = useTranslation(); - const dot = ; - return withTooltip ? {dot} : dot; -} - function StatusLegend() { const { t } = useTranslation(); return (
- {(["running", "ok", "idle", "error", "skipped"] as const).map((k) => ( + {(["running", "ok", "idle", "error", "skipped", "interrupted"] as const).map((k) => ( - + {t(`scheduler.dot.${k}`)} ))} @@ -99,6 +82,132 @@ function StatusLegend() { ); } +// The server sends a row's newest runs, at most this many (RECENT_RUNS in app/scheduler.py). +const MAX_DOTS = 3; + +/** A row's last runs, newest first (leftmost), each saying in its tooltip how it went, when, and who + * started it. A row with no run yet shows one idle dot. While a run goes on the newest dot pulses: + * the job's own claim says so a moment before its run row exists, so that moment gets a dot too. */ +function RunDots({ runs, running }: { runs: RunBrief[]; running: boolean }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const claimed = running && runs[0]?.status !== "running"; + const shown: (RunBrief | null)[] = claimed ? [null, ...runs].slice(0, MAX_DOTS) : runs; + const dots = shown.length === 0 ? [undefined] : shown; + return ( + + {dots.map((run) => { + const hint = + run === undefined + ? t("scheduler.dot.idle") + : run === null + ? t("scheduler.dot.running") + : t("scheduler.dotRun", { + state: t(`scheduler.dot.${run.status}`), + time: relativeTime(run.started_at, lang, t), + trigger: t(`scheduler.trigger.${run.trigger}`), + }); + const k: DotKey = run === undefined ? "idle" : run === null ? "running" : run.status; + return ( + + + + ); + })} + + ); +} + +/** "Last run 2 min ago · ", the line under a row's name: how the newest ended run went + * — its failure, its result, or that the process stopped under it. */ +function LastRun({ row, running, target }: { row: RunFields; running: boolean; target?: string }) { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + return ( +
+ {running + ? t("scheduler.runningNow") + : row.last_finished + ? t("scheduler.lastRun", { time: relativeTime(row.last_finished, lang, t) }) + : t("scheduler.neverRun")} + {target && · {target}} + {row.last_error ? ( + + {" · "} + {jobFailureText(t, row.result, lang) ?? row.last_error} + + ) : row.status === "interrupted" ? ( + · {t("scheduler.interrupted")} + ) : row.result ? ( + · {jobResultText(t, row.result, lang)} + ) : row.last_result ? ( + · {row.last_result} + ) : null} +
+ ); +} + +/** A row's way to its whole history: the activity page's run list, narrowed to this job. */ +function HistoryButton({ job }: { job: string }) { + const { t } = useTranslation(); + const { setPage } = useNavigationActions(); + return ( + + + setPage("activity", () => + window.history.replaceState( + window.history.state, + "", + urlWithParams({ page: "activity", rjob: job }), + ), + ) + } + > + + + + ); +} + +/** A user's run on one target (a channel's full history): its runs as a job's are shown, but it has + * no interval and no "run now" — it needs its target, so it is started where that is chosen. */ +function AdHocRow({ row }: { row: SchedulerAdHoc }) { + const { t } = useTranslation(); + return ( +
+ +
+
+ + + {t(`scheduler.jobs.${row.id}`, row.id)} + + +
+ +
+
+ {row.running ? ( + {t("scheduler.active")} + ) : ( + t("scheduler.adHocStarted") + )} +
+ +
+ ); +} + // Shown for ANY running job. With reported counts it's a determinate bar; for a job that // runs but doesn't report progress (or hasn't yet) it falls back to a generic "working" // label and an indeterminate sliver — so every active run is visible, not just the ones @@ -129,7 +238,7 @@ function JobProgress({ p }: { p: SchedulerJob["progress"] }) { /> ) : ( // Indeterminate: total unknown, so a slim moving sliver instead of a fill. -
+
)}
@@ -186,7 +295,7 @@ function JobRow({ return (
- +
@@ -247,26 +356,7 @@ function JobRow({ )}
-
- {job.running - ? t("scheduler.runningNow") - : job.last_finished - ? t("scheduler.lastRun", { time: relativeTime(job.last_finished, lang, t) }) - : t("scheduler.neverRun")} - {job.last_error ? ( - - {" · "} - {jobFailureText(t, job.result, lang) ?? job.last_error} - - ) : job.result ? ( - · {jobResultText(t, job.result, lang)} - ) : job.last_result ? ( - · {job.last_result} - ) : null} -
+ {job.running && }
@@ -295,6 +385,7 @@ function JobRow({ +
); } @@ -646,6 +737,18 @@ export default function Scheduler() {
+ {/* Ad-hoc runs: a user's work on one target, shown beside the jobs because it spends the same quota */} + {data.ad_hoc.length > 0 && ( +
+ {t("scheduler.adHocTitle")} +
+ {data.ad_hoc.map((row) => ( + + ))} +
+
+ )} + {/* Queue */}
{t("scheduler.queueTitle")} diff --git a/frontend/src/components/SetupWizard.tsx b/frontend/src/components/SetupWizard.tsx index 6989b7f2..03f01ba8 100644 --- a/frontend/src/components/SetupWizard.tsx +++ b/frontend/src/components/SetupWizard.tsx @@ -259,7 +259,9 @@ export default function SetupWizard() { onClick={sendTest} disabled={testState === "sending" || !testTo.trim()} > - + {t("setup.smtp.test")}
diff --git a/frontend/src/components/Stats.tsx b/frontend/src/components/Stats.tsx index 3c895cd2..79dd7daf 100644 --- a/frontend/src/components/Stats.tsx +++ b/frontend/src/components/Stats.tsx @@ -1,67 +1,27 @@ -import { useState } from "react"; import { useTranslation } from "react-i18next"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { qk } from "../lib/queryKeys"; -import { History, Pause, Play, RefreshCw } from "lucide-react"; -import { api, type AdminQuotaRow } from "../lib/api"; -import { formatEta, formatNumber, formatUnit, quotaActionLabel } from "../lib/format"; +import { History, RefreshCw } from "lucide-react"; +import { api } from "../lib/api"; +import { formatEta, formatNumber, quotaActionLabel } from "../lib/format"; import { notify } from "../lib/notifications"; import { notifyYouTubeActionError } from "../lib/youtubeErrors"; import { useWizard } from "./wizardContext"; -import { LS, useTabPersistedState } from "../lib/storage"; import Tooltip from "./ui/Tooltip"; import { LoadingState, StateMessage } from "./ui/QueryState"; -import FilterChipGroup from "./ui/FilterChipGroup"; -import Tabs from "./ui/Tabs"; -import { useTabPanel } from "./ui/tabPanel"; -import { PageToolbar } from "./ui/PageShell"; import { Section } from "./ui/form"; import { SectionLabel } from "./ui/Headings"; import { StatBars, StatTile, StatTiles } from "./ui/StatTiles"; -import { useMe } from "../lib/useMe"; import { notifyActionError } from "../lib/actionErrors"; import { Button } from "./ui/Button"; -import { Pressable } from "./ui/Pressable"; -import { SURFACE } from "./ui/surfaces"; -const RANGES = [7, 30, 90] as const; -type StatsTab = "overview" | "system"; - -// Usage & stats page. "Overview" is per-user (sync status + your own API usage + actions); -// "System" is the admin-only instance-wide quota dashboard + background-sync control. The -// per-user content used to live in Settings → Sync; it moved here so Settings stays config-only. +// Usage & stats page: your sync status, your own API usage and the manual sync actions (they used to +// live in Settings → Sync; they moved here so Settings stays config-only). The instance-wide quota +// view is the activity page's Quota tab. export default function Stats() { - const me = useMe(); - const { t } = useTranslation(); - const isAdmin = me.role === "admin"; - const [tabRaw, setTab] = useTabPersistedState(LS.statsTab, "overview"); - // Clamp at render: only admins may land on the System tab (covers a stale stored value). - const tab: StatsTab = tabRaw === "system" && isAdmin ? "system" : "overview"; - const panel = useTabPanel(); - return ( - // A tab panel only where there are tabs: everyone but an admin sees the Overview alone. -
- {/* Only admins have a second (System) tab; everyone else just sees the Overview. The tabs are - the page's band (B4), so they stay in reach on the long System tab. */} - {isAdmin && ( - -
- ({ - id, - label: t(`stats.tabs.${id}`), - }))} - className="flex flex-wrap gap-1 pb-2" - /> -
- - )} - - {tab === "system" && isAdmin ? : } +
+
); } @@ -230,165 +190,3 @@ function Overview() { ); } - -// Admin-only view: instance-wide quota dashboard + the global background-sync pause toggle. -function SystemStats() { - const { t, i18n } = useTranslation(); - const lang = i18n.language; - const qc = useQueryClient(); - const [days, setDays] = useState(30); - const q = useQuery({ queryKey: qk.adminQuota(days), queryFn: () => api.adminQuota(days) }); - const status = useQuery({ queryKey: qk.myStatus(), queryFn: api.myStatus }); - const pauseResume = useMutation({ - mutationFn: (paused: boolean) => (paused ? api.resumeSync() : api.pauseSync()), - onSuccess: () => qc.invalidateQueries({ queryKey: qk.myStatus() }), - onError: (e, paused) => - notifyActionError( - e, - t(paused ? "errors.writeFailed.syncResume" : "errors.writeFailed.syncPause"), - ), - }); - const data = q.data; - const maxDaily = Math.max(1, ...(data?.daily ?? []).map((d) => d.total)); - const grandTotal = (data?.rows ?? []).reduce((s, r) => s + r.total, 0); - const s = status.data; - - return ( - <> - {s && ( -
- - - -
- )} - -
-

{t("stats.title")}

- ({ - value: String(d), - label: t("stats.rangeDays", { d: formatUnit(d, "day", lang) }), - }))} - value={String(days)} - onChange={(v) => setDays(Number(v))} - /> -
- -

- {t("stats.introBefore")} - {t("stats.introSystem")} - {t("stats.introAfter")} -

- - {q.isError && !data ? ( - q.refetch()}> - {t("common.loadError")} - - ) : q.isLoading ? ( - - ) : !data ? ( -
{t("stats.noData")}
- ) : ( - <> - {/* Daily totals (instance-wide, Pacific days) */} -
-
- {t("stats.dailyTotal", { - d: formatUnit(data.range_days, "day", lang), - units: formatNumber(grandTotal, lang), - })} -
- {data.daily.length === 0 ? ( -
{t("stats.noUsageInRange")}
- ) : ( -
- {data.daily.map((d) => ( -
-
-
- ))} -
- )} -
- - {/* Per-user breakdown */} -
- {data.rows.length === 0 ? ( -
{t("stats.noUsageInRange")}
- ) : ( - data.rows.map((r) => ( - - )) - )} -
- - )} - - ); -} - -function UserRow({ row, max }: { row: AdminQuotaRow; max: number }) { - const { t, i18n } = useTranslation(); - const lang = i18n.language; - const [open, setOpen] = useState(false); - const actions = Object.entries(row.by_action).sort((a, b) => b[1] - a[1]); - const isSystem = row.user_id === null; - return ( -
- setOpen((o) => !o)} - aria-expanded={open} - className="w-full flex items-center gap-3" - > - - {isSystem ? t("stats.system") : row.email} - - - {formatNumber(row.total, lang)} - - - {/* Proportion bar */} -
-
-
- {open && actions.length > 0 && ( -
- {actions.map(([action, units]) => ( -
- {quotaActionLabel(t, action)} - {formatNumber(units, lang)} -
- ))} -
- )} -
- ); -} diff --git a/frontend/src/components/auditColumns.dom.test.tsx b/frontend/src/components/auditColumns.dom.test.tsx deleted file mode 100644 index 43001585..00000000 --- a/frontend/src/components/auditColumns.dom.test.tsx +++ /dev/null @@ -1,205 +0,0 @@ -// @vitest-environment jsdom -// -// An audit row whose server summary is a fixed English line ("rss_poll: ok — new=5, failed=0", -// "log_level = DEBUG", "Background sync paused") reads in the reader's language, rebuilt from the row's -// data through the REAL catalog. A row whose summary is data (an e-mail, a URL) keeps it. -import { afterEach, describe, expect, it } from "vitest"; -import { cleanup, fireEvent, render } from "@testing-library/react"; -import i18n, { setLanguage } from "../i18n"; -import type { AuditRow } from "../lib/api"; -import { auditColumns } from "./auditColumns"; - -// Unmount BEFORE the language goes back: a mounted reader re-renders on the switch, and under a loaded -// run that render landed after the test environment was gone ("window is not defined"). -afterEach(() => { - cleanup(); - setLanguage("en"); -}); - -const row = (over: Partial): AuditRow => ({ - id: 1, - created_at: null, - actor: null, - action: "job_run", - target_type: null, - target_id: null, - summary: null, - before: null, - after: null, - ...over, -}); - -function details(r: AuditRow): string { - setLanguage("hu"); - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const { container } = render(<>{col.render!(r)}); - return [...container.querySelectorAll(".wrap-break-word")].map((e) => e.textContent).join(" | "); -} - -describe("an audit row's details", () => { - it("name a job run and its result in the reader's language", () => { - const r = row({ - target_type: "job", - target_id: "rss_poll", - summary: "rss_poll: ok — new=5, failed=0", - after: { status: "ok", summary: "new=5, failed=0" }, - }); - expect(details(r)).toBe("RSS-lekérdezés (új feltöltések) kész | új videó: 5 · sikertelen: 0"); - }); - - it("keep a failed run's error text, which is diagnostic", () => { - const r = row({ - target_type: "job", - target_id: "enrich", - after: { status: "error", summary: "HTTP 403: quotaExceeded" }, - }); - expect(details(r)).toBe( - "Nem sikerült lefuttatni: Adatdúsítás + élő frissítés | HTTP 403: quotaExceeded", - ); - }); - - it("name a config change's field and values", () => { - const set = row({ - action: "config_set", - target_type: "config", - target_id: "plex_enabled", - summary: "plex_enabled = True", - before: { value: false }, - after: { value: true }, - }); - expect(details(set)).toMatch(/ \| Ki → Be$/); - expect(details(set)).not.toContain("plex_enabled"); - const reset = row({ - action: "config_reset", - target_type: "config", - target_id: "download_upload_max_bytes", - summary: "download_upload_max_bytes → default", - before: { value: 5242880 }, - }); - expect(details(reset)).toMatch(/ \| 5\s242\s880 → alapérték$/); - const secret = row({ - action: "config_set", - target_type: "config", - target_id: "youtube_api_key", - summary: "youtube_api_key = (secret updated)", - }); - expect(details(secret)).toMatch(/ \| titok frissítve/); - }); - - it("name a nested result and a skipped run from the row's data", () => { - const nested = row({ - target_type: "job", - target_id: "people_enrich", - after: { - status: "ok", - summary: "bios={'enriched': 12}, portraits={'cached': 40}", - result: { bios: { enriched: 12 }, portraits: { cached: 40 } }, - }, - }); - expect(details(nested)).not.toContain("{"); - expect(details(nested)).toMatch(/^Személyadatok és portrék \(Plex\) kész \| /); - const skipped = row({ - target_type: "job", - target_id: "rss_poll", - after: { status: "skipped", summary: "sync paused" }, - }); - expect(details(skipped)).toBe("RSS-lekérdezés (új feltöltések) kihagyva | a szinkron szünetel"); - }); - - it("tell a secret's reset from its update, and name a choice", () => { - const reset = row({ - action: "config_reset", - target_type: "config", - target_id: "tmdb_api_key", - summary: "tmdb_api_key → default", - }); - expect(details(reset)).toMatch(/ \| titok visszaállítva/); - const choice = row({ - action: "config_set", - target_type: "config", - target_id: "plex_transcode_hw", - before: { value: "auto" }, - after: { value: "on" }, - }); - expect(details(choice)).toMatch(/ \| Automatikus → GPU$/); - }); - - it("name an interval, a batch and a count in the reader's language", () => { - expect( - details( - row({ - action: "scheduler_interval", - target_type: "job", - target_id: "rss_poll", - summary: "rss_poll → 15 min/run", - after: { interval_minutes: 15 }, - }), - ), - ).toBe("RSS-lekérdezés (új feltöltések) | 15 percenként"); - expect( - details(row({ action: "maintenance_tune", after: { revalidate_batch: 50000 } })), - ).toMatch(/^Futásonként újraellenőrzött videók \| 50\s000$/); - expect( - details( - row({ action: "audit_clear", summary: "Cleared 12 audit entries", after: { cleared: 12 } }), - ), - ).not.toContain("Cleared"); - }); - - it("add nothing to a pause, whose action already says it", () => { - expect(details(row({ action: "sync_pause", summary: "Background sync paused" }))).toBe(""); - }); - - it("keep a summary that is data", () => { - expect(details(row({ action: "invite_add", summary: "someone@example.com" }))).toBe( - "someone@example.com", - ); - }); -}); - -// A folded run (lib/auditGroups) shows its newest run and opens to the earlier ones; a failed run -// is the line to see in a page of routine ones. -describe("a folded run of one job", () => { - const ok = (id: number, enriched: number) => - row({ - id, - target_type: "job", - target_id: "enrich", - after: { status: "ok", result: { enriched } }, - }); - - it("names how many earlier runs it holds, and opens to each", () => { - setLanguage("hu"); - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const runs = [ok(3, 50), ok(2, 40), ok(1, 30)]; - const { getByRole, container } = render(<>{col.render!({ ...runs[0]!, runs })}); - const toggle = getByRole("button", { name: "2 korábbi futás" }); - expect(toggle.getAttribute("aria-expanded")).toBe("false"); - expect(container.querySelector("li")).toBeNull(); - fireEvent.click(toggle); - expect(toggle.getAttribute("aria-expanded")).toBe("true"); - expect([...container.querySelectorAll("li")].map((li) => li.textContent)).toEqual([ - "—dúsítva: 40", - "—dúsítva: 30", - ]); - }); - - it("has no toggle for a lone run", () => { - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const { container } = render(<>{col.render!(ok(1, 5))}); - expect(container.querySelector("button")).toBeNull(); - }); - - it("paints a failed run's line in the danger colour", () => { - const t = i18n.getFixedT("hu"); - const col = auditColumns(t, "hu", []).find((c) => c.key === "details")!; - const failed = row({ target_type: "job", target_id: "enrich", after: { status: "error" } }); - const { container } = render(<>{col.render!(failed)}); - expect(container.querySelector(".wrap-break-word")!.className).toContain("text-danger"); - const { container: fine } = render(<>{col.render!(ok(2, 5))}); - expect(fine.querySelector(".wrap-break-word")!.className).not.toContain("text-danger"); - }); -}); diff --git a/frontend/src/components/auditColumns.tsx b/frontend/src/components/auditColumns.tsx deleted file mode 100644 index 782979f9..00000000 --- a/frontend/src/components/auditColumns.tsx +++ /dev/null @@ -1,171 +0,0 @@ -import type { TFunction } from "i18next"; -import clsx from "clsx"; -import type { AuditRow } from "../lib/api"; -import type { AuditListRow } from "../lib/auditGroups"; -import AuditEarlierRuns from "./AuditEarlierRuns"; -import { formatNumber } from "../lib/format"; -import type { JobResult } from "../lib/api"; -import { jobResultText, jobRunText } from "../lib/jobResult"; -import type { Column } from "./ui/DataTable"; -import { RelativeTime } from "./ui/RelativeTime"; - -function actionLabel(t: TFunction, action: string): string { - // Canonical key → localized label; falls back to the raw key for any unmapped action. - return t(`auditActions.${action}`, action); -} - -function fmt(v: unknown): string { - return v == null || v === "" ? "—" : String(v); -} - -/** A logged config value in the reader's language: a switch reads on/off, a count is grouped, a - * choice is named as the Configuration page names it. */ -function configValue(t: TFunction, key: string, v: unknown, lang: string): string { - if (typeof v === "boolean") return t(v ? "audit.on" : "audit.off"); - if (typeof v === "number") return formatNumber(v, lang); - if (typeof v === "string" && v) return t(`config.choices.${key}.${v}`, { defaultValue: v }); - return fmt(v); -} - -/** A row's `after` as a job-style result (counts), or null when it holds none. */ -function counts(after: Record): JobResult | null { - const out: JobResult = {}; - for (const [k, v] of Object.entries(after)) if (typeof v === "number") out[k] = v; - return Object.keys(out).length ? out : null; -} - -/** The details of the rows whose server `summary` is a fixed English line, rebuilt from the row's - * data in the reader's language: a job run names the job and its result (as the inbox does), a config - * change names the field and its values, an interval or batch change its new value, a count-bearing - * action its counts; a pause/resume says nothing the action has not. `undefined` = not one of these (or - * an older row without the data); the generic summary + change rendering applies. */ -function localDetails( - t: TFunction, - row: AuditRow, - lang: string, -): { head: string | null; sub: string | null } | undefined { - const after = row.after ?? {}; - const before = row.before ?? {}; - if (row.action === "job_run" && row.target_type === "job" && row.target_id) { - const run = jobRunText(t, { job_id: row.target_id, ...after }, lang); - return { head: run.title, sub: run.body }; - } - if ((row.action === "config_set" || row.action === "config_reset") && row.target_id) { - const key = row.target_id; - const head = t(`config.fields.${key}.label`, key); - const reset = row.action === "config_reset"; - // A secret's value is never logged: there is no before/after to show. - if (!row.before && !row.after) - return { head, sub: t(reset ? "audit.secretReset" : "audit.secretUpdated") }; - const to = reset ? t("audit.default") : configValue(t, key, after.value, lang); - return { head, sub: `${configValue(t, key, before.value, lang)} → ${to}` }; - } - if ( - row.action === "scheduler_interval" && - row.target_id && - typeof after.interval_minutes === "number" - ) - return { - head: t(`scheduler.jobs.${row.target_id}`, row.target_id), - sub: t("scheduler.everyMin", { count: after.interval_minutes }), - }; - if (row.action === "maintenance_tune" && typeof after.revalidate_batch === "number") - return { - head: t("scheduler.maintenance.batchLabel"), - sub: formatNumber(after.revalidate_batch, lang), - }; - if (row.action === "audit_clear" || row.action === "discovery_purge") { - const result = counts(after); - return result ? { head: null, sub: jobResultText(t, result, lang) } : undefined; - } - if (row.action === "sync_pause" || row.action === "sync_resume") return { head: null, sub: null }; - return undefined; -} - -/** A compact before→after (or after-only) rendering of the JSON snapshots, e.g. "role: user → admin". */ -function changeText(row: AuditRow): string | null { - const { before, after } = row; - if (before && after) { - return Object.keys(after) - .map((k) => `${k}: ${fmt(before[k])} → ${fmt(after[k])}`) - .join(", "); - } - if (after) { - return Object.entries(after) - .map(([k, v]) => `${k}: ${fmt(v)}`) - .join(", "); - } - return null; -} - -/** Columns for the admin audit-log DataTable. `actionOptions` are the distinct actions present in - * the data (built by the page) so the Action column's select filter only offers real values. */ -export function auditColumns( - t: TFunction, - lang: string, - actionOptions: { value: string; label: string }[], -): Column[] { - return [ - { - key: "when", - header: t("audit.cols.when"), - nowrap: true, - sortable: true, - sortValue: (r) => r.created_at ?? "", - cardPrimary: true, - render: (r) => ( - - ), - }, - { - key: "actor", - header: t("audit.cols.actor"), - sortable: true, - sortValue: (r) => r.actor ?? "", - filter: { kind: "text", get: (r) => r.actor ?? t("audit.system") }, - render: (r) => - r.actor ? ( - {r.actor} - ) : ( - {t("audit.system")} - ), - }, - { - key: "action", - header: t("audit.cols.action"), - nowrap: true, - sortable: true, - sortValue: (r) => actionLabel(t, r.action), - filter: { kind: "select", options: actionOptions, test: (r, v) => r.action === v }, - render: (r) => {actionLabel(t, r.action)}, - }, - { - key: "details", - header: t("audit.cols.details"), - render: (r) => { - const local = localDetails(t, r, lang); - const summary = local ? local.head : r.summary; - const change = local ? local.sub : changeText(r); - // A failed run is the row to see in a page of routine ones: its line in the danger colour. - const failed = r.action === "job_run" && r.after?.status === "error"; - return ( - // wrap-break-word (wrap), NOT truncate: in the auto-layout table a `truncate` (nowrap) cell - // sizes to its longest row, so one long summary/change stretched the whole table past the - // card (the row rules bled into the page). Wrapping keeps every row inside the container. -
- {summary && ( -
{summary}
- )} - {change &&
{change}
} - {r.runs && r.runs.length > 1 && ( - localDetails(t, run, lang)?.sub ?? run.summary} - /> - )} -
- ); - }, - }, - ]; -} diff --git a/frontend/src/components/listNav.wiring.contract.test.ts b/frontend/src/components/listNav.wiring.contract.test.ts index 9d20ef98..a80d94d7 100644 --- a/frontend/src/components/listNav.wiring.contract.test.ts +++ b/frontend/src/components/listNav.wiring.contract.test.ts @@ -24,10 +24,7 @@ const CARDS = [ const LISTS = ["NavGrid", "Carousel", "VirtualGrid"]; /** `path#tag#n` → why it is not a keyboard list. An entry that no longer matches a tag fails the test. */ -const EXEMPT: Record = { - "components/AuditLog.tsx#DataTable#1": - "an admin log read in place: its rows open nothing, so stepping onto them would only move a ring", -}; +const EXEMPT: Record = {}; interface Tag { start: number; diff --git a/frontend/src/components/moduleRegistry.tsx b/frontend/src/components/moduleRegistry.tsx index 27945fcf..63d46ef5 100644 --- a/frontend/src/components/moduleRegistry.tsx +++ b/frontend/src/components/moduleRegistry.tsx @@ -32,8 +32,7 @@ const Stats = lazyWithPreload(() => import("./Stats")); const Scheduler = lazyWithPreload(() => import("./Scheduler")); const ConfigPanel = lazyWithPreload(() => import("./ConfigPanel")); const AdminUsers = lazyWithPreload(() => import("./AdminUsers")); -const AuditLog = lazyWithPreload(() => import("./AuditLog")); -const AdminLogs = lazyWithPreload(() => import("./AdminLogs")); +const Activity = lazyWithPreload(() => import("./Activity")); const SettingsPanel = lazyWithPreload(() => import("./SettingsPanel")); export type PageDef = { @@ -60,8 +59,7 @@ export const PAGE_CONTENT: Record = { scheduler: { Component: Scheduler }, config: { Component: ConfigPanel }, users: { Component: AdminUsers, maxWidth: "48rem" }, - audit: { Component: AuditLog }, - logs: { Component: AdminLogs }, + activity: { Component: Activity }, settings: { Component: SettingsPanel }, }; diff --git a/frontend/src/components/ui/Skeleton.tsx b/frontend/src/components/ui/Skeleton.tsx index 5c1576ff..115fef2c 100644 --- a/frontend/src/components/ui/Skeleton.tsx +++ b/frontend/src/components/ui/Skeleton.tsx @@ -13,6 +13,7 @@ function Skeleton({ className, inline = false }: { className?: string; inline?: return (