From d13483a593065e4d35e06cac439b88407cc4e1da Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 02:23:15 +0200 Subject: [PATCH] feat(config): job intervals live in sysconfig, every setting says when it applies, the unread SponsorBlock default goes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An interval override was a row of its own table beside the one store every other admin knob uses; it is now the `job_interval_` sysconfig key (migration 0110 copies them over and drops the table). The Scheduler page stays its one editor: the Configuration page and the setup wizard neither list nor write those keys, since a write there would skip the reschedule. Each ConfigSpec now states `applies` (required, no default). Every key is `live`, held by a scan that finds no read at import time; a `restart` key is refused until the page can say so. A second scan requires a reader for every key on the Configuration page — `sponsorblock_default` had none since it was added, so it leaves the registry, the translations and the store. --- CLAUDE.md | 2 +- .../versions/0110_intervals_in_sysconfig.py | 79 +++++ backend/app/README.md | 5 +- backend/app/config.py | 2 - backend/app/models.py | 11 - backend/app/routes/config.py | 4 +- backend/app/routes/setup.py | 6 +- backend/app/scheduler.py | 50 +-- backend/app/sysconfig.py | 310 ++++++++++++++---- backend/tests/test_migration_0110.py | 56 ++++ backend/tests/test_scheduler.py | 53 ++- backend/tests/test_sysconfig_registry.py | 70 +++- frontend/src/i18n/backendIds.contract.test.ts | 9 +- frontend/src/i18n/locales/en/config.json | 4 - frontend/src/i18n/locales/hu/config.json | 4 - 15 files changed, 529 insertions(+), 136 deletions(-) create mode 100644 backend/alembic/versions/0110_intervals_in_sysconfig.py create mode 100644 backend/tests/test_migration_0110.py diff --git a/CLAUDE.md b/CLAUDE.md index dba071cb..b57a4030 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,7 +93,7 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory | User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI | | Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` | | Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` and `jobruns` own their session) | -| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter | +| Admin knob storage | `sysconfig` (values) | env / `app_state` / a table of its own (the job intervals had one) | | Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals | | Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers | | FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks | diff --git a/backend/alembic/versions/0110_intervals_in_sysconfig.py b/backend/alembic/versions/0110_intervals_in_sysconfig.py new file mode 100644 index 00000000..4730f333 --- /dev/null +++ b/backend/alembic/versions/0110_intervals_in_sysconfig.py @@ -0,0 +1,79 @@ +"""Job intervals move into sysconfig; the unread SponsorBlock default goes + +Revision ID: 0110_intervals_in_sysconfig +Revises: 0109_job_runs +Create Date: 2026-10-18 + +An admin's interval override lived in its own table, `scheduler_settings`, beside the one store every +other admin knob uses. Each override becomes the `job_interval_` sysconfig key and the table +goes. Only the jobs that still exist are copied: a row for any other id never did anything. + +The `sponsorblock_default` setting was offered on the Configuration page but nothing ever read it, so +its key is gone and its stored value with it — the downgrade cannot bring that value back. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0110_intervals_in_sysconfig" +down_revision: str | None = "0109_job_runs" +branch_labels: str | None = None +depends_on: str | None = None + +# The interval jobs as of this revision (a migration does not read the live code's list). +_JOBS = ( + "rss_poll", + "enrich", + "backfill", + "autotag", + "shorts", + "subscriptions", + "playlist_sync", + "maintenance", + "explore_cleanup", + "download_gc", + "plex_sync", + "plex_watch_sync", + "plex_watch_reconcile", + "tmdb_enrich", + "people_enrich", + "audit_gc", + "log_gc", +) +_PREFIX = "job_interval_" + + +def upgrade() -> None: + op.execute( + sa.text( + "INSERT INTO system_config (key, value, encrypted)" + " SELECT :prefix || job_id, interval_minutes::text, false FROM scheduler_settings" + " WHERE interval_minutes IS NOT NULL AND job_id IN :jobs" + " ON CONFLICT (key) DO NOTHING" + ).bindparams(sa.bindparam("jobs", expanding=True), prefix=_PREFIX, jobs=list(_JOBS)) + ) + op.drop_table("scheduler_settings") + op.execute("DELETE FROM system_config WHERE key = 'sponsorblock_default'") + + +def downgrade() -> None: + op.create_table( + "scheduler_settings", + sa.Column("job_id", sa.String(40), primary_key=True), + sa.Column("interval_minutes", sa.Integer(), nullable=False), + ) + op.execute( + sa.text( + "INSERT INTO scheduler_settings (job_id, interval_minutes)" + " SELECT substr(key, :start), value::int FROM system_config WHERE key IN :keys" + ).bindparams( + sa.bindparam("keys", expanding=True), + start=len(_PREFIX) + 1, + keys=[_PREFIX + j for j in _JOBS], + ) + ) + op.execute( + sa.text("DELETE FROM system_config WHERE key IN :keys").bindparams( + sa.bindparam("keys", expanding=True), keys=[_PREFIX + j for j in _JOBS] + ) + ) diff --git a/backend/app/README.md b/backend/app/README.md index 0dafee5a..7a9ac54f 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -68,8 +68,9 @@ mutation it records, so an audit entry can never exist without its change (or vi - **`app.config.settings`** — boot/env config (DB URL, download root, worker concurrency): read once at startup, not user-tunable at runtime. - **`sysconfig.SPECS`** — the runtime admin knobs (DB-override-or-default). A new admin-tunable value - goes in `SPECS`; don't invent a fourth store. (`state.py`'s `maintenance_batch` predates this and is - slated to fold into `sysconfig` — do not add more like it.) + goes in `SPECS`; don't invent a fourth store. Each spec states when a change takes effect + (`applies`, required): `live` keys are read where they are used, never at import (test-held). The + scheduler's job intervals are `SPECS` keys too, edited on the Scheduler page, not the Configuration page. ## The gate diff --git a/backend/app/config.py b/backend/app/config.py index a77eb37c..b1279f0c 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -274,8 +274,6 @@ class Settings(BaseSettings): job_run_retention_days: int = 180 # On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat". download_layout: DownloadLayout = "plex" - # Default SponsorBlock (skip sponsor segments) for new custom profiles. - sponsorblock_default: bool = False # --- Diagnostics / logging --- # BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at diff --git a/backend/app/models.py b/backend/app/models.py index 880fa7de..2b2d2730 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -701,17 +701,6 @@ class AppState(Base): setup_token_hash: Mapped[str | None] = mapped_column(String(64)) -class SchedulerSetting(Base): - """Admin override for a scheduler job's run interval (minutes). One row per job id; - absent = use the env/config default. DB-backed so it's editable from the admin UI at - runtime without a redeploy (env stays a fallback).""" - - __tablename__ = "scheduler_settings" - - job_id: Mapped[str] = mapped_column(String(40), primary_key=True) - interval_minutes: Mapped[int] = mapped_column(Integer) - - class JobRun(Base): """One run of a background job — scheduled, an admin's "run now", or a user's ad-hoc work on one target — written when it starts and again when it ends (app/jobruns.py), each in its own diff --git a/backend/app/routes/config.py b/backend/app/routes/config.py index e4425ebb..053b3627 100644 --- a/backend/app/routes/config.py +++ b/backend/app/routes/config.py @@ -61,7 +61,7 @@ def set_config( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - s = sysconfig.spec(key) + s = sysconfig.config_page_spec(key) if s is None: raise not_found("config_key") if "value" not in body.model_fields_set: @@ -100,7 +100,7 @@ def reset_config( admin: User = Depends(admin_user), db: Session = Depends(get_db), ) -> dict: - s = sysconfig.spec(key) + s = sysconfig.config_page_spec(key) if s is None: raise not_found("config_key") old = None if s.secret else sysconfig.get(db, key) diff --git a/backend/app/routes/setup.py b/backend/app/routes/setup.py index cddbfcec..7ad48324 100644 --- a/backend/app/routes/setup.py +++ b/backend/app/routes/setup.py @@ -91,12 +91,12 @@ def setup_admin(body: SetupAdminIn, db: Session = Depends(require_setup)) -> dic @router.post("/config") def setup_config(settings_map: dict[str, Any], db: Session = Depends(require_setup)) -> dict: - """Persist provided DB-backed settings (Google creds, SMTP, quota…). Keys must be in the - sysconfig registry; empty values are skipped (left at the default). Genuinely dynamic + """Persist provided DB-backed settings (Google creds, SMTP, quota…). Keys must be the + Configuration page's; empty values are skipped (left at the default). Genuinely dynamic key→value map, so it stays a typed dict rather than a fixed model.""" saved: list[str] = [] for key, value in settings_map.items(): - if sysconfig.spec(key) is None: + if sysconfig.config_page_spec(key) is None: raise refused(400, "unknown_setting", key=key) if value is None or value == "": continue diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 6a3b6ed7..e528ed44 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -11,11 +11,11 @@ from apscheduler.jobstores.base import ConflictingIdError from apscheduler.schedulers.background import BackgroundScheduler from sqlalchemy import select -from app import audit, debugsession, jobruns, logs, logsink, progress, quota +from app import audit, debugsession, jobruns, logs, logsink, progress, quota, sysconfig from app.config import settings from app.db import SessionLocal from app.downloads.gc import run_download_gc -from app.models import Channel, SchedulerSetting +from app.models import Channel from app.plex.sync import sync as run_plex_sync from app.plex.tmdb import run_enrich as run_tmdb_enrich from app.plex.tmdb import run_people_enrich @@ -52,41 +52,19 @@ _manual_actor: contextvars.ContextVar[int | None] = contextvars.ContextVar( "manual_actor", default=None ) -# Each interval job's env/config default period (minutes). The admin can override any of -# these at runtime (stored in scheduler_settings, applied live); see load_intervals. +# Each interval job's env/config default period (minutes). The admin can override any of these at +# runtime from the Scheduler page — a sysconfig key per job (`sysconfig.interval_key`), applied +# live; see load_intervals. JOB_INTERVALS: dict[str, int] = { - "rss_poll": settings.rss_poll_minutes, - "enrich": settings.enrich_interval_minutes, - "backfill": settings.backfill_interval_minutes, - "autotag": settings.autotag_interval_minutes, - "shorts": settings.shorts_probe_interval_minutes, - "subscriptions": settings.subscriptions_resync_minutes, - "playlist_sync": settings.playlist_sync_minutes, - "maintenance": settings.maintenance_interval_minutes, - "explore_cleanup": settings.explore_cleanup_minutes, - "download_gc": settings.download_gc_minutes, - "plex_sync": settings.plex_sync_interval_min, - "plex_watch_sync": settings.plex_watch_sync_interval_min, - "plex_watch_reconcile": settings.plex_watch_reconcile_interval_min, - "tmdb_enrich": settings.tmdb_enrich_interval_min, - "people_enrich": settings.people_enrich_interval_min, - "audit_gc": settings.audit_gc_minutes, - "log_gc": settings.log_gc_minutes, + job: getattr(settings, attr) for job, attr in sysconfig.JOB_INTERVAL_DEFAULTS.items() } - -# Sane bounds for an admin-set interval (minutes). -MIN_INTERVAL = 1 -MAX_INTERVAL = 1440 # one day +MIN_INTERVAL = sysconfig.MIN_INTERVAL +MAX_INTERVAL = sysconfig.MAX_INTERVAL def load_intervals(db) -> dict[str, int]: - """Effective per-job intervals: the env/config defaults overlaid with any admin overrides - saved in scheduler_settings.""" - iv = dict(JOB_INTERVALS) - for row in db.execute(select(SchedulerSetting)).scalars(): - if row.job_id in iv and row.interval_minutes: - iv[row.job_id] = row.interval_minutes - return iv + """Effective per-job intervals: each job's sysconfig override, else its env/config default.""" + return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS} def apply_interval(db, job_id: str, minutes: int) -> int: @@ -95,11 +73,7 @@ def apply_interval(db, job_id: str, minutes: int) -> int: if job_id not in JOB_INTERVALS: raise KeyError(job_id) minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes))) - row = db.get(SchedulerSetting, job_id) - if row is None: - db.add(SchedulerSetting(job_id=job_id, interval_minutes=minutes)) - else: - row.interval_minutes = minutes + sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes) db.commit() if _scheduler is not None and _scheduler.get_job(job_id) is not None: _scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes) @@ -434,7 +408,6 @@ def _audit_gc_job() -> None: # quota_retention_days and finished runs past job_run_retention_days (0 = keep forever, each). # Pure DB, no quota. def work(db): - from app import sysconfig reaped = audit.gc(db, sysconfig.get_int(db, "audit_retention_days")) quota_reaped = quota.gc(db, sysconfig.get_int(db, "quota_retention_days")) @@ -449,7 +422,6 @@ def _log_gc_job() -> None: # Prune log_events rows older than log_retention_days — the limit the privacy policy states, so # this runs hourly rather than daily. Pure DB, no quota. def work(db): - from app import sysconfig days = sysconfig.get_int(db, "log_retention_days") reaped = logsink.gc(db, days) diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 4d91b865..bf6ef7f8 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -8,7 +8,8 @@ stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secr stored in the DB and stay env-only). """ -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Literal from sqlalchemy import select from sqlalchemy.orm import Session @@ -27,6 +28,14 @@ _decrypt_warned: set[str] = set() _coerce_warned: set[tuple[str, str]] = set() +# When a changed value takes effect. "live": the next read obeys it — every consumer reads the key +# where it uses it, never once at import or boot (test-held). "restart": the process has to start +# again. No key needs a restart today, and the Configuration page has no way to say so yet, so a +# "restart" key is refused until the page can (test-held). +Applies = Literal["live", "restart"] +LIVE: Applies = "live" + + @dataclass(frozen=True) class ConfigSpec: key: str @@ -37,6 +46,8 @@ class ConfigSpec: min: int | None = None max: int | None = None allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset" + # Required, with no default: each key states it, so a new one cannot be live by omission. + applies: Applies = field(kw_only=True) @property def choices(self) -> tuple[str, ...]: @@ -45,30 +56,78 @@ class ConfigSpec: return choices_of(Settings.model_fields[self.default_attr].annotation) +# Each interval job of the scheduler and the `settings` attribute holding its default period +# (minutes). Its override is the `job_interval_` key, set from the Scheduler page. +JOB_INTERVAL_DEFAULTS: dict[str, str] = { + "rss_poll": "rss_poll_minutes", + "enrich": "enrich_interval_minutes", + "backfill": "backfill_interval_minutes", + "autotag": "autotag_interval_minutes", + "shorts": "shorts_probe_interval_minutes", + "subscriptions": "subscriptions_resync_minutes", + "playlist_sync": "playlist_sync_minutes", + "maintenance": "maintenance_interval_minutes", + "explore_cleanup": "explore_cleanup_minutes", + "download_gc": "download_gc_minutes", + "plex_sync": "plex_sync_interval_min", + "plex_watch_sync": "plex_watch_sync_interval_min", + "plex_watch_reconcile": "plex_watch_reconcile_interval_min", + "tmdb_enrich": "tmdb_enrich_interval_min", + "people_enrich": "people_enrich_interval_min", + "audit_gc": "audit_gc_minutes", + "log_gc": "log_gc_minutes", +} +# Sane bounds for an admin-set interval (minutes). +MIN_INTERVAL = 1 +MAX_INTERVAL = 1440 # one day + + +def interval_key(job: str) -> str: + return f"job_interval_{job}" + + # Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it # off env into the admin UI; the admin page renders it automatically from this list. A group's keys # stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS. SPECS: tuple[ConfigSpec, ...] = ( # --- Access / registration --- - ConfigSpec("allow_registration", "bool", "access", "allow_registration"), + ConfigSpec("allow_registration", "bool", "access", "allow_registration", applies=LIVE), # --- Email / SMTP (one logical group; the password is the only secret) --- - ConfigSpec("smtp_host", "str", "email", "smtp_host"), - ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535), - ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True), - ConfigSpec("smtp_from", "str", "email", "smtp_from"), - ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), + ConfigSpec("smtp_host", "str", "email", "smtp_host", applies=LIVE), + ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535, applies=LIVE), + ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True, applies=LIVE), + ConfigSpec("smtp_from", "str", "email", "smtp_from", applies=LIVE), + ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True, applies=LIVE), # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- - ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), + ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True, applies=LIVE), # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). - ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True), + ConfigSpec( + "youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True, applies=LIVE + ), # --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the # install wizard / Configuration page so no restart is needed when the creds change. --- - ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True), - ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True), - # --- Quota (shared YouTube Data API daily budget) --- - ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), + ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True, applies=LIVE), ConfigSpec( - "backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000 + "google_client_secret", "str", "google", "google_client_secret", secret=True, applies=LIVE + ), + # --- Quota (shared YouTube Data API daily budget) --- + ConfigSpec( + "quota_daily_budget", + "int", + "quota", + "quota_daily_budget", + min=1, + max=1_000_000, + applies=LIVE, + ), + ConfigSpec( + "backfill_quota_reserve", + "int", + "quota", + "backfill_quota_reserve", + min=0, + max=1_000_000, + applies=LIVE, ), ConfigSpec( "search_daily_limit_per_user", @@ -77,10 +136,19 @@ SPECS: tuple[ConfigSpec, ...] = ( "search_daily_limit_per_user", min=0, max=100_000, + applies=LIVE, ), # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). - ConfigSpec("search_source", "choice", "quota", "search_source"), - ConfigSpec("quota_retention_days", "int", "quota", "quota_retention_days", min=0, max=3650), + ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE), + ConfigSpec( + "quota_retention_days", + "int", + "quota", + "quota_retention_days", + min=0, + max=3650, + applies=LIVE, + ), # --- Backfill (recent-first first pass per channel) --- ConfigSpec( "backfill_recent_max_videos", @@ -89,19 +157,46 @@ SPECS: tuple[ConfigSpec, ...] = ( "backfill_recent_max_videos", min=1, max=100_000, + applies=LIVE, ), ConfigSpec( - "backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500 + "backfill_recent_max_days", + "int", + "backfill", + "backfill_recent_max_days", + min=1, + max=36_500, + applies=LIVE, + ), + ConfigSpec( + "rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168, applies=LIVE ), - ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168), # --- Shorts probe --- ConfigSpec( - "shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600 + "shorts_probe_max_seconds", + "int", + "shorts", + "shorts_probe_max_seconds", + min=1, + max=3_600, + applies=LIVE, + ), + ConfigSpec( + "shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000, applies=LIVE ), - ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000), # --- Batch sizes --- - ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), - ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), + ConfigSpec( + "enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50, applies=LIVE + ), + ConfigSpec( + "autotag_title_sample", + "int", + "batch", + "autotag_title_sample", + min=1, + max=1_000, + applies=LIVE, + ), # --- Maintenance (rolling re-validation of already-known videos) --- # Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin # knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard. @@ -112,16 +207,33 @@ SPECS: tuple[ConfigSpec, ...] = ( "maintenance_revalidate_batch", min=100, max=100_000, + applies=LIVE, ), # --- Discovery cleanup (search + explore ephemeral content) --- # Days an explored-but-unsubscribed channel is kept before the cleanup job purges it. - ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650), - # Days an un-kept live-search result video is kept before the cleanup job removes it. - ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650), - # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- - ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0), ConfigSpec( - "download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0 + "explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650, applies=LIVE + ), + # Days an un-kept live-search result video is kept before the cleanup job removes it. + ConfigSpec( + "search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650, applies=LIVE + ), + # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- + ConfigSpec( + "download_total_max_bytes", + "int", + "downloads", + "download_total_max_bytes", + min=0, + applies=LIVE, + ), + ConfigSpec( + "download_default_max_bytes", + "int", + "downloads", + "download_default_max_bytes", + min=0, + applies=LIVE, ), ConfigSpec( "download_default_max_concurrent", @@ -130,6 +242,7 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_default_max_concurrent", min=1, max=20, + applies=LIVE, ), ConfigSpec( "download_default_max_jobs", @@ -138,6 +251,7 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_default_max_jobs", min=1, max=100_000, + applies=LIVE, ), # min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as # `if cap and size > cap`, so a falsy value disabled it). Now the route passes this number @@ -161,14 +275,27 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_upload_max_bytes", min=1, max=settings.download_upload_max_bytes, + applies=LIVE, ), ConfigSpec( - "download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650 + "download_retention_days", + "int", + "downloads", + "download_retention_days", + min=0, + max=3_650, + applies=LIVE, ), ConfigSpec( - "download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650 + "download_gc_grace_days", + "int", + "downloads", + "download_gc_grace_days", + min=0, + max=3_650, + applies=LIVE, ), - ConfigSpec("download_layout", "choice", "downloads", "download_layout"), + ConfigSpec("download_layout", "choice", "downloads", "download_layout", applies=LIVE), ConfigSpec( "download_worker_concurrency", "int", @@ -176,62 +303,112 @@ SPECS: tuple[ConfigSpec, ...] = ( "download_worker_concurrency", min=1, max=16, + applies=LIVE, ), - ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"), # --- Audit log --- # Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever). - ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650), + ConfigSpec( + "audit_retention_days", + "int", + "audit", + "audit_retention_days", + min=0, + max=3650, + applies=LIVE, + ), # Days the job_runs table keeps a finished background run before the audit_gc job prunes it. - ConfigSpec("job_run_retention_days", "int", "audit", "job_run_retention_days", min=0, max=3650), + ConfigSpec( + "job_run_retention_days", + "int", + "audit", + "job_run_retention_days", + min=0, + max=3650, + applies=LIVE, + ), # --- Diagnostics --- # Runtime log level for the API process. Applied without a restart — see routes/config.py and # main.py's lifespan. - ConfigSpec("log_level", "choice", "diagnostics", "log_level"), + ConfigSpec("log_level", "choice", "diagnostics", "log_level", applies=LIVE), # Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs # carry account-linked request history, and the privacy policy names a limit. - ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90), + ConfigSpec( + "log_retention_days", + "int", + "diagnostics", + "log_retention_days", + min=1, + max=90, + applies=LIVE, + ), # --- Plex integration (optional; local-file playback + Plex metadata) --- - ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"), - ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"), - ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True), - ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"), - ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"), - ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"), - # (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.) - ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16), - ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64), + ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled", applies=LIVE), + ConfigSpec("plex_server_url", "str", "plex", "plex_server_url", applies=LIVE), + ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True, applies=LIVE), + ConfigSpec("plex_path_map", "str", "plex", "plex_path_map", applies=LIVE), + ConfigSpec("plex_libraries", "str", "plex", "plex_libraries", applies=LIVE), + ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map", applies=LIVE), + ConfigSpec( + "plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16, applies=LIVE + ), + ConfigSpec( + "plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64, applies=LIVE + ), # Transcode re-encode tuning. The libx264 knobs apply only to the software fallback. # crf 0–51 (x264 range); max_height 0 = source. - ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"), - ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"), - ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51), + ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw", applies=LIVE), + ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset", applies=LIVE), ConfigSpec( - "plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320 + "plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51, applies=LIVE + ), + ConfigSpec( + "plex_transcode_max_height", + "int", + "plex", + "plex_transcode_max_height", + min=0, + max=4320, + applies=LIVE, ), # Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play). - ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"), - # NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job - # (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS. + ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm", applies=LIVE), # --- TMDB enrichment (optional) — the API key is the only secret --- - ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True), - ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"), + ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True, applies=LIVE), + ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language", applies=LIVE), # --- Bug reports: the Gitea issue tracker a debug session's issue goes to --- - ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"), - ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"), - ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True), - # (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch - # is a config.py default — both stay out of SPECS, like plex_sync_interval_min.) + ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url", applies=LIVE), + ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo", applies=LIVE), + ConfigSpec( + "issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True, applies=LIVE + ), + # --- Scheduler: how often each interval job runs (minutes), one key per job --- + *( + ConfigSpec( + interval_key(job), + "int", + "scheduler", + attr, + min=MIN_INTERVAL, + max=MAX_INTERVAL, + applies=LIVE, + ) + for job, attr in JOB_INTERVAL_DEFAULTS.items() + ), ) # The admin page's sections (the owner's pick): sixteen groups as tabs wrapped to three rows at 133% # zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in -# exactly one section (test-held), so a new group cannot fall off the page. +# exactly one section, or is edited on its own page (test-held), so a new group cannot fall off. SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = ( ("access", ("access", "google", "email")), ("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")), ("media", ("downloads", "plex", "tmdb")), ("system", ("maintenance", "audit", "diagnostics", "issues")), ) +# Groups another admin page edits, so the Configuration page neither lists nor writes them: an +# interval is changed on the Scheduler page, which also reschedules the running job — a write that +# skipped it would leave the job on its old period while the page showed the new one. +OWN_PAGE_GROUPS: tuple[str, ...] = ("scheduler",) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} @@ -240,6 +417,12 @@ def spec(key: str) -> ConfigSpec | None: return _BY_KEY.get(key) +def config_page_spec(key: str) -> ConfigSpec | None: + """`spec`, for the Configuration page: None for a key edited on its own page as well.""" + s = _BY_KEY.get(key) + return None if s is None or s.group in OWN_PAGE_GROUPS else s + + def secrets_manageable() -> bool: """Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY).""" return bool(settings.token_encryption_key) @@ -428,13 +611,16 @@ def reset(db: Session, key: str) -> None: def describe(db: Session) -> dict: - """Admin-UI metadata + current values, grouped. Secret values are never echoed — - only whether an override is set and whether an env default exists.""" + """Admin-UI metadata + current values, grouped — the keys of the Configuration page, not those + edited on their own page. Secret values are never echoed — only whether an override is set and + whether an env default exists.""" groups: dict[str, list[dict]] = {} # Every override in one query: a db.get per key was one round trip each (two for a key with no # row, which the session cannot cache), ~40 ms of the page's ~55 on prod. rows = {r.key: r for r in db.scalars(select(SystemConfig))} for s in SPECS: + if s.group in OWN_PAGE_GROUPS: + continue row = rows.get(s.key) is_set = row is not None item: dict = { diff --git a/backend/tests/test_migration_0110.py b/backend/tests/test_migration_0110.py new file mode 100644 index 00000000..ba3677b8 --- /dev/null +++ b/backend/tests/test_migration_0110.py @@ -0,0 +1,56 @@ +"""Migration 0110's data step, DB-lane: an admin's interval override moves from `scheduler_settings` +into sysconfig and back, a row for a job that no longer exists is not carried over, and the unread +SponsorBlock default leaves the store. + +The schema half is checked by test_migrations.py; a value lost on the way is invisible there. +""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app import scheduler +from app.db import engine + + +def _config(conn) -> dict[str, str]: + return dict(conn.execute(text("SELECT key, value FROM system_config")).tuples().all()) + + +def test_an_interval_override_moves_into_sysconfig_and_back(db): + cfg = Config("alembic.ini") + command.downgrade(cfg, "0109_job_runs") + try: + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO scheduler_settings (job_id, interval_minutes)" + " VALUES ('rss_poll', 25), ('log_gc', 90), ('retired_job', 5)" + ) + ) + conn.execute( + text( + "INSERT INTO system_config (key, value, encrypted) VALUES" + " ('sponsorblock_default', 'true', false), ('quota_daily_budget', '5000', false)" + ) + ) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + assert _config(conn) == { + "job_interval_rss_poll": "25", + "job_interval_log_gc": "90", + "quota_daily_budget": "5000", + } + assert conn.execute(text("SELECT to_regclass('scheduler_settings')")).scalar() is None + assert scheduler.load_intervals(db)["rss_poll"] == 25 + + command.downgrade(cfg, "0109_job_runs") + try: + with engine.connect() as conn: + back = conn.execute(text("SELECT job_id, interval_minutes FROM scheduler_settings")) + assert dict(back.tuples().all()) == {"rss_poll": 25, "log_gc": 90} + assert _config(conn) == {"quota_daily_budget": "5000"} + finally: + command.upgrade(cfg, "head") diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 9150a7a3..91c0c84b 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -12,7 +12,7 @@ from datetime import datetime, timedelta, timezone import pytest from sqlalchemy import select -from app import jobruns, logs, logsink, quota, scheduler, state +from app import jobruns, logs, logsink, quota, scheduler, state, sysconfig from app.models import AuditLog, Channel, JobRun, Notification, QuotaEvent from app.runstatus import RunStatus, RunTrigger @@ -411,3 +411,54 @@ def test_startup_closes_what_the_previous_process_left_running(db, monkeypatch): scheduler.shutdown_scheduler() (run,) = _runs(db, "plex_sync") assert run.status == "interrupted" + + +def test_every_job_has_an_interval_and_every_interval_a_job(): + assert set(scheduler.JOB_FUNCS) == set(scheduler.JOB_INTERVALS) + + +def test_a_stored_interval_is_what_the_job_starts_with(db, monkeypatch): + sysconfig.set_value(db, "job_interval_rss_poll", 25) + db.commit() + monkeypatch.setattr(scheduler.settings, "scheduler_enabled", True) + scheduler.start_scheduler() + try: + sched = scheduler._scheduler + assert sched is not None + assert sched.get_job("rss_poll").trigger.interval == timedelta(minutes=25) + default = scheduler.JOB_INTERVALS["enrich"] + assert sched.get_job("enrich").trigger.interval == timedelta(minutes=default) + finally: + scheduler.shutdown_scheduler() + + +def test_an_interval_set_on_the_scheduler_page_is_stored_and_reschedules_the_job( + db, make_user, as_user, live_scheduler +): + live_scheduler.add_job(lambda: None, "interval", minutes=10, id="backfill") + admin = as_user(make_user(role="admin")) + r = admin.patch("/api/admin/scheduler/jobs/backfill", json={"interval_minutes": 45}) + assert r.status_code == 200, r.text + assert live_scheduler.get_job("backfill").trigger.interval == timedelta(minutes=45) + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == 45 + assert scheduler.load_intervals(db)["backfill"] == 45 + # The Configuration page neither lists an interval nor writes one past the reschedule. + assert ( + admin.patch("/api/admin/config/job_interval_backfill", json={"value": 5}).status_code == 404 + ) + assert admin.delete("/api/admin/config/job_interval_backfill").status_code == 404 + assert "scheduler" not in admin.get("/api/admin/config").json()["groups"] + db.expire_all() + assert sysconfig.get_int(db, "job_interval_backfill") == 45 + + +def test_the_setup_wizard_does_not_write_an_interval(db): + from fastapi import HTTPException + + from app.routes.setup import setup_config + + with pytest.raises(HTTPException) as refused: + setup_config({"job_interval_rss_poll": 5}, db) + assert refused.value.status_code == 400 + assert setup_config({"quota_daily_budget": 5000}, db)["saved"] == ["quota_daily_budget"] diff --git a/backend/tests/test_sysconfig_registry.py b/backend/tests/test_sysconfig_registry.py index 8a2f5d9b..87333fa2 100644 --- a/backend/tests/test_sysconfig_registry.py +++ b/backend/tests/test_sysconfig_registry.py @@ -15,6 +15,9 @@ boot, sysconfig holds a stored override to it, and the admin page renders a pick The DB lane (`db` fixture) covers the store; the rest is pure logic. """ +import ast +from pathlib import Path + import pytest from pydantic import ValidationError @@ -37,7 +40,8 @@ def test_each_group_is_one_run_of_keys(): def test_every_group_sits_in_exactly_one_section(): listed = [g for _, members in sysconfig.SECTIONS for g in members] assert len(listed) == len(set(listed)), listed - assert set(listed) == {s.group for s in sysconfig.SPECS} + assert set(listed) | set(sysconfig.OWN_PAGE_GROUPS) == {s.group for s in sysconfig.SPECS} + assert not set(listed) & set(sysconfig.OWN_PAGE_GROUPS) assert [sid for sid, _ in sysconfig.SECTIONS] == ["access", "youtube", "media", "system"] @@ -112,3 +116,67 @@ def test_set_value_stores_the_canonical_spelling_and_refuses_anything_else(db): with pytest.raises(ValueError, match="scrape, api"): sysconfig.set_value(db, "search_source", "bing") db.rollback() + + +# --- when a changed value takes effect -------------------------------------------------------------- + +_APP = Path(__file__).resolve().parent.parent / "app" +_READERS = {"get", "get_str", "get_int", "get_bool"} + + +def _key_reads() -> list[tuple[str, str, bool]]: + """Every `sysconfig.get*(db, "")` in the app outside the registry: (key, where, inside a + function).""" + found: list[tuple[str, str, bool]] = [] + + def walk(node: ast.AST, path: Path, in_fn: bool) -> None: + for child in ast.iter_child_nodes(node): + fn = in_fn or isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + if ( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Attribute) + and isinstance(child.func.value, ast.Name) + and child.func.value.id == "sysconfig" + and child.func.attr in _READERS + and len(child.args) >= 2 + and isinstance(child.args[1], ast.Constant) + and isinstance(child.args[1].value, str) + ): + found.append((child.args[1].value, f"{path.name}:{child.lineno}", in_fn)) + walk(child, path, fn) + + for path in _APP.rglob("*.py"): + if path.name != "sysconfig.py": + walk(ast.parse(path.read_text(encoding="utf-8")), path, False) + return found + + +def test_every_key_states_when_it_applies_and_none_needs_a_restart_yet(): + with pytest.raises(TypeError): + sysconfig.ConfigSpec("k", "int", "quota", "quota_daily_budget") # pyright: ignore[reportCallIssue] + # The Configuration page cannot tell the admin that a setting waits for a restart; a key that + # does needs that first (a badge on it and a banner once it changed). + assert {s.applies for s in sysconfig.SPECS} == {"live"} + + +def test_a_live_key_is_read_where_it_is_used_never_at_import(): + reads = _key_reads() + assert len(reads) > 50, len( + reads + ) # the scan sees the reads (it would pass vacuously otherwise) + assert [(k, at) for k, at, in_fn in reads if not in_fn] == [] + + +def test_every_key_on_the_configuration_page_has_a_reader(): + read = {k for k, _, _ in _key_reads()} + page = {s.key for s in sysconfig.SPECS if s.group not in sysconfig.OWN_PAGE_GROUPS} + assert sorted(page - read) == [] + + +def test_the_intervals_are_keys_of_the_scheduler_page_only(): + keys = {s.key for s in sysconfig.SPECS if s.group == "scheduler"} + assert keys == {sysconfig.interval_key(j) for j in sysconfig.JOB_INTERVAL_DEFAULTS} + assert all(len(k) <= 64 for k in keys) # system_config.key + assert sysconfig.config_page_spec("job_interval_rss_poll") is None + assert sysconfig.config_page_spec("quota_daily_budget") is not None + assert "scheduler" not in sysconfig.describe(_FakeDb())["groups"] diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts index 7c6b8ba5..6e1c9b9d 100644 --- a/frontend/src/i18n/backendIds.contract.test.ts +++ b/frontend/src/i18n/backendIds.contract.test.ts @@ -10,7 +10,7 @@ import { vocabKey } from "../lib/vocab"; // the registries straight from the backend sources: // - `sysconfig.SPECS` → `config.groups.`, `config.fields..label` + `.hint`, and for a // "choice" key `config.choices..