peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(config): job intervals live in sysconfig, every setting says when it applies, the unread SponsorBlock default goes

Browse Source 
An interval override was a row of its own table beside the one store every other admin knob uses; it
is now the `job_interval_<job>` sysconfig key (migration 0110 copies them over and drops the table).
The Scheduler page stays its one editor: the Configuration page and the setup wizard neither list nor
write those keys, since a write there would skip the reschedule.

Each ConfigSpec now states `applies` (required, no default). Every key is `live`, held by a scan that
finds no read at import time; a `restart` key is refused until the page can say so. A second scan
requires a reader for every key on the Configuration page — `sponsorblock_default` had none since it
was added, so it leaves the registry, the translations and the store.
This commit is contained in:
peter
2026-10-18 02:23:15 +02:00
parent 3bab2e7dc7
commit d13483a593
15 changed files with 529 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sysconfig.py
+248 -62
View File
@@ -8,7 +8,8 @@ stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secr
stored in the DB and stay env-only).
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Literal
from sqlalchemy import select
from sqlalchemy.orm import Session
@@ -27,6 +28,14 @@ _decrypt_warned: set[str] = set()
_coerce_warned: set[tuple[str, str]] = set()
# When a changed value takes effect. "live": the next read obeys it — every consumer reads the key
# where it uses it, never once at import or boot (test-held). "restart": the process has to start
# again. No key needs a restart today, and the Configuration page has no way to say so yet, so a
# "restart" key is refused until the page can (test-held).
Applies = Literal["live", "restart"]
LIVE: Applies = "live"
@dataclass(frozen=True)
class ConfigSpec:
key: str
@@ -37,6 +46,8 @@ class ConfigSpec:
min: int | None = None
max: int | None = None
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
# Required, with no default: each key states it, so a new one cannot be live by omission.
applies: Applies = field(kw_only=True)
@property
def choices(self) -> tuple[str, ...]:
@@ -45,30 +56,78 @@ class ConfigSpec:
return choices_of(Settings.model_fields[self.default_attr].annotation)
# Each interval job of the scheduler and the `settings` attribute holding its default period
# (minutes). Its override is the `job_interval_<job>` key, set from the Scheduler page.
JOB_INTERVAL_DEFAULTS: dict[str, str] = {
"rss_poll": "rss_poll_minutes",
"enrich": "enrich_interval_minutes",
"backfill": "backfill_interval_minutes",
"autotag": "autotag_interval_minutes",
"shorts": "shorts_probe_interval_minutes",
"subscriptions": "subscriptions_resync_minutes",
"playlist_sync": "playlist_sync_minutes",
"maintenance": "maintenance_interval_minutes",
"explore_cleanup": "explore_cleanup_minutes",
"download_gc": "download_gc_minutes",
"plex_sync": "plex_sync_interval_min",
"plex_watch_sync": "plex_watch_sync_interval_min",
"plex_watch_reconcile": "plex_watch_reconcile_interval_min",
"tmdb_enrich": "tmdb_enrich_interval_min",
"people_enrich": "people_enrich_interval_min",
"audit_gc": "audit_gc_minutes",
"log_gc": "log_gc_minutes",
}
# Sane bounds for an admin-set interval (minutes).
MIN_INTERVAL = 1
MAX_INTERVAL = 1440 # one day
def interval_key(job: str) -> str:
return f"job_interval_{job}"
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
# off env into the admin UI; the admin page renders it automatically from this list. A group's keys
# stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS.
SPECS: tuple[ConfigSpec, ...] = (
# --- Access / registration ---
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
ConfigSpec("allow_registration", "bool", "access", "allow_registration", applies=LIVE),
# --- Email / SMTP (one logical group; the password is the only secret) ---
ConfigSpec("smtp_host", "str", "email", "smtp_host"),
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True),
ConfigSpec("smtp_from", "str", "email", "smtp_from"),
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
ConfigSpec("smtp_host", "str", "email", "smtp_host", applies=LIVE),
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535, applies=LIVE),
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True, applies=LIVE),
ConfigSpec("smtp_from", "str", "email", "smtp_from", applies=LIVE),
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True, applies=LIVE),
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True, applies=LIVE),
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
ConfigSpec(
"youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True, applies=LIVE
),
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
# install wizard / Configuration page so no restart is needed when the creds change. ---
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
# --- Quota (shared YouTube Data API daily budget) ---
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True, applies=LIVE),
ConfigSpec(
"backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000
"google_client_secret", "str", "google", "google_client_secret", secret=True, applies=LIVE
),
# --- Quota (shared YouTube Data API daily budget) ---
ConfigSpec(
"quota_daily_budget",
"int",
"quota",
"quota_daily_budget",
min=1,
max=1_000_000,
applies=LIVE,
),
ConfigSpec(
"backfill_quota_reserve",
"int",
"quota",
"backfill_quota_reserve",
min=0,
max=1_000_000,
applies=LIVE,
),
ConfigSpec(
"search_daily_limit_per_user",
@@ -77,10 +136,19 @@ SPECS: tuple[ConfigSpec, ...] = (
"search_daily_limit_per_user",
min=0,
max=100_000,
applies=LIVE,
),
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
ConfigSpec("search_source", "choice", "quota", "search_source"),
ConfigSpec("quota_retention_days", "int", "quota", "quota_retention_days", min=0, max=3650),
ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE),
ConfigSpec(
"quota_retention_days",
"int",
"quota",
"quota_retention_days",
min=0,
max=3650,
applies=LIVE,
),
# --- Backfill (recent-first first pass per channel) ---
ConfigSpec(
"backfill_recent_max_videos",
@@ -89,19 +157,46 @@ SPECS: tuple[ConfigSpec, ...] = (
"backfill_recent_max_videos",
min=1,
max=100_000,
applies=LIVE,
),
ConfigSpec(
"backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500
"backfill_recent_max_days",
"int",
"backfill",
"backfill_recent_max_days",
min=1,
max=36_500,
applies=LIVE,
),
ConfigSpec(
"rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168, applies=LIVE
),
ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168),
# --- Shorts probe ---
ConfigSpec(
"shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600
"shorts_probe_max_seconds",
"int",
"shorts",
"shorts_probe_max_seconds",
min=1,
max=3_600,
applies=LIVE,
),
ConfigSpec(
"shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000, applies=LIVE
),
ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
# --- Batch sizes ---
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
ConfigSpec(
"enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50, applies=LIVE
),
ConfigSpec(
"autotag_title_sample",
"int",
"batch",
"autotag_title_sample",
min=1,
max=1_000,
applies=LIVE,
),
# --- Maintenance (rolling re-validation of already-known videos) ---
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
@@ -112,16 +207,33 @@ SPECS: tuple[ConfigSpec, ...] = (
"maintenance_revalidate_batch",
min=100,
max=100_000,
applies=LIVE,
),
# --- Discovery cleanup (search + explore ephemeral content) ---
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
# Days an un-kept live-search result video is kept before the cleanup job removes it.
ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650),
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0),
ConfigSpec(
"download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0
"explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650, applies=LIVE
),
# Days an un-kept live-search result video is kept before the cleanup job removes it.
ConfigSpec(
"search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650, applies=LIVE
),
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
ConfigSpec(
"download_total_max_bytes",
"int",
"downloads",
"download_total_max_bytes",
min=0,
applies=LIVE,
),
ConfigSpec(
"download_default_max_bytes",
"int",
"downloads",
"download_default_max_bytes",
min=0,
applies=LIVE,
),
ConfigSpec(
"download_default_max_concurrent",
@@ -130,6 +242,7 @@ SPECS: tuple[ConfigSpec, ...] = (
"download_default_max_concurrent",
min=1,
max=20,
applies=LIVE,
),
ConfigSpec(
"download_default_max_jobs",
@@ -138,6 +251,7 @@ SPECS: tuple[ConfigSpec, ...] = (
"download_default_max_jobs",
min=1,
max=100_000,
applies=LIVE,
),
# min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as
# `if cap and size > cap`, so a falsy value disabled it). Now the route passes this number
@@ -161,14 +275,27 @@ SPECS: tuple[ConfigSpec, ...] = (
"download_upload_max_bytes",
min=1,
max=settings.download_upload_max_bytes,
applies=LIVE,
),
ConfigSpec(
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
"download_retention_days",
"int",
"downloads",
"download_retention_days",
min=0,
max=3_650,
applies=LIVE,
),
ConfigSpec(
"download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650
"download_gc_grace_days",
"int",
"downloads",
"download_gc_grace_days",
min=0,
max=3_650,
applies=LIVE,
),
ConfigSpec("download_layout", "choice", "downloads", "download_layout"),
ConfigSpec("download_layout", "choice", "downloads", "download_layout", applies=LIVE),
ConfigSpec(
"download_worker_concurrency",
"int",
@@ -176,62 +303,112 @@ SPECS: tuple[ConfigSpec, ...] = (
"download_worker_concurrency",
min=1,
max=16,
applies=LIVE,
),
ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"),
# --- Audit log ---
# Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever).
ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650),
ConfigSpec(
"audit_retention_days",
"int",
"audit",
"audit_retention_days",
min=0,
max=3650,
applies=LIVE,
),
# Days the job_runs table keeps a finished background run before the audit_gc job prunes it.
ConfigSpec("job_run_retention_days", "int", "audit", "job_run_retention_days", min=0, max=3650),
ConfigSpec(
"job_run_retention_days",
"int",
"audit",
"job_run_retention_days",
min=0,
max=3650,
applies=LIVE,
),
# --- Diagnostics ---
# Runtime log level for the API process. Applied without a restart — see routes/config.py and
# main.py's lifespan.
ConfigSpec("log_level", "choice", "diagnostics", "log_level"),
ConfigSpec("log_level", "choice", "diagnostics", "log_level", applies=LIVE),
# Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs
# carry account-linked request history, and the privacy policy names a limit.
ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90),
ConfigSpec(
"log_retention_days",
"int",
"diagnostics",
"log_retention_days",
min=1,
max=90,
applies=LIVE,
),
# --- Plex integration (optional; local-file playback + Plex metadata) ---
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"),
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"),
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64),
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled", applies=LIVE),
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url", applies=LIVE),
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True, applies=LIVE),
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map", applies=LIVE),
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries", applies=LIVE),
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map", applies=LIVE),
ConfigSpec(
"plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16, applies=LIVE
),
ConfigSpec(
"plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64, applies=LIVE
),
# Transcode re-encode tuning. The libx264 knobs apply only to the software fallback.
# crf 0–51 (x264 range); max_height 0 = source.
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"),
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"),
ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51),
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw", applies=LIVE),
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset", applies=LIVE),
ConfigSpec(
"plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320
"plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51, applies=LIVE
),
ConfigSpec(
"plex_transcode_max_height",
"int",
"plex",
"plex_transcode_max_height",
min=0,
max=4320,
applies=LIVE,
),
# Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play).
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"),
# NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job
# (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS.
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm", applies=LIVE),
# --- TMDB enrichment (optional) — the API key is the only secret ---
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True),
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"),
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True, applies=LIVE),
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language", applies=LIVE),
# --- Bug reports: the Gitea issue tracker a debug session's issue goes to ---
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"),
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"),
ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True),
# (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url", applies=LIVE),
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo", applies=LIVE),
ConfigSpec(
"issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True, applies=LIVE
),
# --- Scheduler: how often each interval job runs (minutes), one key per job ---
*(
ConfigSpec(
interval_key(job),
"int",
"scheduler",
attr,
min=MIN_INTERVAL,
max=MAX_INTERVAL,
applies=LIVE,
)
for job, attr in JOB_INTERVAL_DEFAULTS.items()
),
)
# The admin page's sections (the owner's pick): sixteen groups as tabs wrapped to three rows at 133%
# zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in
# exactly one section (test-held), so a new group cannot fall off the page.
# exactly one section, or is edited on its own page (test-held), so a new group cannot fall off.
SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = (
("access", ("access", "google", "email")),
("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")),
("media", ("downloads", "plex", "tmdb")),
("system", ("maintenance", "audit", "diagnostics", "issues")),
)
# Groups another admin page edits, so the Configuration page neither lists nor writes them: an
# interval is changed on the Scheduler page, which also reschedules the running job — a write that
# skipped it would leave the job on its old period while the page showed the new one.
OWN_PAGE_GROUPS: tuple[str, ...] = ("scheduler",)
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
@@ -240,6 +417,12 @@ def spec(key: str) -> ConfigSpec | None:
return _BY_KEY.get(key)
def config_page_spec(key: str) -> ConfigSpec | None:
"""`spec`, for the Configuration page: None for a key edited on its own page as well."""
s = _BY_KEY.get(key)
return None if s is None or s.group in OWN_PAGE_GROUPS else s
def secrets_manageable() -> bool:
"""Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY)."""
return bool(settings.token_encryption_key)
@@ -428,13 +611,16 @@ def reset(db: Session, key: str) -> None:
def describe(db: Session) -> dict:
"""Admin-UI metadata + current values, grouped. Secret values are never echoed —
only whether an override is set and whether an env default exists."""
"""Admin-UI metadata + current values, grouped — the keys of the Configuration page, not those
edited on their own page. Secret values are never echoed — only whether an override is set and
whether an env default exists."""
groups: dict[str, list[dict]] = {}
# Every override in one query: a db.get per key was one round trip each (two for a key with no
# row, which the session cannot cache), ~40 ms of the page's ~55 on prod.
rows = {r.key: r for r in db.scalars(select(SystemConfig))}
for s in SPECS:
if s.group in OWN_PAGE_GROUPS:
continue
row = rows.get(s.key)
is_set = row is not None
item: dict = {