peter
fad656988a
Backend - A run's outcome write sits outside the job's own try: a database blip after a run that succeeded no longer records it as failed (the row waits for the next start's sweep). - An interrupted run keeps what it spent (the sweep sums its quota rows, as finish does). - The Scheduler's ad-hoc row says running while any backfill runs, not only the newest. - recent_runs reads each job's last runs off its index instead of ranking the whole table. - An interval change is stored with its audit row in one transaction, then rescheduled. - One API instance per database is the stated contract (.env.example, jobruns). - Turning full history on for a channel already complete records no run. - Migration 0109 round-trips interrupted and ad-hoc runs; the run words' DB CHECKs are test-held to the enums; the account filter is one helper for the four activity reads. - An audit row says how many of its request's log lines are still kept. Frontend - An "All" period reaches every row still kept (runs and admin actions outlive 90 days). - Back/Forward is followed in place by the server log: a closing dialog no longer turns Live off; a pivot still opens a fresh log. - Runs moved over from the audit log read their one-liner in the reader's language. - Filter choices or accounts that failed to load say so beside the filters, with a retry. - An old server-log link keeps its account, text and range; a remembered retired page opens its tab. - A request whose lines are gone says so instead of linking to an empty list. - The log lines and the history rows share one row frame, and the server log one list frame with the other tabs; an empty list is announced; each day's quota is readable.
649 lines
26 KiB
Python
649 lines
26 KiB
Python
"""Admin-editable system configuration: a DB-override layer over the env/config defaults.
|
||
|
||
The registry (SPECS) is the single source of truth for which config keys can be overridden
|
||
from the admin Configuration page, plus each key's type, admin-UI group, env/config default,
|
||
bounds, and whether it's a secret. The resolver returns the DB override if one is set, else
|
||
``settings.<default_attr>`` (same DB-override-or-default pattern as app.state). Secrets are
|
||
stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secrets can't be
|
||
stored in the DB and stay env-only).
|
||
"""
|
||
|
||
from dataclasses import dataclass, field
|
||
from typing import Literal
|
||
|
||
from sqlalchemy import select
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app import logs, security
|
||
from app.config import Settings, canonical_choice, choices_of, settings
|
||
from app.models import SystemConfig
|
||
|
||
log = logs.get_logger(__name__)
|
||
# Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per
|
||
# key, not once per request (bounded by the number of secret config keys).
|
||
_decrypt_warned: set[str] = set()
|
||
# The same for a stored value that no longer coerces (a choice list that shrank in a release): it
|
||
# fails on EVERY read of a hot knob until someone saves it again. Keyed with the value, so saving a
|
||
# different bad value is reported afresh (bounded by the keys times the values ever saved).
|
||
_coerce_warned: set[tuple[str, str]] = set()
|
||
|
||
|
||
# When a changed value takes effect. "live": the next read obeys it — every consumer reads the key
|
||
# where it uses it, never once at import (test-held: a read outside a function fails). A value read
|
||
# once at startup is live only when its writer also applies it (the job intervals reschedule, the log
|
||
# level is set as it is saved). "restart": the process has to start
|
||
# again. No key needs a restart today, and the Configuration page has no way to say so yet, so a
|
||
# "restart" key is refused until the page can (test-held).
|
||
Applies = Literal["live", "restart"]
|
||
LIVE: Applies = "live"
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ConfigSpec:
|
||
key: str
|
||
type: str # "int" | "str" | "bool" | "choice"
|
||
group: str # admin-UI grouping (logically related keys stay together)
|
||
default_attr: str # attribute on `settings` holding the env/config default
|
||
secret: bool = False
|
||
min: int | None = None
|
||
max: int | None = None
|
||
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
|
||
# Required, with no default: each key states it, so a new one cannot be live by omission.
|
||
applies: Applies = field(kw_only=True)
|
||
|
||
@property
|
||
def choices(self) -> tuple[str, ...]:
|
||
"""A choice key's allowed values, read from its setting's Literal type in config.py — the one
|
||
place the list is written. The admin page renders them as a picker."""
|
||
return choices_of(Settings.model_fields[self.default_attr].annotation)
|
||
|
||
|
||
# Each interval job of the scheduler and the `settings` attribute holding its default period
|
||
# (minutes). Its override is the `job_interval_<job>` key, set from the Scheduler page.
|
||
JOB_INTERVAL_DEFAULTS: dict[str, str] = {
|
||
"rss_poll": "rss_poll_minutes",
|
||
"enrich": "enrich_interval_minutes",
|
||
"backfill": "backfill_interval_minutes",
|
||
"autotag": "autotag_interval_minutes",
|
||
"shorts": "shorts_probe_interval_minutes",
|
||
"subscriptions": "subscriptions_resync_minutes",
|
||
"playlist_sync": "playlist_sync_minutes",
|
||
"maintenance": "maintenance_interval_minutes",
|
||
"explore_cleanup": "explore_cleanup_minutes",
|
||
"download_gc": "download_gc_minutes",
|
||
"plex_sync": "plex_sync_interval_min",
|
||
"plex_watch_sync": "plex_watch_sync_interval_min",
|
||
"plex_watch_reconcile": "plex_watch_reconcile_interval_min",
|
||
"tmdb_enrich": "tmdb_enrich_interval_min",
|
||
"people_enrich": "people_enrich_interval_min",
|
||
"audit_gc": "audit_gc_minutes",
|
||
"log_gc": "log_gc_minutes",
|
||
}
|
||
# Sane bounds for an admin-set interval (minutes).
|
||
MIN_INTERVAL = 1
|
||
MAX_INTERVAL = 1440 # one day
|
||
|
||
|
||
def interval_key(job: str) -> str:
|
||
return f"job_interval_{job}"
|
||
|
||
|
||
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
||
# off env into the admin UI; the admin page renders it automatically from this list. A group's keys
|
||
# stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS.
|
||
SPECS: tuple[ConfigSpec, ...] = (
|
||
# --- Access / registration ---
|
||
ConfigSpec("allow_registration", "bool", "access", "allow_registration", applies=LIVE),
|
||
# --- Email / SMTP (one logical group; the password is the only secret) ---
|
||
ConfigSpec("smtp_host", "str", "email", "smtp_host", applies=LIVE),
|
||
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535, applies=LIVE),
|
||
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True, applies=LIVE),
|
||
ConfigSpec("smtp_from", "str", "email", "smtp_from", applies=LIVE),
|
||
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True, applies=LIVE),
|
||
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True, applies=LIVE),
|
||
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
||
ConfigSpec(
|
||
"youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True, applies=LIVE
|
||
),
|
||
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
||
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True, applies=LIVE),
|
||
ConfigSpec(
|
||
"google_client_secret", "str", "google", "google_client_secret", secret=True, applies=LIVE
|
||
),
|
||
# --- Quota (shared YouTube Data API daily budget) ---
|
||
ConfigSpec(
|
||
"quota_daily_budget",
|
||
"int",
|
||
"quota",
|
||
"quota_daily_budget",
|
||
min=1,
|
||
max=1_000_000,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"backfill_quota_reserve",
|
||
"int",
|
||
"quota",
|
||
"backfill_quota_reserve",
|
||
min=0,
|
||
max=1_000_000,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"search_daily_limit_per_user",
|
||
"int",
|
||
"quota",
|
||
"search_daily_limit_per_user",
|
||
min=0,
|
||
max=100_000,
|
||
applies=LIVE,
|
||
),
|
||
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
|
||
ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE),
|
||
ConfigSpec(
|
||
"quota_retention_days",
|
||
"int",
|
||
"quota",
|
||
"quota_retention_days",
|
||
min=0,
|
||
max=3650,
|
||
applies=LIVE,
|
||
),
|
||
# --- Backfill (recent-first first pass per channel) ---
|
||
ConfigSpec(
|
||
"backfill_recent_max_videos",
|
||
"int",
|
||
"backfill",
|
||
"backfill_recent_max_videos",
|
||
min=1,
|
||
max=100_000,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"backfill_recent_max_days",
|
||
"int",
|
||
"backfill",
|
||
"backfill_recent_max_days",
|
||
min=1,
|
||
max=36_500,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168, applies=LIVE
|
||
),
|
||
# --- Shorts probe ---
|
||
ConfigSpec(
|
||
"shorts_probe_max_seconds",
|
||
"int",
|
||
"shorts",
|
||
"shorts_probe_max_seconds",
|
||
min=1,
|
||
max=3_600,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000, applies=LIVE
|
||
),
|
||
# --- Batch sizes ---
|
||
ConfigSpec(
|
||
"enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50, applies=LIVE
|
||
),
|
||
ConfigSpec(
|
||
"autotag_title_sample",
|
||
"int",
|
||
"batch",
|
||
"autotag_title_sample",
|
||
min=1,
|
||
max=1_000,
|
||
applies=LIVE,
|
||
),
|
||
# --- Maintenance (rolling re-validation of already-known videos) ---
|
||
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
|
||
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
|
||
ConfigSpec(
|
||
"maintenance_revalidate_batch",
|
||
"int",
|
||
"maintenance",
|
||
"maintenance_revalidate_batch",
|
||
min=100,
|
||
max=100_000,
|
||
applies=LIVE,
|
||
),
|
||
# --- Discovery cleanup (search + explore ephemeral content) ---
|
||
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
|
||
ConfigSpec(
|
||
"explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650, applies=LIVE
|
||
),
|
||
# Days an un-kept live-search result video is kept before the cleanup job removes it.
|
||
ConfigSpec(
|
||
"search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650, applies=LIVE
|
||
),
|
||
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
|
||
ConfigSpec(
|
||
"download_total_max_bytes",
|
||
"int",
|
||
"downloads",
|
||
"download_total_max_bytes",
|
||
min=0,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"download_default_max_bytes",
|
||
"int",
|
||
"downloads",
|
||
"download_default_max_bytes",
|
||
min=0,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"download_default_max_concurrent",
|
||
"int",
|
||
"downloads",
|
||
"download_default_max_concurrent",
|
||
min=1,
|
||
max=20,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"download_default_max_jobs",
|
||
"int",
|
||
"downloads",
|
||
"download_default_max_jobs",
|
||
min=1,
|
||
max=100_000,
|
||
applies=LIVE,
|
||
),
|
||
# min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as
|
||
# `if cap and size > cap`, so a falsy value disabled it). Now the route passes this number
|
||
# straight to `capped_stream_to`, where 0 would refuse every upload instead — the opposite
|
||
# failure, and the reason `config.download_upload_max_bytes` carries its own `ge=1`. Either way
|
||
# a stored 0 is wrong, so it is floored at 1; the overall footprint is separately bounded by the
|
||
# byte quota.
|
||
# `max` is not cosmetic: `bodylimit.ROUTE_MAX_BODY_BYTES` refuses an upload body over exactly
|
||
# this number at the edge, before the route's own check ever sees it. An admin allowed to raise
|
||
# the knob past the edge would get 413s from a layer the admin page never mentions, so the two
|
||
# are pinned together here and `tests/test_body_limit.py` holds them that way. Below the edge is
|
||
# the useful direction and the one the knob exists for: a lowered knob is what makes the route's
|
||
# own 422 — the one carrying this figure — the answer the user gets. `_coerce` also
|
||
# REJECTS a stored value outside these bounds on read (falling back to the env default, with a
|
||
# warning) — neither bound has always been what it is now, so a legal-when-written override can
|
||
# be sitting in the database already.
|
||
ConfigSpec(
|
||
"download_upload_max_bytes",
|
||
"int",
|
||
"downloads",
|
||
"download_upload_max_bytes",
|
||
min=1,
|
||
max=settings.download_upload_max_bytes,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"download_retention_days",
|
||
"int",
|
||
"downloads",
|
||
"download_retention_days",
|
||
min=0,
|
||
max=3_650,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec(
|
||
"download_gc_grace_days",
|
||
"int",
|
||
"downloads",
|
||
"download_gc_grace_days",
|
||
min=0,
|
||
max=3_650,
|
||
applies=LIVE,
|
||
),
|
||
ConfigSpec("download_layout", "choice", "downloads", "download_layout", applies=LIVE),
|
||
ConfigSpec(
|
||
"download_worker_concurrency",
|
||
"int",
|
||
"downloads",
|
||
"download_worker_concurrency",
|
||
min=1,
|
||
max=16,
|
||
applies=LIVE,
|
||
),
|
||
# --- Audit log ---
|
||
# Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever).
|
||
ConfigSpec(
|
||
"audit_retention_days",
|
||
"int",
|
||
"audit",
|
||
"audit_retention_days",
|
||
min=0,
|
||
max=3650,
|
||
applies=LIVE,
|
||
),
|
||
# Days the job_runs table keeps a finished background run before the audit_gc job prunes it.
|
||
ConfigSpec(
|
||
"job_run_retention_days",
|
||
"int",
|
||
"audit",
|
||
"job_run_retention_days",
|
||
min=0,
|
||
max=3650,
|
||
applies=LIVE,
|
||
),
|
||
# --- Diagnostics ---
|
||
# Runtime log level for the API process. Applied without a restart — see routes/config.py and
|
||
# main.py's lifespan.
|
||
ConfigSpec("log_level", "choice", "diagnostics", "log_level", applies=LIVE),
|
||
# Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs
|
||
# carry account-linked request history, and the privacy policy names a limit.
|
||
ConfigSpec(
|
||
"log_retention_days",
|
||
"int",
|
||
"diagnostics",
|
||
"log_retention_days",
|
||
min=1,
|
||
max=90,
|
||
applies=LIVE,
|
||
),
|
||
# --- Plex integration (optional; local-file playback + Plex metadata) ---
|
||
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled", applies=LIVE),
|
||
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url", applies=LIVE),
|
||
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True, applies=LIVE),
|
||
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map", applies=LIVE),
|
||
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries", applies=LIVE),
|
||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map", applies=LIVE),
|
||
ConfigSpec(
|
||
"plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16, applies=LIVE
|
||
),
|
||
ConfigSpec(
|
||
"plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64, applies=LIVE
|
||
),
|
||
# Transcode re-encode tuning. The libx264 knobs apply only to the software fallback.
|
||
# crf 0–51 (x264 range); max_height 0 = source.
|
||
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw", applies=LIVE),
|
||
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset", applies=LIVE),
|
||
ConfigSpec(
|
||
"plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51, applies=LIVE
|
||
),
|
||
ConfigSpec(
|
||
"plex_transcode_max_height",
|
||
"int",
|
||
"plex",
|
||
"plex_transcode_max_height",
|
||
min=0,
|
||
max=4320,
|
||
applies=LIVE,
|
||
),
|
||
# Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play).
|
||
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm", applies=LIVE),
|
||
# --- TMDB enrichment (optional) — the API key is the only secret ---
|
||
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True, applies=LIVE),
|
||
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language", applies=LIVE),
|
||
# --- Bug reports: the Gitea issue tracker a debug session's issue goes to ---
|
||
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url", applies=LIVE),
|
||
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo", applies=LIVE),
|
||
ConfigSpec(
|
||
"issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True, applies=LIVE
|
||
),
|
||
# --- Scheduler: how often each interval job runs (minutes), one key per job ---
|
||
*(
|
||
ConfigSpec(
|
||
interval_key(job),
|
||
"int",
|
||
"scheduler",
|
||
attr,
|
||
min=MIN_INTERVAL,
|
||
max=MAX_INTERVAL,
|
||
applies=LIVE,
|
||
)
|
||
for job, attr in JOB_INTERVAL_DEFAULTS.items()
|
||
),
|
||
)
|
||
|
||
# The admin page's sections (the owner's pick): sixteen groups as tabs wrapped to three rows at 133%
|
||
# zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in
|
||
# exactly one section, or is edited on its own page (test-held), so a new group cannot fall off.
|
||
SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||
("access", ("access", "google", "email")),
|
||
("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")),
|
||
("media", ("downloads", "plex", "tmdb")),
|
||
("system", ("maintenance", "audit", "diagnostics", "issues")),
|
||
)
|
||
# Groups another admin page edits, so the Configuration page neither lists nor writes them: an
|
||
# interval is changed on the Scheduler page, which also reschedules the running job — a write that
|
||
# skipped it would leave the job on its old period while the page showed the new one.
|
||
OWN_PAGE_GROUPS: tuple[str, ...] = ("scheduler",)
|
||
|
||
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
||
|
||
|
||
def spec(key: str) -> ConfigSpec | None:
|
||
return _BY_KEY.get(key)
|
||
|
||
|
||
def config_page_spec(key: str) -> ConfigSpec | None:
|
||
"""`spec`, for the Configuration page: None for a key edited on its own page as well."""
|
||
s = _BY_KEY.get(key)
|
||
return None if s is None or s.group in OWN_PAGE_GROUPS else s
|
||
|
||
|
||
def secrets_manageable() -> bool:
|
||
"""Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY)."""
|
||
return bool(settings.token_encryption_key)
|
||
|
||
|
||
def _default(s: ConfigSpec):
|
||
return getattr(settings, s.default_attr)
|
||
|
||
|
||
class InvalidSetting(ValueError):
|
||
"""A value `set_value` refuses, as a reason the UI translates (`errors.refused.setting_<reason>`)
|
||
plus what the sentence needs — the English message is for logs only. A ValueError still, so a
|
||
caller that only needs "refused or not" keeps working."""
|
||
|
||
def __init__(self, key: str, reason: str, **params: object):
|
||
super().__init__(f"{key}: {reason} {params}")
|
||
self.key = key
|
||
self.reason = reason
|
||
self.params = params
|
||
|
||
|
||
def _choice(s: ConfigSpec, raw: object) -> str:
|
||
"""The canonical spelling of `raw` among the key's choices (any letter case), or InvalidSetting."""
|
||
option = canonical_choice(s.choices, raw)
|
||
if option is None:
|
||
raise InvalidSetting(s.key, "not_a_choice", choices=", ".join(s.choices))
|
||
return option
|
||
|
||
|
||
class OutOfRange(ValueError):
|
||
"""A stored int override that no longer satisfies its spec's bounds."""
|
||
|
||
|
||
def _coerce(s: ConfigSpec, raw: str):
|
||
if s.type == "int":
|
||
# Bounds are checked on READ, not only in `set_value`. Bounds get ADDED to a spec
|
||
# (download_upload_max_bytes gained a `max` when SEC-20 started enforcing the same number at
|
||
# the edge; its `min` went 0 → 1) and env defaults get LOWERED, and either leaves
|
||
# a stored value that was legal when it was written. Serving it is how one layer ends up
|
||
# acting on a number another layer refuses, with the admin page showing a value outside the
|
||
# range it renders.
|
||
#
|
||
# It raises rather than CLAMPING, and that distinction cost a round: clamping is a silent
|
||
# rewrite, and silently rewriting is exactly what this module refuses to do elsewhere. The
|
||
# case that settles it is a stored 0 for download_upload_max_bytes — legal when `min` was 0,
|
||
# and meaning "no per-file cap" to the route that reads it. Clamped to `min=1` it becomes a
|
||
# ONE-BYTE cap and uploads stop working with nothing in the log. Raising drops it into the
|
||
# same fallback the uncoercible path uses: the env default, and one warning.
|
||
v = int(raw)
|
||
if (s.min is not None and v < s.min) or (s.max is not None and v > s.max):
|
||
raise OutOfRange(f"{v} is outside [{s.min}, {s.max}]")
|
||
return v
|
||
if s.type == "bool":
|
||
return raw == "true"
|
||
if s.type == "choice":
|
||
# A stored value that is no longer a choice (the list shrank) reads as the default, like a
|
||
# non-numeric int override does — see get().
|
||
return _choice(s, raw)
|
||
return raw
|
||
|
||
|
||
def _raw_override(db: Session, key: str) -> str | None:
|
||
"""Stored (decrypted) override string for `key`, or None if no override is set.
|
||
|
||
A decrypt failure — a rotated or removed ``TOKEN_ENCRYPTION_KEY`` vs the key the value was written
|
||
with — must NOT crash the caller. `_raw_override` sits on hot paths (`/api/me` via `google_enabled`,
|
||
login, every YouTube scheduler job, account purge), so a raw ``InvalidToken`` here would 500 the
|
||
whole app and the SPA wouldn't even boot. Fall back to the env/config default instead, and warn once
|
||
per key so the operator gets a real signal without a per-request traceback. Recovery is documented in
|
||
docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2)
|
||
"""
|
||
return _raw_of(db.get(SystemConfig, key), key)
|
||
|
||
|
||
def _raw_of(row: SystemConfig | None, key: str) -> str | None:
|
||
"""`_raw_override` over a row already read (describe() reads them all in one query)."""
|
||
if row is None or row.value is None:
|
||
return None
|
||
if not row.encrypted:
|
||
return row.value
|
||
try:
|
||
return security.decrypt(row.value)
|
||
except Exception: # noqa: BLE001 — Fernet InvalidToken / malformed ciphertext (key mismatch)
|
||
if key not in _decrypt_warned:
|
||
_decrypt_warned.add(key)
|
||
log.warning(
|
||
"config %r could not be decrypted (TOKEN_ENCRYPTION_KEY changed or missing?) — falling "
|
||
"back to the env/config default; the key must not change once secrets are stored",
|
||
key,
|
||
)
|
||
return None
|
||
|
||
|
||
def get(db: Session, key: str):
|
||
"""Effective value: the DB override (typed) if set, else the env/config default."""
|
||
return _effective(_BY_KEY[key], _raw_override(db, key))
|
||
|
||
|
||
def _effective(s: ConfigSpec, raw: str | None):
|
||
"""`get` over an override already read."""
|
||
if raw is None:
|
||
return _default(s)
|
||
if raw == "" and not s.allow_empty:
|
||
# For most keys an empty override means "fall back to the env default". Keys that opt
|
||
# into allow_empty (e.g. smtp_user, youtube_api_proxy) treat "" as a real value — so an
|
||
# admin can blank one to DISABLE it rather than being stuck with the env default.
|
||
return _default(s)
|
||
try:
|
||
return _coerce(s, raw)
|
||
except (TypeError, ValueError) as exc:
|
||
# The admin set this in the UI and it is quietly not in force: the app runs on the env
|
||
# default while the settings page goes on showing the value that was saved. An OutOfRange is
|
||
# the same situation arrived at from the other direction — the value was fine when it was
|
||
# written and the spec moved under it — so it gets the same treatment and its own code.
|
||
event = (
|
||
"sysconfig.override_out_of_range"
|
||
if isinstance(exc, OutOfRange)
|
||
else "sysconfig.override_uncoercible"
|
||
)
|
||
if (s.key, raw) not in _coerce_warned:
|
||
_coerce_warned.add((s.key, raw))
|
||
log.warning(
|
||
"a stored setting is not usable; the default is used instead",
|
||
extra=logs.event(event, key=s.key, reason=type(exc).__name__),
|
||
)
|
||
return _default(s)
|
||
|
||
|
||
def get_str(db: Session, key: str) -> str:
|
||
v = get(db, key)
|
||
return "" if v is None else str(v)
|
||
|
||
|
||
def get_int(db: Session, key: str) -> int:
|
||
return int(get(db, key))
|
||
|
||
|
||
def get_bool(db: Session, key: str) -> bool:
|
||
return bool(get(db, key))
|
||
|
||
|
||
def set_value(db: Session, key: str, value) -> None:
|
||
"""Validate, coerce, and stage a DB override for `key` (secrets are encrypted). Flushes but does
|
||
NOT commit — the caller owns the transaction, so the change lands together with its audit row
|
||
(B-core-10). The staged value is visible to a same-session `get()` before the commit."""
|
||
s = _BY_KEY[key]
|
||
if s.type == "int":
|
||
try:
|
||
v = int(value)
|
||
except (TypeError, ValueError):
|
||
raise InvalidSetting(key, "not_a_number") from None
|
||
if s.min is not None and v < s.min:
|
||
raise InvalidSetting(key, "too_low", min=s.min)
|
||
if s.max is not None and v > s.max:
|
||
raise InvalidSetting(key, "too_high", max=s.max)
|
||
raw = str(v)
|
||
elif s.type == "bool":
|
||
raw = "true" if value else "false"
|
||
elif s.type == "choice":
|
||
raw = _choice(s, value)
|
||
else:
|
||
raw = "" if value is None else str(value)
|
||
|
||
stored, encrypted = raw, False
|
||
if s.secret:
|
||
if not secrets_manageable():
|
||
raise InvalidSetting(key, "no_encryption_key")
|
||
stored, encrypted = security.encrypt(raw) or "", True
|
||
|
||
row = db.get(SystemConfig, key)
|
||
if row is None:
|
||
row = SystemConfig(key=key)
|
||
db.add(row)
|
||
row.value = stored
|
||
row.encrypted = encrypted
|
||
db.flush()
|
||
|
||
|
||
def reset(db: Session, key: str) -> None:
|
||
"""Remove the DB override for `key` (fall back to the env/config default). Flushes but does NOT
|
||
commit — the caller owns the transaction (B-core-10)."""
|
||
row = db.get(SystemConfig, key)
|
||
if row is not None:
|
||
db.delete(row)
|
||
db.flush()
|
||
|
||
|
||
def describe(db: Session) -> dict:
|
||
"""Admin-UI metadata + current values, grouped — the keys of the Configuration page, not those
|
||
edited on their own page. Secret values are never echoed — only whether an override is set and
|
||
whether an env default exists."""
|
||
groups: dict[str, list[dict]] = {}
|
||
# Every override in one query: a db.get per key was one round trip each (two for a key with no
|
||
# row, which the session cannot cache), ~40 ms of the page's ~55 on prod.
|
||
rows = {r.key: r for r in db.scalars(select(SystemConfig))}
|
||
for s in SPECS:
|
||
if s.group in OWN_PAGE_GROUPS:
|
||
continue
|
||
row = rows.get(s.key)
|
||
is_set = row is not None
|
||
item: dict = {
|
||
"key": s.key,
|
||
"type": s.type,
|
||
"group": s.group,
|
||
"secret": s.secret,
|
||
"min": s.min,
|
||
"max": s.max,
|
||
"is_set": is_set, # an override row exists
|
||
"allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row
|
||
"choices": list(s.choices), # the picker's options; [] for any other type
|
||
}
|
||
if s.secret:
|
||
item["default_is_set"] = bool(_default(s))
|
||
else:
|
||
item["value"] = _effective(s, _raw_of(row, s.key))
|
||
item["default"] = _default(s)
|
||
groups.setdefault(s.group, []).append(item)
|
||
sections = [
|
||
{"id": sid, "groups": [g for g in members if g in groups]} for sid, members in SECTIONS
|
||
]
|
||
return {"groups": groups, "sections": sections, "secrets_manageable": secrets_manageable()}
|