peter
be156fc52b
- NavRailGroup: one "Admin" rail row opening a panel of the admin pages (links, so a middle-click still opens a tab); the rail fits 133% zoom without a hidden scroll - sysconfig.SECTIONS orders the config groups under four headings (every group in exactly one, test-held); the page shows them as Settings' side list (Tabs `group` headings); the lone registration group reads "Registration" under "Access" - StatTiles / StatTile / StatBars: the Statistics overview's figures as tiles, the per-operation quota use as horizontal bars (a zero draws none)
446 lines
21 KiB
Python
446 lines
21 KiB
Python
"""Admin-editable system configuration: a DB-override layer over the env/config defaults.
|
||
|
||
The registry (SPECS) is the single source of truth for which config keys can be overridden
|
||
from the admin Configuration page, plus each key's type, admin-UI group, env/config default,
|
||
bounds, and whether it's a secret. The resolver returns the DB override if one is set, else
|
||
``settings.<default_attr>`` (same DB-override-or-default pattern as app.state). Secrets are
|
||
stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secrets can't be
|
||
stored in the DB and stay env-only).
|
||
"""
|
||
|
||
from dataclasses import dataclass
|
||
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app import logs, security
|
||
from app.config import Settings, canonical_choice, choices_of, settings
|
||
from app.models import SystemConfig
|
||
|
||
log = logs.get_logger(__name__)
|
||
# Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per
|
||
# key, not once per request (bounded by the number of secret config keys).
|
||
_decrypt_warned: set[str] = set()
|
||
# The same for a stored value that no longer coerces (a choice list that shrank in a release): it
|
||
# fails on EVERY read of a hot knob until someone saves it again. Keyed with the value, so saving a
|
||
# different bad value is reported afresh (bounded by the keys times the values ever saved).
|
||
_coerce_warned: set[tuple[str, str]] = set()
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ConfigSpec:
|
||
key: str
|
||
type: str # "int" | "str" | "bool" | "choice"
|
||
group: str # admin-UI grouping (logically related keys stay together)
|
||
default_attr: str # attribute on `settings` holding the env/config default
|
||
secret: bool = False
|
||
min: int | None = None
|
||
max: int | None = None
|
||
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
|
||
|
||
@property
|
||
def choices(self) -> tuple[str, ...]:
|
||
"""A choice key's allowed values, read from its setting's Literal type in config.py — the one
|
||
place the list is written. The admin page renders them as a picker."""
|
||
return choices_of(Settings.model_fields[self.default_attr].annotation)
|
||
|
||
|
||
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
||
# off env into the admin UI; the admin page renders it automatically from this list. A group's keys
|
||
# stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS.
|
||
SPECS: tuple[ConfigSpec, ...] = (
|
||
# --- Access / registration ---
|
||
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
||
# --- Email / SMTP (one logical group; the password is the only secret) ---
|
||
ConfigSpec("smtp_host", "str", "email", "smtp_host"),
|
||
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
|
||
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True),
|
||
ConfigSpec("smtp_from", "str", "email", "smtp_from"),
|
||
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
|
||
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
|
||
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
||
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
|
||
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
||
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
|
||
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
|
||
# --- Quota (shared YouTube Data API daily budget) ---
|
||
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
|
||
ConfigSpec(
|
||
"backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000
|
||
),
|
||
ConfigSpec(
|
||
"search_daily_limit_per_user",
|
||
"int",
|
||
"quota",
|
||
"search_daily_limit_per_user",
|
||
min=0,
|
||
max=100_000,
|
||
),
|
||
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
|
||
ConfigSpec("search_source", "choice", "quota", "search_source"),
|
||
# --- Backfill (recent-first first pass per channel) ---
|
||
ConfigSpec(
|
||
"backfill_recent_max_videos",
|
||
"int",
|
||
"backfill",
|
||
"backfill_recent_max_videos",
|
||
min=1,
|
||
max=100_000,
|
||
),
|
||
ConfigSpec(
|
||
"backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500
|
||
),
|
||
ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168),
|
||
# --- Shorts probe ---
|
||
ConfigSpec(
|
||
"shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600
|
||
),
|
||
ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
|
||
# --- Batch sizes ---
|
||
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
|
||
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
|
||
# --- Maintenance (rolling re-validation of already-known videos) ---
|
||
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
|
||
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
|
||
ConfigSpec(
|
||
"maintenance_revalidate_batch",
|
||
"int",
|
||
"maintenance",
|
||
"maintenance_revalidate_batch",
|
||
min=100,
|
||
max=100_000,
|
||
),
|
||
# --- Discovery cleanup (search + explore ephemeral content) ---
|
||
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
|
||
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
|
||
# Days an un-kept live-search result video is kept before the cleanup job removes it.
|
||
ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650),
|
||
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
|
||
ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0),
|
||
ConfigSpec(
|
||
"download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0
|
||
),
|
||
ConfigSpec(
|
||
"download_default_max_concurrent",
|
||
"int",
|
||
"downloads",
|
||
"download_default_max_concurrent",
|
||
min=1,
|
||
max=20,
|
||
),
|
||
ConfigSpec(
|
||
"download_default_max_jobs",
|
||
"int",
|
||
"downloads",
|
||
"download_default_max_jobs",
|
||
min=1,
|
||
max=100_000,
|
||
),
|
||
# min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as
|
||
# `if cap and size > cap`, so a falsy value disabled it). Since P1 the route passes this number
|
||
# straight to `capped_stream_to`, where 0 would refuse every upload instead — the opposite
|
||
# failure, and the reason `config.download_upload_max_bytes` carries its own `ge=1`. Either way
|
||
# a stored 0 is wrong, so it is floored at 1; the overall footprint is separately bounded by the
|
||
# byte quota.
|
||
# `max` is not cosmetic: `bodylimit.ROUTE_MAX_BODY_BYTES` refuses an upload body over exactly
|
||
# this number at the edge, before the route's own check ever sees it. An admin allowed to raise
|
||
# the knob past the edge would get 413s from a layer the admin page never mentions, so the two
|
||
# are pinned together here and `tests/test_body_limit.py` holds them that way. Below the edge is
|
||
# the useful direction and the one the knob exists for: a lowered knob is what makes the route's
|
||
# own 422 — the one carrying this figure — the answer the user gets. `_coerce` also
|
||
# REJECTS a stored value outside these bounds on read (falling back to the env default, with a
|
||
# warning) — neither bound has always been what it is now, so a legal-when-written override can
|
||
# be sitting in the database already.
|
||
ConfigSpec(
|
||
"download_upload_max_bytes",
|
||
"int",
|
||
"downloads",
|
||
"download_upload_max_bytes",
|
||
min=1,
|
||
max=settings.download_upload_max_bytes,
|
||
),
|
||
ConfigSpec(
|
||
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
|
||
),
|
||
ConfigSpec(
|
||
"download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650
|
||
),
|
||
ConfigSpec("download_layout", "choice", "downloads", "download_layout"),
|
||
ConfigSpec(
|
||
"download_worker_concurrency",
|
||
"int",
|
||
"downloads",
|
||
"download_worker_concurrency",
|
||
min=1,
|
||
max=16,
|
||
),
|
||
ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"),
|
||
# --- Audit log ---
|
||
# Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever).
|
||
ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650),
|
||
# --- Diagnostics (E22) ---
|
||
# Runtime log level for the API process. Applied without a restart — see routes/config.py and
|
||
# main.py's lifespan.
|
||
ConfigSpec("log_level", "choice", "diagnostics", "log_level"),
|
||
# Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs
|
||
# carry account-linked request history, and the privacy policy names a limit.
|
||
ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90),
|
||
# --- Plex integration (optional; local-file playback + Plex metadata) ---
|
||
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"),
|
||
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"),
|
||
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
|
||
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
|
||
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
|
||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
|
||
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||
ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64),
|
||
# Transcode re-encode tuning (E6 S1). The libx264 knobs apply only to the software fallback.
|
||
# crf 0–51 (x264 range); max_height 0 = source.
|
||
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"),
|
||
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"),
|
||
ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51),
|
||
ConfigSpec(
|
||
"plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320
|
||
),
|
||
# Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play).
|
||
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"),
|
||
# NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job
|
||
# (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS.
|
||
# --- TMDB enrichment (E5 S2; optional) — the API key is the only secret ---
|
||
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True),
|
||
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"),
|
||
# --- Bug reports (E16): the Gitea issue tracker a debug session's issue goes to ---
|
||
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"),
|
||
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"),
|
||
ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True),
|
||
# (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch
|
||
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
|
||
)
|
||
|
||
# The admin page's sections (E31, the owner's pick): sixteen groups as tabs wrapped to three rows at 133%
|
||
# zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in
|
||
# exactly one section (test-held), so a new group cannot fall off the page.
|
||
SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||
("access", ("access", "google", "email")),
|
||
("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")),
|
||
("media", ("downloads", "plex", "tmdb")),
|
||
("system", ("maintenance", "audit", "diagnostics", "issues")),
|
||
)
|
||
|
||
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
||
|
||
|
||
def spec(key: str) -> ConfigSpec | None:
|
||
return _BY_KEY.get(key)
|
||
|
||
|
||
def secrets_manageable() -> bool:
|
||
"""Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY)."""
|
||
return bool(settings.token_encryption_key)
|
||
|
||
|
||
def _default(s: ConfigSpec):
|
||
return getattr(settings, s.default_attr)
|
||
|
||
|
||
class InvalidSetting(ValueError):
|
||
"""A value `set_value` refuses, as a reason the UI translates (`errors.refused.setting_<reason>`)
|
||
plus what the sentence needs — the English message is for logs only. A ValueError still, so a
|
||
caller that only needs "refused or not" keeps working."""
|
||
|
||
def __init__(self, key: str, reason: str, **params: object):
|
||
super().__init__(f"{key}: {reason} {params}")
|
||
self.key = key
|
||
self.reason = reason
|
||
self.params = params
|
||
|
||
|
||
def _choice(s: ConfigSpec, raw: object) -> str:
|
||
"""The canonical spelling of `raw` among the key's choices (any letter case), or InvalidSetting."""
|
||
option = canonical_choice(s.choices, raw)
|
||
if option is None:
|
||
raise InvalidSetting(s.key, "not_a_choice", choices=", ".join(s.choices))
|
||
return option
|
||
|
||
|
||
class OutOfRange(ValueError):
|
||
"""A stored int override that no longer satisfies its spec's bounds."""
|
||
|
||
|
||
def _coerce(s: ConfigSpec, raw: str):
|
||
if s.type == "int":
|
||
# Bounds are checked on READ, not only in `set_value`. Bounds get ADDED to a spec
|
||
# (download_upload_max_bytes gained a `max` when SEC-20 started enforcing the same number at
|
||
# the edge; its `min` went 0 → 1 during E7) and env defaults get LOWERED, and either leaves
|
||
# a stored value that was legal when it was written. Serving it is how one layer ends up
|
||
# acting on a number another layer refuses, with the admin page showing a value outside the
|
||
# range it renders.
|
||
#
|
||
# It raises rather than CLAMPING, and that distinction cost a round: clamping is a silent
|
||
# rewrite, and silently rewriting is exactly what this module refuses to do elsewhere. The
|
||
# case that settles it is a stored 0 for download_upload_max_bytes — legal when `min` was 0,
|
||
# and meaning "no per-file cap" to the route that reads it. Clamped to `min=1` it becomes a
|
||
# ONE-BYTE cap and uploads stop working with nothing in the log. Raising drops it into the
|
||
# same fallback the uncoercible path uses: the env default, and one warning.
|
||
v = int(raw)
|
||
if (s.min is not None and v < s.min) or (s.max is not None and v > s.max):
|
||
raise OutOfRange(f"{v} is outside [{s.min}, {s.max}]")
|
||
return v
|
||
if s.type == "bool":
|
||
return raw == "true"
|
||
if s.type == "choice":
|
||
# A stored value that is no longer a choice (the list shrank) reads as the default, like a
|
||
# non-numeric int override does — see get().
|
||
return _choice(s, raw)
|
||
return raw
|
||
|
||
|
||
def _raw_override(db: Session, key: str) -> str | None:
|
||
"""Stored (decrypted) override string for `key`, or None if no override is set.
|
||
|
||
A decrypt failure — a rotated or removed ``TOKEN_ENCRYPTION_KEY`` vs the key the value was written
|
||
with — must NOT crash the caller. `_raw_override` sits on hot paths (`/api/me` via `google_enabled`,
|
||
login, every YouTube scheduler job, account purge), so a raw ``InvalidToken`` here would 500 the
|
||
whole app and the SPA wouldn't even boot. Fall back to the env/config default instead, and warn once
|
||
per key so the operator gets a real signal without a per-request traceback. Recovery is documented in
|
||
docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2)
|
||
"""
|
||
row = db.get(SystemConfig, key)
|
||
if row is None or row.value is None:
|
||
return None
|
||
if not row.encrypted:
|
||
return row.value
|
||
try:
|
||
return security.decrypt(row.value)
|
||
except Exception: # noqa: BLE001 — Fernet InvalidToken / malformed ciphertext (key mismatch)
|
||
if key not in _decrypt_warned:
|
||
_decrypt_warned.add(key)
|
||
log.warning(
|
||
"config %r could not be decrypted (TOKEN_ENCRYPTION_KEY changed or missing?) — falling "
|
||
"back to the env/config default; the key must not change once secrets are stored",
|
||
key,
|
||
)
|
||
return None
|
||
|
||
|
||
def get(db: Session, key: str):
|
||
"""Effective value: the DB override (typed) if set, else the env/config default."""
|
||
s = _BY_KEY[key]
|
||
raw = _raw_override(db, key)
|
||
if raw is None:
|
||
return _default(s)
|
||
if raw == "" and not s.allow_empty:
|
||
# For most keys an empty override means "fall back to the env default". Keys that opt
|
||
# into allow_empty (e.g. smtp_user, youtube_api_proxy) treat "" as a real value — so an
|
||
# admin can blank one to DISABLE it rather than being stuck with the env default.
|
||
return _default(s)
|
||
try:
|
||
return _coerce(s, raw)
|
||
except (TypeError, ValueError) as exc:
|
||
# The admin set this in the UI and it is quietly not in force: the app runs on the env
|
||
# default while the settings page goes on showing the value that was saved. An OutOfRange is
|
||
# the same situation arrived at from the other direction — the value was fine when it was
|
||
# written and the spec moved under it — so it gets the same treatment and its own code.
|
||
event = (
|
||
"sysconfig.override_out_of_range"
|
||
if isinstance(exc, OutOfRange)
|
||
else "sysconfig.override_uncoercible"
|
||
)
|
||
if (s.key, raw) not in _coerce_warned:
|
||
_coerce_warned.add((s.key, raw))
|
||
log.warning(
|
||
"a stored setting is not usable; the default is used instead",
|
||
extra=logs.event(event, key=s.key, reason=type(exc).__name__),
|
||
)
|
||
return _default(s)
|
||
|
||
|
||
def get_str(db: Session, key: str) -> str:
|
||
v = get(db, key)
|
||
return "" if v is None else str(v)
|
||
|
||
|
||
def get_int(db: Session, key: str) -> int:
|
||
return int(get(db, key))
|
||
|
||
|
||
def get_bool(db: Session, key: str) -> bool:
|
||
return bool(get(db, key))
|
||
|
||
|
||
def set_value(db: Session, key: str, value) -> None:
|
||
"""Validate, coerce, and stage a DB override for `key` (secrets are encrypted). Flushes but does
|
||
NOT commit — the caller owns the transaction, so the change lands together with its audit row
|
||
(B-core-10). The staged value is visible to a same-session `get()` before the commit."""
|
||
s = _BY_KEY[key]
|
||
if s.type == "int":
|
||
try:
|
||
v = int(value)
|
||
except (TypeError, ValueError):
|
||
raise InvalidSetting(key, "not_a_number") from None
|
||
if s.min is not None and v < s.min:
|
||
raise InvalidSetting(key, "too_low", min=s.min)
|
||
if s.max is not None and v > s.max:
|
||
raise InvalidSetting(key, "too_high", max=s.max)
|
||
raw = str(v)
|
||
elif s.type == "bool":
|
||
raw = "true" if value else "false"
|
||
elif s.type == "choice":
|
||
raw = _choice(s, value)
|
||
else:
|
||
raw = "" if value is None else str(value)
|
||
|
||
stored, encrypted = raw, False
|
||
if s.secret:
|
||
if not secrets_manageable():
|
||
raise InvalidSetting(key, "no_encryption_key")
|
||
stored, encrypted = security.encrypt(raw) or "", True
|
||
|
||
row = db.get(SystemConfig, key)
|
||
if row is None:
|
||
row = SystemConfig(key=key)
|
||
db.add(row)
|
||
row.value = stored
|
||
row.encrypted = encrypted
|
||
db.flush()
|
||
|
||
|
||
def reset(db: Session, key: str) -> None:
|
||
"""Remove the DB override for `key` (fall back to the env/config default). Flushes but does NOT
|
||
commit — the caller owns the transaction (B-core-10)."""
|
||
row = db.get(SystemConfig, key)
|
||
if row is not None:
|
||
db.delete(row)
|
||
db.flush()
|
||
|
||
|
||
def describe(db: Session) -> dict:
|
||
"""Admin-UI metadata + current values, grouped. Secret values are never echoed —
|
||
only whether an override is set and whether an env default exists."""
|
||
groups: dict[str, list[dict]] = {}
|
||
for s in SPECS:
|
||
is_set = db.get(SystemConfig, s.key) is not None
|
||
item: dict = {
|
||
"key": s.key,
|
||
"type": s.type,
|
||
"group": s.group,
|
||
"secret": s.secret,
|
||
"min": s.min,
|
||
"max": s.max,
|
||
"is_set": is_set, # an override row exists
|
||
"allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row
|
||
"choices": list(s.choices), # the picker's options; [] for any other type
|
||
}
|
||
if s.secret:
|
||
item["default_is_set"] = bool(_default(s))
|
||
else:
|
||
item["value"] = get(db, s.key)
|
||
item["default"] = _default(s)
|
||
groups.setdefault(s.group, []).append(item)
|
||
sections = [
|
||
{"id": sid, "groups": [g for g in members if g in groups]} for sid, members in SECTIONS
|
||
]
|
||
return {"groups": groups, "sections": sections, "secrets_manageable": secrets_manageable()}
|