"""Admin-editable system configuration: a DB-override layer over the env/config defaults. The registry (SPECS) is the single source of truth for which config keys can be overridden from the admin Configuration page, plus each key's type, admin-UI group, env/config default, bounds, and whether it's a secret. The resolver returns the DB override if one is set, else ``settings.`` (same DB-override-or-default pattern as app.state). Secrets are stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secrets can't be stored in the DB and stay env-only). """ from dataclasses import dataclass from sqlalchemy.orm import Session from app import logs, security from app.config import Settings, canonical_choice, choices_of, settings from app.models import SystemConfig log = logs.get_logger(__name__) # Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per # key, not once per request (bounded by the number of secret config keys). _decrypt_warned: set[str] = set() # The same for a stored value that no longer coerces (a choice list that shrank in a release): it # fails on EVERY read of a hot knob until someone saves it again. Keyed with the value, so saving a # different bad value is reported afresh (bounded by the keys times the values ever saved). _coerce_warned: set[tuple[str, str]] = set() @dataclass(frozen=True) class ConfigSpec: key: str type: str # "int" | "str" | "bool" | "choice" group: str # admin-UI grouping (logically related keys stay together) default_attr: str # attribute on `settings` holding the env/config default secret: bool = False min: int | None = None max: int | None = None allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset" @property def choices(self) -> tuple[str, ...]: """A choice key's allowed values, read from its setting's Literal type in config.py — the one place the list is written. The admin page renders them as a picker.""" return choices_of(Settings.model_fields[self.default_attr].annotation) # Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it # off env into the admin UI; the admin page renders it automatically from this list. A group's keys # stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS. SPECS: tuple[ConfigSpec, ...] = ( # --- Access / registration --- ConfigSpec("allow_registration", "bool", "access", "allow_registration"), # --- Email / SMTP (one logical group; the password is the only secret) --- ConfigSpec("smtp_host", "str", "email", "smtp_host"), ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535), ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True), ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True), # --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the # install wizard / Configuration page so no restart is needed when the creds change. --- ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True), ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True), # --- Quota (shared YouTube Data API daily budget) --- ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), ConfigSpec( "backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000 ), ConfigSpec( "search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000, ), # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). ConfigSpec("search_source", "choice", "quota", "search_source"), # --- Backfill (recent-first first pass per channel) --- ConfigSpec( "backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000, ), ConfigSpec( "backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500 ), ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168), # --- Shorts probe --- ConfigSpec( "shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600 ), ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000), # --- Batch sizes --- ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), # --- Maintenance (rolling re-validation of already-known videos) --- # Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin # knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard. ConfigSpec( "maintenance_revalidate_batch", "int", "maintenance", "maintenance_revalidate_batch", min=100, max=100_000, ), # --- Discovery cleanup (search + explore ephemeral content) --- # Days an explored-but-unsubscribed channel is kept before the cleanup job purges it. ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650), # Days an un-kept live-search result video is kept before the cleanup job removes it. ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650), # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0), ConfigSpec( "download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0 ), ConfigSpec( "download_default_max_concurrent", "int", "downloads", "download_default_max_concurrent", min=1, max=20, ), ConfigSpec( "download_default_max_jobs", "int", "downloads", "download_default_max_jobs", min=1, max=100_000, ), # min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as # `if cap and size > cap`, so a falsy value disabled it). Since P1 the route passes this number # straight to `capped_stream_to`, where 0 would refuse every upload instead — the opposite # failure, and the reason `config.download_upload_max_bytes` carries its own `ge=1`. Either way # a stored 0 is wrong, so it is floored at 1; the overall footprint is separately bounded by the # byte quota. # `max` is not cosmetic: `bodylimit.ROUTE_MAX_BODY_BYTES` refuses an upload body over exactly # this number at the edge, before the route's own check ever sees it. An admin allowed to raise # the knob past the edge would get 413s from a layer the admin page never mentions, so the two # are pinned together here and `tests/test_body_limit.py` holds them that way. Below the edge is # the useful direction and the one the knob exists for: a lowered knob is what makes the route's # own 422 — the one carrying this figure — the answer the user gets. `_coerce` also # REJECTS a stored value outside these bounds on read (falling back to the env default, with a # warning) — neither bound has always been what it is now, so a legal-when-written override can # be sitting in the database already. ConfigSpec( "download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=1, max=settings.download_upload_max_bytes, ), ConfigSpec( "download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650 ), ConfigSpec( "download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650 ), ConfigSpec("download_layout", "choice", "downloads", "download_layout"), ConfigSpec( "download_worker_concurrency", "int", "downloads", "download_worker_concurrency", min=1, max=16, ), ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"), # --- Audit log --- # Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever). ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650), # --- Diagnostics (E22) --- # Runtime log level for the API process. Applied without a restart — see routes/config.py and # main.py's lifespan. ConfigSpec("log_level", "choice", "diagnostics", "log_level"), # Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs # carry account-linked request history, and the privacy policy names a limit. ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90), # --- Plex integration (optional; local-file playback + Plex metadata) --- ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"), ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"), ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True), ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"), ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"), ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"), # (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.) ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16), ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64), # Transcode re-encode tuning (E6 S1). The libx264 knobs apply only to the software fallback. # crf 0–51 (x264 range); max_height 0 = source. ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"), ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"), ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51), ConfigSpec( "plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320 ), # Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play). ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"), # NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job # (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS. # --- TMDB enrichment (E5 S2; optional) — the API key is the only secret --- ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True), ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"), # --- Bug reports (E16): the Gitea issue tracker a debug session's issue goes to --- ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"), ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"), ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True), # (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch # is a config.py default — both stay out of SPECS, like plex_sync_interval_min.) ) # The admin page's sections (E31, the owner's pick): sixteen groups as tabs wrapped to three rows at 133% # zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in # exactly one section (test-held), so a new group cannot fall off the page. SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = ( ("access", ("access", "google", "email")), ("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")), ("media", ("downloads", "plex", "tmdb")), ("system", ("maintenance", "audit", "diagnostics", "issues")), ) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} def spec(key: str) -> ConfigSpec | None: return _BY_KEY.get(key) def secrets_manageable() -> bool: """Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY).""" return bool(settings.token_encryption_key) def _default(s: ConfigSpec): return getattr(settings, s.default_attr) class InvalidSetting(ValueError): """A value `set_value` refuses, as a reason the UI translates (`errors.refused.setting_`) plus what the sentence needs — the English message is for logs only. A ValueError still, so a caller that only needs "refused or not" keeps working.""" def __init__(self, key: str, reason: str, **params: object): super().__init__(f"{key}: {reason} {params}") self.key = key self.reason = reason self.params = params def _choice(s: ConfigSpec, raw: object) -> str: """The canonical spelling of `raw` among the key's choices (any letter case), or InvalidSetting.""" option = canonical_choice(s.choices, raw) if option is None: raise InvalidSetting(s.key, "not_a_choice", choices=", ".join(s.choices)) return option class OutOfRange(ValueError): """A stored int override that no longer satisfies its spec's bounds.""" def _coerce(s: ConfigSpec, raw: str): if s.type == "int": # Bounds are checked on READ, not only in `set_value`. Bounds get ADDED to a spec # (download_upload_max_bytes gained a `max` when SEC-20 started enforcing the same number at # the edge; its `min` went 0 → 1 during E7) and env defaults get LOWERED, and either leaves # a stored value that was legal when it was written. Serving it is how one layer ends up # acting on a number another layer refuses, with the admin page showing a value outside the # range it renders. # # It raises rather than CLAMPING, and that distinction cost a round: clamping is a silent # rewrite, and silently rewriting is exactly what this module refuses to do elsewhere. The # case that settles it is a stored 0 for download_upload_max_bytes — legal when `min` was 0, # and meaning "no per-file cap" to the route that reads it. Clamped to `min=1` it becomes a # ONE-BYTE cap and uploads stop working with nothing in the log. Raising drops it into the # same fallback the uncoercible path uses: the env default, and one warning. v = int(raw) if (s.min is not None and v < s.min) or (s.max is not None and v > s.max): raise OutOfRange(f"{v} is outside [{s.min}, {s.max}]") return v if s.type == "bool": return raw == "true" if s.type == "choice": # A stored value that is no longer a choice (the list shrank) reads as the default, like a # non-numeric int override does — see get(). return _choice(s, raw) return raw def _raw_override(db: Session, key: str) -> str | None: """Stored (decrypted) override string for `key`, or None if no override is set. A decrypt failure — a rotated or removed ``TOKEN_ENCRYPTION_KEY`` vs the key the value was written with — must NOT crash the caller. `_raw_override` sits on hot paths (`/api/me` via `google_enabled`, login, every YouTube scheduler job, account purge), so a raw ``InvalidToken`` here would 500 the whole app and the SPA wouldn't even boot. Fall back to the env/config default instead, and warn once per key so the operator gets a real signal without a per-request traceback. Recovery is documented in docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2) """ row = db.get(SystemConfig, key) if row is None or row.value is None: return None if not row.encrypted: return row.value try: return security.decrypt(row.value) except Exception: # noqa: BLE001 — Fernet InvalidToken / malformed ciphertext (key mismatch) if key not in _decrypt_warned: _decrypt_warned.add(key) log.warning( "config %r could not be decrypted (TOKEN_ENCRYPTION_KEY changed or missing?) — falling " "back to the env/config default; the key must not change once secrets are stored", key, ) return None def get(db: Session, key: str): """Effective value: the DB override (typed) if set, else the env/config default.""" s = _BY_KEY[key] raw = _raw_override(db, key) if raw is None: return _default(s) if raw == "" and not s.allow_empty: # For most keys an empty override means "fall back to the env default". Keys that opt # into allow_empty (e.g. smtp_user, youtube_api_proxy) treat "" as a real value — so an # admin can blank one to DISABLE it rather than being stuck with the env default. return _default(s) try: return _coerce(s, raw) except (TypeError, ValueError) as exc: # The admin set this in the UI and it is quietly not in force: the app runs on the env # default while the settings page goes on showing the value that was saved. An OutOfRange is # the same situation arrived at from the other direction — the value was fine when it was # written and the spec moved under it — so it gets the same treatment and its own code. event = ( "sysconfig.override_out_of_range" if isinstance(exc, OutOfRange) else "sysconfig.override_uncoercible" ) if (s.key, raw) not in _coerce_warned: _coerce_warned.add((s.key, raw)) log.warning( "a stored setting is not usable; the default is used instead", extra=logs.event(event, key=s.key, reason=type(exc).__name__), ) return _default(s) def get_str(db: Session, key: str) -> str: v = get(db, key) return "" if v is None else str(v) def get_int(db: Session, key: str) -> int: return int(get(db, key)) def get_bool(db: Session, key: str) -> bool: return bool(get(db, key)) def set_value(db: Session, key: str, value) -> None: """Validate, coerce, and stage a DB override for `key` (secrets are encrypted). Flushes but does NOT commit — the caller owns the transaction, so the change lands together with its audit row (B-core-10). The staged value is visible to a same-session `get()` before the commit.""" s = _BY_KEY[key] if s.type == "int": try: v = int(value) except (TypeError, ValueError): raise InvalidSetting(key, "not_a_number") from None if s.min is not None and v < s.min: raise InvalidSetting(key, "too_low", min=s.min) if s.max is not None and v > s.max: raise InvalidSetting(key, "too_high", max=s.max) raw = str(v) elif s.type == "bool": raw = "true" if value else "false" elif s.type == "choice": raw = _choice(s, value) else: raw = "" if value is None else str(value) stored, encrypted = raw, False if s.secret: if not secrets_manageable(): raise InvalidSetting(key, "no_encryption_key") stored, encrypted = security.encrypt(raw) or "", True row = db.get(SystemConfig, key) if row is None: row = SystemConfig(key=key) db.add(row) row.value = stored row.encrypted = encrypted db.flush() def reset(db: Session, key: str) -> None: """Remove the DB override for `key` (fall back to the env/config default). Flushes but does NOT commit — the caller owns the transaction (B-core-10).""" row = db.get(SystemConfig, key) if row is not None: db.delete(row) db.flush() def describe(db: Session) -> dict: """Admin-UI metadata + current values, grouped. Secret values are never echoed — only whether an override is set and whether an env default exists.""" groups: dict[str, list[dict]] = {} for s in SPECS: is_set = db.get(SystemConfig, s.key) is not None item: dict = { "key": s.key, "type": s.type, "group": s.group, "secret": s.secret, "min": s.min, "max": s.max, "is_set": is_set, # an override row exists "allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row "choices": list(s.choices), # the picker's options; [] for any other type } if s.secret: item["default_is_set"] = bool(_default(s)) else: item["value"] = get(db, s.key) item["default"] = _default(s) groups.setdefault(s.group, []).append(item) sections = [ {"id": sid, "groups": [g for g in members if g in groups]} for sid, members in SECTIONS ] return {"groups": groups, "sections": sections, "secrets_manageable": secrets_manageable()}