"""Admin-editable system configuration: a DB-override layer over the env/config defaults. The registry (SPECS) is the single source of truth for which config keys can be overridden from the admin Configuration page, plus each key's type, admin-UI group, env/config default, bounds, and whether it's a secret. The resolver returns the DB override if one is set, else ``settings.`` (same DB-override-or-default pattern as app.state). Secrets are stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secrets can't be stored in the DB and stay env-only). """ import logging from dataclasses import dataclass from sqlalchemy.orm import Session from app import security from app.config import settings from app.models import SystemConfig log = logging.getLogger("siftlode.config") # Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per # key, not once per request (bounded by the number of secret config keys). _decrypt_warned: set[str] = set() @dataclass(frozen=True) class ConfigSpec: key: str type: str # "int" | "str" | "bool" group: str # admin-UI grouping (logically related keys stay together) default_attr: str # attribute on `settings` holding the env/config default secret: bool = False min: int | None = None max: int | None = None allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset" # Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it # off env into the admin UI; the admin page renders it automatically from this list. SPECS: tuple[ConfigSpec, ...] = ( # --- Email / SMTP (one logical group; the password is the only secret) --- ConfigSpec("smtp_host", "str", "email", "smtp_host"), ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535), ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True), ConfigSpec("smtp_from", "str", "email", "smtp_from"), ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True), # --- Quota (shared YouTube Data API daily budget) --- ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), ConfigSpec( "backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000 ), ConfigSpec( "search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000, ), # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). ConfigSpec("search_source", "str", "quota", "search_source"), # --- Backfill (recent-first first pass per channel) --- ConfigSpec( "backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000, ), ConfigSpec( "backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500 ), # --- Shorts probe --- ConfigSpec( "shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600 ), ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000), # --- Batch sizes --- ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), # --- Maintenance (rolling re-validation of already-known videos) --- # Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin # knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard. ConfigSpec( "maintenance_revalidate_batch", "int", "maintenance", "maintenance_revalidate_batch", min=100, max=100_000, ), # --- Discovery cleanup (search + explore ephemeral content) --- # Days an explored-but-unsubscribed channel is kept before the cleanup job purges it. ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650), # Days an un-kept live-search result video is kept before the cleanup job removes it. ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650), # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True), # --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the # install wizard / Configuration page so no restart is needed when the creds change. --- ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True), ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True), # --- Access / registration --- ConfigSpec("allow_registration", "bool", "access", "allow_registration"), # --- Download center (yt-dlp) — per-user defaults + retention/GC + layout --- ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0), ConfigSpec( "download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0 ), ConfigSpec( "download_default_max_concurrent", "int", "downloads", "download_default_max_concurrent", min=1, max=20, ), ConfigSpec( "download_default_max_jobs", "int", "downloads", "download_default_max_jobs", min=1, max=100_000, ), # min=1, NOT 0: the per-file cap is enforced as `if cap and size > cap`, so a stored 0 would read # as falsy and SILENTLY disable the limit (an unbounded streamed write to the volume). Floor it at # 1 so the cap is always active; the overall footprint is separately bounded by the byte quota. ConfigSpec("download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=1), ConfigSpec( "download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650 ), ConfigSpec( "download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650 ), ConfigSpec("download_layout", "str", "downloads", "download_layout"), ConfigSpec( "download_worker_concurrency", "int", "downloads", "download_worker_concurrency", min=1, max=16, ), ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"), # --- Audit log --- # Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever). ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650), # --- Plex integration (optional; local-file playback + Plex metadata) --- ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"), ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"), ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True), ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"), ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"), ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"), # (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.) ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16), # Transcode re-encode tuning (E6 S1). hw = "auto"|"on"|"off" (free-text, like search_source); the # libx264 knobs apply only to the software fallback. crf 0–51 (x264 range); max_height 0 = source. ConfigSpec("plex_transcode_hw", "str", "plex", "plex_transcode_hw"), ConfigSpec("plex_transcode_preset", "str", "plex", "plex_transcode_preset"), ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51), ConfigSpec( "plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320 ), # NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job # (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS. # --- TMDB enrichment (E5 S2; optional) — the API key is the only secret --- ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True), ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"), # (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch # is a config.py default — both stay out of SPECS, like plex_sync_interval_min.) ) _BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS} def spec(key: str) -> ConfigSpec | None: return _BY_KEY.get(key) def secrets_manageable() -> bool: """Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY).""" return bool(settings.token_encryption_key) def _default(s: ConfigSpec): return getattr(settings, s.default_attr) def _coerce(s: ConfigSpec, raw: str): if s.type == "int": return int(raw) if s.type == "bool": return raw == "true" return raw def _raw_override(db: Session, key: str) -> str | None: """Stored (decrypted) override string for `key`, or None if no override is set. A decrypt failure — a rotated or removed ``TOKEN_ENCRYPTION_KEY`` vs the key the value was written with — must NOT crash the caller. `_raw_override` sits on hot paths (`/api/me` via `google_enabled`, login, every YouTube scheduler job, account purge), so a raw ``InvalidToken`` here would 500 the whole app and the SPA wouldn't even boot. Fall back to the env/config default instead, and warn once per key so the operator gets a real signal without a per-request traceback. Recovery is documented in docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2) """ row = db.get(SystemConfig, key) if row is None or row.value is None: return None if not row.encrypted: return row.value try: return security.decrypt(row.value) except Exception: # noqa: BLE001 — Fernet InvalidToken / malformed ciphertext (key mismatch) if key not in _decrypt_warned: _decrypt_warned.add(key) log.warning( "config %r could not be decrypted (TOKEN_ENCRYPTION_KEY changed or missing?) — falling " "back to the env/config default; the key must not change once secrets are stored", key, ) return None def get(db: Session, key: str): """Effective value: the DB override (typed) if set, else the env/config default.""" s = _BY_KEY[key] raw = _raw_override(db, key) if raw is None: return _default(s) if raw == "" and not s.allow_empty: # For most keys an empty override means "fall back to the env default". Keys that opt # into allow_empty (e.g. smtp_user, youtube_api_proxy) treat "" as a real value — so an # admin can blank one to DISABLE it rather than being stuck with the env default. return _default(s) try: return _coerce(s, raw) except (TypeError, ValueError): return _default(s) def get_str(db: Session, key: str) -> str: v = get(db, key) return "" if v is None else str(v) def get_int(db: Session, key: str) -> int: return int(get(db, key)) def get_bool(db: Session, key: str) -> bool: return bool(get(db, key)) def set_value(db: Session, key: str, value) -> None: """Validate, coerce, and stage a DB override for `key` (secrets are encrypted). Flushes but does NOT commit — the caller owns the transaction, so the change lands together with its audit row (B-core-10). The staged value is visible to a same-session `get()` before the commit.""" s = _BY_KEY[key] if s.type == "int": v = int(value) if s.min is not None and v < s.min: raise ValueError(f"{key} must be >= {s.min}") if s.max is not None and v > s.max: raise ValueError(f"{key} must be <= {s.max}") raw = str(v) elif s.type == "bool": raw = "true" if value else "false" else: raw = "" if value is None else str(value) stored, encrypted = raw, False if s.secret: if not secrets_manageable(): raise RuntimeError("TOKEN_ENCRYPTION_KEY is not configured") stored, encrypted = security.encrypt(raw) or "", True row = db.get(SystemConfig, key) if row is None: row = SystemConfig(key=key) db.add(row) row.value = stored row.encrypted = encrypted db.flush() def reset(db: Session, key: str) -> None: """Remove the DB override for `key` (fall back to the env/config default). Flushes but does NOT commit — the caller owns the transaction (B-core-10).""" row = db.get(SystemConfig, key) if row is not None: db.delete(row) db.flush() def describe(db: Session) -> dict: """Admin-UI metadata + current values, grouped. Secret values are never echoed — only whether an override is set and whether an env default exists.""" groups: dict[str, list[dict]] = {} for s in SPECS: is_set = db.get(SystemConfig, s.key) is not None item: dict = { "key": s.key, "type": s.type, "group": s.group, "secret": s.secret, "min": s.min, "max": s.max, "is_set": is_set, # an override row exists "allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row } if s.secret: item["default_is_set"] = bool(_default(s)) else: item["value"] = get(db, s.key) item["default"] = _default(s) groups.setdefault(s.group, []).append(item) return {"groups": groups, "secrets_manageable": secrets_manageable()}