from pathlib import Path from typing import Literal, get_args, get_origin from pydantic import Field, ValidationInfo, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict # Shipped placeholder; production must override it. Used by the startup guard below. _DEFAULT_SECRET_KEY = "change-me-session-key" def _version_from_file() -> str: """The committed VERSION file (copied into the image at /app/VERSION) is the single source of truth for the app version; falls back to 'dev' for un-built/local runs.""" for p in (Path(__file__).resolve().parents[1] / "VERSION", Path("VERSION")): try: v = p.read_text(encoding="utf-8").strip() if v: return v # silent-ok: this runs at import, and `app.logs` imports THIS module — a log call here # would be a circular import. The loop tries the other location, then falls back to "dev". except OSError: continue return "dev" # Closed value sets. A setting typed with one of these is a CHOICE: pydantic rejects anything else at # boot, and sysconfig derives the admin picker's options from the same Literal — so the list lives in # exactly one place and a consumer never has to guess what a typo should fall back to. SearchSource = Literal["scrape", "api"] DownloadLayout = Literal["plex", "flat"] LogLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR"] LogFormat = Literal["json", "text"] TranscodeHw = Literal["auto", "on", "off"] # libx264's presets, fastest first (`placebo` left out on purpose: no measurable gain for its cost). TranscodePreset = Literal[ "ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow" ] def choices_of(annotation: object) -> tuple[str, ...]: """The allowed values of a choice-typed setting, or () for any other type.""" return tuple(get_args(annotation)) if get_origin(annotation) is Literal else () def canonical_choice(options: tuple[str, ...], raw: object) -> str | None: """The option `raw` names, in its canonical spelling (any letter case, surrounding space ignored), or None. The ONE matching rule for a choice value from the env and from the admin API, so a spelling one accepts the other cannot refuse.""" folded = str(raw).strip().casefold() return next((option for option in options if option.casefold() == folded), None) class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", extra="ignore") app_name: str = "Siftlode" # Version from the committed VERSION file (an explicit APP_VERSION env still overrides); # git_sha/build_date are injected at image build time. Surfaced by /api/version. app_version: str = _version_from_file() git_sha: str = "unknown" build_date: str = "" database_url: str = "postgresql+psycopg://siftlode:siftlode@db:5432/siftlode" # Session cookie signing key. secret_key: str = _DEFAULT_SECRET_KEY # Fernet key (urlsafe base64, 32 bytes) for encrypting stored refresh tokens. token_encryption_key: str = "" google_client_id: str = "" google_client_secret: str = "" oauth_redirect_url: str = "http://localhost:8080/auth/callback" # Comma-separated invite list / admin list of Google account emails. allowed_emails: str = "" admin_emails: str = "" # Whether anyone may submit an email+password registration (still gated by email # verification + admin approval before they can sign in). Admin-tunable via the DB. allow_registration: bool = True # Origin of a separately served frontend dev server (enables CORS). Empty in production. frontend_origin: str = "" # Reverse-proxy IPs whose X-Forwarded-For we trust for the real client IP (rate limiting). This # is the address the proxy CONNECTS FROM as seen by the app — e.g. the WireGuard peer IP of the # VPS Caddy front door. A request from any other peer (someone hitting the app port directly) # is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via # XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev # deploy. Comma-separated. See ratelimit.client_key + the deploy docs. trusted_proxy_ips: str = "" # --- Outbound email (onboarding: access-request + approval notices) --- # Gmail SMTP + App Password by default. All optional: if unset, email is skipped # (fail-soft) and onboarding still works via in-app notifications. smtp_host: str = "" smtp_port: int = 587 smtp_user: str = "" smtp_password: str = "" smtp_from: str = "" # e.g. "Siftlode "; falls back to smtp_user # --- Sync / YouTube Data API --- # Optional API key for public reads (channels/videos/playlistItems). When set it is # preferred for shared backfill/enrichment so it doesn't depend on a specific user. youtube_api_key: str = "" # Optional HTTP(S) proxy for outbound YouTube Data API calls. Set this to send the # scheduler's googleapis traffic through a fixed-IP host so an IP-restricted API key # keeps working even when this host's public IP is dynamic. youtube_api_proxy: str = "" # Daily quota budget (units). Default leaves headroom under the 10,000/day free limit. quota_daily_budget: int = 9000 # Per-user cap on live YouTube searches per day. With the official API (search_source="api") # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable. search_daily_limit_per_user: int = 70 # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list, # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks. search_source: SearchSource = "scrape" # Recent-first backfill: how far back to fetch on the first pass per channel. backfill_recent_max_videos: int = 100 backfill_recent_max_days: int = 365 # A channel whose RSS feed has not answered for this many hours gets its newest uploads read # through the Data API instead (1 unit), at most once per this many hours. 0 = never. rss_fallback_hours: int = 2 # Shorts are confirmed by probing youtube.com/shorts/. Only videos at or below # this duration (and not livestreams) are probed; longer videos are never Shorts. shorts_probe_max_seconds: int = 180 shorts_probe_batch: int = 150 shorts_probe_interval_minutes: int = 2 # videos.list accepts up to 50 ids per call. enrich_batch_size: int = 50 # --- Background scheduler --- scheduler_enabled: bool = True rss_poll_minutes: int = 20 enrich_interval_minutes: int = 3 backfill_interval_minutes: int = 10 # Keep this many quota units in reserve so scheduled backfill never starves # interactive syncs / enrichment of fresh videos. backfill_quota_reserve: int = 2000 # --- Auto-tagging / feed defaults --- # Number of recent video titles sampled per channel for language detection. autotag_title_sample: int = 40 autotag_interval_minutes: int = 30 subscriptions_resync_minutes: int = 360 playlist_sync_minutes: int = 360 # How often the shared demo account is auto-reset to a clean baseline (communal sandbox). demo_reset_minutes: int = 720 # --- Maintenance / validation job --- # Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods # are in days because that's the granularity that matters; the rolling re-validation # re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a # ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units. maintenance_interval_minutes: int = 1440 maintenance_revalidate_batch: int = 15000 # Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist # items). Deleted/private/paywalled get a recovery window; abandoned upcoming and # confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them). maintenance_grace_days: int = 7 # --- Channel-explore cleanup job --- # Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never- # kept channel (no subscription, no per-user state on its videos) is hard-deleted this many # days after it was last explored — the grace clock is ExploredChannel.created_at. explore_cleanup_minutes: int = 1440 explore_grace_days: int = 7 # How long an un-kept live-search result (a via_search video nobody watched / saved / # subscribed to) is kept before the same cleanup job removes it — "as if never added". search_grace_days: int = 7 # live_status values hidden from the feed by default. Completed-stream VODs # ("was_live") are real watchable content and stay visible. feed_default_hidden_live: str = "live,upcoming" # --- Download center (yt-dlp) --- # Root directory (inside the container) for downloaded media — a Docker volume/bind mount. # Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it. download_root: str = "/downloads" # Whether THIS process runs the download worker loop. On for the dedicated worker # container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled. worker_enabled: bool = False # Base URL of the bgutil PO-token provider sidecar. When set, the download worker asks it # for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality # formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot"). download_pot_base_url: str = "http://bgutil-pot:4416" # yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO # token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's # per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the # worker retries with the FALLBACK set — the POT-capable web clients (web needs the token + # Deno) plus android for audio. Both admin-tunable when YouTube shifts what works. # A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients # of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there. # See adapters.YouTubeAdapter._COOKIE_CAPABLE. download_player_clients: str = "android_vr" download_player_clients_fallback: str = "web_safari,web,android" @staticmethod def _clients(raw: str) -> list[str]: return [c.strip() for c in raw.split(",") if c.strip()] @property def player_client_list(self) -> list[str]: return self._clients(self.download_player_clients) @property def player_client_fallback_list(self) -> list[str]: return self._clients(self.download_player_clients_fallback) # How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED). download_worker_concurrency: int = 2 # How often (empty pending queue) the worker polls for a new job, in seconds. download_poll_seconds: int = 5 # --- download-subprocess watchdog (each download/edit runs in a supervised child process so a # wedged yt-dlp/ffmpeg can be hard-killed; a thread cannot) --- # Kill a running download if its job row hasn't advanced (no progress written) for this long. # Generous: a live download writes progress ~1/s, so only a genuine 0-progress stall trips it. download_stall_timeout_seconds: int = 150 # Absolute ceiling on a single download/edit regardless of progress (a legitimately huge file # still finishes well under this; a runaway is capped). download_wall_clock_max_seconds: int = 7200 # 2 hours # How often the supervisor checks the child's liveness + the job row. download_supervisor_poll_seconds: int = 3 # Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation). download_kill_grace_seconds: int = 5 # --- worker-child resource de-prioritization --- # The download/edit child (yt-dlp + ffmpeg) shares ONE LAN box with the API, Postgres, and Plex # playback. Without a priority handicap a CPU re-encode pegs every core for ~30s and starves web # requests + playback until it finishes. The child lowers its OWN priority at startup — inherited # by every ffmpeg/yt-dlp descendant — so any foreground work always wins. Each knob is best-effort # and independently disable-able with 0. download_child_nice: int = 10 # CPU nice increment applied to the child; 0 = don't renice download_child_ionice_class: int = 3 # best-effort IO class (3 = idle); 0 = leave IO untouched # Cap the browser-safe re-encode's encoder threads so a transcode can't claim every core. 0 = # reserve one core for foreground work (max(1, cpu_count - 1)); >0 pins exactly that many. download_ffmpeg_threads: int = 0 # Throttle the retention GC's whole-tree orphan file sweep so its rglob can't monopolize disk IO # in the API process: sleep `pause_ms` after every `batch` files touched. pause_ms=0 disables. download_gc_sweep_pause_ms: int = 20 download_gc_sweep_batch: int = 400 # Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the # least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit). download_total_max_bytes: int = 0 # Default per-user quota (admin-tunable; a per-user download_quotas row overrides). download_default_max_bytes: int = 5_368_709_120 # 5 GiB download_default_max_concurrent: int = 2 download_default_max_jobs: int = 50 # Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker # finishes), an upload's size is known at ingest, so this bounds a single file directly; the # user's overall byte footprint is still governed by download_default_max_bytes. # ge=1: this number is also the EDGE ceiling (`bodylimit.ROUTE_MAX_BODY_BYTES`), where 0 is a # real limit of zero rather than the route's "no per-file cap" sentinel — an instance set to 0 # would refuse every upload, including the admin page's own way back. To lift the cap in # practice, set a large number. download_upload_max_bytes: int = Field(default=524_288_000, ge=1) # 500 MiB # Retention: an asset expires this many days after its last access; the GC job then # deletes it (once no job references it) after an additional grace window. download_retention_days: int = 30 download_gc_grace_days: int = 2 download_gc_minutes: int = 360 # Admin audit log: how long to keep rows (days; 0 = keep forever) + how often the GC runs. audit_retention_days: int = 180 audit_gc_minutes: int = 1440 # On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat". download_layout: DownloadLayout = "plex" # Default SponsorBlock (skip sponsor segments) for new custom profiles. sponsorblock_default: bool = False # --- Diagnostics / logging (E22) --- # BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at # startup and on change without a restart. This is what a process logs at before it can reach # the database — and the only knob the worker/runner children read, since they are short-lived. # TRACE (below DEBUG) is the "Teljes" diagnostics level; see app/logs.py. log_level: LogLevel = "INFO" # "json" (one object per line — what Loki and the log_events sink ingest) or "text" (a readable # line for a local terminal). Redaction applies to both. log_format: LogFormat = "json" # Also write events to the `log_events` table (app/logsink.py): WARNING and above, the request # lines, and — once they exist — client batches. Off only where no table is wanted (the tests). log_db_sink: bool = True # Days a `log_events` row is kept (the privacy policy promises this), and how often the log_gc job # enforces it — hourly, so a row never outlives the promise by more than an hour. log_retention_days: int = 7 log_gc_minutes: int = 60 # --- Plex integration (optional) --- # The Plex module plays the LOCAL physical media file directly (our own ffmpeg/direct-serve); # Plex is used only for metadata + optional watch-state sync. Requires the Plex server to be # reachable from THIS host and its media reachable on a local mount (see plex_path_map). plex_enabled: bool = False # Base URL of the Plex Media Server reachable from this host, e.g. http://192.168.1.100:32400. plex_server_url: str = "" # Plex server admin token (X-Plex-Token). Server-side only (metadata + image proxy); never # sent to the browser. Stored encrypted when set via the admin Config page. plex_server_token: str = "" # Path map: Plex's on-disk media paths → this host's read-only mount, so Siftlode can open the # physical file for direct playback. Newline/comma-separated "plexPrefix=localPrefix" pairs; # the longest matching prefix wins. E.g. "/data=/plex-media". Empty = identical on both sides. plex_path_map: str = "" # Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections. plex_libraries: str = "" # HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU # section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN # twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1". plex_overlay_map: str = "" # Stable client identifier Siftlode presents to Plex (X-Plex-Client-Identifier). plex_client_id: str = "siftlode-server" # How often the catalog sync job mirrors Plex metadata, in minutes. plex_sync_interval_min: int = 360 # How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C): # a cheap history + on-deck pull plus a re-push of any states that failed their immediate push. # 15 min (S2): with Plex now the source of truth (prompt write-back on the ~10s checkpoint + # Plex-authoritative pull), a faster incremental pull keeps Siftlode close to Plex-side changes. plex_watch_sync_interval_min: int = 15 # How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan # that catches what the incremental history feed can't — notably an un-watch done on the Plex # side — so it runs far less often (daily) than the incremental pass. plex_watch_reconcile_interval_min: int = 1440 # Max concurrent transcode sessions (playable="transcode"). Low by default — the software-fallback # re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve # and remux (stream-copy) are not capped by this. plex_max_transcodes: int = 4 # Max concurrent playback sessions of ANY kind — remux (stream-copy) included. Cheap on the CPU but # each one is a whole segmented copy of the film on the scratch volume, written faster than # realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`). plex_max_sessions: int = 8 # --- Transcode (playable="transcode") re-encode tuning (E6 S1) --- # Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the # source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render # node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`. plex_transcode_hw: TranscodeHw = "auto" # Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf` # trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output # height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always # encodes at source resolution. # Loudness normalisation of the downmixed stereo audio on the HLS path (EBU R128 `loudnorm`). # OFF by default: it makes a film ~11 dB louder and narrows its dynamic range, which is a taste # call, not a fix. Only affects files that go through remux/transcode — a direct-played file is # served untouched, but those are browser-native stereo anyway, not the 5.1 mixes this is for. plex_audio_loudnorm: bool = False plex_transcode_preset: TranscodePreset = "veryfast" plex_transcode_crf: int = 20 plex_transcode_max_height: int = 1080 # Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls` # dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast # local disk, so the default is fine there. plex_hls_dir: str = "" # --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) --- # TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the # admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget / # revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the # stored canonical guid, and SERVED FROM THE DB — never a live TMDB call at serve time. tmdb_api_key: str = "" tmdb_api_base: str = "https://api.themoviedb.org/3" # Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default. tmdb_language: str = "en-US" # --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the # composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's # log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). --- issues_gitea_url: str = "" issues_gitea_repo: str = "" issues_gitea_token: str = "" # How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes. tmdb_enrich_interval_min: int = 120 # Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited). tmdb_enrich_batch: int = 100 # How often the people job (E30 S6) fetches a batch of person bios from TMDB and portraits through Plex, # in minutes, and how many of each per run: ~18k linked people at 300 per 10 min is about ten hours # once, then only the newcomers. people_enrich_interval_min: int = 10 people_enrich_batch: int = 300 @property def allowed_email_set(self) -> set[str]: return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()} @property def admin_email_set(self) -> set[str]: return {e.strip().lower() for e in self.admin_emails.split(",") if e.strip()} @property def trusted_proxy_ip_set(self) -> frozenset[str]: return frozenset(p.strip() for p in self.trusted_proxy_ips.split(",") if p.strip()) @property def app_base(self) -> str: """Public origin of the deployed app, derived from the OAuth redirect URL (.../auth/callback). The single source for user-facing links (verification, reset, approval emails); dev and prod differ, so links must always come from config.""" u = self.oauth_redirect_url return u.split("/auth/")[0] if "/auth/" in u else u.rstrip("/") @property def session_https_only(self) -> bool: """Mark the session cookie Secure when we're served over HTTPS. We treat an https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev.""" return self.oauth_redirect_url.lower().startswith("https://") @field_validator("*", mode="before") @classmethod def _canonical_choice(cls, value: object, info: ValidationInfo) -> object: """A choice setting accepts its value in any letter case (`LOG_LEVEL=debug`) and stores the canonical spelling; an unknown value still fails the boot with pydantic's own message.""" field = cls.model_fields.get(info.field_name or "") options = choices_of(field.annotation) if field else () if options and isinstance(value, str): return canonical_choice(options, value) or value return value @model_validator(mode="after") def _enforce_production_secrets(self) -> "Settings": """Fail fast rather than boot a public instance with the shipped placeholder signing key (which would let anyone forge an admin session) or without token encryption. Local dev (http redirect URL) keeps the convenient defaults.""" if not self.session_https_only: return self # local/dev: allow the placeholder defaults if self.secret_key == _DEFAULT_SECRET_KEY or len(self.secret_key) < 32: raise ValueError( "SECRET_KEY must be a unique random value of at least 32 chars in " "production (an https OAUTH_REDIRECT_URL was detected). Generate one " 'with: python -c "import secrets; print(secrets.token_urlsafe(48))"' ) if not self.token_encryption_key: raise ValueError( "TOKEN_ENCRYPTION_KEY must be set in production so refresh tokens are " "encrypted at rest. Generate one with: " 'python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"' ) return self settings = Settings()