peter
915319fa35
Deep-review round 2 on the round-1 fix: - Race (medium): the round-1 identity guard did the supersede check and the respawn in TWO separate _lock sections. A newer same-key session installed in the gap was still torn down by _spawn's unconditional pop+kill (the round-1 comment even had the direction backwards). Fold the check into _spawn via a new replace_only= param: the identity test and the pop+replace now share ONE lock, so the respawn is a true no-op when superseded/culled. - Honest states (low): the superseded branch returned current_session(key), which is None when the key was cap-culled during our out-of-lock wait; the route turns None into a dishonest 'Media file not found on disk' 404 for a file that exists. Yield 'current_session(key) or s' so a culled session degrades to the route's retryable 'not available yet' path instead. - Hygiene: _want_hw docstring corrected (it is a test-only shim now; production decides via _select_encode) and the two stale _want_hw references refreshed (stream.py header, config.py plex_transcode_hw). - tests: the cap-cull no-respawn/no-None sub-case; the _opencl_present except OSError branch (glob raising, not just empty); tightened a tautological -vf assertion. Backend gate green (ruff + full pytest).
346 lines
20 KiB
Python
346 lines
20 KiB
Python
from pathlib import Path
|
|
|
|
from pydantic import model_validator
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
# Shipped placeholder; production must override it. Used by the startup guard below.
|
|
_DEFAULT_SECRET_KEY = "change-me-session-key"
|
|
|
|
|
|
def _version_from_file() -> str:
|
|
"""The committed VERSION file (copied into the image at /app/VERSION) is the single
|
|
source of truth for the app version; falls back to 'dev' for un-built/local runs."""
|
|
for p in (Path(__file__).resolve().parents[1] / "VERSION", Path("VERSION")):
|
|
try:
|
|
v = p.read_text(encoding="utf-8").strip()
|
|
if v:
|
|
return v
|
|
except OSError:
|
|
continue
|
|
return "dev"
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
|
|
|
app_name: str = "Siftlode"
|
|
# Version from the committed VERSION file (an explicit APP_VERSION env still overrides);
|
|
# git_sha/build_date are injected at image build time. Surfaced by /api/version.
|
|
app_version: str = _version_from_file()
|
|
git_sha: str = "unknown"
|
|
build_date: str = ""
|
|
|
|
database_url: str = "postgresql+psycopg://siftlode:siftlode@db:5432/siftlode"
|
|
|
|
# Session cookie signing key.
|
|
secret_key: str = _DEFAULT_SECRET_KEY
|
|
# Fernet key (urlsafe base64, 32 bytes) for encrypting stored refresh tokens.
|
|
token_encryption_key: str = ""
|
|
|
|
google_client_id: str = ""
|
|
google_client_secret: str = ""
|
|
oauth_redirect_url: str = "http://localhost:8080/auth/callback"
|
|
|
|
# Comma-separated invite list / admin list of Google account emails.
|
|
allowed_emails: str = ""
|
|
admin_emails: str = ""
|
|
|
|
# Whether anyone may submit an email+password registration (still gated by email
|
|
# verification + admin approval before they can sign in). Admin-tunable via the DB.
|
|
allow_registration: bool = True
|
|
|
|
# Origin of a separately served frontend dev server (enables CORS). Empty in production.
|
|
frontend_origin: str = ""
|
|
|
|
# Reverse-proxy IPs whose X-Forwarded-For we trust for the real client IP (rate limiting). This
|
|
# is the address the proxy CONNECTS FROM as seen by the app — e.g. the WireGuard peer IP of the
|
|
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
|
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
|
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
|
# deploy. Comma-separated. See routes.auth._client_ip + the deploy docs.
|
|
trusted_proxy_ips: str = ""
|
|
|
|
# --- Outbound email (onboarding: access-request + approval notices) ---
|
|
# Gmail SMTP + App Password by default. All optional: if unset, email is skipped
|
|
# (fail-soft) and onboarding still works via in-app notifications.
|
|
smtp_host: str = ""
|
|
smtp_port: int = 587
|
|
smtp_user: str = ""
|
|
smtp_password: str = ""
|
|
smtp_from: str = "" # e.g. "Siftlode <addr@gmail.com>"; falls back to smtp_user
|
|
|
|
# --- Sync / YouTube Data API ---
|
|
# Optional API key for public reads (channels/videos/playlistItems). When set it is
|
|
# preferred for shared backfill/enrichment so it doesn't depend on a specific user.
|
|
youtube_api_key: str = ""
|
|
# Optional HTTP(S) proxy for outbound YouTube Data API calls. Set this to send the
|
|
# scheduler's googleapis traffic through a fixed-IP host so an IP-restricted API key
|
|
# keeps working even when this host's public IP is dynamic.
|
|
youtube_api_proxy: str = ""
|
|
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
|
|
quota_daily_budget: int = 9000
|
|
# Per-user cap on live YouTube searches per day. With the official API (search_source="api")
|
|
# each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
|
|
# the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
|
|
search_daily_limit_per_user: int = 70
|
|
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
|
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
|
search_source: str = "scrape"
|
|
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
|
backfill_recent_max_videos: int = 100
|
|
backfill_recent_max_days: int = 365
|
|
|
|
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
|
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
|
shorts_probe_max_seconds: int = 180
|
|
shorts_probe_batch: int = 150
|
|
shorts_probe_interval_minutes: int = 2
|
|
# videos.list accepts up to 50 ids per call.
|
|
enrich_batch_size: int = 50
|
|
|
|
# --- Background scheduler ---
|
|
scheduler_enabled: bool = True
|
|
rss_poll_minutes: int = 20
|
|
enrich_interval_minutes: int = 3
|
|
backfill_interval_minutes: int = 10
|
|
# Keep this many quota units in reserve so scheduled backfill never starves
|
|
# interactive syncs / enrichment of fresh videos.
|
|
backfill_quota_reserve: int = 2000
|
|
|
|
# --- Auto-tagging / feed defaults ---
|
|
# Number of recent video titles sampled per channel for language detection.
|
|
autotag_title_sample: int = 40
|
|
autotag_interval_minutes: int = 30
|
|
subscriptions_resync_minutes: int = 360
|
|
playlist_sync_minutes: int = 360
|
|
# How often the shared demo account is auto-reset to a clean baseline (communal sandbox).
|
|
demo_reset_minutes: int = 720
|
|
# --- Maintenance / validation job ---
|
|
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
|
|
# are in days because that's the granularity that matters; the rolling re-validation
|
|
# re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a
|
|
# ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units.
|
|
maintenance_interval_minutes: int = 1440
|
|
maintenance_revalidate_batch: int = 15000
|
|
# Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist
|
|
# items). Deleted/private/paywalled get a recovery window; abandoned upcoming and
|
|
# confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them).
|
|
maintenance_grace_days: int = 7
|
|
# --- Channel-explore cleanup job ---
|
|
# Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never-
|
|
# kept channel (no subscription, no per-user state on its videos) is hard-deleted this many
|
|
# days after it was last explored — the grace clock is ExploredChannel.created_at.
|
|
explore_cleanup_minutes: int = 1440
|
|
explore_grace_days: int = 7
|
|
# How long an un-kept live-search result (a via_search video nobody watched / saved /
|
|
# subscribed to) is kept before the same cleanup job removes it — "as if never added".
|
|
search_grace_days: int = 7
|
|
# live_status values hidden from the feed by default. Completed-stream VODs
|
|
# ("was_live") are real watchable content and stay visible.
|
|
feed_default_hidden_live: str = "live,upcoming"
|
|
|
|
# --- Download center (yt-dlp) ---
|
|
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
|
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
|
download_root: str = "/downloads"
|
|
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
|
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
|
worker_enabled: bool = False
|
|
# Base URL of the bgutil PO-token provider sidecar. When set, the download worker asks it
|
|
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
|
|
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
|
|
download_pot_base_url: str = "http://bgutil-pot:4416"
|
|
# yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO
|
|
# token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's
|
|
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
|
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
|
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
|
download_player_clients: str = "android_vr"
|
|
download_player_clients_fallback: str = "web_safari,web,android"
|
|
|
|
@staticmethod
|
|
def _clients(raw: str) -> list[str]:
|
|
return [c.strip() for c in raw.split(",") if c.strip()]
|
|
|
|
@property
|
|
def player_client_list(self) -> list[str]:
|
|
return self._clients(self.download_player_clients)
|
|
|
|
@property
|
|
def player_client_fallback_list(self) -> list[str]:
|
|
return self._clients(self.download_player_clients_fallback)
|
|
|
|
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
|
download_worker_concurrency: int = 2
|
|
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
|
download_poll_seconds: int = 5
|
|
# --- download-subprocess watchdog (each download/edit runs in a supervised child process so a
|
|
# wedged yt-dlp/ffmpeg can be hard-killed; a thread cannot) ---
|
|
# Kill a running download if its job row hasn't advanced (no progress written) for this long.
|
|
# Generous: a live download writes progress ~1/s, so only a genuine 0-progress stall trips it.
|
|
download_stall_timeout_seconds: int = 150
|
|
# Absolute ceiling on a single download/edit regardless of progress (a legitimately huge file
|
|
# still finishes well under this; a runaway is capped).
|
|
download_wall_clock_max_seconds: int = 7200 # 2 hours
|
|
# How often the supervisor checks the child's liveness + the job row.
|
|
download_supervisor_poll_seconds: int = 3
|
|
# Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation).
|
|
download_kill_grace_seconds: int = 5
|
|
# --- worker-child resource de-prioritization ---
|
|
# The download/edit child (yt-dlp + ffmpeg) shares ONE LAN box with the API, Postgres, and Plex
|
|
# playback. Without a priority handicap a CPU re-encode pegs every core for ~30s and starves web
|
|
# requests + playback until it finishes. The child lowers its OWN priority at startup — inherited
|
|
# by every ffmpeg/yt-dlp descendant — so any foreground work always wins. Each knob is best-effort
|
|
# and independently disable-able with 0.
|
|
download_child_nice: int = 10 # CPU nice increment applied to the child; 0 = don't renice
|
|
download_child_ionice_class: int = 3 # best-effort IO class (3 = idle); 0 = leave IO untouched
|
|
# Cap the browser-safe re-encode's encoder threads so a transcode can't claim every core. 0 =
|
|
# reserve one core for foreground work (max(1, cpu_count - 1)); >0 pins exactly that many.
|
|
download_ffmpeg_threads: int = 0
|
|
# Throttle the retention GC's whole-tree orphan file sweep so its rglob can't monopolize disk IO
|
|
# in the API process: sleep `pause_ms` after every `batch` files touched. pause_ms=0 disables.
|
|
download_gc_sweep_pause_ms: int = 20
|
|
download_gc_sweep_batch: int = 400
|
|
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
|
|
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
|
|
download_total_max_bytes: int = 0
|
|
# Default per-user quota (admin-tunable; a per-user download_quotas row overrides).
|
|
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
|
download_default_max_concurrent: int = 2
|
|
download_default_max_jobs: int = 50
|
|
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
|
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
|
# user's overall byte footprint is still governed by download_default_max_bytes.
|
|
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
|
# Retention: an asset expires this many days after its last access; the GC job then
|
|
# deletes it (once no job references it) after an additional grace window.
|
|
download_retention_days: int = 30
|
|
download_gc_grace_days: int = 2
|
|
download_gc_minutes: int = 360
|
|
# Admin audit log: how long to keep rows (days; 0 = keep forever) + how often the GC runs.
|
|
audit_retention_days: int = 180
|
|
audit_gc_minutes: int = 1440
|
|
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
|
download_layout: str = "plex"
|
|
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
|
sponsorblock_default: bool = False
|
|
|
|
# --- Plex integration (optional) ---
|
|
# The Plex module plays the LOCAL physical media file directly (our own ffmpeg/direct-serve);
|
|
# Plex is used only for metadata + optional watch-state sync. Requires the Plex server to be
|
|
# reachable from THIS host and its media reachable on a local mount (see plex_path_map).
|
|
plex_enabled: bool = False
|
|
# Base URL of the Plex Media Server reachable from this host, e.g. http://192.168.1.100:32400.
|
|
plex_server_url: str = ""
|
|
# Plex server admin token (X-Plex-Token). Server-side only (metadata + image proxy); never
|
|
# sent to the browser. Stored encrypted when set via the admin Config page.
|
|
plex_server_token: str = ""
|
|
# Path map: Plex's on-disk media paths → this host's read-only mount, so Siftlode can open the
|
|
# physical file for direct playback. Newline/comma-separated "plexPrefix=localPrefix" pairs;
|
|
# the longest matching prefix wins. E.g. "/data=/plex-media". Empty = identical on both sides.
|
|
plex_path_map: str = ""
|
|
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
|
|
plex_libraries: str = ""
|
|
# HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
|
# section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN
|
|
# twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1".
|
|
plex_overlay_map: str = ""
|
|
# Stable client identifier Siftlode presents to Plex (X-Plex-Client-Identifier).
|
|
plex_client_id: str = "siftlode-server"
|
|
# How often the catalog sync job mirrors Plex metadata, in minutes.
|
|
plex_sync_interval_min: int = 360
|
|
# How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C):
|
|
# a cheap history + on-deck pull plus a re-push of any states that failed their immediate push.
|
|
# 15 min (S2): with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
|
# Plex-authoritative pull), a faster incremental pull keeps Siftlode close to Plex-side changes.
|
|
plex_watch_sync_interval_min: int = 15
|
|
# How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan
|
|
# that catches what the incremental history feed can't — notably an un-watch done on the Plex
|
|
# side — so it runs far less often (daily) than the incremental pass.
|
|
plex_watch_reconcile_interval_min: int = 1440
|
|
# Max concurrent transcode sessions (playable="transcode"). Low by default — the software-fallback
|
|
# re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve
|
|
# and remux (stream-copy) are not capped by this.
|
|
plex_max_transcodes: int = 4
|
|
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
|
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
|
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
|
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
|
plex_transcode_hw: str = "auto"
|
|
# Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf`
|
|
# trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output
|
|
# height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always
|
|
# encodes at source resolution.
|
|
plex_transcode_preset: str = "veryfast"
|
|
plex_transcode_crf: int = 20
|
|
plex_transcode_max_height: int = 1080
|
|
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
|
# dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
|
|
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
|
|
# local disk, so the default is fine there.
|
|
plex_hls_dir: str = ""
|
|
|
|
# --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) ---
|
|
# TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the
|
|
# admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget /
|
|
# revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the
|
|
# stored canonical guid, and SERVED FROM THE DB — never a live TMDB call at serve time.
|
|
tmdb_api_key: str = ""
|
|
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
|
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
|
tmdb_language: str = "en-US"
|
|
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
|
tmdb_enrich_interval_min: int = 120
|
|
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
|
tmdb_enrich_batch: int = 100
|
|
|
|
@property
|
|
def allowed_email_set(self) -> set[str]:
|
|
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
|
|
|
@property
|
|
def admin_email_set(self) -> set[str]:
|
|
return {e.strip().lower() for e in self.admin_emails.split(",") if e.strip()}
|
|
|
|
@property
|
|
def trusted_proxy_ip_set(self) -> frozenset[str]:
|
|
return frozenset(p.strip() for p in self.trusted_proxy_ips.split(",") if p.strip())
|
|
|
|
@property
|
|
def app_base(self) -> str:
|
|
"""Public origin of the deployed app, derived from the OAuth redirect URL
|
|
(.../auth/callback). The single source for user-facing links (verification, reset,
|
|
approval emails); dev and prod differ, so links must always come from config."""
|
|
u = self.oauth_redirect_url
|
|
return u.split("/auth/")[0] if "/auth/" in u else u.rstrip("/")
|
|
|
|
@property
|
|
def session_https_only(self) -> bool:
|
|
"""Mark the session cookie Secure when we're served over HTTPS. We treat an
|
|
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
|
return self.oauth_redirect_url.lower().startswith("https://")
|
|
|
|
@model_validator(mode="after")
|
|
def _enforce_production_secrets(self) -> "Settings":
|
|
"""Fail fast rather than boot a public instance with the shipped placeholder
|
|
signing key (which would let anyone forge an admin session) or without token
|
|
encryption. Local dev (http redirect URL) keeps the convenient defaults."""
|
|
if not self.session_https_only:
|
|
return self # local/dev: allow the placeholder defaults
|
|
if self.secret_key == _DEFAULT_SECRET_KEY or len(self.secret_key) < 32:
|
|
raise ValueError(
|
|
"SECRET_KEY must be a unique random value of at least 32 chars in "
|
|
"production (an https OAUTH_REDIRECT_URL was detected). Generate one "
|
|
'with: python -c "import secrets; print(secrets.token_urlsafe(48))"'
|
|
)
|
|
if not self.token_encryption_key:
|
|
raise ValueError(
|
|
"TOKEN_ENCRYPTION_KEY must be set in production so refresh tokens are "
|
|
"encrypted at rest. Generate one with: "
|
|
'python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"'
|
|
)
|
|
return self
|
|
|
|
|
|
settings = Settings()
|