import asyncio import math import mimetypes from contextlib import asynccontextmanager, suppress from pathlib import Path from fastapi import FastAPI, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from sqlalchemy.exc import DataError from starlette.middleware.sessions import SessionMiddleware # Logging is configured HERE, before the app's own modules are imported, so a line one of them logs # WHILE importing already has the format (a logger is configured whenever it is created — # app/logs.py — but a line emitted before `configure_logging` goes out raw). ONE format for the api, # the worker, the download child and alembic alike — this replaces the old uvicorn --log-config + # the hand-rolled fallback that used to live here, and uvicorn's access log is replaced by the # `http.request` middleware below. The imports after it are E402 on purpose, each one marked. from app import logs from app.logs import configure_logging, current_level, log_requests, set_level configure_logging("api") log = logs.get_logger(__name__) from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first from app.config import settings # noqa: E402 — logging is configured first from app.db import SessionLocal, engine # noqa: E402 — logging is configured first import app.tables # noqa: E402 — every table, before the first query from app.downloads import service as download_service # noqa: E402 — logging is configured first from app.downloads import storage # noqa: E402 — logging is configured first from app.plex import stream as plex_stream # noqa: E402 — logging is configured first from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first from app.routes import ( # noqa: E402 — logging is configured first admin, admin_logs as admin_logs_routes, audit as audit_routes, auth as auth_routes, channels, config as config_routes, debug_sessions as debug_sessions_routes, diagnostics as diagnostics_routes, discover as discover_routes, downloads, feed, files as files_routes, health, me, notifications, playlists, plex as plex_routes, public as public_routes, quota, runs as runs_routes, saved_views, scheduler as scheduler_routes, search as search_routes, setup as setup_routes, sync, tags, tokens as tokens_routes, version, ) from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first @asynccontextmanager async def lifespan(app: FastAPI): log.info("Siftlode starting up") problem = storage.files_root_problem(settings.files_root) if problem: # The rest of the app still serves; the files library cannot (the worker refuses to start). log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem) # The stored copy of the log (log_events). The entrypoint has already migrated the schema. logsink.start() # First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print # the wizard URL so the operator can open it (the app runs in setup mode until they finish). with SessionLocal() as db: # The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap # default, so a level chosen in the UI survives a restart without touching the environment. # Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from # booting — an unreachable or not-yet-migrated database has to start into the env level. try: set_level(sysconfig.get_str(db, "log_level")) except Exception: # any DB/config failure keeps the bootstrap level # With the traceback: this catch spans a connection, a query and a coercion at boot, # and `%s` of the exception does not say which of them raised. log.exception("could not read the stored log level; staying at %s", current_level()) if not problem: try: left = download_service.library_left_behind(db, settings.files_root) except Exception: # a database not reachable yet: the check is a hint, never a gate log.exception("could not check the files library against its database") else: if left: log.critical( "The files library is empty but the database names files: the volume " "mounted at %s is not the one that holds them. See docs/self-hosting.md.", settings.files_root, ) if not state.is_configured(db): token = state.rotate_setup_token(db) url = f"{settings.app_base}/setup?token={token}" log.warning( "FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n" " Open the install wizard (internal access only):\n %s", url, # log-pii-ok: this instance's OWN setup URL, and `?token=` is redacted ) # No HLS session survives a restart (they live in this process), so anything still on disk is # a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not # awaited: uvicorn does not serve a single request until lifespan startup returns, so awaiting # the sweep would hold the port shut for as long as the rmtree runs (gigabytes of `.ts` after a # long uptime) — the 502 + failed post-deploy health probe this is meant to avoid. A thread # alone doesn't buy that; not waiting does. The sweep only touches directories no live session # owns, so racing it against early playback is safe. sweep = asyncio.create_task(_sweep_hls_orphans()) reaper = asyncio.create_task(_reap_hls_sessions()) # Fire and forget as well: a rebuild after an ICU upgrade takes seconds, and the title sorts serve # meanwhile (CONCURRENTLY) in the old index's order, as they did before the restart. heal = asyncio.create_task(_heal_collation_drift()) start_scheduler() try: yield finally: log.info("Siftlode shutting down") # Awaited, not just cancelled: a bare `.cancel()` lets the loop close with the task still # pending ("Task was destroyed but it is pending!" — noise that reads like a bug during # triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own # bookkeeping honest. for task in (sweep, reaper, heal): task.cancel() with suppress(asyncio.CancelledError): await task shutdown_scheduler() # Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for # the rest of the read (see plex_subtitles.shutdown). plex_subtitles.shutdown() logsink.stop() # last: write what the shutdown itself just logged # How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`). _HLS_REAP_EVERY_S = 60.0 async def _reap_hls_sessions() -> None: """The periodic reaper plex/stream.py's docstring promises. `reap_idle` used to run only inside the NEXT session request, so after the last playback of the day an abandoned session — its ffmpeg still working through the film, its directory a whole segmented copy of it — lived until somebody pressed play again or the process restarted. A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead held one whole segmented copy of a film per item on the download volume.""" while True: await asyncio.sleep(_HLS_REAP_EVERY_S) try: await asyncio.to_thread(plex_stream.reap_idle) except Exception: log.exception("HLS session reap failed") async def _heal_collation_drift() -> None: """Background startup check of the ICU collation the title-sort indexes are built in (see `collation.heal_drift`). A failure is logged, never fatal: the sorts still answer, only possibly in the old order.""" try: await asyncio.to_thread(collation.heal_drift, engine) except asyncio.CancelledError: raise except Exception: log.exception("ICU collation drift check failed") async def _sweep_hls_orphans() -> None: """Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here must never take the app down, and a cancelled shutdown is not one.""" try: swept = await asyncio.to_thread(plex_stream.sweep_orphans) except asyncio.CancelledError: raise except Exception: log.exception("HLS orphan sweep failed") return if swept: log.info("swept %d orphaned HLS segment directories", swept) app = FastAPI(title=settings.app_name, lifespan=lifespan) @app.exception_handler(RequestValidationError) async def _validation_error(request: Request, exc: RequestValidationError) -> JSONResponse: """FastAPI's own 422, in its own shape — except that it can be ENCODED. The JSON parser accepts `NaN`/`Infinity`, a float field refuses them, and the refusal echoes the value back: the default handler then failed to serialise its own error (JSON has no NaN) and the request became a 500 with a traceback. A non-finite value is echoed as its name instead.""" finite = {float: lambda x: x if math.isfinite(x) else str(x)} return JSONResponse( status_code=422, content={"detail": jsonable_encoder(exc.errors(), custom_encoder=finite)} ) @app.exception_handler(DataError) async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse: """A NUL byte in a string the request sent is a 422, on every route at once. Postgres cannot hold NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not hold: an earlier fix guarded the by-rating_key lookup and missed the rest. Any other DataError is left alone — re-raised, it stays the 500 it always was.""" if "NUL (0x00)" not in str(exc.orig): raise exc return JSONResponse( status_code=422, content={"detail": {"code": "invalid_input", "reason": "nul"}} ) app.add_middleware( SessionMiddleware, secret_key=settings.secret_key, same_site="lax", # required so the cookie rides the OAuth redirect back from Google https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod) ) # Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or # /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render. _SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version") @app.middleware("http") async def setup_gate(request, call_next): """Lock the app to the install wizard until it's configured. Skips the DB entirely once setup completed in this process (the cached fast-path), so there's no steady-state overhead.""" if not state.setup_complete(): path = request.url.path if path.startswith(("/api/", "/auth/")) and not path.startswith(_SETUP_OPEN_PREFIXES): with SessionLocal() as db: configured = state.is_configured_cached(db) if not configured: return JSONResponse({"detail": "This instance isn't set up yet."}, status_code=503) return await call_next(request) # SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL # before the first dependency, so without this the anonymous login route is a free memory/time sink. # # WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this # comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a # stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly # OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints: # # * inside CORS and the security headers, so the 413 is decorated like every other response. A # refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang, # and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what # bodylimit.py exists to avoid. # * outside the setup gate, the session and every auth dependency under the router — none of which # run for a body that is already too large. The setup gate matters specifically: on an instance # that is not yet configured it opens a DB session per request, which is precisely the anonymous # pre-auth work SEC-20 exists to remove. app.add_middleware(bodylimit.BodyLimitMiddleware) if settings.frontend_origin: app.add_middleware( CORSMiddleware, allow_origins=[settings.frontend_origin], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) @app.middleware("http") async def static_cache_headers(request, call_next): """Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given /assets URL never changes content and a browser can hold it forever — this is what makes repeat loads instant and clears the Lighthouse "efficient cache lifetimes" audit. index.html stays no-cache (set on its own FileResponse) so a deploy is picked up at once; other SPA-root static files (welcome images, favicon, robots.txt) get a moderate TTL in the SPA fallback below.""" response = await call_next(request) if request.url.path.startswith("/assets/"): response.headers["Cache-Control"] = "public, max-age=31536000, immutable" return response # Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning: # nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to # external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be # iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a # library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and # Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates # the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and # suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy # (script/style/connect sources) is deferred — the SPA's inline styles + the YouTube/Plex embeds make # it real work — and HSTS belongs on the TLS-terminating edge (the Caddy front door), not here. _SECURITY_HEADERS = { "X-Content-Type-Options": "nosniff", "Referrer-Policy": "strict-origin-when-cross-origin", "Content-Security-Policy": "frame-ancestors 'none'", "X-Frame-Options": "DENY", "Permissions-Policy": "camera=(), microphone=(), geolocation=()", } @app.middleware("http") async def security_headers(request, call_next): response = await call_next(request) for name, value in _SECURITY_HEADERS.items(): response.headers.setdefault(name, value) return response # The access log (app/logs.py): request id, route template, status, duration — one line, never a # query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole # request, including what every layer above does — the body cap included, so a 413 is still logged # with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware, # which is why the account comes from `request.state.user_id` (set by the auth dependency, the one # place a session is actually resolved into an account) rather than from `request.session`. app.middleware("http")(log_requests) app.include_router(health.router) app.include_router(auth_routes.router) app.include_router(sync.router) app.include_router(tags.router) app.include_router(feed.router) app.include_router(discover_routes.router) app.include_router(search_routes.router) app.include_router(me.router) app.include_router(diagnostics_routes.router) app.include_router(debug_sessions_routes.router) app.include_router(notifications.router) app.include_router(channels.router) app.include_router(playlists.router) app.include_router(saved_views.router) app.include_router(plex_routes.router) app.include_router(downloads.router) app.include_router(downloads.admin_router) app.include_router(files_routes.router) app.include_router(tokens_routes.router) app.include_router(public_routes.router) app.include_router(admin.router) app.include_router(config_routes.router) app.include_router(scheduler_routes.router) app.include_router(audit_routes.router) app.include_router(admin_logs_routes.router) app.include_router(runs_routes.router) app.include_router(quota.router) app.include_router(version.router) app.include_router(setup_routes.router) # Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the # filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream). mimetypes.add_type("image/webp", ".webp") mimetypes.add_type("image/avif", ".avif") # The built SPA (populated by the Docker frontend build stage). STATIC_DIR = Path(__file__).parent / "static_spa" # index.html is unhashed and references the content-hashed /assets bundles, so it MUST NOT be # heuristically cached — otherwise, after a deploy, a browser keeps serving the old index.html # (pointing at the previous bundle) and runs stale code until a hard refresh. `no-cache` lets it # stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The # hashed bundles under /assets can cache forever (a new build changes their filename). INDEX_HTML = STATIC_DIR / "index.html" # A public share link's own page: the player alone, without the app's bundle (frontend/watch.html). WATCH_HTML = STATIC_DIR / "watch.html" INDEX_HEADERS = {"Cache-Control": "no-cache"} app.mount( "/assets", StaticFiles(directory=STATIC_DIR / "assets", check_dir=False), name="assets", ) @app.get("/") async def index() -> FileResponse: return FileResponse(INDEX_HTML, headers=INDEX_HEADERS) @app.get("/watch/{token}") async def watch_page(token: str): """Serve the public player page for a share link, with per-video Open Graph tags injected so the link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser ignores the extra tags and boots WatchPage; invalid/expired/password links fall back to the generic card.""" from app.downloads import og with SessionLocal() as db: rendered = og.render_watch_html(token, WATCH_HTML, db) if rendered is None: return FileResponse(WATCH_HTML, headers=INDEX_HEADERS) return HTMLResponse(rendered, headers=INDEX_HEADERS) @app.get("/{full_path:path}") async def spa_fallback(full_path: str) -> FileResponse: # Client-side routes fall back to index.html; real API/asset paths are matched above. if full_path.startswith(("api/", "auth/", "healthz", "assets/")): raise HTTPException(status_code=404) # Serve real files that live at the SPA root (Vite copies public/ there — e.g. the landing # screenshots under /welcome/, favicon). /assets is already mounted above; everything else # that isn't a real file is a client-side route → index.html. Guard against path traversal. if full_path: candidate = (STATIC_DIR / full_path).resolve() if candidate.is_file() and STATIC_DIR.resolve() in candidate.parents: # Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have # stable names, so a moderate cache is safe and satisfies the cache-lifetime audit. return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"}) # A /watch/ path the route above does not match (a trailing segment, no token) is still the # player's: WatchPage reads the token off the path and says when a link is gone. if full_path.startswith("watch/"): return FileResponse(WATCH_HTML, headers=INDEX_HEADERS) return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)