siftlode/backend/app/main.py

import asyncio
import math
import mimetypes
from contextlib import asynccontextmanager, suppress
from pathlib import Path

from fastapi import FastAPI, HTTPException, Request
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from sqlalchemy.exc import DataError
from starlette.middleware.sessions import SessionMiddleware

# Logging is configured HERE, before the app's own modules are imported, so a line one of them logs
# WHILE importing already has the format (a logger is configured whenever it is created —
# app/logs.py — but a line emitted before `configure_logging` goes out raw). ONE format for the api,
# the worker, the download child and alembic alike — this replaces the old uvicorn --log-config +
# the hand-rolled fallback that used to live here, and uvicorn's access log is replaced by the
# `http.request` middleware below. The imports after it are E402 on purpose, each one marked.
from app import logs
from app.logs import configure_logging, current_level, log_requests, set_level

configure_logging("api")

log = logs.get_logger(__name__)
from app import bodylimit, collation, logsink, state, sysconfig  # noqa: E402 — logging is configured first
from app.config import settings  # noqa: E402 — logging is configured first
from app.db import SessionLocal, engine  # noqa: E402 — logging is configured first
from app.downloads import service as download_service  # noqa: E402 — logging is configured first
from app.downloads import storage  # noqa: E402 — logging is configured first
from app.plex import stream as plex_stream  # noqa: E402 — logging is configured first
from app.plex import subtitles as plex_subtitles  # noqa: E402 — logging is configured first
from app.routes import (  # noqa: E402 — logging is configured first
    admin,
    admin_logs as admin_logs_routes,
    audit as audit_routes,
    auth as auth_routes,
    channels,
    config as config_routes,
    debug_sessions as debug_sessions_routes,
    diagnostics as diagnostics_routes,
    discover as discover_routes,
    downloads,
    feed,
    files as files_routes,
    health,
    me,
    notifications,
    playlists,
    plex as plex_routes,
    public as public_routes,
    quota,
    runs as runs_routes,
    saved_views,
    scheduler as scheduler_routes,
    search as search_routes,
    setup as setup_routes,
    sync,
    tags,
    tokens as tokens_routes,
    version,
)
from app.scheduler import shutdown_scheduler, start_scheduler  # noqa: E402 — logging is configured first


@asynccontextmanager
async def lifespan(app: FastAPI):
    log.info("Siftlode starting up")
    problem = storage.files_root_problem(settings.files_root)
    if problem:
        # The rest of the app still serves; the files library cannot (the worker refuses to start).
        log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem)
    # The stored copy of the log (log_events). The entrypoint has already migrated the schema.
    logsink.start()
    # First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
    # the wizard URL so the operator can open it (the app runs in setup mode until they finish).
    with SessionLocal() as db:
        # The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap
        # default, so a level chosen in the UI survives a restart without touching the environment.
        # Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from
        # booting — an unreachable or not-yet-migrated database has to start into the env level.
        try:
            set_level(sysconfig.get_str(db, "log_level"))
        except Exception:  # any DB/config failure keeps the bootstrap level
            # With the traceback: this catch spans a connection, a query and a coercion at boot,
            # and `%s` of the exception does not say which of them raised.
            log.exception("could not read the stored log level; staying at %s", current_level())
        if not problem:
            try:
                left = download_service.library_left_behind(db, settings.files_root)
            except Exception:  # a database not reachable yet: the check is a hint, never a gate
                log.exception("could not check the files library against its database")
            else:
                if left:
                    log.critical(
                        "The files library is empty but the database names files: the volume "
                        "mounted at %s is not the one that holds them. See docs/self-hosting.md.",
                        settings.files_root,
                    )
        if not state.is_configured(db):
            token = state.rotate_setup_token(db)
            url = f"{settings.app_base}/setup?token={token}"
            log.warning(
                "FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n"
                "    Open the install wizard (internal access only):\n    %s",
                url,  # log-pii-ok: this instance's OWN setup URL, and `?token=` is redacted
            )
    # No HLS session survives a restart (they live in this process), so anything still on disk is
    # a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not
    # awaited: uvicorn does not serve a single request until lifespan startup returns, so awaiting
    # the sweep would hold the port shut for as long as the rmtree runs (gigabytes of `.ts` after a
    # long uptime) — the 502 + failed post-deploy health probe this is meant to avoid. A thread
    # alone doesn't buy that; not waiting does. The sweep only touches directories no live session
    # owns, so racing it against early playback is safe.
    sweep = asyncio.create_task(_sweep_hls_orphans())
    reaper = asyncio.create_task(_reap_hls_sessions())
    # Fire and forget as well: a rebuild after an ICU upgrade takes seconds, and the title sorts serve
    # meanwhile (CONCURRENTLY) in the old index's order, as they did before the restart.
    heal = asyncio.create_task(_heal_collation_drift())
    start_scheduler()
    try:
        yield
    finally:
        log.info("Siftlode shutting down")
        # Awaited, not just cancelled: a bare `.cancel()` lets the loop close with the task still
        # pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
        # triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
        # bookkeeping honest.
        for task in (sweep, reaper, heal):
            task.cancel()
            with suppress(asyncio.CancelledError):
                await task
        shutdown_scheduler()
        # Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for
        # the rest of the read (see plex_subtitles.shutdown).
        plex_subtitles.shutdown()
        logsink.stop()  # last: write what the shutdown itself just logged


# How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`).
_HLS_REAP_EVERY_S = 60.0


async def _reap_hls_sessions() -> None:
    """The periodic reaper plex/stream.py's docstring promises. `reap_idle` used to run only inside
    the NEXT session request, so after the last playback of the day an abandoned session — its
    ffmpeg still working through the film, its directory a whole segmented copy of it — lived until
    somebody pressed play again or the process restarted.

    A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
    player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
    invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
    held one whole segmented copy of a film per item on the download volume."""
    while True:
        await asyncio.sleep(_HLS_REAP_EVERY_S)
        try:
            await asyncio.to_thread(plex_stream.reap_idle)
        except Exception:
            log.exception("HLS session reap failed")


async def _heal_collation_drift() -> None:
    """Background startup check of the ICU collation the title-sort indexes are built in (see
    `collation.heal_drift`). A failure is logged, never fatal: the sorts still answer, only possibly in
    the old order."""
    try:
        await asyncio.to_thread(collation.heal_drift, engine)
    except asyncio.CancelledError:
        raise
    except Exception:
        log.exception("ICU collation drift check failed")


async def _sweep_hls_orphans() -> None:
    """Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here
    must never take the app down, and a cancelled shutdown is not one."""
    try:
        swept = await asyncio.to_thread(plex_stream.sweep_orphans)
    except asyncio.CancelledError:
        raise
    except Exception:
        log.exception("HLS orphan sweep failed")
        return
    if swept:
        log.info("swept %d orphaned HLS segment directories", swept)


app = FastAPI(title=settings.app_name, lifespan=lifespan)


@app.exception_handler(RequestValidationError)
async def _validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
    """FastAPI's own 422, in its own shape — except that it can be ENCODED. The JSON parser accepts
    `NaN`/`Infinity`, a float field refuses them, and the refusal echoes the value back: the default
    handler then failed to serialise its own error (JSON has no NaN) and the request became a 500
    with a traceback. A non-finite value is echoed as its name instead."""
    finite = {float: lambda x: x if math.isfinite(x) else str(x)}
    return JSONResponse(
        status_code=422, content={"detail": jsonable_encoder(exc.errors(), custom_encoder=finite)}
    )


@app.exception_handler(DataError)
async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
    """A NUL byte in a string the request sent is a 422, on every route at once. Postgres cannot hold
    NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
    an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
    search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
    hold: an earlier fix guarded the by-rating_key lookup and missed the rest. Any other DataError
    is left alone — re-raised, it stays the 500 it always was."""
    if "NUL (0x00)" not in str(exc.orig):
        raise exc
    return JSONResponse(
        status_code=422, content={"detail": {"code": "invalid_input", "reason": "nul"}}
    )


app.add_middleware(
    SessionMiddleware,
    secret_key=settings.secret_key,
    same_site="lax",  # required so the cookie rides the OAuth redirect back from Google
    https_only=settings.session_https_only,  # Secure flag when served over HTTPS (prod)
)

# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")


@app.middleware("http")
async def setup_gate(request, call_next):
    """Lock the app to the install wizard until it's configured. Skips the DB entirely once setup
    completed in this process (the cached fast-path), so there's no steady-state overhead."""
    if not state.setup_complete():
        path = request.url.path
        if path.startswith(("/api/", "/auth/")) and not path.startswith(_SETUP_OPEN_PREFIXES):
            with SessionLocal() as db:
                configured = state.is_configured_cached(db)
            if not configured:
                return JSONResponse({"detail": "This instance isn't set up yet."}, status_code=503)
    return await call_next(request)


# SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL
# before the first dependency, so without this the anonymous login route is a free memory/time sink.
#
# WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this
# comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a
# stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly
# OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints:
#
#   * inside CORS and the security headers, so the 413 is decorated like every other response. A
#     refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang,
#     and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what
#     bodylimit.py exists to avoid.
#   * outside the setup gate, the session and every auth dependency under the router — none of which
#     run for a body that is already too large. The setup gate matters specifically: on an instance
#     that is not yet configured it opens a DB session per request, which is precisely the anonymous
#     pre-auth work SEC-20 exists to remove.
app.add_middleware(bodylimit.BodyLimitMiddleware)

if settings.frontend_origin:
    app.add_middleware(
        CORSMiddleware,
        allow_origins=[settings.frontend_origin],
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )


@app.middleware("http")
async def static_cache_headers(request, call_next):
    """Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given
    /assets URL never changes content and a browser can hold it forever — this is what makes
    repeat loads instant and clears the Lighthouse "efficient cache lifetimes" audit. index.html
    stays no-cache (set on its own FileResponse) so a deploy is picked up at once; other
    SPA-root static files (welcome images, favicon, robots.txt) get a moderate TTL in the SPA
    fallback below."""
    response = await call_next(request)
    if request.url.path.startswith("/assets/"):
        response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
    return response


# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
# iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a
# library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
# (script/style/connect sources) is deferred — the SPA's inline styles + the YouTube/Plex embeds make
# it real work — and HSTS belongs on the TLS-terminating edge (the Caddy front door), not here.
_SECURITY_HEADERS = {
    "X-Content-Type-Options": "nosniff",
    "Referrer-Policy": "strict-origin-when-cross-origin",
    "Content-Security-Policy": "frame-ancestors 'none'",
    "X-Frame-Options": "DENY",
    "Permissions-Policy": "camera=(), microphone=(), geolocation=()",
}


@app.middleware("http")
async def security_headers(request, call_next):
    response = await call_next(request)
    for name, value in _SECURITY_HEADERS.items():
        response.headers.setdefault(name, value)
    return response


# The access log (app/logs.py): request id, route template, status, duration — one line, never a
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
# request, including what every layer above does — the body cap included, so a 413 is still logged
# with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware,
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
# place a session is actually resolved into an account) rather than from `request.session`.
app.middleware("http")(log_requests)


app.include_router(health.router)
app.include_router(auth_routes.router)
app.include_router(sync.router)
app.include_router(tags.router)
app.include_router(feed.router)
app.include_router(discover_routes.router)
app.include_router(search_routes.router)
app.include_router(me.router)
app.include_router(diagnostics_routes.router)
app.include_router(debug_sessions_routes.router)
app.include_router(notifications.router)
app.include_router(channels.router)
app.include_router(playlists.router)
app.include_router(saved_views.router)
app.include_router(plex_routes.router)
app.include_router(downloads.router)
app.include_router(downloads.admin_router)
app.include_router(files_routes.router)
app.include_router(tokens_routes.router)
app.include_router(public_routes.router)
app.include_router(admin.router)
app.include_router(config_routes.router)
app.include_router(scheduler_routes.router)
app.include_router(audit_routes.router)
app.include_router(admin_logs_routes.router)
app.include_router(runs_routes.router)
app.include_router(quota.router)
app.include_router(version.router)
app.include_router(setup_routes.router)

# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
mimetypes.add_type("image/webp", ".webp")
mimetypes.add_type("image/avif", ".avif")

# The built SPA (populated by the Docker frontend build stage).
STATIC_DIR = Path(__file__).parent / "static_spa"
# index.html is unhashed and references the content-hashed /assets bundles, so it MUST NOT be
# heuristically cached — otherwise, after a deploy, a browser keeps serving the old index.html
# (pointing at the previous bundle) and runs stale code until a hard refresh. `no-cache` lets it
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
# hashed bundles under /assets can cache forever (a new build changes their filename).
INDEX_HTML = STATIC_DIR / "index.html"
# A public share link's own page: the player alone, without the app's bundle (frontend/watch.html).
WATCH_HTML = STATIC_DIR / "watch.html"
INDEX_HEADERS = {"Cache-Control": "no-cache"}
app.mount(
    "/assets",
    StaticFiles(directory=STATIC_DIR / "assets", check_dir=False),
    name="assets",
)


@app.get("/")
async def index() -> FileResponse:
    return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)


@app.get("/watch/{token}")
async def watch_page(token: str):
    """Serve the public player page for a share link, with per-video Open Graph tags injected so the
    link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser
    ignores the extra tags and boots WatchPage; invalid/expired/password links fall back to the
    generic card."""
    from app.downloads import og

    with SessionLocal() as db:
        rendered = og.render_watch_html(token, WATCH_HTML, db)
    if rendered is None:
        return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
    return HTMLResponse(rendered, headers=INDEX_HEADERS)


@app.get("/{full_path:path}")
async def spa_fallback(full_path: str) -> FileResponse:
    # Client-side routes fall back to index.html; real API/asset paths are matched above.
    if full_path.startswith(("api/", "auth/", "healthz", "assets/")):
        raise HTTPException(status_code=404)
    # Serve real files that live at the SPA root (Vite copies public/ there — e.g. the landing
    # screenshots under /welcome/, favicon). /assets is already mounted above; everything else
    # that isn't a real file is a client-side route → index.html. Guard against path traversal.
    if full_path:
        candidate = (STATIC_DIR / full_path).resolve()
        if candidate.is_file() and STATIC_DIR.resolve() in candidate.parents:
            # Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have
            # stable names, so a moderate cache is safe and satisfies the cache-lifetime audit.
            return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"})
    # A /watch/ path the route above does not match (a trailing segment, no token) is still the
    # player's: WatchPage reads the token off the path and says when a link is gone.
    if full_path.startswith("watch/"):
        return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
    return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)