peter
54a6bbf07f
- GC: the expiry warning marks each file conditionally on the expiry it saw, one commit each (a file deleted meanwhile no longer fails the run; one re-timed keeps its fresh expiry unwarned); eviction counts the disk again when a candidate was freed meanwhile, so it never evicts for space already free. - Folders: a retention change locks the folder and its files, and a finishing file reads the retention under a share lock, so neither misses the other; a folder delete locks the files it takes home (a file moved out meanwhile stays put, one deleted meanwhile no longer fails the delete). - Upload: the folder is checked before the probe and held only once the job names it. - Dedup enqueue locks the cached asset, so a release meanwhile yields a fresh one instead of a 500. - Mount check reads the kernel's mount table instead of comparing devices (the vfs storage driver gave a mounted volume the device of /), and only inside a container; the API also logs a fresh, empty volume in the library's place. - Compose files fall back to DOWNLOAD_HOST_PATH; the upgrade notes cover the from-source pull. - Folder routes call owned_or_404 directly.
412 lines
20 KiB
Python
412 lines
20 KiB
Python
import asyncio
|
|
import math
|
|
import mimetypes
|
|
from contextlib import asynccontextmanager, suppress
|
|
from pathlib import Path
|
|
|
|
from fastapi import FastAPI, HTTPException, Request
|
|
from fastapi.encoders import jsonable_encoder
|
|
from fastapi.exceptions import RequestValidationError
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from sqlalchemy.exc import DataError
|
|
from starlette.middleware.sessions import SessionMiddleware
|
|
|
|
# Logging is configured HERE, before the app's own modules are imported, so a line one of them logs
|
|
# WHILE importing already has the format (a logger is configured whenever it is created —
|
|
# app/logs.py — but a line emitted before `configure_logging` goes out raw). ONE format for the api,
|
|
# the worker, the download child and alembic alike — this replaces the old uvicorn --log-config +
|
|
# the hand-rolled fallback that used to live here, and uvicorn's access log is replaced by the
|
|
# `http.request` middleware below. The imports after it are E402 on purpose, each one marked.
|
|
from app import logs
|
|
from app.logs import configure_logging, current_level, log_requests, set_level
|
|
|
|
configure_logging("api")
|
|
|
|
log = logs.get_logger(__name__)
|
|
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
|
from app.config import settings # noqa: E402 — logging is configured first
|
|
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
|
from app.downloads import service as download_service # noqa: E402 — logging is configured first
|
|
from app.downloads import storage # noqa: E402 — logging is configured first
|
|
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
|
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
|
from app.routes import ( # noqa: E402 — logging is configured first
|
|
admin,
|
|
admin_logs as admin_logs_routes,
|
|
audit as audit_routes,
|
|
auth as auth_routes,
|
|
channels,
|
|
config as config_routes,
|
|
debug_sessions as debug_sessions_routes,
|
|
diagnostics as diagnostics_routes,
|
|
discover as discover_routes,
|
|
downloads,
|
|
feed,
|
|
files as files_routes,
|
|
health,
|
|
me,
|
|
notifications,
|
|
playlists,
|
|
plex as plex_routes,
|
|
public as public_routes,
|
|
quota,
|
|
saved_views,
|
|
scheduler as scheduler_routes,
|
|
search as search_routes,
|
|
setup as setup_routes,
|
|
sync,
|
|
tags,
|
|
version,
|
|
)
|
|
from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
log.info("Siftlode starting up")
|
|
problem = storage.files_root_problem(settings.files_root)
|
|
if problem:
|
|
# The rest of the app still serves; the files library cannot (the worker refuses to start).
|
|
log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem)
|
|
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
|
logsink.start()
|
|
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
|
# the wizard URL so the operator can open it (the app runs in setup mode until they finish).
|
|
with SessionLocal() as db:
|
|
# The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap
|
|
# default, so a level chosen in the UI survives a restart without touching the environment.
|
|
# Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from
|
|
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
|
try:
|
|
set_level(sysconfig.get_str(db, "log_level"))
|
|
except Exception: # any DB/config failure keeps the bootstrap level
|
|
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
|
# and `%s` of the exception does not say which of them raised.
|
|
log.exception("could not read the stored log level; staying at %s", current_level())
|
|
if not problem:
|
|
try:
|
|
left = download_service.library_left_behind(db, settings.files_root)
|
|
except Exception: # a database not reachable yet: the check is a hint, never a gate
|
|
log.exception("could not check the files library against its database")
|
|
else:
|
|
if left:
|
|
log.critical(
|
|
"The files library is empty but the database names files: the volume "
|
|
"mounted at %s is not the one that holds them. See docs/self-hosting.md.",
|
|
settings.files_root,
|
|
)
|
|
if not state.is_configured(db):
|
|
token = state.rotate_setup_token(db)
|
|
url = f"{settings.app_base}/setup?token={token}"
|
|
log.warning(
|
|
"FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n"
|
|
" Open the install wizard (internal access only):\n %s",
|
|
url, # log-pii-ok: this instance's OWN setup URL, and `?token=` is redacted
|
|
)
|
|
# No HLS session survives a restart (they live in this process), so anything still on disk is
|
|
# a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not
|
|
# awaited: uvicorn does not serve a single request until lifespan startup returns, so awaiting
|
|
# the sweep would hold the port shut for as long as the rmtree runs (gigabytes of `.ts` after a
|
|
# long uptime) — the 502 + failed post-deploy health probe this is meant to avoid. A thread
|
|
# alone doesn't buy that; not waiting does. The sweep only touches directories no live session
|
|
# owns, so racing it against early playback is safe.
|
|
sweep = asyncio.create_task(_sweep_hls_orphans())
|
|
reaper = asyncio.create_task(_reap_hls_sessions())
|
|
# Fire and forget as well: a rebuild after an ICU upgrade takes seconds, and the title sorts serve
|
|
# meanwhile (CONCURRENTLY) in the old index's order, as they did before the restart.
|
|
heal = asyncio.create_task(_heal_collation_drift())
|
|
start_scheduler()
|
|
try:
|
|
yield
|
|
finally:
|
|
log.info("Siftlode shutting down")
|
|
# Awaited, not just cancelled: a bare `.cancel()` lets the loop close with the task still
|
|
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
|
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
|
# bookkeeping honest.
|
|
for task in (sweep, reaper, heal):
|
|
task.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await task
|
|
shutdown_scheduler()
|
|
# Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for
|
|
# the rest of the read (see plex_subtitles.shutdown).
|
|
plex_subtitles.shutdown()
|
|
logsink.stop() # last: write what the shutdown itself just logged
|
|
|
|
|
|
# How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`).
|
|
_HLS_REAP_EVERY_S = 60.0
|
|
|
|
|
|
async def _reap_hls_sessions() -> None:
|
|
"""The periodic reaper plex/stream.py's docstring promises. `reap_idle` used to run only inside
|
|
the NEXT session request, so after the last playback of the day an abandoned session — its
|
|
ffmpeg still working through the film, its directory a whole segmented copy of it — lived until
|
|
somebody pressed play again or the process restarted.
|
|
|
|
A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
|
|
player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
|
|
invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
|
|
held one whole segmented copy of a film per item on the download volume."""
|
|
while True:
|
|
await asyncio.sleep(_HLS_REAP_EVERY_S)
|
|
try:
|
|
await asyncio.to_thread(plex_stream.reap_idle)
|
|
except Exception:
|
|
log.exception("HLS session reap failed")
|
|
|
|
|
|
async def _heal_collation_drift() -> None:
|
|
"""Background startup check of the ICU collation the title-sort indexes are built in (see
|
|
`collation.heal_drift`). A failure is logged, never fatal: the sorts still answer, only possibly in
|
|
the old order."""
|
|
try:
|
|
await asyncio.to_thread(collation.heal_drift, engine)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
log.exception("ICU collation drift check failed")
|
|
|
|
|
|
async def _sweep_hls_orphans() -> None:
|
|
"""Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here
|
|
must never take the app down, and a cancelled shutdown is not one."""
|
|
try:
|
|
swept = await asyncio.to_thread(plex_stream.sweep_orphans)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
log.exception("HLS orphan sweep failed")
|
|
return
|
|
if swept:
|
|
log.info("swept %d orphaned HLS segment directories", swept)
|
|
|
|
|
|
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
|
|
|
|
|
@app.exception_handler(RequestValidationError)
|
|
async def _validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
|
"""FastAPI's own 422, in its own shape — except that it can be ENCODED. The JSON parser accepts
|
|
`NaN`/`Infinity`, a float field refuses them, and the refusal echoes the value back: the default
|
|
handler then failed to serialise its own error (JSON has no NaN) and the request became a 500
|
|
with a traceback. A non-finite value is echoed as its name instead."""
|
|
finite = {float: lambda x: x if math.isfinite(x) else str(x)}
|
|
return JSONResponse(
|
|
status_code=422, content={"detail": jsonable_encoder(exc.errors(), custom_encoder=finite)}
|
|
)
|
|
|
|
|
|
@app.exception_handler(DataError)
|
|
async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
|
|
"""A NUL byte in a string the request sent is a 422, on every route at once. Postgres cannot hold
|
|
NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
|
|
an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
|
|
search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
|
|
hold: an earlier fix guarded the by-rating_key lookup and missed the rest. Any other DataError
|
|
is left alone — re-raised, it stays the 500 it always was."""
|
|
if "NUL (0x00)" not in str(exc.orig):
|
|
raise exc
|
|
return JSONResponse(
|
|
status_code=422, content={"detail": {"code": "invalid_input", "reason": "nul"}}
|
|
)
|
|
|
|
|
|
app.add_middleware(
|
|
SessionMiddleware,
|
|
secret_key=settings.secret_key,
|
|
same_site="lax", # required so the cookie rides the OAuth redirect back from Google
|
|
https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod)
|
|
)
|
|
|
|
# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
|
|
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
|
|
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")
|
|
|
|
|
|
@app.middleware("http")
|
|
async def setup_gate(request, call_next):
|
|
"""Lock the app to the install wizard until it's configured. Skips the DB entirely once setup
|
|
completed in this process (the cached fast-path), so there's no steady-state overhead."""
|
|
if not state.setup_complete():
|
|
path = request.url.path
|
|
if path.startswith(("/api/", "/auth/")) and not path.startswith(_SETUP_OPEN_PREFIXES):
|
|
with SessionLocal() as db:
|
|
configured = state.is_configured_cached(db)
|
|
if not configured:
|
|
return JSONResponse({"detail": "This instance isn't set up yet."}, status_code=503)
|
|
return await call_next(request)
|
|
|
|
|
|
# SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL
|
|
# before the first dependency, so without this the anonymous login route is a free memory/time sink.
|
|
#
|
|
# WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this
|
|
# comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a
|
|
# stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly
|
|
# OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints:
|
|
#
|
|
# * inside CORS and the security headers, so the 413 is decorated like every other response. A
|
|
# refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang,
|
|
# and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what
|
|
# bodylimit.py exists to avoid.
|
|
# * outside the setup gate, the session and every auth dependency under the router — none of which
|
|
# run for a body that is already too large. The setup gate matters specifically: on an instance
|
|
# that is not yet configured it opens a DB session per request, which is precisely the anonymous
|
|
# pre-auth work SEC-20 exists to remove.
|
|
app.add_middleware(bodylimit.BodyLimitMiddleware)
|
|
|
|
if settings.frontend_origin:
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=[settings.frontend_origin],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
|
|
@app.middleware("http")
|
|
async def static_cache_headers(request, call_next):
|
|
"""Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given
|
|
/assets URL never changes content and a browser can hold it forever — this is what makes
|
|
repeat loads instant and clears the Lighthouse "efficient cache lifetimes" audit. index.html
|
|
stays no-cache (set on its own FileResponse) so a deploy is picked up at once; other
|
|
SPA-root static files (welcome images, favicon, robots.txt) get a moderate TTL in the SPA
|
|
fallback below."""
|
|
response = await call_next(request)
|
|
if request.url.path.startswith("/assets/"):
|
|
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
|
return response
|
|
|
|
|
|
# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
|
|
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
|
|
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
|
|
# iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a
|
|
# library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and
|
|
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
|
|
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
|
|
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
|
|
# (script/style/connect sources) is deferred — the SPA's inline styles + the YouTube/Plex embeds make
|
|
# it real work — and HSTS belongs on the TLS-terminating edge (the Caddy front door), not here.
|
|
_SECURITY_HEADERS = {
|
|
"X-Content-Type-Options": "nosniff",
|
|
"Referrer-Policy": "strict-origin-when-cross-origin",
|
|
"Content-Security-Policy": "frame-ancestors 'none'",
|
|
"X-Frame-Options": "DENY",
|
|
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
|
|
}
|
|
|
|
|
|
@app.middleware("http")
|
|
async def security_headers(request, call_next):
|
|
response = await call_next(request)
|
|
for name, value in _SECURITY_HEADERS.items():
|
|
response.headers.setdefault(name, value)
|
|
return response
|
|
|
|
|
|
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
|
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
|
# request, including what every layer above does — the body cap included, so a 413 is still logged
|
|
# with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware,
|
|
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
|
# place a session is actually resolved into an account) rather than from `request.session`.
|
|
app.middleware("http")(log_requests)
|
|
|
|
|
|
app.include_router(health.router)
|
|
app.include_router(auth_routes.router)
|
|
app.include_router(sync.router)
|
|
app.include_router(tags.router)
|
|
app.include_router(feed.router)
|
|
app.include_router(discover_routes.router)
|
|
app.include_router(search_routes.router)
|
|
app.include_router(me.router)
|
|
app.include_router(diagnostics_routes.router)
|
|
app.include_router(debug_sessions_routes.router)
|
|
app.include_router(notifications.router)
|
|
app.include_router(channels.router)
|
|
app.include_router(playlists.router)
|
|
app.include_router(saved_views.router)
|
|
app.include_router(plex_routes.router)
|
|
app.include_router(downloads.router)
|
|
app.include_router(downloads.admin_router)
|
|
app.include_router(files_routes.router)
|
|
app.include_router(public_routes.router)
|
|
app.include_router(admin.router)
|
|
app.include_router(config_routes.router)
|
|
app.include_router(scheduler_routes.router)
|
|
app.include_router(audit_routes.router)
|
|
app.include_router(admin_logs_routes.router)
|
|
app.include_router(quota.router)
|
|
app.include_router(version.router)
|
|
app.include_router(setup_routes.router)
|
|
|
|
# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
|
|
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
|
|
mimetypes.add_type("image/webp", ".webp")
|
|
mimetypes.add_type("image/avif", ".avif")
|
|
|
|
# The built SPA (populated by the Docker frontend build stage).
|
|
STATIC_DIR = Path(__file__).parent / "static_spa"
|
|
# index.html is unhashed and references the content-hashed /assets bundles, so it MUST NOT be
|
|
# heuristically cached — otherwise, after a deploy, a browser keeps serving the old index.html
|
|
# (pointing at the previous bundle) and runs stale code until a hard refresh. `no-cache` lets it
|
|
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
|
|
# hashed bundles under /assets can cache forever (a new build changes their filename).
|
|
INDEX_HTML = STATIC_DIR / "index.html"
|
|
# A public share link's own page: the player alone, without the app's bundle (frontend/watch.html).
|
|
WATCH_HTML = STATIC_DIR / "watch.html"
|
|
INDEX_HEADERS = {"Cache-Control": "no-cache"}
|
|
app.mount(
|
|
"/assets",
|
|
StaticFiles(directory=STATIC_DIR / "assets", check_dir=False),
|
|
name="assets",
|
|
)
|
|
|
|
|
|
@app.get("/")
|
|
async def index() -> FileResponse:
|
|
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
|
|
|
|
|
@app.get("/watch/{token}")
|
|
async def watch_page(token: str):
|
|
"""Serve the public player page for a share link, with per-video Open Graph tags injected so the
|
|
link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser
|
|
ignores the extra tags and boots WatchPage; invalid/expired/password links fall back to the
|
|
generic card."""
|
|
from app.downloads import og
|
|
|
|
with SessionLocal() as db:
|
|
rendered = og.render_watch_html(token, WATCH_HTML, db)
|
|
if rendered is None:
|
|
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
|
return HTMLResponse(rendered, headers=INDEX_HEADERS)
|
|
|
|
|
|
@app.get("/{full_path:path}")
|
|
async def spa_fallback(full_path: str) -> FileResponse:
|
|
# Client-side routes fall back to index.html; real API/asset paths are matched above.
|
|
if full_path.startswith(("api/", "auth/", "healthz", "assets/")):
|
|
raise HTTPException(status_code=404)
|
|
# Serve real files that live at the SPA root (Vite copies public/ there — e.g. the landing
|
|
# screenshots under /welcome/, favicon). /assets is already mounted above; everything else
|
|
# that isn't a real file is a client-side route → index.html. Guard against path traversal.
|
|
if full_path:
|
|
candidate = (STATIC_DIR / full_path).resolve()
|
|
if candidate.is_file() and STATIC_DIR.resolve() in candidate.parents:
|
|
# Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have
|
|
# stable names, so a moderate cache is safe and satisfies the cache-lifetime audit.
|
|
return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"})
|
|
# A /watch/ path the route above does not match (a trailing segment, no token) is still the
|
|
# player's: WatchPage reads the token off the path and says when a link is gone.
|
|
if full_path.startswith("watch/"):
|
|
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
|
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|