A deployment printed four different log formats and none of them could be joined up: uvicorn's log_config.json for the API, a hand-rolled StreamHandler in main.py for anything not started by uvicorn, logging.basicConfig() in the worker and in the download child (no timestamp at all), and alembic's fileConfig. app/logs.py replaces all four with one JSON line per event (OpenTelemetry's log data model, no SDK): ts, level, severity, source, logger, event code, message, attributes, plus the correlation the context carries — request id, account, download job. Redaction is applied at the HANDLER, not at the call sites: every message, every attribute and every traceback is scrubbed on the way out, so a secret cannot leak by being forgotten at one of ~125 call sites. Passwords, tokens, cookies, secret config values and email addresses never reach a line. The access log is ours now (`http.request`): the route template instead of the concrete path, the status, the duration and the account — and never the query string, which uvicorn's access log printed on every single request (`?account=`, `?source=`, the `?token=` of a share link). A well-formed client request id is accepted and echoed in X-Request-Id, so a browser line and a server line join up. The account comes from `request.state`, filled by the auth dependency: the middleware is the outermost one, so it runs outside SessionMiddleware and in a different task context. `log_level` becomes an admin knob (sysconfig), applied without a restart and re-applied at startup — best-effort on purpose, since a logging knob must never be what stops the app from booting. Every compose service gets bounded json-file logs (3×50 MB): Docker keeps every line forever by default, and the API now writes one line per request. Verified on the localdev stack: JSON everywhere including alembic and the worker, the query strings gone, a supplied X-Request-Id echoed and logged, and three /healthz hits of which only the one made while the level was DEBUG produced a line.
142 lines
5.8 KiB
YAML
142 lines
5.8 KiB
YAML
# Self-contained local development stack — Postgres + API + scheduler, all on this machine.
|
||
#
|
||
# Fully decoupled: its own local database (a Docker volume) and its own scheduler, so there
|
||
# is no shared DB and no migration coupling with any other instance. Edit code, rebuild, and
|
||
# everything (including the background sync) runs right here.
|
||
#
|
||
# docker compose -f docker-compose.localdev.yml up -d --build
|
||
#
|
||
# Browse at http://localhost:8080 — Google login works because the OAuth redirect is
|
||
# http://localhost:8080/auth/callback. The API runs `alembic upgrade head` on startup, so it
|
||
# applies local migrations to THIS local DB only. Seed the catalog once with a pg_dump from
|
||
# another instance if you want real data (see RUNBOOK).
|
||
|
||
# Bounded container logs. Docker's json-file driver keeps every line forever by default, and the API
|
||
# now writes one structured line per request (E22 S2) — 3×50 MB per service is days of history on a
|
||
# busy instance and a hard ceiling on the disk it can take.
|
||
x-logging: &logging
|
||
driver: json-file
|
||
options:
|
||
max-size: "50m"
|
||
max-file: "3"
|
||
|
||
services:
|
||
db:
|
||
image: postgres:16-alpine
|
||
environment:
|
||
POSTGRES_USER: ${POSTGRES_USER:-siftlode}
|
||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-siftlode}
|
||
POSTGRES_DB: ${POSTGRES_DB:-siftlode}
|
||
volumes:
|
||
- pgdata:/var/lib/postgresql/data
|
||
security_opt:
|
||
- apparmor:unconfined
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-siftlode} -d ${POSTGRES_DB:-siftlode}"]
|
||
interval: 5s
|
||
timeout: 5s
|
||
retries: 12
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
api:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
args:
|
||
APP_VERSION: ${APP_VERSION:-dev}
|
||
GIT_SHA: ${GIT_SHA:-unknown}
|
||
BUILD_DATE: ${BUILD_DATE:-}
|
||
env_file: .env
|
||
environment:
|
||
# Own local database (overrides whatever DATABASE_URL is in .env).
|
||
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
|
||
# This instance owns its scheduler now (its own DB, so no double-write concern).
|
||
SCHEDULER_ENABLED: "true"
|
||
# Download center: the API serves finished files (it does NOT run the worker loop).
|
||
DOWNLOAD_ROOT: /downloads
|
||
WORKER_ENABLED: "false"
|
||
# Plex HLS remux scratch on fast container-local storage (the /downloads bind-mount is a slow
|
||
# Windows Docker Desktop mount in dev; prod uses fast local disk so it keeps the default).
|
||
PLEX_HLS_DIR: /var/tmp/plex-hls
|
||
depends_on:
|
||
db:
|
||
condition: service_healthy
|
||
# Harmless on Docker Desktop; needed on some nested/hardened Docker hosts.
|
||
security_opt:
|
||
- apparmor:unconfined
|
||
ports:
|
||
- "${APP_PORT:-8080}:8000"
|
||
volumes:
|
||
# Downloaded media lives on the host so you can inspect the generated Plex tree.
|
||
# Gitignored. The worker writes here; the API reads it to serve local downloads.
|
||
- ./downloads:/downloads
|
||
# Plex media, read-only (see the plex_media volume note below). Maps to plex_path_map.
|
||
- plex_media:/plex-media:ro
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
# Dedicated download worker: same image, runs the yt-dlp job loop instead of the API.
|
||
# It shares the DB (job queue) and the downloads mount with the API.
|
||
worker:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
args:
|
||
APP_VERSION: ${APP_VERSION:-dev}
|
||
GIT_SHA: ${GIT_SHA:-unknown}
|
||
BUILD_DATE: ${BUILD_DATE:-}
|
||
command: ["python", "-m", "app.worker"]
|
||
# tini as PID 1: reaps orphaned ffmpeg/yt-dlp grandchildren when the watchdog hard-kills a
|
||
# download child's process group (the worker reaps its own direct children via subprocess.wait).
|
||
init: true
|
||
env_file: .env
|
||
environment:
|
||
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
|
||
# The worker must not also run the scheduler; the API owns that.
|
||
SCHEDULER_ENABLED: "false"
|
||
DOWNLOAD_ROOT: /downloads
|
||
WORKER_ENABLED: "true"
|
||
depends_on:
|
||
db:
|
||
condition: service_healthy
|
||
bgutil-pot:
|
||
condition: service_started
|
||
security_opt:
|
||
- apparmor:unconfined
|
||
volumes:
|
||
- ./downloads:/downloads
|
||
- plex_media:/plex-media:ro
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
# PO-token provider: mints YouTube Proof-of-Origin tokens on demand so the worker bypasses
|
||
# bot-detection and gets high-quality formats. The worker reaches it at http://bgutil-pot:4416
|
||
# (DOWNLOAD_POT_BASE_URL). Pin the tag to match the plugin version in requirements.txt.
|
||
bgutil-pot:
|
||
image: brainicism/bgutil-ytdlp-pot-provider:1.3.1
|
||
init: true
|
||
security_opt:
|
||
- apparmor:unconfined
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
volumes:
|
||
pgdata:
|
||
# OPTIONAL (Plex integration dev testing): the Plex media, mounted READ-ONLY so the container can
|
||
# play the physical file locally (this dev box isn't the Plex host). Only referenced when the Plex
|
||
# module is enabled; harmless otherwise (Docker mounts it lazily on first container use). On prod
|
||
# (LXC on the Plex host) this is a plain host bind-mount.
|
||
#
|
||
# Local bind to a few test movies/shows kept on this box — the full nas.lan SMB share was too slow.
|
||
# Point PLEX_MEDIA_HOST_PATH (.env, gitignored, so no site path lives in the tracked compose) at the
|
||
# Docker-Desktop host path (e.g. /run/desktop/mnt/host/d/plex-test-media for D:\plex-test-media),
|
||
# mirroring the Plex server's /data/<lib>/… layout so plex_path_map (/data=/plex-media) resolves.
|
||
# (The prior CIFS/PLEX_SMB_* approach for the whole nas.lan share lives in git history.)
|
||
plex_media:
|
||
driver: local
|
||
driver_opts:
|
||
type: none
|
||
o: bind
|
||
device: "${PLEX_MEDIA_HOST_PATH:-/run/desktop/mnt/host/d/plex-test-media}"
|