peter
815083bcc4
- The queue keeps a done job by when it became done (ready_at), not by its last write: renaming, keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished. - GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a second release that frees a file another user still holds. - A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit whose parent expired still goes back to its folder, not to Uploads. - A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes that file home too, instead of failing on the foreign key. - The hash route reads at most two files at once (429 past that) and holds no connection meanwhile. - 0104's backfill dates old files by their request date only: no notice prod wrote names its job. - The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up. - Stale comments about the /downloads mount and the Downloads page.
113 lines
4.2 KiB
YAML
113 lines
4.2 KiB
YAML
# Bounded container logs. Docker's json-file driver keeps every line forever by default, and the API
|
||
# now writes one structured line per request — 3×50 MB per service is days of history on a
|
||
# busy instance and a hard ceiling on the disk it can take.
|
||
x-logging: &logging
|
||
driver: json-file
|
||
options:
|
||
max-size: "50m"
|
||
max-file: "3"
|
||
|
||
services:
|
||
db:
|
||
image: postgres:16-alpine
|
||
environment:
|
||
POSTGRES_USER: ${POSTGRES_USER:-siftlode}
|
||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-siftlode}
|
||
POSTGRES_DB: ${POSTGRES_DB:-siftlode}
|
||
volumes:
|
||
- pgdata:/var/lib/postgresql/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-siftlode} -d ${POSTGRES_DB:-siftlode}"]
|
||
interval: 5s
|
||
timeout: 5s
|
||
retries: 12
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
api:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
args:
|
||
APP_VERSION: ${APP_VERSION:-dev}
|
||
GIT_SHA: ${GIT_SHA:-unknown}
|
||
BUILD_DATE: ${BUILD_DATE:-}
|
||
env_file: .env
|
||
environment:
|
||
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
|
||
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs
|
||
# the yt-dlp job loop.
|
||
FILES_ROOT: /files
|
||
WORKER_ENABLED: "false"
|
||
depends_on:
|
||
db:
|
||
condition: service_healthy
|
||
ports:
|
||
- "${APP_PORT:-8080}:8000"
|
||
volumes:
|
||
# Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
|
||
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
|
||
- ${FILES_HOST_PATH:-files}:/files
|
||
# Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set
|
||
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the
|
||
# admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty
|
||
# named volume — harmless, and the Plex module is off by default.
|
||
- ${PLEX_MEDIA_HOST_PATH:-plex_media}:/plex-media:ro
|
||
healthcheck:
|
||
test: ["CMD", "python", "-c", "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:8000/healthz').status == 200 else 1)"]
|
||
interval: 15s
|
||
timeout: 5s
|
||
retries: 5
|
||
start_period: 25s
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
|
||
# DB (job queue) + the /files mount with the API.
|
||
worker:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
args:
|
||
APP_VERSION: ${APP_VERSION:-dev}
|
||
GIT_SHA: ${GIT_SHA:-unknown}
|
||
BUILD_DATE: ${BUILD_DATE:-}
|
||
command: ["python", "-m", "app.worker"]
|
||
# tini as PID 1: reaps orphaned ffmpeg/yt-dlp grandchildren when the watchdog hard-kills a
|
||
# download child's process group (the worker reaps its own direct children via subprocess.wait).
|
||
init: true
|
||
env_file: .env
|
||
environment:
|
||
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
|
||
SCHEDULER_ENABLED: "false"
|
||
FILES_ROOT: /files
|
||
WORKER_ENABLED: "true"
|
||
depends_on:
|
||
db:
|
||
condition: service_healthy
|
||
# Start only after the API is healthy (it applies the DB migrations on startup).
|
||
api:
|
||
condition: service_healthy
|
||
bgutil-pot:
|
||
condition: service_started
|
||
volumes:
|
||
- ${FILES_HOST_PATH:-files}:/files
|
||
# Same read-only Plex library as the API — the worker resolves library paths too.
|
||
- ${PLEX_MEDIA_HOST_PATH:-plex_media}:/plex-media:ro
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
# PO-token provider sidecar: mints YouTube Proof-of-Origin tokens so the worker beats bot-detection
|
||
# and unlocks high-quality formats (reached at http://bgutil-pot:4416, the config default).
|
||
bgutil-pot:
|
||
image: brainicism/bgutil-ytdlp-pot-provider:1.3.1
|
||
init: true
|
||
logging: *logging
|
||
restart: unless-stopped
|
||
|
||
volumes:
|
||
pgdata:
|
||
files:
|
||
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
|
||
plex_media:
|