chore(test): the DB lane runs on every core — each pytest-xdist worker gets its own database

This commit is contained in:
2026-10-16 01:13:31 +02:00
parent 2b6d968833
commit 9251918e08
7 changed files with 116 additions and 4 deletions
+4 -1
View File
@@ -16,6 +16,9 @@ services:
POSTGRES_DB: siftlode_test POSTGRES_DB: siftlode_test
tmpfs: tmpfs:
- /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run
# One database per pytest-xdist worker, each through the app's own pool (5 + 10 overflow): the default 100
# connections would run out at 8 workers.
command: ["postgres", "-c", "max_connections=300"]
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"] test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"]
interval: 1s interval: 1s
@@ -37,4 +40,4 @@ services:
- ./:/app - ./:/app
working_dir: /app working_dir: /app
# Coverage is on demand, not in the gate: see docs/development.md. # Coverage is on demand, not in the gate: see docs/development.md.
command: ["python", "-m", "pytest"] command: ["python", "-m", "pytest", "-n", "auto"]
+6 -1
View File
@@ -7,4 +7,9 @@
# not an unknown marker that quietly does nothing. # not an unknown marker that quietly does nothing.
testpaths = tests testpaths = tests
pythonpath = . pythonpath = .
addopts = -q -rs --strict-markers # `-p tests.db_per_worker`: under pytest-xdist (`-n`) every worker gets its own database — DATABASE_URL + `_gw<N>`,
# created on first use — because the `db` fixture's TRUNCATE-per-test isolation assumes one process per database.
# It is a plugin so it runs before conftest imports the app (whose settings read DATABASE_URL once). Serial: no-op.
# The file is a verbatim copy of a shared tool (see its docstring); change the source, not the copy.
addopts = -q -rs --strict-markers -p tests.db_per_worker
db_per_worker_env = DATABASE_URL
+3
View File
@@ -16,3 +16,6 @@ pytest-cov==7.1.0
ruff==0.15.21 ruff==0.15.21
httpx2==2.13.0 httpx2==2.13.0
pip-audit==2.10.1 pip-audit==2.10.1
# pytest-xdist: the DB lane runs on every core (`-n auto` in docker-compose.test.yml); tests/db_per_worker.py gives
# each worker its own database. Same tests, same strictness — only side by side.
pytest-xdist==3.8.0
+94
View File
@@ -0,0 +1,94 @@
"""Give every pytest-xdist worker its own database, so a suite that shares one test database can run in parallel.
Generic (source of truth: the memory repo's tools/pytest-parallel/db_per_worker.py); a project carries a verbatim copy
next to its conftest — tests often run in a container that cannot see this repo — and
`tools/pytest-parallel/sync-check.sh <copy>` reports drift. Keep the copy byte-identical; change it here.
Use, as a pytest plugin (it then runs before any conftest, i.e. before the app reads its settings):
pytest.ini: addopts = -p tests.db_per_worker (the copy's import path; `python -m pytest` puts cwd on sys.path)
db_per_worker_env = DATABASE_URL (the environment variable that holds the test database)
or call `apply("<ENV VAR>")` yourself before the app is imported. The variable may hold:
- a Postgres URL (`postgresql://…/name`, `postgresql+psycopg://…`): the worker gets `name_gw<N>`, created from the
base database's server if missing (empty — the project's own schema fixture migrates it, as for one process);
- a SQLite URL (`sqlite:///path.db`) or a plain file path: the worker gets `path-gw<N>.db`.
Without xdist (no PYTEST_XDIST_WORKER) or with the variable unset it changes nothing, so a serial run is unaffected.
"""
from __future__ import annotations
import os
import re
from urllib.parse import urlsplit, urlunsplit
import pytest
_PG = re.compile(r"^postgres(?:ql)?(?:\+\w+)?://", re.I)
def pytest_addoption(parser: pytest.Parser) -> None:
parser.addini(
"db_per_worker_env",
"env var holding the test database (a Postgres URL, SQLite URL or file path)",
)
@pytest.hookimpl(tryfirst=True)
def pytest_load_initial_conftests(early_config: pytest.Config) -> None:
var = early_config.getini("db_per_worker_env")
if var:
apply(str(var))
def apply(var: str) -> str | None:
worker = os.environ.get("PYTEST_XDIST_WORKER")
value = os.environ.get(var)
if not worker or not value:
return value
if _PG.match(value):
new = _postgres(value, worker)
else:
new = _file(value, worker)
os.environ[var] = new
return new
def _postgres(url: str, worker: str) -> str:
parts = urlsplit(url)
base = parts.path.lstrip("/")
name = f"{base}_{worker}"
if not re.fullmatch(r"\w+", name):
raise ValueError(f"db_per_worker: refusing an unusual database name {name!r}")
# The server, through the BASE database (it exists; `postgres` may not be reachable for this role).
plain = urlunsplit(
(re.sub(r"\+\w+$", "", parts.scheme), parts.netloc, parts.path, parts.query, "")
)
_create_if_missing(plain, name)
return urlunsplit((parts.scheme, parts.netloc, f"/{name}", parts.query, parts.fragment))
def _create_if_missing(dsn: str, name: str) -> None:
try:
import psycopg # psycopg 3
conn = psycopg.connect(dsn, autocommit=True)
except ImportError:
import psycopg2 # fallback for projects on psycopg2
conn = psycopg2.connect(dsn)
conn.autocommit = True
try:
cur = conn.cursor()
cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,))
if cur.fetchone() is None:
cur.execute(f'CREATE DATABASE "{name}"') # name checked to be \w+ above
finally:
conn.close()
def _file(value: str, worker: str) -> str:
prefix = ""
m = re.match(r"^(sqlite(?:\+\w+)?:///)(.*)$", value, re.I)
if m:
prefix, value = m.group(1), m.group(2)
root, ext = os.path.splitext(value)
return f"{prefix}{root}-{worker}{ext}"
+3
View File
@@ -107,6 +107,9 @@ Coverage is on demand, never enforced and not in the gate (E35): `npm run test -
`frontend/`, and for the backend `frontend/`, and for the backend
`docker compose -f backend/docker-compose.test.yml run --rm test python -m pytest --cov=app` (then `docker compose -f backend/docker-compose.test.yml run --rm test python -m pytest --cov=app` (then
`… down -v`). `… down -v`).
The DB lane runs on every core (`pytest -n auto`, pytest-xdist): `tests/db_per_worker.py` gives each worker its own
database (`siftlode_test_gw<N>`, created and migrated on first use), so the same tests run side by side. A serial
run (no `-n`) still works and uses `siftlode_test` itself.
The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release
requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails; requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails;
exceptions in `.trivyignore.yaml`, each with a reason and an expiry). exceptions in `.trivyignore.yaml`, each with a reason and an expiry).
+4 -1
View File
@@ -552,7 +552,10 @@ describe("the gate register (docs/gates.md)", () => {
"[pytest]", "[pytest]",
"testpaths = tests", "testpaths = tests",
"pythonpath = .", "pythonpath = .",
"addopts = -q -rs --strict-markers", // `-p tests.db_per_worker` selects nothing and filters nothing: under `-n` it hands each worker its
// own database (DATABASE_URL + `_gw<N>`), so the same tests can run side by side
"addopts = -q -rs --strict-markers -p tests.db_per_worker",
"db_per_worker_env = DATABASE_URL",
]); ]);
}); });
+2 -1
View File
@@ -1232,6 +1232,7 @@
" POSTGRES_DB: siftlode_test", " POSTGRES_DB: siftlode_test",
" tmpfs:", " tmpfs:",
" - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run", " - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run",
" command: [\"postgres\", \"-c\", \"max_connections=300\"]",
" healthcheck:", " healthcheck:",
" test: [\"CMD-SHELL\", \"pg_isready -U siftlode -d siftlode_test\"]", " test: [\"CMD-SHELL\", \"pg_isready -U siftlode -d siftlode_test\"]",
" interval: 1s", " interval: 1s",
@@ -1250,7 +1251,7 @@
" volumes:", " volumes:",
" - ./:/app", " - ./:/app",
" working_dir: /app", " working_dir: /app",
" command: [\"python\", \"-m\", \"pytest\"]" " command: [\"python\", \"-m\", \"pytest\", \"-n\", \"auto\"]"
], ],
"frontend/.prettierignore": [ "frontend/.prettierignore": [
"dist", "dist",