chore(test): the DB lane runs on every core — each pytest-xdist worker gets its own database
This commit is contained in:
@@ -16,6 +16,9 @@ services:
|
||||
POSTGRES_DB: siftlode_test
|
||||
tmpfs:
|
||||
- /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run
|
||||
# One database per pytest-xdist worker, each through the app's own pool (5 + 10 overflow): the default 100
|
||||
# connections would run out at 8 workers.
|
||||
command: ["postgres", "-c", "max_connections=300"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"]
|
||||
interval: 1s
|
||||
@@ -37,4 +40,4 @@ services:
|
||||
- ./:/app
|
||||
working_dir: /app
|
||||
# Coverage is on demand, not in the gate: see docs/development.md.
|
||||
command: ["python", "-m", "pytest"]
|
||||
command: ["python", "-m", "pytest", "-n", "auto"]
|
||||
|
||||
+6
-1
@@ -7,4 +7,9 @@
|
||||
# not an unknown marker that quietly does nothing.
|
||||
testpaths = tests
|
||||
pythonpath = .
|
||||
addopts = -q -rs --strict-markers
|
||||
# `-p tests.db_per_worker`: under pytest-xdist (`-n`) every worker gets its own database — DATABASE_URL + `_gw<N>`,
|
||||
# created on first use — because the `db` fixture's TRUNCATE-per-test isolation assumes one process per database.
|
||||
# It is a plugin so it runs before conftest imports the app (whose settings read DATABASE_URL once). Serial: no-op.
|
||||
# The file is a verbatim copy of a shared tool (see its docstring); change the source, not the copy.
|
||||
addopts = -q -rs --strict-markers -p tests.db_per_worker
|
||||
db_per_worker_env = DATABASE_URL
|
||||
|
||||
@@ -16,3 +16,6 @@ pytest-cov==7.1.0
|
||||
ruff==0.15.21
|
||||
httpx2==2.13.0
|
||||
pip-audit==2.10.1
|
||||
# pytest-xdist: the DB lane runs on every core (`-n auto` in docker-compose.test.yml); tests/db_per_worker.py gives
|
||||
# each worker its own database. Same tests, same strictness — only side by side.
|
||||
pytest-xdist==3.8.0
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Give every pytest-xdist worker its own database, so a suite that shares one test database can run in parallel.
|
||||
|
||||
Generic (source of truth: the memory repo's tools/pytest-parallel/db_per_worker.py); a project carries a verbatim copy
|
||||
next to its conftest — tests often run in a container that cannot see this repo — and
|
||||
`tools/pytest-parallel/sync-check.sh <copy>` reports drift. Keep the copy byte-identical; change it here.
|
||||
|
||||
Use, as a pytest plugin (it then runs before any conftest, i.e. before the app reads its settings):
|
||||
pytest.ini: addopts = -p tests.db_per_worker (the copy's import path; `python -m pytest` puts cwd on sys.path)
|
||||
db_per_worker_env = DATABASE_URL (the environment variable that holds the test database)
|
||||
or call `apply("<ENV VAR>")` yourself before the app is imported. The variable may hold:
|
||||
- a Postgres URL (`postgresql://…/name`, `postgresql+psycopg://…`): the worker gets `name_gw<N>`, created from the
|
||||
base database's server if missing (empty — the project's own schema fixture migrates it, as for one process);
|
||||
- a SQLite URL (`sqlite:///path.db`) or a plain file path: the worker gets `path-gw<N>.db`.
|
||||
Without xdist (no PYTEST_XDIST_WORKER) or with the variable unset it changes nothing, so a serial run is unaffected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import pytest
|
||||
|
||||
_PG = re.compile(r"^postgres(?:ql)?(?:\+\w+)?://", re.I)
|
||||
|
||||
|
||||
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
parser.addini(
|
||||
"db_per_worker_env",
|
||||
"env var holding the test database (a Postgres URL, SQLite URL or file path)",
|
||||
)
|
||||
|
||||
|
||||
@pytest.hookimpl(tryfirst=True)
|
||||
def pytest_load_initial_conftests(early_config: pytest.Config) -> None:
|
||||
var = early_config.getini("db_per_worker_env")
|
||||
if var:
|
||||
apply(str(var))
|
||||
|
||||
|
||||
def apply(var: str) -> str | None:
|
||||
worker = os.environ.get("PYTEST_XDIST_WORKER")
|
||||
value = os.environ.get(var)
|
||||
if not worker or not value:
|
||||
return value
|
||||
if _PG.match(value):
|
||||
new = _postgres(value, worker)
|
||||
else:
|
||||
new = _file(value, worker)
|
||||
os.environ[var] = new
|
||||
return new
|
||||
|
||||
|
||||
def _postgres(url: str, worker: str) -> str:
|
||||
parts = urlsplit(url)
|
||||
base = parts.path.lstrip("/")
|
||||
name = f"{base}_{worker}"
|
||||
if not re.fullmatch(r"\w+", name):
|
||||
raise ValueError(f"db_per_worker: refusing an unusual database name {name!r}")
|
||||
# The server, through the BASE database (it exists; `postgres` may not be reachable for this role).
|
||||
plain = urlunsplit(
|
||||
(re.sub(r"\+\w+$", "", parts.scheme), parts.netloc, parts.path, parts.query, "")
|
||||
)
|
||||
_create_if_missing(plain, name)
|
||||
return urlunsplit((parts.scheme, parts.netloc, f"/{name}", parts.query, parts.fragment))
|
||||
|
||||
|
||||
def _create_if_missing(dsn: str, name: str) -> None:
|
||||
try:
|
||||
import psycopg # psycopg 3
|
||||
|
||||
conn = psycopg.connect(dsn, autocommit=True)
|
||||
except ImportError:
|
||||
import psycopg2 # fallback for projects on psycopg2
|
||||
|
||||
conn = psycopg2.connect(dsn)
|
||||
conn.autocommit = True
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,))
|
||||
if cur.fetchone() is None:
|
||||
cur.execute(f'CREATE DATABASE "{name}"') # name checked to be \w+ above
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _file(value: str, worker: str) -> str:
|
||||
prefix = ""
|
||||
m = re.match(r"^(sqlite(?:\+\w+)?:///)(.*)$", value, re.I)
|
||||
if m:
|
||||
prefix, value = m.group(1), m.group(2)
|
||||
root, ext = os.path.splitext(value)
|
||||
return f"{prefix}{root}-{worker}{ext}"
|
||||
@@ -107,6 +107,9 @@ Coverage is on demand, never enforced and not in the gate (E35): `npm run test -
|
||||
`frontend/`, and for the backend
|
||||
`docker compose -f backend/docker-compose.test.yml run --rm test python -m pytest --cov=app` (then
|
||||
`… down -v`).
|
||||
The DB lane runs on every core (`pytest -n auto`, pytest-xdist): `tests/db_per_worker.py` gives each worker its own
|
||||
database (`siftlode_test_gw<N>`, created and migrated on first use), so the same tests run side by side. A serial
|
||||
run (no `-n`) still works and uses `siftlode_test` itself.
|
||||
The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release
|
||||
requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails;
|
||||
exceptions in `.trivyignore.yaml`, each with a reason and an expiry).
|
||||
|
||||
@@ -552,7 +552,10 @@ describe("the gate register (docs/gates.md)", () => {
|
||||
"[pytest]",
|
||||
"testpaths = tests",
|
||||
"pythonpath = .",
|
||||
"addopts = -q -rs --strict-markers",
|
||||
// `-p tests.db_per_worker` selects nothing and filters nothing: under `-n` it hands each worker its
|
||||
// own database (DATABASE_URL + `_gw<N>`), so the same tests can run side by side
|
||||
"addopts = -q -rs --strict-markers -p tests.db_per_worker",
|
||||
"db_per_worker_env = DATABASE_URL",
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1232,6 +1232,7 @@
|
||||
" POSTGRES_DB: siftlode_test",
|
||||
" tmpfs:",
|
||||
" - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run",
|
||||
" command: [\"postgres\", \"-c\", \"max_connections=300\"]",
|
||||
" healthcheck:",
|
||||
" test: [\"CMD-SHELL\", \"pg_isready -U siftlode -d siftlode_test\"]",
|
||||
" interval: 1s",
|
||||
@@ -1250,7 +1251,7 @@
|
||||
" volumes:",
|
||||
" - ./:/app",
|
||||
" working_dir: /app",
|
||||
" command: [\"python\", \"-m\", \"pytest\"]"
|
||||
" command: [\"python\", \"-m\", \"pytest\", \"-n\", \"auto\"]"
|
||||
],
|
||||
"frontend/.prettierignore": [
|
||||
"dist",
|
||||
|
||||
Reference in New Issue
Block a user