chore(test): the DB lane runs on every core — each pytest-xdist worker gets its own database
This commit is contained in:
@@ -16,6 +16,9 @@ services:
|
|||||||
POSTGRES_DB: siftlode_test
|
POSTGRES_DB: siftlode_test
|
||||||
tmpfs:
|
tmpfs:
|
||||||
- /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run
|
- /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run
|
||||||
|
# One database per pytest-xdist worker, each through the app's own pool (5 + 10 overflow): the default 100
|
||||||
|
# connections would run out at 8 workers.
|
||||||
|
command: ["postgres", "-c", "max_connections=300"]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"]
|
test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"]
|
||||||
interval: 1s
|
interval: 1s
|
||||||
@@ -37,4 +40,4 @@ services:
|
|||||||
- ./:/app
|
- ./:/app
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
# Coverage is on demand, not in the gate: see docs/development.md.
|
# Coverage is on demand, not in the gate: see docs/development.md.
|
||||||
command: ["python", "-m", "pytest"]
|
command: ["python", "-m", "pytest", "-n", "auto"]
|
||||||
|
|||||||
+6
-1
@@ -7,4 +7,9 @@
|
|||||||
# not an unknown marker that quietly does nothing.
|
# not an unknown marker that quietly does nothing.
|
||||||
testpaths = tests
|
testpaths = tests
|
||||||
pythonpath = .
|
pythonpath = .
|
||||||
addopts = -q -rs --strict-markers
|
# `-p tests.db_per_worker`: under pytest-xdist (`-n`) every worker gets its own database — DATABASE_URL + `_gw<N>`,
|
||||||
|
# created on first use — because the `db` fixture's TRUNCATE-per-test isolation assumes one process per database.
|
||||||
|
# It is a plugin so it runs before conftest imports the app (whose settings read DATABASE_URL once). Serial: no-op.
|
||||||
|
# The file is a verbatim copy of a shared tool (see its docstring); change the source, not the copy.
|
||||||
|
addopts = -q -rs --strict-markers -p tests.db_per_worker
|
||||||
|
db_per_worker_env = DATABASE_URL
|
||||||
|
|||||||
@@ -16,3 +16,6 @@ pytest-cov==7.1.0
|
|||||||
ruff==0.15.21
|
ruff==0.15.21
|
||||||
httpx2==2.13.0
|
httpx2==2.13.0
|
||||||
pip-audit==2.10.1
|
pip-audit==2.10.1
|
||||||
|
# pytest-xdist: the DB lane runs on every core (`-n auto` in docker-compose.test.yml); tests/db_per_worker.py gives
|
||||||
|
# each worker its own database. Same tests, same strictness — only side by side.
|
||||||
|
pytest-xdist==3.8.0
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""Give every pytest-xdist worker its own database, so a suite that shares one test database can run in parallel.
|
||||||
|
|
||||||
|
Generic (source of truth: the memory repo's tools/pytest-parallel/db_per_worker.py); a project carries a verbatim copy
|
||||||
|
next to its conftest — tests often run in a container that cannot see this repo — and
|
||||||
|
`tools/pytest-parallel/sync-check.sh <copy>` reports drift. Keep the copy byte-identical; change it here.
|
||||||
|
|
||||||
|
Use, as a pytest plugin (it then runs before any conftest, i.e. before the app reads its settings):
|
||||||
|
pytest.ini: addopts = -p tests.db_per_worker (the copy's import path; `python -m pytest` puts cwd on sys.path)
|
||||||
|
db_per_worker_env = DATABASE_URL (the environment variable that holds the test database)
|
||||||
|
or call `apply("<ENV VAR>")` yourself before the app is imported. The variable may hold:
|
||||||
|
- a Postgres URL (`postgresql://…/name`, `postgresql+psycopg://…`): the worker gets `name_gw<N>`, created from the
|
||||||
|
base database's server if missing (empty — the project's own schema fixture migrates it, as for one process);
|
||||||
|
- a SQLite URL (`sqlite:///path.db`) or a plain file path: the worker gets `path-gw<N>.db`.
|
||||||
|
Without xdist (no PYTEST_XDIST_WORKER) or with the variable unset it changes nothing, so a serial run is unaffected.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
_PG = re.compile(r"^postgres(?:ql)?(?:\+\w+)?://", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||||
|
parser.addini(
|
||||||
|
"db_per_worker_env",
|
||||||
|
"env var holding the test database (a Postgres URL, SQLite URL or file path)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.hookimpl(tryfirst=True)
|
||||||
|
def pytest_load_initial_conftests(early_config: pytest.Config) -> None:
|
||||||
|
var = early_config.getini("db_per_worker_env")
|
||||||
|
if var:
|
||||||
|
apply(str(var))
|
||||||
|
|
||||||
|
|
||||||
|
def apply(var: str) -> str | None:
|
||||||
|
worker = os.environ.get("PYTEST_XDIST_WORKER")
|
||||||
|
value = os.environ.get(var)
|
||||||
|
if not worker or not value:
|
||||||
|
return value
|
||||||
|
if _PG.match(value):
|
||||||
|
new = _postgres(value, worker)
|
||||||
|
else:
|
||||||
|
new = _file(value, worker)
|
||||||
|
os.environ[var] = new
|
||||||
|
return new
|
||||||
|
|
||||||
|
|
||||||
|
def _postgres(url: str, worker: str) -> str:
|
||||||
|
parts = urlsplit(url)
|
||||||
|
base = parts.path.lstrip("/")
|
||||||
|
name = f"{base}_{worker}"
|
||||||
|
if not re.fullmatch(r"\w+", name):
|
||||||
|
raise ValueError(f"db_per_worker: refusing an unusual database name {name!r}")
|
||||||
|
# The server, through the BASE database (it exists; `postgres` may not be reachable for this role).
|
||||||
|
plain = urlunsplit(
|
||||||
|
(re.sub(r"\+\w+$", "", parts.scheme), parts.netloc, parts.path, parts.query, "")
|
||||||
|
)
|
||||||
|
_create_if_missing(plain, name)
|
||||||
|
return urlunsplit((parts.scheme, parts.netloc, f"/{name}", parts.query, parts.fragment))
|
||||||
|
|
||||||
|
|
||||||
|
def _create_if_missing(dsn: str, name: str) -> None:
|
||||||
|
try:
|
||||||
|
import psycopg # psycopg 3
|
||||||
|
|
||||||
|
conn = psycopg.connect(dsn, autocommit=True)
|
||||||
|
except ImportError:
|
||||||
|
import psycopg2 # fallback for projects on psycopg2
|
||||||
|
|
||||||
|
conn = psycopg2.connect(dsn)
|
||||||
|
conn.autocommit = True
|
||||||
|
try:
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,))
|
||||||
|
if cur.fetchone() is None:
|
||||||
|
cur.execute(f'CREATE DATABASE "{name}"') # name checked to be \w+ above
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _file(value: str, worker: str) -> str:
|
||||||
|
prefix = ""
|
||||||
|
m = re.match(r"^(sqlite(?:\+\w+)?:///)(.*)$", value, re.I)
|
||||||
|
if m:
|
||||||
|
prefix, value = m.group(1), m.group(2)
|
||||||
|
root, ext = os.path.splitext(value)
|
||||||
|
return f"{prefix}{root}-{worker}{ext}"
|
||||||
@@ -107,6 +107,9 @@ Coverage is on demand, never enforced and not in the gate (E35): `npm run test -
|
|||||||
`frontend/`, and for the backend
|
`frontend/`, and for the backend
|
||||||
`docker compose -f backend/docker-compose.test.yml run --rm test python -m pytest --cov=app` (then
|
`docker compose -f backend/docker-compose.test.yml run --rm test python -m pytest --cov=app` (then
|
||||||
`… down -v`).
|
`… down -v`).
|
||||||
|
The DB lane runs on every core (`pytest -n auto`, pytest-xdist): `tests/db_per_worker.py` gives each worker its own
|
||||||
|
database (`siftlode_test_gw<N>`, created and migrated on first use), so the same tests run side by side. A serial
|
||||||
|
run (no `-n`) still works and uses `siftlode_test` itself.
|
||||||
The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release
|
The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release
|
||||||
requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails;
|
requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails;
|
||||||
exceptions in `.trivyignore.yaml`, each with a reason and an expiry).
|
exceptions in `.trivyignore.yaml`, each with a reason and an expiry).
|
||||||
|
|||||||
@@ -552,7 +552,10 @@ describe("the gate register (docs/gates.md)", () => {
|
|||||||
"[pytest]",
|
"[pytest]",
|
||||||
"testpaths = tests",
|
"testpaths = tests",
|
||||||
"pythonpath = .",
|
"pythonpath = .",
|
||||||
"addopts = -q -rs --strict-markers",
|
// `-p tests.db_per_worker` selects nothing and filters nothing: under `-n` it hands each worker its
|
||||||
|
// own database (DATABASE_URL + `_gw<N>`), so the same tests can run side by side
|
||||||
|
"addopts = -q -rs --strict-markers -p tests.db_per_worker",
|
||||||
|
"db_per_worker_env = DATABASE_URL",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1232,6 +1232,7 @@
|
|||||||
" POSTGRES_DB: siftlode_test",
|
" POSTGRES_DB: siftlode_test",
|
||||||
" tmpfs:",
|
" tmpfs:",
|
||||||
" - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run",
|
" - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run",
|
||||||
|
" command: [\"postgres\", \"-c\", \"max_connections=300\"]",
|
||||||
" healthcheck:",
|
" healthcheck:",
|
||||||
" test: [\"CMD-SHELL\", \"pg_isready -U siftlode -d siftlode_test\"]",
|
" test: [\"CMD-SHELL\", \"pg_isready -U siftlode -d siftlode_test\"]",
|
||||||
" interval: 1s",
|
" interval: 1s",
|
||||||
@@ -1250,7 +1251,7 @@
|
|||||||
" volumes:",
|
" volumes:",
|
||||||
" - ./:/app",
|
" - ./:/app",
|
||||||
" working_dir: /app",
|
" working_dir: /app",
|
||||||
" command: [\"python\", \"-m\", \"pytest\"]"
|
" command: [\"python\", \"-m\", \"pytest\", \"-n\", \"auto\"]"
|
||||||
],
|
],
|
||||||
"frontend/.prettierignore": [
|
"frontend/.prettierignore": [
|
||||||
"dist",
|
"dist",
|
||||||
|
|||||||
Reference in New Issue
Block a user