From 9251918e08f6294a14df7077a4134996fd73e351 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 16 Oct 2026 01:13:31 +0200 Subject: [PATCH] =?UTF-8?q?chore(test):=20the=20DB=20lane=20runs=20on=20ev?= =?UTF-8?q?ery=20core=20=E2=80=94=20each=20pytest-xdist=20worker=20gets=20?= =?UTF-8?q?its=20own=20database?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/docker-compose.test.yml | 5 +- backend/pytest.ini | 7 +- backend/requirements-dev.txt | 3 + backend/tests/db_per_worker.py | 94 +++++++++++++++++++++++++ docs/development.md | 3 + frontend/src/lib/gates.contract.test.ts | 5 +- frontend/src/lib/gates.snapshot.json | 3 +- 7 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 backend/tests/db_per_worker.py diff --git a/backend/docker-compose.test.yml b/backend/docker-compose.test.yml index e3d18c67..c493d648 100644 --- a/backend/docker-compose.test.yml +++ b/backend/docker-compose.test.yml @@ -16,6 +16,9 @@ services: POSTGRES_DB: siftlode_test tmpfs: - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run + # One database per pytest-xdist worker, each through the app's own pool (5 + 10 overflow): the default 100 + # connections would run out at 8 workers. + command: ["postgres", "-c", "max_connections=300"] healthcheck: test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"] interval: 1s @@ -37,4 +40,4 @@ services: - ./:/app working_dir: /app # Coverage is on demand, not in the gate: see docs/development.md. - command: ["python", "-m", "pytest"] + command: ["python", "-m", "pytest", "-n", "auto"] diff --git a/backend/pytest.ini b/backend/pytest.ini index cc36472b..697f5814 100644 --- a/backend/pytest.ini +++ b/backend/pytest.ini @@ -7,4 +7,9 @@ # not an unknown marker that quietly does nothing. testpaths = tests pythonpath = . -addopts = -q -rs --strict-markers +# `-p tests.db_per_worker`: under pytest-xdist (`-n`) every worker gets its own database — DATABASE_URL + `_gw`, +# created on first use — because the `db` fixture's TRUNCATE-per-test isolation assumes one process per database. +# It is a plugin so it runs before conftest imports the app (whose settings read DATABASE_URL once). Serial: no-op. +# The file is a verbatim copy of a shared tool (see its docstring); change the source, not the copy. +addopts = -q -rs --strict-markers -p tests.db_per_worker +db_per_worker_env = DATABASE_URL diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index e681a5f8..d8179199 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -16,3 +16,6 @@ pytest-cov==7.1.0 ruff==0.15.21 httpx2==2.13.0 pip-audit==2.10.1 +# pytest-xdist: the DB lane runs on every core (`-n auto` in docker-compose.test.yml); tests/db_per_worker.py gives +# each worker its own database. Same tests, same strictness — only side by side. +pytest-xdist==3.8.0 diff --git a/backend/tests/db_per_worker.py b/backend/tests/db_per_worker.py new file mode 100644 index 00000000..ab7b60c0 --- /dev/null +++ b/backend/tests/db_per_worker.py @@ -0,0 +1,94 @@ +"""Give every pytest-xdist worker its own database, so a suite that shares one test database can run in parallel. + +Generic (source of truth: the memory repo's tools/pytest-parallel/db_per_worker.py); a project carries a verbatim copy +next to its conftest — tests often run in a container that cannot see this repo — and +`tools/pytest-parallel/sync-check.sh ` reports drift. Keep the copy byte-identical; change it here. + +Use, as a pytest plugin (it then runs before any conftest, i.e. before the app reads its settings): + pytest.ini: addopts = -p tests.db_per_worker (the copy's import path; `python -m pytest` puts cwd on sys.path) + db_per_worker_env = DATABASE_URL (the environment variable that holds the test database) +or call `apply("")` yourself before the app is imported. The variable may hold: + - a Postgres URL (`postgresql://…/name`, `postgresql+psycopg://…`): the worker gets `name_gw`, created from the + base database's server if missing (empty — the project's own schema fixture migrates it, as for one process); + - a SQLite URL (`sqlite:///path.db`) or a plain file path: the worker gets `path-gw.db`. +Without xdist (no PYTEST_XDIST_WORKER) or with the variable unset it changes nothing, so a serial run is unaffected. +""" + +from __future__ import annotations + +import os +import re +from urllib.parse import urlsplit, urlunsplit + +import pytest + +_PG = re.compile(r"^postgres(?:ql)?(?:\+\w+)?://", re.I) + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addini( + "db_per_worker_env", + "env var holding the test database (a Postgres URL, SQLite URL or file path)", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_load_initial_conftests(early_config: pytest.Config) -> None: + var = early_config.getini("db_per_worker_env") + if var: + apply(str(var)) + + +def apply(var: str) -> str | None: + worker = os.environ.get("PYTEST_XDIST_WORKER") + value = os.environ.get(var) + if not worker or not value: + return value + if _PG.match(value): + new = _postgres(value, worker) + else: + new = _file(value, worker) + os.environ[var] = new + return new + + +def _postgres(url: str, worker: str) -> str: + parts = urlsplit(url) + base = parts.path.lstrip("/") + name = f"{base}_{worker}" + if not re.fullmatch(r"\w+", name): + raise ValueError(f"db_per_worker: refusing an unusual database name {name!r}") + # The server, through the BASE database (it exists; `postgres` may not be reachable for this role). + plain = urlunsplit( + (re.sub(r"\+\w+$", "", parts.scheme), parts.netloc, parts.path, parts.query, "") + ) + _create_if_missing(plain, name) + return urlunsplit((parts.scheme, parts.netloc, f"/{name}", parts.query, parts.fragment)) + + +def _create_if_missing(dsn: str, name: str) -> None: + try: + import psycopg # psycopg 3 + + conn = psycopg.connect(dsn, autocommit=True) + except ImportError: + import psycopg2 # fallback for projects on psycopg2 + + conn = psycopg2.connect(dsn) + conn.autocommit = True + try: + cur = conn.cursor() + cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,)) + if cur.fetchone() is None: + cur.execute(f'CREATE DATABASE "{name}"') # name checked to be \w+ above + finally: + conn.close() + + +def _file(value: str, worker: str) -> str: + prefix = "" + m = re.match(r"^(sqlite(?:\+\w+)?:///)(.*)$", value, re.I) + if m: + prefix, value = m.group(1), m.group(2) + root, ext = os.path.splitext(value) + return f"{prefix}{root}-{worker}{ext}" diff --git a/docs/development.md b/docs/development.md index bd9171e3..5a8d8114 100644 --- a/docs/development.md +++ b/docs/development.md @@ -107,6 +107,9 @@ Coverage is on demand, never enforced and not in the gate (E35): `npm run test - `frontend/`, and for the backend `docker compose -f backend/docker-compose.test.yml run --rm test python -m pytest --cov=app` (then `… down -v`). +The DB lane runs on every core (`pytest -n auto`, pytest-xdist): `tests/db_per_worker.py` gives each worker its own +database (`siftlode_test_gw`, created and migrated on first use), so the same tests run side by side. A serial +run (no `-n`) still works and uses `siftlode_test` itself. The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails; exceptions in `.trivyignore.yaml`, each with a reason and an expiry). diff --git a/frontend/src/lib/gates.contract.test.ts b/frontend/src/lib/gates.contract.test.ts index 9f793a86..220a4303 100644 --- a/frontend/src/lib/gates.contract.test.ts +++ b/frontend/src/lib/gates.contract.test.ts @@ -552,7 +552,10 @@ describe("the gate register (docs/gates.md)", () => { "[pytest]", "testpaths = tests", "pythonpath = .", - "addopts = -q -rs --strict-markers", + // `-p tests.db_per_worker` selects nothing and filters nothing: under `-n` it hands each worker its + // own database (DATABASE_URL + `_gw`), so the same tests can run side by side + "addopts = -q -rs --strict-markers -p tests.db_per_worker", + "db_per_worker_env = DATABASE_URL", ]); }); diff --git a/frontend/src/lib/gates.snapshot.json b/frontend/src/lib/gates.snapshot.json index a478db2d..ce60c8c7 100644 --- a/frontend/src/lib/gates.snapshot.json +++ b/frontend/src/lib/gates.snapshot.json @@ -1232,6 +1232,7 @@ " POSTGRES_DB: siftlode_test", " tmpfs:", " - /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run", + " command: [\"postgres\", \"-c\", \"max_connections=300\"]", " healthcheck:", " test: [\"CMD-SHELL\", \"pg_isready -U siftlode -d siftlode_test\"]", " interval: 1s", @@ -1250,7 +1251,7 @@ " volumes:", " - ./:/app", " working_dir: /app", - " command: [\"python\", \"-m\", \"pytest\"]" + " command: [\"python\", \"-m\", \"pytest\", \"-n\", \"auto\"]" ], "frontend/.prettierignore": [ "dist",