diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml deleted file mode 100644 index 7bd1487a..00000000 --- a/.gitea/workflows/check.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green -# independent of any one machine. Two jobs match the two local lanes: -# frontend — tsc (3 projects) + eslint + prettier --check + knip (both modes) + vitest + vite build -# + npm audit of the production dependencies -# backend — ruff + basedpyright + the full pytest suite (pure-logic + the DB-backed lane) against -# a real Postgres + pip-audit of the lock -# -# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner -# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact -# second the frontend one ends, every time, and the pair costs ~4 + ~6 = **~10 minutes from the -# push**. Anything that waits on this gate has to budget for the sum, not the max; the ops -# `release` read-back timed out on its first real run for believing otherwise. -# -# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend -# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose -# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned -# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that -# failed to come up is a hard error, never a silent skip. -# -# The frontend container's node major is NOT free: it must equal the prod build image's and the dev -# machine's, because vitest/tsc read the engine's own Intl and web-API implementations and those split -# between majors. `.nvmrc` is the single source of truth and -# `frontend/src/lib/nodeEngine.contract.test.ts` fails this gate if any of them drift apart. -name: gate - -# Branch pushes only. A release pushes `main` and then the `v` tag, and a bare `on: [push]` -# treated the tag as a second push of the IDENTICAL commit — every release ran this whole gate -# twice, to the same verdict, for nothing (runs #81-#90 are five such pairs). `branches: ['**']` -# keeps every branch covered and drops tag refs. -# -# It has to be `branches`, NOT the `tags-ignore` this obviously wants to be. Gitea's matchPushEvent -# requires EVERY named filter to match, and each one bails on the wrong ref kind: `tags-ignore` -# starts with `if !refName.IsTag() { break }`, so a filter meant to skip tags rejects every BRANCH -# push and the gate stops running entirely. `branches` bails the same way on a tag, which is the -# direction we want. (Measured in go-gitea v1.27.3 modules/actions/workflows.go, not assumed — it -# differs from GitHub, where `tags-ignore` alone leaves branches unfiltered.) -# -# If that filter were ever wrong the gate would not run at all — which is why `siftlode release` -# now ends by reading the verdict back for the sha it pushed, and says so out loud when there is -# no run for it. A gate that silently stops running is the failure this repo already lived through. -on: - push: - branches: ["**"] - pull_request: - -jobs: - frontend: - runs-on: ubuntu-latest - container: node:24-bookworm - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - - run: npm ci - - run: npm run typecheck - - run: npm run lint - - run: npm run format:check - - run: npm run knip - - run: npm run test - # The production bundle (E32): before this, `vite build` — and devOnlyGuard, which fails it if - # dev-only code reaches the bundle — first ran inside the image build, after main was pushed. - - run: npx vite build - # What ships to the browser: a known CVE at high or above in a production dependency fails. - # (The dev tooling's own advisories are burned down by hand; see docs/gates.md.) - - run: npm audit --omit=dev --audit-level=high - - backend: - runs-on: ubuntu-latest - # A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node - # so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach - # GitHub's python-versions download, so it must be baked into the image. - # - # This node runs the checkout action, never our code, so it is deliberately OUTSIDE the engine - # contract the frontend job is held to — it tracks `.nvmrc` as hygiene (node 20 went EOL in April - # 2026), but a missing upstream tag here must never be able to block the gate. - container: nikolaik/python-nodejs:python3.12-nodejs24 - services: - postgres: - image: postgres:16-alpine - env: - POSTGRES_USER: siftlode - POSTGRES_PASSWORD: siftlode - POSTGRES_DB: siftlode_test - options: >- - --health-cmd "pg_isready -U siftlode -d siftlode_test" - --health-interval 2s - --health-timeout 3s - --health-retries 30 - env: - DATABASE_URL: postgresql+psycopg://siftlode:siftlode@postgres:5432/siftlode_test - SIFTLODE_TEST_REQUIRE_DB: "1" - steps: - - uses: actions/checkout@v4 - # ffmpeg for the poster/thumbnail tests; postgresql-client for the readiness wait below. - - run: apt-get update && apt-get install -y --no-install-recommends ffmpeg postgresql-client - - name: Wait for Postgres - run: until pg_isready -h postgres -U siftlode -d siftlode_test; do sleep 1; done - - run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt - - name: ruff - run: python -m ruff check --config backend/ruff.toml backend scripts - - name: ruff format --check - run: python -m ruff format --check --config backend/ruff.toml backend scripts - # The ninth lane (E27 S3). Run from backend/ so it resolves exactly as the local lane does — - # pyrightconfig.json's include/extraPaths are relative to the config, but the baseline is - # found relative to the working directory. New errors fail; the ~200 pre-existing ones are in - # .basedpyright/baseline.json and burn down epic by epic. Matching is per (file, rule, column - # span) plus a count, so a new error of the SAME shape in a file where a sibling was just - # fixed is absorbed — burn the hot files down first. - # - # The second step is not redundant. basedpyright TIGHTENS the baseline in place when the count - # drops, with no --writebaseline, and CI throws its workspace away — so without this a - # burn-down is green here and red on the developer's `siftlode check`, which has the same - # guard. Two gates over one tree must not answer differently. - - name: basedpyright - run: python -m basedpyright --project pyrightconfig.json - working-directory: backend - - name: basedpyright baseline is committed - run: git diff --exit-code -- backend/.basedpyright/baseline.json - - name: pytest - run: python -m pytest - working-directory: backend - # Dependency CVE audit over the pinned lock. BLOCKING since E32: it ran advisory-only, and when - # it was finally read it had three urllib3 advisories with a fix available (bumped in the same - # change). An advisory without a fix is answered with `--ignore-vuln ` here, a reason and a - # date in the comment, and a line in docs/gates.md. - - name: pip-audit - run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock diff --git a/CLAUDE.md b/CLAUDE.md index 4eb2e8ee..99230748 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,14 +13,14 @@ router-free navigation provider. ## The gate — never merge or ship red The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + the lock check + typecheck (3 tsc projects) + -eslint + prettier --check + knip (both modes) + vitest + vite build + pytest**, then a coverage report (no threshold). It runs +eslint + prettier --check + knip (both modes) + vitest + vite build + pytest + pip-audit + npm audit**, then a coverage report (no threshold). It runs before every `release` merges and on every `publish`, which **refuses to build on failure — there is deliberately no `--skip` flag**; a release also requires the Playwright suite, and the image must pass a Trivy scan before it is pushed. Run it and get it green **before** any merge, release, or handing work to review. -- Full gate (agents with the private ops dispatcher): `siftlode check`. -- Directly, without the dispatcher: the commands in [`docs/development.md`](docs/development.md) § The gate (frontend lanes, `ruff` over `backend scripts`, basedpyright in the test image, and `pytest` — the DB-lane needs Postgres). -- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every file and the CI/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: a new code file or any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped). +- Full gate: `bash scripts/check.sh` (`lint` for the static lanes only); the private ops dispatcher's `siftlode check` runs the same script. Single lanes: [`docs/development.md`](docs/development.md) § The gate. +- **There is no CI** (owner, 2026-10-04): this gate, run here, is the only one — green before every hand-over, and `release` refuses to merge without it. +- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every file and the gate/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: a new code file or any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped). - **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`. - New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`. diff --git a/Dockerfile b/Dockerfile index c1c69b2a..2b9f9f77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG BUILD_DATE= # Stage 1: build the frontend SPA # The major is pinned to `.nvmrc` and held there by frontend/src/lib/nodeEngine.contract.test.ts: -# this image, the CI gate's image and the dev machine must be ONE engine (see that test for why). +# this image and the dev machine the gate runs on must be ONE engine (see that test for why). FROM node:24-alpine AS frontend WORKDIR /fe # Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules. @@ -79,8 +79,8 @@ RUN printf 'Types: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm\nCo && rm -rf /var/lib/apt/lists/* # Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is -# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats -# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header). +# byte-reproducible and the exact versions are the ones the gate's pip-audit scanned — not whatever +# floats up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header). # pip itself is removed once it has installed the lock (E32): nothing runs it at runtime, and it carries # its own VENDORED copies of urllib3, msgpack and setuptools, which Trivy reports as the image's own — # an attack surface and a CVE stream for a tool the running app never calls. diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index b82f6104..6480647e 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -9,8 +9,10 @@ # writes is keyed to the diagnostics THIS version produces. # pytest-cov: the coverage REPORT the gate prints (E32 D4) — a number to read, deliberately with no # threshold; "every regression test seen red" is the bar, not a percentage. +# pip-audit: the gate's audit of requirements.lock (scripts/check.sh), blocking at any severity. basedpyright==1.40.1 pytest==8.4.2 pytest-cov==7.1.0 ruff==0.15.21 httpx2==2.13.0 +pip-audit==2.10.1 diff --git a/backend/requirements.lock b/backend/requirements.lock index eb3457d1..2611fa16 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -1,9 +1,9 @@ # Pinned dependency lock for reproducible builds (SEC-5). GENERATED — do not hand-edit. # Source of truth is requirements.txt (the floor ranges); regenerate after editing it with: # docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort -# The prod Dockerfile + Dockerfile.test + CI install THIS file; pip-audit scans it in CI (blocking -# since E32). A security fix may be ONE pin bumped in place instead of a full regeneration, which -# would move every other package too — then `pip install -r requirements.lock && pip check` must +# The prod Dockerfile + Dockerfile.test install THIS file; the gate's pip-audit (scripts/check.sh) +# scans it, blocking since E32. A security fix may be ONE pin bumped in place instead of a full +# regeneration, which would move every other package too — then `pip install -r requirements.lock && pip check` must # pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177, E32). APScheduler==3.11.3 Authlib==1.7.2 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 3c25394f..05c88c34 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -39,7 +39,7 @@ _MISSING = object() # A would-be silent skip (no DB, or no ffmpeg) is a hard error unless the run says otherwise — the # suite must not print green while the DB-backed / ffmpeg tests were quietly not run. ON by default -# since E32: as an opt-in only the compose lane and CI set, a hand-run `pytest` skipped ~60 DB tests +# since E32: as an opt-in only the compose lane set, a hand-run `pytest` skipped ~60 DB tests # and still passed. `SIFTLODE_TEST_REQUIRE_DB=0` is the explicit "pure-logic only" run. _REQUIRE_FULL = os.getenv("SIFTLODE_TEST_REQUIRE_DB", "1") != "0" diff --git a/backend/tests/test_python_engine_contract.py b/backend/tests/test_python_engine_contract.py index 49a940aa..f7298c8a 100644 --- a/backend/tests/test_python_engine_contract.py +++ b/backend/tests/test_python_engine_contract.py @@ -9,7 +9,7 @@ independent literal, there was no `.python-version`, no `requires-python`, and r **The contract is a chain, and it is split across two test files because no single runner can see every link.** This suite runs inside the test image, which mounts `backend/` and nothing else, so it -cannot read the repo-root Dockerfile or the workflow. It therefore holds the link it CAN see, and the +cannot read the repo-root Dockerfile. It therefore holds the link it CAN see, and the one that matters most here: the interpreter actually running == backend/pyrightconfig.json == backend/ruff.toml @@ -17,7 +17,7 @@ one that matters most here: and `frontend/src/lib/pythonEngine.contract.test.ts` — which runs on the host, with the whole tree — holds the rest of the chain: - /.python-version == pyrightconfig.json == ruff.toml == Dockerfile == Dockerfile.test == check.yml + /.python-version == pyrightconfig.json == ruff.toml == Dockerfile == Dockerfile.test Together they close the loop. To move the version: edit `/.python-version`, then follow the failures in both files. Every place they name is a place that would otherwise have been forgotten. diff --git a/docs/development.md b/docs/development.md index 090f54ac..03b8cfae 100644 --- a/docs/development.md +++ b/docs/development.md @@ -5,7 +5,7 @@ How to run Siftlode locally and pass the gate. Conventions for the backend code ## Node -One major, everywhere. `.nvmrc` pins it; the CI container, the `Dockerfile`'s frontend build stage, +One major, everywhere. `.nvmrc` pins it; the `Dockerfile`'s frontend build stage, `frontend/package.json`'s `engines` and `@types/node` must all match it — and so must the machine you run the gate on. `frontend/src/lib/nodeEngine.contract.test.ts` fails the gate the moment any of them drift apart, including when yours is the odd one out. `nvm use` in the repo root picks it up. @@ -38,9 +38,20 @@ decisions and the chosen patterns, rendered with the app's real CSS. It never re ## The gate -A change is done when all of this is green. CI runs the same checks server-side -(`.gitea/workflows/check.yml`), so "main is always green". What each gate deliberately does NOT -check — and why — is the [gate register](gates.md); a test fails when a narrowing is missing from it. +A change is done when this is green, from the repo root: + +```bash +bash scripts/check.sh # every lane +bash scripts/check.sh lint # the static lanes only (no tests, build or audits) — the quick loop +``` + +It runs every lane even after one fails and names each red one at the end. What it runs, and why +each lane runs where it does, is in the script's comments; `frontend/src/lib/gates.contract.test.ts` +holds its lines, so a lane dropped or narrowed there fails a test. What each gate deliberately does +NOT check — and why — is the [gate register](gates.md). There is no CI: the gate runs here, and a +release runs it (plus e2e and the CVE scan) before anything is merged. + +The lanes, for running one by hand: **Backend** @@ -58,6 +69,10 @@ docker compose -f backend/docker-compose.test.yml run --rm --no-deps --build tes # coverage total. The backend tree is bind-mounted, so it always tests your working files. docker compose -f backend/docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from test docker compose -f backend/docker-compose.test.yml down -v + +# the lock against the advisory databases, at any severity +docker compose -f backend/docker-compose.test.yml run --rm --no-deps test \ + pip-audit -r requirements.lock --no-deps --disable-pip ``` A missing database or ffmpeg is a hard error, not a skip — so the gate can't print green while the @@ -75,6 +90,7 @@ npm run format:check # prettier npm run knip # unused files/exports/deps — and unused by what SHIPS (--production) npm run test # vitest (incl. the compiler, size/complexity and gate-register contracts) npx vite build # the production bundle (devOnlyGuard fails it on dev-only code) +npm audit --omit=dev --audit-level=high # what ships to the browser, at high or above ``` Coverage is reported, never enforced: `npm run test -- --coverage`, and the pytest lane's last line. diff --git a/docs/gates.md b/docs/gates.md index b1d7b907..f1ebf3b8 100644 --- a/docs/gates.md +++ b/docs/gates.md @@ -14,10 +14,10 @@ name its invariant. Whatever is left narrowed is listed here, and not an oversight; a narrowing missing from here is a test failure. **The snapshot.** ESLint's effective config for every code file `eslint .` reaches, and the lines -that run the lanes (`.gitea/workflows/check.yml`, `backend/docker-compose.test.yml`, +that run the lanes (`scripts/check.sh`, `backend/docker-compose.test.yml`, `backend/Dockerfile.test`, the npm scripts, `vitest.config.ts`, `.prettierignore`), are committed in [`frontend/src/lib/gates.snapshot.json`](../frontend/src/lib/gates.snapshot.json). The test fails on -ANY difference — a rule, an option, a glob that moves one file to another config, a new ignore, a CI +ANY difference — a rule, an option, a glob that moves one file to another config, a new ignore, a gate flag. So a change to a gate is always a diff of that file in review, never a shape a detector failed to model (E32 review: three rounds of a cleverer detector each found a bypass of the last one's fixes). Regenerate it with `GATE_SNAPSHOT=update npx vitest run src/lib/gates.contract.test.ts` @@ -117,17 +117,17 @@ project checks needs a row. | Scope | Narrowed | Why | Measured | | --- | --- | --- | --- | | check | e2e (Playwright) not in `check` | It needs the :8080 stack built from the tree (minutes). It runs in `check-full`, and `release` rebuilds the stack and requires it (E32 D2) — nothing ships without it. | 5 specs × 3 engines | -| CI | no e2e, no Trivy | The lab runner has no browser stack, and Trivy's DB download is not reachable from it. Both run in `release` / `publish` on the dev machine, before anything is pushed. | — | +| pushes | no server-side gate (no CI) | Owner decision 2026-10-04. The gate (`scripts/check.sh`) runs on the dev machine before work is handed over, and `release` runs it, then e2e, before anything is merged and Trivy before the image is pushed — the CI only repeated that, after the merge. Its runner shared the git host's container, and one run there starved Gitea of memory until it stopped answering. What it added — a fresh `npm ci` on Linux — the image build does at publish. | — | | coverage | no threshold | Owner decision E32 D4: reported on both sides, never enforced. "Every regression test seen red" is the bar; a percentage invites tests written to move it. | frontend 80% / backend 78% of lines (E32 S1) | | CVE gate | HIGH/CRITICAL with a fix available only | `--ignore-unfixed` + `--severity HIGH,CRITICAL`: a finding with no fix cannot be acted on by an upgrade; it shows in the scan output. A fixable one fails the publish, and the only way past it is `.trivyignore.yaml`. | 0 open (E32 S1) | -| npm audit (CI) | `--omit=dev --audit-level=high` | What ships to the browser, at high or above. The dev toolchain's advisories were burned down by hand in E32 S1 (18 → 0). | 0 | +| npm audit | `--omit=dev --audit-level=high` | What ships to the browser, at high or above. The dev toolchain's advisories were burned down by hand in E32 S1 (18 → 0). | 0 | | React Compiler | the functions it skips, pinned | `components/compilerCoverage.contract.test.ts` — the list may only shrink. End state: only VirtualGrid (TanStack Virtual, an IncompatibleLibrary) remains. | 93 (E32 S1) → 29 (E32 S3) → 1, VirtualGrid (E32 S4) | | size / complexity | today's offenders as ceilings | `src/lib/sourceSize.contract.test.ts`, `backend/tests/test_source_size.py`: listed files and functions may not grow, nothing new may cross the limit, an entry back under it must be deleted. A function is held by its name, not its rank in the file. | files >800 lines: 13 FE / 9 BE; functions over the limit: 64 FE / 32 BE (E32 review) | | size / complexity | shipped code only — not tests, test kits, e2e or `scripts/` | The ratchet guards what runs in production. A test file is long because it is a list of cases, and a test kit is test code; splitting either moves lines without making anything easier to change. `scripts/` is release and seed tooling, not in the image. | outside it: tests >800 lines 2 FE / 5 BE; functions over the limit 2 FE test code (`gates.contract.test.ts` `eslintNarrowings` 23, E32 review round 3; `compilerSkips.testkit.ts` 25), 1 BE test (11), 1 script (`e2e_seed.py` `main` 13) | ## Accepted findings -Entries in `.trivyignore.yaml` (CVE gate) and `--ignore-vuln` flags on CI's `pip-audit` step. Each +Entries in `.trivyignore.yaml` (CVE gate) and `--ignore-vuln` flags on the gate's `pip-audit` lane (`scripts/check.sh`). Each must have a row here (Scope `accepted`) with its reason and its expiry; the test fails on an entry without one, and on a `.trivyignore.yaml` entry with no `statement` or no `expired_at`. diff --git a/frontend/src/lib/gates.contract.test.ts b/frontend/src/lib/gates.contract.test.ts index b826c17d..ed065311 100644 --- a/frontend/src/lib/gates.contract.test.ts +++ b/frontend/src/lib/gates.contract.test.ts @@ -105,7 +105,7 @@ function restrictedEntries(entry: Linter.RuleEntry | undefined): Set { // the lines that run the lanes. Any change fails until the snapshot is regenerated, so a narrowing // arrives as a diff of `gates.snapshot.json` in review instead of slipping past a detector. (Three // review rounds kept finding shapes the detector this replaced did not model — a `basePath` global -// ignore, a widened glob filed under a narrow row, a substring check on a CI line. A comparison +// ignore, a widened glob filed under a narrow row, a substring check on a gate line. A comparison // against everything has no shape to miss.) // // Regenerate: GATE_SNAPSHOT=update npx vitest run src/lib/gates.contract.test.ts @@ -285,7 +285,7 @@ function meaningful(path: string): string[] { }); } const lanes = () => ({ - ".gitea/workflows/check.yml": meaningful(".gitea/workflows/check.yml"), + "scripts/check.sh": meaningful("scripts/check.sh"), "backend/docker-compose.test.yml": meaningful("backend/docker-compose.test.yml"), "backend/Dockerfile.test": meaningful("backend/Dockerfile.test"), "frontend/package.json scripts": ( @@ -485,19 +485,17 @@ describe("the gate register (docs/gates.md)", () => { }); // Every ruff lane passes `--config backend/ruff.toml`, which makes ruff ignore any other config // file it would discover (measured: a nested `ruff.toml` selecting the `D` rules changed nothing). - // So that flag is what makes this file the whole configuration. The snapshot holds every CI line; - // these two are spelled out here because the reason is (round 3: a substring check let an extra - // `--config "lint.extend-ignore=…"` override through). - it("runs every CI ruff step on backend/ruff.toml and nothing else", () => { - const steps = read(".gitea/workflows/check.yml") + // So that flag is what makes this file the whole configuration. The snapshot holds every line of + // the gate script; these two are spelled out because the reason is (round 3: a substring check let + // an extra `--config "lint.extend-ignore=…"` override through). + it("runs both ruff lanes on backend/ruff.toml and nothing else", () => { + const lanes = read("scripts/check.sh") .split("\n") - .filter((l) => /\bruff\b/.test(l) && !l.trim().startsWith("#")) + .filter((l) => /python -m ruff (check|format)/.test(l)) .map((l) => l.trim()); - expect(steps).toEqual([ - "- name: ruff", - "run: python -m ruff check --config backend/ruff.toml backend scripts", - "- name: ruff format --check", - "run: python -m ruff format --check --config backend/ruff.toml backend scripts", + expect(lanes).toEqual([ + "ruff_check() { ruff_pinned && python -m ruff check --config backend/ruff.toml backend scripts; }", + "ruff_format() { ruff_pinned && python -m ruff format --check --config backend/ruff.toml backend scripts; }", ]); }); @@ -577,9 +575,9 @@ describe("the gate register (docs/gates.md)", () => { expires: /\n\s*expired_at:\s*\d{4}-\d{2}-\d{2}/.test(chunk), })); expect(entries.filter((e) => !e.statement || !e.expires).map((e) => e.id)).toEqual([]); - const pipIgnores = read(".gitea/workflows/check.yml") + const pipIgnores = read("scripts/check.sh") .split("\n") - .filter((l) => !l.trim().startsWith("#")) // the step's comment explains the flag; it is not one + .filter((l) => !l.trim().startsWith("#")) // the lane's comment explains the flag; it is not one .flatMap((l) => [...l.matchAll(/--ignore-vuln[ =](\S+)/g)].map((m) => m[1]!)); const found = [...entries.map((e) => e.id), ...pipIgnores].map((id): [string, string] => [ "accepted", diff --git a/frontend/src/lib/gates.snapshot.json b/frontend/src/lib/gates.snapshot.json index 502789dc..00f878f1 100644 --- a/frontend/src/lib/gates.snapshot.json +++ b/frontend/src/lib/gates.snapshot.json @@ -2222,68 +2222,6 @@ } }, "lanes": { - ".gitea/workflows/check.yml": [ - "name: gate", - "on:", - " push:", - " branches: [\"**\"]", - " pull_request:", - "jobs:", - " frontend:", - " runs-on: ubuntu-latest", - " container: node:24-bookworm", - " defaults:", - " run:", - " working-directory: frontend", - " steps:", - " - uses: actions/checkout@v4", - " - run: npm ci", - " - run: npm run typecheck", - " - run: npm run lint", - " - run: npm run format:check", - " - run: npm run knip", - " - run: npm run test", - " - run: npx vite build", - " - run: npm audit --omit=dev --audit-level=high", - " backend:", - " runs-on: ubuntu-latest", - " container: nikolaik/python-nodejs:python3.12-nodejs24", - " services:", - " postgres:", - " image: postgres:16-alpine", - " env:", - " POSTGRES_USER: siftlode", - " POSTGRES_PASSWORD: siftlode", - " POSTGRES_DB: siftlode_test", - " options: >-", - " --health-cmd \"pg_isready -U siftlode -d siftlode_test\"", - " --health-interval 2s", - " --health-timeout 3s", - " --health-retries 30", - " env:", - " DATABASE_URL: postgresql+psycopg://siftlode:siftlode@postgres:5432/siftlode_test", - " SIFTLODE_TEST_REQUIRE_DB: \"1\"", - " steps:", - " - uses: actions/checkout@v4", - " - run: apt-get update && apt-get install -y --no-install-recommends ffmpeg postgresql-client", - " - name: Wait for Postgres", - " run: until pg_isready -h postgres -U siftlode -d siftlode_test; do sleep 1; done", - " - run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt", - " - name: ruff", - " run: python -m ruff check --config backend/ruff.toml backend scripts", - " - name: ruff format --check", - " run: python -m ruff format --check --config backend/ruff.toml backend scripts", - " - name: basedpyright", - " run: python -m basedpyright --project pyrightconfig.json", - " working-directory: backend", - " - name: basedpyright baseline is committed", - " run: git diff --exit-code -- backend/.basedpyright/baseline.json", - " - name: pytest", - " run: python -m pytest", - " working-directory: backend", - " - name: pip-audit", - " run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock" - ], "backend/Dockerfile.test": [ "FROM python:3.12-slim", "WORKDIR /app", @@ -2366,6 +2304,77 @@ " },", " },", "});" + ], + "scripts/check.sh": [ + "set -u", + "cd \"$(dirname \"$0\")/..\" || exit 2", + "mode=\"${1:-all}\"", + "case \"$mode\" in all | lint) ;; *) echo \"usage: scripts/check.sh [lint]\" >&2; exit 2 ;; esac", + "export MSYS_NO_PATHCONV=1 # Git Bash: do not rewrite the in-container paths below", + "TESTCOMPOSE=backend/docker-compose.test.yml", + "failed=()", + "lane() { # lane — run it, remember a failure, never stop", + " local name=\"$1\"", + " shift", + " echo \">> $name…\"", + " \"$@\" || failed+=(\"$name\")", + "}", + "ruff_pinned() {", + " local pin host", + " pin=\"$(sed -n 's/^ruff==\\([0-9.]*\\).*/\\1/p' backend/requirements-dev.txt)\"", + " host=\"$(python -m ruff --version 2>/dev/null | awk '{print $2}')\"", + " [ -n \"$pin\" ] && [ \"$host\" = \"$pin\" ] && return 0", + " echo \"ruff ${host:-missing} on the host, ${pin:-nothing} pinned — python -m pip install ruff==$pin\" >&2", + " return 1", + "}", + "ruff_check() { ruff_pinned && python -m ruff check --config backend/ruff.toml backend scripts; }", + "ruff_format() { ruff_pinned && python -m ruff format --check --config backend/ruff.toml backend scripts; }", + "basedpyright() {", + " local before", + " before=\"$(git hash-object backend/.basedpyright/baseline.json)\"", + " docker compose -f \"$TESTCOMPOSE\" run --rm --no-deps --build test \\", + " python -m basedpyright --project pyrightconfig.json || return 1", + " if [ \"$(git hash-object backend/.basedpyright/baseline.json)\" != \"$before\" ]; then", + " echo \"this run rewrote backend/.basedpyright/baseline.json (errors were fixed) — commit it\" >&2", + " return 1", + " fi", + " git diff HEAD --quiet -- backend/.basedpyright/baseline.json && return 0", + " echo \"backend/.basedpyright/baseline.json is uncommitted — commit it\" >&2", + " return 1", + "}", + "frontend() { (cd frontend && \"$@\"); }", + "lane \"ruff\" ruff_check", + "lane \"ruff format --check\" ruff_format", + "lane \"basedpyright (new errors only)\" basedpyright", + "lane \"lock check (frontend/node_modules == package-lock.json)\" node scripts/check-node-modules.mjs", + "lane \"tsc (src + configs + e2e)\" frontend npm run --silent typecheck", + "lane \"eslint (warnings fail too)\" frontend npm run --silent lint", + "lane \"prettier --check\" frontend npm run --silent format:check", + "lane \"knip (unused, and unused by what ships)\" frontend npm run --silent knip", + "pytest_lane() { # the full suite against a throwaway Postgres; --pull keeps the base image current", + " docker compose -f \"$TESTCOMPOSE\" build --pull -q test || return 1", + " local rc=0", + " docker compose -f \"$TESTCOMPOSE\" up --build --abort-on-container-exit --exit-code-from test || rc=1", + " docker compose -f \"$TESTCOMPOSE\" down -v >/dev/null 2>&1", + " return $rc", + "}", + "pip_audit() {", + " docker compose -f \"$TESTCOMPOSE\" run --rm --no-deps test \\", + " pip-audit -r requirements.lock --no-deps --disable-pip", + "}", + "if [ \"$mode\" = all ]; then", + " lane \"vitest (+ coverage report)\" frontend npm run --silent test -- --coverage", + " lane \"vite build\" frontend npx vite build --logLevel warn", + " lane \"pytest (pure-logic + DB lane)\" pytest_lane", + " lane \"pip-audit (backend lock)\" pip_audit", + " lane \"npm audit (production dependencies)\" frontend npm audit --omit=dev --audit-level=high", + "fi", + "if [ ${#failed[@]} -eq 0 ]; then", + " echo \">> gate ($mode): green ✓\"", + " exit 0", + "fi", + "echo \">> gate ($mode): RED — ${failed[*]}\" >&2", + "exit 1" ] } } diff --git a/frontend/src/lib/nodeEngine.contract.test.ts b/frontend/src/lib/nodeEngine.contract.test.ts index 90ae85bf..02a98dca 100644 --- a/frontend/src/lib/nodeEngine.contract.test.ts +++ b/frontend/src/lib/nodeEngine.contract.test.ts @@ -48,18 +48,6 @@ it("builds the shipped bundle on the pinned engine", () => { ).toBe(pinned); }); -it("runs the server gate on the pinned engine", () => { - // The frontend job only — the backend job's node runs the checkout action, never our code, so it is - // deliberately not part of this contract (see the note in check.yml). - expect( - majorIn( - readRoot(".gitea/workflows/check.yml"), - /^\s*container: node:(\d+)-bookworm$/m, - "check.yml", - ), - ).toBe(pinned); -}); - it("declares the pinned engine to npm and to tsc", () => { expect(pkg.engines?.node).toBe(`${pinned}.x`); // Types a major ahead of the runtime make tsc green over APIs the shipped engine does not have — diff --git a/frontend/src/lib/pythonEngine.contract.test.ts b/frontend/src/lib/pythonEngine.contract.test.ts index b2637f03..e4b5b709 100644 --- a/frontend/src/lib/pythonEngine.contract.test.ts +++ b/frontend/src/lib/pythonEngine.contract.test.ts @@ -11,12 +11,12 @@ import { expect, it } from "vitest"; // // It lives in the FRONTEND suite for one boring reason: this is the only runner that can see the // whole repository. The backend suite runs inside the test image, which mounts `backend/` alone, so -// it cannot read the repo-root Dockerfile or the workflow. It holds the other half of the chain — +// it cannot read the repo-root Dockerfile. It holds the other half of the chain — // including "the interpreter running these tests IS the pinned one", which only it can assert: // // backend/tests/test_python_engine_contract.py : running interpreter == pyrightconfig == ruff.toml // here : .python-version == pyrightconfig == ruff.toml -// == Dockerfile == Dockerfile.test == check.yml +// == Dockerfile == Dockerfile.test // // To move the version: edit `.python-version`, then follow the failures in both files. const readRoot = (rel: string) => readFileSync(new URL(`../../../${rel}`, import.meta.url), "utf8"); @@ -55,18 +55,6 @@ it("tests on the pinned version", () => { ).toBe(pinned); }); -it("runs the server gate on the pinned version", () => { - // The backend job's container. Its NODE is deliberately outside the Node contract (it only runs - // the checkout action), but its PYTHON runs our code and our tests, so it is inside this one. - expect( - versionIn( - readRoot(".gitea/workflows/check.yml"), - /^\s*container: nikolaik\/python-nodejs:python(\d+)\.(\d+)-nodejs\d+$/m, - "check.yml", - ), - ).toBe(pinned); -}); - it("type-checks against the pinned version", () => { // A type checker aimed at another version green-lights APIs the shipped interpreter does not have // — the same defect as the image split, moved to check time. diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100644 index 00000000..562d8f50 --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# The gate — every lane, in one place, run from the repo root (E32, after CI went: the lab runner +# duplicated what this already ran before every release, and one CI run starved the git host of +# memory until it stopped answering). The release dispatcher calls this; so can anyone with Docker +# and the pinned toolchain. `frontend/src/lib/gates.contract.test.ts` holds its lines, so a lane +# dropped or narrowed here fails a test like any other gate change. +# +# bash scripts/check.sh every lane (what a release requires, before e2e and the CVE scan) +# bash scripts/check.sh lint the static lanes only: no tests, no build, no audits +# +# Each lane runs even when an earlier one failed, and the summary names every red one: a gate that +# stops at the first failure hides the second until the first is fixed. The backend lanes run in the +# test image (backend/Dockerfile.test): requirements.lock will not install on Windows (uvloop), and a +# type checker or auditor without the real packages reports nonsense. +set -u +cd "$(dirname "$0")/.." || exit 2 +mode="${1:-all}" +case "$mode" in all | lint) ;; *) echo "usage: scripts/check.sh [lint]" >&2; exit 2 ;; esac + +export MSYS_NO_PATHCONV=1 # Git Bash: do not rewrite the in-container paths below +TESTCOMPOSE=backend/docker-compose.test.yml +failed=() +lane() { # lane — run it, remember a failure, never stop + local name="$1" + shift + echo ">> $name…" + "$@" || failed+=("$name") +} + +# --- static lanes ------------------------------------------------------------------------------------ + +# Host ruff, held to the pin in requirements-dev.txt: a newer ruff silently changes what "formatted" +# means. `--config backend/ruff.toml` on both: it makes that file the WHOLE configuration (ruff then +# ignores every other config it would discover) and gives scripts/ the backend's rule set. +ruff_pinned() { + local pin host + pin="$(sed -n 's/^ruff==\([0-9.]*\).*/\1/p' backend/requirements-dev.txt)" + host="$(python -m ruff --version 2>/dev/null | awk '{print $2}')" + [ -n "$pin" ] && [ "$host" = "$pin" ] && return 0 + echo "ruff ${host:-missing} on the host, ${pin:-nothing} pinned — python -m pip install ruff==$pin" >&2 + return 1 +} +ruff_check() { ruff_pinned && python -m ruff check --config backend/ruff.toml backend scripts; } +ruff_format() { ruff_pinned && python -m ruff format --check --config backend/ruff.toml backend scripts; } + +# New errors only: the pre-existing ones are in backend/.basedpyright/baseline.json. basedpyright +# TIGHTENS that file by itself when errors go, and the tree is bind-mounted read-write, so a green run +# can leave it rewritten: that fails here until it is committed, or the next run's stack reads stale. +basedpyright() { + local before + before="$(git hash-object backend/.basedpyright/baseline.json)" + docker compose -f "$TESTCOMPOSE" run --rm --no-deps --build test \ + python -m basedpyright --project pyrightconfig.json || return 1 + if [ "$(git hash-object backend/.basedpyright/baseline.json)" != "$before" ]; then + echo "this run rewrote backend/.basedpyright/baseline.json (errors were fixed) — commit it" >&2 + return 1 + fi + git diff HEAD --quiet -- backend/.basedpyright/baseline.json && return 0 + echo "backend/.basedpyright/baseline.json is uncommitted — commit it" >&2 + return 1 +} + +frontend() { (cd frontend && "$@"); } + +lane "ruff" ruff_check +lane "ruff format --check" ruff_format +lane "basedpyright (new errors only)" basedpyright +# every frontend lane judges the HOST node_modules: this proves it is the tree the lock pins +lane "lock check (frontend/node_modules == package-lock.json)" node scripts/check-node-modules.mjs +lane "tsc (src + configs + e2e)" frontend npm run --silent typecheck +lane "eslint (warnings fail too)" frontend npm run --silent lint +lane "prettier --check" frontend npm run --silent format:check +lane "knip (unused, and unused by what ships)" frontend npm run --silent knip + +# --- tests, build, audits -------------------------------------------------------------------------- + +pytest_lane() { # the full suite against a throwaway Postgres; --pull keeps the base image current + docker compose -f "$TESTCOMPOSE" build --pull -q test || return 1 + local rc=0 + docker compose -f "$TESTCOMPOSE" up --build --abort-on-container-exit --exit-code-from test || rc=1 + docker compose -f "$TESTCOMPOSE" down -v >/dev/null 2>&1 + return $rc +} +# Every pinned backend package against the advisory databases, at any severity — the lock is a full +# `pip freeze`, so it is audited as written, without a resolver. An advisory with no fix is answered +# with `--ignore-vuln ` here and a row in docs/gates.md (the register test reads this line). +pip_audit() { + docker compose -f "$TESTCOMPOSE" run --rm --no-deps test \ + pip-audit -r requirements.lock --no-deps --disable-pip +} + +if [ "$mode" = all ]; then + # --coverage prints the summary: read, never a threshold (E32 D4) + lane "vitest (+ coverage report)" frontend npm run --silent test -- --coverage + # the production bundle, and devOnlyGuard, which fails it when dev-only code reaches it + lane "vite build" frontend npx vite build --logLevel warn + lane "pytest (pure-logic + DB lane)" pytest_lane + lane "pip-audit (backend lock)" pip_audit + # what ships to the browser, at high or above; the dev toolchain is burned down by hand + lane "npm audit (production dependencies)" frontend npm audit --omit=dev --audit-level=high +fi + +if [ ${#failed[@]} -eq 0 ]; then + echo ">> gate ($mode): green ✓" + exit 0 +fi +echo ">> gate ($mode): RED — ${failed[*]}" >&2 +exit 1