From bc856ab2af9bb8089ff632521ee834e38d389aed Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 20:09:58 +0200 Subject: [PATCH 01/37] test: warm cold dynamic imports before the timed cases Under a full coverage run on every core, a test file's first cold import (0.6-0.8 s alone) outran the 5 s case timeout and the half-done import failed every later case in the file: the vitest lane went red in 2 of 5 runs with no defect. A beforeAll warm-up in the 4 files of that pattern; 5 of 5 full runs green after. --- frontend/src/lib/accountPinnedStores.dom.test.ts | 11 ++++++++++- frontend/src/lib/diagnostics.dom.test.ts | 11 ++++++++++- frontend/src/lib/diagnostics.session.dom.test.ts | 10 +++++++++- frontend/src/lib/verifyEntry.dom.test.tsx | 10 +++++++++- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/accountPinnedStores.dom.test.ts b/frontend/src/lib/accountPinnedStores.dom.test.ts index 04e9ab0c..7f69c363 100644 --- a/frontend/src/lib/accountPinnedStores.dom.test.ts +++ b/frontend/src/lib/accountPinnedStores.dom.test.ts @@ -4,7 +4,7 @@ // tab (bookmark, typed URL: no pin in sessionStorage) that is before BootGate pins the account, so they // read as nobody's; they must read again on the pin. The notification store used to keep its empty, // nobody's history, and the tab's first notice replaced the account's stored history with it. -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; const entry = (id: number, message: string) => ({ id, @@ -24,6 +24,15 @@ const storedMessages = (account: number) => }[] ).map((n) => n.message); +// Warm the transform cache once (see verifyEntry.dom.test.tsx): the first cold import costs ~0.8 s +// alone and outran the 5 s case timeout under a full coverage run. Transformed code outlives +// vi.resetModules, so each case still gets fresh module instances, only faster. +beforeAll(async () => { + await import("./notifications"); + await import("./hints"); + await import("./api/core"); +}, 60_000); + beforeEach(() => { localStorage.clear(); sessionStorage.clear(); diff --git a/frontend/src/lib/diagnostics.dom.test.ts b/frontend/src/lib/diagnostics.dom.test.ts index 57ff9feb..e4219a09 100644 --- a/frontend/src/lib/diagnostics.dom.test.ts +++ b/frontend/src/lib/diagnostics.dom.test.ts @@ -3,7 +3,7 @@ // E22 S5: the consented level applied to the tab, and the stream it opens. The module keeps its state // at module level (what is applied, the queue), so every test imports a fresh copy — and the log // from that same copy, or it would be reading a different ring. -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; const sent: unknown[][] = []; const unloaded: unknown[][] = []; @@ -24,6 +24,15 @@ vi.mock("./api", async () => { }; }); +// Warm the transform cache once (see verifyEntry.dom.test.tsx): the first cold import costs ~0.8 s +// alone and outran the 5 s case timeout under a full coverage run. Transformed code outlives +// vi.resetModules, so each case still gets fresh module instances, only faster. +beforeAll(async () => { + await import("./diagnostics"); + await import("./log"); + await import("./api"); +}, 60_000); + beforeEach(() => { vi.useFakeTimers(); vi.resetModules(); diff --git a/frontend/src/lib/diagnostics.session.dom.test.ts b/frontend/src/lib/diagnostics.session.dom.test.ts index 0cb2ec0a..96e2d8f4 100644 --- a/frontend/src/lib/diagnostics.session.dom.test.ts +++ b/frontend/src/lib/diagnostics.session.dom.test.ts @@ -4,7 +4,7 @@ // consent), never lowers a raised one, tags every event it keeps, and its last seconds reach the // server before Stop. Its own file: `lib/diagnostics` and `lib/log` keep module-level state, so every // test imports fresh copies of both, as diagnostics.dom.test does. -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; const sent: { event: string; debug_session_id?: string }[][] = []; @@ -45,6 +45,14 @@ const recording = (level: "DEBUG" | "TRACE", minutes = 60, over = {}) => ({ ...over, }); +// Warm the transform cache once (see verifyEntry.dom.test.tsx): the first cold import costs ~0.8 s +// alone and outran the 5 s case timeout under a full coverage run. Transformed code outlives +// vi.resetModules, so each case still gets fresh module instances, only faster. +beforeAll(async () => { + await import("./diagnostics"); + await import("./log"); +}, 60_000); + let current: typeof import("./diagnostics") | null = null; beforeEach(() => { vi.useFakeTimers(); diff --git a/frontend/src/lib/verifyEntry.dom.test.tsx b/frontend/src/lib/verifyEntry.dom.test.tsx index 728c00c8..3b20de3c 100644 --- a/frontend/src/lib/verifyEntry.dom.test.tsx +++ b/frontend/src/lib/verifyEntry.dom.test.tsx @@ -7,9 +7,17 @@ // // entryUrl snapshots window.location at IMPORT time, so each case resets the module registry and sets // the URL before importing — that is the behaviour under test, not a workaround. -import { describe, it, expect, afterEach, vi } from "vitest"; +import { describe, it, expect, afterEach, beforeAll, vi } from "vitest"; import { render, screen, cleanup, act } from "@testing-library/react"; +// The first import of the hook transforms its whole graph (./api and everything under it): ~0.6 s +// alone, past the 5 s case timeout when the full suite runs with coverage on every core — and the +// case it timed out in left a half-done import that failed every case after it. Transformed code +// outlives vi.resetModules, so paying it once here, with room, leaves each case its warm ~60 ms. +beforeAll(async () => { + await import("./useVerifyEntry"); +}, 60_000); + afterEach(() => { cleanup(); vi.resetModules(); From 27f48b7a8bf6009504d3869f7d544c0a67915699 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 20:09:58 +0200 Subject: [PATCH 02/37] feat(gate): check.sh --brief prints a red lane's first error lines only --- frontend/src/lib/gates.snapshot.json | 35 +++++++++++++++++++++---- scripts/check.sh | 38 +++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/gates.snapshot.json b/frontend/src/lib/gates.snapshot.json index e208ef36..6ca4bde5 100644 --- a/frontend/src/lib/gates.snapshot.json +++ b/frontend/src/lib/gates.snapshot.json @@ -1436,7 +1436,8 @@ "e2e/helpers.ts", "e2e/nav.spec.ts", "e2e/player.spec.ts", - "e2e/settings.spec.ts" + "e2e/settings.spec.ts", + "e2e/speed.measure.ts" ], "js configs": ["eslint.config.js", "postcss.config.js"], "lib tests": [ @@ -2309,16 +2310,40 @@ "scripts/check.sh": [ "set -u", "cd \"$(dirname \"$0\")/..\" || exit 2", - "mode=\"${1:-all}\"", - "case \"$mode\" in all | lint) ;; *) echo \"usage: scripts/check.sh [lint]\" >&2; exit 2 ;; esac", + "mode=all", + "brief=0", + "for arg in \"$@\"; do", + " case \"$arg\" in", + " all | lint) mode=\"$arg\" ;;", + " --brief) brief=1 ;;", + " esac", + "done", "export MSYS_NO_PATHCONV=1 # Git Bash: do not rewrite the in-container paths below", "TESTCOMPOSE=backend/docker-compose.test.yml", "failed=()", "lane() { # lane — run it, remember a failure, never stop", " local name=\"$1\"", " shift", - " echo \">> $name…\"", - " \"$@\" || failed+=(\"$name\")", + " if [ \"$brief\" = 0 ]; then", + " echo \">> $name…\"", + " \"$@\" || failed+=(\"$name\")", + " return 0", + " fi", + " local out", + " out=\"$(mktemp)\"", + " if \"$@\" >\"$out\" 2>&1; then", + " echo \">> $name: ok\"", + " else", + " failed+=(\"$name\")", + " echo \">> $name: FAILED\"", + " brief_excerpt \"$out\"", + " fi", + " rm -f \"$out\"", + "}", + "brief_excerpt() {", + " local hits", + " hits=\"$(grep -E -i -A 2 -m 20 'error|fail|✗|×|warn|would reformat|unused|vulnerab|found [0-9]' \"$1\")\"", + " if [ -n \"$hits\" ]; then printf '%s\\n' \"$hits\"; else tail -n 20 \"$1\"; fi", "}", "ruff_pinned() {", " local pin host", diff --git a/scripts/check.sh b/scripts/check.sh index 562d8f50..b7bfb36f 100644 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -7,6 +7,8 @@ # # bash scripts/check.sh every lane (what a release requires, before e2e and the CVE scan) # bash scripts/check.sh lint the static lanes only: no tests, no build, no audits +# … --brief (either mode) a lane's output only when it fails, and only its first +# error lines — for an agent, whose context the full log would flood # # Each lane runs even when an earlier one failed, and the summary names every red one: a gate that # stops at the first failure hides the second until the first is fixed. The backend lanes run in the @@ -14,8 +16,15 @@ # type checker or auditor without the real packages reports nonsense. set -u cd "$(dirname "$0")/.." || exit 2 -mode="${1:-all}" -case "$mode" in all | lint) ;; *) echo "usage: scripts/check.sh [lint]" >&2; exit 2 ;; esac +mode=all +brief=0 +for arg in "$@"; do + case "$arg" in + all | lint) mode="$arg" ;; + --brief) brief=1 ;; + *) echo "usage: scripts/check.sh [lint] [--brief]" >&2; exit 2 ;; + esac +done export MSYS_NO_PATHCONV=1 # Git Bash: do not rewrite the in-container paths below TESTCOMPOSE=backend/docker-compose.test.yml @@ -23,8 +32,29 @@ failed=() lane() { # lane — run it, remember a failure, never stop local name="$1" shift - echo ">> $name…" - "$@" || failed+=("$name") + if [ "$brief" = 0 ]; then + echo ">> $name…" + "$@" || failed+=("$name") + return 0 + fi + local out + out="$(mktemp)" + if "$@" >"$out" 2>&1; then + echo ">> $name: ok" + else + failed+=("$name") + echo ">> $name: FAILED" + brief_excerpt "$out" + fi + rm -f "$out" +} +# The lines that say what broke: the first 20 error-looking ones, each with the two lines after it +# (knip names the file on the line after "Unused files"); a log with none of them (a tool that +# words its failures some other way) shows its tail instead, which is where a summary sits. +brief_excerpt() { + local hits + hits="$(grep -E -i -A 2 -m 20 'error|fail|✗|×|warn|would reformat|unused|vulnerab|found [0-9]' "$1")" + if [ -n "$hits" ]; then printf '%s\n' "$hits"; else tail -n 20 "$1"; fi } # --- static lanes ------------------------------------------------------------------------------------ From 11d35910fd3b14da1a800fb1ae8828aeb079884f Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 20:09:58 +0200 Subject: [PATCH 03/37] =?UTF-8?q?feat(tooling):=20lint-changed.sh=20?= =?UTF-8?q?=E2=80=94=20fast=20lint=20lanes=20on=20the=20uncommitted=20file?= =?UTF-8?q?s,=20usable=20as=20a=20Stop=20hook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/lint-changed.sh | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 scripts/lint-changed.sh diff --git a/scripts/lint-changed.sh b/scripts/lint-changed.sh new file mode 100644 index 00000000..f4feb716 --- /dev/null +++ b/scripts/lint-changed.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# The fast lint lanes on the uncommitted files only — ruff (check + format) on Python, eslint + +# prettier on the frontend — for a check after every edit instead of a full gate run. It is a +# convenience, never the gate: scripts/check.sh stays what a merge or a release requires. +# +# bash scripts/lint-changed.sh lint what `git status` shows as changed or new +# bash scripts/lint-changed.sh --hook the same, as a Claude Code Stop hook: silent and exit 0 when +# clean; otherwise the findings on stderr and exit 2, which +# hands them back to the agent before it ends its turn +# +# Measured 2026-10-04: one cold eslint run (type-aware) takes 3.4–4.7 s, so this runs once per turn +# (Stop), not after every edit — a warm daemon (eslint_d) would add a dependency and a stray process. +set -u +cd "$(dirname "$0")/.." || exit 2 +hook=0 +[ "${1:-}" = --hook ] && hook=1 + +if [ "$hook" = 1 ]; then + # A Stop hook that blocked once is told so on the next stop; let that one through, or an unfixable + # finding would keep the agent in a loop. + grep -q '"stop_hook_active": *true' && exit 0 +fi + +mapfile -t changed < <(git status --porcelain --untracked-files=all | sed -E 's/^.. //; s/.* -> //' | + while read -r f; do [ -f "$f" ] && printf '%s\n' "$f"; done) +py=() fe=() +for f in "${changed[@]}"; do + case "$f" in + backend/*.py | scripts/*.py) py+=("$f") ;; + frontend/*.ts | frontend/*.tsx | frontend/*.js | frontend/*.mjs) fe+=("${f#frontend/}") ;; + esac +done + +report="$( + if [ ${#py[@]} -gt 0 ]; then + python -m ruff check --quiet --config backend/ruff.toml "${py[@]}" 2>&1 + python -m ruff format --check --quiet --config backend/ruff.toml "${py[@]}" 2>&1 + fi + if [ ${#fe[@]} -gt 0 ]; then + (cd frontend && npx eslint --max-warnings 0 "${fe[@]}" 2>&1) + (cd frontend && npx prettier --check --log-level warn "${fe[@]}" 2>&1) + fi +)" + +if [ -z "$report" ]; then + [ "$hook" = 1 ] || echo "lint-changed: clean (${#py[@]} py, ${#fe[@]} frontend)" + exit 0 +fi +if [ "$hook" = 1 ]; then + printf 'lint-changed found problems in the uncommitted files:\n%s\n' "$report" >&2 + exit 2 +fi +printf '%s\n' "$report" +exit 1 From c2bf8fb7f6621b6128aba716fd5bb0e40fc15026 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 20:09:58 +0200 Subject: [PATCH 04/37] =?UTF-8?q?feat(tooling):=20audit=5Fmetrics.py=20?= =?UTF-8?q?=E2=80=94=20size,=20complexity,=20gate=20weight=20and=20a=20spe?= =?UTF-8?q?ed=20baseline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + docs/development.md | 16 ++- frontend/e2e/speed.measure.ts | 128 ++++++++++++++++++++++ frontend/knip.jsonc | 2 +- frontend/playwright.config.ts | 12 ++ scripts/audit_metrics.py | 199 ++++++++++++++++++++++++++++++++++ scripts/fe_complexity.mjs | 45 ++++++++ 7 files changed, 399 insertions(+), 4 deletions(-) create mode 100644 frontend/e2e/speed.measure.ts create mode 100644 scripts/audit_metrics.py create mode 100644 scripts/fe_complexity.mjs diff --git a/.gitignore b/.gitignore index d034f201..04718ff8 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ Thumbs.db # TypeScript incremental build artifacts — machine-local, absolute-path-bearing, and no gate lane # reads or regenerates them, so a committed one is stale from the commit that adds it. *.tsbuildinfo +frontend/e2e/.speed/ diff --git a/docs/development.md b/docs/development.md index 03b8cfae..e23464b1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -16,10 +16,10 @@ next. A gate that is green on a major you do not ship is green about nothing. ## Run it locally -The whole stack (Postgres + API + worker) comes up from the localdev compose file: +The whole stack (Postgres + API + worker) comes up from the base compose file plus the localdev override: ```bash -docker compose -f docker-compose.localdev.yml up --build +docker compose -f docker-compose.yml -f docker-compose.localdev.yml up --build ``` Browse at **http://localhost:8080** (Google login works — the OAuth redirect is @@ -43,9 +43,19 @@ A change is done when this is green, from the repo root: ```bash bash scripts/check.sh # every lane bash scripts/check.sh lint # the static lanes only (no tests, build or audits) — the quick loop +bash scripts/check.sh --brief # either mode: one line per lane, output only for a red one +bash scripts/lint-changed.sh # ruff + eslint + prettier on the uncommitted files only ``` -It runs every lane even after one fails and names each red one at the end. What it runs, and why +It runs every lane even after one fails and names each red one at the end. `lint-changed.sh` is a +convenience, not the gate; with `--hook` it doubles as a Claude Code Stop hook (silent when clean, +the findings handed back otherwise). + +The codebase's health numbers — size, complexity, test ratio, gate weight, comment noise — come from +`python scripts/audit_metrics.py` (seconds; it reports, it never fails). `--speed` adds the speed +baseline (`frontend/e2e/speed.measure.ts`: the feed's first card and a shared `/watch` link's +time-to-play against the :8080 stack, with `E2E_PASSWORD` set). An epic that changes structure +compares its before and after. What it runs, and why each lane runs where it does, is in the script's comments; `frontend/src/lib/gates.contract.test.ts` holds its lines, so a lane dropped or narrowed there fails a test. What each gate deliberately does NOT check — and why — is the [gate register](gates.md). There is no CI: the gate runs here, and a diff --git a/frontend/e2e/speed.measure.ts b/frontend/e2e/speed.measure.ts new file mode 100644 index 00000000..f723f0cd --- /dev/null +++ b/frontend/e2e/speed.measure.ts @@ -0,0 +1,128 @@ +import { test, type Page } from "@playwright/test"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; + +/** + * The speed baseline: numbers, not assertions — read by `scripts/audit_metrics.py --speed`, which + * runs this as the `speed` project (never part of the e2e suite: its file name is not a `.spec`). + * Each reading is the median of RUNS cold page loads (a fresh browser context, so no HTTP cache), + * against the localdev stack and the seeded E2E account. It is a baseline for before/after within + * one machine, not an absolute figure: the same measurement on a phone over the internet is slower. + * + * Not measured here: the in-app players (the E2E account owns no download, Plex item or watch + * history to open), and any iOS reading (Playwright's WebKit decodes media through the host OS — + * see playwright.config.ts) — the owner's phone gives that one by hand. + */ + +const RUNS = 5; +const OUT = "e2e/.speed/latest.json"; +const CONTAINER = process.env.E2E_DB_CONTAINER || "siftlode-db-1"; + +const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] ?? NaN; + +/** JavaScript the page pulled in, uncompressed: the same on every stack (localdev serves it raw, + * prod gzips it at the proxy), and what the browser has to parse. */ +const scriptBytes = (page: Page) => + page.evaluate(() => + (performance.getEntriesByType("resource") as PerformanceResourceTiming[]) + .filter((r) => r.initiatorType === "script" || r.name.endsWith(".js")) + .reduce((sum, r) => sum + r.decodedBodySize, 0), + ); + +/** Public share links on the local stack that could play: no password, not expired, asset ready. + * The page itself is the judge of the last step (its file must be on THIS disk — a localdev DB + * seeded from prod lists assets whose files never came along), so the caller probes them. */ +function shareTokens(): string[] { + const sql = + "select l.token from download_links l join download_jobs j on j.id = l.job_id" + + " join media_assets a on a.id = j.asset_id" + + " where l.password_hash is null and (l.expires_at is null or l.expires_at > now())" + + " and a.status = 'ready' order by l.id"; + try { + const out = execFileSync( + "docker", + ["exec", CONTAINER, "psql", "-U", "siftlode", "-d", "siftlode", "-tAc", sql], + { encoding: "utf8" }, + ); + return out + .split("\n") + .map((t) => t.trim()) + .filter(Boolean); + } catch { + return []; + } +} + +test("speed baseline", async ({ browser, page: page0 }) => { + test.setTimeout(180_000); + const result: Record = { + measured_at: new Date().toISOString(), + runs: RUNS, + }; + + // Feed: navigation start → the first video card on screen. + const feedMs: number[] = []; + let feedJs = 0; + for (let i = 0; i < RUNS; i++) { + const ctx = await browser.newContext(); + const page = await ctx.newPage(); + await page.goto("/"); + // Timed inside the page, every frame: an `expect` would read the clock on its own retry + // schedule (100 ms … 1 s steps), not when the card appeared. + const at = await page.waitForFunction( + () => document.querySelector('[data-testid="video-card"]') && performance.now(), + null, + { polling: "raf", timeout: 30_000 }, + ); + feedMs.push((await at.jsonValue()) as number); + feedJs = await scriptBytes(page); + await ctx.close(); + } + result.feed_first_card_ms = Math.round(median(feedMs)); + result.feed_js_raw_bytes = feedJs; + + // /watch: navigation start → the shared video playing (started muted by script, standing in for + // the visitor's tap — the page has no autoplay). + let token: string | null = null; + for (const t of shareTokens()) { + if ((await page0.request.get(`/api/public/watch/${t}`)).ok()) { + token = t; + break; + } + } + if (token) { + const watchMs: number[] = []; + let watchJs = 0; + for (let i = 0; i < RUNS; i++) { + const ctx = await browser.newContext({ storageState: { cookies: [], origins: [] } }); + const page = await ctx.newPage(); + await page.goto(`/watch/${token}`); + await page.waitForFunction(() => document.querySelector("video, audio"), null, { + polling: "raf", + timeout: 30_000, + }); + const video = page.locator("video, audio").first(); + watchMs.push( + await video.evaluate( + (el: HTMLMediaElement) => + new Promise((resolve, reject) => { + el.muted = true; + el.addEventListener("playing", () => resolve(performance.now()), { once: true }); + el.play().catch(reject); + }), + ), + ); + watchJs = await scriptBytes(page); + await ctx.close(); + } + result.watch_time_to_play_ms = Math.round(median(watchMs)); + result.watch_js_raw_bytes = watchJs; + } else { + result.watch_time_to_play_ms = null; + result.watch_js_raw_bytes = null; + result.watch_note = "no playable share link on the local stack"; + } + + mkdirSync("e2e/.speed", { recursive: true }); + writeFileSync(OUT, JSON.stringify(result, null, 2) + "\n"); +}); diff --git a/frontend/knip.jsonc b/frontend/knip.jsonc index 7fa7cc10..e3ac4ab9 100644 --- a/frontend/knip.jsonc +++ b/frontend/knip.jsonc @@ -15,6 +15,6 @@ ], "playwright": { "config": ["playwright.config.ts"], - "entry": ["e2e/**/*.@(spec|setup).ts"], + "entry": ["e2e/**/*.@(spec|setup|measure).ts"], }, } diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 45b26084..922b061f 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -61,5 +61,17 @@ export default defineConfig({ use: { ...devices["iPad (gen 7) landscape"], storageState: STORAGE_STATE }, dependencies: ["setup"], }, + // The speed baseline (e2e/speed.measure.ts): readings, not tests. Only when PW_SPEED is set — + // `scripts/audit_metrics.py --speed` sets it — so the e2e suite never runs it. + ...(process.env.PW_SPEED + ? [ + { + name: "speed", + testMatch: /\.measure\.ts/, + use: { ...devices["Desktop Chrome"], storageState: STORAGE_STATE }, + dependencies: ["setup"], + }, + ] + : []), ], }); diff --git a/scripts/audit_metrics.py b/scripts/audit_metrics.py new file mode 100644 index 00000000..f34129ca --- /dev/null +++ b/scripts/audit_metrics.py @@ -0,0 +1,199 @@ +"""The codebase's health numbers in one deterministic run: size, complexity, tests, governance +weight, comment noise and the footprint of features marked for removal. It REPORTS, it never fails +— what used to be a gate (the size/complexity ratchet) is read here instead, and an epic compares +its before and after. + + python scripts/audit_metrics.py the static numbers (seconds), as JSON on stdout + python scripts/audit_metrics.py --speed plus the speed baseline: frontend/e2e/speed.measure.ts + against the localdev stack (E2E_PASSWORD must be set; + the ops wrapper exports it) + +The readings are counts of files and lines, so a number moves only when the code does. +""" + +import ast +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BE_APP = ROOT / "backend" / "app" +BE_TESTS = ROOT / "backend" / "tests" +FE_SRC = ROOT / "frontend" / "src" +BIG_FILE = 800 +BE_COMPLEXITY = 10 # ruff's C901 default, the backend ratchet's limit +FE_COMPLEXITY = 20 # the frontend ratchet's limit + +EPIC_CODE = re.compile(r"\b(E\d{2}|R\d{1,2})(\s?S\d+)?\b") +NARRATION = re.compile(r"\b(used to|no longer)\b", re.IGNORECASE) + + +def _is_fe_test(p: Path) -> bool: + return bool(re.search(r"\.(test|testkit)\.tsx?$", p.name)) + + +def _files(root: Path, pattern: str) -> list[Path]: + return sorted(p for p in root.rglob(pattern) if "__pycache__" not in p.parts) + + +def _lines(paths: list[Path]) -> list[tuple[Path, list[str]]]: + return [(p, p.read_text(encoding="utf-8").splitlines()) for p in paths] + + +def _comment_lines(lines: list[str], python: bool) -> int: + """Comment lines; for Python the docstrings count too — this codebase writes its "why" there.""" + if not python: + return sum(1 for line in lines if line.lstrip().startswith(("//", "/*", "*"))) + hashes = sum(1 for line in lines if line.lstrip().startswith("#")) + docs = 0 + for node in ast.walk(ast.parse("\n".join(lines))): + body = getattr(node, "body", None) + if not isinstance(body, list) or not body: + continue + first = body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + docs += (first.end_lineno or first.lineno) - first.lineno + 1 + return hashes + docs + + +def _size(be: list[tuple[Path, list[str]]], fe: list[tuple[Path, list[str]]]) -> dict[str, object]: + be_tests = _lines(_files(BE_TESTS, "*.py")) + fe_tests = _lines([p for p in _files(FE_SRC, "*.ts*") if _is_fe_test(p)]) + be_n = sum(len(ls) for _, ls in be) + fe_n = sum(len(ls) for _, ls in fe) + return { + "backend_lines": be_n, + "frontend_lines": fe_n, + "backend_test_ratio": round(sum(len(ls) for _, ls in be_tests) / be_n, 2), + "frontend_test_ratio": round(sum(len(ls) for _, ls in fe_tests) / fe_n, 2), + "files_over_800": sum(1 for _, ls in be + fe if len(ls) > BIG_FILE), + "largest_file": max(((len(ls), p) for p, ls in be + fe), key=lambda t: t[0])[1] + .relative_to(ROOT) + .as_posix(), + "tables": len(re.findall(r'__tablename__ = "', (BE_APP / "models.py").read_text("utf-8"))), + } + + +def _complexity() -> dict[str, object]: + ruff = subprocess.run( + [sys.executable, "-m", "ruff", "check", "--config", "backend/ruff.toml", "--select", "C901"] + + ["--output-format", "json", "--exit-zero", "backend/app"], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + found = [ + int(m) for d in json.loads(ruff.stdout) for m in re.findall(r"\((\d+) > ", d["message"]) + ] + fe = subprocess.run( + ["node", "scripts/fe_complexity.mjs", str(FE_COMPLEXITY)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + fe_json = json.loads(fe.stdout) + return { + f"backend_functions_over_{BE_COMPLEXITY}": len(found), + "backend_max_complexity": max(found, default=0), + f"frontend_functions_over_{FE_COMPLEXITY}": fe_json["over"], + "frontend_max_complexity": fe_json["max"], + } + + +def _noise(be: list[tuple[Path, list[str]]], fe: list[tuple[Path, list[str]]]) -> dict[str, object]: + every = be + fe + epic = [(p, ln) for p, ls in every for ln in ls if EPIC_CODE.search(ln)] + be_c = sum(_comment_lines(ls, True) for _, ls in be) + fe_c = sum(_comment_lines(ls, False) for _, ls in fe) + return { + "epic_code_lines": len(epic), + "epic_code_files": len({p for p, _ in epic}), + "history_narration_lines": sum(1 for _, ls in every for ln in ls if NARRATION.search(ln)), + "backend_comment_share": round(be_c / sum(len(ls) for _, ls in be), 3), + "frontend_comment_share": round(fe_c / sum(len(ls) for _, ls in fe), 3), + } + + +def _governance() -> dict[str, object]: + eslint = (ROOT / "frontend" / "eslint.config.js").read_text("utf-8") + check = (ROOT / "scripts" / "check.sh").read_text("utf-8") + contract = [p for p in _files(FE_SRC, "*.contract.test.ts*")] + return { + "gate_lanes": len(re.findall(r'^\s*lane "', check, re.MULTILINE)), + "local_eslint_rules": len(set(re.findall(r'"local/([a-z0-9-]+)"', eslint))), + "frontend_contract_tests": len(contract), + "gates_snapshot_lines": len( + (FE_SRC / "lib" / "gates.snapshot.json").read_text("utf-8").splitlines() + ), + "claude_md_lines": len((ROOT / "CLAUDE.md").read_text("utf-8").splitlines()), + "gates_md_lines": len((ROOT / "docs" / "gates.md").read_text("utf-8").splitlines()), + } + + +def _footprint( + be: list[tuple[Path, list[str]]], fe: list[tuple[Path, list[str]]] +) -> dict[str, int]: + """Files that still mention a feature marked for removal: 0 once it is gone.""" + patterns = { + "demo_files": re.compile(r"\bdemo|isDemo|is_demo", re.IGNORECASE), + "chat_files": re.compile(r"\bchat\b|\bmessages?\.py|\bChat[A-Z]|message_keys"), + "access_request_files": re.compile(r"access[ _-]?request|AccessRequest", re.IGNORECASE), + } + return { + k: sum(1 for _, ls in be + fe if any(rx.search(ln) for ln in ls)) + for k, rx in patterns.items() + } + + +def _speed() -> dict[str, object]: + env = {**os.environ, "PW_SPEED": "1"} + out = ROOT / "frontend" / "e2e" / ".speed" / "latest.json" + out.unlink(missing_ok=True) # never report an earlier run's readings + run = subprocess.run( + ["npx", "playwright", "test", "--project=speed", "--reporter=line"], + cwd=ROOT / "frontend", + env=env, + capture_output=True, + text=True, + encoding="utf-8", + shell=os.name == "nt", # npx is a .cmd on Windows + check=False, # a failed run is reported in the result, not raised + ) + if run.returncode != 0 or not out.exists(): + lines = [ln.strip() for ln in (run.stdout + run.stderr).splitlines() if ln.strip()] + first_error = next( + (i for i, ln in enumerate(lines) if "Error" in ln), max(0, len(lines) - 15) + ) + return {"error": "the speed run failed", "output": lines[first_error : first_error + 15]} + return json.loads(out.read_text("utf-8")) + + +def main() -> int: + be = _lines(_files(BE_APP, "*.py")) + fe = _lines( + [p for p in _files(FE_SRC, "*.ts*") if not _is_fe_test(p) and not p.name.endswith(".d.ts")] + ) + report: dict[str, object] = { + "size": _size(be, fe), + "complexity": _complexity(), + "noise": _noise(be, fe), + "governance": _governance(), + "footprint": _footprint(be, fe), + } + if "--speed" in sys.argv[1:]: + report["speed"] = _speed() + sys.stdout.write(json.dumps(report, indent=2) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fe_complexity.mjs b/scripts/fe_complexity.mjs new file mode 100644 index 00000000..64f8a806 --- /dev/null +++ b/scripts/fe_complexity.mjs @@ -0,0 +1,45 @@ +// Frontend functions over a cyclomatic-complexity limit, counted for scripts/audit_metrics.py: a bare +// `complexity` rule run (no type info, no other rules), so it takes seconds, not a full lint. Prints +// one JSON object: {"limit": N, "over": count, "max": highest}. Shipped code only — tests and test +// kits are lists of cases, not logic anyone has to change. +// +// node scripts/fe_complexity.mjs [limit] (default 20, the frontend's ratchet limit) +import { readdirSync, readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SRC = fileURLToPath(new URL("../frontend/src/", import.meta.url)); +const require = createRequire(new URL("../frontend/package.json", import.meta.url)); +const { Linter } = require("eslint"); +const tseslint = require("typescript-eslint"); + +const limit = Number(process.argv[2] ?? 20); +const files = readdirSync(SRC, { recursive: true }) + .map((p) => p.split("\\").join("/")) + .filter( + (p) => /\.(ts|tsx)$/.test(p) && !/\.(test|testkit)\.tsx?$/.test(p) && !p.endsWith(".d.ts"), + ); + +const linter = new Linter(); +const config = [ + { + files: ["**/*.{ts,tsx}"], + languageOptions: { + parser: tseslint.parser, + parserOptions: { ecmaFeatures: { jsx: true } }, + }, + rules: { complexity: ["error", limit] }, + }, +]; +let over = 0; +let max = 0; +for (const f of files) { + for (const m of linter.verify(readFileSync(join(SRC, f), "utf8"), config, f)) { + const n = Number(/complexity of (\d+)/.exec(m.message)?.[1]); + if (!n) continue; + over += 1; + max = Math.max(max, n); + } +} +console.log(JSON.stringify({ limit, over, max })); From a499f002090224c2d6a5a7e41cc77be4a146768a Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 20:09:58 +0200 Subject: [PATCH 05/37] refactor(compose): localdev is an override of docker-compose.yml, not a copy --- docker-compose.localdev.yml | 118 ++++++------------------------------ 1 file changed, 17 insertions(+), 101 deletions(-) diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml index 88dab27d..202ca5b8 100644 --- a/docker-compose.localdev.yml +++ b/docker-compose.localdev.yml @@ -1,138 +1,54 @@ -# Self-contained local development stack — Postgres + API + scheduler, all on this machine. +# Self-contained local development stack — Postgres + API + scheduler + worker, all on this machine. +# An OVERRIDE of docker-compose.yml, not a copy of it: only what local development changes lives +# here, so the two cannot drift apart (they had — the worker here lost its wait for the API's +# migrations). # -# Fully decoupled: its own local database (a Docker volume) and its own scheduler, so there -# is no shared DB and no migration coupling with any other instance. Edit code, rebuild, and -# everything (including the background sync) runs right here. +# docker compose -f docker-compose.yml -f docker-compose.localdev.yml up -d --build # -# docker compose -f docker-compose.localdev.yml up -d --build -# -# Browse at http://localhost:8080 — Google login works because the OAuth redirect is -# http://localhost:8080/auth/callback. The API runs `alembic upgrade head` on startup, so it -# applies local migrations to THIS local DB only. Seed the catalog once with a pg_dump from -# another instance if you want real data (see RUNBOOK). - -# Bounded container logs. Docker's json-file driver keeps every line forever by default, and the API -# now writes one structured line per request (E22 S2) — 3×50 MB per service is days of history on a -# busy instance and a hard ceiling on the disk it can take. -x-logging: &logging - driver: json-file - options: - max-size: "50m" - max-file: "3" +# Fully decoupled: its own local database (a Docker volume) and its own scheduler, so there is no +# shared DB and no migration coupling with any other instance. Browse at http://localhost:8080 — +# Google login works because the OAuth redirect is http://localhost:8080/auth/callback. The API runs +# `alembic upgrade head` on startup, so it applies local migrations to THIS local DB only. Seed the +# catalog once with a pg_dump from another instance if you want real data (see RUNBOOK). services: db: - image: postgres:16-alpine - environment: - POSTGRES_USER: ${POSTGRES_USER:-siftlode} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-siftlode} - POSTGRES_DB: ${POSTGRES_DB:-siftlode} - volumes: - - pgdata:/var/lib/postgresql/data + # Harmless on Docker Desktop; needed on some nested/hardened Docker hosts (every service below). security_opt: - apparmor:unconfined - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-siftlode} -d ${POSTGRES_DB:-siftlode}"] - interval: 5s - timeout: 5s - retries: 12 - logging: *logging - restart: unless-stopped api: - build: - context: . - dockerfile: Dockerfile - args: - APP_VERSION: ${APP_VERSION:-dev} - GIT_SHA: ${GIT_SHA:-unknown} - BUILD_DATE: ${BUILD_DATE:-} - env_file: .env environment: - # Own local database (overrides whatever DATABASE_URL is in .env). - DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode} - # This instance owns its scheduler now (its own DB, so no double-write concern). + # This instance owns its scheduler (its own DB, so no double-write concern). SCHEDULER_ENABLED: "true" - # Download center: the API serves finished files (it does NOT run the worker loop). - DOWNLOAD_ROOT: /downloads - WORKER_ENABLED: "false" # Plex HLS remux scratch on fast container-local storage (the /downloads bind-mount is a slow # Windows Docker Desktop mount in dev; prod uses fast local disk so it keeps the default). PLEX_HLS_DIR: /var/tmp/plex-hls - depends_on: - db: - condition: service_healthy - # Harmless on Docker Desktop; needed on some nested/hardened Docker hosts. security_opt: - apparmor:unconfined - ports: - - "${APP_PORT:-8080}:8000" volumes: - # Downloaded media lives on the host so you can inspect the generated Plex tree. - # Gitignored. The worker writes here; the API reads it to serve local downloads. + # Downloaded media lives on the host so you can inspect the generated Plex tree. Gitignored. - ./downloads:/downloads - # Plex media, read-only (see the plex_media volume note below). Maps to plex_path_map. - plex_media:/plex-media:ro - logging: *logging - restart: unless-stopped - # Dedicated download worker: same image, runs the yt-dlp job loop instead of the API. - # It shares the DB (job queue) and the downloads mount with the API. worker: - build: - context: . - dockerfile: Dockerfile - args: - APP_VERSION: ${APP_VERSION:-dev} - GIT_SHA: ${GIT_SHA:-unknown} - BUILD_DATE: ${BUILD_DATE:-} - command: ["python", "-m", "app.worker"] - # tini as PID 1: reaps orphaned ffmpeg/yt-dlp grandchildren when the watchdog hard-kills a - # download child's process group (the worker reaps its own direct children via subprocess.wait). - init: true - env_file: .env - environment: - DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode} - # The worker must not also run the scheduler; the API owns that. - SCHEDULER_ENABLED: "false" - DOWNLOAD_ROOT: /downloads - WORKER_ENABLED: "true" - depends_on: - db: - condition: service_healthy - bgutil-pot: - condition: service_started security_opt: - apparmor:unconfined volumes: - ./downloads:/downloads - plex_media:/plex-media:ro - logging: *logging - restart: unless-stopped - # PO-token provider: mints YouTube Proof-of-Origin tokens on demand so the worker bypasses - # bot-detection and gets high-quality formats. The worker reaches it at http://bgutil-pot:4416 - # (DOWNLOAD_POT_BASE_URL). Pin the tag to match the plugin version in requirements.txt. bgutil-pot: - image: brainicism/bgutil-ytdlp-pot-provider:1.3.1 - init: true security_opt: - apparmor:unconfined - logging: *logging - restart: unless-stopped volumes: - pgdata: # OPTIONAL (Plex integration dev testing): the Plex media, mounted READ-ONLY so the container can # play the physical file locally (this dev box isn't the Plex host). Only referenced when the Plex - # module is enabled; harmless otherwise (Docker mounts it lazily on first container use). On prod - # (LXC on the Plex host) this is a plain host bind-mount. - # - # Local bind to a few test movies/shows kept on this box — the full nas.lan SMB share was too slow. - # Point PLEX_MEDIA_HOST_PATH (.env, gitignored, so no site path lives in the tracked compose) at the - # Docker-Desktop host path (e.g. /run/desktop/mnt/host/d/plex-test-media for D:\plex-test-media), - # mirroring the Plex server's /data//… layout so plex_path_map (/data=/plex-media) resolves. - # (The prior CIFS/PLEX_SMB_* approach for the whole nas.lan share lives in git history.) + # module is enabled; harmless otherwise. Point PLEX_MEDIA_HOST_PATH (.env, gitignored, so no site + # path lives in the tracked compose) at the Docker-Desktop host path (e.g. + # /run/desktop/mnt/host/d/plex-test-media for D:\plex-test-media), mirroring the Plex server's + # /data//… layout so plex_path_map (/data=/plex-media) resolves. plex_media: driver: local driver_opts: From 7f08c510bf1ffcac2b21be64f0759ad057000efc Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 21:02:03 +0200 Subject: [PATCH 06/37] feat(auth)!: remove the public access-request endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /auth/request-access had no caller left: the login page's form was gone, and registration and a refused Google sign-in already land in the same admin approval. A third public write path with no protection of its own (owner decision, E34). Its one guard moves to where every path passes: upsert_pending_invite no longer opens a pending row for an address the env already lets in. The registration path lacked it — registering b.ob@gmail.com beside an env-granted bob@gmail.com opened a pending row, and is_allowed answers from the invite rows once any exist, so the real owner was locked out. The tests that drove the endpoint now drive registration or the shared function. --- backend/app/mailtext.py | 2 +- backend/app/routes/auth.py | 53 ++++++-------------------- backend/tests/test_mail_language.py | 59 +++++++++-------------------- backend/tests/test_review_round.py | 23 +++++------ frontend/src/lib/api/auth.ts | 9 +---- 5 files changed, 39 insertions(+), 107 deletions(-) diff --git a/backend/app/mailtext.py b/backend/app/mailtext.py index 68485609..f8be6152 100644 --- a/backend/app/mailtext.py +++ b/backend/app/mailtext.py @@ -53,7 +53,7 @@ def lang_for_user(user: User | None) -> str: def lang_for_email(db: Session, email: str) -> str: """The language for an address that may not have an account yet: their own account preference, - else the language they filled our access-request form in, else English. + else the language their access request came in (the locale Google reported), else English. Matched CANONICALLY like every other identity lookup in this epic, and it walks past rows that carry no usable code instead of stopping at the first one — prod really does hold two accounts diff --git a/backend/app/routes/auth.py b/backend/app/routes/auth.py index 2fbca25b..d3396c40 100644 --- a/backend/app/routes/auth.py +++ b/backend/app/routes/auth.py @@ -29,6 +29,7 @@ from app.auth import ( WRITE_SCOPES, current_user, env_admin_canonicals, + env_allowed_canonicals, find_user_by_email, get_or_create_demo_user, google_enabled, @@ -72,7 +73,6 @@ _reset_limiter = RateLimiter(max_events=5, window_seconds=300) # Separate from _reset_limiter on purpose: opening a link must not eat the budget for REQUESTING one. _reset_check_limiter = RateLimiter(max_events=10, window_seconds=300) _verify_resend_limiter = RateLimiter(max_events=5, window_seconds=300) -_request_access_limiter = RateLimiter(max_events=5, window_seconds=300) _suspend_email_limiter = RateLimiter(max_events=1, window_seconds=3600) @@ -196,9 +196,9 @@ def upsert_pending_invite(db: Session, email: str, lang: str | None = None) -> I not reopen a decision that has already been made for that mailbox. The row is still STORED under the address as typed, because that is where mail has to go. - `lang` is the language of the surface the request came from — the sign-in page, or the locale - Google reported for a refused sign-in. The latest one wins: someone who asks again from a - Hungarian page is telling us something newer than last time. Anything unsupported is ignored + `lang` is the language the request came in — the locale Google reported for a refused sign-in. + The latest one wins: someone who asks again under another locale is telling us something newer + than last time. Anything unsupported is ignored rather than stored, so a forged body cannot put junk in the column. The commit contract is unchanged: this still commits ONLY when it writes something. Callers rely @@ -215,6 +215,12 @@ def upsert_pending_invite(db: Session, email: str, lang: str | None = None) -> I .scalars() .all() ) + # An address the env already lets in has nothing to ask for, and a pending row for it would be + # worse than nothing: `is_allowed` answers from the invite rows once any exist, so a stranger + # registering a sibling spelling (`b.ob@` beside an env-granted `bob@`) would lock the real + # owner out of a grant no admin could see. + if not rows and canon in env_allowed_canonicals(): + return InviteRequest(None, False) # An approved row wins, so a denied duplicate can't mask an approval. Then a row that is not # DENIED, preferring the exact spelling — a denied sibling must never be picked while a pending # row exists, or the reopen branch below would fire and put one person in the admin's queue @@ -628,43 +634,6 @@ async def logout(request: Request): return JSONResponse({"ok": True, "switched": False}) -class AccessRequestIn(EmailIn): - # The language of the page this request was made from. Same reasoning as registration's `lang` - # (and the same shape): it is the ONLY moment this preference exists anywhere the server can - # see, since the requester has no account to carry it, and it decides what language the approval - # or denial mail is written in. - lang: str | None = None - - -@router.post("/request-access") -async def request_access( - body: AccessRequestIn, - request: Request, - background: BackgroundTasks, - db: Session = Depends(get_db), -) -> dict: - """Public: ask for access. Idempotent — repeat requests for the same email don't - duplicate or re-spam admins (upsert answers `is_new=False` for an already-pending row). - - Answers a uniform ``"pending"`` for EVERY outcome — already-allowed, newly-pending, - already-pending, and rate-limited alike. The old ``"approved"`` reply for an allow-listed - address made this a public enumeration oracle for the allow-list (which the rest of the module - is careful to avoid, cf. demo_login); an already-allowed user can still just sign in with Google. - Rate-limited per client IP like register/login/reset, and a block returns "pending" too so the - endpoint can't be probed for throttling either.""" - email = (body.email or "").strip().lower() - if not valid_email(email): - raise refused(400, "invalid_email") - if not _request_access_limiter.allow(client_key(request)): - return {"ok": True, "status": "pending"} - if is_allowed(db, email): - return {"ok": True, "status": "pending"} # already allowed — don't reveal it - req = upsert_pending_invite(db, email, body.lang) - if req.is_new: - notify_admins_of_request(db, background, email) - return {"ok": True, "status": "pending"} - - @router.post("/demo") def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -> dict: """Hidden demo entry: a whitelisted email logs straight into the shared demo account, @@ -918,7 +887,7 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non db.commit() if email_ok: send_verification_link(db, user) - # Fired unconditionally, unlike the other two callers which gate on a fresh invite row: for + # Fired unconditionally, unlike the Google callback, which gates on a fresh invite row: for # an address the admin already approved, nothing lands in the Access-requests queue, so this # is the only place the account's existence is announced at all. Delivery is a separate # matter — with no working SMTP `_send` drops it, which is why the pre-approved registrant diff --git a/backend/tests/test_mail_language.py b/backend/tests/test_mail_language.py index f5ef1b7d..b5316fbd 100644 --- a/backend/tests/test_mail_language.py +++ b/backend/tests/test_mail_language.py @@ -131,6 +131,12 @@ def test_the_lookup_is_canonical_like_every_other_identity_match(db, make_user): # --- capturing the signal at the one moment it exists --------------------------------------------- +def _ask(db, lang: str) -> None: + """A pending request carrying the language it was made in — what a refused Google sign-in records + (with the locale Google reported), through the one function every entry point shares.""" + auth_routes.upsert_pending_invite(db, "asking@example.com", lang) + + def test_registration_records_the_language_of_the_sign_up_page(db, client): r = client.post( "/auth/register", json={"email": "signup@example.com", "password": _PW, "lang": "hu"} @@ -141,31 +147,16 @@ def test_registration_records_the_language_of_the_sign_up_page(db, client): assert (user.preferences or {}).get("language") == "hu" -def test_an_access_request_records_the_language_it_was_made_in(db, client): - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "hu"} - ).status_code - == 200 - ) +def test_an_access_request_records_the_language_it_was_made_in(db): + _ask(db, "hu") inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one() assert inv.lang == "hu" -def test_asking_again_from_another_language_updates_the_row(db, client): - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "en"} - ).status_code - == 200 - ) - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "hu"} - ).status_code - == 200 - ) +def test_asking_again_from_another_language_updates_the_row(db): + _ask(db, "en") + _ask(db, "hu") db.expire_all() inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one() @@ -174,13 +165,8 @@ def test_asking_again_from_another_language_updates_the_row(db, client): assert db.execute(select(Invite)).scalars().all() == [inv] -def test_an_unsupported_language_is_ignored_rather_than_stored(db, client): - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "../../etc"} - ).status_code - == 200 - ) +def test_an_unsupported_language_is_ignored_rather_than_stored(db): + _ask(db, "../../etc") inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one() assert inv.lang is None @@ -222,21 +208,15 @@ def test_the_confirmation_mail_is_written_in_the_accounts_language(db, client, m def test_the_approval_mail_answers_in_the_language_the_request_was_made_in( - db, client, make_user, as_user, monkeypatch + db, make_user, as_user, monkeypatch ): from app import email as email_mod monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True) sent = _bodies(monkeypatch) admin = make_user(email="admin@example.com", role="admin") - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "hu"} - ).status_code - == 200 - ) + _ask(db, "hu") inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one() - sent.clear() # drop the admin notification; this test is about the reply to the requester r = as_user(admin).post(f"/api/admin/invites/{inv.id}/approve") @@ -246,18 +226,13 @@ def test_the_approval_mail_answers_in_the_language_the_request_was_made_in( assert subject == mailtext.MAIL_TEXT["access_approved.subject"]["hu"] -def test_a_denial_answers_in_that_language_too(db, client, make_user, as_user, monkeypatch): +def test_a_denial_answers_in_that_language_too(db, make_user, as_user, monkeypatch): from app import email as email_mod monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True) sent = _bodies(monkeypatch) admin = make_user(email="admin@example.com", role="admin") - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "hu"} - ).status_code - == 200 - ) + _ask(db, "hu") inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one() sent.clear() diff --git a/backend/tests/test_review_round.py b/backend/tests/test_review_round.py index 6574e296..68a6fb25 100644 --- a/backend/tests/test_review_round.py +++ b/backend/tests/test_review_round.py @@ -175,7 +175,7 @@ def test_a_stripped_account_keeps_its_recovery_stamp(db, client, smtp_on, make_u assert row.password_hash is None and row.password_revoked_at is not None -def test_a_denied_sibling_is_not_reopened_while_a_pending_row_exists(db, client): +def test_a_denied_sibling_is_not_reopened_while_a_pending_row_exists(db): # Canonical siblings exist on prod. Picking the denied one reopens it and re-notifies every # admin, putting one person in the queue twice — and the exact-spelling tie-break made that # deterministic rather than merely possible. @@ -187,9 +187,7 @@ def test_a_denied_sibling_is_not_reopened_while_a_pending_row_exists(db, client) ) db.commit() - assert ( - client.post("/auth/request-access", json={"email": "pairone@gmail.com"}).status_code == 200 - ) + auth_routes.upsert_pending_invite(db, "pairone@gmail.com") db.expire_all() rows = {i.email: i.status for i in db.execute(select(Invite)).scalars().all()} @@ -210,12 +208,13 @@ def test_the_env_allow_list_is_matched_canonically(db, monkeypatch): def test_an_env_granted_address_cannot_be_shadowed_by_a_stranger(db, client, monkeypatch): - # `is_allowed` short-circuits on ANY invite row for the canonical form, so before this an - # unauthenticated caller could request access under a sibling spelling and permanently shadow an - # env grant. With the env list matched canonically the request never opens a row. + # `is_allowed` short-circuits on ANY invite row for the canonical form, so an unauthenticated + # caller registering under a sibling spelling would open a pending row and permanently shadow an + # env grant. The env list is matched canonically, so the registration never opens a row. monkeypatch.setattr(settings, "allowed_emails", "bob@gmail.com") - assert client.post("/auth/request-access", json={"email": "b.ob@gmail.com"}).status_code == 200 + r = client.post("/auth/register", json={"email": "b.ob@gmail.com", "password": _PW}) + assert r.status_code == 200 assert db.execute(select(Invite)).scalars().all() == [] assert auth_core.is_allowed(db, "bob@gmail.com") is True @@ -351,12 +350,8 @@ def test_two_admins_two_languages_two_mails(db, client, make_user, smtp_on, wire make_user(email="hu.admin@example.com", role="admin", preferences={"language": "hu"}) make_user(email="en.admin@example.com", role="admin", preferences={"language": "en"}) - assert ( - client.post( - "/auth/request-access", json={"email": "asking@example.com", "lang": "en"} - ).status_code - == 200 - ) + r = client.post("/auth/register", json={"email": "asking@example.com", "password": _PW}) + assert r.status_code == 200 notices = [w for w in wire if "asking@example.com" in w[1]] assert len(notices) == 2 diff --git a/frontend/src/lib/api/auth.ts b/frontend/src/lib/api/auth.ts index b556b425..bdd29365 100644 --- a/frontend/src/lib/api/auth.ts +++ b/frontend/src/lib/api/auth.ts @@ -127,14 +127,7 @@ export const authApi = { { quiet: true }, ), - // --- onboarding: access requests + demo entry --- - // Same reasoning as register: the requester has no account to carry a language preference, so the - // page they asked from is what decides the language of the approval or denial mail. - requestAccess: (email: string): Promise<{ status: string }> => - req("/auth/request-access", { - method: "POST", - body: JSON.stringify({ email, lang: uiLang() }), - }), + // --- onboarding: demo entry --- // Hidden demo entry: a whitelisted email logs into the shared demo account, no Google // sign-in. Returns whether a session was established. demoLogin: (email: string): Promise<{ authenticated: boolean }> => From cca331583627f5ade8119335616f170f70f51fed Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 4 Oct 2026 21:02:03 +0200 Subject: [PATCH 07/37] feat!: remove direct messaging (chat) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E2EE direct messages, the chat dock, the Messages module, the message WebSocket and its realtime registry, and the messages / message_keys tables (migration 0097; the downgrade restores the empty schema). Prod held two welcome notes, no message between people and one key — never used (owner decision, E34). WebSocket now has no owner module: the lint bans it everywhere. Privacy policy updated: it no longer promises E2EE for a feature that is gone. --- CLAUDE.md | 2 +- backend/.basedpyright/baseline.json | 90 ---- .../alembic/versions/0097_drop_messages.py | 69 +++ backend/app/README.md | 2 +- backend/app/auth.py | 7 +- backend/app/logs.py | 1 - backend/app/main.py | 2 - backend/app/models.py | 63 +-- backend/app/realtime.py | 68 --- backend/app/routes/messages.py | 479 ------------------ backend/app/worker.py | 2 +- .../tests/fixtures/log_redaction_cases.json | 2 +- backend/tests/test_collation.py | 21 +- backend/tests/test_logs.py | 8 +- backend/tests/test_messages_welcome.py | 53 -- backend/tests/test_route_conventions.py | 4 +- docs/gates.md | 1 - frontend/eslint.config.js | 10 +- frontend/src/App.tsx | 2 - frontend/src/components/ChatDock.tsx | 131 ----- frontend/src/components/ChatThread.tsx | 243 --------- frontend/src/components/KeyGate.tsx | 185 ------- .../components/Messages.keyboard.dom.test.tsx | 70 --- frontend/src/components/Messages.tsx | 338 ------------ frontend/src/components/NavSidebar.tsx | 19 +- frontend/src/components/PlexPlayer.tsx | 2 +- frontend/src/components/UploadDialog.tsx | 2 +- .../src/components/formAlerts.dom.test.tsx | 20 +- frontend/src/components/legal/LegalLayout.tsx | 2 +- .../src/components/legal/PrivacyPolicy.tsx | 22 +- frontend/src/components/modulePages.tsx | 10 - frontend/src/components/moduleRegistry.tsx | 3 +- .../src/components/ui/Avatar.dom.test.tsx | 4 +- frontend/src/components/ui/Avatar.tsx | 2 +- frontend/src/components/ui/NavRail.tsx | 2 +- frontend/src/components/ui/README.md | 5 +- frontend/src/design-lab/batchB.tsx | 2 - frontend/src/i18n/locales/en/errors.json | 6 - frontend/src/i18n/locales/en/keys.json | 1 - frontend/src/i18n/locales/en/messages.json | 41 -- frontend/src/i18n/locales/en/nav.json | 1 - frontend/src/i18n/locales/en/settings.json | 2 +- frontend/src/i18n/locales/hu/errors.json | 6 - frontend/src/i18n/locales/hu/keys.json | 1 - frontend/src/i18n/locales/hu/messages.json | 41 -- frontend/src/i18n/locales/hu/nav.json | 1 - frontend/src/i18n/locales/hu/settings.json | 2 +- frontend/src/index.css | 26 - frontend/src/lib/api.ts | 3 - frontend/src/lib/api/facade.test.ts | 1 - frontend/src/lib/api/messaging.ts | 72 --- frontend/src/lib/chatDock.dom.test.ts | 52 -- frontend/src/lib/chatDock.ts | 113 ----- frontend/src/lib/e2ee.test.ts | 180 ------- frontend/src/lib/e2ee.ts | 253 --------- frontend/src/lib/gates.snapshot.json | 73 +-- frontend/src/lib/history.ts | 2 +- frontend/src/lib/keymap.ts | 1 - frontend/src/lib/log.ts | 2 +- frontend/src/lib/messagesSocket.test.ts | 74 --- frontend/src/lib/messagesSocket.ts | 137 ----- frontend/src/lib/messaging.ts | 66 --- frontend/src/lib/modules.ts | 9 - .../src/lib/networkBoundary.contract.test.ts | 1 - frontend/src/lib/pageMeta.test.ts | 1 - frontend/src/lib/queryKeys.test.ts | 3 - frontend/src/lib/queryKeys.ts | 8 +- frontend/src/lib/sourceSize.contract.test.ts | 1 - frontend/src/lib/storage.ts | 1 - frontend/src/lib/urlState.ts | 1 - scripts/audit_metrics.py | 3 +- 71 files changed, 122 insertions(+), 3011 deletions(-) create mode 100644 backend/alembic/versions/0097_drop_messages.py delete mode 100644 backend/app/realtime.py delete mode 100644 backend/app/routes/messages.py delete mode 100644 backend/tests/test_messages_welcome.py delete mode 100644 frontend/src/components/ChatDock.tsx delete mode 100644 frontend/src/components/ChatThread.tsx delete mode 100644 frontend/src/components/KeyGate.tsx delete mode 100644 frontend/src/components/Messages.keyboard.dom.test.tsx delete mode 100644 frontend/src/components/Messages.tsx delete mode 100644 frontend/src/i18n/locales/en/messages.json delete mode 100644 frontend/src/i18n/locales/hu/messages.json delete mode 100644 frontend/src/lib/api/messaging.ts delete mode 100644 frontend/src/lib/chatDock.dom.test.ts delete mode 100644 frontend/src/lib/chatDock.ts delete mode 100644 frontend/src/lib/e2ee.test.ts delete mode 100644 frontend/src/lib/e2ee.ts delete mode 100644 frontend/src/lib/messagesSocket.test.ts delete mode 100644 frontend/src/lib/messagesSocket.ts delete mode 100644 frontend/src/lib/messaging.ts diff --git a/CLAUDE.md b/CLAUDE.md index 99230748..6335df25 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,7 +63,7 @@ how it gets built. Concretely: `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, `downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`, -`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, `collation`, +`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`, `security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json index 2a942acc..9ffc4757 100644 --- a/backend/.basedpyright/baseline.json +++ b/backend/.basedpyright/baseline.json @@ -824,96 +824,6 @@ } } ], - "./app/routes/messages.py": [ - { - "code": "reportCallIssue", - "range": { - "startColumn": 11, - "endColumn": 43, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportCallIssue", - "range": { - "startColumn": 13, - "endColumn": 5, - "lineCount": 7 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 8, - "endColumn": 15, - "lineCount": 5 - } - }, - { - "code": "reportReturnType", - "range": { - "startColumn": 15, - "endColumn": 72, - "lineCount": 1 - } - }, - { - "code": "reportCallIssue", - "range": { - "startColumn": 30, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 41, - "endColumn": 44, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 36, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 71, - "endColumn": 73, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 27, - "endColumn": 29, - "lineCount": 1 - } - } - ], "./app/routes/playlists.py": [ { "code": "reportOptionalOperand", diff --git a/backend/alembic/versions/0097_drop_messages.py b/backend/alembic/versions/0097_drop_messages.py new file mode 100644 index 00000000..a3d56ce2 --- /dev/null +++ b/backend/alembic/versions/0097_drop_messages.py @@ -0,0 +1,69 @@ +"""messages: drop direct messaging (the owner's decision, 2026-10-04) + +Chat — end-to-end encrypted direct messages plus the server's welcome note — was never used: prod held +two welcome notes, no message between people, and one key. Its tables go with its code. The rows are +not migrated anywhere; the nightly backup holds them. + +The downgrade restores the schema as it stood (0026 + 0027), empty: a message can't be brought back +from a migration, and the E2EE key blobs were only ever readable by their owner's browser anyway. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0097_drop_messages" +down_revision: str | None = "0096_videos_sort_indexes" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.drop_table("message_keys") + op.drop_index("ix_messages_created_at", table_name="messages") + op.drop_index("ix_messages_recipient_id", table_name="messages") + op.drop_index("ix_messages_sender_id", table_name="messages") + op.drop_table("messages") + + +def downgrade() -> None: + op.create_table( + "messages", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "sender_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=True + ), + sa.Column( + "recipient_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("body", sa.Text(), nullable=True), + sa.Column("read_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column("kind", sa.String(length=16), nullable=False, server_default="user"), + sa.Column("ciphertext", sa.Text(), nullable=True), + sa.Column("iv", sa.String(length=32), nullable=True), + ) + op.create_index("ix_messages_sender_id", "messages", ["sender_id"]) + op.create_index("ix_messages_recipient_id", "messages", ["recipient_id"]) + op.create_index("ix_messages_created_at", "messages", ["created_at"]) + op.create_table( + "message_keys", + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column("public_key", sa.Text(), nullable=False), + sa.Column("wrapped_private_key", sa.Text(), nullable=False), + sa.Column("salt", sa.String(length=64), nullable=False), + sa.Column("wrap_iv", sa.String(length=32), nullable=False), + sa.Column("key_check", sa.Text(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + ) diff --git a/backend/app/README.md b/backend/app/README.md index b7c885c5..f200d455 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -21,7 +21,7 @@ Three layers, by *who* runs the code — not by feature: `issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker. 3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`, - `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging + `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core diff --git a/backend/app/auth.py b/backend/app/auth.py index cbee737c..2460042e 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -278,8 +278,7 @@ ACTIVE_ACCOUNT_HEADER = "x-siftlode-account" def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]: """Which account this connection acts as, and whether that's the session's default account. - Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this - exact per-tab resolution instead of re-implementing it. + Takes an HTTPConnection rather than a Request: it reads only the session and the headers. The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser) plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or @@ -297,8 +296,8 @@ def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int """ default_id = conn.session.get("user_id") wallet = conn.session.get("account_ids") or [] - # Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a - # plain file download). Both are wallet-gated below. + # Header for normal XHR; ?account= for a context that can't set headers (a plain file + # download). Both are wallet-gated below. hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account") if hdr: try: diff --git a/backend/app/logs.py b/backend/app/logs.py index d25d5380..1b099320 100644 --- a/backend/app/logs.py +++ b/backend/app/logs.py @@ -688,7 +688,6 @@ _POLLED_ROUTES = frozenset( { "/api/me", "/api/me/notifications/unread_count", - "/api/messages/unread_count", "/api/downloads", "/api/downloads/index", "/api/downloads/usage", diff --git a/backend/app/main.py b/backend/app/main.py index 1376e980..37d39045 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -44,7 +44,6 @@ from app.routes import ( # noqa: E402 — logging is configured first feed, health, me, - messages, notifications, playlists, plex as plex_routes, @@ -311,7 +310,6 @@ app.include_router(me.router) app.include_router(diagnostics_routes.router) app.include_router(debug_sessions_routes.router) app.include_router(notifications.router) -app.include_router(messages.router) app.include_router(channels.router) app.include_router(playlists.router) app.include_router(saved_views.router) diff --git a/backend/app/models.py b/backend/app/models.py index 2e406a39..5d5d22fd 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -59,7 +59,7 @@ POSITION_MAX_S = 2**31 - 1 class TimestampMixin: """Adds a server-defaulted `created_at`. Models that index created_at or order it - differently (QuotaEvent, Notification, Message) keep their own column on purpose.""" + differently (QuotaEvent, Notification) keep their own column on purpose.""" created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) @@ -814,44 +814,6 @@ class Notification(Base): ) -class Message(Base): - """A message in the notification module (phase 2). - - Two kinds share this table: - - `kind="user"`: a private, END-TO-END ENCRYPTED direct message between two users. The - server only ever stores `ciphertext` + `iv` (base64); it never holds the plaintext or - the keys, so not even an admin can read it. A conversation is just the set of messages - between two users — no separate thread entity; queries group by the {sender, recipient} - pair. - - `kind="system"`: a server-authored plaintext message (e.g. the welcome, or future admin - announcements). `sender_id` is NULL and `body` holds the text. These are non-private - boilerplate, readable without any E2EE key setup, and surface as a "Siftlode" conversation. - - `read_at` is stamped when the recipient opens the thread (NULL = unread, feeds the nav - badge). The recipient FK cascades (and sender, for user messages) so a GDPR account erasure - removes that user's messages outright.""" - - __tablename__ = "messages" - - id: Mapped[int] = mapped_column(primary_key=True) - kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user") - # NULL for system messages (no human sender). - sender_id: Mapped[int | None] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), index=True - ) - recipient_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), index=True - ) - # System messages carry plaintext `body`; user messages carry `ciphertext`+`iv` (base64). - body: Mapped[str | None] = mapped_column(Text) - ciphertext: Mapped[str | None] = mapped_column(Text) - iv: Mapped[str | None] = mapped_column(String(32)) - read_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), server_default=func.now(), index=True - ) - - class MediaAsset(Base, TimestampMixin): """A physical downloaded media file — the SHARED cache layer of the download center. @@ -1094,29 +1056,6 @@ class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin): cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged -class MessageKey(Base, TimestampMixin): - """A user's end-to-end-encryption key material for direct messaging (phase 2). - - One row per user. `public_key` (base64 SPKI, ECDH P-256) is shared with others so they can - derive the pairwise conversation key. `wrapped_private_key` is the user's private key - encrypted IN THE BROWSER with a key derived (PBKDF2) from a message passphrase the server - never sees — so the server stores only an opaque blob it cannot unwrap. `salt`/`wrap_iv` - parameterise that derivation; `key_check` is a small ciphertext the client decrypts to - verify a passphrase on unlock. The server is purely a key directory + blob store; it can - never read any message.""" - - __tablename__ = "message_keys" - - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), primary_key=True - ) - public_key: Mapped[str] = mapped_column(Text) - wrapped_private_key: Mapped[str] = mapped_column(Text) - salt: Mapped[str] = mapped_column(String(64)) - wrap_iv: Mapped[str] = mapped_column(String(32)) - key_check: Mapped[str] = mapped_column(Text) - - # --- Plex integration (optional) ------------------------------------------------------------- # A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is # from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states diff --git a/backend/app/realtime.py b/backend/app/realtime.py deleted file mode 100644 index 0f29e086..00000000 --- a/backend/app/realtime.py +++ /dev/null @@ -1,68 +0,0 @@ -"""In-process WebSocket connection registry for live server→client push. - -Generic groundwork (notification module phase 2 is the first user): connections are kept in -this process's memory, keyed by user_id, so a request handler can push an event to all of a -user's open tabs/devices. Single-uvicorn-process assumption — if we ever run multiple workers, -replace the in-memory map with a Redis pub/sub fan-out behind the same `push()` interface (same -note as app.ratelimit). - -`push()` is intentionally SYNC-callable: ordinary (sync) route handlers run in a threadpool, so -they schedule the actual send onto the WebSocket event loop via run_coroutine_threadsafe. The -connect/disconnect/_push coroutines all run on that one loop, so the connection map is only ever -mutated from a single thread. -""" - -import asyncio -from collections import defaultdict - -from starlette.websockets import WebSocket -from app import logs - -log = logs.get_logger(__name__) - - -class ConnectionManager: - def __init__(self) -> None: - self._conns: dict[int, set[WebSocket]] = defaultdict(set) - self._loop: asyncio.AbstractEventLoop | None = None - - async def connect(self, user_id: int, ws: WebSocket) -> None: - self._loop = asyncio.get_running_loop() - await ws.accept() - self._conns[user_id].add(ws) - - async def disconnect(self, user_id: int, ws: WebSocket) -> None: - conns = self._conns.get(user_id) - if conns: - conns.discard(ws) - if not conns: - self._conns.pop(user_id, None) - - async def _push(self, user_id: int, payload: dict) -> None: - for ws in list(self._conns.get(user_id, ())): - try: - await ws.send_json(payload) - except Exception as exc: # noqa: BLE001 — one dead socket must not stop the others - # Stale socket; it'll be cleaned up by its own disconnect handler. But this is - # also where a live update goes missing: the UI simply never changes, and there - # is no request behind it whose failure anyone could look up. - log.debug( - "a live update could not be pushed to a socket", - extra=logs.event( - "realtime.push_failed", user_id=user_id, reason=type(exc).__name__ - ), - ) - - def push(self, user_id: int, payload: dict) -> None: - """Schedule a push onto the WebSocket event loop. Safe to call from a sync route - handler (threadpool). No-op if no WebSocket has ever connected (no loop captured).""" - loop = self._loop - if loop is None: - return - try: - asyncio.run_coroutine_threadsafe(self._push(user_id, payload), loop) - except Exception: # noqa: BLE001 — a closed loop or socket must not fail the caller - log.debug("ws push failed", exc_info=True) - - -manager = ConnectionManager() diff --git a/backend/app/routes/messages.py b/backend/app/routes/messages.py deleted file mode 100644 index 28270603..00000000 --- a/backend/app/routes/messages.py +++ /dev/null @@ -1,479 +0,0 @@ -"""Direct messaging (notification module, phase 2): end-to-end encrypted user-to-user chat -plus server-authored system messages, with live WebSocket delivery. - -E2EE model: every private message is encrypted IN THE BROWSER under a key both parties derive -from ECDH(their private key, the other's public key). The server only ever stores ciphertext + -iv and acts as a key directory (public keys) and an opaque blob store (each user's private key, -wrapped client-side with a passphrase the server never sees). So not even an admin can read a -user-to-user conversation. `kind="system"` messages (the welcome; future announcements) are -plain server boilerplate — non-private, readable without any key setup, shown as a "Siftlode" -conversation. - -Every endpoint is gated by `require_human` (the shared demo account can't message). Metadata -(who messaged whom, when, read state) is NOT encrypted — standard for E2EE and needed for -routing and unread counts. -""" - -from datetime import datetime, timezone - -from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect -from pydantic import BaseModel -from sqlalchemy import and_, case, delete, func, or_, select -from sqlalchemy.orm import Session - -from app.mailtext import lang_for_user -from app.collation import collated -from app.auth import require_human, resolved_user_id -from app.db import SessionLocal, get_db -from app.models import Message, MessageKey, User -from app.ratelimit import RateLimiter -from app.realtime import manager -from app.security import verify_password -from app.userscope import is_messageable_user, messageable_clauses -from app.utils import iso -from app.refusal import refused -from app.routes._common import not_found - -router = APIRouter(prefix="/api/messages", tags=["messages"]) - -MAX_CIPHERTEXT = 8000 # base64 of a generous plaintext + AES-GCM overhead -SYSTEM_PARTNER_ID = 0 # synthetic conversation partner for system messages -SYSTEM_NAME = "Siftlode" -_send_limiter = RateLimiter(max_events=30, window_seconds=60) -_reset_key_limiter = RateLimiter(max_events=5, window_seconds=300) - -# The welcome is rendered server-side at creation time, so it carries its own translations -# (one per supported UI language). Kept short and non-private (every new user gets the same text). -WELCOME = { - "en": ( - "Welcome to Siftlode! 👋\n\n" - "This is your private message inbox. Conversations with other members are end-to-end " - "encrypted — only you and the person you're chatting with can read them, not even an " - "admin. To start messaging someone, set up secure messaging with a passphrase first.\n\n" - "Enjoy your Feed!" - ), - "hu": ( - "Üdv a Siftlode-on! 👋\n\n" - "Ez a privát üzenet-postafiókod. A többi taggal folytatott beszélgetések végpontig " - "titkosítottak — csak te és a beszélgetőpartnered olvashatja őket, még az admin sem. " - "Ha üzenni szeretnél valakinek, előbb állítsd be a biztonságos üzenetküldést egy " - "jelszóval.\n\n" - "Jó böngészést!" - ), -} - - -class KeySetupIn(BaseModel): - public_key: str - wrapped_private_key: str - salt: str - wrap_iv: str - key_check: str - - -class KeyResetIn(BaseModel): - # Present only for password accounts; a Google-only account has no password to re-verify. - current_password: str | None = None - - -class SendIn(BaseModel): - recipient_id: int - ciphertext: str - iv: str - - -def _user_label(u: User) -> str: - return u.display_name or u.email.split("@")[0] - - -def _serialize_user(u: User) -> dict: - return {"id": u.id, "name": _user_label(u), "avatar_url": u.avatar_url} - - -def _system_partner() -> dict: - return {"id": SYSTEM_PARTNER_ID, "name": SYSTEM_NAME, "avatar_url": None} - - -# The welcome's opening line in every language — how a stored copy is recognised. It was rendered -# once, in the language of the day it was created, so an account that later switched language kept -# reading it in the old one (E31 S1: the owner's, in English). Recognised here, it is served in the -# reader's language instead. Derived from WELCOME itself, so a change to an OPENING line, or a -# language taken out of WELCOME, orphans the copies already stored: they are served as stored. The -# one retired language (German, 2026-07) left none on prod (checked 2026-10-02). -_WELCOME_OPENINGS = {text.partition("\n")[0] for text in WELCOME.values()} - - -def _system_body(body: str | None, lang: str | None) -> str | None: - """A system message's text for a reader of `lang`: the welcome in their language, any other as - stored.""" - if body is None or lang is None or body.partition("\n")[0] not in _WELCOME_OPENINGS: - return body - return WELCOME.get(lang, WELCOME["en"]) - - -def _serialize_msg(m: Message, lang: str | None = None) -> dict: - """`lang`: the reader's language, for a system message's text (a user message is ciphertext).""" - return { - "id": m.id, - "kind": m.kind, - "sender_id": m.sender_id, - "recipient_id": m.recipient_id, - "body": _system_body(m.body, lang) if m.kind == "system" else m.body, - "ciphertext": m.ciphertext, - "iv": m.iv, - "read_at": iso(m.read_at), - "created_at": iso(m.created_at), - } - - -def ensure_welcome(db: Session, user: User) -> None: - """Create the one-time system welcome message for a (human) user the first time it's needed. - Idempotent: guarded on whether any system message already exists for them.""" - if user.is_demo: - return - exists = db.scalar( - select(Message.id).where(Message.recipient_id == user.id, Message.kind == "system").limit(1) - ) - if exists: - return - lang = (user.preferences or {}).get("language") - body = WELCOME.get(lang, WELCOME["en"]) - m = Message(kind="system", sender_id=None, recipient_id=user.id, body=body) - db.add(m) - db.commit() - db.refresh(m) - manager.push(user.id, {"type": "message", "message": _serialize_msg(m, lang_for_user(user))}) - - -# --- E2EE key directory / blob store ----------------------------------------- - - -@router.get("/keys/me") -def my_key(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict: - """My own key bundle. `configured` says whether secure messaging is set up; if so the - wrapped blob + params are returned so this device can unlock with the passphrase.""" - k = db.get(MessageKey, user.id) - if k is None: - return {"configured": False} - return { - "configured": True, - "public_key": k.public_key, - "wrapped_private_key": k.wrapped_private_key, - "salt": k.salt, - "wrap_iv": k.wrap_iv, - "key_check": k.key_check, - } - - -@router.post("/keys") -def setup_keys( - payload: KeySetupIn, - user: User = Depends(require_human), - db: Session = Depends(get_db), -) -> dict: - """Store this user's public key + passphrase-wrapped private key. Set-once: changing the - key would orphan every existing conversation key, so re-setup is rejected here.""" - if db.get(MessageKey, user.id) is not None: - raise refused(409, "messaging_set_up") - db.add( - MessageKey( - user_id=user.id, - public_key=payload.public_key, - wrapped_private_key=payload.wrapped_private_key, - salt=payload.salt, - wrap_iv=payload.wrap_iv, - key_check=payload.key_check, - ) - ) - db.commit() - return {"ok": True, "configured": True} - - -@router.post("/keys/reset") -def reset_keys( - payload: KeyResetIn, - user: User = Depends(require_human), - db: Session = Depends(get_db), -) -> dict: - """Forgotten-passphrase escape hatch: drop this user's secure-messaging key so they can set a - NEW passphrase (the set-once guard in setup_keys otherwise makes a lost passphrase a permanent - wall). Destroys the user's entire E2EE history BY DESIGN — every conversation key is derived - from this keypair, so replacing it orphans every stored ciphertext in both directions. Those - now-undecryptable `user` messages are deleted here (a clean slate for both parties) rather than - left as permanent "can't decrypt" clutter; `system` messages (plaintext) are untouched. - - An account WITH a password must re-authenticate: a hijacked session must not be able to silently - reset messaging and then read the victim's future messages under a new passphrase. A password- - less (Google-only) account has nothing to re-verify, so it relies on its live session plus the - client-side danger confirmation.""" - if not _reset_key_limiter.allow(str(user.id)): - raise refused(429, "too_many_attempts") - if user.password_hash and not verify_password( - payload.current_password or "", user.password_hash - ): - raise refused(403, "wrong_password") - # Delete the now-dead ciphertext first (both directions), then the wrapped keypair itself so a - # fresh setup is allowed. Order doesn't matter (no FK between them), but do it in one commit. - db.execute( - delete(Message).where( - Message.kind == "user", - or_(Message.sender_id == user.id, Message.recipient_id == user.id), - ) - ) - key = db.get(MessageKey, user.id) - if key is not None: - db.delete(key) - db.commit() - return {"ok": True, "configured": False} - - -@router.get("/keys/{user_id}") -def public_key( - user_id: int, user: User = Depends(require_human), db: Session = Depends(get_db) -) -> dict: - """A member's public key, for deriving the pairwise conversation key. MB2 (like get_thread): - resolve the target only if they're messageable OR we already share a thread, else the same - "User not found" as a non-existent id — so this can't be probed to enumerate user_ids or to - fetch keys for suspended/deactivated/demo accounts.""" - cond = and_( - Message.kind == "user", - or_( - and_(Message.sender_id == user.id, Message.recipient_id == user_id), - and_(Message.sender_id == user_id, Message.recipient_id == user.id), - ), - ) - p = db.get(User, user_id) - has_history = db.scalar(select(Message.id).where(cond).limit(1)) - if p is None or (not is_messageable_user(p) and not has_history): - raise not_found("user") - k = db.get(MessageKey, user_id) - if k is None: - raise refused(404, "no_messaging") - return {"user_id": user_id, "public_key": k.public_key} - - -# --- directory + conversations ----------------------------------------------- - - -@router.get("/users") -def list_recipients( - user: User = Depends(require_human), db: Session = Depends(get_db) -) -> list[dict]: - """Member directory for the recipient picker: messageable members (except me) who have set - up secure messaging (so a conversation key can be derived).""" - rows = ( - db.execute( - select(User) - .join(MessageKey, MessageKey.user_id == User.id) - .where(User.id != user.id, *messageable_clauses()) - .order_by(collated(func.coalesce(User.display_name, User.email), lang_for_user(user))) - ) - .scalars() - .all() - ) - return [_serialize_user(u) for u in rows] - - -@router.get("/conversations") -def list_conversations(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict: - """One entry per conversation: each user partner (E2EE) plus the system "Siftlode" thread, - with the latest message + my unread count, newest first. Lazily creates the welcome on the - first open. User-message previews are ciphertext — the client decrypts them.""" - ensure_welcome(db, user) - # The conversation partner for a message, from this user's perspective (system → 0). - partner_expr = case( - (Message.kind == "system", SYSTEM_PARTNER_ID), - (Message.sender_id == user.id, Message.recipient_id), - else_=Message.sender_id, - ) - mine = or_(Message.sender_id == user.id, Message.recipient_id == user.id) - # Latest message per conversation in ONE query (DISTINCT ON, Postgres) instead of loading the - # user's whole message history into Python — bounded by the number of conversations. - latest = ( - db.execute( - select(Message) - .where(mine) - .distinct(partner_expr) - .order_by(partner_expr, Message.created_at.desc()) - ) - .scalars() - .all() - ) - # Unread incoming, grouped per conversation. - unread = dict( - db.execute( - select(partner_expr, func.count()) - .where(mine, Message.recipient_id == user.id, Message.read_at.is_(None)) - .group_by(partner_expr) - ).all() - ) - - def _pid(m: Message) -> int: - if m.kind == "system": - return SYSTEM_PARTNER_ID - return m.recipient_id if m.sender_id == user.id else m.sender_id - - user_ids = [_pid(m) for m in latest if m.kind != "system"] - partners = {} - if user_ids: - partners = { - u.id: u for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all() - } - items = [] - for m in latest: - pid = _pid(m) - if pid == SYSTEM_PARTNER_ID: - partner = _system_partner() - elif pid in partners: - partner = _serialize_user(partners[pid]) - else: - continue - items.append( - { - "partner": partner, - "last_message": _serialize_msg(m, lang_for_user(user)), - "unread": int(unread.get(pid, 0)), - } - ) - items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True) - return {"items": items} - - -@router.get("/conversations/{partner_id}") -def get_thread( - partner_id: int, - mark_read: bool = True, - user: User = Depends(require_human), - db: Session = Depends(get_db), -) -> dict: - """Full thread with a partner (or the system thread when partner_id == 0), oldest first. - Stamps unread incoming messages read unless mark_read=false.""" - if partner_id == SYSTEM_PARTNER_ID: - ensure_welcome(db, user) - partner = _system_partner() - cond = and_(Message.recipient_id == user.id, Message.kind == "system") - else: - p = db.get(User, partner_id) - cond = and_( - Message.kind == "user", - or_( - and_(Message.sender_id == user.id, Message.recipient_id == partner_id), - and_(Message.sender_id == partner_id, Message.recipient_id == user.id), - ), - ) - # MB2: don't leak arbitrary user identities. Resolve a partner only if they're messageable - # OR we already share a thread (so a conversation with a since-suspended/deactivated user - # still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle. - has_history = db.scalar(select(Message.id).where(cond).limit(1)) - if p is None or (not is_messageable_user(p) and not has_history): - raise not_found("user") - partner = _serialize_user(p) - msgs = ( - db.execute(select(Message).where(cond).order_by(Message.created_at.asc())).scalars().all() - ) - if mark_read: - now = datetime.now(timezone.utc) - changed = False - for m in msgs: - if m.recipient_id == user.id and m.read_at is None: - m.read_at = now - changed = True - if changed: - db.commit() - # MB3: ping this user's OTHER tabs/devices that their unread dropped so the badge - # refreshes live instead of waiting for the next 20s poll. The client re-fetches the - # count on this signal, so no number is carried here. - manager.push(user.id, {"type": "unread"}) - return {"partner": partner, "items": [_serialize_msg(m, lang_for_user(user)) for m in msgs]} - - -@router.post("") -def send_message( - payload: SendIn, - user: User = Depends(require_human), - db: Session = Depends(get_db), -) -> dict: - """Send an end-to-end-encrypted message. The server stores only the ciphertext the client - produced, then pushes it live to the recipient (and the sender's other devices).""" - if not _send_limiter.allow(f"msg:{user.id}"): - raise refused(429, "message_rate") - ct = (payload.ciphertext or "").strip() - iv = (payload.iv or "").strip() - if not ct or not iv: - raise refused(400, "message_empty") - if len(ct) > MAX_CIPHERTEXT: - raise refused(400, "message_too_long") - if payload.recipient_id == user.id: - raise refused(400, "message_self") - recipient = db.get(User, payload.recipient_id) - if not is_messageable_user(recipient): - raise not_found("recipient") - if db.get(MessageKey, recipient.id) is None: - raise refused(404, "no_messaging") - m = Message(kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv) - db.add(m) - db.commit() - db.refresh(m) - # Carry both parties so each recipient can open/flash the right dock window (the partner is - # whichever of from/to isn't them). - event = { - "type": "message", - "message": _serialize_msg(m), - "from": _serialize_user(user), - "to": _serialize_user(recipient), - } - manager.push(recipient.id, event) - manager.push(user.id, event) # the sender's other open devices - return _serialize_msg(m) - - -@router.get("/unread_count") -def unread_count(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict: - n = db.scalar( - select(func.count()) - .select_from(Message) - .where(Message.recipient_id == user.id, Message.read_at.is_(None)) - ) - return {"count": n or 0} - - -# --- live delivery ----------------------------------------------------------- - - -@router.websocket("/ws") -async def messages_ws(ws: WebSocket) -> None: - """Live push channel: authenticated by the session cookie, registers the connection so - sent messages reach this user's open tabs instantly. We don't consume client→server frames - (sending goes through POST /api/messages); the receive loop only detects disconnect.""" - # Which signed-in account this socket acts as — the SHARED per-tab resolution (a wallet-gated - # ?account=, the session default when none is named, nobody for an account no longer signed in - # here). A browser WebSocket can't send the X-Siftlode-Account header, so resolved_user_id falls - # through to the ?account= query param. - uid, _ = resolved_user_id(ws) - if not uid: - await ws.close(code=1008) - return - db = SessionLocal() - try: - user = db.get(User, uid) - # SA4: honour server-side session revocation on the live channel too — reject a cookie whose - # recorded epoch is behind the account's current one (mirrors current_user). Without this a - # copied cookie could keep receiving pushes after a password reset / "log out everywhere". - epochs = ws.session.get("epochs") or {} - epoch_ok = user is not None and int(epochs.get(str(uid), 0)) == (user.session_epoch or 0) - ok = epoch_ok and is_messageable_user(user) - finally: - db.close() - if not ok: - await ws.close(code=1008) - return - await manager.connect(uid, ws) - try: - while True: - await ws.receive_text() - # silent-ok: the ordinary way a chat socket ends — a closed tab. The `finally` below is the - # whole handling, and nothing failed. - except WebSocketDisconnect: - pass - finally: - await manager.disconnect(uid, ws) diff --git a/backend/app/worker.py b/backend/app/worker.py index 4536cb04..20c48d76 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -11,7 +11,7 @@ Design: instantly (no re-download); a fresh asset is claimed pending→downloading by exactly one worker (a DB compare-and-set), so two jobs for the same video+format never double-download. * Live progress is written to the job row (short-lived sessions, throttled ~1/s) — the - frontend polls it; the worker can't reach the API's in-process WebSocket registry. + frontend polls it (the worker is a separate process with no channel to the browser). * Pause/cancel are cooperative: the progress hook re-reads the job status and aborts the yt-dlp download if it's no longer `running`. * Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset. diff --git a/backend/tests/fixtures/log_redaction_cases.json b/backend/tests/fixtures/log_redaction_cases.json index c89a7cc2..12ac0a0f 100644 --- a/backend/tests/fixtures/log_redaction_cases.json +++ b/backend/tests/fixtures/log_redaction_cases.json @@ -31,7 +31,7 @@ } }, { - "name": "E2EE material and cookies", + "name": "ciphertext, passphrases and cookies", "key": "", "value": { "ciphertext": "AAAA", diff --git a/backend/tests/test_collation.py b/backend/tests/test_collation.py index 2558be23..1c4f3f97 100644 --- a/backend/tests/test_collation.py +++ b/backend/tests/test_collation.py @@ -221,30 +221,13 @@ def _others(db, n_offset: int = 0) -> list[User]: return users -def test_the_share_and_message_recipients_are_in_the_readers_alphabet( - db, monkeypatch, override_dep -): - from app.models import MessageKey - +def test_the_share_recipients_are_in_the_readers_alphabet(db, monkeypatch, override_dep): user = _user(db, "hu") - others = _others(db) - db.add_all( - MessageKey( - user_id=u.id, - public_key="k", - wrapped_private_key="w", - salt="s", - wrap_iv="i", - key_check="c", - ) - for u in others - ) + _others(db) db.commit() client = _client(db, user, monkeypatch, override_dep) emails = [r["email"] for r in client.get("/api/downloads/recipients").json()] assert emails == [f"{w.lower()}@example.com" for w in HU] - names = [r["name"] for r in client.get("/api/messages/users").json()] - assert names == HU def test_the_blocked_channels_are_in_the_readers_alphabet(db, monkeypatch, override_dep): diff --git a/backend/tests/test_logs.py b/backend/tests/test_logs.py index 0d77f296..587a1919 100644 --- a/backend/tests/test_logs.py +++ b/backend/tests/test_logs.py @@ -219,11 +219,11 @@ def _client() -> TestClient: raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) return {"ok": True} - @app.get("/api/messages/unread_count") + @app.get("/api/me/notifications/unread_count") def _unread(): return {"count": 0} - @app.post("/api/messages/unread_count") + @app.post("/api/me/notifications/unread_count") def _unread_write(): return {"ok": True} @@ -337,11 +337,11 @@ def test_a_polled_read_is_debug_a_write_to_the_same_route_is_not(sink): http = logging.getLogger("siftlode.http") http.setLevel(logs.TRACE) client = _client() - r = client.get("/api/messages/unread_count") + r = client.get("/api/me/notifications/unread_count") assert sink.objects()[-1]["level"] == "DEBUG" # ...and says so, for the browser's copy of the line (frontend lib/http.ts). assert r.headers["X-Log-Quiet"] == "1" - r = client.post("/api/messages/unread_count") + r = client.post("/api/me/notifications/unread_count") assert sink.objects()[-1]["level"] == "INFO" assert "X-Log-Quiet" not in r.headers assert client.get("/api/videos/abc").status_code == 200 diff --git a/backend/tests/test_messages_welcome.py b/backend/tests/test_messages_welcome.py deleted file mode 100644 index 8d279450..00000000 --- a/backend/tests/test_messages_welcome.py +++ /dev/null @@ -1,53 +0,0 @@ -"""E31 S1: the welcome message is rendered once, when the account is created, in the language of that day. -An account that switched language afterwards kept reading it in the old one (the owner's: English under a -Hungarian UI). It is served in the reader's language now; any other system message stays as stored. -Needs the DB.""" - -from fastapi.testclient import TestClient - -import app.state as state -from app.auth import current_user -from app.db import get_db -from app.main import app -from app.models import Message, User -from app.routes.messages import WELCOME - - -def _client(db, user, monkeypatch, override_dep) -> TestClient: - monkeypatch.setattr(state, "_configured_cache", True) - override_dep(get_db, db.info["request"]) - override_dep(current_user, lambda: user) - return TestClient(app) - - -def _user_with_english_welcome(db) -> User: - """An account whose welcome was written in English, then switched to Hungarian.""" - user = User(email="welcome-hu@example.com", preferences={"language": "hu"}) - db.add(user) - db.flush() - db.add(Message(kind="system", sender_id=None, recipient_id=user.id, body=WELCOME["en"])) - db.commit() - return user - - -def test_the_welcome_is_read_in_the_readers_language(db, monkeypatch, override_dep): - user = _user_with_english_welcome(db) - client = _client(db, user, monkeypatch, override_dep) - - thread = client.get("/api/messages/conversations/0").json()["items"] - assert [m["body"] for m in thread] == [WELCOME["hu"]] - - convs = client.get("/api/messages/conversations").json()["items"] - assert [c["last_message"]["body"] for c in convs if c["partner"]["id"] == 0] == [WELCOME["hu"]] - - -def test_another_system_message_is_served_as_stored(db, monkeypatch, override_dep): - user = _user_with_english_welcome(db) - db.add( - Message(kind="system", sender_id=None, recipient_id=user.id, body="Maintenance tonight.") - ) - db.commit() - client = _client(db, user, monkeypatch, override_dep) - - bodies = [m["body"] for m in client.get("/api/messages/conversations/0").json()["items"]] - assert bodies == [WELCOME["hu"], "Maintenance tonight."] diff --git a/backend/tests/test_route_conventions.py b/backend/tests/test_route_conventions.py index c3850fe0..513515ca 100644 --- a/backend/tests/test_route_conventions.py +++ b/backend/tests/test_route_conventions.py @@ -23,11 +23,9 @@ COMMITTING_HELPERS = { "search.py:_ingest_candidates": "stages around YouTube calls: the stubs must survive an enrich " "that fails", "search.py:_classify_shorts": "the last stage of that ingest, after its probes", - "auth.py:upsert_pending_invite": "commits only when it writes — the contract its three " + "auth.py:upsert_pending_invite": "commits only when it writes — the contract its two " "callers rely on (`_register_account` persists its own flushed user because a no-op here " "commits nothing)", - "messages.py:ensure_welcome": "the websocket push after it must announce a row the client " - "can already fetch", "debug_sessions.py:_end_send": "gives back the claim a send took, in a `finally`: it runs when " "the handler is already unwinding and has no commit left to reach", } diff --git a/docs/gates.md b/docs/gates.md index f1ebf3b8..096f327a 100644 --- a/docs/gates.md +++ b/docs/gates.md @@ -55,7 +55,6 @@ them itself (`no-undef`, `no-redeclare`, `no-dupe-keys`, …), nor the formattin | e2e | `@typescript-eslint/no-floating-promises` — narrowed | The same `allowForKnownSafeCalls` list (the typed rules are one object for both scopes). | 6 names | | lib/http.ts | `no-restricted-globals` — narrowed | A network owner (`NETWORK_OWNERS` in eslint.config.js): the one module that may call `fetch` (`send`, which `req` and the public pages go through). Every other network global stays restricted there. | 1 global | | lib/api/core.ts | `no-restricted-globals` — narrowed | A network owner: `uploadReq` is the one `XMLHttpRequest` (upload progress). | 1 global | -| lib/messagesSocket.ts | `no-restricted-globals` — narrowed | A network owner: the chat socket is the one `WebSocket`. | 1 global | | src tests · ui tests · lib tests | `no-restricted-globals`, `no-restricted-properties` — off | A test stands a double in for the network (`vi.stubGlobal("fetch", …)`) and may reach it directly. | — | | src tests · ui tests · lib tests | `local/literal-a11y-text`, `local/button-primitive`, `local/input-primitive`, `local/surface-role`, `local/glyph-link` — off | A test renders throwaway markup (a raw button as a focus target, an untranslated label) that is not product UI. | — | | FocusSafeButton.tsx + AnchoredPopover.tsx | `local/focus-safe-disabled` — off | FocusSafeButton: it IS the focus-safe primitive the rule points everyone else to. | — | diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 77659e09..1493e88f 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -25,13 +25,13 @@ import playwright from "eslint-plugin-playwright"; // a `-- ` description (eslint-comments/require-description makes that mandatory). // E22 S4: every way out of the browser goes through the one module that LOGS it — a request line -// with a request id (lib/http.ts), the upload XHR (lib/api/core.ts), the messages socket -// (lib/messagesSocket.ts). A bare `fetch` elsewhere is a request the client log never sees, which is -// exactly how the baseline survey found four. Each owner is exempt from its OWN global only. +// with a request id (lib/http.ts), the upload XHR (lib/api/core.ts). A bare `fetch` elsewhere is a +// request the client log never sees, which is exactly how the baseline survey found four. Each owner +// is exempt from its OWN global only; a global with no owner (null) has no exempt file at all. const NETWORK_OWNERS = { fetch: "src/lib/http.ts", XMLHttpRequest: "src/lib/api/core.ts", - WebSocket: "src/lib/messagesSocket.ts", + WebSocket: null, EventSource: null, }; const networkGlobals = (exempt) => @@ -495,7 +495,7 @@ const suppressionRules = { // The two type-aware rules (E32 D3: a curated pair, not strictTypeChecked — ~2000 hits there, none // real in the sample). A floating promise is a rejection nobody handles: the audit found two real -// ones (ChatDock's device key store, VideoEditor's play()). `void` marks a deliberate fire-and-forget +// ones (the chat dock's device key store, since removed, and VideoEditor's play()). `void` marks a deliberate fire-and-forget // whose callee reports its own failure. React Query's invalidate/refetch family never rejects // unless asked to (`throwOnError`), so those calls are known-safe by name. const typedRules = { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 2b0db8f3..9a878d99 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -35,7 +35,6 @@ import BackToTop from "./components/ui/BackToTop"; import PageShell from "./components/ui/PageShell"; import { SkipLink } from "./components/ui/SkipLink"; import { pageTitleKey } from "./lib/pageMeta"; -import ChatDock from "./components/ChatDock"; import DebugRecorder from "./components/DebugRecorder"; import Toaster from "./components/ui/Toaster"; import KeyboardHelp from "./components/ui/KeyboardHelp"; @@ -459,7 +458,6 @@ export default function App() { "take me to X" the same way. Their keys are bound by useGlobalKeymap above. */} - {me && !me.is_demo && } {/* The debug recorder (E16): its start dialog, and its pill docked under the toasts. */} {me && !me.is_demo && } diff --git a/frontend/src/components/ChatDock.tsx b/frontend/src/components/ChatDock.tsx deleted file mode 100644 index db984d3d..00000000 --- a/frontend/src/components/ChatDock.tsx +++ /dev/null @@ -1,131 +0,0 @@ -import { useEffect } from "react"; -import { qk } from "../lib/queryKeys"; -import { useTranslation } from "react-i18next"; -import { useQueryClient } from "@tanstack/react-query"; -import { ChevronDown, ChevronUp, X } from "lucide-react"; -import { - closeChat, - initDock, - notifyIncoming, - toggleMinimize, - useDockChats, - type DockChat, -} from "../lib/chatDock"; -import { onMessage, onUnread } from "../lib/messagesSocket"; -import * as e2ee from "../lib/e2ee"; -import { errorAttrs, log } from "../lib/log"; -import Avatar from "./ui/Avatar"; -import ChatThread from "./ChatThread"; -import Overlay from "./ui/Overlay"; -import { IconButton } from "./ui/IconButton"; -import { Pressable } from "./ui/Pressable"; -import { SURFACE } from "./ui/surfaces"; - -// The Messenger-style floating chat dock: open conversations live here, bottom-right, across -// page navigation (state persisted per user). Always mounted (for human users) so it also owns -// the app-wide live-message subscription and restores this device's unlock state on load. -export default function ChatDock({ meId }: { meId: number }) { - const qc = useQueryClient(); - const chats = useDockChats(); - - // Restore the per-device unlocked key and the persisted dock windows once, app-wide. - useEffect(() => { - initDock(meId); - // The device key store can be unopenable (storage disabled, some private windows): messages then - // stay locked behind the passphrase, as on a new device — said in the log, not thrown at nobody. - e2ee - .loadFromDevice(meId) - .catch((err: unknown) => - log.warn( - "ui.e2ee_device_key_unreadable", - "the device key store could not be read", - errorAttrs(err), - ), - ); - }, [meId]); - - // Live delivery: refresh the conversation list, unread badge, and the affected thread; and - // for an INCOMING message, open or flash that partner's dock window. - useEffect( - () => - onMessage(({ message: m, from }) => { - qc.invalidateQueries({ queryKey: qk.conversations() }); - qc.invalidateQueries({ queryKey: qk.messageUnread() }); - const partnerId = m.sender_id === meId ? m.recipient_id : m.sender_id; - qc.invalidateQueries({ queryKey: qk.thread(partnerId) }); - // Auto-open/flash only for messages from someone else (not my own echo) and only for - // real user conversations. - if (m.kind === "user" && m.sender_id !== meId && from) { - notifyIncoming({ id: from.id, name: from.name, avatar_url: from.avatar_url }); - } - }), - [qc, meId], - ); - - // Live badge sync: another tab/device read a thread → refresh this tab's unread count + list. - useEffect( - () => - onUnread(() => { - qc.invalidateQueries({ queryKey: qk.messageUnread() }); - qc.invalidateQueries({ queryKey: qk.conversations() }); - }), - [qc], - ); - - if (chats.length === 0) return null; - return ( - -
- {chats.map((c) => ( - - ))} -
-
- ); -} - -function ChatWindow({ chat, meId }: { chat: DockChat; meId: number }) { - const { t } = useTranslation(); - return ( -
- {/* One-shot attention flash on a new message; keyed by the counter so it replays each time. */} - {chat.flash > 0 && ( -
- )} -
- toggleMinimize(chat.partnerId)} - className="flex items-center gap-2 min-w-0 flex-1" - title={chat.minimized ? t("messages.expand") : t("messages.minimize")} - > - - {chat.name} - - toggleMinimize(chat.partnerId)} - label={chat.minimized ? t("messages.expand") : t("messages.minimize")} - > - {chat.minimized ? : } - - closeChat(chat.partnerId)} label={t("messages.close")}> - - -
- {!chat.minimized && ( -
- -
- )} -
- ); -} diff --git a/frontend/src/components/ChatThread.tsx b/frontend/src/components/ChatThread.tsx deleted file mode 100644 index 80b84671..00000000 --- a/frontend/src/components/ChatThread.tsx +++ /dev/null @@ -1,243 +0,0 @@ -import { useEffect, useEffectEvent, useRef, useState } from "react"; -import { qk } from "../lib/queryKeys"; -import { useTranslation } from "react-i18next"; -import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { Send } from "lucide-react"; -import { api, type Message, type MessageUser } from "../lib/api"; -import { useLiveQuery } from "../lib/useLiveQuery"; -import { useScrollFade } from "../lib/useScrollFade"; -import { IconButton } from "./ui/IconButton"; -import { LoadingState, StateMessage } from "./ui/QueryState"; -import { notifyActionError } from "../lib/actionErrors"; -import { partnerPub, POLL_MS, refreshPartnerPub, useKeyState } from "../lib/messaging"; -import * as e2ee from "../lib/e2ee"; -import KeyGate from "./KeyGate"; -import { Textarea } from "./ui/Input"; -import { RelativeTime } from "./ui/RelativeTime"; - -// The message list + composer for one conversation, shared by the full Messages page and the -// floating dock window. Handles client-side decryption and encrypt-on-send; the surrounding -// chrome (back button / minimize / close) is the caller's. Self-contained key gating: prompts -// setup/unlock (server-truth) for a user conversation until messaging is ready. -export default function ChatThread({ - meId, - partnerId, - isSystem, -}: { - meId: number; - partnerId: number; - isSystem: boolean; -}) { - const { t } = useTranslation(); - const qc = useQueryClient(); - const [draft, setDraft] = useState(""); - const [plain, setPlain] = useState>({}); - const bottomRef = useRef(null); - const { ref: fadeRef, style: fadeStyle } = useScrollFade(); - const ready = useKeyState() === "ready"; - const needsKey = !isSystem && !ready; - - const q = useLiveQuery<{ partner: MessageUser; items: Message[] }>( - qk.thread(partnerId), - () => api.thread(partnerId), - { intervalMs: POLL_MS, enabled: !needsKey }, - ); - const items = q.data?.items ?? []; - // The two effects below take the FETCH as their trigger and read the payload through this effect - // event, so their dep lists name everything they touch without making the identity of `q.data` a - // trigger. - const latestThread = useEffectEvent(() => q.data); - - // Decrypt the thread when fresh server data arrives. Keyed on dataUpdatedAt (not `items`): the - // `?? []` fallback is a new array reference every render, which would otherwise re-run this - // effect (and setPlain a new object) on every render while data is loading — a render storm. - useEffect(() => { - if (isSystem || !ready) return; - let cancelled = false; - const thread = latestThread(); - // Every await inside is caught per message (a failure reads "can't decrypt"), so this never rejects. - void (async () => { - const out: Record = {}; - // Fetch the partner key lazily inside the try (only when there's a ciphertext message) and - // once — so an empty thread never fetches, and a fetch failure (e.g. a partner who reset and - // hasn't re-set-up, so has no key) degrades to "can't decrypt" instead of rejecting the pass. - let pub: string | null = null; - let refreshed = false; - for (const m of thread?.items ?? []) { - if (!m.ciphertext || !m.iv) continue; - try { - if (!pub) pub = await partnerPub(partnerId); - out[m.id] = await e2ee.decryptFrom(partnerId, pub, m.ciphertext, m.iv); - } catch { - // A decrypt failure can mean the partner rotated their keypair (a passphrase reset), so - // this device's cached public key is stale. Refresh it ONCE per pass and retry — that - // also re-derives future sends under the current key. A genuinely bad message still ends - // up "can't decrypt". - if (!refreshed) { - refreshed = true; - try { - pub = await refreshPartnerPub(partnerId); - out[m.id] = await e2ee.decryptFrom(partnerId, pub, m.ciphertext, m.iv); - continue; - } catch { - /* fall through to the error label below */ - } - } - out[m.id] = "⚠ " + t("messages.cantDecrypt"); - } - } - if (!cancelled) setPlain(out); - })(); - return () => { - cancelled = true; - }; - // `dataUpdatedAt` is the trigger and the payload is read through the effect event above — see - // Messages, where keying this family on `q.data` itself produced a render loop. - }, [q.dataUpdatedAt, ready, isSystem, partnerId, t]); - - // Opening the thread (or a new incoming message arriving) marks messages read server-side, so - // refresh the badges — but ONLY when the incoming-message count actually changed, not on every - // 20s poll (which would needlessly refetch /conversations + /unread_count each tick). - const lastIncoming = useRef(-1); - const lastPartner = useRef(partnerId); - useEffect(() => { - if (!q.dataUpdatedAt) return; - // The Messages page reuses one ChatThread instance across conversation switches (not keyed by - // partnerId), so reset the counter when the partner changes — otherwise switching to a thread - // with the same incoming-count would skip the open-marks-read refresh. - if (lastPartner.current !== partnerId) { - lastPartner.current = partnerId; - lastIncoming.current = -1; - } - const incoming = (latestThread()?.items ?? []).filter((m) => m.recipient_id === meId).length; - if (incoming !== lastIncoming.current) { - lastIncoming.current = incoming; - qc.invalidateQueries({ queryKey: qk.conversations() }); - qc.invalidateQueries({ queryKey: qk.messageUnread() }); - } - }, [q.dataUpdatedAt, partnerId, meId, qc]); - - // Scroll to the newest message when the count changes (open / new message) — NOT on `plain`, - // which gets a fresh object on every poll's decrypt pass and would yank a scrolled-up reader - // back to the bottom even when nothing new arrived. - useEffect(() => { - bottomRef.current?.scrollIntoView({ block: "end" }); - }, [items.length]); - - const send = useMutation({ - mutationFn: async (text: string) => { - const pub = await partnerPub(partnerId); - const { ciphertext, iv } = await e2ee.encryptFor(partnerId, pub, text); - return api.sendMessage(partnerId, ciphertext, iv); - }, - onSuccess: () => { - setDraft(""); - qc.invalidateQueries({ queryKey: qk.thread(partnerId) }); - qc.invalidateQueries({ queryKey: qk.conversations() }); - }, - onError: (e) => { - // The draft is kept (only cleared on success) so the user can retry. 400/409/422/500 already - // raised the global error dialog; 404 (recipient gone) and 429 (rate-limited) are "caller- - // handled" in api.req (no modal), so surface those here or the send fails silently. - // - // Through the shared helper, which knows both of those rules: it no-ops on anything already - // reported, and it prefers the server's reason only when that reason is a STRUCTURED detail. - // The hand-written copy that used to live here did `e.detail || …`, so a Hungarian reader got - // the route's English sentence — "That member hasn't set up messaging yet." — which is - // exactly what round 5 changed `notifyActionError` to stop doing. A second copy of a contract - // is how the fix misses a caller (round 6). - notifyActionError(e, t("messages.sendFailed")); - }, - }); - - const submit = () => { - const text = draft.trim(); - if (text && !send.isPending) send.mutate(text); - }; - - if (needsKey) { - return ( -
-
- -
-
- ); - } - - return ( - <> -
- {q.isError && !q.data ? ( -
- q.refetch()}> - {t("common.loadError")} - -
- ) : q.isPending ? ( -
- -
- ) : items.length === 0 ? ( -
{t("messages.threadEmpty")}
- ) : ( - items.map((m) => { - const mine = m.sender_id === meId; - const text = m.kind === "system" ? (m.body ?? "") : (plain[m.id] ?? "…"); - return ( -
-
- {text} - -
-
- ); - }) - )} -
-
- - {isSystem ? ( -
- {t("messages.systemReadOnly")} -
- ) : ( -
-