fix(ops): make backup/restore work, and give upgrades a pin and a rollback

DX-13: both shell scripts ran bare 'docker compose', which has no config file to find in the layout docs/self-hosting.md tells people to use (only docker-compose.selfhost.yml is downloaded) — they now fall back to it, while a source checkout keeps resolving docker-compose.yml as before. The two PowerShell scripts moved the dump through the PS pipeline, which corrupts binary on 5.1 and cannot express -Encoding Byte at all on 7 ('Byte' is not a supported encoding name), so they simply failed there; they now write inside the container and 'docker compose cp' it across. backup.sh also wrote its output straight into backups/, leaving a truncated file that looks like a real backup when pg_dump fails — it writes a .part and promotes on success. Both restores report a non-zero pg_restore rather than printing 'Restored' over it. Docs now state that a backup is the dump PLUS .env: without TOKEN_ENCRYPTION_KEY every encrypted secret in the restored database is unreadable, which the old one-line 'back up the volume' never said. DX-14: 'latest' moves with every release and migrations run on every boot, so self-hosters had no way to choose when they move and no way back. Docs gain IMAGE_TAG pinning, a rollback recipe (image-only vs schema-bearing) and the migration-failed-at-boot case; CHANGELOG.md is generated from releaseNotes.ts so the file cannot drift from what the app shows, and 'siftlode release' refuses on a stale one rather than regenerating it into the clean tree it just demanded. Verified: dump + restore round-trip on both platforms against localdev into a scratch database (8 users, 262822 videos, schema 0082_invite_lang), and IMAGE_TAG=0.80.0 resolves on both services in 'compose config'.
2026-09-05 00:50:07 +02:00
parent e23128b4a1
commit ca4e559cc7
8 changed files with 2061 additions and 17 deletions
@@ -1,10 +1,43 @@
-# Dumps the Siftlode Postgres database to backupssiftlode-<timestamp>.dump (custom format).
+# Dumps the Siftlode Postgres database to backups\siftlode-<timestamp>.dump (custom format).
 # Run from the project root:  .\scripts\backup.ps1
+#
+# A dump is only HALF a backup. The other half is your .env — it holds TOKEN_ENCRYPTION_KEY, without
+# which every encrypted value in the restored database (Google credentials, SMTP password, Plex
+# token, stored refresh tokens) is unreadable. See docs/self-hosting.md § Backups.
 $ErrorActionPreference = "Stop"
+
+# The dump is moved with `docker compose cp`, never through the PowerShell pipeline. Piping a binary
+# stream corrupts it on Windows PowerShell 5.1 (text decoding), and on PowerShell 7 the old
+# `-Encoding Byte` workaround does not even exist any more — it throws "'Byte' is not a supported
+# encoding name", so this script simply failed there.
+$remote = "/tmp/siftlode-backup.dump"
+
+# `docker compose` finds docker-compose.yml on its own and honours $env:COMPOSE_FILE. The documented
+# self-host layout (docs/self-hosting.md § Get the files) has NEITHER — just the selfhost file.
+if (-not $env:COMPOSE_FILE -and -not (Test-Path "docker-compose.yml") -and (Test-Path "docker-compose.selfhost.yml")) {
+    $env:COMPOSE_FILE = "docker-compose.selfhost.yml"
+}
+
 $user = if ($env:POSTGRES_USER) { $env:POSTGRES_USER } else { "siftlode" }
 $dbname = if ($env:POSTGRES_DB) { $env:POSTGRES_DB } else { "siftlode" }
 New-Item -ItemType Directory -Force -Path "backups" | Out-Null
 $ts = Get-Date -Format "yyyyMMdd-HHmmss"
 $out = "backups\siftlode-$ts.dump"
-docker compose exec -T db pg_dump -U $user -Fc $dbname | Set-Content -NoNewline -Encoding Byte $out
+
+docker compose exec -T db pg_dump -U $user -Fc $dbname -f $remote
+if ($LASTEXITCODE -ne 0) {
+    docker compose exec -T db rm -f $remote 2>$null
+    throw "BACKUP FAILED — pg_dump returned $LASTEXITCODE; no file was written."
+}
+
+docker compose cp "db:$remote" $out
+$cpCode = $LASTEXITCODE
+docker compose exec -T db rm -f $remote 2>$null
+if ($cpCode -ne 0) { throw "BACKUP FAILED — could not copy the dump out of the container (exit $cpCode)." }
+if (-not (Test-Path $out) -or (Get-Item $out).Length -eq 0) {
+    if (Test-Path $out) { Remove-Item $out }
+    throw "BACKUP FAILED — the dump came out empty; no backup was written."
+}
+
 Write-Output "Wrote $out"
+Write-Output "Remember: back up your .env too (TOKEN_ENCRYPTION_KEY) — this dump alone cannot be fully restored."
@@ -1,11 +1,39 @@
 #!/bin/sh
 # Dumps the Siftlode Postgres database to backups/siftlode-<timestamp>.dump (custom format).
 # Run from the project root:  ./scripts/backup.sh
+#
+# A dump is only HALF a backup. The other half is your `.env` — it holds TOKEN_ENCRYPTION_KEY,
+# without which every encrypted value in the restored database (Google credentials, SMTP password,
+# Plex token, stored refresh tokens) is unreadable. See docs/self-hosting.md § Backups.
 set -e
+
+# `docker compose` finds docker-compose.yml on its own and honours $COMPOSE_FILE. The documented
+# self-host layout (docs/self-hosting.md § Get the files) has NEITHER — just the selfhost file — so
+# without this the command below failed there with "no configuration file provided".
+if [ -z "${COMPOSE_FILE:-}" ] && [ ! -f docker-compose.yml ] && [ -f docker-compose.selfhost.yml ]; then
+  COMPOSE_FILE=docker-compose.selfhost.yml
+  export COMPOSE_FILE
+fi
+
 USER_NAME="${POSTGRES_USER:-siftlode}"
 DB_NAME="${POSTGRES_DB:-siftlode}"
 mkdir -p backups
 TS=$(date +%Y%m%d-%H%M%S)
 OUT="backups/siftlode-$TS.dump"
-docker compose exec -T db pg_dump -U "$USER_NAME" -Fc "$DB_NAME" > "$OUT"
+
+# Write to a temp name and only promote it on success: a failed pg_dump would otherwise leave a
+# truncated (often 0-byte) file sitting in backups/ that looks exactly like a real backup.
+TMP="$OUT.part"
+if ! docker compose exec -T db pg_dump -U "$USER_NAME" -Fc "$DB_NAME" > "$TMP"; then
+  rm -f "$TMP"
+  echo "BACKUP FAILED — pg_dump returned an error; no file was written." >&2
+  exit 1
+fi
+if [ ! -s "$TMP" ]; then
+  rm -f "$TMP"
+  echo "BACKUP FAILED — pg_dump produced an empty file; no backup was written." >&2
+  exit 1
+fi
+mv "$TMP" "$OUT"
 echo "Wrote $OUT"
+echo "Remember: back up your .env too (TOKEN_ENCRYPTION_KEY) — this dump alone cannot be fully restored."
@@ -0,0 +1,86 @@
+// Generates CHANGELOG.md from frontend/src/lib/releaseNotes.ts — the same entries the app shows
+// under Release Notes, so a self-hoster deciding whether to upgrade reads the same text an
+// in-app user does, without having to install the app first (DX-14).
+//
+// Why derive rather than hand-write: two lists of "what changed in 0.80.0" WILL drift, and the
+// in-app one is the one that gets written during a release. This makes the file a build product of
+// it. RELEASE_NOTES is a plain data literal by design, so it is imported by stripping the single TS
+// type annotation off the declaration — no transpiler, no dependency.
+//
+//   node scripts/gen-changelog.mjs           # write CHANGELOG.md
+//   node scripts/gen-changelog.mjs --check   # exit 1 if CHANGELOG.md is stale (used by `siftlode release`)
+
+import { readFile, writeFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const SOURCE = path.join(root, "frontend", "src", "lib", "releaseNotes.ts");
+const TARGET = path.join(root, "CHANGELOG.md");
+
+async function loadEntries() {
+  const src = await readFile(SOURCE, "utf8");
+  const decl = "export const RELEASE_NOTES";
+  const at = src.indexOf(decl);
+  if (at === -1) {
+    throw new Error(`${SOURCE} no longer declares ${decl} — the generator's assumption broke, fix it rather than hand-writing the changelog.`);
+  }
+  const body = src.slice(at).replace(/^export const RELEASE_NOTES\s*:[^=]*=/, "export const RELEASE_NOTES =");
+  const mod = await import(`data:text/javascript,${encodeURIComponent(body)}`);
+  if (!Array.isArray(mod.RELEASE_NOTES) || mod.RELEASE_NOTES.length === 0) {
+    throw new Error("RELEASE_NOTES imported as something other than a non-empty array.");
+  }
+  return mod.RELEASE_NOTES;
+}
+
+function section(title, items) {
+  if (!items || items.length === 0) return "";
+  return `### ${title}\n\n${items.map((line) => `- ${line}`).join("\n")}\n\n`;
+}
+
+function render(entries) {
+  const head = [
+    "# Changelog",
+    "",
+    "Every released version of Siftlode, newest first.",
+    "",
+    "**Generated from `frontend/src/lib/releaseNotes.ts` by `scripts/gen-changelog.mjs` — do not edit by hand;**",
+    "edit the release notes and re-run the generator.",
+    "",
+    "Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).",
+    "",
+    "",
+  ].join("\n");
+
+  const body = entries
+    .map((e) => {
+      const sha = e.sha ? ` (\`${e.sha}\`)` : "";
+      let out = `## ${e.version} — ${e.date}${sha}\n\n`;
+      if (e.summary) out += `${e.summary}\n\n`;
+      out += section("Features", e.features);
+      out += section("Fixes", e.fixes);
+      out += section("Internal", e.chores);
+      return out;
+    })
+    .join("");
+
+  return `${head}${body}`.replace(/\n{3,}$/, "\n");
+}
+
+const entries = await loadEntries();
+const rendered = render(entries);
+
+if (process.argv.includes("--check")) {
+  const current = await readFile(TARGET, "utf8").catch(() => null);
+  if (current !== rendered) {
+    console.error(
+      `CHANGELOG.md is out of date with releaseNotes.ts (newest entry: ${entries[0].version}).\n` +
+        "Run: node scripts/gen-changelog.mjs   — then commit the result.",
+    );
+    process.exit(1);
+  }
+  console.log(`CHANGELOG.md is current (${entries.length} releases, newest ${entries[0].version}).`);
+} else {
+  await writeFile(TARGET, rendered, "utf8");
+  console.log(`Wrote CHANGELOG.md — ${entries.length} releases, newest ${entries[0].version}.`);
+}
@@ -1,9 +1,35 @@
 # Restores a Siftlode Postgres dump created by backup.ps1 into the running db container.
-# Usage:  .\scripts\restore.ps1 backupssiftlode-YYYYMMDD-HHMMSS.dump
+# Usage:  .\scripts\restore.ps1 backups\siftlode-YYYYMMDD-HHMMSS.dump
+#
+# Restoring the database does NOT restore the secrets that decrypt it. If this dump is from an
+# instance with a different TOKEN_ENCRYPTION_KEY than the current .env, the encrypted sysconfig
+# values and refresh tokens will not be readable. See docs/self-hosting.md § Backups.
 param([Parameter(Mandatory = $true)][string]$DumpFile)
 $ErrorActionPreference = "Stop"
+
+# See backup.ps1: the dump travels with `docker compose cp`, never through the PowerShell pipeline,
+# which corrupts binary on 5.1 and cannot express `-Encoding Byte` at all on 7.
+$remote = "/tmp/siftlode-restore.dump"
+
+if (-not $env:COMPOSE_FILE -and -not (Test-Path "docker-compose.yml") -and (Test-Path "docker-compose.selfhost.yml")) {
+    $env:COMPOSE_FILE = "docker-compose.selfhost.yml"
+}
+
 $user = if ($env:POSTGRES_USER) { $env:POSTGRES_USER } else { "siftlode" }
 $dbname = if ($env:POSTGRES_DB) { $env:POSTGRES_DB } else { "siftlode" }
 if (-not (Test-Path $DumpFile)) { throw "Dump file not found: $DumpFile" }
-Get-Content -Encoding Byte $DumpFile | docker compose exec -T db pg_restore -U $user -d $dbname --clean --if-exists
+
+docker compose cp $DumpFile "db:$remote"
+if ($LASTEXITCODE -ne 0) { throw "RESTORE FAILED — could not copy the dump into the container (exit $LASTEXITCODE)." }
+
+docker compose exec -T db pg_restore -U $user -d $dbname --clean --if-exists $remote
+$restoreCode = $LASTEXITCODE
+docker compose exec -T db rm -f $remote 2>$null
+
+# pg_restore exits non-zero for ignorable warnings too (--clean --if-exists on a fresh database
+# reports every missing object it tried to drop), so a non-zero code is reported as "check this",
+# not swallowed and not dressed up as success.
+if ($restoreCode -ne 0) {
+    throw "pg_restore reported errors while restoring $DumpFile (exit $restoreCode) — read the output above before treating this instance as restored."
+}
 Write-Output "Restored $DumpFile"
@@ -1,11 +1,31 @@
 #!/bin/sh
 # Restores a Siftlode Postgres dump created by backup.sh into the running db container.
 # Usage:  ./scripts/restore.sh backups/siftlode-YYYYMMDD-HHMMSS.dump
+#
+# Restoring the database does NOT restore the secrets that decrypt it. If this dump is from an
+# instance with a different TOKEN_ENCRYPTION_KEY than the current .env, the encrypted sysconfig
+# values and refresh tokens will not be readable. See docs/self-hosting.md § Backups.
 set -e
+
+# See backup.sh: the documented self-host layout ships only docker-compose.selfhost.yml, so plain
+# `docker compose` has no configuration file to find there.
+if [ -z "${COMPOSE_FILE:-}" ] && [ ! -f docker-compose.yml ] && [ -f docker-compose.selfhost.yml ]; then
+  COMPOSE_FILE=docker-compose.selfhost.yml
+  export COMPOSE_FILE
+fi
+
 FILE="$1"
 if [ -z "$FILE" ]; then echo "Usage: restore.sh <dumpfile>"; exit 1; fi
 if [ ! -f "$FILE" ]; then echo "Dump file not found: $FILE"; exit 1; fi
 USER_NAME="${POSTGRES_USER:-siftlode}"
 DB_NAME="${POSTGRES_DB:-siftlode}"
-docker compose exec -T db pg_restore -U "$USER_NAME" -d "$DB_NAME" --clean --if-exists < "$FILE"
-echo "Restored $FILE"
+
+# pg_restore exits non-zero for ignorable warnings too (--clean --if-exists on a fresh database
+# reports every missing object it tried to drop), so a non-zero code is reported as "check this",
+# not swallowed and not dressed up as success.
+if docker compose exec -T db pg_restore -U "$USER_NAME" -d "$DB_NAME" --clean --if-exists < "$FILE"; then
+  echo "Restored $FILE"
+else
+  echo "pg_restore reported errors while restoring $FILE — read the output above before treating this instance as restored." >&2
+  exit 1
+fi