test: prove 2.8 to 2.9.0 upgrade is lossless and fix what it caught

scripts/build-upgrade-fixture.ts

@@ -0,0 +1,159 @@
+/**
+ * Writes src/backend/tests/fixtures/upgrade/{db.sqlite,fixture.env,files/}: a 2.8 install's
+ * data directory with every feature table filled. The schema is 2.8's own
+ * dump, the rows come from rows.ts, and the columns 2.8 encrypted are
+ * encrypted the way 2.8 did it (a v3 system-wrapped data key per user, then
+ * FieldCrypto per field), so the upgrade test has to really decrypt them.
+ *
+ * Usage: npx tsx scripts/build-upgrade-fixture.ts
+ */
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import Database from "better-sqlite3";
+import bcrypt from "bcryptjs";
+import { FieldCrypto } from "../src/backend/utils/field-crypto.js";
+import {
+  ENCRYPTED_COLUMNS,
+  FIXTURE_PASSWORD,
+  RECORDING_FILES,
+  ROWS,
+  type Row,
+} from "../src/backend/tests/fixtures/upgrade/rows.js";
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const FIXTURES = path.join(ROOT, "src/backend/tests/fixtures");
+const OUT = path.join(FIXTURES, "upgrade");
+
+const hex = (label: string) =>
+  crypto
+    .createHash("sha256")
+    .update(`termix-d4-fixture:${label}`)
+    .digest("hex");
+
+const ENV = {
+  JWT_SECRET: hex("jwt") + hex("jwt-2"),
+  DATABASE_KEY: hex("database"),
+  ENCRYPTION_KEY: hex("encryption"),
+  INTERNAL_AUTH_TOKEN: hex("internal"),
+};
+const masterKey = Buffer.from(ENV.ENCRYPTION_KEY, "hex");
+
+/** UserKeyManager's v3 wrap, unchanged since 2.8. */
+function wrapDek(userId: string, dek: Buffer): string {
+  const wrapKey = Buffer.from(
+    crypto.hkdfSync(
+      "sha256",
+      masterKey,
+      Buffer.alloc(0),
+      `termix:dek-wrap:v3:${userId}`,
+      32,
+    ),
+  );
+  const iv = crypto.randomBytes(12);
+  const cipher = crypto.createCipheriv("aes-256-gcm", wrapKey, iv);
+  cipher.setAAD(Buffer.from(userId, "utf8"));
+  const ct = Buffer.concat([cipher.update(dek), cipher.final()]);
+  return JSON.stringify({
+    v: 3,
+    alg: "aes-256-gcm",
+    iv: iv.toString("base64"),
+    ct: ct.toString("base64"),
+    tag: cipher.getAuthTag().toString("base64"),
+    createdAt: "2026-05-01T10:00:00.000Z",
+  });
+}
+
+const deks = new Map<string, Buffer>();
+const dekFor = (userId: string) => {
+  let dek = deks.get(userId);
+  if (!dek) {
+    dek = Buffer.from(hex(`dek:${userId}`), "hex");
+    deks.set(userId, dek);
+  }
+  return dek;
+};
+
+function prepareRow(table: string, row: Row): Row {
+  const out: Row = { ...row };
+  if (table === "users" && out.password_hash === undefined) {
+    out.password_hash = bcrypt.hashSync(FIXTURE_PASSWORD, 10);
+  }
+  for (const [encTable, column, field, userColumn] of ENCRYPTED_COLUMNS) {
+    if (encTable !== table || typeof out[column] !== "string" || !out[column])
+      continue;
+    const userId = String(out[userColumn]);
+    const recordId = table === "users" ? userId : String(out.id);
+    out[column] = FieldCrypto.encryptField(
+      out[column] as string,
+      dekFor(userId),
+      recordId,
+      field,
+    );
+  }
+  return out;
+}
+
+function insert(sqlite: Database.Database, table: string, row: Row): void {
+  const keys = Object.keys(row);
+  sqlite
+    .prepare(
+      `INSERT INTO "${table}" (${keys.map((key) => `"${key}"`).join(", ")}) VALUES (${keys.map(() => "?").join(", ")})`,
+    )
+    .run(...keys.map((key) => row[key]));
+}
+
+fs.mkdirSync(OUT, { recursive: true });
+const dbPath = path.join(OUT, "db.sqlite");
+fs.rmSync(dbPath, { force: true });
+
+const sqlite = new Database(dbPath);
+sqlite.pragma("journal_mode = DELETE");
+sqlite.exec(
+  fs.readFileSync(path.join(FIXTURES, "sqlite-2.8-schema.sql"), "utf8"),
+);
+sqlite.pragma("foreign_keys = ON");
+
+sqlite.transaction(() => {
+  for (const [table, rows] of Object.entries(ROWS)) {
+    for (const row of rows) insert(sqlite, table, prepareRow(table, row));
+  }
+  for (const user of ROWS.users) {
+    const userId = String(user.id);
+    insert(sqlite, "settings", {
+      key: `user_dek_v3_${userId}`,
+      value: wrapDek(userId, dekFor(userId)),
+    });
+  }
+})();
+
+const violations = sqlite.pragma("foreign_key_check") as unknown[];
+if (violations.length > 0) {
+  throw new Error(`Foreign key violations: ${JSON.stringify(violations)}`);
+}
+sqlite.exec("VACUUM");
+sqlite.close();
+
+fs.writeFileSync(
+  path.join(OUT, "fixture.env"),
+  Object.entries(ENV)
+    .map(([key, value]) => `${key}=${value}`)
+    .join("\n") + "\n",
+);
+
+// The recording files 2.8 left in DATA_DIR, which the rows point at.
+const files = path.join(OUT, "files");
+fs.rmSync(files, { recursive: true, force: true });
+for (const [kind, relative] of Object.entries(RECORDING_FILES)) {
+  const file = path.join(files, relative);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(
+    file,
+    `d4 ${kind} recording
+`,
+  );
+}
+
+console.log(`Wrote ${path.relative(ROOT, dbPath)} and fixture.env`);

scripts/lib/upgrade-sql.mjs

@@ -0,0 +1,124 @@
+/**
+ * Every SQL statement a 2.8 install's Postgres or MySQL database runs on its
+ * way to 2.9.0, in the order it runs them: core's drizzle migrations added
+ * since release-2.8.0-tag, then each bundled plugin's own migrations in the
+ * order the loader activates them. The data copies that run after that
+ * (src/backend/upgrade/) are TypeScript, not SQL, and are covered by the
+ * upgrade test and scripts/upgrade-check.sh.
+ *
+ * Snapshot-tested by scripts/upgrade-sql.test.ts, so any change to the
+ * upgrade path shows up as a diff in review.
+ *
+ * Usage: node scripts/lib/upgrade-sql.mjs postgres|mysql
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ROOT = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  "../..",
+);
+
+/** The last core migration release-2.8.0-tag shipped, per dialect. */
+export const LAST_28_MIGRATION = {
+  postgres: "0024_flimsy_forge",
+  mysql: "0023_even_beast",
+};
+
+/** Plugin ids in the order the loader activates them: dependencies first. */
+export function pluginActivationOrder(root = ROOT) {
+  const dir = path.join(root, "plugins");
+  const manifests = new Map(
+    fs
+      .readdirSync(dir, { withFileTypes: true })
+      .filter(
+        (entry) =>
+          entry.isDirectory() &&
+          fs.existsSync(path.join(dir, entry.name, "manifest.json")),
+      )
+      .map((entry) => [
+        entry.name,
+        JSON.parse(
+          fs.readFileSync(path.join(dir, entry.name, "manifest.json"), "utf8"),
+        ),
+      ])
+      .sort(([a], [b]) => a.localeCompare(b)),
+  );
+
+  const order = [];
+  const done = new Set();
+  const visit = (id) => {
+    if (done.has(id) || !manifests.has(id)) return;
+    done.add(id);
+    const manifest = manifests.get(id);
+    for (const dependency of Object.keys(manifest.dependencies ?? {})) {
+      visit(dependency);
+    }
+    for (const dependency of Object.keys(manifest.optionalDependencies ?? {})) {
+      visit(dependency);
+    }
+    order.push(id);
+  };
+  for (const id of manifests.keys()) visit(id);
+  return order;
+}
+
+function section(title, sql) {
+  return `-- ==== ${title} ====\n${sql.trim()}\n`;
+}
+
+export function buildUpgradeSql(dialect, root = ROOT) {
+  if (!(dialect in LAST_28_MIGRATION)) {
+    throw new Error(`unsupported dialect "${dialect}"`);
+  }
+  const parts = [
+    `-- Termix 2.8.0 to 2.9.0 on ${dialect}, generated by scripts/lib/upgrade-sql.mjs\n`,
+  ];
+
+  const coreDir = path.join(root, "drizzle", dialect);
+  const journal = JSON.parse(
+    fs.readFileSync(path.join(coreDir, "meta", "_journal.json"), "utf8"),
+  );
+  const tags = journal.entries.map((entry) => entry.tag);
+  const last = tags.indexOf(LAST_28_MIGRATION[dialect]);
+  if (last === -1) {
+    throw new Error(
+      `${LAST_28_MIGRATION[dialect]} is not in the ${dialect} journal`,
+    );
+  }
+  for (const tag of tags.slice(last + 1)) {
+    parts.push(
+      section(
+        `core ${tag}`,
+        fs.readFileSync(path.join(coreDir, `${tag}.sql`), "utf8"),
+      ),
+    );
+  }
+
+  for (const id of pluginActivationOrder(root)) {
+    const dir = path.join(root, "plugins", id, "migrations", dialect);
+    if (!fs.existsSync(dir)) continue;
+    for (const file of fs
+      .readdirSync(dir)
+      .filter((name) => name.endsWith(".sql"))
+      .sort()) {
+      parts.push(
+        section(
+          `plugin ${id} ${file}`,
+          fs.readFileSync(path.join(dir, file), "utf8"),
+        ),
+      );
+    }
+  }
+
+  return parts.join("\n").replace(/\r\n/g, "\n");
+}
+
+if (
+  process.argv[1] &&
+  fileURLToPath(import.meta.url) === path.resolve(process.argv[1])
+) {
+  process.stdout.write(buildUpgradeSql(process.argv[2] ?? "postgres"));
+}

scripts/upgrade-check.compose.yml

@@ -0,0 +1,35 @@
+# Throwaway Postgres and MySQL servers for scripts/upgrade-check.sh. Data
+# lives in tmpfs, so every run starts from an empty server.
+name: termix-upgrade-check
+
+services:
+  postgres:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: termix
+      POSTGRES_PASSWORD: termix
+      POSTGRES_DB: termix
+    ports:
+      - "55433:5432"
+    tmpfs:
+      - /var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U termix -d termix"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+
+  mysql:
+    image: mysql:8
+    environment:
+      MYSQL_ROOT_PASSWORD: termix
+      MYSQL_DATABASE: termix
+    ports:
+      - "33307:3306"
+    tmpfs:
+      - /var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ptermix"]
+      interval: 2s
+      timeout: 5s
+      retries: 60

scripts/upgrade-check.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Upgrades the 2.8 fixture install to 2.9.0 on real Postgres and MySQL
+# servers and runs the same checks the SQLite upgrade test runs.
+#
+# Needs Docker and a built tree (npm run build). Run from anywhere:
+#   bash scripts/upgrade-check.sh            # both engines
+#   bash scripts/upgrade-check.sh postgres   # one engine
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+COMPOSE=(docker compose -f "$ROOT/scripts/upgrade-check.compose.yml")
+TAG="release-2.8.0-tag"
+if [ "$#" -gt 0 ]; then
+  DIALECTS=("$@")
+else
+  DIALECTS=(postgres mysql)
+fi
+
+cd "$ROOT"
+
+if [ ! -f dist/plugins/snippets/dist/backend.js ]; then
+  echo "dist/plugins is missing. Run npm run build first." >&2
+  exit 1
+fi
+if ! git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then
+  echo "Tag $TAG is missing. Run git fetch --tags first." >&2
+  exit 1
+fi
+
+LEGACY="$(mktemp -d)"
+cleanup() {
+  "${COMPOSE[@]}" down -v >/dev/null 2>&1 || true
+  rm -rf "$LEGACY"
+}
+trap cleanup EXIT
+
+# 2.8's own drizzle migrations build the 2.8 schema on each server.
+git archive "$TAG" drizzle | tar -x -C "$LEGACY"
+
+"${COMPOSE[@]}" up -d --wait
+
+status=0
+for dialect in "${DIALECTS[@]}"; do
+  case "$dialect" in
+    postgres) url="postgres://termix:termix@127.0.0.1:55433/termix" ;;
+    mysql) url="mysql://root:termix@127.0.0.1:33307/termix" ;;
+    *)
+      echo "Unknown dialect $dialect (postgres or mysql)" >&2
+      exit 2
+      ;;
+  esac
+  echo
+  echo "== 2.8 to 2.9.0 on $dialect"
+  if ! TEST_DIALECT="$dialect" TEST_DATABASE_URL="$url" \
+    UPGRADE_LEGACY_MIGRATIONS_DIR="$LEGACY/drizzle" \
+    npx vitest run src/backend/tests/plugins/upgrade-fixture.test.ts \
+    --project backend; then
+    status=1
+  fi
+done
+
+exit "$status"

scripts/upgrade-sql.test.ts

@@ -0,0 +1,44 @@
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { LEGACY_TABLE_OWNERS } from "../packages/plugin-sdk/src/db";
+import { buildUpgradeSql } from "./lib/upgrade-sql.mjs";
+
+const SNAPSHOTS = path.resolve(
+  __dirname,
+  "../src/backend/tests/fixtures/upgrade",
+);
+
+const RENAME = {
+  postgres: /^ALTER TABLE "([a-z0-9_]+)" RENAME TO "(p_[a-z0-9_]+)";$/gm,
+  mysql: /^RENAME TABLE `([a-z0-9_]+)` TO `(p_[a-z0-9_]+)`;$/gm,
+};
+
+describe.each(["postgres", "mysql"] as const)(
+  "the 2.8 to 2.9.0 upgrade SQL on %s",
+  (dialect) => {
+    const sql = buildUpgradeSql(dialect);
+    const statements = sql.replace(/^--.*$/gm, "");
+
+    it("matches the reviewed snapshot", async () => {
+      await expect(sql).toMatchFileSnapshot(
+        path.join(SNAPSHOTS, `upgrade-${dialect}.sql`),
+      );
+    });
+
+    it("adopts every legacy table by renaming it, never by dropping it", () => {
+      const renamed = new Set(
+        [...sql.matchAll(RENAME[dialect])].map((match) => match[1]),
+      );
+      for (const table of Object.keys(LEGACY_TABLE_OWNERS)) {
+        expect(renamed, table).toContain(table);
+        expect(statements).not.toMatch(
+          new RegExp(`DROP TABLE[^;]*[\`"]${table}[\`"]`, "i"),
+        );
+      }
+    });
+
+    it("never drops a column", () => {
+      expect(statements).not.toMatch(/DROP COLUMN/i);
+    });
+  },
+);