feat: give plugins their own tables, migrations and sync entities

packages/plugin-sdk/cli/commands/migrations.mjs

@@ -0,0 +1,227 @@
+/**
+ * Generates a plugin's migrations from its table definitions.
+ *
+ * Not drizzle-kit. Core's own SQLite schema is hand-written DDL and only the
+ * client-server engines use drizzle migrations, so a plugin on drizzle-kit
+ * would still need a hand-rolled SQLite path, three configs and three
+ * generated schema modules. This diffs the definitions against a committed
+ * snapshot and writes plain SQL for all three dialects, which is what the
+ * server's runner applies.
+ *
+ * Usage, from a plugin directory:
+ *   termix-plugin migrations [name]
+ *   termix-plugin migrations --check    verifies nothing is undeclared
+ *
+ * The plugin exports its definitions from src/backend/tables.ts as `tables`.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import * as esbuild from "esbuild";
+import { BACKEND_EXTERNALS } from "../lib/externals.mjs";
+import { readManifest, resolveEntry } from "../lib/plugin-dir.mjs";
+
+const DIALECTS = ["sqlite", "postgres", "mysql"];
+
+const TABLE_ENTRIES = [
+  "src/backend/tables.ts",
+  "src/backend/tables.mjs",
+  "src/backend/tables.js",
+];
+
+/** Loads the plugin's table definitions by bundling them to a temp file. */
+async function loadDefinitions(cwd) {
+  const entry = resolveEntry(cwd, TABLE_ENTRIES);
+  if (!entry) return null;
+
+  // Emitted inside the plugin rather than in the system temp dir: the SDK
+  // stays external to the bundle, so the output has to sit somewhere node can
+  // resolve @termix/plugin-sdk from.
+  const outdir = fs.mkdtempSync(path.join(cwd, ".termix-tables-"));
+  const outfile = path.join(outdir, "tables.mjs");
+
+  try {
+    await esbuild.build({
+      entryPoints: [entry],
+      outfile,
+      bundle: true,
+      format: "esm",
+      platform: "node",
+      target: "node22",
+      logLevel: "warning",
+      external: BACKEND_EXTERNALS,
+    });
+
+    const module = await import(`file://${outfile.split(path.sep).join("/")}`);
+    const tables = module.tables ?? module.default;
+    if (!Array.isArray(tables)) {
+      throw new Error(
+        `${path.relative(cwd, entry)} must export a "tables" array of defineTable() results`,
+      );
+    }
+    return tables;
+  } finally {
+    fs.rmSync(outdir, { recursive: true, force: true });
+  }
+}
+
+function snapshotPath(cwd) {
+  return path.join(cwd, "migrations", "snapshot.json");
+}
+
+function readSnapshot(cwd) {
+  const file = snapshotPath(cwd);
+  if (!fs.existsSync(file)) return { version: 1, tables: {} };
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+/** What a definition looks like in the snapshot, so a diff can compare them. */
+function snapshotTable(definition) {
+  return {
+    columns: definition.columns,
+    indexes: definition.indexes,
+  };
+}
+
+function nextSequence(cwd) {
+  let highest = 0;
+  for (const dialect of DIALECTS) {
+    const dir = path.join(cwd, "migrations", dialect);
+    if (!fs.existsSync(dir)) continue;
+    for (const file of fs.readdirSync(dir)) {
+      const match = /^(\d{4})_/.exec(file);
+      if (match) highest = Math.max(highest, Number(match[1]));
+    }
+  }
+  return highest + 1;
+}
+
+/**
+ * What changed since the snapshot.
+ *
+ * New tables and new columns are generated. A changed or removed column is
+ * reported rather than guessed at: renames and type changes need a data
+ * decision this cannot make, so the author writes that migration by hand.
+ */
+function diff(previous, definitions) {
+  const created = [];
+  const added = [];
+  const manual = [];
+
+  for (const definition of definitions) {
+    const before = previous.tables[definition.name];
+    if (!before) {
+      created.push(definition);
+      continue;
+    }
+
+    for (const [property, column] of Object.entries(definition.columns)) {
+      const existing = before.columns[property];
+      if (!existing) {
+        added.push({ definition, property, column });
+        continue;
+      }
+      if (JSON.stringify(existing) !== JSON.stringify(column)) {
+        manual.push(`${definition.name}.${property} changed type or modifiers`);
+      }
+    }
+
+    for (const property of Object.keys(before.columns)) {
+      if (!definition.columns[property]) {
+        manual.push(`${definition.name}.${property} was removed`);
+      }
+    }
+  }
+
+  for (const name of Object.keys(previous.tables)) {
+    if (!definitions.some((definition) => definition.name === name)) {
+      manual.push(`table ${name} was removed`);
+    }
+  }
+
+  return { created, added, manual };
+}
+
+export async function migrations({ cwd, args }) {
+  const manifest = readManifest(cwd);
+  const pluginId = manifest.id ?? path.basename(cwd);
+  const check = args.includes("--check");
+  const name = args.find((arg) => !arg.startsWith("--")) ?? "update";
+
+  const definitions = await loadDefinitions(cwd);
+  if (definitions === null) {
+    console.log(`${pluginId}: no table definitions, nothing to generate`);
+    return;
+  }
+
+  const { createTableSql, addColumnSql } = await loadEmitter();
+
+  const previous = readSnapshot(cwd);
+  const { created, added, manual } = diff(previous, definitions);
+
+  if (manual.length > 0) {
+    for (const problem of manual) console.error(`  ${problem}`);
+    throw new Error(
+      `${pluginId}: these changes need a hand-written migration, because a rename or a type change is a data decision.`,
+    );
+  }
+
+  if (created.length === 0 && added.length === 0) {
+    console.log(`${pluginId}: migrations are up to date`);
+    return;
+  }
+
+  if (check) {
+    throw new Error(
+      `${pluginId}: table definitions have changed but no migration was generated. Run: termix-plugin migrations`,
+    );
+  }
+
+  const sequence = String(nextSequence(cwd)).padStart(4, "0");
+  const file = `${sequence}_${name.replace(/[^a-z0-9_]/gi, "_").toLowerCase()}.sql`;
+
+  for (const dialect of DIALECTS) {
+    const statements = [];
+    for (const definition of created) {
+      statements.push(...createTableSql(dialect, pluginId, definition));
+    }
+    for (const entry of added) {
+      statements.push(
+        addColumnSql(dialect, pluginId, entry.definition, entry.property),
+      );
+    }
+
+    const dir = path.join(cwd, "migrations", dialect);
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(
+      path.join(dir, file),
+      `-- ${pluginId} ${sequence}: ${name}\n-- Generated by termix-plugin migrations. Review before committing.\n\n${statements.join("\n")}\n`,
+    );
+  }
+
+  const snapshot = { version: 1, tables: {} };
+  for (const definition of definitions) {
+    snapshot.tables[definition.name] = snapshotTable(definition);
+  }
+  fs.writeFileSync(snapshotPath(cwd), `${JSON.stringify(snapshot, null, 2)}\n`);
+
+  console.log(
+    `${pluginId}: wrote ${DIALECTS.length} dialect files for ${sequence}_${name}`,
+  );
+}
+
+/**
+ * The DDL emitter, shared with the server.
+ *
+ * One mapping, two consumers: the SQL written here is the SQL the server's
+ * migration runner applies, so a second copy could drift into producing a
+ * migration the runner cannot reproduce.
+ */
+async function loadEmitter() {
+  const entry = new URL("../../dist/ddl.js", import.meta.url);
+  if (!fs.existsSync(fileURLToPath(entry))) {
+    throw new Error("The plugin SDK is not built yet. Run: npm run build:sdk");
+  }
+  return import(entry.href);
+}

packages/plugin-sdk/cli/commands/validate.mjs

@@ -2,6 +2,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { readManifest } from "../lib/plugin-dir.mjs";
+import { LEGACY_TABLE_OWNERS as LEGACY_TABLES } from "../../dist/db.js";
 
 /**
  * The manifest rules live in src/manifest.ts, which the server uses too, so
@@ -35,6 +36,8 @@ export async function validate({ cwd }) {
     }
   }
 
+  problems.push(...validateMigrations(cwd, raw.id ?? path.basename(cwd)));
+
   if (problems.length > 0) {
     for (const problem of problems) console.error(`  ${problem}`);
     throw new Error(`${raw.id ?? path.basename(cwd)}: manifest is not valid.`);
@@ -42,3 +45,77 @@ export async function validate({ cwd }) {
 
   console.log(`ok  ${raw.id ?? path.basename(cwd)}`);
 }
+
+const DIALECTS = ["sqlite", "postgres", "mysql"];
+
+const CREATE_OR_ALTER =
+  /(?:CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TABLE(?:\s+IF\s+NOT\s+EXISTS)?|ALTER\s+TABLE|DROP\s+TABLE(?:\s+IF\s+EXISTS)?|TRUNCATE\s+TABLE)\s+([`"']?)([a-zA-Z0-9_]+)\1/gi;
+
+/**
+ * Checks that a plugin's migrations only touch tables it owns.
+ *
+ * The server enforces this too, because a plugin installed from a tarball
+ * never ran this command. Here it is a build-time error with a file name
+ * attached, rather than a plugin that fails to activate later.
+ */
+function validateMigrations(cwd, pluginId) {
+  const problems = [];
+  const prefix = `p_${String(pluginId).replace(/-/g, "_")}_`;
+  const root = path.join(cwd, "migrations");
+  if (!fs.existsSync(root)) return problems;
+
+  const present = DIALECTS.filter((dialect) =>
+    fs.existsSync(path.join(root, dialect)),
+  );
+
+  const byDialect = new Map();
+  for (const dialect of present) {
+    const dir = path.join(root, dialect);
+    const files = fs
+      .readdirSync(dir)
+      .filter((file) => file.endsWith(".sql"))
+      .sort();
+    byDialect.set(dialect, files);
+
+    for (const file of files) {
+      if (!/^\d{4}_[a-z0-9_]+\.sql$/.test(file)) {
+        problems.push(
+          `migrations/${dialect}/${file} must be named NNNN_name.sql, lower snake_case`,
+        );
+      }
+
+      const sql = fs.readFileSync(path.join(dir, file), "utf8");
+      CREATE_OR_ALTER.lastIndex = 0;
+      let match;
+      while ((match = CREATE_OR_ALTER.exec(sql)) !== null) {
+        const table = match[2];
+        if (table.startsWith(prefix)) continue;
+        if (LEGACY_TABLES[table] === pluginId) continue;
+        problems.push(
+          `migrations/${dialect}/${file} touches "${table}", which is not prefixed "${prefix}"`,
+        );
+      }
+    }
+  }
+
+  // A migration that exists for one engine and not another leaves that
+  // deployment without the table, which is how alert_rules ended up
+  // SQLite-only in core.
+  const [first, ...rest] = present;
+  for (const dialect of rest) {
+    const a = byDialect.get(first);
+    const b = byDialect.get(dialect);
+    for (const file of a) {
+      if (!b.includes(file)) {
+        problems.push(`migrations/${dialect}/${file} is missing`);
+      }
+    }
+    for (const file of b) {
+      if (!a.includes(file)) {
+        problems.push(`migrations/${first}/${file} is missing`);
+      }
+    }
+  }
+
+  return problems;
+}

packages/plugin-sdk/cli/index.mjs

@@ -10,8 +10,9 @@ import { build } from "./commands/build.mjs";
 import { validate } from "./commands/validate.mjs";
 import { test } from "./commands/test.mjs";
 import { pack } from "./commands/pack.mjs";
+import { migrations } from "./commands/migrations.mjs";
 
-const COMMANDS = { build, validate, test, pack };
+const COMMANDS = { build, validate, test, pack, migrations };
 
 const [command, ...args] = process.argv.slice(2);
 
@@ -24,6 +25,7 @@ if (!command || command === "--help" || command === "-h") {
       "  validate   Check manifest.json and the files it names",
       "  test       Run the plugin's vitest suite",
       "  pack       Write a .tgz of the built plugin",
+      "  migrations Generate migrations from the plugin's table definitions",
     ].join("\n"),
   );
   process.exit(command ? 0 : 1);