feat: give plugins their own tables, migrations and sync entities

packages/plugin-sdk/src/backend.ts

@@ -11,6 +11,7 @@
  */
 
 import type { PluginManifest } from "./manifest.js";
+import type { PluginTableDefinition } from "./db.js";
 
 /**
  * Thrown when a plugin calls a guarded member without the capability.
@@ -56,6 +57,75 @@ export interface PluginKeyValue {
   list: () => Promise<string[]>;
 }
 
+/**
+ * The plugin's own tables.
+ *
+ * `define` registers a table definition and returns the Drizzle table object
+ * to query through `db`. Both require db:own.
+ *
+ * Scoped by name: every table carries the p_<id>_ prefix, and `refs` exposes
+ * users and ssh_data read-only so a plugin can join against them without being
+ * able to write them. In-process code could reach around all of this - see
+ * "What this protects, and what it does not" in ARCHITECTURE.md. The
+ * capability, the prefix, lint and review are the contract, not a sandbox.
+ */
+export interface PluginDatabase {
+  /** Registers a definition and returns its queryable table object. */
+  define: <T = unknown>(definition: PluginTableDefinition) => Promise<T>;
+  /** Drizzle handle, scoped to this plugin's tables. */
+  client: <T = unknown>() => Promise<T>;
+  /** Read-only references to the core tables a plugin may point at. */
+  refs: <T = unknown>() => Promise<T>;
+}
+
+/** A row's shape on the wire, before it is written to a table. */
+export type SyncRow = Record<string, unknown>;
+
+export interface SyncEntityReference {
+  /** Stored column holding a local numeric id, e.g. "credentialId". */
+  field: string;
+  /** Wire field holding the portable id, e.g. "credentialSyncId". */
+  syncField: string;
+  /** The entity the id points at. */
+  entityType: string;
+}
+
+export interface SyncEntityRegistration {
+  /** Stable wire name. Never change it: tombstones and remote rows match on it. */
+  type: string;
+  /** The table, as returned by ctx.db.define. */
+  table: unknown;
+  /** Column holding the owning user id. Defaults to "userId". */
+  userColumn?: string;
+  /** Fields DataCrypto translates between plaintext wire and encrypted row. */
+  encryptedFields?: readonly string[];
+  /** Local-id to sync-id translations applied on the way out and back. */
+  references?: readonly SyncEntityReference[];
+  /** Lower sorts first. Reference targets must sort before their referrers. */
+  order?: number;
+  /** Fields that must never be overwritten by an inbound payload. */
+  readOnlyFields?: readonly string[];
+  /** One row per user rather than many, keyed on the owner. */
+  singleton?: boolean;
+  /**
+   * Escape hatch for a row whose references are not plain columns, such as ids
+   * embedded in a JSON blob. Runs instead of `references`, not alongside it.
+   */
+  serialize?: (
+    row: SyncRow,
+    resolveSyncId: (entityType: string, id: number) => Promise<string | null>,
+  ) => Promise<SyncRow>;
+  deserialize?: (
+    row: SyncRow,
+    resolveId: (entityType: string, syncId: string) => Promise<number | null>,
+  ) => Promise<SyncRow>;
+}
+
+/** Adds an entity to remote sync between a desktop backend and a server. */
+export interface PluginSync {
+  registerEntity: (entity: SyncEntityRegistration) => void;
+}
+
 export interface PluginRegistry {
   provide: <T>(key: string, value: T) => void;
   consume: <T>(key: string) => T | undefined;
@@ -95,6 +165,8 @@ export interface PluginContext {
   readonly log: PluginLogger;
   readonly events: PluginEvents;
   readonly kv: PluginKeyValue;
+  readonly db: PluginDatabase;
+  readonly sync: PluginSync;
   readonly registry: PluginRegistry;
   readonly services: PluginServices;
   readonly secrets: PluginSecrets;
@@ -133,3 +205,4 @@ export function definePlugin(plugin: PluginModule): PluginModule {
 }
 
 export type { PluginManifest } from "./manifest.js";
+export type { PluginTableDefinition } from "./db.js";

packages/plugin-sdk/src/db.ts

@@ -0,0 +1,263 @@
+/**
+ * Table definitions a plugin owns.
+ *
+ * A definition is a plain descriptor, not a live Drizzle table. Core turns it
+ * into one at runtime (src/backend/plugins/table-builder.ts) and emits the DDL
+ * for whichever dialect the server is on, which keeps this entry a description
+ * of the contract rather than an implementation a plugin could reach around.
+ *
+ * Why one table object and three DDL emitters, rather than three table
+ * objects: the sqlite-core definitions already encode correctly for every
+ * engine at query time. text and integer encode as themselves; a boolean
+ * writes 1/0, which Postgres and MySQL both accept; real is a plain number.
+ * What genuinely differs between the engines is DDL. See the header of
+ * scripts/generate-dialect-schema.cjs, which this mapping is taken from.
+ */
+
+export type PluginColumnType =
+  | "id"
+  | "text"
+  | "varchar"
+  | "integer"
+  | "bigint"
+  | "boolean"
+  | "timestamp"
+  | "json"
+  | "encryptedText"
+  | "refUser"
+  | "refHost";
+
+export interface PluginColumn {
+  readonly type: PluginColumnType;
+  /** varchar only. The ceiling MySQL can index. */
+  readonly length?: number;
+  readonly notNull?: boolean;
+  readonly primaryKey?: boolean;
+  readonly unique?: boolean;
+  readonly defaultValue?: string | number | boolean | null;
+  /** timestamp only: default CURRENT_TIMESTAMP. */
+  readonly defaultNow?: boolean;
+}
+
+/** The longest string MySQL will index, and so the width of every key column. */
+export const KEY_LENGTH = 255;
+
+interface Builder<T extends PluginColumn = PluginColumn> {
+  notNull: () => Builder<T>;
+  primaryKey: () => Builder<T>;
+  unique: () => Builder<T>;
+  default: (value: string | number | boolean | null) => Builder<T>;
+  defaultNow: () => Builder<T>;
+  readonly column: PluginColumn;
+}
+
+function builder(column: PluginColumn): Builder {
+  const next = (patch: Partial<PluginColumn>) =>
+    builder({ ...column, ...patch });
+  return {
+    column,
+    notNull: () => next({ notNull: true }),
+    primaryKey: () => next({ primaryKey: true, notNull: true }),
+    unique: () => next({ unique: true }),
+    default: (defaultValue) => next({ defaultValue }),
+    defaultNow: () => next({ defaultNow: true }),
+  };
+}
+
+/** Auto-incrementing surrogate key: integer on sqlite, serial, auto_increment. */
+export const id = () =>
+  builder({ type: "id", primaryKey: true, notNull: true });
+
+/** Unbounded string. Never indexable: MySQL cannot index TEXT. */
+export const text = () => builder({ type: "text" });
+
+/** Bounded string. Indexable on every engine. */
+export const varchar = (length: number = KEY_LENGTH) =>
+  builder({ type: "varchar", length });
+
+export const integer = () => builder({ type: "integer" });
+export const bigint = () => builder({ type: "bigint" });
+export const boolean = () => builder({ type: "boolean" });
+
+/** Stored as text on every engine. See src/backend/database/repositories/sql-timestamp.ts. */
+export const timestamp = () => builder({ type: "timestamp" });
+
+/** JSON held as text. Never indexable, for the same reason as text(). */
+export const json = () => builder({ type: "json" });
+
+/** Text encrypted with the owning user's key before it is written. */
+export const encryptedText = () => builder({ type: "encryptedText" });
+
+/** Foreign key to users.id, cascading on delete. */
+export const refUser = () =>
+  builder({ type: "refUser", length: KEY_LENGTH, notNull: true });
+
+/** Foreign key to ssh_data.id, cascading on delete. */
+export const refHost = () => builder({ type: "refHost", notNull: true });
+
+export interface PluginTableIndex {
+  readonly name: string;
+  readonly columns: readonly string[];
+  readonly unique?: boolean;
+}
+
+export interface PluginTableOptions {
+  readonly indexes?: readonly PluginTableIndex[];
+  readonly uniques?: readonly PluginTableIndex[];
+}
+
+export interface PluginTableDefinition {
+  /** The unprefixed name as the plugin wrote it. */
+  readonly name: string;
+  readonly columns: Readonly<Record<string, PluginColumn>>;
+  readonly indexes: readonly PluginTableIndex[];
+}
+
+/** Columns that can never carry an index, because MySQL cannot index them. */
+const UNINDEXABLE: ReadonlySet<PluginColumnType> = new Set([
+  "text",
+  "json",
+  "encryptedText",
+]);
+
+const NAME_PATTERN = /^[a-z][a-z0-9_]*$/;
+
+type Columns = Record<string, Builder | PluginColumn>;
+
+function unwrap(value: Builder | PluginColumn): PluginColumn {
+  return "column" in value ? value.column : value;
+}
+
+/**
+ * Declares a table this plugin owns.
+ *
+ * The name is unprefixed here; core prefixes it with p_<plugin id>_ when the
+ * definition is registered, so a plugin cannot name a table into another
+ * plugin's namespace or into core's.
+ */
+export function defineTable(
+  name: string,
+  columns: Columns,
+  options: PluginTableOptions = {},
+): PluginTableDefinition {
+  if (!NAME_PATTERN.test(name)) {
+    throw new Error(
+      `Table name "${name}" must be lower snake_case and start with a letter`,
+    );
+  }
+
+  const resolved: Record<string, PluginColumn> = {};
+  for (const [key, value] of Object.entries(columns)) {
+    resolved[key] = unwrap(value);
+  }
+
+  if (Object.keys(resolved).length === 0) {
+    throw new Error(`Table "${name}" declares no columns`);
+  }
+
+  const indexes = [
+    ...(options.uniques ?? []).map((entry) => ({ ...entry, unique: true })),
+    ...(options.indexes ?? []),
+  ];
+
+  // Checked here rather than at migration time so the error names the table
+  // and column while the author is still looking at the definition.
+  for (const index of indexes) {
+    if (!NAME_PATTERN.test(index.name)) {
+      throw new Error(
+        `Index name "${index.name}" must be lower snake_case and start with a letter`,
+      );
+    }
+    if (index.columns.length === 0) {
+      throw new Error(`Index "${index.name}" names no columns`);
+    }
+    for (const column of index.columns) {
+      const declared = resolved[column];
+      if (!declared) {
+        throw new Error(
+          `Index "${index.name}" names "${column}", which table "${name}" does not declare`,
+        );
+      }
+      if (UNINDEXABLE.has(declared.type)) {
+        throw new Error(
+          `Index "${index.name}" covers "${column}", a ${declared.type} column. ` +
+            `MySQL cannot index an unbounded column: use varchar(n) instead.`,
+        );
+      }
+    }
+  }
+
+  return { name, columns: resolved, indexes };
+}
+
+/** p_<id with - as _>_ - the prefix every table a plugin owns carries. */
+export function tablePrefix(pluginId: string): string {
+  return `p_${pluginId.replace(/-/g, "_")}_`;
+}
+
+/** The physical table name for a definition owned by a plugin. */
+export function prefixedTableName(pluginId: string, name: string): string {
+  return `${tablePrefix(pluginId)}${name}`;
+}
+
+/**
+ * Legacy core tables, and the one plugin allowed to adopt each.
+ *
+ * Adoption is how a feature keeps its rows when it moves out of core: the
+ * table is renamed into the plugin's namespace rather than copied. One owner
+ * per table, because two plugins adopting the same one would mean whichever
+ * activated second found it already gone.
+ *
+ * Shared with the CLI so a migration that reaches for someone else's table is
+ * caught at build time as well as at activation.
+ */
+export const LEGACY_TABLE_OWNERS: Readonly<Record<string, string>> = {
+  fleets: "fleets",
+  fleet_members: "fleets",
+  fleet_inventory: "fleets",
+  user_workspaces: "workspaces",
+  network_topology: "network-topology",
+  snippets: "snippets",
+  snippet_folders: "snippets",
+  snippet_access: "snippets",
+  c2s_tunnel_presets: "tunnels",
+  command_history: "ssh-terminal",
+  file_manager_recent: "file-manager",
+  file_manager_pinned: "file-manager",
+  file_manager_shortcuts: "file-manager",
+  transfer_recent: "file-manager",
+  tmux_session_tags: "tmux-monitor",
+  session_shares: "session-sharing",
+  session_share_participants: "session-sharing",
+  collab_rooms: "session-sharing",
+  collab_room_members: "session-sharing",
+  session_recordings: "session-recording",
+  host_metrics_preferences: "host-metrics",
+  host_health_checks: "host-metrics",
+  host_health_history: "host-metrics",
+  host_metrics_history: "host-metrics",
+  proxmox_stats_preferences: "proxmox",
+  proxmox_node_history: "proxmox",
+  automations: "automations",
+  automation_trigger_state: "automations",
+  automation_schedules: "automations",
+  automation_runs: "automations",
+  automation_run_steps: "automations",
+  automation_channels: "automations",
+  ai_providers: "ai",
+  ai_conversations: "ai",
+  ai_messages: "ai",
+  ai_proposals: "ai",
+  homepage_items: "homepage",
+  homepage_layouts: "homepage",
+  dashboard_service_links: "homepage",
+  secret_sources: "secret-sources",
+  opkssh_tokens: "opkssh",
+  vault_profiles: "vault",
+  vault_tokens: "vault",
+  termix_identities: "termix-identity",
+  termix_identity_keys: "termix-identity",
+  termix_identity_ca: "termix-identity",
+  webauthn_credentials: "webauthn",
+  sso_providers: "sso",
+};

packages/plugin-sdk/src/ddl.ts

@@ -0,0 +1,220 @@
+/**
+ * The DDL a table definition becomes, per dialect.
+ *
+ * Lives in the SDK because two things need the identical mapping: the CLI,
+ * which writes a plugin's migration files, and the server, which applies them.
+ * A second copy would drift and produce a migration the runner could not
+ * reproduce.
+ *
+ * The mapping itself is the one scripts/generate-dialect-schema.cjs applies to
+ * core's schema, kept deliberately in step with it.
+ */
+
+import { KEY_LENGTH, prefixedTableName } from "./db.js";
+import type {
+  PluginColumn,
+  PluginTableDefinition,
+  PluginTableIndex,
+} from "./db.js";
+
+/** Which engine the DDL is for. Matches core's DatabaseDialect exactly. */
+export type SqlDialect = "sqlite" | "postgres" | "mysql";
+
+/** The core tables a plugin may point a foreign key at, and nothing else. */
+export const REFERENCEABLE = {
+  refUser: { table: "users", column: "id" },
+  refHost: { table: "ssh_data", column: "id" },
+} as const;
+
+function camelToSnake(value: string): string {
+  return value.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
+}
+
+/** The SQL column name for a declared property. */
+export function columnName(property: string, column?: PluginColumn): string {
+  void column;
+  return camelToSnake(property);
+}
+
+function quote(dialect: SqlDialect, identifier: string): string {
+  if (dialect === "mysql") return "`" + identifier + "`";
+  return '"' + identifier + '"';
+}
+
+/**
+ * The physical type for a column on one dialect.
+ *
+ * Mirrors generate-dialect-schema.cjs: an autoincrement key is serial /
+ * auto_increment, a boolean is native off sqlite, and anything carrying a key
+ * has to be varchar because MySQL will not index TEXT.
+ */
+function columnType(
+  dialect: SqlDialect,
+  column: PluginColumn,
+  keyed: boolean,
+): string {
+  switch (column.type) {
+    case "id":
+      if (dialect === "postgres") return "serial";
+      if (dialect === "mysql") return "int AUTO_INCREMENT";
+      return "integer";
+    case "integer":
+      return dialect === "mysql" ? "int" : "integer";
+    case "bigint":
+      return dialect === "sqlite" ? "integer" : "bigint";
+    case "boolean":
+      return dialect === "sqlite" ? "integer" : "boolean";
+    case "varchar":
+      return dialect === "sqlite"
+        ? "text"
+        : `varchar(${column.length ?? KEY_LENGTH})`;
+    case "refUser":
+      return dialect === "sqlite" ? "text" : `varchar(${KEY_LENGTH})`;
+    case "refHost":
+      return dialect === "mysql" ? "int" : "integer";
+    default:
+      // A keyed string has to be indexable, so it narrows to varchar even
+      // though it was declared unbounded. defineTable refuses to index a
+      // text/json column, so in practice this only catches a primary key.
+      if (keyed && dialect !== "sqlite") return `varchar(${KEY_LENGTH})`;
+      return "text";
+  }
+}
+
+function defaultClause(dialect: SqlDialect, column: PluginColumn): string {
+  if (column.defaultNow) {
+    // MySQL rejects a bare DEFAULT CURRENT_TIMESTAMP on a text column; an
+    // expression default is written parenthesised. MariaDB takes either.
+    return dialect === "mysql"
+      ? " DEFAULT (CURRENT_TIMESTAMP)"
+      : " DEFAULT CURRENT_TIMESTAMP";
+  }
+  if (column.defaultValue === undefined) return "";
+  const value = column.defaultValue;
+  if (value === null) return " DEFAULT NULL";
+  if (typeof value === "boolean") {
+    return dialect === "sqlite"
+      ? ` DEFAULT ${value ? 1 : 0}`
+      : ` DEFAULT ${value ? "true" : "false"}`;
+  }
+  if (typeof value === "number") return ` DEFAULT ${value}`;
+  return ` DEFAULT '${String(value).replace(/'/g, "''")}'`;
+}
+
+/** Every column that carries a key, and so must be indexable on MySQL. */
+export function keyedColumns(
+  definition: PluginTableDefinition,
+): ReadonlySet<string> {
+  const keyed = new Set<string>();
+  for (const [property, column] of Object.entries(definition.columns)) {
+    if (column.primaryKey || column.unique) keyed.add(property);
+    if (column.type === "refUser" || column.type === "refHost") {
+      keyed.add(property);
+    }
+  }
+  for (const index of definition.indexes) {
+    for (const property of index.columns) keyed.add(property);
+  }
+  return keyed;
+}
+
+function indexStatement(
+  dialect: SqlDialect,
+  physical: string,
+  definition: PluginTableDefinition,
+  entry: PluginTableIndex,
+): string {
+  const columns = entry.columns
+    .map((property) =>
+      quote(dialect, columnName(property, definition.columns[property])),
+    )
+    .join(", ");
+  const unique = entry.unique ? "UNIQUE " : "";
+  // MySQL has no CREATE INDEX IF NOT EXISTS. A plugin migration runs once,
+  // recorded in plugin_migrations, so the guard is the ledger, not the DDL.
+  const guard = dialect === "mysql" ? "" : "IF NOT EXISTS ";
+  return `CREATE ${unique}INDEX ${guard}${quote(dialect, entry.name)} ON ${quote(dialect, physical)} (${columns});`;
+}
+
+/** The CREATE TABLE and CREATE INDEX statements for one definition. */
+export function createTableSql(
+  dialect: SqlDialect,
+  pluginId: string,
+  definition: PluginTableDefinition,
+): string[] {
+  const physical = prefixedTableName(pluginId, definition.name);
+  const keyed = keyedColumns(definition);
+
+  const lines: string[] = [];
+  const constraints: string[] = [];
+
+  for (const [property, column] of Object.entries(definition.columns)) {
+    const name = columnName(property, column);
+    const type = columnType(dialect, column, keyed.has(property));
+
+    let line = `  ${quote(dialect, name)} ${type}`;
+    if (column.type === "id") {
+      line += " PRIMARY KEY";
+      if (dialect === "sqlite") line += " AUTOINCREMENT";
+    } else {
+      if (column.primaryKey) line += " PRIMARY KEY";
+      if (column.notNull) line += " NOT NULL";
+      if (column.unique) line += " UNIQUE";
+      line += defaultClause(dialect, column);
+    }
+    lines.push(line);
+
+    const reference =
+      column.type === "refUser"
+        ? REFERENCEABLE.refUser
+        : column.type === "refHost"
+          ? REFERENCEABLE.refHost
+          : null;
+    if (reference) {
+      constraints.push(
+        `  FOREIGN KEY (${quote(dialect, name)}) REFERENCES ${quote(dialect, reference.table)} (${quote(dialect, reference.column)}) ON DELETE CASCADE`,
+      );
+    }
+  }
+
+  const body = [...lines, ...constraints].join(",\n");
+  const statements = [
+    `CREATE TABLE IF NOT EXISTS ${quote(dialect, physical)} (\n${body}\n);`,
+  ];
+
+  for (const entry of definition.indexes) {
+    statements.push(indexStatement(dialect, physical, definition, entry));
+  }
+
+  return statements;
+}
+
+/** DROP statement for a definition, for removePluginData. */
+/** DROP statement for a definition, for removePluginData. */
+export function dropTableSql(
+  dialect: SqlDialect,
+  pluginId: string,
+  definition: PluginTableDefinition,
+): string {
+  const physical = prefixedTableName(pluginId, definition.name);
+  return `DROP TABLE IF EXISTS ${quote(dialect, physical)};`;
+}
+
+/** The ALTER that adds one newly declared column. */
+export function addColumnSql(
+  dialect: SqlDialect,
+  pluginId: string,
+  definition: PluginTableDefinition,
+  property: string,
+): string {
+  const physical = prefixedTableName(pluginId, definition.name);
+  const column = definition.columns[property];
+  const keyed = keyedColumns(definition).has(property);
+  const name = columnName(property, column);
+
+  let line = `${quote(dialect, name)} ${columnType(dialect, column, keyed)}`;
+  if (column.notNull) line += " NOT NULL";
+  line += defaultClause(dialect, column);
+
+  return `ALTER TABLE ${quote(dialect, physical)} ADD COLUMN ${line};`;
+}

packages/plugin-sdk/src/testing.ts

@@ -16,6 +16,8 @@ import type {
 } from "./backend.js";
 import type { TermixApp } from "./frontend.js";
 import type { PluginManifest } from "./manifest.js";
+import type { PluginTableDefinition } from "./db.js";
+import type { SyncEntityRegistration } from "./backend.js";
 
 export interface FakeContextOptions {
   pluginId?: string;
@@ -32,6 +34,10 @@ export interface FakePluginContext {
   emitted: Array<{ topic: string; payload: unknown }>;
   /** Backing store behind ctx.kv. */
   kv: Map<string, unknown>;
+  /** Table definitions registered through ctx.db.define, in order. */
+  tables: PluginTableDefinition[];
+  /** Sync entities registered through ctx.sync.registerEntity, in order. */
+  syncEntities: SyncEntityRegistration[];
 }
 
 function noopLogger(): PluginLogger {
@@ -55,6 +61,8 @@ export function createFakeContext(
   const disposals: Array<() => void | Promise<void>> = [];
   const emitted: Array<{ topic: string; payload: unknown }> = [];
   const kv = new Map<string, unknown>();
+  const tables: PluginTableDefinition[] = [];
+  const syncEntities: SyncEntityRegistration[] = [];
   const listeners = new Map<string, Set<(payload: unknown) => void>>();
   let actor = options.actor;
 
@@ -105,6 +113,27 @@ export function createFakeContext(
       list: async () => [...kv.keys()],
     },
 
+    db: {
+      // No engine behind this one. A test that needs real SQL should drive the
+      // runtime rather than the double.
+      define: async (definition) => {
+        tables.push(definition);
+        return undefined as never;
+      },
+      client: async () => {
+        throw new Error("createFakeContext does not implement db.client");
+      },
+      refs: async () => {
+        throw new Error("createFakeContext does not implement db.refs");
+      },
+    },
+
+    sync: {
+      registerEntity: (entity) => {
+        syncEntities.push(entity);
+      },
+    },
+
     registry: {
       provide: () => {},
       consume: () => undefined,
@@ -139,7 +168,7 @@ export function createFakeContext(
     currentActor: () => actor,
   };
 
-  return { ctx, disposals, emitted, kv };
+  return { ctx, disposals, emitted, kv, tables, syncEntities };
 }
 
 export interface MockContextOptions {
@@ -203,9 +232,25 @@ export function createMockCtx(
   };
 
   const guardedKv = ctx.kv;
+  const guardedDb = ctx.db;
   const gatedCtx: PluginContext = {
     ...ctx,
 
+    db: {
+      define: async (definition) => {
+        require("db:own");
+        return guardedDb.define(definition);
+      },
+      client: async () => {
+        require("db:own");
+        return guardedDb.client();
+      },
+      refs: async () => {
+        require("db:own");
+        return guardedDb.refs();
+      },
+    },
+
     kv: {
       get: async (key) => {
         require("kv:own");