feat: give plugins their own tables, migrations and sync entities

This commit is contained in:
LukeGus committed 2026-09-22 15:49:54 -05:00
1 parent 5796186a1c
commit c496a093ca
47 files changed
+32595 -290

No files matched your search

@@ -0,0 +1,11 @@
CREATE TABLE `plugin_migrations` (
`id` int AUTO_INCREMENT NOT NULL,
`plugin_id` varchar(255) NOT NULL,
`migration_id` varchar(255) NOT NULL,
`checksum` text NOT NULL,
`applied_at` text NOT NULL DEFAULT (CURRENT_TIMESTAMP),
CONSTRAINT `plugin_migrations_id` PRIMARY KEY(`id`),
CONSTRAINT `idx_plugin_migrations_plugin_migration` UNIQUE(`plugin_id`,`migration_id`)
);
--> statement-breakpoint
ALTER TABLE `plugin_migrations` ADD CONSTRAINT `plugin_migrations_plugin_id_plugins_id_fk` FOREIGN KEY (`plugin_id`) REFERENCES `plugins`(`id`) ON DELETE cascade ON UPDATE no action;
File diff suppressed because it is too large. Load diff
+7
View File
@@ -190,6 +190,13 @@
"when": 1790102859925,
"tag": "0026_empty_black_bird",
"breakpoints": true
},
{
"idx": 27,
"version": "5",
"when": 1790109037476,
"tag": "0027_watery_multiple_man",
"breakpoints": true
}
]
}
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE "plugin_migrations" (
"id" serial PRIMARY KEY NOT NULL,
"plugin_id" varchar(255) NOT NULL,
"migration_id" varchar(255) NOT NULL,
"checksum" text NOT NULL,
"applied_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL
);
--> statement-breakpoint
ALTER TABLE "plugin_migrations" ADD CONSTRAINT "plugin_migrations_plugin_id_plugins_id_fk" FOREIGN KEY ("plugin_id") REFERENCES "public"."plugins"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE UNIQUE INDEX "idx_plugin_migrations_plugin_migration" ON "plugin_migrations" USING btree ("plugin_id","migration_id");
File diff suppressed because it is too large. Load diff
+7
View File
@@ -197,6 +197,13 @@
"when": 1790102859387,
"tag": "0027_sharp_guardian",
"breakpoints": true
},
{
"idx": 28,
"version": "7",
"when": 1790109036999,
"tag": "0028_whole_zemo",
"breakpoints": true
}
]
}
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE `plugin_migrations` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`plugin_id` text NOT NULL,
`migration_id` text NOT NULL,
`checksum` text NOT NULL,
`applied_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL,
FOREIGN KEY (`plugin_id`) REFERENCES `plugins`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `idx_plugin_migrations_plugin_migration` ON `plugin_migrations` (`plugin_id`,`migration_id`);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -169,6 +169,13 @@
"when": 1790102858837,
"tag": "0023_talented_shooting_star",
"breakpoints": true
},
{
"idx": 24,
"version": "6",
"when": 1790109036510,
"tag": "0024_public_hemingway",
"breakpoints": true
}
]
}
+5
View File
@@ -1,3 +1,8 @@
// The fallback list, for a server older than GET /sync/entity-types.
//
// Current servers return the entity types and their order, which is the only
// way this process learns about one a plugin added. This array is what those
// servers would have synced anyway, so an old pairing keeps working.
const SYNCED_ENTITY_TYPES = Object.freeze([
// Ordered by reference dependency: hosts and snippets resolve credential,
// vault and folder syncIds, so those have to exist on the other side first.
+42 -2
View File
@@ -311,7 +311,10 @@ class RemoteSyncEngine {
const state = readJson(getRemoteSyncStatePath(), { entities: {} });
let sawAuthFailure = false;
for (const entityType of SYNCED_ENTITY_TYPES) {
const remoteBase = config.serverUrl.replace(/\/$/, "");
const entityTypes = await this.resolveEntityTypes(remoteBase, remoteJwt);
for (const entityType of entityTypes) {
const entityState = state.entities[entityType] || {
lastPulledAt: null,
lastPushedAt: null,
@@ -319,7 +322,7 @@ class RemoteSyncEngine {
const result = await this.syncEntity({
entityType,
remoteBaseUrl: config.serverUrl.replace(/\/$/, ""),
remoteBaseUrl: remoteBase,
remoteJwt,
since: entityState.lastPulledAt,
});
@@ -374,6 +377,43 @@ class RemoteSyncEngine {
return this.status;
}
/**
* Which entities to sync, and in what order.
*
* Asked of the remote server, because a plugin installed there can add one
* and this process has no way to know. Intersected with what the embedded
* backend supports: the two sides may have different plugins installed, and
* pushing an entity the local backend has never heard of just 400s.
*
* A server older than this endpoint 404s, and the frozen list is what it
* would have synced anyway.
*/
async resolveEntityTypes(remoteBaseUrl, remoteJwt) {
const fetchTypes = async (base, token) => {
try {
const body = await this.fetchJson(`${base}/sync/entity-types`, token);
const types = (body?.entityTypes || [])
.slice()
.sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
.map((entity) => entity.type)
.filter((type) => typeof type === "string" && type);
return types.length > 0 ? types : null;
} catch {
return null;
}
};
const remote = await fetchTypes(remoteBaseUrl, remoteJwt);
if (!remote) return SYNCED_ENTITY_TYPES;
const local = await fetchTypes(EMBEDDED_BASE_URL, this.localJwt);
if (!local) return remote;
const supported = new Set(local);
const shared = remote.filter((type) => supported.has(type));
return shared.length > 0 ? shared : SYNCED_ENTITY_TYPES;
}
async fetchJson(url, token, options = {}) {
const res = await fetchRemoteSync(url, {
...options,
+22 -4
View File
@@ -1,6 +1,26 @@
// Sort writes per destination: response order is lost when both sides are merged.
//
// Keyed on the self-reference field the rows actually carry rather than on the
// entity name, so an entity a plugin adds gets the same treatment without this
// process needing the server's entity registry.
const SELF_REFERENCE_FIELDS = { hosts: "parentHostSyncId" };
function selfReferenceField(entityType, writes) {
const known = SELF_REFERENCE_FIELDS[entityType];
if (known) return known;
// A plugin entity declares its own; detect it from the payload.
const sample = writes.find((write) => write && write.row);
if (!sample) return null;
const candidate = Object.keys(sample.row).find(
(key) => key === `parent${entityType}SyncId`,
);
return candidate || null;
}
function orderSyncWrites(entityType, writes) {
if (entityType !== "hosts") return writes;
const field = selfReferenceField(entityType, writes);
if (!field) return writes;
const destinations = new Map();
for (const write of writes) {
if (!destinations.has(write.baseUrl)) {
@@ -16,9 +36,7 @@ function orderSyncWrites(entityType, writes) {
if (visiting.has(write))
throw new Error("Cyclic parent host sync dependency");
visiting.add(write);
const parent = destinations
.get(write.baseUrl)
.get(write.row.parentHostSyncId);
const parent = destinations.get(write.baseUrl).get(write.row[field]);
if (parent) visit(parent);
visiting.delete(write);
visited.add(write);
+64 -14
View File
@@ -27,6 +27,8 @@ dependencies. Entry points:
| Entry | Contents |
| --------------------------------- | ---------------------------------------------------------- |
| `@termix/plugin-sdk/backend` | `PluginContext`, `definePlugin()`, `PluginCapabilityError` |
| `@termix/plugin-sdk/db` | `defineTable()`, the column builders, the legacy-table map |
| `@termix/plugin-sdk/ddl` | The per-dialect DDL emitter, shared with the CLI |
| `@termix/plugin-sdk/frontend` | The `app` object types (stubs until A7) |
| `@termix/plugin-sdk/manifest` | Manifest types, validation, the JSON schema |
| `@termix/plugin-sdk/capabilities` | The capability catalog |
@@ -76,13 +78,50 @@ revokes one does not get it handed back on the next restart.
### 5. Data
Plugins own their tables through SDK table definitions and ship migrations for
sqlite, pg and mysql. Table names are prefixed `p_<id with - as _>_`. Legacy
core tables are adopted by rename in the plugin's first migration. `ctx.kv` is
for small key/value state. Core `schema.ts` ends up holding only core tables.
Plugins own their tables. A plugin declares them with `defineTable()` from
`@termix/plugin-sdk/db` and ships migrations for `sqlite`, `postgres` and
`mysql` under `migrations/<dialect>/NNNN_name.sql`. Table names are prefixed
`p_<id with - as _>_`, which `defineTable` adds. `ctx.kv` is for small
key/value state. Core `schema.ts` ends up holding only core tables.
**A3** delivers the table definitions and migration runner. A1 ships `ctx.kv`
only.
**One table object, three DDL emitters.** The sqlite-core definitions already
encode correctly on every engine at query time, which is why core's own 44
repositories use `schema.ts` on all three. Only DDL genuinely differs, and
that mapping lives in `@termix/plugin-sdk/ddl` so `termix-plugin migrations`
and the server's runner cannot drift into disagreeing.
**MySQL cannot index TEXT.** Any column carrying a key becomes
`varchar(255)`, and `defineTable` refuses an index on `text()` or `json()`
outright rather than silently truncating it.
**Migrations are applied at activation**, before `activate()` runs, so a
plugin's first query never meets a missing table. Applied migrations are
recorded in `plugin_migrations` with a checksum, which makes them immutable:
editing one that already ran blocks that plugin rather than leaving two
installs with quietly different schemas. A migration that throws marks its
plugin failed and core keeps booting.
**Adoption** is how a feature keeps its rows when it moves out of core. The
legacy table is renamed into the plugin's namespace
(`fleets` -> `p_fleets_fleets`) rather than copied, so nothing is duplicated
and no row is migrated one at a time. `LEGACY_TABLE_OWNERS` maps each legacy
table to the one plugin allowed to adopt it, enforced by the CLI at build time
and by the runner at activation, because a plugin installed from a tarball
never ran the CLI.
**Anything DDL-only in `db/index.ts` is SQLite-only.** `alert_rules`,
`alert_rule_channels` and `alert_firings` were created only there and never
declared in `schema.ts`, so they never existed on Postgres or MySQL at all.
Automations replaced them and A3 drops them. A core table needs all three
artifacts in lockstep: the `schema.ts` declaration, the `index.ts` DDL, and the
regenerated `schema.pg.ts`/`schema.mysql.ts` plus drizzle migrations.
**Sync entities are registered, not hardcoded.** `ctx.sync.registerEntity`
adds an entity to remote sync; core registers its own ten the same way in
`database/routes/sync-entities.ts`. The wire names are unchanged, because
existing tombstones and rows on the far side match on those strings. The
Electron client asks the server for the ordered list and falls back to its
frozen array for a server that predates the endpoint.
### 6. HTTP and WebSockets
@@ -175,7 +214,9 @@ plugins/<id>/
src/backend/index.ts exports activate(ctx) and deactivate()
src/frontend/index.tsx exports activate(app) and deactivate() (A7)
locales/en.json English strings; Crowdin translates the rest
migrations/{sqlite,pg,mysql}/ owned tables (A3)
migrations/{sqlite,postgres,mysql}/ owned tables, one .sql per dialect
migrations/snapshot.json diff basis for the generator
src/backend/tables.ts defineTable() definitions
tests/backend/ vitest, node
tests/frontend/ vitest, jsdom
README.md CHANGELOG.md
@@ -364,7 +405,8 @@ Built per plugin in `src/backend/plugins/ctx.ts` and passed to `activate`.
| `ctx.disposables.add` | none | **A1** |
| `ctx.asUser(userId, fn)` | none, always audited | **A1** |
| `ctx.currentActor()` | none | **A1** |
| `ctx.db` | `db:own` | A3 |
| `ctx.db.define` / `.client` / `.refs` | `db:own` | **A3** |
| `ctx.sync.registerEntity` | none | **A3** |
| `ctx.http.route` / `ctx.ws` | `network:serve` | A4 |
| `ctx.hosts.*` | `hosts:read` / `hosts:write` | A4 |
| `ctx.ssh.*` | `ssh:connect`, `credentials:use` | A4 |
@@ -454,6 +496,13 @@ architecture did.
- **Protection from malicious code.** A plugin that wants to bypass the gate
can. Nothing here stops it.
`ctx.db` is a clear example. It hands back a handle scoped to the plugin's own
tables, and the `p_<id>_` prefix, the CLI check and the runner's check all say
a plugin may only touch its own. None of that is enforced by the engine: a
plugin holding that handle can write any table in the database, and `ctx.db.refs`
hands it `users` and `ssh_data` outright. The prefix, the capability, the audit
line, lint and review are the contract. The engine is not.
So the honest boundary is: capabilities stop accidental and casual overreach,
and make every privileged call auditable. What stops malicious code is
**signing, review and the kill list** - the same trust model as any other
@@ -473,12 +522,13 @@ until one is, this document does not pretend otherwise.
runs from the plugin's own directory, so a plugin that moves to its own repo
keeps working unchanged.
| Command | What it does |
| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `termix-plugin build` | esbuild bundles `src/backend/index.ts` to `dist/backend.js` (ESM, node22) and `src/frontend/index.tsx` to `dist/frontend.js` (ESM, browser), then copies `locales/` and `migrations/` into `dist/`. Imported CSS lands in `dist/frontend.css`. |
| `termix-plugin validate` | Runs the SDK's `parseManifest` and checks the files the manifest names exist. |
| `termix-plugin test` | Runs the plugin's vitest suite. |
| `termix-plugin pack` | Writes a `.tgz` of the manifest, `dist/`, locales, migrations and README. **D3** adds signing. |
| Command | What it does |
| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `termix-plugin build` | esbuild bundles `src/backend/index.ts` to `dist/backend.js` (ESM, node22) and `src/frontend/index.tsx` to `dist/frontend.js` (ESM, browser), then copies `locales/` and `migrations/` into `dist/`. Imported CSS lands in `dist/frontend.css`. |
| `termix-plugin validate` | Runs the SDK's `parseManifest`, checks the files the manifest names exist, and checks that migrations only touch this plugin's tables and exist for every dialect. |
| `termix-plugin migrations` | Diffs `src/backend/tables.ts` against `migrations/snapshot.json` and writes one `.sql` per dialect. `--check` fails when a definition changed without a migration. A rename or a type change is refused rather than guessed at, because that is a data decision. |
| `termix-plugin test` | Runs the plugin's vitest suite. |
| `termix-plugin pack` | Writes a `.tgz` of the manifest, `dist/`, locales, migrations and README. **D3** adds signing. |
`npm run build:plugins` builds every plugin and stages the result in
`dist/plugins/<id>/`, which is what `getBundledPluginsDir()` resolves to and
@@ -0,0 +1,227 @@
/**
* Generates a plugin's migrations from its table definitions.
*
* Not drizzle-kit. Core's own SQLite schema is hand-written DDL and only the
* client-server engines use drizzle migrations, so a plugin on drizzle-kit
* would still need a hand-rolled SQLite path, three configs and three
* generated schema modules. This diffs the definitions against a committed
* snapshot and writes plain SQL for all three dialects, which is what the
* server's runner applies.
*
* Usage, from a plugin directory:
* termix-plugin migrations [name]
* termix-plugin migrations --check verifies nothing is undeclared
*
* The plugin exports its definitions from src/backend/tables.ts as `tables`.
*/
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import * as esbuild from "esbuild";
import { BACKEND_EXTERNALS } from "../lib/externals.mjs";
import { readManifest, resolveEntry } from "../lib/plugin-dir.mjs";
const DIALECTS = ["sqlite", "postgres", "mysql"];
const TABLE_ENTRIES = [
"src/backend/tables.ts",
"src/backend/tables.mjs",
"src/backend/tables.js",
];
/** Loads the plugin's table definitions by bundling them to a temp file. */
async function loadDefinitions(cwd) {
const entry = resolveEntry(cwd, TABLE_ENTRIES);
if (!entry) return null;
// Emitted inside the plugin rather than in the system temp dir: the SDK
// stays external to the bundle, so the output has to sit somewhere node can
// resolve @termix/plugin-sdk from.
const outdir = fs.mkdtempSync(path.join(cwd, ".termix-tables-"));
const outfile = path.join(outdir, "tables.mjs");
try {
await esbuild.build({
entryPoints: [entry],
outfile,
bundle: true,
format: "esm",
platform: "node",
target: "node22",
logLevel: "warning",
external: BACKEND_EXTERNALS,
});
const module = await import(`file://${outfile.split(path.sep).join("/")}`);
const tables = module.tables ?? module.default;
if (!Array.isArray(tables)) {
throw new Error(
`${path.relative(cwd, entry)} must export a "tables" array of defineTable() results`,
);
}
return tables;
} finally {
fs.rmSync(outdir, { recursive: true, force: true });
}
}
function snapshotPath(cwd) {
return path.join(cwd, "migrations", "snapshot.json");
}
function readSnapshot(cwd) {
const file = snapshotPath(cwd);
if (!fs.existsSync(file)) return { version: 1, tables: {} };
return JSON.parse(fs.readFileSync(file, "utf8"));
}
/** What a definition looks like in the snapshot, so a diff can compare them. */
function snapshotTable(definition) {
return {
columns: definition.columns,
indexes: definition.indexes,
};
}
function nextSequence(cwd) {
let highest = 0;
for (const dialect of DIALECTS) {
const dir = path.join(cwd, "migrations", dialect);
if (!fs.existsSync(dir)) continue;
for (const file of fs.readdirSync(dir)) {
const match = /^(\d{4})_/.exec(file);
if (match) highest = Math.max(highest, Number(match[1]));
}
}
return highest + 1;
}
/**
* What changed since the snapshot.
*
* New tables and new columns are generated. A changed or removed column is
* reported rather than guessed at: renames and type changes need a data
* decision this cannot make, so the author writes that migration by hand.
*/
function diff(previous, definitions) {
const created = [];
const added = [];
const manual = [];
for (const definition of definitions) {
const before = previous.tables[definition.name];
if (!before) {
created.push(definition);
continue;
}
for (const [property, column] of Object.entries(definition.columns)) {
const existing = before.columns[property];
if (!existing) {
added.push({ definition, property, column });
continue;
}
if (JSON.stringify(existing) !== JSON.stringify(column)) {
manual.push(`${definition.name}.${property} changed type or modifiers`);
}
}
for (const property of Object.keys(before.columns)) {
if (!definition.columns[property]) {
manual.push(`${definition.name}.${property} was removed`);
}
}
}
for (const name of Object.keys(previous.tables)) {
if (!definitions.some((definition) => definition.name === name)) {
manual.push(`table ${name} was removed`);
}
}
return { created, added, manual };
}
export async function migrations({ cwd, args }) {
const manifest = readManifest(cwd);
const pluginId = manifest.id ?? path.basename(cwd);
const check = args.includes("--check");
const name = args.find((arg) => !arg.startsWith("--")) ?? "update";
const definitions = await loadDefinitions(cwd);
if (definitions === null) {
console.log(`${pluginId}: no table definitions, nothing to generate`);
return;
}
const { createTableSql, addColumnSql } = await loadEmitter();
const previous = readSnapshot(cwd);
const { created, added, manual } = diff(previous, definitions);
if (manual.length > 0) {
for (const problem of manual) console.error(` ${problem}`);
throw new Error(
`${pluginId}: these changes need a hand-written migration, because a rename or a type change is a data decision.`,
);
}
if (created.length === 0 && added.length === 0) {
console.log(`${pluginId}: migrations are up to date`);
return;
}
if (check) {
throw new Error(
`${pluginId}: table definitions have changed but no migration was generated. Run: termix-plugin migrations`,
);
}
const sequence = String(nextSequence(cwd)).padStart(4, "0");
const file = `${sequence}_${name.replace(/[^a-z0-9_]/gi, "_").toLowerCase()}.sql`;
for (const dialect of DIALECTS) {
const statements = [];
for (const definition of created) {
statements.push(...createTableSql(dialect, pluginId, definition));
}
for (const entry of added) {
statements.push(
addColumnSql(dialect, pluginId, entry.definition, entry.property),
);
}
const dir = path.join(cwd, "migrations", dialect);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(
path.join(dir, file),
`-- ${pluginId} ${sequence}: ${name}\n-- Generated by termix-plugin migrations. Review before committing.\n\n${statements.join("\n")}\n`,
);
}
const snapshot = { version: 1, tables: {} };
for (const definition of definitions) {
snapshot.tables[definition.name] = snapshotTable(definition);
}
fs.writeFileSync(snapshotPath(cwd), `${JSON.stringify(snapshot, null, 2)}\n`);
console.log(
`${pluginId}: wrote ${DIALECTS.length} dialect files for ${sequence}_${name}`,
);
}
/**
* The DDL emitter, shared with the server.
*
* One mapping, two consumers: the SQL written here is the SQL the server's
* migration runner applies, so a second copy could drift into producing a
* migration the runner cannot reproduce.
*/
async function loadEmitter() {
const entry = new URL("../../dist/ddl.js", import.meta.url);
if (!fs.existsSync(fileURLToPath(entry))) {
throw new Error("The plugin SDK is not built yet. Run: npm run build:sdk");
}
return import(entry.href);
}
@@ -2,6 +2,7 @@ import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { readManifest } from "../lib/plugin-dir.mjs";
import { LEGACY_TABLE_OWNERS as LEGACY_TABLES } from "../../dist/db.js";
/**
* The manifest rules live in src/manifest.ts, which the server uses too, so
@@ -35,6 +36,8 @@ export async function validate({ cwd }) {
}
}
problems.push(...validateMigrations(cwd, raw.id ?? path.basename(cwd)));
if (problems.length > 0) {
for (const problem of problems) console.error(` ${problem}`);
throw new Error(`${raw.id ?? path.basename(cwd)}: manifest is not valid.`);
@@ -42,3 +45,77 @@ export async function validate({ cwd }) {
console.log(`ok ${raw.id ?? path.basename(cwd)}`);
}
const DIALECTS = ["sqlite", "postgres", "mysql"];
const CREATE_OR_ALTER =
/(?:CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TABLE(?:\s+IF\s+NOT\s+EXISTS)?|ALTER\s+TABLE|DROP\s+TABLE(?:\s+IF\s+EXISTS)?|TRUNCATE\s+TABLE)\s+([`"']?)([a-zA-Z0-9_]+)\1/gi;
/**
* Checks that a plugin's migrations only touch tables it owns.
*
* The server enforces this too, because a plugin installed from a tarball
* never ran this command. Here it is a build-time error with a file name
* attached, rather than a plugin that fails to activate later.
*/
function validateMigrations(cwd, pluginId) {
const problems = [];
const prefix = `p_${String(pluginId).replace(/-/g, "_")}_`;
const root = path.join(cwd, "migrations");
if (!fs.existsSync(root)) return problems;
const present = DIALECTS.filter((dialect) =>
fs.existsSync(path.join(root, dialect)),
);
const byDialect = new Map();
for (const dialect of present) {
const dir = path.join(root, dialect);
const files = fs
.readdirSync(dir)
.filter((file) => file.endsWith(".sql"))
.sort();
byDialect.set(dialect, files);
for (const file of files) {
if (!/^\d{4}_[a-z0-9_]+\.sql$/.test(file)) {
problems.push(
`migrations/${dialect}/${file} must be named NNNN_name.sql, lower snake_case`,
);
}
const sql = fs.readFileSync(path.join(dir, file), "utf8");
CREATE_OR_ALTER.lastIndex = 0;
let match;
while ((match = CREATE_OR_ALTER.exec(sql)) !== null) {
const table = match[2];
if (table.startsWith(prefix)) continue;
if (LEGACY_TABLES[table] === pluginId) continue;
problems.push(
`migrations/${dialect}/${file} touches "${table}", which is not prefixed "${prefix}"`,
);
}
}
}
// A migration that exists for one engine and not another leaves that
// deployment without the table, which is how alert_rules ended up
// SQLite-only in core.
const [first, ...rest] = present;
for (const dialect of rest) {
const a = byDialect.get(first);
const b = byDialect.get(dialect);
for (const file of a) {
if (!b.includes(file)) {
problems.push(`migrations/${dialect}/${file} is missing`);
}
}
for (const file of b) {
if (!a.includes(file)) {
problems.push(`migrations/${first}/${file} is missing`);
}
}
}
return problems;
}
+3 -1
View File
@@ -10,8 +10,9 @@ import { build } from "./commands/build.mjs";
import { validate } from "./commands/validate.mjs";
import { test } from "./commands/test.mjs";
import { pack } from "./commands/pack.mjs";
import { migrations } from "./commands/migrations.mjs";
const COMMANDS = { build, validate, test, pack };
const COMMANDS = { build, validate, test, pack, migrations };
const [command, ...args] = process.argv.slice(2);
@@ -24,6 +25,7 @@ if (!command || command === "--help" || command === "-h") {
" validate Check manifest.json and the files it names",
" test Run the plugin's vitest suite",
" pack Write a .tgz of the built plugin",
" migrations Generate migrations from the plugin's table definitions",
].join("\n"),
);
process.exit(command ? 0 : 1);
+8
View File
@@ -12,6 +12,14 @@
"types": "./dist/backend.d.ts",
"default": "./dist/backend.js"
},
"./db": {
"types": "./dist/db.d.ts",
"default": "./dist/db.js"
},
"./ddl": {
"types": "./dist/ddl.d.ts",
"default": "./dist/ddl.js"
},
"./frontend": {
"types": "./dist/frontend.d.ts",
"default": "./dist/frontend.js"
+73
View File
@@ -11,6 +11,7 @@
*/
import type { PluginManifest } from "./manifest.js";
import type { PluginTableDefinition } from "./db.js";
/**
* Thrown when a plugin calls a guarded member without the capability.
@@ -56,6 +57,75 @@ export interface PluginKeyValue {
list: () => Promise<string[]>;
}
/**
* The plugin's own tables.
*
* `define` registers a table definition and returns the Drizzle table object
* to query through `db`. Both require db:own.
*
* Scoped by name: every table carries the p_<id>_ prefix, and `refs` exposes
* users and ssh_data read-only so a plugin can join against them without being
* able to write them. In-process code could reach around all of this - see
* "What this protects, and what it does not" in ARCHITECTURE.md. The
* capability, the prefix, lint and review are the contract, not a sandbox.
*/
export interface PluginDatabase {
/** Registers a definition and returns its queryable table object. */
define: <T = unknown>(definition: PluginTableDefinition) => Promise<T>;
/** Drizzle handle, scoped to this plugin's tables. */
client: <T = unknown>() => Promise<T>;
/** Read-only references to the core tables a plugin may point at. */
refs: <T = unknown>() => Promise<T>;
}
/** A row's shape on the wire, before it is written to a table. */
export type SyncRow = Record<string, unknown>;
export interface SyncEntityReference {
/** Stored column holding a local numeric id, e.g. "credentialId". */
field: string;
/** Wire field holding the portable id, e.g. "credentialSyncId". */
syncField: string;
/** The entity the id points at. */
entityType: string;
}
export interface SyncEntityRegistration {
/** Stable wire name. Never change it: tombstones and remote rows match on it. */
type: string;
/** The table, as returned by ctx.db.define. */
table: unknown;
/** Column holding the owning user id. Defaults to "userId". */
userColumn?: string;
/** Fields DataCrypto translates between plaintext wire and encrypted row. */
encryptedFields?: readonly string[];
/** Local-id to sync-id translations applied on the way out and back. */
references?: readonly SyncEntityReference[];
/** Lower sorts first. Reference targets must sort before their referrers. */
order?: number;
/** Fields that must never be overwritten by an inbound payload. */
readOnlyFields?: readonly string[];
/** One row per user rather than many, keyed on the owner. */
singleton?: boolean;
/**
* Escape hatch for a row whose references are not plain columns, such as ids
* embedded in a JSON blob. Runs instead of `references`, not alongside it.
*/
serialize?: (
row: SyncRow,
resolveSyncId: (entityType: string, id: number) => Promise<string | null>,
) => Promise<SyncRow>;
deserialize?: (
row: SyncRow,
resolveId: (entityType: string, syncId: string) => Promise<number | null>,
) => Promise<SyncRow>;
}
/** Adds an entity to remote sync between a desktop backend and a server. */
export interface PluginSync {
registerEntity: (entity: SyncEntityRegistration) => void;
}
export interface PluginRegistry {
provide: <T>(key: string, value: T) => void;
consume: <T>(key: string) => T | undefined;
@@ -95,6 +165,8 @@ export interface PluginContext {
readonly log: PluginLogger;
readonly events: PluginEvents;
readonly kv: PluginKeyValue;
readonly db: PluginDatabase;
readonly sync: PluginSync;
readonly registry: PluginRegistry;
readonly services: PluginServices;
readonly secrets: PluginSecrets;
@@ -133,3 +205,4 @@ export function definePlugin(plugin: PluginModule): PluginModule {
}
export type { PluginManifest } from "./manifest.js";
export type { PluginTableDefinition } from "./db.js";
+263
View File
@@ -0,0 +1,263 @@
/**
* Table definitions a plugin owns.
*
* A definition is a plain descriptor, not a live Drizzle table. Core turns it
* into one at runtime (src/backend/plugins/table-builder.ts) and emits the DDL
* for whichever dialect the server is on, which keeps this entry a description
* of the contract rather than an implementation a plugin could reach around.
*
* Why one table object and three DDL emitters, rather than three table
* objects: the sqlite-core definitions already encode correctly for every
* engine at query time. text and integer encode as themselves; a boolean
* writes 1/0, which Postgres and MySQL both accept; real is a plain number.
* What genuinely differs between the engines is DDL. See the header of
* scripts/generate-dialect-schema.cjs, which this mapping is taken from.
*/
export type PluginColumnType =
| "id"
| "text"
| "varchar"
| "integer"
| "bigint"
| "boolean"
| "timestamp"
| "json"
| "encryptedText"
| "refUser"
| "refHost";
export interface PluginColumn {
readonly type: PluginColumnType;
/** varchar only. The ceiling MySQL can index. */
readonly length?: number;
readonly notNull?: boolean;
readonly primaryKey?: boolean;
readonly unique?: boolean;
readonly defaultValue?: string | number | boolean | null;
/** timestamp only: default CURRENT_TIMESTAMP. */
readonly defaultNow?: boolean;
}
/** The longest string MySQL will index, and so the width of every key column. */
export const KEY_LENGTH = 255;
interface Builder<T extends PluginColumn = PluginColumn> {
notNull: () => Builder<T>;
primaryKey: () => Builder<T>;
unique: () => Builder<T>;
default: (value: string | number | boolean | null) => Builder<T>;
defaultNow: () => Builder<T>;
readonly column: PluginColumn;
}
function builder(column: PluginColumn): Builder {
const next = (patch: Partial<PluginColumn>) =>
builder({ ...column, ...patch });
return {
column,
notNull: () => next({ notNull: true }),
primaryKey: () => next({ primaryKey: true, notNull: true }),
unique: () => next({ unique: true }),
default: (defaultValue) => next({ defaultValue }),
defaultNow: () => next({ defaultNow: true }),
};
}
/** Auto-incrementing surrogate key: integer on sqlite, serial, auto_increment. */
export const id = () =>
builder({ type: "id", primaryKey: true, notNull: true });
/** Unbounded string. Never indexable: MySQL cannot index TEXT. */
export const text = () => builder({ type: "text" });
/** Bounded string. Indexable on every engine. */
export const varchar = (length: number = KEY_LENGTH) =>
builder({ type: "varchar", length });
export const integer = () => builder({ type: "integer" });
export const bigint = () => builder({ type: "bigint" });
export const boolean = () => builder({ type: "boolean" });
/** Stored as text on every engine. See src/backend/database/repositories/sql-timestamp.ts. */
export const timestamp = () => builder({ type: "timestamp" });
/** JSON held as text. Never indexable, for the same reason as text(). */
export const json = () => builder({ type: "json" });
/** Text encrypted with the owning user's key before it is written. */
export const encryptedText = () => builder({ type: "encryptedText" });
/** Foreign key to users.id, cascading on delete. */
export const refUser = () =>
builder({ type: "refUser", length: KEY_LENGTH, notNull: true });
/** Foreign key to ssh_data.id, cascading on delete. */
export const refHost = () => builder({ type: "refHost", notNull: true });
export interface PluginTableIndex {
readonly name: string;
readonly columns: readonly string[];
readonly unique?: boolean;
}
export interface PluginTableOptions {
readonly indexes?: readonly PluginTableIndex[];
readonly uniques?: readonly PluginTableIndex[];
}
export interface PluginTableDefinition {
/** The unprefixed name as the plugin wrote it. */
readonly name: string;
readonly columns: Readonly<Record<string, PluginColumn>>;
readonly indexes: readonly PluginTableIndex[];
}
/** Columns that can never carry an index, because MySQL cannot index them. */
const UNINDEXABLE: ReadonlySet<PluginColumnType> = new Set([
"text",
"json",
"encryptedText",
]);
const NAME_PATTERN = /^[a-z][a-z0-9_]*$/;
type Columns = Record<string, Builder | PluginColumn>;
function unwrap(value: Builder | PluginColumn): PluginColumn {
return "column" in value ? value.column : value;
}
/**
* Declares a table this plugin owns.
*
* The name is unprefixed here; core prefixes it with p_<plugin id>_ when the
* definition is registered, so a plugin cannot name a table into another
* plugin's namespace or into core's.
*/
export function defineTable(
name: string,
columns: Columns,
options: PluginTableOptions = {},
): PluginTableDefinition {
if (!NAME_PATTERN.test(name)) {
throw new Error(
`Table name "${name}" must be lower snake_case and start with a letter`,
);
}
const resolved: Record<string, PluginColumn> = {};
for (const [key, value] of Object.entries(columns)) {
resolved[key] = unwrap(value);
}
if (Object.keys(resolved).length === 0) {
throw new Error(`Table "${name}" declares no columns`);
}
const indexes = [
...(options.uniques ?? []).map((entry) => ({ ...entry, unique: true })),
...(options.indexes ?? []),
];
// Checked here rather than at migration time so the error names the table
// and column while the author is still looking at the definition.
for (const index of indexes) {
if (!NAME_PATTERN.test(index.name)) {
throw new Error(
`Index name "${index.name}" must be lower snake_case and start with a letter`,
);
}
if (index.columns.length === 0) {
throw new Error(`Index "${index.name}" names no columns`);
}
for (const column of index.columns) {
const declared = resolved[column];
if (!declared) {
throw new Error(
`Index "${index.name}" names "${column}", which table "${name}" does not declare`,
);
}
if (UNINDEXABLE.has(declared.type)) {
throw new Error(
`Index "${index.name}" covers "${column}", a ${declared.type} column. ` +
`MySQL cannot index an unbounded column: use varchar(n) instead.`,
);
}
}
}
return { name, columns: resolved, indexes };
}
/** p_<id with - as _>_ - the prefix every table a plugin owns carries. */
export function tablePrefix(pluginId: string): string {
return `p_${pluginId.replace(/-/g, "_")}_`;
}
/** The physical table name for a definition owned by a plugin. */
export function prefixedTableName(pluginId: string, name: string): string {
return `${tablePrefix(pluginId)}${name}`;
}
/**
* Legacy core tables, and the one plugin allowed to adopt each.
*
* Adoption is how a feature keeps its rows when it moves out of core: the
* table is renamed into the plugin's namespace rather than copied. One owner
* per table, because two plugins adopting the same one would mean whichever
* activated second found it already gone.
*
* Shared with the CLI so a migration that reaches for someone else's table is
* caught at build time as well as at activation.
*/
export const LEGACY_TABLE_OWNERS: Readonly<Record<string, string>> = {
fleets: "fleets",
fleet_members: "fleets",
fleet_inventory: "fleets",
user_workspaces: "workspaces",
network_topology: "network-topology",
snippets: "snippets",
snippet_folders: "snippets",
snippet_access: "snippets",
c2s_tunnel_presets: "tunnels",
command_history: "ssh-terminal",
file_manager_recent: "file-manager",
file_manager_pinned: "file-manager",
file_manager_shortcuts: "file-manager",
transfer_recent: "file-manager",
tmux_session_tags: "tmux-monitor",
session_shares: "session-sharing",
session_share_participants: "session-sharing",
collab_rooms: "session-sharing",
collab_room_members: "session-sharing",
session_recordings: "session-recording",
host_metrics_preferences: "host-metrics",
host_health_checks: "host-metrics",
host_health_history: "host-metrics",
host_metrics_history: "host-metrics",
proxmox_stats_preferences: "proxmox",
proxmox_node_history: "proxmox",
automations: "automations",
automation_trigger_state: "automations",
automation_schedules: "automations",
automation_runs: "automations",
automation_run_steps: "automations",
automation_channels: "automations",
ai_providers: "ai",
ai_conversations: "ai",
ai_messages: "ai",
ai_proposals: "ai",
homepage_items: "homepage",
homepage_layouts: "homepage",
dashboard_service_links: "homepage",
secret_sources: "secret-sources",
opkssh_tokens: "opkssh",
vault_profiles: "vault",
vault_tokens: "vault",
termix_identities: "termix-identity",
termix_identity_keys: "termix-identity",
termix_identity_ca: "termix-identity",
webauthn_credentials: "webauthn",
sso_providers: "sso",
};
+220
View File
@@ -0,0 +1,220 @@
/**
* The DDL a table definition becomes, per dialect.
*
* Lives in the SDK because two things need the identical mapping: the CLI,
* which writes a plugin's migration files, and the server, which applies them.
* A second copy would drift and produce a migration the runner could not
* reproduce.
*
* The mapping itself is the one scripts/generate-dialect-schema.cjs applies to
* core's schema, kept deliberately in step with it.
*/
import { KEY_LENGTH, prefixedTableName } from "./db.js";
import type {
PluginColumn,
PluginTableDefinition,
PluginTableIndex,
} from "./db.js";
/** Which engine the DDL is for. Matches core's DatabaseDialect exactly. */
export type SqlDialect = "sqlite" | "postgres" | "mysql";
/** The core tables a plugin may point a foreign key at, and nothing else. */
export const REFERENCEABLE = {
refUser: { table: "users", column: "id" },
refHost: { table: "ssh_data", column: "id" },
} as const;
function camelToSnake(value: string): string {
return value.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
}
/** The SQL column name for a declared property. */
export function columnName(property: string, column?: PluginColumn): string {
void column;
return camelToSnake(property);
}
function quote(dialect: SqlDialect, identifier: string): string {
if (dialect === "mysql") return "`" + identifier + "`";
return '"' + identifier + '"';
}
/**
* The physical type for a column on one dialect.
*
* Mirrors generate-dialect-schema.cjs: an autoincrement key is serial /
* auto_increment, a boolean is native off sqlite, and anything carrying a key
* has to be varchar because MySQL will not index TEXT.
*/
function columnType(
dialect: SqlDialect,
column: PluginColumn,
keyed: boolean,
): string {
switch (column.type) {
case "id":
if (dialect === "postgres") return "serial";
if (dialect === "mysql") return "int AUTO_INCREMENT";
return "integer";
case "integer":
return dialect === "mysql" ? "int" : "integer";
case "bigint":
return dialect === "sqlite" ? "integer" : "bigint";
case "boolean":
return dialect === "sqlite" ? "integer" : "boolean";
case "varchar":
return dialect === "sqlite"
? "text"
: `varchar(${column.length ?? KEY_LENGTH})`;
case "refUser":
return dialect === "sqlite" ? "text" : `varchar(${KEY_LENGTH})`;
case "refHost":
return dialect === "mysql" ? "int" : "integer";
default:
// A keyed string has to be indexable, so it narrows to varchar even
// though it was declared unbounded. defineTable refuses to index a
// text/json column, so in practice this only catches a primary key.
if (keyed && dialect !== "sqlite") return `varchar(${KEY_LENGTH})`;
return "text";
}
}
function defaultClause(dialect: SqlDialect, column: PluginColumn): string {
if (column.defaultNow) {
// MySQL rejects a bare DEFAULT CURRENT_TIMESTAMP on a text column; an
// expression default is written parenthesised. MariaDB takes either.
return dialect === "mysql"
? " DEFAULT (CURRENT_TIMESTAMP)"
: " DEFAULT CURRENT_TIMESTAMP";
}
if (column.defaultValue === undefined) return "";
const value = column.defaultValue;
if (value === null) return " DEFAULT NULL";
if (typeof value === "boolean") {
return dialect === "sqlite"
? ` DEFAULT ${value ? 1 : 0}`
: ` DEFAULT ${value ? "true" : "false"}`;
}
if (typeof value === "number") return ` DEFAULT ${value}`;
return ` DEFAULT '${String(value).replace(/'/g, "''")}'`;
}
/** Every column that carries a key, and so must be indexable on MySQL. */
export function keyedColumns(
definition: PluginTableDefinition,
): ReadonlySet<string> {
const keyed = new Set<string>();
for (const [property, column] of Object.entries(definition.columns)) {
if (column.primaryKey || column.unique) keyed.add(property);
if (column.type === "refUser" || column.type === "refHost") {
keyed.add(property);
}
}
for (const index of definition.indexes) {
for (const property of index.columns) keyed.add(property);
}
return keyed;
}
function indexStatement(
dialect: SqlDialect,
physical: string,
definition: PluginTableDefinition,
entry: PluginTableIndex,
): string {
const columns = entry.columns
.map((property) =>
quote(dialect, columnName(property, definition.columns[property])),
)
.join(", ");
const unique = entry.unique ? "UNIQUE " : "";
// MySQL has no CREATE INDEX IF NOT EXISTS. A plugin migration runs once,
// recorded in plugin_migrations, so the guard is the ledger, not the DDL.
const guard = dialect === "mysql" ? "" : "IF NOT EXISTS ";
return `CREATE ${unique}INDEX ${guard}${quote(dialect, entry.name)} ON ${quote(dialect, physical)} (${columns});`;
}
/** The CREATE TABLE and CREATE INDEX statements for one definition. */
export function createTableSql(
dialect: SqlDialect,
pluginId: string,
definition: PluginTableDefinition,
): string[] {
const physical = prefixedTableName(pluginId, definition.name);
const keyed = keyedColumns(definition);
const lines: string[] = [];
const constraints: string[] = [];
for (const [property, column] of Object.entries(definition.columns)) {
const name = columnName(property, column);
const type = columnType(dialect, column, keyed.has(property));
let line = ` ${quote(dialect, name)} ${type}`;
if (column.type === "id") {
line += " PRIMARY KEY";
if (dialect === "sqlite") line += " AUTOINCREMENT";
} else {
if (column.primaryKey) line += " PRIMARY KEY";
if (column.notNull) line += " NOT NULL";
if (column.unique) line += " UNIQUE";
line += defaultClause(dialect, column);
}
lines.push(line);
const reference =
column.type === "refUser"
? REFERENCEABLE.refUser
: column.type === "refHost"
? REFERENCEABLE.refHost
: null;
if (reference) {
constraints.push(
` FOREIGN KEY (${quote(dialect, name)}) REFERENCES ${quote(dialect, reference.table)} (${quote(dialect, reference.column)}) ON DELETE CASCADE`,
);
}
}
const body = [...lines, ...constraints].join(",\n");
const statements = [
`CREATE TABLE IF NOT EXISTS ${quote(dialect, physical)} (\n${body}\n);`,
];
for (const entry of definition.indexes) {
statements.push(indexStatement(dialect, physical, definition, entry));
}
return statements;
}
/** DROP statement for a definition, for removePluginData. */
/** DROP statement for a definition, for removePluginData. */
export function dropTableSql(
dialect: SqlDialect,
pluginId: string,
definition: PluginTableDefinition,
): string {
const physical = prefixedTableName(pluginId, definition.name);
return `DROP TABLE IF EXISTS ${quote(dialect, physical)};`;
}
/** The ALTER that adds one newly declared column. */
export function addColumnSql(
dialect: SqlDialect,
pluginId: string,
definition: PluginTableDefinition,
property: string,
): string {
const physical = prefixedTableName(pluginId, definition.name);
const column = definition.columns[property];
const keyed = keyedColumns(definition).has(property);
const name = columnName(property, column);
let line = `${quote(dialect, name)} ${columnType(dialect, column, keyed)}`;
if (column.notNull) line += " NOT NULL";
line += defaultClause(dialect, column);
return `ALTER TABLE ${quote(dialect, physical)} ADD COLUMN ${line};`;
}
+46 -1
View File
@@ -16,6 +16,8 @@ import type {
} from "./backend.js";
import type { TermixApp } from "./frontend.js";
import type { PluginManifest } from "./manifest.js";
import type { PluginTableDefinition } from "./db.js";
import type { SyncEntityRegistration } from "./backend.js";
export interface FakeContextOptions {
pluginId?: string;
@@ -32,6 +34,10 @@ export interface FakePluginContext {
emitted: Array<{ topic: string; payload: unknown }>;
/** Backing store behind ctx.kv. */
kv: Map<string, unknown>;
/** Table definitions registered through ctx.db.define, in order. */
tables: PluginTableDefinition[];
/** Sync entities registered through ctx.sync.registerEntity, in order. */
syncEntities: SyncEntityRegistration[];
}
function noopLogger(): PluginLogger {
@@ -55,6 +61,8 @@ export function createFakeContext(
const disposals: Array<() => void | Promise<void>> = [];
const emitted: Array<{ topic: string; payload: unknown }> = [];
const kv = new Map<string, unknown>();
const tables: PluginTableDefinition[] = [];
const syncEntities: SyncEntityRegistration[] = [];
const listeners = new Map<string, Set<(payload: unknown) => void>>();
let actor = options.actor;
@@ -105,6 +113,27 @@ export function createFakeContext(
list: async () => [...kv.keys()],
},
db: {
// No engine behind this one. A test that needs real SQL should drive the
// runtime rather than the double.
define: async (definition) => {
tables.push(definition);
return undefined as never;
},
client: async () => {
throw new Error("createFakeContext does not implement db.client");
},
refs: async () => {
throw new Error("createFakeContext does not implement db.refs");
},
},
sync: {
registerEntity: (entity) => {
syncEntities.push(entity);
},
},
registry: {
provide: () => {},
consume: () => undefined,
@@ -139,7 +168,7 @@ export function createFakeContext(
currentActor: () => actor,
};
return { ctx, disposals, emitted, kv };
return { ctx, disposals, emitted, kv, tables, syncEntities };
}
export interface MockContextOptions {
@@ -203,9 +232,25 @@ export function createMockCtx(
};
const guardedKv = ctx.kv;
const guardedDb = ctx.db;
const gatedCtx: PluginContext = {
...ctx,
db: {
define: async (definition) => {
require("db:own");
return guardedDb.define(definition);
},
client: async () => {
require("db:own");
return guardedDb.client();
},
refs: async () => {
require("db:own");
return guardedDb.refs();
},
},
kv: {
get: async (key) => {
require("kv:own");
+50
View File
@@ -132,6 +132,56 @@ check(
await hosts.deleteForUser(userId, host.id);
check("host really deleted", await hosts.findById(host.id), null);
// A plugin's own tables. The DDL is emitted from a table definition rather
// than written by hand, so this is what proves the emitter's output is valid
// on a real engine - the unit tests can only snapshot the string.
const { defineTable, id, varchar, text, refUser, timestamp } = await import(
"../packages/plugin-sdk/dist/db.js"
);
const { createTableSql, dropTableSql } = await import(
"../packages/plugin-sdk/dist/ddl.js"
);
const { sql: raw } = await import("drizzle-orm");
const pluginTable = defineTable(
"verify_note",
{
id: id(),
userId: refUser(),
title: varchar(200).notNull(),
body: text(),
createdAt: timestamp().notNull().defaultNow(),
},
{ indexes: [{ name: "idx_verify_note_user", columns: ["userId"] }] },
);
const pluginId = "verify";
try {
for (const statement of createTableSql(dialect, pluginId, pluginTable)) {
await db.execute(raw.raw(statement));
}
check("plugin table DDL is accepted", true, true);
await db.execute(
raw.raw(
`INSERT INTO p_verify_verify_note (user_id, title) VALUES ('${userId}', 'from verify')`,
),
);
const rows = await db.execute(
raw.raw("SELECT title FROM p_verify_verify_note"),
);
const list = Array.isArray(rows) ? rows : (rows?.rows ?? []);
check("plugin table round-trips a row", list.length > 0, true);
} catch (error) {
check(`plugin table DDL is accepted (${error.message})`, false, true);
} finally {
try {
await db.execute(raw.raw(dropTableSql(dialect, pluginId, pluginTable)));
} catch {
// Best effort: the scratch database is the caller's to clean.
}
}
console.log(
failures === 0
? `\n${dialect}: all checks passed\n`
+22 -69
View File
@@ -741,6 +741,16 @@ async function initializeCompleteDatabase(): Promise<void> {
FOREIGN KEY (plugin_id) REFERENCES plugins (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS plugin_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
plugin_id TEXT NOT NULL,
migration_id TEXT NOT NULL,
checksum TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (plugin_id, migration_id),
FOREIGN KEY (plugin_id) REFERENCES plugins (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS api_keys (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
@@ -2440,29 +2450,20 @@ const migrateSchema = () => {
// --- proxmox-node-history end ---
// --- alerts begin ---
try {
sqlite.prepare("SELECT id FROM alert_rules LIMIT 1").get();
} catch {
// alert_rules, alert_rule_channels and alert_firings were only ever created
// here, never declared in schema.ts, so they never existed on Postgres or
// MySQL at all. Automations replaced them; drop the SQLite leftovers.
for (const table of [
"alert_firings",
"alert_rule_channels",
"alert_rules",
]) {
try {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS alert_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
host_id INTEGER REFERENCES ssh_data(id) ON DELETE CASCADE,
name TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
trigger_type TEXT NOT NULL,
threshold_value REAL,
threshold_duration_seconds INTEGER,
cooldown_minutes INTEGER NOT NULL DEFAULT 15,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`);
} catch (createError) {
databaseLogger.warn("Failed to create alert_rules table", {
sqlite.exec(`DROP TABLE IF EXISTS ${table};`);
} catch (dropError) {
databaseLogger.warn(`Failed to drop legacy ${table} table`, {
operation: "schema_migration",
error: createError,
error: dropError,
});
}
}
@@ -2489,54 +2490,6 @@ const migrateSchema = () => {
});
}
}
try {
sqlite.prepare("SELECT id FROM alert_rule_channels LIMIT 1").get();
} catch {
try {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS alert_rule_channels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id INTEGER NOT NULL REFERENCES alert_rules(id) ON DELETE CASCADE,
channel_id INTEGER NOT NULL REFERENCES notification_channels(id) ON DELETE CASCADE
);
`);
} catch (createError) {
databaseLogger.warn("Failed to create alert_rule_channels table", {
operation: "schema_migration",
error: createError,
});
}
}
try {
sqlite.prepare("SELECT id FROM alert_firings LIMIT 1").get();
} catch {
try {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS alert_firings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
rule_id INTEGER NOT NULL REFERENCES alert_rules(id) ON DELETE CASCADE,
host_id INTEGER NOT NULL,
host_name TEXT NOT NULL,
fired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
resolved_at TEXT,
value REAL,
message TEXT NOT NULL,
severity TEXT NOT NULL DEFAULT 'warning',
acknowledged INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_alert_firings_user_fired
ON alert_firings (user_id, fired_at DESC);
`);
} catch (createError) {
databaseLogger.warn("Failed to create alert_firings table", {
operation: "schema_migration",
error: createError,
});
}
}
// --- alerts end ---
// --- automations begin ---
+29
View File
@@ -2252,4 +2252,33 @@ export const pluginStorage = mysqlTable(
],
);
/**
* Which of a plugin's migrations have been applied.
*
* The checksum is what makes an already-applied migration immutable: editing
* one that has run blocks that plugin rather than silently leaving two
* databases with different shapes. Cascades with the plugin so removing its
* data leaves no ledger behind.
*/
export const pluginMigrations = mysqlTable(
"plugin_migrations",
{
id: int("id").autoincrement().primaryKey(),
pluginId: varchar("plugin_id", { length: 255 })
.notNull()
.references(() => plugins.id, { onDelete: "cascade" }),
migrationId: varchar("migration_id", { length: 255 }).notNull(),
checksum: text("checksum").notNull(),
appliedAt: text("applied_at")
.notNull()
.default(sql`(CURRENT_TIMESTAMP)`),
},
(table) => [
uniqueIndex("idx_plugin_migrations_plugin_migration").on(
table.pluginId,
table.migrationId,
),
],
);
// --- plugins end ---
+29
View File
@@ -2253,4 +2253,33 @@ export const pluginStorage = pgTable(
],
);
/**
* Which of a plugin's migrations have been applied.
*
* The checksum is what makes an already-applied migration immutable: editing
* one that has run blocks that plugin rather than silently leaving two
* databases with different shapes. Cascades with the plugin so removing its
* data leaves no ledger behind.
*/
export const pluginMigrations = pgTable(
"plugin_migrations",
{
id: serial("id").primaryKey(),
pluginId: varchar("plugin_id", { length: 255 })
.notNull()
.references(() => plugins.id, { onDelete: "cascade" }),
migrationId: varchar("migration_id", { length: 255 }).notNull(),
checksum: text("checksum").notNull(),
appliedAt: text("applied_at")
.notNull()
.default(sql`CURRENT_TIMESTAMP`),
},
(table) => [
uniqueIndex("idx_plugin_migrations_plugin_migration").on(
table.pluginId,
table.migrationId,
),
],
);
// --- plugins end ---
+29
View File
@@ -2251,4 +2251,33 @@ export const pluginStorage = sqliteTable(
],
);
/**
* Which of a plugin's migrations have been applied.
*
* The checksum is what makes an already-applied migration immutable: editing
* one that has run blocks that plugin rather than silently leaving two
* databases with different shapes. Cascades with the plugin so removing its
* data leaves no ledger behind.
*/
export const pluginMigrations = sqliteTable(
"plugin_migrations",
{
id: integer("id").primaryKey({ autoIncrement: true }),
pluginId: text("plugin_id")
.notNull()
.references(() => plugins.id, { onDelete: "cascade" }),
migrationId: text("migration_id").notNull(),
checksum: text("checksum").notNull(),
appliedAt: text("applied_at")
.notNull()
.default(sql`CURRENT_TIMESTAMP`),
},
(table) => [
uniqueIndex("idx_plugin_migrations_plugin_migration").on(
table.pluginId,
table.migrationId,
),
],
);
// --- plugins end ---
@@ -34,6 +34,7 @@ import { OpenTabRepository } from "./open-tab-repository.js";
import { OpksshTokenRepository } from "./opkssh-token-repository.js";
import { PluginRepository } from "./plugin-repository.js";
import { PluginStorageRepository } from "./plugin-storage-repository.js";
import { PluginMigrationRepository } from "./plugin-migration-repository.js";
import { PluginPermissionGrantRepository } from "./plugin-permission-grant-repository.js";
import { PluginRegistryRepository } from "./plugin-registry-repository.js";
import { PluginInstallCountRepository } from "./plugin-install-count-repository.js";
@@ -393,6 +394,13 @@ export function createCurrentPluginStorageRepository(): PluginStorageRepository
);
}
export function createCurrentPluginMigrationRepository(): PluginMigrationRepository {
return new PluginMigrationRepository(
createCurrentRepositoryContext(),
createCurrentRepositoryWriteHook("plugin_migration_repository_write"),
);
}
export function createCurrentPluginRegistryRepository(): PluginRegistryRepository {
return new PluginRegistryRepository(
createCurrentRepositoryContext(),
@@ -0,0 +1,73 @@
import { and, eq } from "drizzle-orm";
import { pluginMigrations } from "../db/schema.js";
import { insertReturning } from "./returning.js";
import { rowsAffected } from "./mutation-result.js";
import type { DatabaseContext } from "./database-context.js";
export type PluginMigrationRecord = typeof pluginMigrations.$inferSelect;
/**
* Which of a plugin's migrations have run.
*
* The checksum is the reason this is a table rather than a file marker: it
* makes an applied migration immutable, so an edited one is caught instead of
* leaving two installs with quietly different schemas.
*/
export class PluginMigrationRepository {
constructor(
private readonly context: DatabaseContext,
private readonly onWrite?: () => void | Promise<void>,
) {}
async listByPlugin(pluginId: string): Promise<PluginMigrationRecord[]> {
return this.context.drizzle
.select()
.from(pluginMigrations)
.where(eq(pluginMigrations.pluginId, pluginId));
}
async record(
pluginId: string,
migrationId: string,
checksum: string,
now = new Date().toISOString(),
): Promise<void> {
await insertReturning(this.context, pluginMigrations, {
pluginId,
migrationId,
checksum,
appliedAt: now,
});
await this.afterWrite();
}
async deleteByPlugin(pluginId: string): Promise<number> {
const result = await this.context.drizzle
.delete(pluginMigrations)
.where(eq(pluginMigrations.pluginId, pluginId));
const affected = rowsAffected(result);
if (affected > 0) await this.afterWrite();
return affected;
}
async has(pluginId: string, migrationId: string): Promise<boolean> {
const rows = await this.context.drizzle
.select()
.from(pluginMigrations)
.where(
and(
eq(pluginMigrations.pluginId, pluginId),
eq(pluginMigrations.migrationId, migrationId),
),
)
.limit(1);
return rows.length > 0;
}
private async afterWrite(): Promise<void> {
await this.onWrite?.();
}
}
@@ -33,6 +33,16 @@ export class PluginStorageRepository {
return rows.map((row) => row.key);
}
/** How many keys this plugin holds, for the per-plugin key cap. */
async countKeys(pluginId: string): Promise<number> {
const rows = await this.context.drizzle
.select()
.from(pluginStorage)
.where(eq(pluginStorage.pluginId, pluginId));
return rows.length;
}
async set(
pluginId: string,
key: string,
@@ -5,17 +5,16 @@ import type { DatabaseContext } from "./database-context.js";
export type SyncTombstoneRecord = typeof syncTombstones.$inferSelect;
export type SyncEntityType =
| "hosts"
| "sshCredentials"
| "sshFolders"
| "snippets"
| "snippetFolders"
| "vaultProfiles"
| "dashboardServiceLinks"
| "homepageItems"
| "userPreferences"
| "networkTopology";
/**
* The wire name of a synced entity.
*
* A string rather than a union: the set is whatever core and the installed
* plugins have registered, so it cannot be known at compile time. The column
* was already free-form text with no enum constraint, so nothing in the
* database changes. sync.ts validates a name through the registry before it
* reaches a query.
*/
export type SyncEntityType = string;
export class SyncTombstoneRepository {
constructor(
+67
View File
@@ -461,4 +461,71 @@ router.delete(
},
);
/**
* @openapi
* /plugins/{id}/data:
* delete:
* summary: Delete everything a plugin stores
* description: Drops the plugin's own tables and clears its key/value state, capability grants and migration ledger. Disabling a plugin never touches its data; this is the explicit uninstall path, and it cannot be undone. The plugin is deactivated first so nothing is writing while its tables go.
* tags:
* - Plugins
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: string
* responses:
* 200:
* description: What was removed.
* 404:
* description: No such plugin.
* 500:
* description: Failed to remove the plugin's data.
*/
router.delete(
"/:id/data",
authenticateJWT,
requireManagePlugins,
async (req: Request, res: Response) => {
const pluginId = String(req.params.id);
const userId = (req as AuthenticatedRequest).userId;
try {
const record = await createCurrentPluginRepository().findById(pluginId);
if (!record) {
res.status(404).json({ error: "No such plugin" });
return;
}
// Stopped first: dropping a table out from under a running plugin turns
// its next query into an error rather than a clean shutdown.
const { loader } = getPluginRuntime();
if (loader.get(pluginId)?.state === "active") {
await loader.deactivate(pluginId);
}
const { removePluginData } = await import("../../plugins/data.js");
const removed = await removePluginData(pluginId);
databaseLogger.warn(`Removed all data for plugin ${pluginId}`, {
operation: "plugin_remove_data",
pluginId,
userId,
tables: removed.tables.length,
kvKeys: removed.kvKeys,
});
res.json({ id: pluginId, removed });
} catch (error) {
databaseLogger.error(
`Failed to remove data for plugin ${pluginId}`,
error instanceof Error ? error : new Error(String(error)),
{ operation: "plugin_remove_data" },
);
res.status(500).json({ error: "Failed to remove the plugin's data" });
}
},
);
export default router;
@@ -0,0 +1,179 @@
/**
* Core's own sync entities, registered the same way a plugin registers one.
*
* These were a hardcoded ENTITY_CONFIG in sync.ts, a SyncEntityType union in
* the tombstone repository, a REFERENCES map in sync-references.ts and a
* frozen array in electron/. Four lists that had to agree and had no test
* saying they did - which is how networkTopology came to be missing from one
* of them. Now there is one registration per entity and the rest is derived.
*
* The order values reproduce the dependency order the Electron array encoded:
* anything referenced by another entity syncs first, and networkTopology
* syncs last because it holds host ids inside a JSON blob.
*/
import {
hosts,
sshCredentials,
sshFolders,
snippets,
snippetFolders,
vaultProfiles,
dashboardServiceLinks,
homepageItems,
userPreferences,
networkTopology,
} from "../db/schema.js";
import {
CORE_OWNER,
registerEntity,
listEntities,
} from "../../plugins/sync-registry.js";
import { mapTopologyHostIds } from "./sync-references.js";
const CREDENTIAL_REFERENCE = {
field: "credentialId",
syncField: "credentialSyncId",
entityType: "sshCredentials",
} as const;
const HOST_REFERENCES = [
CREDENTIAL_REFERENCE,
{
field: "rdpCredentialId",
syncField: "rdpCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vncCredentialId",
syncField: "vncCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "telnetCredentialId",
syncField: "telnetCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vaultProfileId",
syncField: "vaultProfileSyncId",
entityType: "vaultProfiles",
},
{
field: "parentHostId",
syncField: "parentHostSyncId",
entityType: "hosts",
},
] as const;
let registered = false;
/**
* Registers core's entities. Idempotent, because the sync routes and the
* Electron entity endpoint both need the registry primed and neither can
* assume it ran first.
*/
export function registerCoreSyncEntities(): void {
if (registered && listEntities().length > 0) return;
registered = true;
// 10, 20, 30... so a plugin can slot an entity between two core ones
// without every order value having to be renumbered.
registerEntity(CORE_OWNER, {
type: "sshCredentials",
table: sshCredentials,
order: 10,
});
registerEntity(CORE_OWNER, {
type: "vaultProfiles",
table: vaultProfiles,
order: 20,
});
registerEntity(CORE_OWNER, {
type: "sshFolders",
table: sshFolders,
order: 30,
references: [CREDENTIAL_REFERENCE],
});
registerEntity(CORE_OWNER, {
type: "snippetFolders",
table: snippetFolders,
order: 40,
});
registerEntity(CORE_OWNER, {
type: "hosts",
table: hosts,
order: 50,
// connectionOrigin says how this device reached the host, which is never
// true of the other side.
readOnlyFields: ["connectionOrigin"],
references: HOST_REFERENCES,
encryptedFields: ["ssh_data"],
});
registerEntity(CORE_OWNER, {
type: "snippets",
table: snippets,
order: 60,
});
registerEntity(CORE_OWNER, {
type: "dashboardServiceLinks",
table: dashboardServiceLinks,
order: 70,
});
registerEntity(CORE_OWNER, {
type: "homepageItems",
table: homepageItems,
order: 80,
});
registerEntity(CORE_OWNER, {
type: "userPreferences",
table: userPreferences,
order: 90,
singleton: true,
readOnlyFields: ["storageMode"],
});
registerEntity(CORE_OWNER, {
type: "networkTopology",
table: networkTopology,
order: 100,
singleton: true,
// Host ids live inside the topology JSON rather than in a column, so the
// generic reference machinery cannot reach them.
serialize: async (row, resolveSyncId) => ({
...row,
topology: await mapTopologyHostIds(
row.topology as string | null | undefined,
async (id) => {
const numericId = Number(id);
if (!Number.isInteger(numericId)) return null;
return resolveSyncId("hosts", numericId);
},
),
}),
deserialize: async (row, resolveId) => ({
...row,
topology: await mapTopologyHostIds(
row.topology as string | null | undefined,
async (syncId) => {
const id = await resolveId("hosts", syncId);
return id === null ? null : String(id);
},
),
}),
});
}
/** The tables a reference may point at, derived from the registrations. */
export const ENCRYPTED_ENTITY_TABLES: Readonly<Record<string, string>> = {
hosts: "ssh_data",
sshCredentials: "ssh_credentials",
};
+28 -66
View File
@@ -1,6 +1,16 @@
import type { SyncEntityType } from "../repositories/sync-tombstone-repository.js";
import { getEntity } from "../../plugins/sync-registry.js";
export type SyncReferenceEntity = "hosts" | "sshCredentials" | "vaultProfiles";
/**
* An entity another entity's rows may point at.
*
* A plain string rather than a union: the set is whatever is registered, and a
* plugin can both register an entity and reference one. sync.ts resolves the
* name through the registry, so an unknown one is an error there rather than a
* silent fallthrough - which is what the old three-way if/else did, routing
* anything that was not hosts or sshCredentials to vaultProfiles.
*/
export type SyncReferenceEntity = string;
interface TopologyNode {
data?: { id?: string; [key: string]: unknown };
@@ -25,7 +35,7 @@ interface Topology {
* leave group ids untouched. Nodes/edges that fail to resolve (host deleted,
* or not synced to this side yet) are dropped rather than left dangling.
*/
async function mapTopologyHostIds(
export async function mapTopologyHostIds(
topologyJson: string | null | undefined,
mapId: (id: string) => Promise<string | null>,
): Promise<string | null> {
@@ -81,46 +91,16 @@ interface SyncReference {
entityType: SyncReferenceEntity;
}
const CREDENTIAL_REFERENCE: SyncReference = {
field: "credentialId",
syncField: "credentialSyncId",
entityType: "sshCredentials",
};
const HOST_REFERENCES: SyncReference[] = [
CREDENTIAL_REFERENCE,
{
field: "rdpCredentialId",
syncField: "rdpCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vncCredentialId",
syncField: "vncCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "telnetCredentialId",
syncField: "telnetCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vaultProfileId",
syncField: "vaultProfileSyncId",
entityType: "vaultProfiles",
},
{
field: "parentHostId",
syncField: "parentHostSyncId",
entityType: "hosts",
},
];
export function orderSyncRows(
entityType: SyncEntityType,
rows: Record<string, unknown>[],
): Record<string, unknown>[] {
if (entityType !== "hosts") return rows;
// Only an entity that references itself needs its rows ordered among
// themselves. Today that is hosts, through parentHostId.
const selfReference = referencesFor(entityType).find(
(reference) => reference.entityType === entityType,
);
if (!selfReference) return rows;
const bySyncId = new Map(
rows
@@ -136,7 +116,7 @@ export function orderSyncRows(
if (visiting.has(row)) return;
visiting.add(row);
const parentSyncId = row.parentHostSyncId;
const parentSyncId = row[selfReference.syncField];
if (typeof parentSyncId === "string") {
const parent = bySyncId.get(parentSyncId);
if (parent) visit(parent);
@@ -151,10 +131,9 @@ export function orderSyncRows(
return ordered;
}
const REFERENCES: Partial<Record<SyncEntityType, SyncReference[]>> = {
hosts: HOST_REFERENCES,
sshFolders: [CREDENTIAL_REFERENCE],
};
function referencesFor(entityType: SyncEntityType): readonly SyncReference[] {
return (getEntity(entityType)?.references ?? []) as readonly SyncReference[];
}
export async function serializeSyncReferences(
entityType: SyncEntityType,
@@ -166,19 +145,10 @@ export async function serializeSyncReferences(
): Promise<Record<string, unknown>> {
const serialized = { ...row };
if (entityType === "networkTopology") {
serialized.topology = await mapTopologyHostIds(
serialized.topology as string | null | undefined,
async (id) => {
const numericId = Number(id);
if (!Number.isInteger(numericId)) return null;
return resolveSyncId("hosts", numericId);
},
);
return serialized;
}
const custom = getEntity(entityType)?.serialize;
if (custom) return custom(serialized, resolveSyncId);
for (const reference of REFERENCES[entityType] ?? []) {
for (const reference of referencesFor(entityType)) {
const id = serialized[reference.field];
serialized[reference.syncField] =
typeof id === "number"
@@ -199,18 +169,10 @@ export async function deserializeSyncReferences(
): Promise<Record<string, unknown>> {
const deserialized = { ...row };
if (entityType === "networkTopology") {
deserialized.topology = await mapTopologyHostIds(
deserialized.topology as string | null | undefined,
async (syncId) => {
const id = await resolveId("hosts", syncId);
return id === null ? null : String(id);
},
);
return deserialized;
}
const custom = getEntity(entityType)?.deserialize;
if (custom) return custom(deserialized, resolveId);
for (const reference of REFERENCES[entityType] ?? []) {
for (const reference of referencesFor(entityType)) {
const syncId = deserialized[reference.syncField];
delete deserialized[reference.syncField];
delete deserialized[reference.field];
+95 -116
View File
@@ -1,17 +1,6 @@
import express, { type Request, type Response } from "express";
import { and, eq, type SQL } from "drizzle-orm";
import {
hosts,
sshCredentials,
sshFolders,
snippets,
snippetFolders,
vaultProfiles,
dashboardServiceLinks,
homepageItems,
userPreferences,
networkTopology,
} from "../db/schema.js";
import { hosts } from "../db/schema.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { DataCrypto } from "../../utils/data-crypto.js";
import { databaseLogger } from "../../utils/logger.js";
@@ -22,6 +11,15 @@ import {
createCurrentSyncTombstoneRepository,
} from "../repositories/factory.js";
import type { SyncEntityType } from "../repositories/sync-tombstone-repository.js";
import {
getEntity,
hasEntity,
listEntities,
} from "../../plugins/sync-registry.js";
import {
ENCRYPTED_ENTITY_TABLES,
registerCoreSyncEntities,
} from "./sync-entities.js";
import {
deserializeSyncReferences,
orderSyncRows,
@@ -31,66 +29,44 @@ import {
import { timestampAtOrAfter } from "../sync-timestamp.js";
import { validateParentHostId } from "./host-parent-validation.js";
// Primed at import so every consumer of this module - the routes below, the
// reference resolvers, the Electron entity-types endpoint - sees the same
// registry regardless of which one runs first.
registerCoreSyncEntities();
const router = express.Router();
const authManager = AuthManager.getInstance();
const authenticateJWT = authManager.createAuthMiddleware();
// Encrypted tables need DataCrypto to translate between the wire payload
// (plaintext) and the stored row (encrypted). Everything else is stored
// and synced as-is.
const ENCRYPTED_ENTITY_TABLES: Partial<Record<SyncEntityType, string>> = {
hosts: "ssh_data",
sshCredentials: "ssh_credentials",
};
/**
* What a sync entity looks like once it is registered.
*
* The table was a hand-maintained union of ten concrete drizzle types, which a
* plugin-provided table could never join. It is the same deliberate
* approximation repositories/database-context.ts documents: the query-builder
* surface used here is identical across the engines and across tables, and
* every use already casts to reach .syncId and .updatedAt.
*/
type SyncTable = typeof hosts;
interface EntityConfig {
table:
| typeof hosts
| typeof sshCredentials
| typeof sshFolders
| typeof snippets
| typeof snippetFolders
| typeof vaultProfiles
| typeof dashboardServiceLinks
| typeof homepageItems
| typeof userPreferences
| typeof networkTopology;
// Fields that only make sense on the device that created the row, or
// that are managed elsewhere and must never be overwritten by a sync
// payload from the other side.
readOnlyFields: string[];
singleton?: boolean;
function entityConfig(entityType: SyncEntityType) {
const entity = getEntity(entityType);
if (!entity) {
throw new Error(`Unknown sync entity "${entityType}"`);
}
return {
table: entity.table as SyncTable,
readOnlyFields: entity.readOnlyFields,
singleton: entity.singleton === true,
};
}
const ENTITY_CONFIG: Record<SyncEntityType, EntityConfig> = {
hosts: {
table: hosts,
readOnlyFields: ["connectionOrigin"],
},
sshCredentials: { table: sshCredentials, readOnlyFields: [] },
sshFolders: { table: sshFolders, readOnlyFields: [] },
snippets: { table: snippets, readOnlyFields: [] },
snippetFolders: { table: snippetFolders, readOnlyFields: [] },
vaultProfiles: { table: vaultProfiles, readOnlyFields: [] },
dashboardServiceLinks: { table: dashboardServiceLinks, readOnlyFields: [] },
homepageItems: { table: homepageItems, readOnlyFields: [] },
userPreferences: {
table: userPreferences,
readOnlyFields: ["storageMode"],
singleton: true,
},
networkTopology: {
table: networkTopology,
readOnlyFields: [],
singleton: true,
},
};
const VALID_ENTITY_TYPES = new Set(Object.keys(ENTITY_CONFIG));
type RepositoryContext = ReturnType<typeof createCurrentRepositoryContext>;
export function isValidEntityType(value: unknown): value is SyncEntityType {
return typeof value === "string" && VALID_ENTITY_TYPES.has(value);
if (typeof value !== "string") return false;
registerCoreSyncEntities();
return hasEntity(value);
}
/**
@@ -106,7 +82,7 @@ export function locateSyncRow(
userId: string,
syncId: string,
): SQL {
const { table, singleton } = ENTITY_CONFIG[entityType];
const { table, singleton } = entityConfig(entityType);
if (singleton) {
return eq(table.userId, userId);
@@ -118,34 +94,25 @@ export function locateSyncRow(
)!;
}
/**
* The syncId of a referenced row.
*
* Generic over the registry rather than an if/else over three known entities.
* The old chain ended in an unguarded else that returned vaultProfiles, so a
* fourth reference target would have silently resolved against the wrong
* table; an unregistered entity is an error here instead.
*/
async function findReferenceSyncId(
context: RepositoryContext,
entityType: SyncReferenceEntity,
id: number,
userId: string,
): Promise<string | null> {
if (entityType === "hosts") {
const [row] = await context.drizzle
.select({ syncId: hosts.syncId })
.from(hosts)
.where(and(eq(hosts.id, id), eq(hosts.userId, userId)))
.limit(1);
return row?.syncId ?? null;
}
if (entityType === "sshCredentials") {
const [row] = await context.drizzle
.select({ syncId: sshCredentials.syncId })
.from(sshCredentials)
.where(and(eq(sshCredentials.id, id), eq(sshCredentials.userId, userId)))
.limit(1);
return row?.syncId ?? null;
}
const { table } = entityConfig(entityType as SyncEntityType);
const [row] = await context.drizzle
.select({ syncId: vaultProfiles.syncId })
.from(vaultProfiles)
.where(and(eq(vaultProfiles.id, id), eq(vaultProfiles.userId, userId)))
.select({ syncId: table.syncId })
.from(table)
.where(and(eq(table.id, id), eq(table.userId, userId)))
.limit(1);
return row?.syncId ?? null;
}
@@ -156,35 +123,11 @@ async function findReferenceId(
syncId: string,
userId: string,
): Promise<number | null> {
if (entityType === "hosts") {
const [row] = await context.drizzle
.select({ id: hosts.id })
.from(hosts)
.where(and(eq(hosts.syncId, syncId), eq(hosts.userId, userId)))
.limit(1);
return row?.id ?? null;
}
if (entityType === "sshCredentials") {
const [row] = await context.drizzle
.select({ id: sshCredentials.id })
.from(sshCredentials)
.where(
and(
eq(sshCredentials.syncId, syncId),
eq(sshCredentials.userId, userId),
),
)
.limit(1);
return row?.id ?? null;
}
const { table } = entityConfig(entityType as SyncEntityType);
const [row] = await context.drizzle
.select({ id: vaultProfiles.id })
.from(vaultProfiles)
.where(
and(eq(vaultProfiles.syncId, syncId), eq(vaultProfiles.userId, userId)),
)
.select({ id: table.id })
.from(table)
.where(and(eq(table.syncId, syncId), eq(table.userId, userId)))
.limit(1);
return row?.id ?? null;
}
@@ -230,7 +173,7 @@ export function stripWritePayload(
entityType: SyncEntityType,
payload: Record<string, unknown>,
): Record<string, unknown> {
const { readOnlyFields } = ENTITY_CONFIG[entityType];
const { readOnlyFields } = entityConfig(entityType);
const clean = { ...payload };
delete clean.id;
delete clean.userId;
@@ -239,6 +182,42 @@ export function stripWritePayload(
return clean;
}
/**
* @openapi
* /sync/entity-types:
* get:
* summary: List the entity types this server syncs, in dependency order
* description: The desktop app syncs the entities in the order returned here, because an entity referenced by another has to exist on the far side first. A server that predates this endpoint returns 404 and the client falls back to its built-in list.
* tags:
* - Sync
* responses:
* 200:
* description: The ordered entity types.
* 500:
* description: Failed to list entity types.
*/
router.get(
"/entity-types",
authenticateJWT,
async (_req: Request, res: Response) => {
try {
registerCoreSyncEntities();
return res.json({
entityTypes: listEntities().map((entity) => ({
type: entity.type,
order: entity.order,
singleton: entity.singleton === true,
})),
});
} catch (error) {
databaseLogger.error("Failed to list sync entity types", error, {
operation: "sync_entity_types",
});
return res.status(500).json({ error: "Failed to list entity types" });
}
},
);
/**
* @openapi
* /sync/{entityType}:
@@ -280,7 +259,7 @@ router.get(
: null;
try {
const { table, singleton } = ENTITY_CONFIG[entityType];
const { table, singleton } = entityConfig(entityType);
const context = createCurrentRepositoryContext();
const conditions = [eq(table.userId, userId)];
if (since && "updatedAt" in table) {
@@ -352,7 +331,7 @@ router.post(
}
try {
const { table } = ENTITY_CONFIG[entityType];
const { table } = entityConfig(entityType);
const context = createCurrentRepositoryContext();
await context.drizzle
@@ -418,7 +397,7 @@ router.post(
try {
// singleton is still needed below: those tables have no sync_id column
// for the insert to populate.
const { table, singleton } = ENTITY_CONFIG[entityType];
const { table, singleton } = entityConfig(entityType);
const context = createCurrentRepositoryContext();
const locateRow = locateSyncRow(entityType, userId, syncId);
+77 -5
View File
@@ -31,6 +31,8 @@ import { getActor, runAsActor } from "./actor.js";
import { DisposableBag } from "./disposables.js";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
import type { PluginContext, PluginModule } from "@termix/plugin-sdk/backend";
import type { PluginTableDefinition } from "@termix/plugin-sdk/db";
import * as syncRegistry from "./sync-registry.js";
export type { PluginModule };
@@ -38,6 +40,20 @@ export type { PluginModule };
const MAX_KV_KEY_LENGTH = 128;
const MAX_KV_VALUE_BYTES = 256 * 1024;
/**
* How many keys one plugin may hold.
*
* The value cap alone bounds a single row, not the table: a plugin writing
* unique keys in a loop could still fill the database. Configurable because
* the right ceiling depends on the install, not on the plugin.
*/
const DEFAULT_MAX_KV_KEYS = 10_000;
function maxKvKeys(env: NodeJS.ProcessEnv = process.env): number {
const raw = Number(env.PLUGIN_MAX_KV_KEYS);
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_MAX_KV_KEYS;
}
export interface PluginHandle {
module: PluginModule;
bag: DisposableBag;
@@ -169,11 +185,21 @@ export function createPluginContext(
}
const { createCurrentPluginStorageRepository } =
await import("../database/repositories/factory.js");
await createCurrentPluginStorageRepository().set(
pluginId,
key,
serialized,
);
const repository = createCurrentPluginStorageRepository();
// Only a new key can grow the table, so an overwrite is never blocked
// by a full one. Checked before the write, not after.
const existing = await repository.get(pluginId, key);
if (existing === null) {
const limit = maxKvKeys();
if ((await repository.countKeys(pluginId)) >= limit) {
throw new Error(
`Plugin "${pluginId}" has reached the ${limit} key limit for ctx.kv`,
);
}
}
await repository.set(pluginId, key, serialized);
},
{ action: "kv_set" },
);
@@ -200,6 +226,39 @@ export function createPluginContext(
{ action: "kv_list" },
);
const dbDefine = guarded(
manifest,
"db:own",
async (definition: PluginTableDefinition) => {
const { registerTable } = await import("./data.js");
return registerTable(pluginId, definition);
},
{ action: "db_define", details: () => "table definition registered" },
);
const dbClient = guarded(
manifest,
"db:own",
async () => {
const { getDb } = await import("../database/db/index.js");
return getDb();
},
{ action: "db_client" },
);
const dbRefs = guarded(
manifest,
"db:own",
async () => {
// Read-only by convention, not by engine: a plugin holding these can
// write through them. The capability and the audit line are the record
// that it did. See the header of this file.
const schema = await import("../database/db/schema.js");
return { users: schema.users, hosts: schema.hosts };
},
{ action: "db_refs" },
);
return {
pluginId,
manifest,
@@ -237,6 +296,19 @@ export function createPluginContext(
},
},
db: {
define: (definition) => dbDefine(definition) as never,
client: () => dbClient() as never,
refs: () => dbRefs() as never,
},
sync: {
registerEntity: (entity) => {
const dispose = syncRegistry.registerEntity(pluginId, entity);
handle.bag.add(dispose, `sync entity ${entity.type}`);
},
},
kv: {
get: (key) => kvGet(key),
set: (key, value) => kvSet(key, value),
+177
View File
@@ -0,0 +1,177 @@
/**
* The live database side of plugin-owned data.
*
* Holds the table definitions each plugin has registered, builds the runner
* that applies its migrations against the real database, and drops everything
* again on uninstall. The pure logic lives in migrations.ts so it can be
* tested without a database; this file is the part that needs one.
*/
import { sql } from "drizzle-orm";
import { pluginLogger } from "../utils/logger.js";
import { resolveDatabaseDialect } from "../database/db/dialect.js";
import type { DatabaseDialect } from "../database/db/dialect.js";
import type { PluginTableDefinition } from "@termix/plugin-sdk/db";
import { prefixedTableName } from "@termix/plugin-sdk/db";
import { buildTable, dropTableSql } from "./table-builder.js";
import {
applyPluginMigrations,
readMigrations,
tableExistsSql,
type MigrationRunner,
} from "./migrations.js";
/** Definitions registered by a plugin, in the order it declared them. */
const registered = new Map<string, PluginTableDefinition[]>();
/** Built table objects, keyed by plugin id then definition name. */
const built = new Map<string, Map<string, unknown>>();
export function registerTable(
pluginId: string,
definition: PluginTableDefinition,
): unknown {
const definitions = registered.get(pluginId) ?? [];
const existing = definitions.find((entry) => entry.name === definition.name);
if (existing) {
// Re-registering on re-activation is normal: the module is cached, so
// activate runs again against the same definitions.
return getTable(pluginId, definition.name);
}
definitions.push(definition);
registered.set(pluginId, definitions);
const table = buildTable(pluginId, definition);
const tables = built.get(pluginId) ?? new Map<string, unknown>();
tables.set(definition.name, table);
built.set(pluginId, tables);
return table;
}
export function getTable(pluginId: string, name: string): unknown {
return built.get(pluginId)?.get(name);
}
export function listTables(pluginId: string): PluginTableDefinition[] {
return [...(registered.get(pluginId) ?? [])];
}
/** Forgets a plugin's definitions. Used by the tests and by uninstall. */
export function forgetTables(pluginId: string): void {
registered.delete(pluginId);
built.delete(pluginId);
}
export function resetPluginData(): void {
registered.clear();
built.clear();
}
async function execute(statements: string[]): Promise<void> {
const { getDb } = await import("../database/db/index.js");
const db = getDb();
for (const statement of statements) {
await db.run(sql.raw(statement));
}
}
/**
* Builds the runner that applies migrations against the live database.
*
* SQLite gets a transaction; the client-server engines do not, because MySQL
* commits implicitly on DDL so a "transaction" there would only be a promise
* the engine does not keep. A half-applied migration is recorded statement by
* statement either way, so a retry resumes rather than repeating.
*/
export async function createMigrationRunner(
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<MigrationRunner> {
const { createCurrentPluginMigrationRepository } =
await import("../database/repositories/factory.js");
const repository = createCurrentPluginMigrationRepository();
return {
dialect,
listApplied: async (pluginId) => {
const rows = await repository.listByPlugin(pluginId);
return rows.map((row) => ({
migrationId: row.migrationId,
checksum: row.checksum,
}));
},
execute,
record: async (pluginId, migration) => {
await repository.record(pluginId, migration.id, migration.checksum);
},
};
}
/**
* Applies a plugin's migrations. Throws so the loader can fail the plugin.
*
* Returns early when the plugin ships none, before anything asks for a
* database handle: most plugins own no tables, and activating one should not
* require a connection it never uses.
*/
export async function migratePlugin(
pluginId: string,
pluginDir: string,
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<string[]> {
if (readMigrations(pluginDir, dialect).length === 0) return [];
const runner = await createMigrationRunner(dialect);
return applyPluginMigrations(pluginId, pluginDir, runner);
}
/** Whether a table exists, asked in the active dialect's own catalog. */
export async function tableExists(
table: string,
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<boolean> {
const { getDb } = await import("../database/db/index.js");
const rows = (await getDb().all(tableExistsSql(dialect, table))) as unknown[];
return rows.length > 0;
}
/**
* Removes everything a plugin owns.
*
* Tables go in reverse registration order so a table referenced by a later one
* is dropped last. Disabling a plugin never calls this: data outlives being
* turned off, and only an explicit uninstall throws it away.
*/
export async function removePluginData(
pluginId: string,
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<{ tables: string[]; kvKeys: number; migrations: number }> {
const definitions = listTables(pluginId);
const dropped: string[] = [];
for (const definition of [...definitions].reverse()) {
await execute([dropTableSql(dialect, pluginId, definition)]);
dropped.push(prefixedTableName(pluginId, definition.name));
}
const {
createCurrentPluginStorageRepository,
createCurrentPluginMigrationRepository,
createCurrentPluginPermissionGrantRepository,
} = await import("../database/repositories/factory.js");
const kvKeys =
await createCurrentPluginStorageRepository().deleteByPlugin(pluginId);
const migrations =
await createCurrentPluginMigrationRepository().deleteByPlugin(pluginId);
await createCurrentPluginPermissionGrantRepository().deleteByPlugin(pluginId);
forgetTables(pluginId);
pluginLogger.info(
`Removed data for plugin ${pluginId}: ${dropped.length} table(s), ${kvKeys} kv key(s)`,
{ operation: "plugin_remove_data" },
);
return { tables: dropped, kvKeys, migrations };
}
+7
View File
@@ -338,6 +338,13 @@ export class PluginLoader {
);
}
// Before activate, not inside it: a plugin's first line may query its
// own tables, and a migration that fails should stop it starting rather
// than leave it half-running against a schema that is not there. A
// throw here lands in the catch below, which fails this plugin only.
const { migratePlugin } = await import("./data.js");
await migratePlugin(plugin.id, plugin.dir);
const handle = createPluginHandle(plugin.id, { activate, deactivate });
const ctx = createPluginContext(plugin.manifest, handle);
+316
View File
@@ -0,0 +1,316 @@
/**
* Applies a plugin's migrations before it activates.
*
* A plugin ships plain SQL per dialect in migrations/<dialect>/NNNN_name.sql.
* Not drizzle-kit: core's own SQLite schema is hand-written DDL and only the
* client-server engines use drizzle migrations, so a plugin that used
* drizzle-kit would still need a hand-rolled SQLite path. One runner for all
* three keeps a plugin's tables identical everywhere.
*
* Failures are contained. An edited migration that already ran blocks that
* plugin; a migration that throws marks it failed. Core keeps booting either
* way, because one plugin's bad SQL is not a reason to take the server down.
*/
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { sql } from "drizzle-orm";
import { pluginLogger } from "../utils/logger.js";
import type { DatabaseDialect } from "../database/db/dialect.js";
import { tablePrefix, LEGACY_TABLE_OWNERS } from "@termix/plugin-sdk/db";
export { LEGACY_TABLE_OWNERS };
/** Dialect directory names, matching DatabaseDialect exactly. */
const DIALECTS: readonly DatabaseDialect[] = ["sqlite", "postgres", "mysql"];
const FILE_PATTERN = /^(\d{4})_([a-z0-9_]+)\.sql$/;
export interface PluginMigration {
/** The file's stem, e.g. "0001_init". Stable across dialects. */
id: string;
sequence: number;
file: string;
sql: string;
checksum: string;
}
/** Raised when an already-applied migration no longer matches its checksum. */
export class PluginMigrationChecksumError extends Error {
readonly code = "EPLUGINMIGRATIONCHECKSUM";
constructor(pluginId: string, migrationId: string) {
super(
`Plugin "${pluginId}" migration "${migrationId}" has changed since it was applied. ` +
`An applied migration is immutable: add a new one instead of editing this file.`,
);
this.name = "PluginMigrationChecksumError";
}
}
export function checksum(contents: string): string {
// Normalised so a line-ending change between platforms is not a mismatch.
const normalised = contents.replace(/\r\n/g, "\n").trimEnd();
return crypto.createHash("sha256").update(normalised).digest("hex");
}
export function migrationsDir(
pluginDir: string,
dialect: DatabaseDialect,
): string {
return path.join(pluginDir, "migrations", dialect);
}
/** Reads the migrations a plugin ships for one dialect, in sequence order. */
export function readMigrations(
pluginDir: string,
dialect: DatabaseDialect,
): PluginMigration[] {
const dir = migrationsDir(pluginDir, dialect);
if (!fs.existsSync(dir)) return [];
const migrations: PluginMigration[] = [];
for (const file of fs.readdirSync(dir).sort()) {
const match = FILE_PATTERN.exec(file);
if (!match) {
if (file.endsWith(".sql")) {
throw new Error(
`Migration "${file}" must be named NNNN_name.sql, lower snake_case`,
);
}
continue;
}
const contents = fs.readFileSync(path.join(dir, file), "utf8");
migrations.push({
id: file.replace(/\.sql$/, ""),
sequence: Number(match[1]),
file,
sql: contents,
checksum: checksum(contents),
});
}
migrations.sort((a, b) => a.sequence - b.sequence);
return migrations;
}
/**
* Splits a migration file into statements.
*
* Deliberately simple: statements end at a semicolon that is not inside a
* string literal or a comment. A plugin needing more than that should put the
* logic in its own code, not in DDL.
*/
export function splitStatements(source: string): string[] {
const statements: string[] = [];
let current = "";
let quote: string | null = null;
let lineComment = false;
let blockComment = false;
for (let i = 0; i < source.length; i++) {
const char = source[i];
const next = source[i + 1];
if (lineComment) {
current += char;
if (char === "\n") lineComment = false;
continue;
}
if (blockComment) {
current += char;
if (char === "*" && next === "/") {
current += next;
i++;
blockComment = false;
}
continue;
}
if (quote) {
current += char;
// '' inside a quoted string is an escaped quote, not a terminator.
if (char === quote) {
if (next === quote) {
current += next;
i++;
} else {
quote = null;
}
}
continue;
}
if (char === "-" && next === "-") {
lineComment = true;
current += char;
continue;
}
if (char === "/" && next === "*") {
blockComment = true;
current += char;
continue;
}
if (char === "'" || char === '"' || char === "`") {
quote = char;
current += char;
continue;
}
if (char === ";") {
if (current.trim()) statements.push(current.trim());
current = "";
continue;
}
current += char;
}
if (current.trim()) statements.push(current.trim());
return statements;
}
/**
* Tables a migration may create or alter.
*
* A plugin owns the p_<id>_ namespace and nothing else. Enforced here as well
* as in the CLI, because a plugin installed from a tarball never ran the CLI.
*/
const CREATE_OR_ALTER =
/\b(?:CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TABLE(?:\s+IF\s+NOT\s+EXISTS)?|ALTER\s+TABLE|DROP\s+TABLE(?:\s+IF\s+EXISTS)?|TRUNCATE\s+TABLE)\s+([`"']?)([a-zA-Z0-9_]+)\1/gi;
/**
* Checks that every table a migration touches belongs to this plugin.
*
* A legacy table it is allowed to adopt is permitted too, because adoption
* renames it into the plugin's namespace in the same migration.
*/
export function assertOwnedTables(
pluginId: string,
statements: string[],
): void {
const prefix = tablePrefix(pluginId);
for (const statement of statements) {
CREATE_OR_ALTER.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = CREATE_OR_ALTER.exec(statement)) !== null) {
const table = match[2];
if (table.startsWith(prefix)) continue;
if (LEGACY_TABLE_OWNERS[table] === pluginId) continue;
const owner = LEGACY_TABLE_OWNERS[table];
throw new Error(
owner
? `Plugin "${pluginId}" may not touch "${table}": it belongs to the "${owner}" plugin.`
: `Plugin "${pluginId}" may only create or alter tables prefixed "${prefix}", not "${table}".`,
);
}
}
}
export interface AppliedMigration {
migrationId: string;
checksum: string;
}
export interface MigrationRunner {
dialect: DatabaseDialect;
listApplied: (pluginId: string) => Promise<AppliedMigration[]>;
execute: (statements: string[]) => Promise<void>;
record: (pluginId: string, migration: PluginMigration) => Promise<void>;
/** Runs fn inside a transaction where the dialect supports DDL in one. */
transaction?: (fn: () => Promise<void>) => Promise<void>;
}
/**
* Brings one plugin's tables up to date.
*
* Returns the ids applied, so a caller can log what changed. Throws on a
* checksum mismatch or a failing statement; the loader turns either into a
* failed plugin without disturbing the rest of the server.
*/
export async function applyPluginMigrations(
pluginId: string,
pluginDir: string,
runner: MigrationRunner,
): Promise<string[]> {
const migrations = readMigrations(pluginDir, runner.dialect);
if (migrations.length === 0) return [];
const applied = new Map(
(await runner.listApplied(pluginId)).map((row) => [
row.migrationId,
row.checksum,
]),
);
const pending: PluginMigration[] = [];
for (const migration of migrations) {
const previous = applied.get(migration.id);
if (previous === undefined) {
pending.push(migration);
continue;
}
if (previous !== migration.checksum) {
throw new PluginMigrationChecksumError(pluginId, migration.id);
}
}
if (pending.length === 0) return [];
const run = async () => {
for (const migration of pending) {
const statements = splitStatements(migration.sql);
assertOwnedTables(pluginId, statements);
await runner.execute(statements);
await runner.record(pluginId, migration);
}
};
if (runner.transaction) {
await runner.transaction(run);
} else {
await run();
}
pluginLogger.info(
`Applied ${pending.length} migration(s) for plugin ${pluginId}: ${pending
.map((m) => m.id)
.join(", ")}`,
{ operation: "plugin_migrate" },
);
return pending.map((migration) => migration.id);
}
/** The dialect directories a plugin is expected to ship. */
export function expectedDialects(): readonly DatabaseDialect[] {
return DIALECTS;
}
/**
* Renames a legacy core table into a plugin's namespace, or creates it fresh.
*
* Adoption is how a feature keeps its data when it moves out of core. The
* table is renamed rather than copied so nothing is duplicated and nothing has
* to be migrated row by row.
*/
export function adoptLegacyTableSql(
dialect: DatabaseDialect,
legacyName: string,
newName: string,
): string {
if (dialect === "mysql") {
return `RENAME TABLE \`${legacyName}\` TO \`${newName}\`;`;
}
return `ALTER TABLE "${legacyName}" RENAME TO "${newName}";`;
}
/** Whether a table exists, asked in the dialect's own catalog. */
export function tableExistsSql(dialect: DatabaseDialect, table: string) {
if (dialect === "sqlite") {
return sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${table}`;
}
if (dialect === "postgres") {
return sql`SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = ${table}`;
}
return sql`SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${table}`;
}
+100
View File
@@ -0,0 +1,100 @@
/**
* Which entities remote sync carries.
*
* Core registers its ten at boot and a plugin registers its own through
* ctx.sync, so the list is built rather than hardcoded in four places. The
* wire names are unchanged: existing tombstones and rows on the other side of
* a pair match on those strings, so renaming one would orphan data.
*
* Order is dependency order, not declaration order. A row referencing another
* entity is resolved by that entity's syncId, so the target has to exist on
* the far side first. That is why every registration carries an explicit
* order rather than relying on registration sequence.
*/
import { pluginLogger } from "../utils/logger.js";
import type { SyncEntityRegistration } from "@termix/plugin-sdk/backend";
export interface RegisteredSyncEntity extends SyncEntityRegistration {
/** Which plugin registered it, or "core". */
owner: string;
order: number;
userColumn: string;
readOnlyFields: readonly string[];
encryptedFields: readonly string[];
}
const entities = new Map<string, RegisteredSyncEntity>();
/** Core's own entities register under this owner. */
export const CORE_OWNER = "core";
export function registerEntity(
owner: string,
entity: SyncEntityRegistration,
): () => void {
if (!entity.type) {
throw new Error("A sync entity needs a type");
}
const existing = entities.get(entity.type);
if (existing && existing.owner !== owner) {
throw new Error(
`Sync entity "${entity.type}" is already registered by "${existing.owner}"`,
);
}
entities.set(entity.type, {
...entity,
owner,
order: entity.order ?? 100,
userColumn: entity.userColumn ?? "userId",
readOnlyFields: entity.readOnlyFields ?? [],
encryptedFields: entity.encryptedFields ?? [],
});
return () => {
const current = entities.get(entity.type);
// Identity-checked: a plugin that restarted must not revoke the
// registration its own restart installed.
if (current && current.owner === owner) entities.delete(entity.type);
};
}
export function getEntity(type: string): RegisteredSyncEntity | undefined {
return entities.get(type);
}
export function hasEntity(type: string): boolean {
return entities.has(type);
}
/** Every entity, in dependency order. */
export function listEntities(): RegisteredSyncEntity[] {
return [...entities.values()].sort((a, b) => {
if (a.order !== b.order) return a.order - b.order;
return a.type.localeCompare(b.type);
});
}
/** The ordered wire names, which is what the Electron client syncs by. */
export function listEntityTypes(): string[] {
return listEntities().map((entity) => entity.type);
}
export function unregisterByOwner(owner: string): void {
for (const [type, entity] of [...entities]) {
if (entity.owner === owner) entities.delete(type);
}
}
export function resetSyncRegistry(): void {
entities.clear();
}
/** Logged once at boot so a missing entity is visible without a debugger. */
export function logRegisteredEntities(): void {
pluginLogger.info(`Sync entities: ${listEntityTypes().join(", ")}`, {
operation: "sync_registry",
});
}
+104
View File
@@ -0,0 +1,104 @@
/**
* Turns a plugin's table definition into the Drizzle table it is queried
* through.
*
* One table object for every dialect. The sqlite-core definitions already
* encode correctly on each engine at query time, so the repositories use them
* everywhere; only DDL genuinely differs, and that lives in the SDK's ddl
* entry so the CLI and the server share one mapping.
*/
import {
sqliteTable,
text as sqliteText,
integer as sqliteInteger,
index as sqliteIndex,
uniqueIndex as sqliteUniqueIndex,
type AnySQLiteColumn,
} from "drizzle-orm/sqlite-core";
import { sql } from "drizzle-orm";
import { prefixedTableName } from "@termix/plugin-sdk/db";
import type { PluginTableDefinition } from "@termix/plugin-sdk/db";
import { columnName } from "@termix/plugin-sdk/ddl";
type ColumnChain = {
notNull: () => unknown;
primaryKey: () => unknown;
unique: () => unknown;
default: (value: unknown) => unknown;
};
/**
* Builds the queryable table object.
*
* Always sqlite-core, on every dialect. drizzle's three table classes share no
* base class, and the query-builder surface a plugin uses is identical across
* them - the same deliberate approximation repositories/database-context.ts
* documents for core.
*/
export function buildTable(
pluginId: string,
definition: PluginTableDefinition,
): unknown {
const physical = prefixedTableName(pluginId, definition.name);
const columns: Record<string, unknown> = {};
for (const [property, column] of Object.entries(definition.columns)) {
const name = columnName(property, column);
let built: unknown;
switch (column.type) {
case "id":
built = sqliteInteger(name).primaryKey({ autoIncrement: true });
break;
case "integer":
case "bigint":
case "refHost":
built = sqliteInteger(name);
break;
case "boolean":
built = sqliteInteger(name, { mode: "boolean" });
break;
default:
built = sqliteText(name);
break;
}
if (column.type !== "id") {
if (column.primaryKey) built = (built as ColumnChain).primaryKey();
if (column.notNull) built = (built as ColumnChain).notNull();
if (column.unique) built = (built as ColumnChain).unique();
if (column.defaultNow) {
built = (built as ColumnChain).default(sql`CURRENT_TIMESTAMP`);
} else if (column.defaultValue !== undefined) {
built = (built as ColumnChain).default(column.defaultValue);
}
}
columns[property] = built;
}
return sqliteTable(physical, columns as never, (table) =>
definition.indexes.map((entry) => {
const make = entry.unique ? sqliteUniqueIndex : sqliteIndex;
const target = table as unknown as Record<string, AnySQLiteColumn>;
return make(entry.name).on(
...(entry.columns.map((c) => target[c]) as [
AnySQLiteColumn,
...AnySQLiteColumn[],
]),
);
}),
);
}
// The DDL emitters live in the SDK, because termix-plugin writes the same SQL
// this server applies. Re-exported here so core keeps one import site.
export {
createTableSql,
dropTableSql,
addColumnSql,
keyedColumns,
columnName,
REFERENCEABLE,
} from "@termix/plugin-sdk/ddl";
@@ -1,11 +1,19 @@
import { describe, expect, it } from "vitest";
import { beforeAll, describe, expect, it } from "vitest";
import {
deserializeSyncReferences,
orderSyncRows,
serializeSyncReferences,
} from "../../database/routes/sync-references.js";
import { registerCoreSyncEntities } from "../../database/routes/sync-entities.js";
describe("sync references", () => {
// References come from the registry now, not a hardcoded map, so the core
// entities have to be registered before any of this resolves. sync.ts does
// this at import in production.
beforeAll(() => {
registerCoreSyncEntities();
});
it("serializes database-local host IDs as stable sync IDs", async () => {
const row = await serializeSyncReferences(
"hosts",
@@ -30,6 +30,9 @@ vi.mock("../../database/repositories/factory.js", () => ({
delete: async (pluginId: string, key: string) =>
kvStore.delete(`${pluginId}:${key}`),
listKeys: async () => [...kvStore.keys()],
countKeys: async (pluginId: string) =>
[...kvStore.keys()].filter((key) => key.startsWith(`${pluginId}:`))
.length,
}),
}));
+234
View File
@@ -0,0 +1,234 @@
/**
* ctx.db and the kv caps.
*
* db:own was in the catalog from A1 but nothing checked it, which ARCHITECTURE
* calls the anti-pattern outright. These assert that it is checked where the
* privileged thing happens, and audited either way.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
const grants = new Map<string, string[]>();
const auditEntries: Array<Record<string, unknown>> = [];
const kvStore = new Map<string, string>();
vi.mock("../../database/repositories/factory.js", () => ({
createCurrentPluginPermissionGrantRepository: () => ({
listByPlugin: async (pluginId: string) =>
(grants.get(pluginId) ?? []).map((capability) => ({
pluginId,
capability,
})),
}),
createCurrentPluginStorageRepository: () => ({
get: async (pluginId: string, key: string) =>
kvStore.get(`${pluginId}:${key}`) ?? null,
set: async (pluginId: string, key: string, value: string) => {
kvStore.set(`${pluginId}:${key}`, value);
},
delete: async (pluginId: string, key: string) =>
kvStore.delete(`${pluginId}:${key}`),
listKeys: async (pluginId: string) =>
[...kvStore.keys()]
.filter((key) => key.startsWith(`${pluginId}:`))
.map((key) => key.slice(pluginId.length + 1)),
countKeys: async (pluginId: string) =>
[...kvStore.keys()].filter((key) => key.startsWith(`${pluginId}:`))
.length,
}),
}));
vi.mock("../../utils/audit-logger.js", () => ({
logAudit: async (entry: Record<string, unknown>) => {
auditEntries.push(entry);
},
}));
vi.mock("../../database/db/index.js", () => ({
getDb: () => ({ marker: "db-handle" }),
}));
vi.mock("../../database/db/schema.js", () => ({
users: { marker: "users" },
hosts: { marker: "hosts" },
}));
import { createPluginContext, createPluginHandle } from "../../plugins/ctx.js";
import { invalidatePluginPermissionCache } from "../../plugins/permissions.js";
import { resetPluginData } from "../../plugins/data.js";
import {
resetSyncRegistry,
listEntityTypes,
} from "../../plugins/sync-registry.js";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
function manifestFor(pluginId: string, capabilities: string[]): PluginManifest {
return {
id: pluginId,
name: pluginId,
version: "1.0.0",
description: "",
author: { name: "test" },
license: "MIT",
category: "Productivity",
engine: { termix: ">=2.9.0", api: "1" },
capabilities,
} as PluginManifest;
}
function contextFor(pluginId: string, capabilities: string[]) {
const manifest = manifestFor(pluginId, capabilities);
const handle = createPluginHandle(pluginId, { activate: () => {} });
return { ctx: createPluginContext(manifest, handle), handle };
}
beforeEach(() => {
grants.clear();
auditEntries.length = 0;
kvStore.clear();
resetPluginData();
resetSyncRegistry();
invalidatePluginPermissionCache();
delete process.env.PLUGIN_MAX_KV_KEYS;
});
describe("ctx.db capability", () => {
it("refuses define without db:own", async () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
await expect(
ctx.db.define({ name: "thing", columns: {}, indexes: [] }),
).rejects.toThrow(/db:own/);
});
it("refuses define when granted but not declared", async () => {
// A grant for something the manifest never declared is ignored, so
// widening a plugin's reach always needs a manifest the user can see.
grants.set("demo", ["db:own"]);
const { ctx } = contextFor("demo", []);
await expect(
ctx.db.define({ name: "thing", columns: {}, indexes: [] }),
).rejects.toThrow(/db:own/);
});
it("refuses the client handle without db:own", async () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
await expect(ctx.db.client()).rejects.toThrow(/db:own/);
});
it("allows define when declared and granted", async () => {
grants.set("demo", ["db:own"]);
const { ctx } = contextFor("demo", ["db:own"]);
const table = await ctx.db.define({
name: "thing",
columns: { id: { type: "id", primaryKey: true, notNull: true } },
indexes: [],
});
expect(table).toBeTruthy();
});
it("audits a refused call as well as an allowed one", async () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
await expect(ctx.db.client()).rejects.toThrow();
const entry = auditEntries.find((row) => row.action === "plugin_db_client");
expect(entry).toMatchObject({
success: false,
username: "plugin:demo",
resourceId: "demo",
});
});
it("hands back read-only refs to the core tables a plugin may join", async () => {
grants.set("demo", ["db:own"]);
const { ctx } = contextFor("demo", ["db:own"]);
const refs = (await ctx.db.refs()) as Record<string, unknown>;
expect(Object.keys(refs).sort()).toEqual(["hosts", "users"]);
});
});
describe("ctx.kv limits", () => {
it("refuses a key over the length cap", async () => {
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await expect(ctx.kv.set("k".repeat(129), 1)).rejects.toThrow(
/at most 128 characters/,
);
});
it("refuses a value over the size cap", async () => {
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await expect(ctx.kv.set("big", "x".repeat(300_000))).rejects.toThrow(
/byte limit/,
);
});
it("refuses a new key past the key-count cap", async () => {
process.env.PLUGIN_MAX_KV_KEYS = "2";
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await ctx.kv.set("a", 1);
await ctx.kv.set("b", 2);
await expect(ctx.kv.set("c", 3)).rejects.toThrow(/key limit/);
});
it("still allows overwriting an existing key when full", async () => {
// Only a new key grows the table, so a full store is not read-only.
process.env.PLUGIN_MAX_KV_KEYS = "2";
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await ctx.kv.set("a", 1);
await ctx.kv.set("b", 2);
await expect(ctx.kv.set("a", 99)).resolves.toBeUndefined();
expect(await ctx.kv.get("a")).toBe(99);
});
it("counts each plugin's keys separately", async () => {
process.env.PLUGIN_MAX_KV_KEYS = "1";
grants.set("one", ["kv:own"]);
grants.set("two", ["kv:own"]);
await contextFor("one", ["kv:own"]).ctx.kv.set("a", 1);
await expect(
contextFor("two", ["kv:own"]).ctx.kv.set("a", 1),
).resolves.toBeUndefined();
});
});
describe("ctx.sync", () => {
it("registers an entity a plugin owns", () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
ctx.sync.registerEntity({ type: "demoThings", table: {}, order: 200 });
expect(listEntityTypes()).toContain("demoThings");
});
it("drops the registration when the plugin is disposed", async () => {
grants.set("demo", []);
const { ctx, handle } = contextFor("demo", []);
ctx.sync.registerEntity({ type: "demoThings", table: {} });
await handle.bag.disposeAll();
expect(listEntityTypes()).not.toContain("demoThings");
});
});
@@ -0,0 +1,345 @@
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
vi.mock("../../utils/logger.js", () => ({
pluginLogger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
success: vi.fn(),
},
}));
import {
applyPluginMigrations,
assertOwnedTables,
adoptLegacyTableSql,
checksum,
PluginMigrationChecksumError,
readMigrations,
splitStatements,
LEGACY_TABLE_OWNERS,
type MigrationRunner,
type PluginMigration,
} from "../../plugins/migrations.js";
let root: string;
function writeMigration(
pluginDir: string,
dialect: string,
file: string,
body: string,
): void {
const dir = path.join(pluginDir, "migrations", dialect);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, file), body);
}
function createRunner(overrides: Partial<MigrationRunner> = {}) {
const executed: string[][] = [];
const recorded: PluginMigration[] = [];
const applied: { migrationId: string; checksum: string }[] = [];
const runner: MigrationRunner = {
dialect: "sqlite",
listApplied: async () => applied,
execute: async (statements) => {
executed.push(statements);
},
record: async (_pluginId, migration) => {
recorded.push(migration);
applied.push({
migrationId: migration.id,
checksum: migration.checksum,
});
},
...overrides,
};
return { runner, executed, recorded, applied };
}
beforeEach(() => {
root = fs.mkdtempSync(path.join(os.tmpdir(), "termix-migrations-"));
});
afterEach(() => {
fs.rmSync(root, { recursive: true, force: true });
});
describe("readMigrations", () => {
it("reads files in sequence order regardless of directory order", () => {
writeMigration(root, "sqlite", "0002_second.sql", "SELECT 2;");
writeMigration(root, "sqlite", "0010_tenth.sql", "SELECT 10;");
writeMigration(root, "sqlite", "0001_first.sql", "SELECT 1;");
const migrations = readMigrations(root, "sqlite");
expect(migrations.map((m) => m.id)).toEqual([
"0001_first",
"0002_second",
"0010_tenth",
]);
});
it("returns nothing when the plugin ships no migrations", () => {
expect(readMigrations(root, "sqlite")).toEqual([]);
});
it("rejects a .sql file that is not named NNNN_name.sql", () => {
writeMigration(root, "sqlite", "init.sql", "SELECT 1;");
expect(() => readMigrations(root, "sqlite")).toThrow(/NNNN_name\.sql/);
});
it("is insensitive to line endings, so a checkout does not look edited", () => {
expect(checksum("CREATE TABLE a;\nSELECT 1;\n")).toBe(
checksum("CREATE TABLE a;\r\nSELECT 1;\r\n"),
);
});
});
describe("splitStatements", () => {
it("splits on semicolons", () => {
expect(splitStatements("SELECT 1; SELECT 2;")).toEqual([
"SELECT 1",
"SELECT 2",
]);
});
it("keeps a semicolon inside a string literal", () => {
const statements = splitStatements(
"INSERT INTO t VALUES ('a;b'); SELECT 1;",
);
expect(statements).toEqual(["INSERT INTO t VALUES ('a;b')", "SELECT 1"]);
});
it("keeps an escaped quote inside a string literal", () => {
const statements = splitStatements("INSERT INTO t VALUES ('it''s; fine');");
expect(statements).toEqual(["INSERT INTO t VALUES ('it''s; fine')"]);
});
it("keeps a semicolon inside a line comment", () => {
const statements = splitStatements("-- a; comment\nSELECT 1;");
expect(statements).toHaveLength(1);
expect(statements[0]).toContain("SELECT 1");
});
it("keeps a semicolon inside a block comment", () => {
const statements = splitStatements("/* a; b */ SELECT 1;");
expect(statements).toHaveLength(1);
});
it("ignores a trailing statement that is only whitespace", () => {
expect(splitStatements("SELECT 1;\n\n \n")).toEqual(["SELECT 1"]);
});
});
describe("assertOwnedTables", () => {
it("allows a table in the plugin's own namespace", () => {
expect(() =>
assertOwnedTables("my-plugin", [
"CREATE TABLE IF NOT EXISTS p_my_plugin_thing (id integer)",
]),
).not.toThrow();
});
it("rejects a table outside the plugin's namespace", () => {
expect(() =>
assertOwnedTables("my-plugin", ["CREATE TABLE users (id integer)"]),
).toThrow(/may only create or alter tables prefixed "p_my_plugin_"/);
});
it("rejects altering a core table", () => {
expect(() =>
assertOwnedTables("my-plugin", [
"ALTER TABLE ssh_data ADD COLUMN x text",
]),
).toThrow(/p_my_plugin_/);
});
it("allows the one plugin that may adopt a legacy table", () => {
expect(() =>
assertOwnedTables("fleets", [
'ALTER TABLE "fleets" RENAME TO "p_fleets_fleets"',
]),
).not.toThrow();
});
it("names the owner when another plugin tries to adopt its table", () => {
expect(() =>
assertOwnedTables("workspaces", [
'ALTER TABLE "fleets" RENAME TO "p_workspaces_fleets"',
]),
).toThrow(/belongs to the "fleets" plugin/);
});
it("maps every legacy table to exactly one owning plugin", () => {
// A table owned by two plugins would let either one adopt it and lose the
// other's data.
const owners = Object.entries(LEGACY_TABLE_OWNERS);
expect(new Set(owners.map(([table]) => table)).size).toBe(owners.length);
});
});
describe("applyPluginMigrations", () => {
it("applies pending migrations in order and records each one", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
writeMigration(
root,
"sqlite",
"0002_more.sql",
"CREATE TABLE p_demo_b (id integer);",
);
const { runner, recorded } = createRunner();
const ids = await applyPluginMigrations("demo", root, runner);
expect(ids).toEqual(["0001_init", "0002_more"]);
expect(recorded.map((m) => m.id)).toEqual(["0001_init", "0002_more"]);
});
it("skips a migration that has already been applied", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const body = fs.readFileSync(
path.join(root, "migrations", "sqlite", "0001_init.sql"),
"utf8",
);
const { runner, executed } = createRunner({
listApplied: async () => [
{ migrationId: "0001_init", checksum: checksum(body) },
],
});
const ids = await applyPluginMigrations("demo", root, runner);
expect(ids).toEqual([]);
expect(executed).toEqual([]);
});
it("refuses to run when an applied migration was edited", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const { runner } = createRunner({
listApplied: async () => [
{ migrationId: "0001_init", checksum: "stale-checksum" },
],
});
await expect(applyPluginMigrations("demo", root, runner)).rejects.toThrow(
PluginMigrationChecksumError,
);
});
it("does not record a migration whose statements failed", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const { runner, recorded } = createRunner({
execute: async () => {
throw new Error("syntax error");
},
});
await expect(applyPluginMigrations("demo", root, runner)).rejects.toThrow(
"syntax error",
);
expect(recorded).toEqual([]);
});
it("refuses a migration that reaches outside the plugin's namespace", async () => {
writeMigration(root, "sqlite", "0001_init.sql", "DROP TABLE users;");
const { runner, executed } = createRunner();
await expect(applyPluginMigrations("demo", root, runner)).rejects.toThrow(
/p_demo_/,
);
expect(executed).toEqual([]);
});
it("runs inside a transaction when the dialect offers one", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const order: string[] = [];
const { runner } = createRunner({
transaction: async (fn) => {
order.push("begin");
await fn();
order.push("commit");
},
execute: async () => {
order.push("execute");
},
});
await applyPluginMigrations("demo", root, runner);
expect(order).toEqual(["begin", "execute", "commit"]);
});
it("reads the directory for the dialect it was given", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (x);",
);
writeMigration(
root,
"mysql",
"0001_init.sql",
"CREATE TABLE p_demo_a (x int);",
);
const { runner, executed } = createRunner({ dialect: "mysql" });
await applyPluginMigrations("demo", root, runner);
expect(executed[0][0]).toContain("int");
});
});
describe("adoptLegacyTableSql", () => {
it("uses ALTER TABLE RENAME on sqlite and postgres", () => {
expect(adoptLegacyTableSql("sqlite", "fleets", "p_fleets_fleets")).toBe(
'ALTER TABLE "fleets" RENAME TO "p_fleets_fleets";',
);
expect(adoptLegacyTableSql("postgres", "fleets", "p_fleets_fleets")).toBe(
'ALTER TABLE "fleets" RENAME TO "p_fleets_fleets";',
);
});
it("uses RENAME TABLE and backticks on mysql", () => {
expect(adoptLegacyTableSql("mysql", "fleets", "p_fleets_fleets")).toBe(
"RENAME TABLE `fleets` TO `p_fleets_fleets`;",
);
});
});
@@ -0,0 +1,382 @@
/**
* Plugin-owned data against a real engine.
*
* The DDL snapshots in table-ddl.test.ts prove what the emitter writes; this
* proves an engine accepts it, that adoption moves real rows, and that
* removing a plugin's data removes exactly its own.
*/
import Database from "better-sqlite3";
import { drizzle } from "drizzle-orm/better-sqlite3";
import { sql } from "drizzle-orm";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
defineTable,
id,
refUser,
text,
timestamp,
varchar,
} from "@termix/plugin-sdk/db";
import { createTableSql, dropTableSql } from "../../plugins/table-builder.js";
import {
adoptLegacyTableSql,
applyPluginMigrations,
checksum,
PluginMigrationChecksumError,
tableExistsSql,
type MigrationRunner,
} from "../../plugins/migrations.js";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
vi.mock("../../utils/logger.js", () => ({
pluginLogger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
success: vi.fn(),
},
}));
let sqlite: Database.Database;
let db: ReturnType<typeof drizzle>;
let root: string;
const notes = defineTable(
"note",
{
id: id(),
userId: refUser(),
title: varchar(200).notNull(),
body: text(),
createdAt: timestamp().notNull().defaultNow(),
},
{ indexes: [{ name: "idx_note_user", columns: ["userId"] }] },
);
/** The ledger, kept in the same database so a rollback would take it too. */
function createRunner(): MigrationRunner {
return {
dialect: "sqlite",
listApplied: async (pluginId) =>
sqlite
.prepare(
"SELECT migration_id as migrationId, checksum FROM plugin_migrations WHERE plugin_id = ?",
)
.all(pluginId) as { migrationId: string; checksum: string }[],
execute: async (statements) => {
for (const statement of statements) sqlite.exec(statement);
},
record: async (pluginId, migration) => {
sqlite
.prepare(
"INSERT INTO plugin_migrations (plugin_id, migration_id, checksum, applied_at) VALUES (?, ?, ?, ?)",
)
.run(
pluginId,
migration.id,
migration.checksum,
new Date().toISOString(),
);
},
};
}
function writeMigration(dialect: string, file: string, body: string): void {
const dir = path.join(root, "migrations", dialect);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, file), body);
}
function tableExists(name: string): boolean {
return (
(
sqlite
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
)
.all(name) as unknown[]
).length > 0
);
}
beforeEach(() => {
root = fs.mkdtempSync(path.join(os.tmpdir(), "termix-plugin-data-"));
sqlite = new Database(":memory:");
sqlite.exec("PRAGMA foreign_keys = ON");
sqlite.exec(`
CREATE TABLE users (id TEXT PRIMARY KEY, username TEXT NOT NULL);
CREATE TABLE ssh_data (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT);
CREATE TABLE plugin_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
plugin_id TEXT NOT NULL,
migration_id TEXT NOT NULL,
checksum TEXT NOT NULL,
applied_at TEXT NOT NULL,
UNIQUE (plugin_id, migration_id)
);
`);
sqlite
.prepare("INSERT INTO users (id, username) VALUES (?, ?)")
.run("u1", "ada");
db = drizzle(sqlite);
});
afterEach(() => {
sqlite.close();
fs.rmSync(root, { recursive: true, force: true });
});
describe("emitted DDL against a real engine", () => {
it("creates the table and its indexes", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
expect(tableExists("p_demo_note")).toBe(true);
});
it("enforces the foreign key to users", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
expect(() =>
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("nobody", "orphan"),
).toThrow(/FOREIGN KEY/);
});
it("cascades a delete from the referenced user", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("u1", "mine");
sqlite.prepare("DELETE FROM users WHERE id = ?").run("u1");
const rows = sqlite.prepare("SELECT * FROM p_demo_note").all();
expect(rows).toHaveLength(0);
});
it("fills the timestamp default the emitter wrote", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("u1", "mine");
const [row] = sqlite
.prepare("SELECT created_at FROM p_demo_note")
.all() as {
created_at: string;
}[];
expect(row.created_at).toBeTruthy();
});
it("answers the table-exists probe for the active dialect", async () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
const found = await db.all(tableExistsSql("sqlite", "p_demo_note"));
const missing = await db.all(tableExistsSql("sqlite", "p_demo_nothing"));
expect(found).toHaveLength(1);
expect(missing).toHaveLength(0);
});
});
describe("applying migrations for real", () => {
it("creates the tables and records the ledger", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "demo", notes).join("\n"),
);
const applied = await applyPluginMigrations("demo", root, createRunner());
expect(applied).toEqual(["0001_init"]);
expect(tableExists("p_demo_note")).toBe(true);
expect(
sqlite.prepare("SELECT * FROM plugin_migrations").all(),
).toHaveLength(1);
});
it("is a no-op the second time, so a restart does not re-run DDL", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "demo", notes).join("\n"),
);
await applyPluginMigrations("demo", root, createRunner());
const again = await applyPluginMigrations("demo", root, createRunner());
expect(again).toEqual([]);
});
it("blocks the plugin when an applied migration was edited", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "demo", notes).join("\n"),
);
await applyPluginMigrations("demo", root, createRunner());
writeMigration(
"sqlite",
"0001_init.sql",
`${createTableSql("sqlite", "demo", notes).join("\n")}\n-- edited`,
);
await expect(
applyPluginMigrations("demo", root, createRunner()),
).rejects.toThrow(PluginMigrationChecksumError);
});
it("leaves one plugin's failure without touching another's tables", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "good", notes).join("\n"),
);
await applyPluginMigrations("good", root, createRunner());
const brokenRoot = fs.mkdtempSync(path.join(os.tmpdir(), "termix-broken-"));
fs.mkdirSync(path.join(brokenRoot, "migrations", "sqlite"), {
recursive: true,
});
fs.writeFileSync(
path.join(brokenRoot, "migrations", "sqlite", "0001_init.sql"),
"CREATE TABLE p_broken_x (id integer); THIS IS NOT SQL;",
);
await expect(
applyPluginMigrations("broken", brokenRoot, createRunner()),
).rejects.toThrow();
// The good plugin is untouched, and core is still usable.
expect(tableExists("p_good_note")).toBe(true);
expect(sqlite.prepare("SELECT * FROM users").all()).toHaveLength(1);
fs.rmSync(brokenRoot, { recursive: true, force: true });
});
});
describe("adopting a legacy core table", () => {
it("renames the table and keeps every row", async () => {
sqlite.exec(
"CREATE TABLE fleets (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
);
sqlite.prepare("INSERT INTO fleets (name) VALUES (?)").run("production");
sqlite.prepare("INSERT INTO fleets (name) VALUES (?)").run("staging");
writeMigration(
"sqlite",
"0001_adopt.sql",
adoptLegacyTableSql("sqlite", "fleets", "p_fleets_fleets"),
);
await applyPluginMigrations("fleets", root, createRunner());
expect(tableExists("fleets")).toBe(false);
const rows = sqlite
.prepare("SELECT name FROM p_fleets_fleets ORDER BY id")
.all() as { name: string }[];
expect(rows.map((row) => row.name)).toEqual(["production", "staging"]);
});
it("creates the table fresh when there is no legacy one", async () => {
const fresh = defineTable("fleets", { id: id(), name: varchar(100) });
writeMigration(
"sqlite",
"0001_adopt.sql",
createTableSql("sqlite", "fleets", fresh).join("\n"),
);
await applyPluginMigrations("fleets", root, createRunner());
expect(tableExists("p_fleets_fleets")).toBe(true);
expect(sqlite.prepare("SELECT * FROM p_fleets_fleets").all()).toHaveLength(
0,
);
});
it("refuses a plugin adopting a table that belongs to another", async () => {
sqlite.exec("CREATE TABLE fleets (id INTEGER PRIMARY KEY)");
writeMigration(
"sqlite",
"0001_steal.sql",
adoptLegacyTableSql("sqlite", "fleets", "p_workspaces_fleets"),
);
await expect(
applyPluginMigrations("workspaces", root, createRunner()),
).rejects.toThrow(/belongs to the "fleets" plugin/);
// Refused before execution, so the legacy table is still there.
expect(tableExists("fleets")).toBe(true);
});
});
describe("removing a plugin's data", () => {
it("drops the plugin's tables and leaves everything else", async () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
const other = defineTable("thing", { id: id(), name: varchar(50) });
for (const statement of createTableSql("sqlite", "other", other)) {
sqlite.exec(statement);
}
sqlite.exec(dropTableSql("sqlite", "demo", notes));
expect(tableExists("p_demo_note")).toBe(false);
expect(tableExists("p_other_thing")).toBe(true);
expect(tableExists("users")).toBe(true);
});
it("clears only the removed plugin's ledger rows", () => {
for (const pluginId of ["demo", "other"]) {
sqlite
.prepare(
"INSERT INTO plugin_migrations (plugin_id, migration_id, checksum, applied_at) VALUES (?, ?, ?, ?)",
)
.run(pluginId, "0001_init", checksum("x"), new Date().toISOString());
}
sqlite
.prepare("DELETE FROM plugin_migrations WHERE plugin_id = ?")
.run("demo");
const remaining = sqlite
.prepare("SELECT plugin_id FROM plugin_migrations")
.all() as { plugin_id: string }[];
expect(remaining.map((row) => row.plugin_id)).toEqual(["other"]);
});
});
describe("querying through the built table", () => {
it("reads back a row written with raw SQL", async () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("u1", "written");
const rows = (await db.all(sql.raw("SELECT title FROM p_demo_note"))) as {
title: string;
}[];
expect(rows[0].title).toBe("written");
});
});
@@ -0,0 +1,162 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import {
CORE_OWNER,
getEntity,
hasEntity,
listEntities,
listEntityTypes,
registerEntity,
resetSyncRegistry,
unregisterByOwner,
} from "../../plugins/sync-registry.js";
import { registerCoreSyncEntities } from "../../database/routes/sync-entities.js";
import { SYNCED_ENTITY_TYPES } from "../../../../electron/remote-sync-entities.cjs";
beforeEach(() => {
resetSyncRegistry();
});
afterEach(() => {
resetSyncRegistry();
});
describe("sync registry", () => {
it("orders entities by their declared order, then by name", () => {
registerEntity("core", { type: "b", table: {}, order: 20 });
registerEntity("core", { type: "a", table: {}, order: 10 });
registerEntity("core", { type: "c", table: {}, order: 10 });
expect(listEntityTypes()).toEqual(["a", "c", "b"]);
});
it("defaults the user column and the optional lists", () => {
registerEntity("core", { type: "thing", table: {} });
expect(getEntity("thing")).toMatchObject({
userColumn: "userId",
readOnlyFields: [],
encryptedFields: [],
});
});
it("refuses an entity another owner already registered", () => {
registerEntity("core", { type: "hosts", table: {} });
expect(() =>
registerEntity("evil-plugin", { type: "hosts", table: {} }),
).toThrow(/already registered by "core"/);
});
it("lets the same owner re-register, which is what reactivation does", () => {
registerEntity("demo", { type: "thing", table: {}, order: 1 });
registerEntity("demo", { type: "thing", table: {}, order: 2 });
expect(getEntity("thing")?.order).toBe(2);
});
it("removes an entity when its registration is disposed", () => {
const dispose = registerEntity("demo", { type: "thing", table: {} });
expect(hasEntity("thing")).toBe(true);
dispose();
expect(hasEntity("thing")).toBe(false);
});
it("does not let a stale disposer remove a newer registration", () => {
// A plugin that crashed and restarted must not revoke the registration
// its own restart installed.
const stale = registerEntity("demo", { type: "thing", table: {} });
resetSyncRegistry();
registerEntity("other", { type: "thing", table: {} });
stale();
expect(getEntity("thing")?.owner).toBe("other");
});
it("removes everything an owner registered", () => {
registerEntity("demo", { type: "a", table: {} });
registerEntity("demo", { type: "b", table: {} });
registerEntity("core", { type: "c", table: {} });
unregisterByOwner("demo");
expect(listEntityTypes()).toEqual(["c"]);
});
it("needs a type", () => {
expect(() => registerEntity("demo", { type: "", table: {} })).toThrow(
/needs a type/,
);
});
});
describe("core sync entities", () => {
beforeEach(() => {
registerCoreSyncEntities();
});
it("registers every entity the Electron client knows about", () => {
// Nothing asserted this before, which is how networkTopology came to be
// missing from the list in sync.test.ts.
expect([...listEntityTypes()].sort()).toEqual(
[...SYNCED_ENTITY_TYPES].sort(),
);
});
it("keeps the dependency order the frozen Electron array encodes", () => {
expect(listEntityTypes()).toEqual([...SYNCED_ENTITY_TYPES]);
});
it("owns all of them as core", () => {
for (const entity of listEntities()) {
expect(entity.owner).toBe(CORE_OWNER);
}
});
it("sorts every reference target before the entity that points at it", () => {
const order = new Map(
listEntities().map((entity) => [entity.type, entity.order]),
);
for (const entity of listEntities()) {
for (const reference of entity.references ?? []) {
// A self-reference is ordered within the entity by orderSyncRows.
if (reference.entityType === entity.type) continue;
expect(
order.get(reference.entityType),
`${entity.type} -> ${reference.entityType}`,
).toBeLessThan(entity.order);
}
}
});
it("marks the two singletons and nothing else", () => {
const singletons = listEntities()
.filter((entity) => entity.singleton)
.map((entity) => entity.type);
expect(singletons.sort()).toEqual(["networkTopology", "userPreferences"]);
});
it("keeps the read-only fields that must survive a sync payload", () => {
expect(getEntity("hosts")?.readOnlyFields).toEqual(["connectionOrigin"]);
expect(getEntity("userPreferences")?.readOnlyFields).toEqual([
"storageMode",
]);
});
it("gives networkTopology the custom serializer its JSON blob needs", () => {
const topology = getEntity("networkTopology");
expect(topology?.serialize).toBeTypeOf("function");
expect(topology?.deserialize).toBeTypeOf("function");
});
it("is idempotent, because several modules prime it", () => {
const before = listEntityTypes();
registerCoreSyncEntities();
expect(listEntityTypes()).toEqual(before);
});
});
+255
View File
@@ -0,0 +1,255 @@
import { describe, expect, it } from "vitest";
import {
defineTable,
id,
varchar,
text,
json,
boolean,
timestamp,
integer,
bigint,
encryptedText,
refUser,
refHost,
prefixedTableName,
tablePrefix,
} from "@termix/plugin-sdk/db";
import {
createTableSql,
dropTableSql,
addColumnSql,
keyedColumns,
buildTable,
} from "../../plugins/table-builder.js";
/**
* The three-dialect DDL is what verify:dialect would catch against a real
* server, and that needs a live Postgres and MySQL. These snapshots are what
* covers it in a run that has neither.
*/
const definition = defineTable(
"note",
{
id: id(),
userId: refUser(),
hostId: refHost(),
title: varchar(200).notNull(),
body: text(),
meta: json(),
secret: encryptedText(),
pinned: boolean().default(false),
position: integer().default(0),
size: bigint(),
createdAt: timestamp().notNull().defaultNow(),
},
{
uniques: [{ name: "idx_note_user_title", columns: ["userId", "title"] }],
indexes: [{ name: "idx_note_host", columns: ["hostId"] }],
},
);
describe("table naming", () => {
it("prefixes with the plugin id, dashes becoming underscores", () => {
expect(tablePrefix("network-topology")).toBe("p_network_topology_");
expect(prefixedTableName("network-topology", "graph")).toBe(
"p_network_topology_graph",
);
});
});
describe("defineTable validation", () => {
it("refuses an index on an unbounded text column", () => {
expect(() =>
defineTable(
"bad",
{ id: id(), body: text() },
{ indexes: [{ name: "idx_bad", columns: ["body"] }] },
),
).toThrow(/MySQL cannot index an unbounded column/);
});
it("refuses an index on a json column", () => {
expect(() =>
defineTable(
"bad",
{ id: id(), meta: json() },
{ indexes: [{ name: "idx_bad", columns: ["meta"] }] },
),
).toThrow(/MySQL cannot index an unbounded column/);
});
it("refuses an index naming a column the table does not declare", () => {
expect(() =>
defineTable(
"bad",
{ id: id() },
{ indexes: [{ name: "idx_bad", columns: ["nope"] }] },
),
).toThrow(/which table "bad" does not declare/);
});
it("refuses a table name that is not lower snake_case", () => {
expect(() => defineTable("Notes", { id: id() })).toThrow(
/lower snake_case/,
);
});
it("refuses a table with no columns", () => {
expect(() => defineTable("empty", {})).toThrow(/declares no columns/);
});
it("treats uniques as indexes that happen to be unique", () => {
const table = defineTable(
"thing",
{ id: id(), name: varchar(50) },
{ uniques: [{ name: "idx_thing_name", columns: ["name"] }] },
);
expect(table.indexes[0]).toMatchObject({
name: "idx_thing_name",
unique: true,
});
});
});
describe("keyedColumns", () => {
it("counts primary keys, uniques, refs and indexed columns", () => {
const keyed = keyedColumns(definition);
expect(keyed.has("userId")).toBe(true);
expect(keyed.has("hostId")).toBe(true);
expect(keyed.has("title")).toBe(true);
// Not keyed, so it stays TEXT and is not narrowed to varchar(255).
expect(keyed.has("body")).toBe(false);
expect(keyed.has("meta")).toBe(false);
});
});
describe("createTableSql", () => {
it("emits sqlite DDL", () => {
expect(createTableSql("sqlite", "demo", definition).join("\n"))
.toMatchInlineSnapshot(`
"CREATE TABLE IF NOT EXISTS "p_demo_note" (
"id" integer PRIMARY KEY AUTOINCREMENT,
"user_id" text NOT NULL,
"host_id" integer NOT NULL,
"title" text NOT NULL,
"body" text,
"meta" text,
"secret" text,
"pinned" integer DEFAULT 0,
"position" integer DEFAULT 0,
"size" integer,
"created_at" text NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE,
FOREIGN KEY ("host_id") REFERENCES "ssh_data" ("id") ON DELETE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS "idx_note_user_title" ON "p_demo_note" ("user_id", "title");
CREATE INDEX IF NOT EXISTS "idx_note_host" ON "p_demo_note" ("host_id");"
`);
});
it("emits postgres DDL with serial and native booleans", () => {
expect(createTableSql("postgres", "demo", definition).join("\n"))
.toMatchInlineSnapshot(`
"CREATE TABLE IF NOT EXISTS "p_demo_note" (
"id" serial PRIMARY KEY,
"user_id" varchar(255) NOT NULL,
"host_id" integer NOT NULL,
"title" varchar(200) NOT NULL,
"body" text,
"meta" text,
"secret" text,
"pinned" boolean DEFAULT false,
"position" integer DEFAULT 0,
"size" bigint,
"created_at" text NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE,
FOREIGN KEY ("host_id") REFERENCES "ssh_data" ("id") ON DELETE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS "idx_note_user_title" ON "p_demo_note" ("user_id", "title");
CREATE INDEX IF NOT EXISTS "idx_note_host" ON "p_demo_note" ("host_id");"
`);
});
it("emits mysql DDL with backticks and a parenthesised timestamp default", () => {
expect(createTableSql("mysql", "demo", definition).join("\n"))
.toMatchInlineSnapshot(`
"CREATE TABLE IF NOT EXISTS \`p_demo_note\` (
\`id\` int AUTO_INCREMENT PRIMARY KEY,
\`user_id\` varchar(255) NOT NULL,
\`host_id\` int NOT NULL,
\`title\` varchar(200) NOT NULL,
\`body\` text,
\`meta\` text,
\`secret\` text,
\`pinned\` boolean DEFAULT false,
\`position\` int DEFAULT 0,
\`size\` bigint,
\`created_at\` text NOT NULL DEFAULT (CURRENT_TIMESTAMP),
FOREIGN KEY (\`user_id\`) REFERENCES \`users\` (\`id\`) ON DELETE CASCADE,
FOREIGN KEY (\`host_id\`) REFERENCES \`ssh_data\` (\`id\`) ON DELETE CASCADE
);
CREATE UNIQUE INDEX \`idx_note_user_title\` ON \`p_demo_note\` (\`user_id\`, \`title\`);
CREATE INDEX \`idx_note_host\` ON \`p_demo_note\` (\`host_id\`);"
`);
});
it("never leaves an indexed column as TEXT off sqlite", () => {
// MySQL 8 rejects a TEXT column in a key specification. MariaDB accepts
// it, which is why this was missed in core until a real MySQL saw it.
for (const dialect of ["postgres", "mysql"] as const) {
const sql = createTableSql(dialect, "demo", definition).join("\n");
const keyed = keyedColumns(definition);
for (const property of keyed) {
const column = property.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
const line = sql
.split("\n")
.find(
(row) => row.includes(`${column}\``) || row.includes(`"${column}"`),
);
expect(line, `${dialect}.${column}`).not.toMatch(/\s(text|json)[,\s]/);
}
}
});
it("omits IF NOT EXISTS on a mysql index, which does not support it", () => {
const mysql = createTableSql("mysql", "demo", definition);
const indexes = mysql.filter((statement) => statement.includes("INDEX"));
expect(indexes).not.toHaveLength(0);
for (const statement of indexes) {
expect(statement).not.toContain("IF NOT EXISTS");
}
});
});
describe("addColumnSql", () => {
it("emits an ALTER per dialect", () => {
expect(addColumnSql("sqlite", "demo", definition, "body")).toBe(
'ALTER TABLE "p_demo_note" ADD COLUMN "body" text;',
);
expect(addColumnSql("mysql", "demo", definition, "title")).toBe(
"ALTER TABLE `p_demo_note` ADD COLUMN `title` varchar(200) NOT NULL;",
);
});
});
describe("dropTableSql", () => {
it("drops the prefixed table", () => {
expect(dropTableSql("postgres", "demo", definition)).toBe(
'DROP TABLE IF EXISTS "p_demo_note";',
);
});
});
describe("buildTable", () => {
it("builds a queryable drizzle table carrying the prefixed name", async () => {
const { getTableName, getTableColumns } = await import("drizzle-orm");
const table = buildTable("demo", definition) as never;
expect(getTableName(table)).toBe("p_demo_note");
expect(Object.keys(getTableColumns(table))).toContain("userId");
});
});