feat: give plugins their own tables, migrations and sync entities

This commit is contained in:
LukeGus committed 2026-09-22 15:49:54 -05:00
1 parent 5796186a1c
commit c496a093ca
47 files changed
+32595 -290

No files matched your search

+22 -69
View File
@@ -741,6 +741,16 @@ async function initializeCompleteDatabase(): Promise<void> {
FOREIGN KEY (plugin_id) REFERENCES plugins (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS plugin_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
plugin_id TEXT NOT NULL,
migration_id TEXT NOT NULL,
checksum TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (plugin_id, migration_id),
FOREIGN KEY (plugin_id) REFERENCES plugins (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS api_keys (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
@@ -2440,29 +2450,20 @@ const migrateSchema = () => {
// --- proxmox-node-history end ---
// --- alerts begin ---
try {
sqlite.prepare("SELECT id FROM alert_rules LIMIT 1").get();
} catch {
// alert_rules, alert_rule_channels and alert_firings were only ever created
// here, never declared in schema.ts, so they never existed on Postgres or
// MySQL at all. Automations replaced them; drop the SQLite leftovers.
for (const table of [
"alert_firings",
"alert_rule_channels",
"alert_rules",
]) {
try {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS alert_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
host_id INTEGER REFERENCES ssh_data(id) ON DELETE CASCADE,
name TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
trigger_type TEXT NOT NULL,
threshold_value REAL,
threshold_duration_seconds INTEGER,
cooldown_minutes INTEGER NOT NULL DEFAULT 15,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`);
} catch (createError) {
databaseLogger.warn("Failed to create alert_rules table", {
sqlite.exec(`DROP TABLE IF EXISTS ${table};`);
} catch (dropError) {
databaseLogger.warn(`Failed to drop legacy ${table} table`, {
operation: "schema_migration",
error: createError,
error: dropError,
});
}
}
@@ -2489,54 +2490,6 @@ const migrateSchema = () => {
});
}
}
try {
sqlite.prepare("SELECT id FROM alert_rule_channels LIMIT 1").get();
} catch {
try {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS alert_rule_channels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id INTEGER NOT NULL REFERENCES alert_rules(id) ON DELETE CASCADE,
channel_id INTEGER NOT NULL REFERENCES notification_channels(id) ON DELETE CASCADE
);
`);
} catch (createError) {
databaseLogger.warn("Failed to create alert_rule_channels table", {
operation: "schema_migration",
error: createError,
});
}
}
try {
sqlite.prepare("SELECT id FROM alert_firings LIMIT 1").get();
} catch {
try {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS alert_firings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
rule_id INTEGER NOT NULL REFERENCES alert_rules(id) ON DELETE CASCADE,
host_id INTEGER NOT NULL,
host_name TEXT NOT NULL,
fired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
resolved_at TEXT,
value REAL,
message TEXT NOT NULL,
severity TEXT NOT NULL DEFAULT 'warning',
acknowledged INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_alert_firings_user_fired
ON alert_firings (user_id, fired_at DESC);
`);
} catch (createError) {
databaseLogger.warn("Failed to create alert_firings table", {
operation: "schema_migration",
error: createError,
});
}
}
// --- alerts end ---
// --- automations begin ---
+29
View File
@@ -2252,4 +2252,33 @@ export const pluginStorage = mysqlTable(
],
);
/**
* Which of a plugin's migrations have been applied.
*
* The checksum is what makes an already-applied migration immutable: editing
* one that has run blocks that plugin rather than silently leaving two
* databases with different shapes. Cascades with the plugin so removing its
* data leaves no ledger behind.
*/
export const pluginMigrations = mysqlTable(
"plugin_migrations",
{
id: int("id").autoincrement().primaryKey(),
pluginId: varchar("plugin_id", { length: 255 })
.notNull()
.references(() => plugins.id, { onDelete: "cascade" }),
migrationId: varchar("migration_id", { length: 255 }).notNull(),
checksum: text("checksum").notNull(),
appliedAt: text("applied_at")
.notNull()
.default(sql`(CURRENT_TIMESTAMP)`),
},
(table) => [
uniqueIndex("idx_plugin_migrations_plugin_migration").on(
table.pluginId,
table.migrationId,
),
],
);
// --- plugins end ---
+29
View File
@@ -2253,4 +2253,33 @@ export const pluginStorage = pgTable(
],
);
/**
* Which of a plugin's migrations have been applied.
*
* The checksum is what makes an already-applied migration immutable: editing
* one that has run blocks that plugin rather than silently leaving two
* databases with different shapes. Cascades with the plugin so removing its
* data leaves no ledger behind.
*/
export const pluginMigrations = pgTable(
"plugin_migrations",
{
id: serial("id").primaryKey(),
pluginId: varchar("plugin_id", { length: 255 })
.notNull()
.references(() => plugins.id, { onDelete: "cascade" }),
migrationId: varchar("migration_id", { length: 255 }).notNull(),
checksum: text("checksum").notNull(),
appliedAt: text("applied_at")
.notNull()
.default(sql`CURRENT_TIMESTAMP`),
},
(table) => [
uniqueIndex("idx_plugin_migrations_plugin_migration").on(
table.pluginId,
table.migrationId,
),
],
);
// --- plugins end ---
+29
View File
@@ -2251,4 +2251,33 @@ export const pluginStorage = sqliteTable(
],
);
/**
* Which of a plugin's migrations have been applied.
*
* The checksum is what makes an already-applied migration immutable: editing
* one that has run blocks that plugin rather than silently leaving two
* databases with different shapes. Cascades with the plugin so removing its
* data leaves no ledger behind.
*/
export const pluginMigrations = sqliteTable(
"plugin_migrations",
{
id: integer("id").primaryKey({ autoIncrement: true }),
pluginId: text("plugin_id")
.notNull()
.references(() => plugins.id, { onDelete: "cascade" }),
migrationId: text("migration_id").notNull(),
checksum: text("checksum").notNull(),
appliedAt: text("applied_at")
.notNull()
.default(sql`CURRENT_TIMESTAMP`),
},
(table) => [
uniqueIndex("idx_plugin_migrations_plugin_migration").on(
table.pluginId,
table.migrationId,
),
],
);
// --- plugins end ---
@@ -34,6 +34,7 @@ import { OpenTabRepository } from "./open-tab-repository.js";
import { OpksshTokenRepository } from "./opkssh-token-repository.js";
import { PluginRepository } from "./plugin-repository.js";
import { PluginStorageRepository } from "./plugin-storage-repository.js";
import { PluginMigrationRepository } from "./plugin-migration-repository.js";
import { PluginPermissionGrantRepository } from "./plugin-permission-grant-repository.js";
import { PluginRegistryRepository } from "./plugin-registry-repository.js";
import { PluginInstallCountRepository } from "./plugin-install-count-repository.js";
@@ -393,6 +394,13 @@ export function createCurrentPluginStorageRepository(): PluginStorageRepository
);
}
export function createCurrentPluginMigrationRepository(): PluginMigrationRepository {
return new PluginMigrationRepository(
createCurrentRepositoryContext(),
createCurrentRepositoryWriteHook("plugin_migration_repository_write"),
);
}
export function createCurrentPluginRegistryRepository(): PluginRegistryRepository {
return new PluginRegistryRepository(
createCurrentRepositoryContext(),
@@ -0,0 +1,73 @@
import { and, eq } from "drizzle-orm";
import { pluginMigrations } from "../db/schema.js";
import { insertReturning } from "./returning.js";
import { rowsAffected } from "./mutation-result.js";
import type { DatabaseContext } from "./database-context.js";
export type PluginMigrationRecord = typeof pluginMigrations.$inferSelect;
/**
* Which of a plugin's migrations have run.
*
* The checksum is the reason this is a table rather than a file marker: it
* makes an applied migration immutable, so an edited one is caught instead of
* leaving two installs with quietly different schemas.
*/
export class PluginMigrationRepository {
constructor(
private readonly context: DatabaseContext,
private readonly onWrite?: () => void | Promise<void>,
) {}
async listByPlugin(pluginId: string): Promise<PluginMigrationRecord[]> {
return this.context.drizzle
.select()
.from(pluginMigrations)
.where(eq(pluginMigrations.pluginId, pluginId));
}
async record(
pluginId: string,
migrationId: string,
checksum: string,
now = new Date().toISOString(),
): Promise<void> {
await insertReturning(this.context, pluginMigrations, {
pluginId,
migrationId,
checksum,
appliedAt: now,
});
await this.afterWrite();
}
async deleteByPlugin(pluginId: string): Promise<number> {
const result = await this.context.drizzle
.delete(pluginMigrations)
.where(eq(pluginMigrations.pluginId, pluginId));
const affected = rowsAffected(result);
if (affected > 0) await this.afterWrite();
return affected;
}
async has(pluginId: string, migrationId: string): Promise<boolean> {
const rows = await this.context.drizzle
.select()
.from(pluginMigrations)
.where(
and(
eq(pluginMigrations.pluginId, pluginId),
eq(pluginMigrations.migrationId, migrationId),
),
)
.limit(1);
return rows.length > 0;
}
private async afterWrite(): Promise<void> {
await this.onWrite?.();
}
}
@@ -33,6 +33,16 @@ export class PluginStorageRepository {
return rows.map((row) => row.key);
}
/** How many keys this plugin holds, for the per-plugin key cap. */
async countKeys(pluginId: string): Promise<number> {
const rows = await this.context.drizzle
.select()
.from(pluginStorage)
.where(eq(pluginStorage.pluginId, pluginId));
return rows.length;
}
async set(
pluginId: string,
key: string,
@@ -5,17 +5,16 @@ import type { DatabaseContext } from "./database-context.js";
export type SyncTombstoneRecord = typeof syncTombstones.$inferSelect;
export type SyncEntityType =
| "hosts"
| "sshCredentials"
| "sshFolders"
| "snippets"
| "snippetFolders"
| "vaultProfiles"
| "dashboardServiceLinks"
| "homepageItems"
| "userPreferences"
| "networkTopology";
/**
* The wire name of a synced entity.
*
* A string rather than a union: the set is whatever core and the installed
* plugins have registered, so it cannot be known at compile time. The column
* was already free-form text with no enum constraint, so nothing in the
* database changes. sync.ts validates a name through the registry before it
* reaches a query.
*/
export type SyncEntityType = string;
export class SyncTombstoneRepository {
constructor(
+67
View File
@@ -461,4 +461,71 @@ router.delete(
},
);
/**
* @openapi
* /plugins/{id}/data:
* delete:
* summary: Delete everything a plugin stores
* description: Drops the plugin's own tables and clears its key/value state, capability grants and migration ledger. Disabling a plugin never touches its data; this is the explicit uninstall path, and it cannot be undone. The plugin is deactivated first so nothing is writing while its tables go.
* tags:
* - Plugins
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: string
* responses:
* 200:
* description: What was removed.
* 404:
* description: No such plugin.
* 500:
* description: Failed to remove the plugin's data.
*/
router.delete(
"/:id/data",
authenticateJWT,
requireManagePlugins,
async (req: Request, res: Response) => {
const pluginId = String(req.params.id);
const userId = (req as AuthenticatedRequest).userId;
try {
const record = await createCurrentPluginRepository().findById(pluginId);
if (!record) {
res.status(404).json({ error: "No such plugin" });
return;
}
// Stopped first: dropping a table out from under a running plugin turns
// its next query into an error rather than a clean shutdown.
const { loader } = getPluginRuntime();
if (loader.get(pluginId)?.state === "active") {
await loader.deactivate(pluginId);
}
const { removePluginData } = await import("../../plugins/data.js");
const removed = await removePluginData(pluginId);
databaseLogger.warn(`Removed all data for plugin ${pluginId}`, {
operation: "plugin_remove_data",
pluginId,
userId,
tables: removed.tables.length,
kvKeys: removed.kvKeys,
});
res.json({ id: pluginId, removed });
} catch (error) {
databaseLogger.error(
`Failed to remove data for plugin ${pluginId}`,
error instanceof Error ? error : new Error(String(error)),
{ operation: "plugin_remove_data" },
);
res.status(500).json({ error: "Failed to remove the plugin's data" });
}
},
);
export default router;
@@ -0,0 +1,179 @@
/**
* Core's own sync entities, registered the same way a plugin registers one.
*
* These were a hardcoded ENTITY_CONFIG in sync.ts, a SyncEntityType union in
* the tombstone repository, a REFERENCES map in sync-references.ts and a
* frozen array in electron/. Four lists that had to agree and had no test
* saying they did - which is how networkTopology came to be missing from one
* of them. Now there is one registration per entity and the rest is derived.
*
* The order values reproduce the dependency order the Electron array encoded:
* anything referenced by another entity syncs first, and networkTopology
* syncs last because it holds host ids inside a JSON blob.
*/
import {
hosts,
sshCredentials,
sshFolders,
snippets,
snippetFolders,
vaultProfiles,
dashboardServiceLinks,
homepageItems,
userPreferences,
networkTopology,
} from "../db/schema.js";
import {
CORE_OWNER,
registerEntity,
listEntities,
} from "../../plugins/sync-registry.js";
import { mapTopologyHostIds } from "./sync-references.js";
const CREDENTIAL_REFERENCE = {
field: "credentialId",
syncField: "credentialSyncId",
entityType: "sshCredentials",
} as const;
const HOST_REFERENCES = [
CREDENTIAL_REFERENCE,
{
field: "rdpCredentialId",
syncField: "rdpCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vncCredentialId",
syncField: "vncCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "telnetCredentialId",
syncField: "telnetCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vaultProfileId",
syncField: "vaultProfileSyncId",
entityType: "vaultProfiles",
},
{
field: "parentHostId",
syncField: "parentHostSyncId",
entityType: "hosts",
},
] as const;
let registered = false;
/**
* Registers core's entities. Idempotent, because the sync routes and the
* Electron entity endpoint both need the registry primed and neither can
* assume it ran first.
*/
export function registerCoreSyncEntities(): void {
if (registered && listEntities().length > 0) return;
registered = true;
// 10, 20, 30... so a plugin can slot an entity between two core ones
// without every order value having to be renumbered.
registerEntity(CORE_OWNER, {
type: "sshCredentials",
table: sshCredentials,
order: 10,
});
registerEntity(CORE_OWNER, {
type: "vaultProfiles",
table: vaultProfiles,
order: 20,
});
registerEntity(CORE_OWNER, {
type: "sshFolders",
table: sshFolders,
order: 30,
references: [CREDENTIAL_REFERENCE],
});
registerEntity(CORE_OWNER, {
type: "snippetFolders",
table: snippetFolders,
order: 40,
});
registerEntity(CORE_OWNER, {
type: "hosts",
table: hosts,
order: 50,
// connectionOrigin says how this device reached the host, which is never
// true of the other side.
readOnlyFields: ["connectionOrigin"],
references: HOST_REFERENCES,
encryptedFields: ["ssh_data"],
});
registerEntity(CORE_OWNER, {
type: "snippets",
table: snippets,
order: 60,
});
registerEntity(CORE_OWNER, {
type: "dashboardServiceLinks",
table: dashboardServiceLinks,
order: 70,
});
registerEntity(CORE_OWNER, {
type: "homepageItems",
table: homepageItems,
order: 80,
});
registerEntity(CORE_OWNER, {
type: "userPreferences",
table: userPreferences,
order: 90,
singleton: true,
readOnlyFields: ["storageMode"],
});
registerEntity(CORE_OWNER, {
type: "networkTopology",
table: networkTopology,
order: 100,
singleton: true,
// Host ids live inside the topology JSON rather than in a column, so the
// generic reference machinery cannot reach them.
serialize: async (row, resolveSyncId) => ({
...row,
topology: await mapTopologyHostIds(
row.topology as string | null | undefined,
async (id) => {
const numericId = Number(id);
if (!Number.isInteger(numericId)) return null;
return resolveSyncId("hosts", numericId);
},
),
}),
deserialize: async (row, resolveId) => ({
...row,
topology: await mapTopologyHostIds(
row.topology as string | null | undefined,
async (syncId) => {
const id = await resolveId("hosts", syncId);
return id === null ? null : String(id);
},
),
}),
});
}
/** The tables a reference may point at, derived from the registrations. */
export const ENCRYPTED_ENTITY_TABLES: Readonly<Record<string, string>> = {
hosts: "ssh_data",
sshCredentials: "ssh_credentials",
};
+28 -66
View File
@@ -1,6 +1,16 @@
import type { SyncEntityType } from "../repositories/sync-tombstone-repository.js";
import { getEntity } from "../../plugins/sync-registry.js";
export type SyncReferenceEntity = "hosts" | "sshCredentials" | "vaultProfiles";
/**
* An entity another entity's rows may point at.
*
* A plain string rather than a union: the set is whatever is registered, and a
* plugin can both register an entity and reference one. sync.ts resolves the
* name through the registry, so an unknown one is an error there rather than a
* silent fallthrough - which is what the old three-way if/else did, routing
* anything that was not hosts or sshCredentials to vaultProfiles.
*/
export type SyncReferenceEntity = string;
interface TopologyNode {
data?: { id?: string; [key: string]: unknown };
@@ -25,7 +35,7 @@ interface Topology {
* leave group ids untouched. Nodes/edges that fail to resolve (host deleted,
* or not synced to this side yet) are dropped rather than left dangling.
*/
async function mapTopologyHostIds(
export async function mapTopologyHostIds(
topologyJson: string | null | undefined,
mapId: (id: string) => Promise<string | null>,
): Promise<string | null> {
@@ -81,46 +91,16 @@ interface SyncReference {
entityType: SyncReferenceEntity;
}
const CREDENTIAL_REFERENCE: SyncReference = {
field: "credentialId",
syncField: "credentialSyncId",
entityType: "sshCredentials",
};
const HOST_REFERENCES: SyncReference[] = [
CREDENTIAL_REFERENCE,
{
field: "rdpCredentialId",
syncField: "rdpCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vncCredentialId",
syncField: "vncCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "telnetCredentialId",
syncField: "telnetCredentialSyncId",
entityType: "sshCredentials",
},
{
field: "vaultProfileId",
syncField: "vaultProfileSyncId",
entityType: "vaultProfiles",
},
{
field: "parentHostId",
syncField: "parentHostSyncId",
entityType: "hosts",
},
];
export function orderSyncRows(
entityType: SyncEntityType,
rows: Record<string, unknown>[],
): Record<string, unknown>[] {
if (entityType !== "hosts") return rows;
// Only an entity that references itself needs its rows ordered among
// themselves. Today that is hosts, through parentHostId.
const selfReference = referencesFor(entityType).find(
(reference) => reference.entityType === entityType,
);
if (!selfReference) return rows;
const bySyncId = new Map(
rows
@@ -136,7 +116,7 @@ export function orderSyncRows(
if (visiting.has(row)) return;
visiting.add(row);
const parentSyncId = row.parentHostSyncId;
const parentSyncId = row[selfReference.syncField];
if (typeof parentSyncId === "string") {
const parent = bySyncId.get(parentSyncId);
if (parent) visit(parent);
@@ -151,10 +131,9 @@ export function orderSyncRows(
return ordered;
}
const REFERENCES: Partial<Record<SyncEntityType, SyncReference[]>> = {
hosts: HOST_REFERENCES,
sshFolders: [CREDENTIAL_REFERENCE],
};
function referencesFor(entityType: SyncEntityType): readonly SyncReference[] {
return (getEntity(entityType)?.references ?? []) as readonly SyncReference[];
}
export async function serializeSyncReferences(
entityType: SyncEntityType,
@@ -166,19 +145,10 @@ export async function serializeSyncReferences(
): Promise<Record<string, unknown>> {
const serialized = { ...row };
if (entityType === "networkTopology") {
serialized.topology = await mapTopologyHostIds(
serialized.topology as string | null | undefined,
async (id) => {
const numericId = Number(id);
if (!Number.isInteger(numericId)) return null;
return resolveSyncId("hosts", numericId);
},
);
return serialized;
}
const custom = getEntity(entityType)?.serialize;
if (custom) return custom(serialized, resolveSyncId);
for (const reference of REFERENCES[entityType] ?? []) {
for (const reference of referencesFor(entityType)) {
const id = serialized[reference.field];
serialized[reference.syncField] =
typeof id === "number"
@@ -199,18 +169,10 @@ export async function deserializeSyncReferences(
): Promise<Record<string, unknown>> {
const deserialized = { ...row };
if (entityType === "networkTopology") {
deserialized.topology = await mapTopologyHostIds(
deserialized.topology as string | null | undefined,
async (syncId) => {
const id = await resolveId("hosts", syncId);
return id === null ? null : String(id);
},
);
return deserialized;
}
const custom = getEntity(entityType)?.deserialize;
if (custom) return custom(deserialized, resolveId);
for (const reference of REFERENCES[entityType] ?? []) {
for (const reference of referencesFor(entityType)) {
const syncId = deserialized[reference.syncField];
delete deserialized[reference.syncField];
delete deserialized[reference.field];
+95 -116
View File
@@ -1,17 +1,6 @@
import express, { type Request, type Response } from "express";
import { and, eq, type SQL } from "drizzle-orm";
import {
hosts,
sshCredentials,
sshFolders,
snippets,
snippetFolders,
vaultProfiles,
dashboardServiceLinks,
homepageItems,
userPreferences,
networkTopology,
} from "../db/schema.js";
import { hosts } from "../db/schema.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { DataCrypto } from "../../utils/data-crypto.js";
import { databaseLogger } from "../../utils/logger.js";
@@ -22,6 +11,15 @@ import {
createCurrentSyncTombstoneRepository,
} from "../repositories/factory.js";
import type { SyncEntityType } from "../repositories/sync-tombstone-repository.js";
import {
getEntity,
hasEntity,
listEntities,
} from "../../plugins/sync-registry.js";
import {
ENCRYPTED_ENTITY_TABLES,
registerCoreSyncEntities,
} from "./sync-entities.js";
import {
deserializeSyncReferences,
orderSyncRows,
@@ -31,66 +29,44 @@ import {
import { timestampAtOrAfter } from "../sync-timestamp.js";
import { validateParentHostId } from "./host-parent-validation.js";
// Primed at import so every consumer of this module - the routes below, the
// reference resolvers, the Electron entity-types endpoint - sees the same
// registry regardless of which one runs first.
registerCoreSyncEntities();
const router = express.Router();
const authManager = AuthManager.getInstance();
const authenticateJWT = authManager.createAuthMiddleware();
// Encrypted tables need DataCrypto to translate between the wire payload
// (plaintext) and the stored row (encrypted). Everything else is stored
// and synced as-is.
const ENCRYPTED_ENTITY_TABLES: Partial<Record<SyncEntityType, string>> = {
hosts: "ssh_data",
sshCredentials: "ssh_credentials",
};
/**
* What a sync entity looks like once it is registered.
*
* The table was a hand-maintained union of ten concrete drizzle types, which a
* plugin-provided table could never join. It is the same deliberate
* approximation repositories/database-context.ts documents: the query-builder
* surface used here is identical across the engines and across tables, and
* every use already casts to reach .syncId and .updatedAt.
*/
type SyncTable = typeof hosts;
interface EntityConfig {
table:
| typeof hosts
| typeof sshCredentials
| typeof sshFolders
| typeof snippets
| typeof snippetFolders
| typeof vaultProfiles
| typeof dashboardServiceLinks
| typeof homepageItems
| typeof userPreferences
| typeof networkTopology;
// Fields that only make sense on the device that created the row, or
// that are managed elsewhere and must never be overwritten by a sync
// payload from the other side.
readOnlyFields: string[];
singleton?: boolean;
function entityConfig(entityType: SyncEntityType) {
const entity = getEntity(entityType);
if (!entity) {
throw new Error(`Unknown sync entity "${entityType}"`);
}
return {
table: entity.table as SyncTable,
readOnlyFields: entity.readOnlyFields,
singleton: entity.singleton === true,
};
}
const ENTITY_CONFIG: Record<SyncEntityType, EntityConfig> = {
hosts: {
table: hosts,
readOnlyFields: ["connectionOrigin"],
},
sshCredentials: { table: sshCredentials, readOnlyFields: [] },
sshFolders: { table: sshFolders, readOnlyFields: [] },
snippets: { table: snippets, readOnlyFields: [] },
snippetFolders: { table: snippetFolders, readOnlyFields: [] },
vaultProfiles: { table: vaultProfiles, readOnlyFields: [] },
dashboardServiceLinks: { table: dashboardServiceLinks, readOnlyFields: [] },
homepageItems: { table: homepageItems, readOnlyFields: [] },
userPreferences: {
table: userPreferences,
readOnlyFields: ["storageMode"],
singleton: true,
},
networkTopology: {
table: networkTopology,
readOnlyFields: [],
singleton: true,
},
};
const VALID_ENTITY_TYPES = new Set(Object.keys(ENTITY_CONFIG));
type RepositoryContext = ReturnType<typeof createCurrentRepositoryContext>;
export function isValidEntityType(value: unknown): value is SyncEntityType {
return typeof value === "string" && VALID_ENTITY_TYPES.has(value);
if (typeof value !== "string") return false;
registerCoreSyncEntities();
return hasEntity(value);
}
/**
@@ -106,7 +82,7 @@ export function locateSyncRow(
userId: string,
syncId: string,
): SQL {
const { table, singleton } = ENTITY_CONFIG[entityType];
const { table, singleton } = entityConfig(entityType);
if (singleton) {
return eq(table.userId, userId);
@@ -118,34 +94,25 @@ export function locateSyncRow(
)!;
}
/**
* The syncId of a referenced row.
*
* Generic over the registry rather than an if/else over three known entities.
* The old chain ended in an unguarded else that returned vaultProfiles, so a
* fourth reference target would have silently resolved against the wrong
* table; an unregistered entity is an error here instead.
*/
async function findReferenceSyncId(
context: RepositoryContext,
entityType: SyncReferenceEntity,
id: number,
userId: string,
): Promise<string | null> {
if (entityType === "hosts") {
const [row] = await context.drizzle
.select({ syncId: hosts.syncId })
.from(hosts)
.where(and(eq(hosts.id, id), eq(hosts.userId, userId)))
.limit(1);
return row?.syncId ?? null;
}
if (entityType === "sshCredentials") {
const [row] = await context.drizzle
.select({ syncId: sshCredentials.syncId })
.from(sshCredentials)
.where(and(eq(sshCredentials.id, id), eq(sshCredentials.userId, userId)))
.limit(1);
return row?.syncId ?? null;
}
const { table } = entityConfig(entityType as SyncEntityType);
const [row] = await context.drizzle
.select({ syncId: vaultProfiles.syncId })
.from(vaultProfiles)
.where(and(eq(vaultProfiles.id, id), eq(vaultProfiles.userId, userId)))
.select({ syncId: table.syncId })
.from(table)
.where(and(eq(table.id, id), eq(table.userId, userId)))
.limit(1);
return row?.syncId ?? null;
}
@@ -156,35 +123,11 @@ async function findReferenceId(
syncId: string,
userId: string,
): Promise<number | null> {
if (entityType === "hosts") {
const [row] = await context.drizzle
.select({ id: hosts.id })
.from(hosts)
.where(and(eq(hosts.syncId, syncId), eq(hosts.userId, userId)))
.limit(1);
return row?.id ?? null;
}
if (entityType === "sshCredentials") {
const [row] = await context.drizzle
.select({ id: sshCredentials.id })
.from(sshCredentials)
.where(
and(
eq(sshCredentials.syncId, syncId),
eq(sshCredentials.userId, userId),
),
)
.limit(1);
return row?.id ?? null;
}
const { table } = entityConfig(entityType as SyncEntityType);
const [row] = await context.drizzle
.select({ id: vaultProfiles.id })
.from(vaultProfiles)
.where(
and(eq(vaultProfiles.syncId, syncId), eq(vaultProfiles.userId, userId)),
)
.select({ id: table.id })
.from(table)
.where(and(eq(table.syncId, syncId), eq(table.userId, userId)))
.limit(1);
return row?.id ?? null;
}
@@ -230,7 +173,7 @@ export function stripWritePayload(
entityType: SyncEntityType,
payload: Record<string, unknown>,
): Record<string, unknown> {
const { readOnlyFields } = ENTITY_CONFIG[entityType];
const { readOnlyFields } = entityConfig(entityType);
const clean = { ...payload };
delete clean.id;
delete clean.userId;
@@ -239,6 +182,42 @@ export function stripWritePayload(
return clean;
}
/**
* @openapi
* /sync/entity-types:
* get:
* summary: List the entity types this server syncs, in dependency order
* description: The desktop app syncs the entities in the order returned here, because an entity referenced by another has to exist on the far side first. A server that predates this endpoint returns 404 and the client falls back to its built-in list.
* tags:
* - Sync
* responses:
* 200:
* description: The ordered entity types.
* 500:
* description: Failed to list entity types.
*/
router.get(
"/entity-types",
authenticateJWT,
async (_req: Request, res: Response) => {
try {
registerCoreSyncEntities();
return res.json({
entityTypes: listEntities().map((entity) => ({
type: entity.type,
order: entity.order,
singleton: entity.singleton === true,
})),
});
} catch (error) {
databaseLogger.error("Failed to list sync entity types", error, {
operation: "sync_entity_types",
});
return res.status(500).json({ error: "Failed to list entity types" });
}
},
);
/**
* @openapi
* /sync/{entityType}:
@@ -280,7 +259,7 @@ router.get(
: null;
try {
const { table, singleton } = ENTITY_CONFIG[entityType];
const { table, singleton } = entityConfig(entityType);
const context = createCurrentRepositoryContext();
const conditions = [eq(table.userId, userId)];
if (since && "updatedAt" in table) {
@@ -352,7 +331,7 @@ router.post(
}
try {
const { table } = ENTITY_CONFIG[entityType];
const { table } = entityConfig(entityType);
const context = createCurrentRepositoryContext();
await context.drizzle
@@ -418,7 +397,7 @@ router.post(
try {
// singleton is still needed below: those tables have no sync_id column
// for the insert to populate.
const { table, singleton } = ENTITY_CONFIG[entityType];
const { table, singleton } = entityConfig(entityType);
const context = createCurrentRepositoryContext();
const locateRow = locateSyncRow(entityType, userId, syncId);
+77 -5
View File
@@ -31,6 +31,8 @@ import { getActor, runAsActor } from "./actor.js";
import { DisposableBag } from "./disposables.js";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
import type { PluginContext, PluginModule } from "@termix/plugin-sdk/backend";
import type { PluginTableDefinition } from "@termix/plugin-sdk/db";
import * as syncRegistry from "./sync-registry.js";
export type { PluginModule };
@@ -38,6 +40,20 @@ export type { PluginModule };
const MAX_KV_KEY_LENGTH = 128;
const MAX_KV_VALUE_BYTES = 256 * 1024;
/**
* How many keys one plugin may hold.
*
* The value cap alone bounds a single row, not the table: a plugin writing
* unique keys in a loop could still fill the database. Configurable because
* the right ceiling depends on the install, not on the plugin.
*/
const DEFAULT_MAX_KV_KEYS = 10_000;
function maxKvKeys(env: NodeJS.ProcessEnv = process.env): number {
const raw = Number(env.PLUGIN_MAX_KV_KEYS);
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_MAX_KV_KEYS;
}
export interface PluginHandle {
module: PluginModule;
bag: DisposableBag;
@@ -169,11 +185,21 @@ export function createPluginContext(
}
const { createCurrentPluginStorageRepository } =
await import("../database/repositories/factory.js");
await createCurrentPluginStorageRepository().set(
pluginId,
key,
serialized,
);
const repository = createCurrentPluginStorageRepository();
// Only a new key can grow the table, so an overwrite is never blocked
// by a full one. Checked before the write, not after.
const existing = await repository.get(pluginId, key);
if (existing === null) {
const limit = maxKvKeys();
if ((await repository.countKeys(pluginId)) >= limit) {
throw new Error(
`Plugin "${pluginId}" has reached the ${limit} key limit for ctx.kv`,
);
}
}
await repository.set(pluginId, key, serialized);
},
{ action: "kv_set" },
);
@@ -200,6 +226,39 @@ export function createPluginContext(
{ action: "kv_list" },
);
const dbDefine = guarded(
manifest,
"db:own",
async (definition: PluginTableDefinition) => {
const { registerTable } = await import("./data.js");
return registerTable(pluginId, definition);
},
{ action: "db_define", details: () => "table definition registered" },
);
const dbClient = guarded(
manifest,
"db:own",
async () => {
const { getDb } = await import("../database/db/index.js");
return getDb();
},
{ action: "db_client" },
);
const dbRefs = guarded(
manifest,
"db:own",
async () => {
// Read-only by convention, not by engine: a plugin holding these can
// write through them. The capability and the audit line are the record
// that it did. See the header of this file.
const schema = await import("../database/db/schema.js");
return { users: schema.users, hosts: schema.hosts };
},
{ action: "db_refs" },
);
return {
pluginId,
manifest,
@@ -237,6 +296,19 @@ export function createPluginContext(
},
},
db: {
define: (definition) => dbDefine(definition) as never,
client: () => dbClient() as never,
refs: () => dbRefs() as never,
},
sync: {
registerEntity: (entity) => {
const dispose = syncRegistry.registerEntity(pluginId, entity);
handle.bag.add(dispose, `sync entity ${entity.type}`);
},
},
kv: {
get: (key) => kvGet(key),
set: (key, value) => kvSet(key, value),
+177
View File
@@ -0,0 +1,177 @@
/**
* The live database side of plugin-owned data.
*
* Holds the table definitions each plugin has registered, builds the runner
* that applies its migrations against the real database, and drops everything
* again on uninstall. The pure logic lives in migrations.ts so it can be
* tested without a database; this file is the part that needs one.
*/
import { sql } from "drizzle-orm";
import { pluginLogger } from "../utils/logger.js";
import { resolveDatabaseDialect } from "../database/db/dialect.js";
import type { DatabaseDialect } from "../database/db/dialect.js";
import type { PluginTableDefinition } from "@termix/plugin-sdk/db";
import { prefixedTableName } from "@termix/plugin-sdk/db";
import { buildTable, dropTableSql } from "./table-builder.js";
import {
applyPluginMigrations,
readMigrations,
tableExistsSql,
type MigrationRunner,
} from "./migrations.js";
/** Definitions registered by a plugin, in the order it declared them. */
const registered = new Map<string, PluginTableDefinition[]>();
/** Built table objects, keyed by plugin id then definition name. */
const built = new Map<string, Map<string, unknown>>();
export function registerTable(
pluginId: string,
definition: PluginTableDefinition,
): unknown {
const definitions = registered.get(pluginId) ?? [];
const existing = definitions.find((entry) => entry.name === definition.name);
if (existing) {
// Re-registering on re-activation is normal: the module is cached, so
// activate runs again against the same definitions.
return getTable(pluginId, definition.name);
}
definitions.push(definition);
registered.set(pluginId, definitions);
const table = buildTable(pluginId, definition);
const tables = built.get(pluginId) ?? new Map<string, unknown>();
tables.set(definition.name, table);
built.set(pluginId, tables);
return table;
}
export function getTable(pluginId: string, name: string): unknown {
return built.get(pluginId)?.get(name);
}
export function listTables(pluginId: string): PluginTableDefinition[] {
return [...(registered.get(pluginId) ?? [])];
}
/** Forgets a plugin's definitions. Used by the tests and by uninstall. */
export function forgetTables(pluginId: string): void {
registered.delete(pluginId);
built.delete(pluginId);
}
export function resetPluginData(): void {
registered.clear();
built.clear();
}
async function execute(statements: string[]): Promise<void> {
const { getDb } = await import("../database/db/index.js");
const db = getDb();
for (const statement of statements) {
await db.run(sql.raw(statement));
}
}
/**
* Builds the runner that applies migrations against the live database.
*
* SQLite gets a transaction; the client-server engines do not, because MySQL
* commits implicitly on DDL so a "transaction" there would only be a promise
* the engine does not keep. A half-applied migration is recorded statement by
* statement either way, so a retry resumes rather than repeating.
*/
export async function createMigrationRunner(
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<MigrationRunner> {
const { createCurrentPluginMigrationRepository } =
await import("../database/repositories/factory.js");
const repository = createCurrentPluginMigrationRepository();
return {
dialect,
listApplied: async (pluginId) => {
const rows = await repository.listByPlugin(pluginId);
return rows.map((row) => ({
migrationId: row.migrationId,
checksum: row.checksum,
}));
},
execute,
record: async (pluginId, migration) => {
await repository.record(pluginId, migration.id, migration.checksum);
},
};
}
/**
* Applies a plugin's migrations. Throws so the loader can fail the plugin.
*
* Returns early when the plugin ships none, before anything asks for a
* database handle: most plugins own no tables, and activating one should not
* require a connection it never uses.
*/
export async function migratePlugin(
pluginId: string,
pluginDir: string,
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<string[]> {
if (readMigrations(pluginDir, dialect).length === 0) return [];
const runner = await createMigrationRunner(dialect);
return applyPluginMigrations(pluginId, pluginDir, runner);
}
/** Whether a table exists, asked in the active dialect's own catalog. */
export async function tableExists(
table: string,
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<boolean> {
const { getDb } = await import("../database/db/index.js");
const rows = (await getDb().all(tableExistsSql(dialect, table))) as unknown[];
return rows.length > 0;
}
/**
* Removes everything a plugin owns.
*
* Tables go in reverse registration order so a table referenced by a later one
* is dropped last. Disabling a plugin never calls this: data outlives being
* turned off, and only an explicit uninstall throws it away.
*/
export async function removePluginData(
pluginId: string,
dialect: DatabaseDialect = resolveDatabaseDialect(),
): Promise<{ tables: string[]; kvKeys: number; migrations: number }> {
const definitions = listTables(pluginId);
const dropped: string[] = [];
for (const definition of [...definitions].reverse()) {
await execute([dropTableSql(dialect, pluginId, definition)]);
dropped.push(prefixedTableName(pluginId, definition.name));
}
const {
createCurrentPluginStorageRepository,
createCurrentPluginMigrationRepository,
createCurrentPluginPermissionGrantRepository,
} = await import("../database/repositories/factory.js");
const kvKeys =
await createCurrentPluginStorageRepository().deleteByPlugin(pluginId);
const migrations =
await createCurrentPluginMigrationRepository().deleteByPlugin(pluginId);
await createCurrentPluginPermissionGrantRepository().deleteByPlugin(pluginId);
forgetTables(pluginId);
pluginLogger.info(
`Removed data for plugin ${pluginId}: ${dropped.length} table(s), ${kvKeys} kv key(s)`,
{ operation: "plugin_remove_data" },
);
return { tables: dropped, kvKeys, migrations };
}
+7
View File
@@ -338,6 +338,13 @@ export class PluginLoader {
);
}
// Before activate, not inside it: a plugin's first line may query its
// own tables, and a migration that fails should stop it starting rather
// than leave it half-running against a schema that is not there. A
// throw here lands in the catch below, which fails this plugin only.
const { migratePlugin } = await import("./data.js");
await migratePlugin(plugin.id, plugin.dir);
const handle = createPluginHandle(plugin.id, { activate, deactivate });
const ctx = createPluginContext(plugin.manifest, handle);
+316
View File
@@ -0,0 +1,316 @@
/**
* Applies a plugin's migrations before it activates.
*
* A plugin ships plain SQL per dialect in migrations/<dialect>/NNNN_name.sql.
* Not drizzle-kit: core's own SQLite schema is hand-written DDL and only the
* client-server engines use drizzle migrations, so a plugin that used
* drizzle-kit would still need a hand-rolled SQLite path. One runner for all
* three keeps a plugin's tables identical everywhere.
*
* Failures are contained. An edited migration that already ran blocks that
* plugin; a migration that throws marks it failed. Core keeps booting either
* way, because one plugin's bad SQL is not a reason to take the server down.
*/
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { sql } from "drizzle-orm";
import { pluginLogger } from "../utils/logger.js";
import type { DatabaseDialect } from "../database/db/dialect.js";
import { tablePrefix, LEGACY_TABLE_OWNERS } from "@termix/plugin-sdk/db";
export { LEGACY_TABLE_OWNERS };
/** Dialect directory names, matching DatabaseDialect exactly. */
const DIALECTS: readonly DatabaseDialect[] = ["sqlite", "postgres", "mysql"];
const FILE_PATTERN = /^(\d{4})_([a-z0-9_]+)\.sql$/;
export interface PluginMigration {
/** The file's stem, e.g. "0001_init". Stable across dialects. */
id: string;
sequence: number;
file: string;
sql: string;
checksum: string;
}
/** Raised when an already-applied migration no longer matches its checksum. */
export class PluginMigrationChecksumError extends Error {
readonly code = "EPLUGINMIGRATIONCHECKSUM";
constructor(pluginId: string, migrationId: string) {
super(
`Plugin "${pluginId}" migration "${migrationId}" has changed since it was applied. ` +
`An applied migration is immutable: add a new one instead of editing this file.`,
);
this.name = "PluginMigrationChecksumError";
}
}
export function checksum(contents: string): string {
// Normalised so a line-ending change between platforms is not a mismatch.
const normalised = contents.replace(/\r\n/g, "\n").trimEnd();
return crypto.createHash("sha256").update(normalised).digest("hex");
}
export function migrationsDir(
pluginDir: string,
dialect: DatabaseDialect,
): string {
return path.join(pluginDir, "migrations", dialect);
}
/** Reads the migrations a plugin ships for one dialect, in sequence order. */
export function readMigrations(
pluginDir: string,
dialect: DatabaseDialect,
): PluginMigration[] {
const dir = migrationsDir(pluginDir, dialect);
if (!fs.existsSync(dir)) return [];
const migrations: PluginMigration[] = [];
for (const file of fs.readdirSync(dir).sort()) {
const match = FILE_PATTERN.exec(file);
if (!match) {
if (file.endsWith(".sql")) {
throw new Error(
`Migration "${file}" must be named NNNN_name.sql, lower snake_case`,
);
}
continue;
}
const contents = fs.readFileSync(path.join(dir, file), "utf8");
migrations.push({
id: file.replace(/\.sql$/, ""),
sequence: Number(match[1]),
file,
sql: contents,
checksum: checksum(contents),
});
}
migrations.sort((a, b) => a.sequence - b.sequence);
return migrations;
}
/**
* Splits a migration file into statements.
*
* Deliberately simple: statements end at a semicolon that is not inside a
* string literal or a comment. A plugin needing more than that should put the
* logic in its own code, not in DDL.
*/
export function splitStatements(source: string): string[] {
const statements: string[] = [];
let current = "";
let quote: string | null = null;
let lineComment = false;
let blockComment = false;
for (let i = 0; i < source.length; i++) {
const char = source[i];
const next = source[i + 1];
if (lineComment) {
current += char;
if (char === "\n") lineComment = false;
continue;
}
if (blockComment) {
current += char;
if (char === "*" && next === "/") {
current += next;
i++;
blockComment = false;
}
continue;
}
if (quote) {
current += char;
// '' inside a quoted string is an escaped quote, not a terminator.
if (char === quote) {
if (next === quote) {
current += next;
i++;
} else {
quote = null;
}
}
continue;
}
if (char === "-" && next === "-") {
lineComment = true;
current += char;
continue;
}
if (char === "/" && next === "*") {
blockComment = true;
current += char;
continue;
}
if (char === "'" || char === '"' || char === "`") {
quote = char;
current += char;
continue;
}
if (char === ";") {
if (current.trim()) statements.push(current.trim());
current = "";
continue;
}
current += char;
}
if (current.trim()) statements.push(current.trim());
return statements;
}
/**
* Tables a migration may create or alter.
*
* A plugin owns the p_<id>_ namespace and nothing else. Enforced here as well
* as in the CLI, because a plugin installed from a tarball never ran the CLI.
*/
const CREATE_OR_ALTER =
/\b(?:CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TABLE(?:\s+IF\s+NOT\s+EXISTS)?|ALTER\s+TABLE|DROP\s+TABLE(?:\s+IF\s+EXISTS)?|TRUNCATE\s+TABLE)\s+([`"']?)([a-zA-Z0-9_]+)\1/gi;
/**
* Checks that every table a migration touches belongs to this plugin.
*
* A legacy table it is allowed to adopt is permitted too, because adoption
* renames it into the plugin's namespace in the same migration.
*/
export function assertOwnedTables(
pluginId: string,
statements: string[],
): void {
const prefix = tablePrefix(pluginId);
for (const statement of statements) {
CREATE_OR_ALTER.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = CREATE_OR_ALTER.exec(statement)) !== null) {
const table = match[2];
if (table.startsWith(prefix)) continue;
if (LEGACY_TABLE_OWNERS[table] === pluginId) continue;
const owner = LEGACY_TABLE_OWNERS[table];
throw new Error(
owner
? `Plugin "${pluginId}" may not touch "${table}": it belongs to the "${owner}" plugin.`
: `Plugin "${pluginId}" may only create or alter tables prefixed "${prefix}", not "${table}".`,
);
}
}
}
export interface AppliedMigration {
migrationId: string;
checksum: string;
}
export interface MigrationRunner {
dialect: DatabaseDialect;
listApplied: (pluginId: string) => Promise<AppliedMigration[]>;
execute: (statements: string[]) => Promise<void>;
record: (pluginId: string, migration: PluginMigration) => Promise<void>;
/** Runs fn inside a transaction where the dialect supports DDL in one. */
transaction?: (fn: () => Promise<void>) => Promise<void>;
}
/**
* Brings one plugin's tables up to date.
*
* Returns the ids applied, so a caller can log what changed. Throws on a
* checksum mismatch or a failing statement; the loader turns either into a
* failed plugin without disturbing the rest of the server.
*/
export async function applyPluginMigrations(
pluginId: string,
pluginDir: string,
runner: MigrationRunner,
): Promise<string[]> {
const migrations = readMigrations(pluginDir, runner.dialect);
if (migrations.length === 0) return [];
const applied = new Map(
(await runner.listApplied(pluginId)).map((row) => [
row.migrationId,
row.checksum,
]),
);
const pending: PluginMigration[] = [];
for (const migration of migrations) {
const previous = applied.get(migration.id);
if (previous === undefined) {
pending.push(migration);
continue;
}
if (previous !== migration.checksum) {
throw new PluginMigrationChecksumError(pluginId, migration.id);
}
}
if (pending.length === 0) return [];
const run = async () => {
for (const migration of pending) {
const statements = splitStatements(migration.sql);
assertOwnedTables(pluginId, statements);
await runner.execute(statements);
await runner.record(pluginId, migration);
}
};
if (runner.transaction) {
await runner.transaction(run);
} else {
await run();
}
pluginLogger.info(
`Applied ${pending.length} migration(s) for plugin ${pluginId}: ${pending
.map((m) => m.id)
.join(", ")}`,
{ operation: "plugin_migrate" },
);
return pending.map((migration) => migration.id);
}
/** The dialect directories a plugin is expected to ship. */
export function expectedDialects(): readonly DatabaseDialect[] {
return DIALECTS;
}
/**
* Renames a legacy core table into a plugin's namespace, or creates it fresh.
*
* Adoption is how a feature keeps its data when it moves out of core. The
* table is renamed rather than copied so nothing is duplicated and nothing has
* to be migrated row by row.
*/
export function adoptLegacyTableSql(
dialect: DatabaseDialect,
legacyName: string,
newName: string,
): string {
if (dialect === "mysql") {
return `RENAME TABLE \`${legacyName}\` TO \`${newName}\`;`;
}
return `ALTER TABLE "${legacyName}" RENAME TO "${newName}";`;
}
/** Whether a table exists, asked in the dialect's own catalog. */
export function tableExistsSql(dialect: DatabaseDialect, table: string) {
if (dialect === "sqlite") {
return sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${table}`;
}
if (dialect === "postgres") {
return sql`SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = ${table}`;
}
return sql`SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${table}`;
}
+100
View File
@@ -0,0 +1,100 @@
/**
* Which entities remote sync carries.
*
* Core registers its ten at boot and a plugin registers its own through
* ctx.sync, so the list is built rather than hardcoded in four places. The
* wire names are unchanged: existing tombstones and rows on the other side of
* a pair match on those strings, so renaming one would orphan data.
*
* Order is dependency order, not declaration order. A row referencing another
* entity is resolved by that entity's syncId, so the target has to exist on
* the far side first. That is why every registration carries an explicit
* order rather than relying on registration sequence.
*/
import { pluginLogger } from "../utils/logger.js";
import type { SyncEntityRegistration } from "@termix/plugin-sdk/backend";
export interface RegisteredSyncEntity extends SyncEntityRegistration {
/** Which plugin registered it, or "core". */
owner: string;
order: number;
userColumn: string;
readOnlyFields: readonly string[];
encryptedFields: readonly string[];
}
const entities = new Map<string, RegisteredSyncEntity>();
/** Core's own entities register under this owner. */
export const CORE_OWNER = "core";
export function registerEntity(
owner: string,
entity: SyncEntityRegistration,
): () => void {
if (!entity.type) {
throw new Error("A sync entity needs a type");
}
const existing = entities.get(entity.type);
if (existing && existing.owner !== owner) {
throw new Error(
`Sync entity "${entity.type}" is already registered by "${existing.owner}"`,
);
}
entities.set(entity.type, {
...entity,
owner,
order: entity.order ?? 100,
userColumn: entity.userColumn ?? "userId",
readOnlyFields: entity.readOnlyFields ?? [],
encryptedFields: entity.encryptedFields ?? [],
});
return () => {
const current = entities.get(entity.type);
// Identity-checked: a plugin that restarted must not revoke the
// registration its own restart installed.
if (current && current.owner === owner) entities.delete(entity.type);
};
}
export function getEntity(type: string): RegisteredSyncEntity | undefined {
return entities.get(type);
}
export function hasEntity(type: string): boolean {
return entities.has(type);
}
/** Every entity, in dependency order. */
export function listEntities(): RegisteredSyncEntity[] {
return [...entities.values()].sort((a, b) => {
if (a.order !== b.order) return a.order - b.order;
return a.type.localeCompare(b.type);
});
}
/** The ordered wire names, which is what the Electron client syncs by. */
export function listEntityTypes(): string[] {
return listEntities().map((entity) => entity.type);
}
export function unregisterByOwner(owner: string): void {
for (const [type, entity] of [...entities]) {
if (entity.owner === owner) entities.delete(type);
}
}
export function resetSyncRegistry(): void {
entities.clear();
}
/** Logged once at boot so a missing entity is visible without a debugger. */
export function logRegisteredEntities(): void {
pluginLogger.info(`Sync entities: ${listEntityTypes().join(", ")}`, {
operation: "sync_registry",
});
}
+104
View File
@@ -0,0 +1,104 @@
/**
* Turns a plugin's table definition into the Drizzle table it is queried
* through.
*
* One table object for every dialect. The sqlite-core definitions already
* encode correctly on each engine at query time, so the repositories use them
* everywhere; only DDL genuinely differs, and that lives in the SDK's ddl
* entry so the CLI and the server share one mapping.
*/
import {
sqliteTable,
text as sqliteText,
integer as sqliteInteger,
index as sqliteIndex,
uniqueIndex as sqliteUniqueIndex,
type AnySQLiteColumn,
} from "drizzle-orm/sqlite-core";
import { sql } from "drizzle-orm";
import { prefixedTableName } from "@termix/plugin-sdk/db";
import type { PluginTableDefinition } from "@termix/plugin-sdk/db";
import { columnName } from "@termix/plugin-sdk/ddl";
type ColumnChain = {
notNull: () => unknown;
primaryKey: () => unknown;
unique: () => unknown;
default: (value: unknown) => unknown;
};
/**
* Builds the queryable table object.
*
* Always sqlite-core, on every dialect. drizzle's three table classes share no
* base class, and the query-builder surface a plugin uses is identical across
* them - the same deliberate approximation repositories/database-context.ts
* documents for core.
*/
export function buildTable(
pluginId: string,
definition: PluginTableDefinition,
): unknown {
const physical = prefixedTableName(pluginId, definition.name);
const columns: Record<string, unknown> = {};
for (const [property, column] of Object.entries(definition.columns)) {
const name = columnName(property, column);
let built: unknown;
switch (column.type) {
case "id":
built = sqliteInteger(name).primaryKey({ autoIncrement: true });
break;
case "integer":
case "bigint":
case "refHost":
built = sqliteInteger(name);
break;
case "boolean":
built = sqliteInteger(name, { mode: "boolean" });
break;
default:
built = sqliteText(name);
break;
}
if (column.type !== "id") {
if (column.primaryKey) built = (built as ColumnChain).primaryKey();
if (column.notNull) built = (built as ColumnChain).notNull();
if (column.unique) built = (built as ColumnChain).unique();
if (column.defaultNow) {
built = (built as ColumnChain).default(sql`CURRENT_TIMESTAMP`);
} else if (column.defaultValue !== undefined) {
built = (built as ColumnChain).default(column.defaultValue);
}
}
columns[property] = built;
}
return sqliteTable(physical, columns as never, (table) =>
definition.indexes.map((entry) => {
const make = entry.unique ? sqliteUniqueIndex : sqliteIndex;
const target = table as unknown as Record<string, AnySQLiteColumn>;
return make(entry.name).on(
...(entry.columns.map((c) => target[c]) as [
AnySQLiteColumn,
...AnySQLiteColumn[],
]),
);
}),
);
}
// The DDL emitters live in the SDK, because termix-plugin writes the same SQL
// this server applies. Re-exported here so core keeps one import site.
export {
createTableSql,
dropTableSql,
addColumnSql,
keyedColumns,
columnName,
REFERENCEABLE,
} from "@termix/plugin-sdk/ddl";
@@ -1,11 +1,19 @@
import { describe, expect, it } from "vitest";
import { beforeAll, describe, expect, it } from "vitest";
import {
deserializeSyncReferences,
orderSyncRows,
serializeSyncReferences,
} from "../../database/routes/sync-references.js";
import { registerCoreSyncEntities } from "../../database/routes/sync-entities.js";
describe("sync references", () => {
// References come from the registry now, not a hardcoded map, so the core
// entities have to be registered before any of this resolves. sync.ts does
// this at import in production.
beforeAll(() => {
registerCoreSyncEntities();
});
it("serializes database-local host IDs as stable sync IDs", async () => {
const row = await serializeSyncReferences(
"hosts",
@@ -30,6 +30,9 @@ vi.mock("../../database/repositories/factory.js", () => ({
delete: async (pluginId: string, key: string) =>
kvStore.delete(`${pluginId}:${key}`),
listKeys: async () => [...kvStore.keys()],
countKeys: async (pluginId: string) =>
[...kvStore.keys()].filter((key) => key.startsWith(`${pluginId}:`))
.length,
}),
}));
+234
View File
@@ -0,0 +1,234 @@
/**
* ctx.db and the kv caps.
*
* db:own was in the catalog from A1 but nothing checked it, which ARCHITECTURE
* calls the anti-pattern outright. These assert that it is checked where the
* privileged thing happens, and audited either way.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
const grants = new Map<string, string[]>();
const auditEntries: Array<Record<string, unknown>> = [];
const kvStore = new Map<string, string>();
vi.mock("../../database/repositories/factory.js", () => ({
createCurrentPluginPermissionGrantRepository: () => ({
listByPlugin: async (pluginId: string) =>
(grants.get(pluginId) ?? []).map((capability) => ({
pluginId,
capability,
})),
}),
createCurrentPluginStorageRepository: () => ({
get: async (pluginId: string, key: string) =>
kvStore.get(`${pluginId}:${key}`) ?? null,
set: async (pluginId: string, key: string, value: string) => {
kvStore.set(`${pluginId}:${key}`, value);
},
delete: async (pluginId: string, key: string) =>
kvStore.delete(`${pluginId}:${key}`),
listKeys: async (pluginId: string) =>
[...kvStore.keys()]
.filter((key) => key.startsWith(`${pluginId}:`))
.map((key) => key.slice(pluginId.length + 1)),
countKeys: async (pluginId: string) =>
[...kvStore.keys()].filter((key) => key.startsWith(`${pluginId}:`))
.length,
}),
}));
vi.mock("../../utils/audit-logger.js", () => ({
logAudit: async (entry: Record<string, unknown>) => {
auditEntries.push(entry);
},
}));
vi.mock("../../database/db/index.js", () => ({
getDb: () => ({ marker: "db-handle" }),
}));
vi.mock("../../database/db/schema.js", () => ({
users: { marker: "users" },
hosts: { marker: "hosts" },
}));
import { createPluginContext, createPluginHandle } from "../../plugins/ctx.js";
import { invalidatePluginPermissionCache } from "../../plugins/permissions.js";
import { resetPluginData } from "../../plugins/data.js";
import {
resetSyncRegistry,
listEntityTypes,
} from "../../plugins/sync-registry.js";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
function manifestFor(pluginId: string, capabilities: string[]): PluginManifest {
return {
id: pluginId,
name: pluginId,
version: "1.0.0",
description: "",
author: { name: "test" },
license: "MIT",
category: "Productivity",
engine: { termix: ">=2.9.0", api: "1" },
capabilities,
} as PluginManifest;
}
function contextFor(pluginId: string, capabilities: string[]) {
const manifest = manifestFor(pluginId, capabilities);
const handle = createPluginHandle(pluginId, { activate: () => {} });
return { ctx: createPluginContext(manifest, handle), handle };
}
beforeEach(() => {
grants.clear();
auditEntries.length = 0;
kvStore.clear();
resetPluginData();
resetSyncRegistry();
invalidatePluginPermissionCache();
delete process.env.PLUGIN_MAX_KV_KEYS;
});
describe("ctx.db capability", () => {
it("refuses define without db:own", async () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
await expect(
ctx.db.define({ name: "thing", columns: {}, indexes: [] }),
).rejects.toThrow(/db:own/);
});
it("refuses define when granted but not declared", async () => {
// A grant for something the manifest never declared is ignored, so
// widening a plugin's reach always needs a manifest the user can see.
grants.set("demo", ["db:own"]);
const { ctx } = contextFor("demo", []);
await expect(
ctx.db.define({ name: "thing", columns: {}, indexes: [] }),
).rejects.toThrow(/db:own/);
});
it("refuses the client handle without db:own", async () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
await expect(ctx.db.client()).rejects.toThrow(/db:own/);
});
it("allows define when declared and granted", async () => {
grants.set("demo", ["db:own"]);
const { ctx } = contextFor("demo", ["db:own"]);
const table = await ctx.db.define({
name: "thing",
columns: { id: { type: "id", primaryKey: true, notNull: true } },
indexes: [],
});
expect(table).toBeTruthy();
});
it("audits a refused call as well as an allowed one", async () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
await expect(ctx.db.client()).rejects.toThrow();
const entry = auditEntries.find((row) => row.action === "plugin_db_client");
expect(entry).toMatchObject({
success: false,
username: "plugin:demo",
resourceId: "demo",
});
});
it("hands back read-only refs to the core tables a plugin may join", async () => {
grants.set("demo", ["db:own"]);
const { ctx } = contextFor("demo", ["db:own"]);
const refs = (await ctx.db.refs()) as Record<string, unknown>;
expect(Object.keys(refs).sort()).toEqual(["hosts", "users"]);
});
});
describe("ctx.kv limits", () => {
it("refuses a key over the length cap", async () => {
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await expect(ctx.kv.set("k".repeat(129), 1)).rejects.toThrow(
/at most 128 characters/,
);
});
it("refuses a value over the size cap", async () => {
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await expect(ctx.kv.set("big", "x".repeat(300_000))).rejects.toThrow(
/byte limit/,
);
});
it("refuses a new key past the key-count cap", async () => {
process.env.PLUGIN_MAX_KV_KEYS = "2";
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await ctx.kv.set("a", 1);
await ctx.kv.set("b", 2);
await expect(ctx.kv.set("c", 3)).rejects.toThrow(/key limit/);
});
it("still allows overwriting an existing key when full", async () => {
// Only a new key grows the table, so a full store is not read-only.
process.env.PLUGIN_MAX_KV_KEYS = "2";
grants.set("demo", ["kv:own"]);
const { ctx } = contextFor("demo", ["kv:own"]);
await ctx.kv.set("a", 1);
await ctx.kv.set("b", 2);
await expect(ctx.kv.set("a", 99)).resolves.toBeUndefined();
expect(await ctx.kv.get("a")).toBe(99);
});
it("counts each plugin's keys separately", async () => {
process.env.PLUGIN_MAX_KV_KEYS = "1";
grants.set("one", ["kv:own"]);
grants.set("two", ["kv:own"]);
await contextFor("one", ["kv:own"]).ctx.kv.set("a", 1);
await expect(
contextFor("two", ["kv:own"]).ctx.kv.set("a", 1),
).resolves.toBeUndefined();
});
});
describe("ctx.sync", () => {
it("registers an entity a plugin owns", () => {
grants.set("demo", []);
const { ctx } = contextFor("demo", []);
ctx.sync.registerEntity({ type: "demoThings", table: {}, order: 200 });
expect(listEntityTypes()).toContain("demoThings");
});
it("drops the registration when the plugin is disposed", async () => {
grants.set("demo", []);
const { ctx, handle } = contextFor("demo", []);
ctx.sync.registerEntity({ type: "demoThings", table: {} });
await handle.bag.disposeAll();
expect(listEntityTypes()).not.toContain("demoThings");
});
});
@@ -0,0 +1,345 @@
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
vi.mock("../../utils/logger.js", () => ({
pluginLogger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
success: vi.fn(),
},
}));
import {
applyPluginMigrations,
assertOwnedTables,
adoptLegacyTableSql,
checksum,
PluginMigrationChecksumError,
readMigrations,
splitStatements,
LEGACY_TABLE_OWNERS,
type MigrationRunner,
type PluginMigration,
} from "../../plugins/migrations.js";
let root: string;
function writeMigration(
pluginDir: string,
dialect: string,
file: string,
body: string,
): void {
const dir = path.join(pluginDir, "migrations", dialect);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, file), body);
}
function createRunner(overrides: Partial<MigrationRunner> = {}) {
const executed: string[][] = [];
const recorded: PluginMigration[] = [];
const applied: { migrationId: string; checksum: string }[] = [];
const runner: MigrationRunner = {
dialect: "sqlite",
listApplied: async () => applied,
execute: async (statements) => {
executed.push(statements);
},
record: async (_pluginId, migration) => {
recorded.push(migration);
applied.push({
migrationId: migration.id,
checksum: migration.checksum,
});
},
...overrides,
};
return { runner, executed, recorded, applied };
}
beforeEach(() => {
root = fs.mkdtempSync(path.join(os.tmpdir(), "termix-migrations-"));
});
afterEach(() => {
fs.rmSync(root, { recursive: true, force: true });
});
describe("readMigrations", () => {
it("reads files in sequence order regardless of directory order", () => {
writeMigration(root, "sqlite", "0002_second.sql", "SELECT 2;");
writeMigration(root, "sqlite", "0010_tenth.sql", "SELECT 10;");
writeMigration(root, "sqlite", "0001_first.sql", "SELECT 1;");
const migrations = readMigrations(root, "sqlite");
expect(migrations.map((m) => m.id)).toEqual([
"0001_first",
"0002_second",
"0010_tenth",
]);
});
it("returns nothing when the plugin ships no migrations", () => {
expect(readMigrations(root, "sqlite")).toEqual([]);
});
it("rejects a .sql file that is not named NNNN_name.sql", () => {
writeMigration(root, "sqlite", "init.sql", "SELECT 1;");
expect(() => readMigrations(root, "sqlite")).toThrow(/NNNN_name\.sql/);
});
it("is insensitive to line endings, so a checkout does not look edited", () => {
expect(checksum("CREATE TABLE a;\nSELECT 1;\n")).toBe(
checksum("CREATE TABLE a;\r\nSELECT 1;\r\n"),
);
});
});
describe("splitStatements", () => {
it("splits on semicolons", () => {
expect(splitStatements("SELECT 1; SELECT 2;")).toEqual([
"SELECT 1",
"SELECT 2",
]);
});
it("keeps a semicolon inside a string literal", () => {
const statements = splitStatements(
"INSERT INTO t VALUES ('a;b'); SELECT 1;",
);
expect(statements).toEqual(["INSERT INTO t VALUES ('a;b')", "SELECT 1"]);
});
it("keeps an escaped quote inside a string literal", () => {
const statements = splitStatements("INSERT INTO t VALUES ('it''s; fine');");
expect(statements).toEqual(["INSERT INTO t VALUES ('it''s; fine')"]);
});
it("keeps a semicolon inside a line comment", () => {
const statements = splitStatements("-- a; comment\nSELECT 1;");
expect(statements).toHaveLength(1);
expect(statements[0]).toContain("SELECT 1");
});
it("keeps a semicolon inside a block comment", () => {
const statements = splitStatements("/* a; b */ SELECT 1;");
expect(statements).toHaveLength(1);
});
it("ignores a trailing statement that is only whitespace", () => {
expect(splitStatements("SELECT 1;\n\n \n")).toEqual(["SELECT 1"]);
});
});
describe("assertOwnedTables", () => {
it("allows a table in the plugin's own namespace", () => {
expect(() =>
assertOwnedTables("my-plugin", [
"CREATE TABLE IF NOT EXISTS p_my_plugin_thing (id integer)",
]),
).not.toThrow();
});
it("rejects a table outside the plugin's namespace", () => {
expect(() =>
assertOwnedTables("my-plugin", ["CREATE TABLE users (id integer)"]),
).toThrow(/may only create or alter tables prefixed "p_my_plugin_"/);
});
it("rejects altering a core table", () => {
expect(() =>
assertOwnedTables("my-plugin", [
"ALTER TABLE ssh_data ADD COLUMN x text",
]),
).toThrow(/p_my_plugin_/);
});
it("allows the one plugin that may adopt a legacy table", () => {
expect(() =>
assertOwnedTables("fleets", [
'ALTER TABLE "fleets" RENAME TO "p_fleets_fleets"',
]),
).not.toThrow();
});
it("names the owner when another plugin tries to adopt its table", () => {
expect(() =>
assertOwnedTables("workspaces", [
'ALTER TABLE "fleets" RENAME TO "p_workspaces_fleets"',
]),
).toThrow(/belongs to the "fleets" plugin/);
});
it("maps every legacy table to exactly one owning plugin", () => {
// A table owned by two plugins would let either one adopt it and lose the
// other's data.
const owners = Object.entries(LEGACY_TABLE_OWNERS);
expect(new Set(owners.map(([table]) => table)).size).toBe(owners.length);
});
});
describe("applyPluginMigrations", () => {
it("applies pending migrations in order and records each one", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
writeMigration(
root,
"sqlite",
"0002_more.sql",
"CREATE TABLE p_demo_b (id integer);",
);
const { runner, recorded } = createRunner();
const ids = await applyPluginMigrations("demo", root, runner);
expect(ids).toEqual(["0001_init", "0002_more"]);
expect(recorded.map((m) => m.id)).toEqual(["0001_init", "0002_more"]);
});
it("skips a migration that has already been applied", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const body = fs.readFileSync(
path.join(root, "migrations", "sqlite", "0001_init.sql"),
"utf8",
);
const { runner, executed } = createRunner({
listApplied: async () => [
{ migrationId: "0001_init", checksum: checksum(body) },
],
});
const ids = await applyPluginMigrations("demo", root, runner);
expect(ids).toEqual([]);
expect(executed).toEqual([]);
});
it("refuses to run when an applied migration was edited", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const { runner } = createRunner({
listApplied: async () => [
{ migrationId: "0001_init", checksum: "stale-checksum" },
],
});
await expect(applyPluginMigrations("demo", root, runner)).rejects.toThrow(
PluginMigrationChecksumError,
);
});
it("does not record a migration whose statements failed", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const { runner, recorded } = createRunner({
execute: async () => {
throw new Error("syntax error");
},
});
await expect(applyPluginMigrations("demo", root, runner)).rejects.toThrow(
"syntax error",
);
expect(recorded).toEqual([]);
});
it("refuses a migration that reaches outside the plugin's namespace", async () => {
writeMigration(root, "sqlite", "0001_init.sql", "DROP TABLE users;");
const { runner, executed } = createRunner();
await expect(applyPluginMigrations("demo", root, runner)).rejects.toThrow(
/p_demo_/,
);
expect(executed).toEqual([]);
});
it("runs inside a transaction when the dialect offers one", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (id integer);",
);
const order: string[] = [];
const { runner } = createRunner({
transaction: async (fn) => {
order.push("begin");
await fn();
order.push("commit");
},
execute: async () => {
order.push("execute");
},
});
await applyPluginMigrations("demo", root, runner);
expect(order).toEqual(["begin", "execute", "commit"]);
});
it("reads the directory for the dialect it was given", async () => {
writeMigration(
root,
"sqlite",
"0001_init.sql",
"CREATE TABLE p_demo_a (x);",
);
writeMigration(
root,
"mysql",
"0001_init.sql",
"CREATE TABLE p_demo_a (x int);",
);
const { runner, executed } = createRunner({ dialect: "mysql" });
await applyPluginMigrations("demo", root, runner);
expect(executed[0][0]).toContain("int");
});
});
describe("adoptLegacyTableSql", () => {
it("uses ALTER TABLE RENAME on sqlite and postgres", () => {
expect(adoptLegacyTableSql("sqlite", "fleets", "p_fleets_fleets")).toBe(
'ALTER TABLE "fleets" RENAME TO "p_fleets_fleets";',
);
expect(adoptLegacyTableSql("postgres", "fleets", "p_fleets_fleets")).toBe(
'ALTER TABLE "fleets" RENAME TO "p_fleets_fleets";',
);
});
it("uses RENAME TABLE and backticks on mysql", () => {
expect(adoptLegacyTableSql("mysql", "fleets", "p_fleets_fleets")).toBe(
"RENAME TABLE `fleets` TO `p_fleets_fleets`;",
);
});
});
@@ -0,0 +1,382 @@
/**
* Plugin-owned data against a real engine.
*
* The DDL snapshots in table-ddl.test.ts prove what the emitter writes; this
* proves an engine accepts it, that adoption moves real rows, and that
* removing a plugin's data removes exactly its own.
*/
import Database from "better-sqlite3";
import { drizzle } from "drizzle-orm/better-sqlite3";
import { sql } from "drizzle-orm";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
defineTable,
id,
refUser,
text,
timestamp,
varchar,
} from "@termix/plugin-sdk/db";
import { createTableSql, dropTableSql } from "../../plugins/table-builder.js";
import {
adoptLegacyTableSql,
applyPluginMigrations,
checksum,
PluginMigrationChecksumError,
tableExistsSql,
type MigrationRunner,
} from "../../plugins/migrations.js";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
vi.mock("../../utils/logger.js", () => ({
pluginLogger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
success: vi.fn(),
},
}));
let sqlite: Database.Database;
let db: ReturnType<typeof drizzle>;
let root: string;
const notes = defineTable(
"note",
{
id: id(),
userId: refUser(),
title: varchar(200).notNull(),
body: text(),
createdAt: timestamp().notNull().defaultNow(),
},
{ indexes: [{ name: "idx_note_user", columns: ["userId"] }] },
);
/** The ledger, kept in the same database so a rollback would take it too. */
function createRunner(): MigrationRunner {
return {
dialect: "sqlite",
listApplied: async (pluginId) =>
sqlite
.prepare(
"SELECT migration_id as migrationId, checksum FROM plugin_migrations WHERE plugin_id = ?",
)
.all(pluginId) as { migrationId: string; checksum: string }[],
execute: async (statements) => {
for (const statement of statements) sqlite.exec(statement);
},
record: async (pluginId, migration) => {
sqlite
.prepare(
"INSERT INTO plugin_migrations (plugin_id, migration_id, checksum, applied_at) VALUES (?, ?, ?, ?)",
)
.run(
pluginId,
migration.id,
migration.checksum,
new Date().toISOString(),
);
},
};
}
function writeMigration(dialect: string, file: string, body: string): void {
const dir = path.join(root, "migrations", dialect);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, file), body);
}
function tableExists(name: string): boolean {
return (
(
sqlite
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
)
.all(name) as unknown[]
).length > 0
);
}
beforeEach(() => {
root = fs.mkdtempSync(path.join(os.tmpdir(), "termix-plugin-data-"));
sqlite = new Database(":memory:");
sqlite.exec("PRAGMA foreign_keys = ON");
sqlite.exec(`
CREATE TABLE users (id TEXT PRIMARY KEY, username TEXT NOT NULL);
CREATE TABLE ssh_data (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT);
CREATE TABLE plugin_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
plugin_id TEXT NOT NULL,
migration_id TEXT NOT NULL,
checksum TEXT NOT NULL,
applied_at TEXT NOT NULL,
UNIQUE (plugin_id, migration_id)
);
`);
sqlite
.prepare("INSERT INTO users (id, username) VALUES (?, ?)")
.run("u1", "ada");
db = drizzle(sqlite);
});
afterEach(() => {
sqlite.close();
fs.rmSync(root, { recursive: true, force: true });
});
describe("emitted DDL against a real engine", () => {
it("creates the table and its indexes", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
expect(tableExists("p_demo_note")).toBe(true);
});
it("enforces the foreign key to users", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
expect(() =>
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("nobody", "orphan"),
).toThrow(/FOREIGN KEY/);
});
it("cascades a delete from the referenced user", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("u1", "mine");
sqlite.prepare("DELETE FROM users WHERE id = ?").run("u1");
const rows = sqlite.prepare("SELECT * FROM p_demo_note").all();
expect(rows).toHaveLength(0);
});
it("fills the timestamp default the emitter wrote", () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("u1", "mine");
const [row] = sqlite
.prepare("SELECT created_at FROM p_demo_note")
.all() as {
created_at: string;
}[];
expect(row.created_at).toBeTruthy();
});
it("answers the table-exists probe for the active dialect", async () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
const found = await db.all(tableExistsSql("sqlite", "p_demo_note"));
const missing = await db.all(tableExistsSql("sqlite", "p_demo_nothing"));
expect(found).toHaveLength(1);
expect(missing).toHaveLength(0);
});
});
describe("applying migrations for real", () => {
it("creates the tables and records the ledger", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "demo", notes).join("\n"),
);
const applied = await applyPluginMigrations("demo", root, createRunner());
expect(applied).toEqual(["0001_init"]);
expect(tableExists("p_demo_note")).toBe(true);
expect(
sqlite.prepare("SELECT * FROM plugin_migrations").all(),
).toHaveLength(1);
});
it("is a no-op the second time, so a restart does not re-run DDL", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "demo", notes).join("\n"),
);
await applyPluginMigrations("demo", root, createRunner());
const again = await applyPluginMigrations("demo", root, createRunner());
expect(again).toEqual([]);
});
it("blocks the plugin when an applied migration was edited", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "demo", notes).join("\n"),
);
await applyPluginMigrations("demo", root, createRunner());
writeMigration(
"sqlite",
"0001_init.sql",
`${createTableSql("sqlite", "demo", notes).join("\n")}\n-- edited`,
);
await expect(
applyPluginMigrations("demo", root, createRunner()),
).rejects.toThrow(PluginMigrationChecksumError);
});
it("leaves one plugin's failure without touching another's tables", async () => {
writeMigration(
"sqlite",
"0001_init.sql",
createTableSql("sqlite", "good", notes).join("\n"),
);
await applyPluginMigrations("good", root, createRunner());
const brokenRoot = fs.mkdtempSync(path.join(os.tmpdir(), "termix-broken-"));
fs.mkdirSync(path.join(brokenRoot, "migrations", "sqlite"), {
recursive: true,
});
fs.writeFileSync(
path.join(brokenRoot, "migrations", "sqlite", "0001_init.sql"),
"CREATE TABLE p_broken_x (id integer); THIS IS NOT SQL;",
);
await expect(
applyPluginMigrations("broken", brokenRoot, createRunner()),
).rejects.toThrow();
// The good plugin is untouched, and core is still usable.
expect(tableExists("p_good_note")).toBe(true);
expect(sqlite.prepare("SELECT * FROM users").all()).toHaveLength(1);
fs.rmSync(brokenRoot, { recursive: true, force: true });
});
});
describe("adopting a legacy core table", () => {
it("renames the table and keeps every row", async () => {
sqlite.exec(
"CREATE TABLE fleets (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
);
sqlite.prepare("INSERT INTO fleets (name) VALUES (?)").run("production");
sqlite.prepare("INSERT INTO fleets (name) VALUES (?)").run("staging");
writeMigration(
"sqlite",
"0001_adopt.sql",
adoptLegacyTableSql("sqlite", "fleets", "p_fleets_fleets"),
);
await applyPluginMigrations("fleets", root, createRunner());
expect(tableExists("fleets")).toBe(false);
const rows = sqlite
.prepare("SELECT name FROM p_fleets_fleets ORDER BY id")
.all() as { name: string }[];
expect(rows.map((row) => row.name)).toEqual(["production", "staging"]);
});
it("creates the table fresh when there is no legacy one", async () => {
const fresh = defineTable("fleets", { id: id(), name: varchar(100) });
writeMigration(
"sqlite",
"0001_adopt.sql",
createTableSql("sqlite", "fleets", fresh).join("\n"),
);
await applyPluginMigrations("fleets", root, createRunner());
expect(tableExists("p_fleets_fleets")).toBe(true);
expect(sqlite.prepare("SELECT * FROM p_fleets_fleets").all()).toHaveLength(
0,
);
});
it("refuses a plugin adopting a table that belongs to another", async () => {
sqlite.exec("CREATE TABLE fleets (id INTEGER PRIMARY KEY)");
writeMigration(
"sqlite",
"0001_steal.sql",
adoptLegacyTableSql("sqlite", "fleets", "p_workspaces_fleets"),
);
await expect(
applyPluginMigrations("workspaces", root, createRunner()),
).rejects.toThrow(/belongs to the "fleets" plugin/);
// Refused before execution, so the legacy table is still there.
expect(tableExists("fleets")).toBe(true);
});
});
describe("removing a plugin's data", () => {
it("drops the plugin's tables and leaves everything else", async () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
const other = defineTable("thing", { id: id(), name: varchar(50) });
for (const statement of createTableSql("sqlite", "other", other)) {
sqlite.exec(statement);
}
sqlite.exec(dropTableSql("sqlite", "demo", notes));
expect(tableExists("p_demo_note")).toBe(false);
expect(tableExists("p_other_thing")).toBe(true);
expect(tableExists("users")).toBe(true);
});
it("clears only the removed plugin's ledger rows", () => {
for (const pluginId of ["demo", "other"]) {
sqlite
.prepare(
"INSERT INTO plugin_migrations (plugin_id, migration_id, checksum, applied_at) VALUES (?, ?, ?, ?)",
)
.run(pluginId, "0001_init", checksum("x"), new Date().toISOString());
}
sqlite
.prepare("DELETE FROM plugin_migrations WHERE plugin_id = ?")
.run("demo");
const remaining = sqlite
.prepare("SELECT plugin_id FROM plugin_migrations")
.all() as { plugin_id: string }[];
expect(remaining.map((row) => row.plugin_id)).toEqual(["other"]);
});
});
describe("querying through the built table", () => {
it("reads back a row written with raw SQL", async () => {
for (const statement of createTableSql("sqlite", "demo", notes)) {
sqlite.exec(statement);
}
sqlite
.prepare("INSERT INTO p_demo_note (user_id, title) VALUES (?, ?)")
.run("u1", "written");
const rows = (await db.all(sql.raw("SELECT title FROM p_demo_note"))) as {
title: string;
}[];
expect(rows[0].title).toBe("written");
});
});
@@ -0,0 +1,162 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import {
CORE_OWNER,
getEntity,
hasEntity,
listEntities,
listEntityTypes,
registerEntity,
resetSyncRegistry,
unregisterByOwner,
} from "../../plugins/sync-registry.js";
import { registerCoreSyncEntities } from "../../database/routes/sync-entities.js";
import { SYNCED_ENTITY_TYPES } from "../../../../electron/remote-sync-entities.cjs";
beforeEach(() => {
resetSyncRegistry();
});
afterEach(() => {
resetSyncRegistry();
});
describe("sync registry", () => {
it("orders entities by their declared order, then by name", () => {
registerEntity("core", { type: "b", table: {}, order: 20 });
registerEntity("core", { type: "a", table: {}, order: 10 });
registerEntity("core", { type: "c", table: {}, order: 10 });
expect(listEntityTypes()).toEqual(["a", "c", "b"]);
});
it("defaults the user column and the optional lists", () => {
registerEntity("core", { type: "thing", table: {} });
expect(getEntity("thing")).toMatchObject({
userColumn: "userId",
readOnlyFields: [],
encryptedFields: [],
});
});
it("refuses an entity another owner already registered", () => {
registerEntity("core", { type: "hosts", table: {} });
expect(() =>
registerEntity("evil-plugin", { type: "hosts", table: {} }),
).toThrow(/already registered by "core"/);
});
it("lets the same owner re-register, which is what reactivation does", () => {
registerEntity("demo", { type: "thing", table: {}, order: 1 });
registerEntity("demo", { type: "thing", table: {}, order: 2 });
expect(getEntity("thing")?.order).toBe(2);
});
it("removes an entity when its registration is disposed", () => {
const dispose = registerEntity("demo", { type: "thing", table: {} });
expect(hasEntity("thing")).toBe(true);
dispose();
expect(hasEntity("thing")).toBe(false);
});
it("does not let a stale disposer remove a newer registration", () => {
// A plugin that crashed and restarted must not revoke the registration
// its own restart installed.
const stale = registerEntity("demo", { type: "thing", table: {} });
resetSyncRegistry();
registerEntity("other", { type: "thing", table: {} });
stale();
expect(getEntity("thing")?.owner).toBe("other");
});
it("removes everything an owner registered", () => {
registerEntity("demo", { type: "a", table: {} });
registerEntity("demo", { type: "b", table: {} });
registerEntity("core", { type: "c", table: {} });
unregisterByOwner("demo");
expect(listEntityTypes()).toEqual(["c"]);
});
it("needs a type", () => {
expect(() => registerEntity("demo", { type: "", table: {} })).toThrow(
/needs a type/,
);
});
});
describe("core sync entities", () => {
beforeEach(() => {
registerCoreSyncEntities();
});
it("registers every entity the Electron client knows about", () => {
// Nothing asserted this before, which is how networkTopology came to be
// missing from the list in sync.test.ts.
expect([...listEntityTypes()].sort()).toEqual(
[...SYNCED_ENTITY_TYPES].sort(),
);
});
it("keeps the dependency order the frozen Electron array encodes", () => {
expect(listEntityTypes()).toEqual([...SYNCED_ENTITY_TYPES]);
});
it("owns all of them as core", () => {
for (const entity of listEntities()) {
expect(entity.owner).toBe(CORE_OWNER);
}
});
it("sorts every reference target before the entity that points at it", () => {
const order = new Map(
listEntities().map((entity) => [entity.type, entity.order]),
);
for (const entity of listEntities()) {
for (const reference of entity.references ?? []) {
// A self-reference is ordered within the entity by orderSyncRows.
if (reference.entityType === entity.type) continue;
expect(
order.get(reference.entityType),
`${entity.type} -> ${reference.entityType}`,
).toBeLessThan(entity.order);
}
}
});
it("marks the two singletons and nothing else", () => {
const singletons = listEntities()
.filter((entity) => entity.singleton)
.map((entity) => entity.type);
expect(singletons.sort()).toEqual(["networkTopology", "userPreferences"]);
});
it("keeps the read-only fields that must survive a sync payload", () => {
expect(getEntity("hosts")?.readOnlyFields).toEqual(["connectionOrigin"]);
expect(getEntity("userPreferences")?.readOnlyFields).toEqual([
"storageMode",
]);
});
it("gives networkTopology the custom serializer its JSON blob needs", () => {
const topology = getEntity("networkTopology");
expect(topology?.serialize).toBeTypeOf("function");
expect(topology?.deserialize).toBeTypeOf("function");
});
it("is idempotent, because several modules prime it", () => {
const before = listEntityTypes();
registerCoreSyncEntities();
expect(listEntityTypes()).toEqual(before);
});
});
+255
View File
@@ -0,0 +1,255 @@
import { describe, expect, it } from "vitest";
import {
defineTable,
id,
varchar,
text,
json,
boolean,
timestamp,
integer,
bigint,
encryptedText,
refUser,
refHost,
prefixedTableName,
tablePrefix,
} from "@termix/plugin-sdk/db";
import {
createTableSql,
dropTableSql,
addColumnSql,
keyedColumns,
buildTable,
} from "../../plugins/table-builder.js";
/**
* The three-dialect DDL is what verify:dialect would catch against a real
* server, and that needs a live Postgres and MySQL. These snapshots are what
* covers it in a run that has neither.
*/
const definition = defineTable(
"note",
{
id: id(),
userId: refUser(),
hostId: refHost(),
title: varchar(200).notNull(),
body: text(),
meta: json(),
secret: encryptedText(),
pinned: boolean().default(false),
position: integer().default(0),
size: bigint(),
createdAt: timestamp().notNull().defaultNow(),
},
{
uniques: [{ name: "idx_note_user_title", columns: ["userId", "title"] }],
indexes: [{ name: "idx_note_host", columns: ["hostId"] }],
},
);
describe("table naming", () => {
it("prefixes with the plugin id, dashes becoming underscores", () => {
expect(tablePrefix("network-topology")).toBe("p_network_topology_");
expect(prefixedTableName("network-topology", "graph")).toBe(
"p_network_topology_graph",
);
});
});
describe("defineTable validation", () => {
it("refuses an index on an unbounded text column", () => {
expect(() =>
defineTable(
"bad",
{ id: id(), body: text() },
{ indexes: [{ name: "idx_bad", columns: ["body"] }] },
),
).toThrow(/MySQL cannot index an unbounded column/);
});
it("refuses an index on a json column", () => {
expect(() =>
defineTable(
"bad",
{ id: id(), meta: json() },
{ indexes: [{ name: "idx_bad", columns: ["meta"] }] },
),
).toThrow(/MySQL cannot index an unbounded column/);
});
it("refuses an index naming a column the table does not declare", () => {
expect(() =>
defineTable(
"bad",
{ id: id() },
{ indexes: [{ name: "idx_bad", columns: ["nope"] }] },
),
).toThrow(/which table "bad" does not declare/);
});
it("refuses a table name that is not lower snake_case", () => {
expect(() => defineTable("Notes", { id: id() })).toThrow(
/lower snake_case/,
);
});
it("refuses a table with no columns", () => {
expect(() => defineTable("empty", {})).toThrow(/declares no columns/);
});
it("treats uniques as indexes that happen to be unique", () => {
const table = defineTable(
"thing",
{ id: id(), name: varchar(50) },
{ uniques: [{ name: "idx_thing_name", columns: ["name"] }] },
);
expect(table.indexes[0]).toMatchObject({
name: "idx_thing_name",
unique: true,
});
});
});
describe("keyedColumns", () => {
it("counts primary keys, uniques, refs and indexed columns", () => {
const keyed = keyedColumns(definition);
expect(keyed.has("userId")).toBe(true);
expect(keyed.has("hostId")).toBe(true);
expect(keyed.has("title")).toBe(true);
// Not keyed, so it stays TEXT and is not narrowed to varchar(255).
expect(keyed.has("body")).toBe(false);
expect(keyed.has("meta")).toBe(false);
});
});
describe("createTableSql", () => {
it("emits sqlite DDL", () => {
expect(createTableSql("sqlite", "demo", definition).join("\n"))
.toMatchInlineSnapshot(`
"CREATE TABLE IF NOT EXISTS "p_demo_note" (
"id" integer PRIMARY KEY AUTOINCREMENT,
"user_id" text NOT NULL,
"host_id" integer NOT NULL,
"title" text NOT NULL,
"body" text,
"meta" text,
"secret" text,
"pinned" integer DEFAULT 0,
"position" integer DEFAULT 0,
"size" integer,
"created_at" text NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE,
FOREIGN KEY ("host_id") REFERENCES "ssh_data" ("id") ON DELETE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS "idx_note_user_title" ON "p_demo_note" ("user_id", "title");
CREATE INDEX IF NOT EXISTS "idx_note_host" ON "p_demo_note" ("host_id");"
`);
});
it("emits postgres DDL with serial and native booleans", () => {
expect(createTableSql("postgres", "demo", definition).join("\n"))
.toMatchInlineSnapshot(`
"CREATE TABLE IF NOT EXISTS "p_demo_note" (
"id" serial PRIMARY KEY,
"user_id" varchar(255) NOT NULL,
"host_id" integer NOT NULL,
"title" varchar(200) NOT NULL,
"body" text,
"meta" text,
"secret" text,
"pinned" boolean DEFAULT false,
"position" integer DEFAULT 0,
"size" bigint,
"created_at" text NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE,
FOREIGN KEY ("host_id") REFERENCES "ssh_data" ("id") ON DELETE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS "idx_note_user_title" ON "p_demo_note" ("user_id", "title");
CREATE INDEX IF NOT EXISTS "idx_note_host" ON "p_demo_note" ("host_id");"
`);
});
it("emits mysql DDL with backticks and a parenthesised timestamp default", () => {
expect(createTableSql("mysql", "demo", definition).join("\n"))
.toMatchInlineSnapshot(`
"CREATE TABLE IF NOT EXISTS \`p_demo_note\` (
\`id\` int AUTO_INCREMENT PRIMARY KEY,
\`user_id\` varchar(255) NOT NULL,
\`host_id\` int NOT NULL,
\`title\` varchar(200) NOT NULL,
\`body\` text,
\`meta\` text,
\`secret\` text,
\`pinned\` boolean DEFAULT false,
\`position\` int DEFAULT 0,
\`size\` bigint,
\`created_at\` text NOT NULL DEFAULT (CURRENT_TIMESTAMP),
FOREIGN KEY (\`user_id\`) REFERENCES \`users\` (\`id\`) ON DELETE CASCADE,
FOREIGN KEY (\`host_id\`) REFERENCES \`ssh_data\` (\`id\`) ON DELETE CASCADE
);
CREATE UNIQUE INDEX \`idx_note_user_title\` ON \`p_demo_note\` (\`user_id\`, \`title\`);
CREATE INDEX \`idx_note_host\` ON \`p_demo_note\` (\`host_id\`);"
`);
});
it("never leaves an indexed column as TEXT off sqlite", () => {
// MySQL 8 rejects a TEXT column in a key specification. MariaDB accepts
// it, which is why this was missed in core until a real MySQL saw it.
for (const dialect of ["postgres", "mysql"] as const) {
const sql = createTableSql(dialect, "demo", definition).join("\n");
const keyed = keyedColumns(definition);
for (const property of keyed) {
const column = property.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
const line = sql
.split("\n")
.find(
(row) => row.includes(`${column}\``) || row.includes(`"${column}"`),
);
expect(line, `${dialect}.${column}`).not.toMatch(/\s(text|json)[,\s]/);
}
}
});
it("omits IF NOT EXISTS on a mysql index, which does not support it", () => {
const mysql = createTableSql("mysql", "demo", definition);
const indexes = mysql.filter((statement) => statement.includes("INDEX"));
expect(indexes).not.toHaveLength(0);
for (const statement of indexes) {
expect(statement).not.toContain("IF NOT EXISTS");
}
});
});
describe("addColumnSql", () => {
it("emits an ALTER per dialect", () => {
expect(addColumnSql("sqlite", "demo", definition, "body")).toBe(
'ALTER TABLE "p_demo_note" ADD COLUMN "body" text;',
);
expect(addColumnSql("mysql", "demo", definition, "title")).toBe(
"ALTER TABLE `p_demo_note` ADD COLUMN `title` varchar(200) NOT NULL;",
);
});
});
describe("dropTableSql", () => {
it("drops the prefixed table", () => {
expect(dropTableSql("postgres", "demo", definition)).toBe(
'DROP TABLE IF EXISTS "p_demo_note";',
);
});
});
describe("buildTable", () => {
it("builds a queryable drizzle table carrying the prefixed name", async () => {
const { getTableName, getTableColumns } = await import("drizzle-orm");
const table = buildTable("demo", definition) as never;
expect(getTableName(table)).toBe("p_demo_note");
expect(Object.keys(getTableColumns(table))).toContain("userId");
});
});