Files
Termix/src/backend/sync/entities.ts
T
Luke GustafsonandZacharyZcR 6b708106e1 release-2.9.1 (#1557)
* fix(sso): send the PKCE verifier for GitHub login (#1534)

* fix(remote-desktop): clip cursor overflow without disabling zoom (#1535)

* test(remote-desktop): preserve sessions beyond one hour (#1536)

* fix(hosts): expose controls for overflowing editor tabs (#1537)

* fix(hosts): expose controls for overflowing editor tabs

* test(hosts): mock resize observation in admin panel tests

* fix(status): skip TCP probes while host sessions are active (#1538)

* fix(database): batch session activity persistence (#1539)

* docs(api): describe cookie and API key authentication (#1540)

* fix(snippets): repair missing note column on SQLite upgrades (#1541)

* fix(docker): retain stored SSH credential association (#1543)

* fix(file-manager): render transfer toasts without plugin hooks (#1546)

* feat(hosts): keep compact row actions inline (#1547)

* fix(auth): allow retrying unavailable second-factor interfaces (#1549)

* fix(auth): reject unresolved legacy identity provisioning (#1550)

* fix(desktop): ship the plugin SDK inside Mac App Store builds (#1551)

* fix(tmux): pass full session info to the terminal's session picker (#1552)

The tmux.sessions service reduced detected sessions to bare names, but the
picker reads session.name, so every entry rendered blank and selecting one
sent an empty name, which created a new session instead of attaching.

* fix(ssh): read force keyboard-interactive as a boolean when resolving hosts (#1553)

The column holds the text "true"/"false", and the resolver passed it
through as-is. The string "false" is truthy, so the password provider
treated every saved host as forced keyboard-interactive and left the
password out. Jump hops are built straight from the resolved host, so a
password hop whose server doesn't offer keyboard-interactive failed with
"All configured authentication methods failed".

* fix(database): batch database saves for bulk host writes (#1554)

* fix(database): yield to the event loop between database saves

Each SQLite save serializes and encrypts the whole database into new
buffers, which V8 only releases once the event loop turns. Boot
migrations and bulk host import write plugin settings per host and per
key in an awaited loop, so hundreds of full copies piled up and the
container ran out of memory.

* fix(database): save once per bulk host write instead of once per row

On SQLite every repository write force-saves the whole database. Boot
plugin data migrations, host defaults materialization and the bulk
host routes write one row per host per setting, so they serialized and
encrypted the full database hundreds of times in a row.

DatabaseSaveTrigger.batched wraps a function so force saves made while
it runs collapse into a single save when it finishes. Nested scopes fold
into the outer one, and work that outlives its scope saves normally.

* feat(hosts): add instance-wide predefined tag suggestions (#1548)

* feat(hosts): add shared predefined tag suggestions

* style(hosts): format tag catalog routes

* test(database): advance fake timers past the save yield (#1555)

* fix(i18n): complete Simplified Chinese core and plugin translations (#1542)

* fix(i18n): complete Chinese onboarding and navigation labels

* fix(i18n): translate remaining Chinese core and plugin interfaces

* fix(i18n): translate host editor tab overflow controls

* fix(i18n): use consistent Chinese fleet terminology

* fix(i18n): localize hardcoded controls and plugin views

* fix(i18n): translate built-in homepage widget catalog

* test(homepage): follow translated timezone placeholder

* fix(i18n): translate plugin-provided homepage widgets

* fix(i18n): localize feature settings section titles

* fix: 2.8 oidc accounts unable to sign in after upgrading (#1381)

* fix: plugins losing the saved ssh login when copying a host (#1391)

* fix: fleets, proxmox and automations not getting the host sudo password

* fix: host imports running a defaults pass and metrics restart per host (#1384)

* fix: high cpu from status probes and full database saves on every sample (#1300)

* fix: user data export freezing the server (#1393)

* fix: op:// secret references rejected as ssh keys on credentials (#1394)

* fix: slow file deletes from the trash lookups on every delete (#1390)

* fix: add openapi docs and error handling to host tag routes

* test: compare download stream buffers directly so it stops timing out

* chore: drop em dash from host row comment

* chore: update release notes for 2.9.1

* fix: restore space to add host tags and redesign predefined tags editor

* fix: host key silently accepted when the host is missing from the database (#1397)

* fix: clearer host login failed status label and fix its translations (#1396)

* chore: add host key and status label fixes to release notes

* fix: sso and ldap provider dialogs overflowing the screen and using mismatched toggles

* chore: increment ver

* fix(audit): store plugin entries with no acting user as a null user_id (#1556)

Plugin audit entries written outside a request used the literal "system"
as user_id. That column references users.id, so the insert was refused
(Postgres logs it as an FK violation) and the entry was silently dropped.
Write null instead and keep "system" / plugin:<id> in username.

* chore: add sso/ldap dialog and audit log fixes to release notes

* fix: tunnels and host settings missing from shared hosts on desktop

* fix: remote desktop logins missing from shared hosts on desktop

* fix: simplify host status to online/offline and keep it live without a refresh

* chore: sync Crowdin translations for 2.9.1

---------

Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com>
2026-10-04 15:11:41 -05:00

677 lines
21 KiB
TypeScript

/**
* Core's own sync entities, registered the same way a plugin registers one.
*
* Order values leave room between them (10, 30, 50...) so a plugin can slot
* an entity in without renumbering; anything referenced syncs first.
*/
import { and, eq } from "drizzle-orm";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
import {
hosts,
sshCredentials,
sshFolders,
userPreferences,
} from "../database/db/schema.js";
import {
CORE_OWNER,
listEntities,
registerEntity,
} from "../plugins/sync-registry.js";
import { FieldCrypto } from "../utils/field-crypto.js";
import { syncLogger } from "../utils/logger.js";
import { getPluginRuntime } from "../plugins/index.js";
import { singletonSyncId } from "./wire.js";
import { parseDefaultOverrides } from "../../types/host-defaults.js";
import {
exportHostPluginSettings,
importHostPluginSettings,
} from "./host-plugin-settings.js";
import {
exportProtocolLogins,
exportSharedProtocolLogins,
importProtocolLogins,
} from "./host-protocol-auth.js";
import {
coversHostDefault,
eraseHostDefault,
loadAdminDefaults,
loadHostDefaults,
writeAdminDefaults,
writeHostDefault,
} from "./host-defaults.js";
const CREDENTIAL_REFERENCE = {
field: "credentialId",
syncField: "credentialSyncId",
entityType: "sshCredentials",
} as const;
const HOST_REFERENCES = [
CREDENTIAL_REFERENCE,
{
field: "parentHostId",
syncField: "parentHostSyncId",
entityType: "hosts",
},
{ field: "jumpHosts[].hostId", entityType: "hosts" },
] as const;
/** Columns that only mean something on the device that holds them. */
const HOST_LOCAL_FIELDS = [
"connectionOrigin",
"localOnly",
"sharedSource",
"hostKeyLastVerified",
];
/**
* Columns kept until 3.0.0 that nothing reads any more: their values moved
* into plugin host settings, which sync with the host's pluginSettings. They
* stay off the wire so a stale local id never reaches the other side.
*/
const HOST_RETIRED_FIELDS = ["quickActions"];
let registered = false;
/** Registers core's entities. Idempotent. */
export function registerCoreSyncEntities(): void {
if (
registered &&
listEntities().some((entity) => entity.owner === CORE_OWNER)
)
return;
registered = true;
registerEntity(
CORE_OWNER,
{
type: "sshCredentials",
table: sshCredentials,
order: 10,
encryptedFields: FieldCrypto.fieldsFor("ssh_credentials"),
readOnlyFields: ["usageCount", "lastUsed", "sharedSource"],
shouldSync: (row) => !row.sharedSource,
permissions: {
create: "credentials.create",
update: "credentials.edit",
delete: "credentials.delete",
},
afterWrite: async ({ id, userId, created }) => {
if (id === null || created) return;
const { SharedHostSecretsManager } =
await import("../utils/shared-host-secrets-manager.js");
await SharedHostSecretsManager.getInstance().resyncHostsForCredential(
id,
userId,
);
const { SharedCredentialSecretsManager } =
await import("../utils/shared-credential-secrets-manager.js");
await SharedCredentialSecretsManager.getInstance().resyncCredential(
id,
userId,
);
},
remove: async (row, userId) => {
const { deleteOwnedCredential } =
await import("../hosts/delete-credential.js");
await deleteOwnedCredential(userId, row.id as number);
},
},
{
// A credential shared with the user can back their own host.
canReference: async (userId, _ownerId, id) => {
const { findUsableCredential } =
await import("../hosts/usable-credential.js");
return (await findUsableCredential(id, userId)) !== null;
},
},
);
// vaultProfiles (order 20) belongs to the vault plugin.
registerEntity(CORE_OWNER, {
type: "sshFolders",
table: sshFolders,
order: 30,
references: [CREDENTIAL_REFERENCE],
readOnlyFields: ["localOnly"],
shouldSync: (row) => !row.localOnly,
permissions: {
create: "hosts.create",
update: "hosts.edit",
delete: "hosts.delete",
},
});
registerEntity(
CORE_OWNER,
{
type: "hosts",
table: hosts,
order: 50,
encryptedFields: FieldCrypto.fieldsFor("ssh_data"),
readOnlyFields: [...HOST_LOCAL_FIELDS, ...HOST_RETIRED_FIELDS],
references: HOST_REFERENCES,
shouldSync: (row) => !row.localOnly && !row.sharedSource,
permissions: {
create: "hosts.create",
update: "hosts.edit",
delete: "hosts.delete",
},
serialize: async (row, resolveSyncId) => ({
...row,
pluginSettings:
typeof row.id === "number"
? await exportHostPluginSettings(row.id)
: {},
protocolAuth:
typeof row.id === "number"
? await exportProtocolLogins(
row.id,
String(row.userId),
resolveSyncId,
)
: {},
}),
afterWrite: async ({ id, userId, wire, resolveId }) => {
if (id === null) return;
// The server owns host defaults: it keeps what the host set itself
// and resolves the rest. A desktop takes the server's values as sent.
const { getLink } = await import("./client/link-store.js");
const onServer = !(await getLink().catch(() => null));
await importHostPluginSettings(
id,
wire.pluginSettings,
onServer ? parseDefaultOverrides(wire.defaultOverrides) : null,
);
await importProtocolLogins(id, userId, wire.protocolAuth, resolveId);
await dropInvalidParent(userId, id);
if (onServer) {
const { applyDefaultsAfterHostWrite } =
await import("../hosts/defaults/index.js");
await applyDefaultsAfterHostWrite(id);
}
const { SharedHostSecretsManager } =
await import("../utils/shared-host-secrets-manager.js");
await SharedHostSecretsManager.getInstance().resyncHost(id);
},
remove: async (row, userId) => {
const { deleteOwnedHost } = await import("../hosts/delete-host.js");
await deleteOwnedHost(userId, row.id as number);
},
},
{
canReference: async (userId, _ownerId, id) => {
const { PermissionManager } =
await import("../utils/permission-manager.js");
const access = await PermissionManager.getInstance().canAccessHost(
userId,
id,
"view",
);
return access.hasAccess;
},
},
);
// After hosts: a folder's jump chain names hosts by syncId.
registerEntity(
CORE_OWNER,
{ type: "hostDefaults", table: null, order: 60 },
{
load: loadHostDefaults,
write: writeHostDefault,
erase: eraseHostDefault,
covers: (syncId) => coversHostDefault(syncId, activeManifest),
},
);
registerEntity(
CORE_OWNER,
{ type: "hostDefaultsAdmin", table: null, order: 7, singleton: true },
{
readOnly: true,
load: loadAdminDefaults,
write: writeAdminDefaults,
erase: async () => {},
},
);
// dashboardServiceLinks (70) and homepageItems (80) belong to homepage.
registerEntity(CORE_OWNER, {
type: "userPreferences",
table: userPreferences,
order: 90,
singleton: true,
readOnlyFields: ["storageMode"],
// A keybinding parameter a plugin declares with a syncEntity holds a
// local row id of that entity; the wire carries its syncId.
serialize: (row, resolveSyncId) =>
mapKeybindingReferences(row, async (entityType, value) => {
const id = Number(value);
return Number.isInteger(id) ? resolveSyncId(entityType, id) : null;
}),
deserialize: (row, resolveId) =>
mapKeybindingReferences(row, async (entityType, value) => {
const id = await resolveId(entityType, value);
return id === null ? null : String(id);
}),
});
registerEntity(
CORE_OWNER,
{ type: "pluginUserSettings", table: null, order: 95 },
{
load: loadPluginUserSettings,
write: writePluginUserSetting,
erase: erasePluginUserSetting,
covers: (syncId) => !!activeManifest(syncId.split(":")[0]),
},
);
registerEntity(
CORE_OWNER,
{ type: "accountProfile", table: null, order: 5, singleton: true },
{
readOnly: true,
load: loadAccountProfile,
write: writeAccountProfile,
erase: async () => {},
},
);
registerEntity(
CORE_OWNER,
{ type: "sharedCredentials", table: null, order: 15 },
{
readOnly: true,
load: loadSharedCredentials,
write: (userId, wire) => writeSharedCopy("sshCredentials", userId, wire),
erase: (userId, syncId) =>
eraseSharedCopy("sshCredentials", userId, syncId),
},
);
registerEntity(
CORE_OWNER,
{ type: "sharedHosts", table: null, order: 55 },
{
readOnly: true,
load: loadSharedHosts,
write: (userId, wire) => writeSharedCopy("hosts", userId, wire),
erase: (userId, syncId) => eraseSharedCopy("hosts", userId, syncId),
},
);
}
/**
* Rewrites every keybinding parameter whose declaration names a sync entity,
* keeping the column's storage form. A value that does not map becomes null.
*/
export async function mapKeybindingReferences(
row: SyncRow,
map: (entityType: string, value: string) => Promise<string | null>,
): Promise<SyncRow> {
const stored = row.customKeybindings;
if (typeof stored !== "string" || !stored) return row;
let bindings: unknown;
try {
bindings = JSON.parse(stored);
} catch {
return row;
}
if (!Array.isArray(bindings)) return row;
const { findKeybindingAction } =
await import("../database/routes/keybinding-validation.js");
let changed = false;
for (const binding of bindings) {
const action = (binding as { action?: Record<string, unknown> } | null)
?.action;
if (!action || typeof action.type !== "string") continue;
const params = findKeybindingAction(action.type)?.params ?? {};
for (const [name, param] of Object.entries(params)) {
const value = action[name];
if (!param.syncEntity || typeof value !== "string" || !value) continue;
action[name] = await map(param.syncEntity, value);
changed = true;
}
}
return changed
? { ...row, customKeybindings: JSON.stringify(bindings) }
: row;
}
/** A synced parent link that would make a cycle here is dropped. */
async function dropInvalidParent(userId: string, hostId: number) {
const { createCurrentRepositoryContext } =
await import("../database/repositories/factory.js");
const db = createCurrentRepositoryContext().drizzle;
const [row] = await db
.select({ parentHostId: hosts.parentHostId })
.from(hosts)
.where(and(eq(hosts.id, hostId), eq(hosts.userId, userId)))
.limit(1);
if (typeof row?.parentHostId !== "number") return;
const { validateParentHostId } =
await import("../database/routes/host-parent-validation.js");
if (await validateParentHostId(userId, hostId, row.parentHostId)) {
await db
.update(hosts)
.set({ parentHostId: null })
.where(and(eq(hosts.id, hostId), eq(hosts.userId, userId)));
}
}
function activeManifests(): PluginManifest[] {
try {
return getPluginRuntime()
.loader.list()
.filter((plugin) => plugin.state === "active" && plugin.manifest)
.map((plugin) => plugin.manifest as PluginManifest);
} catch {
return [];
}
}
function activeManifest(pluginId: string): PluginManifest | undefined {
return activeManifests().find((manifest) => manifest.id === pluginId);
}
async function loadPluginUserSettings(userId: string): Promise<SyncRow[]> {
const { declaredFields, getSetting } = await import("../plugins/settings.js");
const { createCurrentPluginSettingsRepository } =
await import("../database/repositories/factory.js");
const repository = createCurrentPluginSettingsRepository();
const rows: SyncRow[] = [];
for (const manifest of activeManifests()) {
const fields = declaredFields(manifest, "user");
if (fields.length === 0) continue;
const stored = new Set(
(await repository.getAll(manifest.id, "user", userId))
.filter((row) => row.value !== null)
.map((row) => row.key),
);
for (const field of fields) {
if (!stored.has(field.key)) continue;
rows.push({
syncId: `${manifest.id}:${field.key}`,
pluginId: manifest.id,
key: field.key,
value: await getSetting(manifest, "user", userId, field.key),
});
}
}
return rows;
}
async function writePluginUserSetting(userId: string, wire: SyncRow) {
const manifest = activeManifest(String(wire.pluginId));
if (!manifest) return;
const { setSetting } = await import("../plugins/settings.js");
const error = await setSetting(
manifest,
"user",
userId,
String(wire.key),
wire.value,
);
if (error) throw new Error(error);
}
async function erasePluginUserSetting(userId: string, syncId: string) {
const [pluginId, ...rest] = syncId.split(":");
const { createCurrentPluginSettingsRepository } =
await import("../database/repositories/factory.js");
await createCurrentPluginSettingsRepository().delete(
pluginId,
"user",
userId,
rest.join(":"),
);
}
async function loadAccountProfile(userId: string): Promise<SyncRow[]> {
const { createCurrentUserRepository, createCurrentRoleRepository } =
await import("../database/repositories/factory.js");
const user = await createCurrentUserRepository().findById(userId);
if (!user) return [];
const { PermissionManager } = await import("../utils/permission-manager.js");
const roles = await createCurrentRoleRepository().listUserRoles(userId);
return [
{
syncId: singletonSyncId("accountProfile"),
remoteUserId: userId,
username: user.username,
isAdmin: !!user.isAdmin,
roles: roles.map((role) => role.roleName).sort(),
permissions: (
await PermissionManager.getInstance().getUserPermissions(userId)
).sort(),
},
];
}
async function writeAccountProfile(userId: string, wire: SyncRow) {
const { updateLink } = await import("./client/link-store.js");
await updateLink({
remoteUserId: (wire.remoteUserId as string) ?? null,
remoteUsername: (wire.username as string) ?? null,
account: {
remoteUserId: wire.remoteUserId as string,
username: wire.username as string,
isAdmin: !!wire.isAdmin,
roles: (wire.roles as string[]) ?? [],
permissions: (wire.permissions as string[]) ?? [],
},
});
if (typeof wire.username === "string" && wire.username) {
const { createCurrentUserRepository } =
await import("../database/repositories/factory.js");
const users = createCurrentUserRepository();
const current = await users.findById(userId);
if (current && current.username !== wire.username) {
await users.update(userId, { username: wire.username });
}
}
}
/** Host columns a shared copy never carries as the owner has them. */
const SHARED_HOST_DROP = [
"id",
"userId",
"credentialId",
"parentHostId",
"pluginSettings",
"protocolAuth",
"quickActions",
"createdAt",
"updatedAt",
"isShared",
"permissionLevel",
...HOST_LOCAL_FIELDS,
];
async function loadSharedHosts(userId: string): Promise<SyncRow[]> {
const {
createCurrentRbacAccessRepository,
createCurrentRoleRepository,
createCurrentHostResolutionRepository,
createCurrentUserRepository,
} = await import("../database/repositories/factory.js");
const roleIds = await createCurrentRoleRepository().listUserRoleIds(userId);
const entries =
await createCurrentRbacAccessRepository().listVisibleHostAccessEntries(
userId,
roleIds,
);
const hostRepository = createCurrentHostResolutionRepository();
const owned = await hostRepository.listOwnedHostIds(userId);
const levels = new Map<number, string>();
for (const entry of entries) {
if (owned.has(entry.hostId)) continue;
if (!levels.has(entry.hostId)) {
levels.set(entry.hostId, entry.permissionLevel);
}
}
if (levels.size === 0) return [];
const { resolveHostById } = await import("../hosts/host-resolver.js");
const users = createCurrentUserRepository();
const ownerNames = new Map<string, string>();
const rows: SyncRow[] = [];
for (const [hostId, permissionLevel] of levels) {
try {
const host = (await resolveHostById(
hostId,
userId,
)) as unknown as SyncRow | null;
if (!host || typeof host.syncId !== "string") continue;
const ownerId = String(host.userId);
if (!ownerNames.has(ownerId)) {
ownerNames.set(
ownerId,
(await users.findById(ownerId))?.username ?? "",
);
}
const wire: SyncRow = {};
for (const [field, value] of Object.entries(host)) {
if (SHARED_HOST_DROP.includes(field)) continue;
wire[field] =
value !== null && typeof value === "object"
? JSON.stringify(value)
: value;
}
wire.jumpHosts = await jumpHostsToSyncIds(host.jumpHosts);
wire.pluginSettings = await exportHostPluginSettings(hostId, {
userId,
permissionLevel,
});
wire.protocolAuth = await exportSharedProtocolLogins(host, userId);
wire.syncId = host.syncId;
wire.shared = {
hostId,
owner: ownerNames.get(ownerId) ?? "",
permissionLevel,
};
rows.push(wire);
} catch (error) {
syncLogger.warn("Could not include a shared host in sync", {
operation: "sync_shared_host",
hostId,
error: error instanceof Error ? error.message : String(error),
});
}
}
return rows;
}
async function jumpHostsToSyncIds(value: unknown): Promise<string | null> {
const list = Array.isArray(value) ? value : [];
if (list.length === 0) return null;
const { createCurrentHostRepository } =
await import("../database/repositories/factory.js");
const repository = createCurrentHostRepository();
const mapped = [];
for (const item of list) {
const hostId = (item as { hostId?: unknown })?.hostId;
if (typeof hostId !== "number") continue;
const target = await repository.findById(hostId);
if (target?.syncId) mapped.push({ ...item, hostId: target.syncId });
}
return JSON.stringify(mapped);
}
async function loadSharedCredentials(userId: string): Promise<SyncRow[]> {
const {
createCurrentCredentialAccessRepository,
createCurrentCredentialRepository,
createCurrentRoleRepository,
createCurrentUserRepository,
} = await import("../database/repositories/factory.js");
const roleIds = await createCurrentRoleRepository().listUserRoleIds(userId);
const grants =
await createCurrentCredentialAccessRepository().listSharedWithUser(
userId,
roleIds,
);
if (grants.length === 0) return [];
const { findUsableCredential } =
await import("../hosts/usable-credential.js");
const credentials = createCurrentCredentialRepository();
const users = createCurrentUserRepository();
const rows: SyncRow[] = [];
for (const grant of grants) {
try {
const row = (await credentials.findById(
grant.credentialId,
)) as SyncRow | null;
const usable = (await findUsableCredential(
grant.credentialId,
userId,
)) as unknown as SyncRow | null;
if (!row || !usable || typeof row.syncId !== "string") continue;
rows.push({
syncId: row.syncId,
name: row.name,
description: row.description ?? null,
tags: row.tags ?? null,
authType: usable.authType,
username: usable.username ?? null,
password: usable.password ?? null,
privateKey: usable.privateKey ?? null,
keyPassword: usable.keyPassword ?? null,
keyType: usable.keyType ?? null,
publicKey: usable.publicKey ?? null,
certPublicKey: usable.certPublicKey ?? null,
shared: {
credentialId: grant.credentialId,
owner: (await users.findById(grant.ownerId))?.username ?? "",
permissionLevel: grant.permissionLevel,
},
});
} catch (error) {
syncLogger.warn("Could not include a shared credential in sync", {
operation: "sync_shared_credential",
credentialId: grant.credentialId,
error: error instanceof Error ? error.message : String(error),
});
}
}
return rows;
}
/** Stores a read-only copy of something shared with the linked account. */
async function writeSharedCopy(
entityType: "hosts" | "sshCredentials",
userId: string,
wire: SyncRow,
): Promise<void> {
const { getEntity } = await import("../plugins/sync-registry.js");
const { writeWireRow, createResolvers } = await import("./store.js");
const entity = getEntity(entityType);
if (!entity) return;
const { shared, ...rest } = wire;
await writeWireRow(entity, userId, rest, createResolvers(userId), {
sharedSource: JSON.stringify(shared ?? {}),
});
}
async function eraseSharedCopy(
entityType: "hosts" | "sshCredentials",
userId: string,
syncId: string,
): Promise<void> {
const { getEntity } = await import("../plugins/sync-registry.js");
const { deleteStoredRow, findStoredRow } = await import("./store.js");
const entity = getEntity(entityType);
if (!entity) return;
const row = await findStoredRow(entity, userId, syncId);
if (!row?.sharedSource) return;
await deleteStoredRow(entity, userId, syncId);
}