Ethica/Termix
1
0 Fork 0
mirror of https://github.com/Termix-SSH/Termix.git synced 2026-10-29 22:55:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: finish the automations plugin (B17)

Browse Source
This commit is contained in:
LukeGus committed 2026-09-24 13:05:44 -05:00
1 parent 993a91c899
commit 6b4d1432ab
96 files changed
+25106 -5077
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

plugins/automations/src/backend/actions/host-targets.ts
+43 -42
View File
@@ -1,39 +1,45 @@
import type { HostSelector } from "../../../../../src/types/automations.js";
import { resolveHostById } from "../../../../../src/backend/hosts/host-resolver.js";
import { PermissionManager } from "../../../../../src/backend/utils/permission-manager.js";
import { fleetMemberIds } from "../snippets.js";
import type { StepExecutionContext } from "./types.js";
import type { PluginSshHost } from "@termix/plugin-sdk/backend";
import type { HostSelector } from "../../types.js";
import type { StepExecutionContext, StepRuntime } from "./types.js";
/** A host the caller is allowed to act on. `host` is always resolved. */
/** A host the owner may act on, resolved through core's connect pipeline. */
export interface ResolvedTarget {
id: number;
name: string;
host: NonNullable<Awaited<ReturnType<typeof resolveHostById>>>;
host: PluginSshHost;
}
/**
* Turns a selector into the hosts a step may actually act on.
*
* Access is checked here, at execution time rather than when the automation
* was saved, so a permission revoked after the fact takes effect on the next
* run. resolveHostById performs its own connect-level check and returns null
* when the owner can no longer reach the host.
* Access is checked at execution time rather than when the automation was
* saved, so a permission revoked after the fact takes effect on the next run.
* resolveHost runs as the automation's owner and answers null when they can
* no longer reach the host.
*/
export async function resolveTargets(
selector: HostSelector,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<{ targets: ResolvedTarget[]; skipped: number[] }> {
const ids = await selectorHostIds(selector, context);
const ids = await selectorHostIds(selector, context, runtime);
const targets: ResolvedTarget[] = [];
const skipped: number[] = [];
for (const id of ids) {
const host = await resolveHostById(id, context.userId);
let host: PluginSshHost | null = null;
try {
host = await runtime.ctx.ssh.resolveHost(id);
} catch {
host = null;
}
if (!host) {
skipped.push(id);
continue;
}
targets.push({ id, name: host.name || host.ip, host });
const name =
typeof host.name === "string" && host.name ? host.name : host.ip;
targets.push({ id, name, host });
}
return { targets, skipped };
@@ -42,6 +48,7 @@ export async function resolveTargets(
async function selectorHostIds(
selector: HostSelector,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<number[]> {
switch (selector.kind) {
case "host":
@@ -50,35 +57,29 @@ async function selectorHostIds(
return selector.hostIds;
case "trigger":
return context.triggerHostId ? [context.triggerHostId] : [];
case "fleet":
return fleetHostIds(selector.fleetId, context.userId);
case "all":
return allAccessibleHostIds(context.userId);
case "fleet": {
const fleets = runtime.deps.fleets();
if (typeof fleets.members !== "function") return [];
try {
const members = await fleets.members(selector.fleetId);
return members.map((member) => member.id);
} catch {
return [];
}
}
case "all": {
try {
const hosts = await runtime.ctx.hosts.list();
// The owner's own hosts, as before: "all" never fans out onto hosts
// someone else shared with them.
return hosts
.filter((host) => host.userId === context.userId)
.map((host) => host.id);
} catch {
return [];
}
}
default:
return [];
}
}
async function fleetHostIds(
fleetId: number,
userId: string,
): Promise<number[]> {
return fleetMemberIds(userId, fleetId);
}
async function allAccessibleHostIds(userId: string): Promise<number[]> {
try {
const { createCurrentHostRepository } =
await import("../../../../../src/backend/database/repositories/factory.js");
const hosts = await createCurrentHostRepository().listByUserId(userId);
const ids = hosts.map((host) => host.id);
const allowed =
await PermissionManager.getInstance().filterAccessibleHostIds(
userId,
ids,
);
return ids.filter((id) => allowed.has(id));
} catch {
return [];
}
}
plugins/automations/src/backend/actions/index.ts
+145 -104
View File
@@ -1,15 +1,6 @@
import {
DEFAULT_STEP_TIMEOUT_MS,
type Step,
} from "../../../../../src/types/automations.js";
import type { Client } from "ssh2";
import { DEFAULT_STEP_TIMEOUT_MS, type Step } from "../../types.js";
import { execCommand, execElevated } from "@termix/plugin-sdk/host-commands";
import { withSshConnection } from "../ssh.js";
import { createCurrentNotificationChannelRepository } from "../../../../../src/backend/database/repositories/factory.js";
import { getSnippet, resolveSnippetCommandFor } from "../snippets.js";
import { runDockerAction } from "../docker.js";
import { runTunnelAction } from "../tunnels.js";
import { sendAutomationNotification } from "../notify.js";
import { automationFetch } from "../http.js";
import { renderRecord, renderTemplate } from "../template.js";
import { resolveTargets, type ResolvedTarget } from "./host-targets.js";
import {
@@ -18,8 +9,14 @@ import {
stepTimeout,
type StepExecutionContext,
type StepResult,
type StepRuntime,
} from "./types.js";
/** The admin allowlist of private hosts an HTTP step may reach. */
const PRIVATE_ALLOWLIST_KEY = "notification_private_endpoint_allowlist";
const errorText = (error: unknown) =>
error instanceof Error ? error.message : String(error);
/**
* One executor per step type.
*
@@ -30,22 +27,23 @@ import {
export async function executeStep(
step: Step,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
switch (step.type) {
case "notify":
return runNotify(step, context);
return runNotify(step, context, runtime);
case "http":
return runHttp(step, context);
return runHttp(step, context, runtime);
case "run_command":
return runCommand(step, context);
return runCommand(step, context, runtime);
case "run_snippet":
return runSnippet(step, context);
return runSnippet(step, context, runtime);
case "docker":
return runDocker(step, context);
return runDocker(step, context, runtime);
case "tunnel":
return runTunnel(step, context);
return runTunnel(step, context, runtime);
case "wol":
return runWol(step, context);
return runWol(step, context, runtime);
case "wait":
return runWait(step, context);
case "set_var":
@@ -64,6 +62,7 @@ export async function executeStep(
async function runNotify(
step: Extract<Step, { type: "notify" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const title = renderTemplate(step.title ?? "", context.template);
const body = renderTemplate(step.body ?? "", context.template);
@@ -77,49 +76,45 @@ async function runNotify(
);
}
const repository = createCurrentNotificationChannelRepository();
const channels = await repository.listNotificationChannels(context.userId);
const selected = channels.filter((channel) =>
step.channelIds.includes(channel.id),
);
if (selected.length === 0) {
return fail("Selected notification channels no longer exist");
const trigger = context.template.trigger ?? {};
let result;
try {
result = await runtime.ctx.notify.send(step.channelIds, {
title: title || "Termix automation",
body,
severity: step.severity ?? "warning",
context: {
hostId:
context.template.host?.id ??
(typeof trigger.hostId === "number" ? trigger.hostId : undefined),
hostName:
context.template.host?.name ??
(typeof trigger.hostName === "string" ? trigger.hostName : undefined),
sourceId: context.automationId,
sourceName: title || undefined,
triggerType:
typeof trigger.type === "string" ? trigger.type : undefined,
value: trigger.value,
threshold: trigger.threshold,
},
});
} catch (error) {
return fail(errorText(error));
}
let delivered = 0;
const errors: string[] = [];
for (const channel of selected) {
if (!channel.enabled) continue;
try {
await sendAutomationNotification(
{ id: channel.id, type: channel.type, config: channel.config },
{
title,
body,
severity: step.severity ?? "warning",
context: context.template,
},
);
delivered++;
} catch (error) {
errors.push(
`${channel.name}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
if (delivered === 0) {
const errors = result.failures.map((f) => `${f.name}: ${f.error}`);
if (result.delivered === 0) {
return fail(errors.join("; ") || "No enabled channels to notify");
}
return ok(
`Notified ${delivered} channel(s)${errors.length ? `; ${errors.join("; ")}` : ""}`,
`Notified ${result.delivered} channel(s)${errors.length ? `; ${errors.join("; ")}` : ""}`,
);
}
async function runHttp(
step: Extract<Step, { type: "http" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const url = renderTemplate(step.url, context.template);
const headers = renderRecord(step.headers, context.template);
@@ -132,11 +127,20 @@ async function runHttp(
}
try {
const response = await automationFetch(url, {
// Private delivery needs both the step's opt-in and an exact host in
// the admin allowlist.
const allowPrivateHosts = step.allowPrivateNetwork
? parseAllowlist(
await runtime.ctx.settings.readCore(PRIVATE_ALLOWLIST_KEY),
)
: [];
const response = await runtime.ctx.fetch(url, {
method: step.method,
headers,
headers: body
? { "Content-Type": "application/json", ...(headers ?? {}) }
: headers,
body,
allowPrivateNetwork: step.allowPrivateNetwork,
allowPrivateHosts,
timeoutMs: stepTimeout(context, step.timeoutMs, DEFAULT_STEP_TIMEOUT_MS),
});
@@ -144,16 +148,31 @@ async function runHttp(
const summary = `HTTP ${response.status} ${response.statusText}\n${text}`;
return response.ok ? ok(summary) : fail(`HTTP ${response.status}`, summary);
} catch (error) {
return fail(error instanceof Error ? error.message : String(error));
return fail(errorText(error));
}
}
function parseAllowlist(raw: unknown): string[] {
if (typeof raw !== "string" || !raw) return [];
try {
const value = JSON.parse(raw);
if (!Array.isArray(value)) return [];
return value
.filter((entry): entry is string => typeof entry === "string")
.map((entry) => entry.trim().toLowerCase())
.filter(Boolean);
} catch {
return [];
}
}
async function runCommand(
step: Extract<Step, { type: "run_command" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const command = renderTemplate(step.command, context.template);
return runOnTargets(step.hostSelector, context, async (target) => {
return runOnTargets(step.hostSelector, context, runtime, async (target) => {
if (context.dryRun) {
return { output: `Would run on ${target.name}: ${command}` };
}
@@ -162,6 +181,7 @@ async function runCommand(
command,
step.elevated,
context,
runtime,
step.timeoutMs,
);
});
@@ -170,10 +190,20 @@ async function runCommand(
async function runSnippet(
step: Extract<Step, { type: "run_snippet" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const snippet = await getSnippet(context.userId, step.snippetId);
const snippets = runtime.deps.snippets();
const getSnippet = snippets.get;
const resolveCommand = snippets.resolveCommand;
if (
typeof getSnippet !== "function" ||
typeof resolveCommand !== "function"
) {
return fail("The snippets plugin is not available");
}
const snippet = await getSnippet(step.snippetId).catch(() => null);
if (!snippet) return fail("Snippet not found, or the snippets plugin is off");
if (!snippet) return fail("Snippet not found");
if (snippet.isNote) return fail("Notes cannot be executed on a host");
// Template values only ever reach the snippet through inputValues, never by
@@ -181,18 +211,17 @@ async function runSnippet(
// syntax of their own.
const inputValues = renderRecord(step.inputValues, context.template) ?? {};
return runOnTargets(step.hostSelector, context, async (target) => {
const command = await resolveSnippetCommandFor(
context.userId,
return runOnTargets(step.hostSelector, context, runtime, async (target) => {
const command = await resolveCommand(
step.snippetId,
{
ip: target.host.ip,
username: target.host.username,
port: target.host.port,
name: target.host.name,
name: target.name,
},
inputValues,
);
).catch(() => null);
if (!command) {
return { output: "", error: "Snippet could not be resolved" };
}
@@ -205,6 +234,7 @@ async function runSnippet(
command,
step.elevated,
context,
runtime,
step.timeoutMs,
);
});
@@ -213,40 +243,51 @@ async function runSnippet(
async function runDocker(
step: Extract<Step, { type: "docker" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const container = renderTemplate(step.container, context.template);
if (!container) return fail("Container name is required");
const action = runtime.deps.docker().action;
if (typeof action !== "function") {
return fail("The docker plugin is not available");
}
return runOnTargets(step.hostSelector, context, async (target) => {
return runOnTargets(step.hostSelector, context, runtime, async (target) => {
if (context.dryRun) {
return {
output: `Would ${step.action} container ${container} on ${target.name}`,
};
}
const result = await runDockerAction(
context.userId,
target.id,
container,
step.action,
);
return result.ok
? { output: `Container ${container}: ${step.action} done` }
: { output: "", error: result.error };
try {
await action(target.id, container, step.action);
return { output: `Container ${container}: ${step.action} done` };
} catch (error) {
return { output: "", error: errorText(error) };
}
});
}
async function runTunnel(
step: Extract<Step, { type: "tunnel" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const name = renderTemplate(step.tunnelName, context.template);
if (context.dryRun) return ok(`Would ${step.action} tunnel ${name}`);
const tunnels = runtime.deps.tunnels();
const act = step.action === "connect" ? tunnels.start : tunnels.stop;
if (typeof act !== "function") {
return fail("The tunnels plugin is not available");
}
// A disconnect goes through the tunnels plugin's manual stop, which also
// holds off its retries, so the tunnel stays down.
const result = await runTunnelAction(context.userId, step.action, name);
if (!result.ok) return fail(result.error);
try {
await act(name);
} catch (error) {
return fail(errorText(error));
}
return ok(
step.action === "connect"
? `Tunnel ${name} connected`
@@ -257,26 +298,23 @@ async function runTunnel(
async function runWol(
step: Extract<Step, { type: "wol" }>,
context: StepExecutionContext,
runtime: StepRuntime,
): Promise<StepResult> {
const host = await resolveTargets(
{ kind: "host", hostId: step.hostId },
context,
);
const target = host.targets[0];
if (!target?.host) return fail("Host not found or not accessible");
const host = await runtime.ctx.hosts.get(step.hostId).catch(() => null);
if (!host) return fail("Host not found or not accessible");
const name = host.name || host.ip;
const mac = (target.host as { macAddress?: string }).macAddress;
if (!mac) return fail("Host has no MAC address configured");
if (context.dryRun) return ok(`Would wake ${target.name} (${mac})`);
const wake = runtime.deps.wakeOnLan().wake;
if (typeof wake !== "function") {
return fail("The wake-on-lan plugin is not available");
}
if (context.dryRun) return ok(`Would wake ${name}`);
try {
const { sendWakeOnLan, isValidMac } =
await import("../../../../../src/backend/utils/wake-on-lan.js");
if (!isValidMac(mac)) return fail(`Invalid MAC address: ${mac}`);
await sendWakeOnLan(mac);
return ok(`Sent magic packet to ${target.name}`);
await wake(step.hostId);
return ok(`Sent magic packet to ${name}`);
} catch (error) {
return fail(error instanceof Error ? error.message : String(error));
return fail(errorText(error));
}
}
@@ -310,9 +348,10 @@ async function runSetVar(
async function runOnTargets(
selector: Extract<Step, { type: "run_command" }>["hostSelector"],
context: StepExecutionContext,
runtime: StepRuntime,
run: (target: ResolvedTarget) => Promise<{ output?: string; error?: string }>,
): Promise<StepResult> {
const { targets, skipped } = await resolveTargets(selector, context);
const { targets, skipped } = await resolveTargets(selector, context, runtime);
if (targets.length === 0) {
return fail(
@@ -363,25 +402,27 @@ async function execOnHost(
command: string,
elevated: boolean | undefined,
context: StepExecutionContext,
runtime: StepRuntime,
timeoutMs: number | undefined,
): Promise<{ output?: string; error?: string }> {
const sshHost = host as ResolvedTarget["host"] & { sudoPassword?: string };
const timeout = stepTimeout(context, timeoutMs, DEFAULT_STEP_TIMEOUT_MS);
if (timeout <= 0) return { error: "Run deadline exceeded" };
try {
const result = await withSshConnection(
sshHost as never,
{ pool: "fleet", purpose: "fleet" },
async (client) => {
if (elevated) {
return execElevated(client, command, sshHost.sudoPassword, {
timeoutMs: timeout,
});
}
return execCommand(client, command, timeout);
},
);
const result = await runtime.ctx.ssh.withConnection<
{ stdout: string; stderr: string; code: number | null },
Client
>(host, { pool: "automations", purpose: "fleet" }, async (client) => {
if (elevated) {
return execElevated(
client,
command,
host.sudoPassword as string | undefined,
{ timeoutMs: timeout },
);
}
return execCommand(client, command, timeout);
});
const output = [result.stdout, result.stderr].filter(Boolean).join("\n");
if (result.code === 0 || result.code === null) {
@@ -389,6 +430,6 @@ async function execOnHost(
}
return { error: `Exited with code ${result.code}`, output };
} catch (error) {
return { error: error instanceof Error ? error.message : String(error) };
return { error: errorText(error) };
}
}
plugins/automations/src/backend/actions/types.ts
+9 -1
View File
@@ -1,4 +1,6 @@
import type { Step } from "../../../../../src/types/automations.js";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import type { Step } from "../../types.js";
import type { Deps } from "../deps.js";
import type { TemplateContext } from "../template.js";
/** What every executor returns. `output` is what later steps can read. */
@@ -12,6 +14,12 @@ export interface StepResult {
halt?: { status: "success" | "failed" };
}
/** What the executors reach the rest of Termix through. */
export interface StepRuntime {
ctx: PluginContext;
deps: Deps;
}
export interface StepExecutionContext {
userId: string;
automationId: number;
plugins/automations/src/backend/conditions.ts
+1 -4
View File
@@ -1,7 +1,4 @@
import type {
MetricPath,
Operator,
} from "../../../../src/types/automations.js";
import type { MetricPath, Operator } from "../types.js";
/**
* Operator evaluation and metric extraction.
plugins/automations/src/backend/deps.ts
+121
View File
@@ -0,0 +1,121 @@
import type { PluginServices } from "@termix/plugin-sdk/backend";
import type { DockerEventKind } from "../types.js";
/*
* Other plugins, as automations reaches them. Every call runs as the current
* actor (the automation's owner, set by the engine with ctx.asUser). A
* missing provider answers an empty handle, so each method is probed before
* it is called.
*/
export interface SnippetsAccessV1 {
list?: () => Promise<Array<{ id: number; name: string; isNote?: boolean }>>;
get: (id: number) => Promise<{
id: number;
name: string;
content: string;
isNote: boolean;
} | null>;
resolveCommand: (
id: number,
vars: { ip?: string; username?: string; port?: number; name?: string },
inputValues?: Record<string, string>,
) => Promise<string | null>;
}
export interface FleetsAccessV1 {
list: () => Promise<Array<{ id: number; name: string }>>;
members: (
fleetId: number,
) => Promise<Array<{ id: number; name: string | null; ip: string }>>;
}
export interface TunnelsAccessV1 {
start: (name: string) => Promise<void>;
stop: (name: string) => Promise<void>;
}
export interface DockerContainersV1 {
action: (
hostId: number,
container: string,
action: "start" | "stop" | "restart",
) => Promise<void>;
}
export interface DockerEvent {
hostId: number;
container: string;
event: DockerEventKind;
}
export interface DockerEventsV1 {
subscribe: (
hostId: number,
listener: (event: DockerEvent) => void,
) => Promise<() => void>;
}
export interface MetricsViewersV1 {
register: (
hostId: number,
) => Promise<{ viewerSessionId: string } | { skipped: true; reason: string }>;
heartbeat: (viewerSessionId: string) => boolean;
unregister: (hostId: number, viewerSessionId: string) => void;
}
/** Provided by the wake-on-lan plugin. It resolves the host's MAC itself. */
export interface WakeOnLanV1 {
wake: (hostId: number) => Promise<void>;
}
/** The plugin id behind each service, for "needs <plugin>" messages. */
export const PROVIDERS = {
snippets: "snippets.access",
fleets: "fleets.access",
tunnels: "tunnels.access",
docker: "docker.containers",
"docker-events": "docker.events",
"host-metrics": "host-metrics.viewers",
"wake-on-lan": "wake-on-lan.send",
} as const;
export type ProviderKey = keyof typeof PROVIDERS;
/** The plugin a provider key belongs to. */
export function pluginOf(key: ProviderKey): string {
return key === "docker-events" ? "docker" : key;
}
export type Deps = ReturnType<typeof createDeps>;
export function createDeps(services: PluginServices) {
function handle<T extends object>(name: string): Partial<T> {
try {
return services.get<T>(name) as Partial<T>;
} catch {
return {};
}
}
return {
/** Which providers are running now. */
available(key: ProviderKey): boolean {
return services.providers(PROVIDERS[key]).length > 0;
},
providers(): Record<ProviderKey, boolean> {
const result = {} as Record<ProviderKey, boolean>;
for (const key of Object.keys(PROVIDERS) as ProviderKey[]) {
result[key] = services.providers(PROVIDERS[key]).length > 0;
}
return result;
},
snippets: () => handle<SnippetsAccessV1>(PROVIDERS.snippets),
fleets: () => handle<FleetsAccessV1>(PROVIDERS.fleets),
tunnels: () => handle<TunnelsAccessV1>(PROVIDERS.tunnels),
docker: () => handle<DockerContainersV1>(PROVIDERS.docker),
dockerEvents: () => handle<DockerEventsV1>(PROVIDERS["docker-events"]),
viewers: () => handle<MetricsViewersV1>(PROVIDERS["host-metrics"]),
wakeOnLan: () => handle<WakeOnLanV1>(PROVIDERS["wake-on-lan"]),
};
}
plugins/automations/src/backend/docker-watcher.ts
+67 -92
View File
@@ -1,7 +1,6 @@
import type { AutomationDefinition } from "../../../../src/types/automations.js";
import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
import { subscribeDockerEvents, type DockerEvent } from "./docker.js";
import { onDockerEvent } from "./triggers.js";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import type { Deps, DockerEvent } from "./deps.js";
import type { DockerEvent as TriggerDockerEvent } from "./triggers.js";
/**
* Keeps a docker.events subscription for every host a docker_event trigger
@@ -21,95 +20,71 @@ interface Watch {
subscribedAt: number;
}
const watches = new Map<string, Watch>();
export function createDockerWatcher(
ctx: Pick<PluginContext, "asUser">,
deps: Pick<Deps, "dockerEvents">,
watchedHosts: () => Promise<Map<number, string>>,
onEvent: (event: TriggerDockerEvent) => Promise<void>,
) {
const watches = new Map<string, Watch>();
const keyOf = (hostId: number, userId: string) => `${userId}:${hostId}`;
const keyOf = (hostId: number, userId: string) => `${userId}:${hostId}`;
/** Hosts named by an enabled docker_event trigger, with the owning user. */
export async function listDockerWatchedHosts(): Promise<Map<number, string>> {
const watched = new Map<number, string>();
try {
const rows = await createCurrentAutomationRepository().listAllEnabled();
for (const row of rows) {
let definition: AutomationDefinition;
try {
definition = JSON.parse(row.definition) as AutomationDefinition;
} catch {
continue;
}
const trigger = definition.trigger;
if (trigger?.kind !== "docker_event") continue;
const selector = trigger.hostSelector;
if (selector?.kind === "host") {
watched.set(selector.hostId, row.userId);
} else if (selector?.kind === "hosts") {
for (const hostId of selector.hostIds) watched.set(hostId, row.userId);
}
// Fleet and "all" selectors are deliberately not expanded: watching
// every host a user owns for container state is far too costly.
}
} catch {
return watched;
}
return watched;
}
/** Brings the subscriptions in line with the watched hosts. */
export async function reconcileDockerWatch(
now: number = Date.now(),
watched?: Map<number, string>,
): Promise<void> {
const wanted = watched ?? (await listDockerWatchedHosts());
const wantedKeys = new Set(
[...wanted].map(([hostId, userId]) => keyOf(hostId, userId)),
);
for (const [key, watch] of watches) {
if (wantedKeys.has(key)) continue;
watch.unsubscribe?.();
watches.delete(key);
}
for (const [hostId, userId] of wanted) {
const key = keyOf(hostId, userId);
let watch = watches.get(key);
if (!watch) {
watch = {
listener: (event) => {
void onDockerEvent({
hostId: event.hostId,
ownerUserId: userId,
container: event.container,
event: event.event,
}).catch(() => undefined);
},
unsubscribe: null,
subscribedAt: 0,
};
watches.set(key, watch);
}
if (watch.subscribedAt && now - watch.subscribedAt < RESUBSCRIBE_MS) {
continue;
}
try {
watch.unsubscribe = await subscribeDockerEvents(
userId,
hostId,
watch.listener,
return {
/** Brings the subscriptions in line with the watched hosts. */
async reconcile(now: number = Date.now()): Promise<void> {
const wanted = await watchedHosts();
const wantedKeys = new Set(
[...wanted].map(([hostId, userId]) => keyOf(hostId, userId)),
);
watch.subscribedAt = watch.unsubscribe ? now : 0;
} catch {
watch.subscribedAt = 0;
}
}
}
/** Drops every subscription, for shutdown and tests. */
export function resetDockerWatcher(): void {
for (const watch of watches.values()) watch.unsubscribe?.();
watches.clear();
for (const [key, watch] of watches) {
if (wantedKeys.has(key)) continue;
watch.unsubscribe?.();
watches.delete(key);
}
for (const [hostId, userId] of wanted) {
const key = keyOf(hostId, userId);
let watch = watches.get(key);
if (!watch) {
watch = {
listener: (event) => {
void onEvent({
hostId: event.hostId,
ownerUserId: userId,
container: event.container,
event: event.event,
}).catch(() => undefined);
},
unsubscribe: null,
subscribedAt: 0,
};
watches.set(key, watch);
}
if (watch.subscribedAt && now - watch.subscribedAt < RESUBSCRIBE_MS) {
continue;
}
const subscribe = deps.dockerEvents().subscribe;
if (typeof subscribe !== "function") {
watch.subscribedAt = 0;
continue;
}
try {
const current = watch;
watch.unsubscribe = await ctx.asUser(userId, () =>
subscribe(hostId, current.listener),
);
watch.subscribedAt = now;
} catch {
watch.subscribedAt = 0;
}
}
},
/** Drops every subscription, on deactivate. */
reset(): void {
for (const watch of watches.values()) watch.unsubscribe?.();
watches.clear();
},
};
}
plugins/automations/src/backend/docker.ts
-85
View File
@@ -1,85 +0,0 @@
import type { PluginServices } from "@termix/plugin-sdk/backend";
let current: PluginServices | null = null;
/** Set in activate, cleared on deactivate. */
export function setDockerServices(services: PluginServices | null): void {
current = services;
}
export type DockerEventName = "exited" | "started" | "unhealthy" | "restarting";
export interface DockerEvent {
hostId: number;
container: string;
event: DockerEventName;
}
/** The docker plugin's "docker.containers" service, version 1. */
interface DockerService {
action: (
hostId: number,
container: string,
action: "start" | "stop" | "restart",
) => Promise<void>;
}
/** The docker plugin's "docker.events" service, version 1. */
interface DockerEventsService {
subscribe: (
hostId: number,
listener: (event: DockerEvent) => void,
) => Promise<() => void>;
}
const UNAVAILABLE = "The docker plugin is not available";
function service<T extends object>(
name: string,
userId: string,
): Partial<T> | null {
try {
return current?.get<T>(name, { userId }) ?? null;
} catch {
return null;
}
}
/**
* Starts, stops or restarts a container as userId. Optional: without the
* docker plugin the step fails with a clear message.
*/
export async function runDockerAction(
userId: string,
hostId: number,
container: string,
action: "start" | "stop" | "restart",
): Promise<{ ok: boolean; unavailable?: boolean; error?: string }> {
const docker = service<DockerService>("docker.containers", userId);
if (!docker || typeof docker.action !== "function") {
return { ok: false, unavailable: true, error: UNAVAILABLE };
}
try {
await docker.action(hostId, container, action);
return { ok: true };
} catch (error) {
return {
ok: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
/**
* Subscribes `listener` to container changes on a host as userId, or
* resolves null while the docker plugin is off.
*/
export async function subscribeDockerEvents(
userId: string,
hostId: number,
listener: (event: DockerEvent) => void,
): Promise<(() => void) | null> {
const events = service<DockerEventsService>("docker.events", userId);
if (!events || typeof events.subscribe !== "function") return null;
return events.subscribe(hostId, listener);
}
plugins/automations/src/backend/engine.ts
+91 -59
View File
@@ -1,22 +1,24 @@
import type {
AutomationDefinition,
RunStatus,
Step,
} from "../../../../src/types/automations.js";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import type { AutomationDefinition, RunStatus, Step } from "../types.js";
import {
DEFAULT_MAX_RUN_SECONDS,
MAX_AUTOMATION_DEPTH,
MAX_STEP_OUTPUT_BYTES,
} from "../../../../src/types/automations.js";
import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
import { statsLogger } from "../../../../src/backend/utils/logger.js";
import { resolveHostById } from "../../../../src/backend/hosts/host-resolver.js";
import { notifyAutomationInternalEvent } from "../../../../src/backend/hosts/automation-events.js";
} from "../types.js";
import { executeStep } from "./actions/index.js";
import type { StepExecutionContext, StepResult } from "./actions/types.js";
import { compare } from "./conditions.js";
import type { Deps } from "./deps.js";
import type {
AutomationEngineRow,
AutomationRepository,
} from "./repository.js";
import { missingPlugins } from "./requirements.js";
import { renderTemplate, type TemplateContext } from "./template.js";
/** Emitted when a run fails, so an automation_failed trigger can react. */
export const TOPIC_AUTOMATION_FAILED = "plugin.automations.automation_failed";
export interface RunRequest {
automationId: number;
triggerType: string;
@@ -36,38 +38,53 @@ export interface RunOutcome {
}
/**
* Executes automations.
*
* Both HTTP handlers and the metrics hooks live in this same process, so this
* is a plain singleton rather than anything cross-process. State that has to
* survive a restart (cooldowns, dwell windows) lives in the database; the only
* thing held in memory is the set of runs currently in flight, which is
* meaningless after a restart anyway.
* Executes automations. One per activation. State that has to survive a
* restart (cooldowns, dwell windows) lives in the database; the only thing
* held in memory is the set of runs in flight, which is meaningless after a
* restart anyway.
*/
export class AutomationEngine {
private static instance: AutomationEngine;
private readonly running = new Set<number>();
private readonly queued = new Map<number, number>();
static getInstance(): AutomationEngine {
if (!AutomationEngine.instance) {
AutomationEngine.instance = new AutomationEngine();
}
return AutomationEngine.instance;
}
constructor(
private readonly ctx: PluginContext,
private readonly repository: AutomationRepository,
private readonly deps: Deps,
) {}
isRunning(automationId: number): boolean {
return this.running.has(automationId);
}
/**
* Runs an automation as its owner. Never throws: a failure is the outcome.
*/
async run(request: RunRequest): Promise<RunOutcome> {
const repository = createCurrentAutomationRepository();
const automation = await repository.findById(request.automationId);
let automation: AutomationEngineRow | null;
try {
automation = await this.repository.findById(request.automationId);
} catch (err) {
return { runId: null, status: "failed", error: errorText(err) };
}
if (!automation) {
return { runId: null, status: "failed", error: "Automation not found" };
}
const owner = automation;
try {
return await this.ctx.asUser(owner.userId, () =>
this.runAsOwner(owner, request),
);
} catch (err) {
return { runId: null, status: "failed", error: errorText(err) };
}
}
private async runAsOwner(
automation: AutomationEngineRow,
request: RunRequest,
): Promise<RunOutcome> {
const repository = this.repository;
const depth = request.depth ?? 0;
const ancestry = request.ancestry ?? [];
@@ -99,6 +116,30 @@ export class AutomationEngine {
};
}
// A trigger or step whose plugin is off skips the whole run with the
// reason on record, rather than running half of it.
const missing = missingPlugins(definition, this.deps);
if (missing.length > 0) {
const reason = `Needs the ${missing.join(", ")} plugin${missing.length > 1 ? "s" : ""}`;
this.ctx.log.info(
`Skipping automation ${automation.id} (${request.triggerType}): ${reason}`,
);
const run = await repository.createRun({
automationId: automation.id,
userId: automation.userId,
triggerType: request.triggerType,
triggerContext: JSON.stringify(request.triggerContext ?? {}),
status: "skipped",
parentRunId: request.parentRunId ?? null,
});
await repository.finishRun(run.id, {
status: "skipped",
error: reason,
durationMs: 0,
});
return { runId: run.id, status: "skipped", error: reason };
}
// A second trigger while a run is in flight is recorded as skipped rather
// than dropped silently, so the history explains what happened.
//
@@ -182,10 +223,7 @@ export class AutomationEngine {
let host: TemplateContext["host"];
if (request.triggerHostId) {
try {
const resolved = await resolveHostById(
request.triggerHostId,
automation.userId,
);
const resolved = await this.ctx.hosts.get(request.triggerHostId);
if (resolved) {
host = {
id: request.triggerHostId,
@@ -245,7 +283,7 @@ export class AutomationEngine {
}
} catch (err) {
status = "failed";
error = err instanceof Error ? err.message : String(err);
error = errorText(err);
} finally {
this.running.delete(automation.id);
}
@@ -257,31 +295,21 @@ export class AutomationEngine {
});
if (status === "failed") {
statsLogger.warn(`Automation "${automation.name}" failed`, {
operation: "automation_run_failed",
automationId: automation.id,
runId: run.id,
error,
});
this.ctx.log.warn(
`Automation "${automation.name}" (${automation.id}) failed in run ${run.id}: ${error ?? "unknown error"}`,
);
// An automation_failed handler that itself fails must not re-announce
// its own failure, so the event is not emitted for runs that this event
// already started.
if (request.triggerType !== "internal_event") {
// Emitted onto the bus rather than imported from hosts/: the bus is
// fire-and-forget, so this no longer needs the lazy-import dance that
// was working around the repositories cycle.
notifyAutomationInternalEvent(
"automation_failed",
automation.userId,
undefined,
{
automationId: automation.id,
automationName: automation.name,
runId: run.id,
error: error ?? null,
},
);
this.ctx.events.emit(TOPIC_AUTOMATION_FAILED, {
userId: automation.userId,
automationId: automation.id,
automationName: automation.name,
runId: run.id,
error: error ?? null,
});
}
}
@@ -301,7 +329,7 @@ export class AutomationEngine {
error?: string;
halted?: { status: "success" | "failed" };
}> {
const repository = createCurrentAutomationRepository();
const repository = this.repository;
for (const step of steps) {
if (step.enabled === false) continue;
@@ -382,12 +410,12 @@ export class AutomationEngine {
let result: StepResult;
try {
result = await executeStep(step, context);
result = await executeStep(step, context, {
ctx: this.ctx,
deps: this.deps,
});
} catch (err) {
result = {
success: false,
error: err instanceof Error ? err.message : String(err),
};
result = { success: false, error: errorText(err) };
}
const { text, truncated } = truncate(result.output);
@@ -432,6 +460,10 @@ export class AutomationEngine {
}
}
function errorText(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
/** Keeps a single step's output from bloating the database. */
function truncate(output: string | undefined): {
text: string | null;
plugins/automations/src/backend/headless-viewer.ts
+84 -88
View File
@@ -1,105 +1,101 @@
import { statsLogger } from "../../../../src/backend/utils/logger.js";
import { listAutomationWatchedHosts } from "./triggers.js";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import type { Deps } from "./deps.js";
/**
* Keeps metric collection running for hosts an automation watches.
*
* Heavy metric collection is normally started by a UI viewer and stops when
* the last one leaves, which means threshold rules only ever evaluated while
* somebody had the host open. Automations register a synthetic viewer instead
* of bypassing that mechanism, so a real viewer arriving or leaving still
* behaves exactly as before.
*
* The catch is `cleanupInactiveViewers`, which drops any viewer whose
* heartbeat is older than 120s. Without the heartbeat below, headless polling
* would quietly stop two minutes after it started.
* Host metrics only polls hosts somebody is looking at. Automations register
* as a viewer through the host-metrics.viewers service, as the automation's
* owner, and heartbeat every tick so the viewer is not reaped. A heartbeat
* that comes back false means host-metrics restarted and forgot the viewer,
* so the next tick registers it again.
*/
export interface ViewerRegistry {
registerViewer(hostId: number, sessionId: string, userId: string): void;
unregisterViewer(hostId: number, sessionId: string): void;
updateHeartbeat(sessionId: string): boolean;
interface Registered {
userId: string;
viewerSessionId: string;
}
const SESSION_PREFIX = "automation:";
let registry: ViewerRegistry | null = null;
const registered = new Map<number, string>();
export function setViewerRegistry(next: ViewerRegistry | null): void {
registry = next;
}
export function automationSessionId(hostId: number): string {
return `${SESSION_PREFIX}${hostId}`;
}
/**
* Brings the set of synthetic viewers in line with what the enabled
* automations currently watch, and heartbeats the ones that stay.
*/
export async function reconcileHeadlessViewers(): Promise<{
added: number;
removed: number;
active: number;
}> {
if (!registry) return { added: 0, removed: 0, active: 0 };
let watched: Map<number, string>;
try {
watched = await listAutomationWatchedHosts();
} catch {
return { added: 0, removed: 0, active: registered.size };
}
let added = 0;
let removed = 0;
for (const [hostId, userId] of watched) {
const sessionId = automationSessionId(hostId);
if (registered.has(hostId)) {
// Refresh before the 120s reaper would take it.
registry.updateHeartbeat(sessionId);
continue;
}
export function createHeadlessViewers(
ctx: Pick<PluginContext, "asUser" | "log">,
deps: Pick<Deps, "viewers">,
watchedHosts: () => Promise<Map<number, string>>,
) {
const registered = new Map<number, Registered>();
async function release(hostId: number, entry: Registered): Promise<void> {
registered.delete(hostId);
const unregister = deps.viewers().unregister;
if (typeof unregister !== "function") return;
try {
registry.registerViewer(hostId, sessionId, userId);
registered.set(hostId, userId);
added++;
} catch (error) {
statsLogger.warn("Could not start headless metrics for a host", {
operation: "automation_headless_register_error",
hostId,
error: error instanceof Error ? error.message : String(error),
});
}
}
for (const hostId of [...registered.keys()]) {
if (watched.has(hostId)) continue;
try {
registry.unregisterViewer(hostId, automationSessionId(hostId));
await ctx.asUser(entry.userId, () =>
unregister(hostId, entry.viewerSessionId),
);
} catch {
// Already gone; drop it either way.
}
registered.delete(hostId);
removed++;
}
return { added, removed, active: registered.size };
}
/** Drops every synthetic viewer, for shutdown and tests. */
export function releaseHeadlessViewers(): void {
if (registry) {
for (const hostId of registered.keys()) {
try {
registry.unregisterViewer(hostId, automationSessionId(hostId));
} catch {
// Nothing useful to do during teardown.
return {
/** Brings the viewers in line with what the automations watch. */
async reconcile(): Promise<{ added: number; removed: number }> {
const viewers = deps.viewers();
if (typeof viewers.register !== "function") {
// host-metrics is off: its viewers went with it.
registered.clear();
return { added: 0, removed: 0 };
}
}
}
registered.clear();
const watched = await watchedHosts();
let added = 0;
let removed = 0;
for (const [hostId, entry] of [...registered]) {
if (watched.get(hostId) === entry.userId) continue;
await release(hostId, entry);
removed++;
}
for (const [hostId, userId] of watched) {
const existing = registered.get(hostId);
if (existing) {
const alive = await ctx
.asUser(userId, () => viewers.heartbeat!(existing.viewerSessionId))
.catch(() => false);
if (alive) continue;
registered.delete(hostId);
}
try {
const result = await ctx.asUser(userId, () =>
viewers.register!(hostId),
);
if ("viewerSessionId" in result) {
registered.set(hostId, {
userId,
viewerSessionId: result.viewerSessionId,
});
added++;
}
} catch (error) {
ctx.log.warn(
`Could not start headless metrics for host ${hostId}: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
return { added, removed };
},
/** Drops every viewer, on deactivate. */
async releaseAll(): Promise<void> {
for (const [hostId, entry] of [...registered]) {
await release(hostId, entry);
}
},
registeredHosts: () => [...registered.keys()],
};
}
plugins/automations/src/backend/http.ts
-79
View File
@@ -1,79 +0,0 @@
import { safeOutboundFetch } from "../../../../src/backend/utils/safe-outbound-fetch.js";
import { readNotificationPrivateAllowlist } from "../../../../src/backend/utils/notification-egress.js";
/**
* Outbound HTTP for automation steps and notification channels.
*
* safeOutboundFetch refuses private and loopback addresses, which is the right
* default against SSRF but also blocks the self-hosted ntfy or Gotify sitting
* on a LAN that many installs actually use. Private delivery therefore needs
* both a channel opt-in and an exact host in the administrator allowlist;
* scheme validation, DNS pinning and redirect refusal remain in force.
*/
export interface AutomationFetchOptions {
method?: string;
headers?: Record<string, string>;
body?: string;
allowPrivateNetwork?: boolean;
timeoutMs?: number;
}
export async function automationFetch(
url: string,
options: AutomationFetchOptions = {},
): Promise<Response> {
const {
method = "GET",
headers,
body,
allowPrivateNetwork,
timeoutMs = 30_000,
} = options;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, 1));
const init: RequestInit = {
method,
headers: body
? { "Content-Type": "application/json", ...(headers ?? {}) }
: headers,
body,
signal: controller.signal,
};
try {
if (allowPrivateNetwork) {
return await privateNetworkFetch(url, init);
}
return await safeOutboundFetch(url, init);
} finally {
clearTimeout(timer);
}
}
/**
* The opt-in path still goes through the guarded resolver. Only an exact host
* authorized by an administrator may resolve to a private address.
*/
async function privateNetworkFetch(
rawUrl: string,
init: RequestInit,
): Promise<Response> {
let parsed: URL;
try {
parsed = new URL(rawUrl);
} catch {
throw new Error("Invalid URL");
}
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
throw new Error("Only http and https URLs are allowed");
}
if (parsed.username || parsed.password) {
throw new Error("URLs with embedded credentials are not allowed");
}
const allowlist = await readNotificationPrivateAllowlist();
return safeOutboundFetch(rawUrl, init, allowlist);
}
plugins/automations/src/backend/index.ts
+48 -50
View File
@@ -1,63 +1,61 @@
import type { Router } from "express";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import { setPluginSsh } from "./ssh.js";
import { setPluginServices } from "./snippets.js";
import { setTunnelServices } from "./tunnels.js";
import { setDockerServices } from "./docker.js";
import { startAutomationsService, stopAutomationsService } from "./routes.js";
import {
startAutomationScheduler,
stopAutomationScheduler,
} from "./scheduler.js";
import { onInternalEvent } from "./triggers.js";
import { createDeps } from "./deps.js";
import { createDockerWatcher } from "./docker-watcher.js";
import { AutomationEngine } from "./engine.js";
import { createHeadlessViewers } from "./headless-viewer.js";
import { createAutomationRepository } from "./repository.js";
import { registerRoutes } from "./routes.js";
import { createScheduler, STARTUP_DELAY_MS, TICK_MS } from "./scheduler.js";
import { createAutomationsService } from "./service.js";
import { createTriggers } from "./triggers.js";
/** What the tunnels plugin emits when a tunnel drops without being asked to. */
interface TunnelDisconnectedEvent {
userId?: string;
hostId?: number;
tunnelName?: string;
}
export type { AutomationsAccessV1, AutomationSummary } from "./service.js";
export async function activate(ctx: PluginContext) {
setPluginSsh(ctx.ssh);
ctx.disposables.add(() => setPluginSsh(null));
setPluginServices(ctx.services);
ctx.disposables.add(() => setPluginServices(null));
setTunnelServices(ctx.services);
ctx.disposables.add(() => setTunnelServices(null));
setDockerServices(ctx.services);
ctx.disposables.add(() => setDockerServices(null));
const repository = await createAutomationRepository(ctx.db, async () => {
const channels = await ctx.notify.channels();
return channels.map((channel) => channel.id);
});
const deps = createDeps(ctx.services);
const engine = new AutomationEngine(ctx, repository, deps);
const triggers = createTriggers(repository, engine, ctx.log);
triggers.subscribe(ctx.events);
// Nothing arrives here while the tunnels plugin is off, which is the whole
// of the optional dependency.
ctx.events.on("plugin.tunnels.tunnel_disconnected", (payload) => {
const event = payload as TunnelDisconnectedEvent;
if (!event?.userId) return;
void onInternalEvent({
event: "tunnel_disconnected",
userId: event.userId,
hostId: event.hostId,
details: { tunnelName: event.tunnelName },
}).catch((error: unknown) => {
ctx.log.warn(
`Tunnel disconnect trigger failed: ${
error instanceof Error ? error.message : String(error)
}`,
);
});
const viewers = createHeadlessViewers(ctx, deps, () =>
triggers.watchedHosts("metric_threshold"),
);
const docker = createDockerWatcher(
ctx,
deps,
() => triggers.watchedHosts("docker_event"),
triggers.onDockerEvent,
);
ctx.disposables.add(() => {
docker.reset();
void viewers.releaseAll();
});
startAutomationsService(
const scheduler = createScheduler({
repository,
engine,
log: ctx.log,
reconcile: [() => viewers.reconcile(), (now) => docker.reconcile(now)],
});
ctx.schedule.after(STARTUP_DELAY_MS, () => scheduler.tick());
ctx.schedule.every(TICK_MS, () => scheduler.tick());
registerRoutes(
// The webhook route authenticates on its own per-automation token rather
// than a session, which is the point of an inbound webhook.
ctx.http.router({ public: ["/webhook/:token"] }),
ctx.http.router<Router>({ public: ["/webhook/:token"] }),
{ ctx, repository, engine, deps },
);
startAutomationScheduler();
ctx.log.info(
"Automations routes mounted at /plugin-api/automations; scheduler started",
ctx.services.provide(
"automations.access",
createAutomationsService(ctx, repository, engine, deps),
);
}
export async function deactivate() {
stopAutomationScheduler();
stopAutomationsService();
}
export async function deactivate() {}
plugins/automations/src/backend/notify.ts
-193
View File
@@ -1,193 +0,0 @@
import { statsLogger } from "../../../../src/backend/utils/logger.js";
import { automationFetch } from "./http.js";
import type { TemplateContext } from "./template.js";
/**
* Notification delivery for automations.
*
* The alert engine special-cased Discord because its dispatcher only knew
* webhook and ntfy; here every transport goes through one switch, so adding a
* channel type is a single edit.
*/
export interface AutomationChannel {
id: number;
type: string;
config: string;
}
export interface AutomationNotification {
title: string;
body: string;
severity: "info" | "warning" | "critical";
context?: TemplateContext;
}
const NTFY_PRIORITY: Record<string, number> = {
info: 2,
warning: 3,
critical: 5,
};
const NTFY_TAGS: Record<string, string> = {
info: "information_source",
warning: "warning",
critical: "rotating_light",
};
const DISCORD_COLORS: Record<string, number> = {
info: 3066993,
warning: 16753920,
critical: 15158332,
};
export async function sendAutomationNotification(
channel: AutomationChannel,
notification: AutomationNotification,
): Promise<void> {
let config: Record<string, unknown>;
try {
config = JSON.parse(channel.config) as Record<string, unknown>;
} catch {
throw new Error("Channel configuration is not valid JSON");
}
const allowPrivateNetwork = config.allowPrivateNetwork === true;
switch (channel.type) {
case "webhook":
return sendWebhook(config, notification, allowPrivateNetwork);
case "ntfy":
return sendNtfy(config, notification, allowPrivateNetwork);
case "discord":
return sendDiscord(config, notification, allowPrivateNetwork);
default:
throw new Error(`Unsupported channel type: ${channel.type}`);
}
}
function requireUrl(config: Record<string, unknown>): string {
const url = typeof config.url === "string" ? config.url.trim() : "";
if (!url) throw new Error("Channel is missing a URL");
return url;
}
async function sendWebhook(
config: Record<string, unknown>,
notification: AutomationNotification,
allowPrivateNetwork: boolean,
): Promise<void> {
const url = requireUrl(config);
const method = config.method === "PUT" ? "PUT" : "POST";
const headers =
config.headers && typeof config.headers === "object"
? (config.headers as Record<string, string>)
: {};
const response = await automationFetch(url, {
method,
headers,
body: JSON.stringify({
title: notification.title,
hostName:
notification.context?.host?.name ??
notification.context?.trigger?.hostName,
hostId:
notification.context?.host?.id ?? notification.context?.trigger?.hostId,
ruleName: notification.title,
ruleId: notification.context?.run?.automationId,
triggerType: notification.context?.trigger?.type,
value: notification.context?.trigger?.value,
threshold: notification.context?.trigger?.threshold,
message: notification.body,
severity: notification.severity,
timestamp: new Date().toISOString(),
}),
allowPrivateNetwork,
});
if (!response.ok) {
throw new Error(`HTTP ${response.status} ${response.statusText}`);
}
}
async function sendNtfy(
config: Record<string, unknown>,
notification: AutomationNotification,
allowPrivateNetwork: boolean,
): Promise<void> {
const base = requireUrl(config).replace(/\/$/, "");
const topic = typeof config.topic === "string" ? config.topic.trim() : "";
if (!topic) throw new Error("ntfy channel is missing a topic");
const headers: Record<string, string> = {
Title: notification.title || "Termix automation",
Priority: String(NTFY_PRIORITY[notification.severity] ?? 3),
Tags: NTFY_TAGS[notification.severity] ?? "information_source",
};
if (typeof config.token === "string" && config.token) {
headers.Authorization = `Bearer ${config.token}`;
}
const response = await automationFetch(`${base}/${topic}`, {
method: "POST",
headers,
body: notification.body || notification.title,
allowPrivateNetwork,
});
if (!response.ok) {
throw new Error(`HTTP ${response.status} ${response.statusText}`);
}
}
async function sendDiscord(
config: Record<string, unknown>,
notification: AutomationNotification,
allowPrivateNetwork: boolean,
): Promise<void> {
const url = requireUrl(config);
const payload: Record<string, unknown> = {
embeds: [
{
title: notification.title || "Termix automation",
description: notification.body || undefined,
color: DISCORD_COLORS[notification.severity] ?? 3447003,
timestamp: new Date().toISOString(),
},
],
};
if (typeof config.username === "string" && config.username) {
payload.username = config.username;
}
if (typeof config.avatar_url === "string" && config.avatar_url) {
payload.avatar_url = config.avatar_url;
}
const response = await automationFetch(url, {
method: "POST",
body: JSON.stringify(payload),
allowPrivateNetwork,
});
if (!response.ok) {
const detail = await response.text().catch(() => "");
throw new Error(
`HTTP ${response.status} ${response.statusText}${detail ? `: ${detail}` : ""}`,
);
}
}
/** Fire-and-forget wrapper for callers that must not block on delivery. */
export function sendAutomationNotificationSafely(
channel: AutomationChannel,
notification: AutomationNotification,
): void {
sendAutomationNotification(channel, notification).catch((error) => {
statsLogger.warn("Automation notification failed", {
operation: "automation_notification_error",
channelId: channel.id,
type: channel.type,
error: error instanceof Error ? error.message : String(error),
});
});
}
plugins/automations/src/backend/repository.ts
+738
View File
@@ -0,0 +1,738 @@
import { and, asc, desc, eq, inArray, lt, lte, sql } from "drizzle-orm";
import type { PluginDatabase } from "@termix/plugin-sdk/backend";
import {
automations as automationsDef,
channels as channelsDef,
runSteps as runStepsDef,
runs as runsDef,
schedules as schedulesDef,
triggerState as triggerStateDef,
} from "./tables.js";
/* eslint-disable @typescript-eslint/no-explicit-any */
// Tables come from ctx.db.define, which the SDK hands back untyped, and the
// drizzle handle is the server's own. Typed at this module's edge instead.
type Table = any;
type Drizzle = any;
type Row = Record<string, any>;
/* eslint-enable @typescript-eslint/no-explicit-any */
export interface AutomationRow {
id: number;
user_id: string;
name: string;
description: string | null;
enabled: number;
definition: string;
definition_version: number;
concurrency_policy: string;
max_run_seconds: number;
dry_run: number;
last_run_at: string | null;
last_run_status: string | null;
created_at: string;
updated_at: string;
channels: number[];
}
export interface AutomationRunRow {
id: number;
automation_id: number;
user_id: string;
trigger_type: string;
trigger_context: string | null;
status: string;
started_at: string;
finished_at: string | null;
duration_ms: number | null;
error: string | null;
dry_run: number;
parent_run_id: number | null;
automation_name?: string | null;
}
export interface AutomationRunStepRow {
id: number;
run_id: number;
step_index: number;
step_id: string;
step_type: string;
status: string;
started_at: string;
finished_at: string | null;
output: string | null;
error: string | null;
truncated: number;
}
/** The shape the engine loads. */
export interface AutomationEngineRow {
id: number;
userId: string;
name: string;
enabled: boolean;
definition: string;
concurrencyPolicy: string;
maxRunSeconds: number;
dryRun: boolean;
}
export interface TriggerStateRow {
automationId: number;
stateKey: string;
breachStartedAt: string | null;
lastFiredAt: string | null;
lastValue: number | null;
lastObservedState: string | null;
}
export interface DueScheduleRow {
automationId: number;
cron: string | null;
intervalSeconds: number | null;
timezone: string | null;
nextDueAt: string | null;
}
export type AutomationRepository = Awaited<
ReturnType<typeof createAutomationRepository>
>;
/**
* Automation storage. `ownedChannelIds` answers which of core's notification
* channels the user owns, so a link can never point at another user's.
*/
export async function createAutomationRepository(
db: PluginDatabase,
ownedChannelIds: (userId: string) => Promise<number[]>,
) {
const automations: Table = await db.define(automationsDef);
const triggerState: Table = await db.define(triggerStateDef);
const schedules: Table = await db.define(schedulesDef);
const runs: Table = await db.define(runsDef);
const runSteps: Table = await db.define(runStepsDef);
const channels: Table = await db.define(channelsDef);
const client = () => db.client<Drizzle>();
/** Inserts one row and returns its id. MySQL has no RETURNING. */
async function insertId(
table: Table,
values: Record<string, unknown>,
): Promise<number> {
const drizzle = await client();
if (db.dialect === "mysql") {
const result = await drizzle.insert(table).values(values);
const header = Array.isArray(result) ? result[0] : result;
return Number(header?.insertId);
}
const rows = await drizzle
.insert(table)
.values(values)
.returning({ id: table.id });
return rows[0].id as number;
}
async function listChannelIds(automationId: number): Promise<number[]> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select({ channelId: channels.channelId })
.from(channels)
.where(eq(channels.automationId, automationId));
return rows.map((row) => row.channelId as number);
}
async function replaceChannels(
automationId: number,
userId: string,
channelIds: number[],
): Promise<void> {
const drizzle = await client();
await drizzle
.delete(channels)
.where(eq(channels.automationId, automationId));
if (channelIds.length === 0) return;
const owned = new Set(await ownedChannelIds(userId));
const linked = [...new Set(channelIds)].filter((id) => owned.has(id));
if (linked.length > 0) {
await drizzle
.insert(channels)
.values(linked.map((channelId) => ({ automationId, channelId })));
}
}
async function findForUser(
id: number,
userId: string,
): Promise<AutomationRow | null> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(automations)
.where(and(eq(automations.id, id), eq(automations.userId, userId)))
.limit(1);
if (!rows[0]) return null;
return mapAutomationRow(rows[0], await listChannelIds(id));
}
async function getTriggerState(
automationId: number,
stateKey: string,
): Promise<TriggerStateRow | null> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(triggerState)
.where(
and(
eq(triggerState.automationId, automationId),
eq(triggerState.stateKey, stateKey),
),
)
.limit(1);
return rows[0] ? mapTriggerStateRow(rows[0]) : null;
}
return {
async list(userId: string): Promise<AutomationRow[]> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(automations)
.where(eq(automations.userId, userId))
.orderBy(asc(automations.name));
if (rows.length === 0) return [];
// One query for every automation's channels rather than one per row.
const links: Row[] = await drizzle
.select({
automationId: channels.automationId,
channelId: channels.channelId,
})
.from(channels)
.where(
inArray(
channels.automationId,
rows.map((row) => row.id),
),
);
const byAutomation = new Map<number, number[]>();
for (const link of links) {
const list = byAutomation.get(link.automationId) ?? [];
list.push(link.channelId);
byAutomation.set(link.automationId, list);
}
return rows.map((row) => mapAutomationRow(row, byAutomation.get(row.id)));
},
findForUser,
async findById(id: number): Promise<AutomationEngineRow | null> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(automations)
.where(eq(automations.id, id))
.limit(1);
return rows[0] ? mapEngineRow(rows[0]) : null;
},
async create(input: {
userId: string;
name: string;
description?: string | null;
enabled?: boolean;
definition: string;
concurrencyPolicy?: string;
maxRunSeconds?: number;
dryRun?: boolean;
channels?: number[];
}): Promise<AutomationRow> {
const now = new Date().toISOString();
const id = await insertId(automations, {
userId: input.userId,
name: input.name,
description: input.description ?? null,
enabled: input.enabled ?? true,
definition: input.definition,
concurrencyPolicy: input.concurrencyPolicy ?? "skip",
maxRunSeconds: input.maxRunSeconds ?? 300,
dryRun: input.dryRun ?? false,
createdAt: now,
updatedAt: now,
});
await replaceChannels(id, input.userId, input.channels ?? []);
await db.persist();
const created = await findForUser(id, input.userId);
if (!created) throw new Error("Automation could not be read back");
return created;
},
async update(
id: number,
userId: string,
input: {
name?: string;
description?: string | null;
enabled?: boolean;
definition?: string;
concurrencyPolicy?: string;
maxRunSeconds?: number;
dryRun?: boolean;
channels?: number[];
},
): Promise<AutomationRow | null> {
const existing = await findForUser(id, userId);
if (!existing) return null;
const { channels: channelIds, ...fields } = input;
const drizzle = await client();
if (Object.keys(fields).length > 0) {
await drizzle
.update(automations)
.set({ ...fields, updatedAt: new Date().toISOString() })
.where(and(eq(automations.id, id), eq(automations.userId, userId)));
}
if (channelIds) await replaceChannels(id, userId, channelIds);
await db.persist();
return findForUser(id, userId);
},
async delete(id: number, userId: string): Promise<boolean> {
const existing = await findForUser(id, userId);
if (!existing) return false;
const drizzle = await client();
await drizzle
.delete(automations)
.where(and(eq(automations.id, id), eq(automations.userId, userId)));
await db.persist();
return true;
},
/**
* Enabled automations owned by one user. Wildcard targets must never
* reach across users.
*/
async listEnabledForUser(userId: string): Promise<AutomationEngineRow[]> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(automations)
.where(
and(eq(automations.enabled, true), eq(automations.userId, userId)),
);
return rows.map(mapEngineRow);
},
async listAllEnabled(): Promise<AutomationEngineRow[]> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(automations)
.where(eq(automations.enabled, true));
return rows.map(mapEngineRow);
},
getTriggerState,
async upsertTriggerState(input: {
automationId: number;
stateKey: string;
breachStartedAt?: string | null;
lastFiredAt?: string | null;
lastValue?: number | null;
lastObservedState?: string | null;
}): Promise<void> {
const existing = await getTriggerState(
input.automationId,
input.stateKey,
);
const updatedAt = new Date().toISOString();
const drizzle = await client();
if (existing) {
const values: Record<string, unknown> = { updatedAt };
if (input.breachStartedAt !== undefined)
values.breachStartedAt = input.breachStartedAt;
if (input.lastFiredAt !== undefined)
values.lastFiredAt = input.lastFiredAt;
if (input.lastValue !== undefined) values.lastValue = input.lastValue;
if (input.lastObservedState !== undefined)
values.lastObservedState = input.lastObservedState;
await drizzle
.update(triggerState)
.set(values)
.where(
and(
eq(triggerState.automationId, input.automationId),
eq(triggerState.stateKey, input.stateKey),
),
);
} else {
await drizzle.insert(triggerState).values({
automationId: input.automationId,
stateKey: input.stateKey,
breachStartedAt: input.breachStartedAt ?? null,
lastFiredAt: input.lastFiredAt ?? null,
lastValue: input.lastValue ?? null,
lastObservedState: input.lastObservedState ?? null,
updatedAt,
});
}
await db.persist({ lazy: true });
},
async clearBreach(automationId: number, stateKey: string): Promise<void> {
const drizzle = await client();
await drizzle
.update(triggerState)
.set({ breachStartedAt: null, updatedAt: new Date().toISOString() })
.where(
and(
eq(triggerState.automationId, automationId),
eq(triggerState.stateKey, stateKey),
),
);
await db.persist({ lazy: true });
},
/** Dwell windows the scheduler has to re-check without a fresh sample. */
async listOpenBreaches(): Promise<TriggerStateRow[]> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(triggerState)
.where(sql`${triggerState.breachStartedAt} IS NOT NULL`);
return rows.map(mapTriggerStateRow);
},
async upsertSchedule(input: {
automationId: number;
cron: string | null;
intervalSeconds: number | null;
timezone: string | null;
nextDueAt: string | null;
}): Promise<void> {
const drizzle = await client();
const existing: Row[] = await drizzle
.select({ id: schedules.id })
.from(schedules)
.where(eq(schedules.automationId, input.automationId))
.limit(1);
if (existing[0]) {
await drizzle
.update(schedules)
.set({
cron: input.cron,
intervalSeconds: input.intervalSeconds,
timezone: input.timezone,
nextDueAt: input.nextDueAt,
})
.where(eq(schedules.automationId, input.automationId));
} else {
await drizzle.insert(schedules).values(input);
}
await db.persist();
},
async deleteSchedule(automationId: number): Promise<void> {
const drizzle = await client();
await drizzle
.delete(schedules)
.where(eq(schedules.automationId, automationId));
await db.persist();
},
/** Schedules due at or before `now`, joined to their enabled automation. */
async listDueSchedules(now: string): Promise<DueScheduleRow[]> {
const drizzle = await client();
return drizzle
.select({
automationId: schedules.automationId,
cron: schedules.cron,
intervalSeconds: schedules.intervalSeconds,
timezone: schedules.timezone,
nextDueAt: schedules.nextDueAt,
})
.from(schedules)
.innerJoin(automations, eq(automations.id, schedules.automationId))
.where(
and(eq(automations.enabled, true), lte(schedules.nextDueAt, now)),
);
},
async markScheduleTicked(
automationId: number,
nextDueAt: string | null,
lastTickAt: string,
): Promise<void> {
const drizzle = await client();
await drizzle
.update(schedules)
.set({ nextDueAt, lastTickAt })
.where(eq(schedules.automationId, automationId));
await db.persist({ lazy: true });
},
async createRun(input: {
automationId: number;
userId: string;
triggerType: string;
triggerContext?: string | null;
status: string;
dryRun?: boolean;
parentRunId?: number | null;
}): Promise<{ id: number }> {
const id = await insertId(runs, {
automationId: input.automationId,
userId: input.userId,
triggerType: input.triggerType,
triggerContext: input.triggerContext ?? null,
status: input.status,
startedAt: new Date().toISOString(),
dryRun: input.dryRun ?? false,
parentRunId: input.parentRunId ?? null,
});
await db.persist({ lazy: true });
return { id };
},
async finishRun(
runId: number,
input: {
status: string;
error?: string | null;
durationMs?: number | null;
},
): Promise<void> {
const drizzle = await client();
await drizzle
.update(runs)
.set({
status: input.status,
error: input.error ?? null,
finishedAt: new Date().toISOString(),
durationMs: input.durationMs ?? null,
})
.where(eq(runs.id, runId));
const run: Row[] = await drizzle
.select({ automationId: runs.automationId, startedAt: runs.startedAt })
.from(runs)
.where(eq(runs.id, runId))
.limit(1);
if (run[0]) {
await drizzle
.update(automations)
.set({ lastRunAt: run[0].startedAt, lastRunStatus: input.status })
.where(eq(automations.id, run[0].automationId));
}
await db.persist();
},
async listRuns(
userId: string,
options: { automationId?: number; limit?: number; offset?: number } = {},
): Promise<AutomationRunRow[]> {
const limit = Math.min(Math.max(options.limit ?? 50, 1), 200);
const offset = Math.max(options.offset ?? 0, 0);
const where = options.automationId
? and(
eq(runs.userId, userId),
eq(runs.automationId, options.automationId),
)
: eq(runs.userId, userId);
const drizzle = await client();
const rows: Row[] = await drizzle
.select({ run: runs, automationName: automations.name })
.from(runs)
.leftJoin(automations, eq(automations.id, runs.automationId))
.where(where)
.orderBy(desc(runs.startedAt), desc(runs.id))
.limit(limit)
.offset(offset);
return rows.map((row) => ({
...mapRunRow(row.run),
automation_name: row.automationName ?? null,
}));
},
async findRunForUser(
runId: number,
userId: string,
): Promise<AutomationRunRow | null> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(runs)
.where(and(eq(runs.id, runId), eq(runs.userId, userId)))
.limit(1);
return rows[0] ? mapRunRow(rows[0]) : null;
},
async createRunStep(input: {
runId: number;
stepIndex: number;
stepId: string;
stepType: string;
status: string;
}): Promise<number> {
const id = await insertId(runSteps, {
...input,
startedAt: new Date().toISOString(),
});
await db.persist({ lazy: true });
return id;
},
async finishRunStep(
stepRowId: number,
input: {
status: string;
output?: string | null;
error?: string | null;
truncated?: boolean;
},
): Promise<void> {
const drizzle = await client();
await drizzle
.update(runSteps)
.set({
status: input.status,
output: input.output ?? null,
error: input.error ?? null,
truncated: input.truncated ?? false,
finishedAt: new Date().toISOString(),
})
.where(eq(runSteps.id, stepRowId));
await db.persist({ lazy: true });
},
async listRunSteps(runId: number): Promise<AutomationRunStepRow[]> {
const drizzle = await client();
const rows: Row[] = await drizzle
.select()
.from(runSteps)
.where(eq(runSteps.runId, runId))
.orderBy(asc(runSteps.stepIndex));
return rows.map(mapRunStepRow);
},
/** Trims run history. Called from the scheduler's daily sweep. */
async pruneRunsOlderThan(days: number): Promise<number> {
const cutoff = new Date(Date.now() - days * 86400000).toISOString();
const drizzle = await client();
const old: Row[] = await drizzle
.select({ id: runs.id })
.from(runs)
.where(lt(runs.startedAt, cutoff));
if (old.length === 0) return 0;
await drizzle.delete(runs).where(lt(runs.startedAt, cutoff));
await db.persist();
return old.length;
},
/** Marks runs left behind by a crash so they do not block concurrency. */
async failStaleRunningRuns(olderThanIso: string): Promise<number> {
const where = and(
eq(runs.status, "running"),
lt(runs.startedAt, olderThanIso),
);
const drizzle = await client();
const stale: Row[] = await drizzle
.select({ id: runs.id })
.from(runs)
.where(where);
if (stale.length === 0) return 0;
await drizzle
.update(runs)
.set({
status: "failed",
error: "Interrupted by a server restart",
finishedAt: new Date().toISOString(),
})
.where(where);
await db.persist();
return stale.length;
},
};
}
function mapAutomationRow(row: Row, channelIds: number[] = []): AutomationRow {
return {
id: row.id,
user_id: row.userId,
name: row.name,
description: row.description ?? null,
enabled: row.enabled ? 1 : 0,
definition: row.definition,
definition_version: row.definitionVersion,
concurrency_policy: row.concurrencyPolicy,
max_run_seconds: row.maxRunSeconds,
dry_run: row.dryRun ? 1 : 0,
last_run_at: row.lastRunAt ?? null,
last_run_status: row.lastRunStatus ?? null,
created_at: row.createdAt,
updated_at: row.updatedAt,
channels: channelIds,
};
}
function mapEngineRow(row: Row): AutomationEngineRow {
return {
id: row.id,
userId: row.userId,
name: row.name,
enabled: !!row.enabled,
definition: row.definition,
concurrencyPolicy: row.concurrencyPolicy,
maxRunSeconds: row.maxRunSeconds,
dryRun: !!row.dryRun,
};
}
function mapTriggerStateRow(row: Row): TriggerStateRow {
return {
automationId: row.automationId,
stateKey: row.stateKey,
breachStartedAt: row.breachStartedAt ?? null,
lastFiredAt: row.lastFiredAt ?? null,
lastValue: row.lastValue ?? null,
lastObservedState: row.lastObservedState ?? null,
};
}
function mapRunRow(row: Row): AutomationRunRow {
return {
id: row.id,
automation_id: row.automationId,
user_id: row.userId,
trigger_type: row.triggerType,
trigger_context: row.triggerContext ?? null,
status: row.status,
started_at: row.startedAt,
finished_at: row.finishedAt ?? null,
duration_ms: row.durationMs ?? null,
error: row.error ?? null,
dry_run: row.dryRun ? 1 : 0,
parent_run_id: row.parentRunId ?? null,
};
}
function mapRunStepRow(row: Row): AutomationRunStepRow {
return {
id: row.id,
run_id: row.runId,
step_index: row.stepIndex,
step_id: row.stepId,
step_type: row.stepType,
status: row.status,
started_at: row.startedAt,
finished_at: row.finishedAt ?? null,
output: row.output ?? null,
error: row.error ?? null,
truncated: row.truncated ? 1 : 0,
};
}
plugins/automations/src/backend/requirements.ts
+93
View File
@@ -0,0 +1,93 @@
import type { AutomationDefinition, HostSelector, Step } from "../types.js";
import { pluginOf, type Deps, type ProviderKey } from "./deps.js";
/**
* The providers a definition cannot run without. A trigger or step whose
* plugin is off makes the whole automation wait for it, rather than half of
* it running.
*/
export function requiredProviders(
definition: AutomationDefinition | null | undefined,
): ProviderKey[] {
const needed = new Set<ProviderKey>();
if (!definition) return [];
const selector = (value: HostSelector | undefined) => {
if (value?.kind === "fleet") needed.add("fleets");
};
const trigger = definition.trigger;
switch (trigger?.kind) {
case "metric_threshold":
case "health_check":
needed.add("host-metrics");
selector(trigger.hostSelector);
break;
case "docker_event":
needed.add("docker-events");
selector(trigger.hostSelector);
break;
case "host_status":
selector(trigger.hostSelector);
break;
case "internal_event":
if (trigger.event === "tunnel_disconnected") needed.add("tunnels");
selector(trigger.hostSelector);
break;
default:
break;
}
const walk = (steps: Step[] | undefined) => {
for (const step of steps ?? []) {
if (step.enabled === false) continue;
switch (step.type) {
case "run_snippet":
needed.add("snippets");
selector(step.hostSelector);
break;
case "run_command":
selector(step.hostSelector);
break;
case "docker":
needed.add("docker");
selector(step.hostSelector);
break;
case "tunnel":
needed.add("tunnels");
break;
case "wol":
needed.add("wake-on-lan");
break;
case "if":
walk(step.then);
walk(step.else);
break;
default:
break;
}
}
};
walk(definition.steps);
return [...needed];
}
/** Plugin ids the definition needs that are not running, deduplicated. */
export function missingPlugins(
definition: AutomationDefinition | null | undefined,
deps: Pick<Deps, "available">,
): string[] {
const missing = requiredProviders(definition)
.filter((key) => !deps.available(key))
.map(pluginOf);
return [...new Set(missing)];
}
export function parseDefinition(raw: string): AutomationDefinition | null {
try {
return JSON.parse(raw) as AutomationDefinition;
} catch {
return null;
}
}
plugins/automations/src/backend/routes.ts
+545 -755
View File
File diff suppressed because it is too large. Load diff
plugins/automations/src/backend/scheduler.ts
+135 -194
View File
@@ -1,217 +1,158 @@
import type { AutomationDefinition } from "../../../../src/types/automations.js";
import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
import { DataCrypto } from "../../../../src/backend/utils/data-crypto.js";
import { statsLogger } from "../../../../src/backend/utils/logger.js";
import type { PluginLogger } from "@termix/plugin-sdk/backend";
import type { AutomationDefinition } from "../types.js";
import { computeNextDueAt } from "./cron.js";
import { hasDwelled, isCoolingDown } from "./conditions.js";
import { reconcileDockerWatch, resetDockerWatcher } from "./docker-watcher.js";
import { AutomationEngine } from "./engine.js";
import { reconcileHeadlessViewers } from "./headless-viewer.js";
import {
subscribeAutomationTriggers,
unsubscribeAutomationTriggers,
} from "./triggers.js";
import type { AutomationEngine } from "./engine.js";
import type { AutomationRepository } from "./repository.js";
/**
* The one timer the automations feature owns.
*
* Every other periodic job in the backend is its own module-level setInterval;
* this deliberately is not one per automation. A single tick handles due
* schedules, dwell windows that need re-checking without a fresh sample, the
* synthetic viewer heartbeat, and history pruning.
* The one tick the automations plugin runs. It handles due schedules, dwell
* windows that need re-checking without a fresh sample, the watchers that
* keep metrics and Docker events flowing for watched hosts, and history
* pruning. activate() drives it through ctx.schedule.
*/
const TICK_MS = 15_000;
const STARTUP_DELAY_MS = 30_000;
export const TICK_MS = 15_000;
export const STARTUP_DELAY_MS = 30_000;
const PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
const RUN_RETENTION_DAYS = 30;
/** A "running" row older than this belongs to a process that is gone. */
const STALE_RUN_MS = 6 * 60 * 60 * 1000;
let tickTimer: NodeJS.Timeout | null = null;
let startupTimer: NodeJS.Timeout | null = null;
let lastPruneAt = 0;
let ticking = false;
export function startAutomationScheduler(): void {
if (tickTimer) return;
// Event-driven triggers come through the bus; the interval below only drives
// the schedule-based ones.
subscribeAutomationTriggers();
startupTimer = setTimeout(() => {
void tick();
}, STARTUP_DELAY_MS);
startupTimer.unref?.();
tickTimer = setInterval(() => {
void tick();
}, TICK_MS);
tickTimer.unref?.();
export interface SchedulerDeps {
repository: AutomationRepository;
engine: Pick<AutomationEngine, "run">;
log: PluginLogger;
/** Watchers brought in line with the enabled automations every tick. */
reconcile: Array<(now: number) => Promise<unknown>>;
}
export function stopAutomationScheduler(): void {
if (tickTimer) clearInterval(tickTimer);
if (startupTimer) clearTimeout(startupTimer);
tickTimer = null;
startupTimer = null;
unsubscribeAutomationTriggers();
resetDockerWatcher();
}
export function createScheduler(deps: SchedulerDeps) {
const { repository, engine, log } = deps;
let lastPruneAt = 0;
let ticking = false;
/** Exposed for tests; the interval calls this. */
export async function tick(now: Date = new Date()): Promise<void> {
// A slow tick must not overlap the next one.
if (ticking) return;
ticking = true;
const errorText = (error: unknown) =>
error instanceof Error ? error.message : String(error);
try {
await reconcileHeadlessViewers().catch(() => undefined);
await runDueSchedules(now);
await recheckOpenBreaches(now);
await reconcileDockerWatch(now.getTime()).catch(() => undefined);
await pruneIfDue(now);
} catch (error) {
statsLogger.warn("Automation scheduler tick failed", {
operation: "automation_scheduler_tick_error",
error: error instanceof Error ? error.message : String(error),
});
} finally {
ticking = false;
}
}
async function runDueSchedules(now: Date): Promise<void> {
const due = await repository.listDueSchedules(now.toISOString());
async function runDueSchedules(now: Date): Promise<void> {
const repository = createCurrentAutomationRepository();
const due = await repository.listDueSchedules(now.toISOString());
for (const schedule of due) {
const automation = await repository.findById(schedule.automationId);
if (!automation) continue;
// Background work can only touch a user's data while their key resolves.
if (!canAccess(automation.userId)) {
statsLogger.warn("Skipping scheduled automation: data key unavailable", {
operation: "automation_schedule_locked",
automationId: automation.id,
});
continue;
}
const nextDueAt = computeNextDueAt(
{
cron: schedule.cron,
intervalSeconds: schedule.intervalSeconds,
timezone: schedule.timezone,
},
now,
);
await repository.markScheduleTicked(
schedule.automationId,
nextDueAt,
now.toISOString(),
);
AutomationEngine.getInstance()
.run({
automationId: schedule.automationId,
triggerType: "schedule",
triggerContext: { scheduledFor: now.toISOString() },
})
.catch((error) => {
statsLogger.warn("Scheduled automation failed to start", {
operation: "automation_schedule_error",
automationId: schedule.automationId,
error: error instanceof Error ? error.message : String(error),
});
});
}
}
/**
* Fires sustained breaches whose window has elapsed.
*
* Without this a dwell window only completes when another sample happens to
* arrive, so a breach that starts just before polling stops would never fire.
*/
async function recheckOpenBreaches(now: Date): Promise<void> {
const repository = createCurrentAutomationRepository();
const open = await repository.listOpenBreaches();
const nowMs = now.getTime();
for (const state of open) {
const automation = await repository.findById(state.automationId);
if (!automation || !automation.enabled) continue;
if (!canAccess(automation.userId)) continue;
let definition: AutomationDefinition;
try {
definition = JSON.parse(automation.definition) as AutomationDefinition;
} catch {
continue;
}
const trigger = definition.trigger;
if (trigger?.kind !== "metric_threshold") continue;
if (!trigger.forSeconds) continue;
if (!hasDwelled(state.breachStartedAt, trigger.forSeconds, nowMs)) continue;
if (isCoolingDown(state.lastFiredAt, trigger.cooldownMinutes, nowMs)) {
continue;
}
const hostId = Number(state.stateKey.split(":")[0]);
await repository.upsertTriggerState({
automationId: automation.id,
stateKey: state.stateKey,
lastFiredAt: now.toISOString(),
});
AutomationEngine.getInstance()
.run({
automationId: automation.id,
triggerType: "metric_threshold",
triggerContext: {
hostId,
value: state.lastValue,
threshold: trigger.value,
metric: trigger.metric.path,
sustained: true,
for (const schedule of due) {
const nextDueAt = computeNextDueAt(
{
cron: schedule.cron,
intervalSeconds: schedule.intervalSeconds,
timezone: schedule.timezone,
},
triggerHostId: Number.isFinite(hostId) ? hostId : undefined,
})
.catch(() => undefined);
}
}
now,
);
await repository.markScheduleTicked(
schedule.automationId,
nextDueAt,
now.toISOString(),
);
async function pruneIfDue(now: Date): Promise<void> {
if (now.getTime() - lastPruneAt < PRUNE_INTERVAL_MS) return;
lastPruneAt = now.getTime();
const repository = createCurrentAutomationRepository();
try {
await repository.failStaleRunningRuns(
new Date(now.getTime() - STALE_RUN_MS).toISOString(),
);
const deleted = await repository.pruneRunsOlderThan(RUN_RETENTION_DAYS);
if (deleted > 0) {
statsLogger.info(`Pruned ${deleted} old automation run(s)`, {
operation: "automation_run_prune",
deleted,
});
void engine
.run({
automationId: schedule.automationId,
triggerType: "schedule",
triggerContext: { scheduledFor: now.toISOString() },
})
.catch((error: unknown) =>
log.warn(
`Scheduled automation ${schedule.automationId} failed to start: ${errorText(error)}`,
),
);
}
} catch (error) {
statsLogger.warn("Automation run pruning failed", {
operation: "automation_run_prune_error",
error: error instanceof Error ? error.message : String(error),
});
}
}
function canAccess(userId: string): boolean {
try {
return DataCrypto.canUserAccessData(userId);
} catch {
return false;
/**
* Fires sustained breaches whose window has elapsed. Without this a dwell
* window only completes when another sample happens to arrive.
*/
async function recheckOpenBreaches(now: Date): Promise<void> {
const open = await repository.listOpenBreaches();
const nowMs = now.getTime();
for (const state of open) {
const automation = await repository.findById(state.automationId);
if (!automation || !automation.enabled) continue;
let definition: AutomationDefinition;
try {
definition = JSON.parse(automation.definition) as AutomationDefinition;
} catch {
continue;
}
const trigger = definition.trigger;
if (trigger?.kind !== "metric_threshold") continue;
if (!trigger.forSeconds) continue;
if (!hasDwelled(state.breachStartedAt, trigger.forSeconds, nowMs)) {
continue;
}
if (isCoolingDown(state.lastFiredAt, trigger.cooldownMinutes, nowMs)) {
continue;
}
const hostId = Number(state.stateKey.split(":")[0]);
await repository.upsertTriggerState({
automationId: automation.id,
stateKey: state.stateKey,
lastFiredAt: now.toISOString(),
});
void engine
.run({
automationId: automation.id,
triggerType: "metric_threshold",
triggerContext: {
hostId,
value: state.lastValue,
threshold: trigger.value,
metric: trigger.metric.path,
sustained: true,
},
triggerHostId: Number.isFinite(hostId) ? hostId : undefined,
})
.catch(() => undefined);
}
}
async function pruneIfDue(now: Date): Promise<void> {
if (now.getTime() - lastPruneAt < PRUNE_INTERVAL_MS) return;
lastPruneAt = now.getTime();
try {
await repository.failStaleRunningRuns(
new Date(now.getTime() - STALE_RUN_MS).toISOString(),
);
const deleted = await repository.pruneRunsOlderThan(RUN_RETENTION_DAYS);
if (deleted > 0) log.info(`Pruned ${deleted} old automation run(s)`);
} catch (error) {
log.warn(`Automation run pruning failed: ${errorText(error)}`);
}
}
return {
async tick(now: Date = new Date()): Promise<void> {
// A slow tick must not overlap the next one.
if (ticking) return;
ticking = true;
try {
for (const reconcile of deps.reconcile) {
await reconcile(now.getTime()).catch((error: unknown) =>
log.warn(`Automation watcher failed: ${errorText(error)}`),
);
}
await runDueSchedules(now);
await recheckOpenBreaches(now);
await pruneIfDue(now);
} catch (error) {
log.warn(`Automation scheduler tick failed: ${errorText(error)}`);
} finally {
ticking = false;
}
},
};
}
plugins/automations/src/backend/service.ts
+139
View File
@@ -0,0 +1,139 @@
import type { PluginContext } from "@termix/plugin-sdk/backend";
import type { AutomationDefinition, RunStatus } from "../types.js";
import type { Deps } from "./deps.js";
import type { AutomationEngine } from "./engine.js";
import type { AutomationRepository } from "./repository.js";
import { missingPlugins, parseDefinition } from "./requirements.js";
import { syncSchedule } from "./routes.js";
import { validateDefinition } from "./validate.js";
export interface AutomationSummary {
id: number;
name: string;
description: string | null;
enabled: boolean;
triggerKind: string | null;
lastRunAt: string | null;
lastRunStatus: string | null;
missingPlugins: string[];
}
/**
* The "automations.access" service, version 1. Runs as the caller's user,
* gated on automations.view; create and run also need automations.create and
* automations.run.
*/
export interface AutomationsAccessV1 {
list: () => Promise<AutomationSummary[]>;
get: (
id: number,
) => Promise<
(AutomationSummary & { definition: AutomationDefinition | null }) | null
>;
/**
* Validates the definition the same way the create route does. Starts
* disabled unless `enabled` is set, so an automation nobody watched run
* does not start firing on its own.
*/
create: (input: {
name: string;
description?: string | null;
definition: unknown;
enabled?: boolean;
}) => Promise<{ id: number; name: string }>;
run: (
id: number,
options?: { dryRun?: boolean },
) => Promise<{ runId: number | null; status: RunStatus; error?: string }>;
}
export function createAutomationsService(
ctx: PluginContext,
repository: AutomationRepository,
engine: Pick<AutomationEngine, "run">,
deps: Deps,
): AutomationsAccessV1 {
const actor = (method: string) => {
const userId = ctx.currentActor();
if (!userId) throw new Error(`automations.access.${method} needs an actor`);
return userId;
};
const summarize = (
row: Awaited<ReturnType<AutomationRepository["list"]>>[number],
) => {
const definition = parseDefinition(row.definition);
if (definition?.trigger?.kind === "webhook") {
definition.trigger = { ...definition.trigger, tokenHash: "" };
}
return {
summary: {
id: row.id,
name: row.name,
description: row.description,
enabled: !!row.enabled,
triggerKind: definition?.trigger?.kind ?? null,
lastRunAt: row.last_run_at,
lastRunStatus: row.last_run_status,
missingPlugins: missingPlugins(definition, deps),
},
definition,
};
};
return {
list: async () => {
const rows = await repository.list(actor("list"));
return rows.map((row) => summarize(row).summary);
},
get: async (id) => {
const row = await repository.findForUser(id, actor("get"));
if (!row) return null;
const { summary, definition } = summarize(row);
return { ...summary, definition };
},
create: async (input) => {
const userId = actor("create");
if (!(await ctx.rbac.has("create"))) {
throw new Error('Creating automations needs "automations.create"');
}
const name = typeof input.name === "string" ? input.name.trim() : "";
if (!name) throw new Error("Name is required");
const validated = validateDefinition(input.definition);
if (!validated.ok || !validated.definition) {
throw new Error(
validated.error ?? "The automation definition is invalid",
);
}
if (validated.definition.trigger.kind === "webhook") {
// A webhook's token is only ever shown by the create route.
throw new Error("Webhook automations are created from the editor");
}
const created = await repository.create({
userId,
name,
description: input.description ?? null,
definition: JSON.stringify(validated.definition),
enabled: input.enabled ?? false,
});
await syncSchedule(repository, created.id, validated.definition);
return { id: created.id, name: created.name };
},
run: async (id, options = {}) => {
const userId = actor("run");
if (!(await ctx.rbac.has("run"))) {
throw new Error('Running automations needs "automations.run"');
}
const existing = await repository.findForUser(id, userId);
if (!existing) throw new Error("Automation not found");
return engine.run({
automationId: id,
triggerType: "manual",
triggerContext: { manual: true, requestedBy: userId },
dryRun: options.dryRun === true,
});
},
};
}
plugins/automations/src/backend/snippets.ts
-91
View File
@@ -1,91 +0,0 @@
import type { PluginServices } from "@termix/plugin-sdk/backend";
let current: PluginServices | null = null;
/** Set in activate, cleared on deactivate. */
export function setPluginServices(services: PluginServices | null): void {
current = services;
}
/** Another plugin's snippets, as ctx.services.get("snippets.access") returns them. */
interface SnippetsAccess {
get: (id: number) => Promise<{
id: number;
name: string;
content: string;
isNote: boolean;
} | null>;
resolveCommand: (
id: number,
vars: { ip?: string; username?: string; port?: number; name?: string },
inputValues?: Record<string, string>,
) => Promise<string | null>;
}
/**
* A snippet's command, resolved against a target host and run as userId.
* Null when the snippets plugin is off, the user may not use it, or the
* snippet does not exist. Optional: the run_snippet step fails with a clear
* message without it, rather than the plugin failing to activate.
*/
export async function resolveSnippetCommandFor(
userId: string,
id: number,
vars: { ip?: string; username?: string; port?: number; name?: string },
inputValues?: Record<string, string>,
): Promise<string | null> {
if (!current) return null;
try {
return await current
.get<SnippetsAccess>("snippets.access", { userId })
.resolveCommand(id, vars, inputValues);
} catch {
return null;
}
}
export async function getSnippet(
userId: string,
id: number,
): Promise<{
id: number;
name: string;
content: string;
isNote: boolean;
} | null> {
if (!current) return null;
try {
return await current
.get<SnippetsAccess>("snippets.access", { userId })
.get(id);
} catch {
return null;
}
}
/** Another plugin's fleets, as ctx.services.get("fleets.access") returns them. */
interface FleetsAccess {
members: (
fleetId: number,
) => Promise<Array<{ id: number; name: string | null; ip: string }>>;
}
/**
* A fleet's effective member host ids, for a "fleet" target selector. Empty
* when the fleets plugin is off, the user may not use it, or the fleet does
* not exist - the step then runs against no hosts rather than failing.
*/
export async function fleetMemberIds(
userId: string,
fleetId: number,
): Promise<number[]> {
if (!current) return [];
try {
const members = await current
.get<FleetsAccess>("fleets.access", { userId })
.members(fleetId);
return members.map((member) => member.id);
} catch {
return [];
}
}
plugins/automations/src/backend/ssh.ts
-36
View File
@@ -1,36 +0,0 @@
import type { Client } from "ssh2";
import type {
PluginSsh,
PluginSshConnection,
PluginSshConnectOptions,
PluginSshHost,
} from "@termix/plugin-sdk/backend";
let current: PluginSsh | null = null;
/** Set in activate, cleared on deactivate. */
export function setPluginSsh(ssh: PluginSsh | null): void {
current = ssh;
}
export function pluginSsh(): PluginSsh {
if (!current) throw new Error("The plugin is not active");
return current;
}
/** withConnection with ssh2's Client type filled in. */
export function withSshConnection<T>(
host: number | PluginSshHost,
options: PluginSshConnectOptions & { pool: string },
fn: (client: Client) => Promise<T>,
): Promise<T> {
return pluginSsh().withConnection<T, Client>(host, options, fn);
}
/** connect with ssh2's Client type filled in. */
export function connectSsh(
host: number | PluginSshHost,
options?: PluginSshConnectOptions,
): Promise<PluginSshConnection<Client>> {
return pluginSsh().connect<Client>(host, options);
}
plugins/automations/src/backend/tables.ts
+197
View File
@@ -0,0 +1,197 @@
import {
adoptLegacyTable,
boolean,
defineTable,
id,
integer,
real,
refUser,
text,
timestamp,
varchar,
} from "@termix/plugin-sdk/db";
/*
* Adopted from core's automation tables, so column and index names are the
* legacy ones and the rename carries every row across.
*
* The links between these tables, and a channel link to core's
* notification_channels, are plain integers here: the SDK only references
* users and ssh_data. The adoption migrations keep the legacy FOREIGN KEY
* clauses by hand, so deleting an automation still cascades.
*/
export const automations = adoptLegacyTable(
"automations",
defineTable(
"automations",
{
id: id(),
userId: refUser(),
name: text().notNull(),
description: text(),
enabled: boolean().notNull().default(true),
// The whole trigger and steps graph, read and written as a unit.
definition: text().notNull(),
definitionVersion: integer().notNull().default(1),
concurrencyPolicy: text().notNull().default("skip"),
maxRunSeconds: integer().notNull().default(300),
dryRun: boolean().notNull().default(false),
lastRunAt: text(),
lastRunStatus: text(),
createdAt: timestamp().notNull().defaultNow(),
updatedAt: timestamp().notNull().defaultNow(),
},
{
indexes: [
{ name: "idx_automations_user", columns: ["userId", "enabled"] },
],
},
),
);
/**
* Durable per-target trigger state, so cooldowns and dwell windows survive a
* restart. state_key scopes it to what the trigger watches ("<hostId>",
* "<hostId>:/data", "<hostId>:<container>").
*/
export const triggerState = adoptLegacyTable(
"automation_trigger_state",
defineTable(
"trigger_state",
{
id: id(),
automationId: integer().notNull(),
stateKey: varchar().notNull(),
breachStartedAt: text(),
lastFiredAt: text(),
lastValue: real(),
lastObservedState: text(),
updatedAt: timestamp().notNull().defaultNow(),
},
{
uniques: [
{
name: "idx_automation_trigger_state_key",
columns: ["automationId", "stateKey"],
},
],
},
),
);
export const schedules = adoptLegacyTable(
"automation_schedules",
defineTable(
"schedules",
{
id: id(),
automationId: integer().notNull(),
cron: text(),
intervalSeconds: integer(),
timezone: text(),
nextDueAt: timestamp(),
lastTickAt: text(),
},
{
uniques: [
{
name: "idx_automation_schedules_automation",
columns: ["automationId"],
},
],
indexes: [
{ name: "idx_automation_schedules_due", columns: ["nextDueAt"] },
],
},
),
);
export const runs = adoptLegacyTable(
"automation_runs",
defineTable(
"runs",
{
id: id(),
automationId: integer().notNull(),
userId: refUser(),
triggerType: text().notNull(),
triggerContext: text(),
status: text().notNull(),
startedAt: timestamp().notNull().defaultNow(),
finishedAt: text(),
durationMs: integer(),
error: text(),
dryRun: boolean().notNull().default(false),
// Set when one automation invoked another, so a chain can be traced.
parentRunId: integer(),
},
{
indexes: [
{
name: "idx_automation_runs_automation",
columns: ["automationId", "startedAt"],
},
{ name: "idx_automation_runs_user", columns: ["userId", "startedAt"] },
],
},
),
);
export const runSteps = adoptLegacyTable(
"automation_run_steps",
defineTable(
"run_steps",
{
id: id(),
runId: integer().notNull(),
stepIndex: integer().notNull(),
stepId: text().notNull(),
stepType: text().notNull(),
status: text().notNull(),
startedAt: timestamp().notNull().defaultNow(),
finishedAt: text(),
output: text(),
error: text(),
truncated: boolean().notNull().default(false),
},
{
indexes: [
{
name: "idx_automation_run_steps_run",
columns: ["runId", "stepIndex"],
},
],
},
),
);
/** Which of core's notification channels an automation is linked to. */
export const channels = adoptLegacyTable(
"automation_channels",
defineTable(
"channels",
{
id: id(),
automationId: integer().notNull(),
channelId: integer().notNull(),
},
{
uniques: [
{
name: "idx_automation_channels_pair",
columns: ["automationId", "channelId"],
},
],
},
),
);
export const tables = [
automations,
triggerState,
schedules,
runs,
runSteps,
channels,
];
plugins/automations/src/backend/triggers.ts
+383 -362
View File
@@ -1,11 +1,9 @@
import type { PluginEvents, PluginLogger } from "@termix/plugin-sdk/backend";
import type { AutomationDefinition, HostSelector, Trigger } from "../types.js";
import type {
AutomationDefinition,
HostSelector,
Trigger,
} from "../../../../src/types/automations.js";
import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
import type { AutomationEngineRow } from "../../../../src/backend/database/repositories/automation-repository.js";
import { statsLogger } from "../../../../src/backend/utils/logger.js";
AutomationEngineRow,
AutomationRepository,
} from "./repository.js";
import {
compare,
extractMetricValue,
@@ -15,11 +13,7 @@ import {
severityForValue,
type MetricsSnapshot,
} from "./conditions.js";
import { AutomationEngine } from "./engine.js";
import {
pluginEvents,
TOPICS,
} from "../../../../src/backend/plugins/events.js";
import { TOPIC_AUTOMATION_FAILED, type AutomationEngine } from "./engine.js";
/**
* Matches events against automation triggers and decides what fires.
@@ -68,377 +62,404 @@ interface LoadedAutomation {
definition: AutomationDefinition;
}
async function loadEnabledFor(userId: string): Promise<LoadedAutomation[]> {
try {
const rows =
await createCurrentAutomationRepository().listEnabledForUser(userId);
const loaded: LoadedAutomation[] = [];
for (const row of rows) {
try {
loaded.push({
row,
definition: JSON.parse(row.definition) as AutomationDefinition,
});
} catch {
// A malformed definition should not stop the others from evaluating.
export type Triggers = ReturnType<typeof createTriggers>;
/** Built once per activation. */
export function createTriggers(
repository: AutomationRepository,
engine: Pick<AutomationEngine, "run">,
log: PluginLogger,
) {
async function loadEnabledFor(userId: string): Promise<LoadedAutomation[]> {
try {
const rows = await repository.listEnabledForUser(userId);
const loaded: LoadedAutomation[] = [];
for (const row of rows) {
try {
loaded.push({
row,
definition: JSON.parse(row.definition) as AutomationDefinition,
});
} catch {
// A malformed definition should not stop the others from evaluating.
}
}
return loaded;
} catch {
return [];
}
return loaded;
} catch {
return [];
}
}
/** Whether a selector covers a host. Ownership is checked by the caller. */
function selectorCoversHost(
selector: HostSelector | undefined,
hostId: number,
): boolean {
if (!selector) return true;
switch (selector.kind) {
case "all":
case "trigger":
return true;
case "host":
return selector.hostId === hostId;
case "hosts":
return selector.hostIds.includes(hostId);
case "fleet":
// Fleet membership is resolved at execution time; evaluate optimistically
// so a fleet-scoped trigger still reaches the engine.
return true;
default:
return false;
/** Whether a selector covers a host. Ownership is checked by the caller. */
function selectorCoversHost(
selector: HostSelector | undefined,
hostId: number,
): boolean {
if (!selector) return true;
switch (selector.kind) {
case "all":
case "trigger":
return true;
case "host":
return selector.hostId === hostId;
case "hosts":
return selector.hostIds.includes(hostId);
case "fleet":
// Fleet membership is resolved at execution time; evaluate optimistically
// so a fleet-scoped trigger still reaches the engine.
return true;
default:
return false;
}
}
}
async function fire(
automation: AutomationEngineRow,
stateKey: string,
triggerType: string,
triggerContext: Record<string, unknown>,
hostId?: number,
): Promise<void> {
const repository = createCurrentAutomationRepository();
await repository.upsertTriggerState({
automationId: automation.id,
stateKey,
lastFiredAt: new Date().toISOString(),
});
AutomationEngine.getInstance()
.run({
async function fire(
automation: AutomationEngineRow,
stateKey: string,
triggerType: string,
triggerContext: Record<string, unknown>,
hostId?: number,
): Promise<void> {
await repository.upsertTriggerState({
automationId: automation.id,
triggerType,
triggerContext,
triggerHostId: hostId,
})
.catch((error) => {
statsLogger.warn("Automation run failed to start", {
operation: "automation_trigger_error",
stateKey,
lastFiredAt: new Date().toISOString(),
});
engine
.run({
automationId: automation.id,
error: error instanceof Error ? error.message : String(error),
triggerType,
triggerContext,
triggerHostId: hostId,
})
.catch((error: unknown) => {
log.warn(
`Automation ${automation.id} failed to start: ${
error instanceof Error ? error.message : String(error)
}`,
);
});
});
}
/**
* Metric samples. Called for every polled host, including hosts polled only
* because an automation asked for them.
*/
export async function onMetrics(event: MetricEvent): Promise<void> {
const automations = await loadEnabledFor(event.ownerUserId);
const repository = createCurrentAutomationRepository();
const now = Date.now();
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "metric_threshold") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
const value = extractMetricValue(event.metrics, trigger.metric);
if (value === null) continue;
const stateKey = metricStateKey(event.hostId, trigger.metric);
const state = await repository.getTriggerState(row.id, stateKey);
const breaching = compare(value, trigger.operator, trigger.value);
if (!breaching) {
if (state?.breachStartedAt) {
await repository.clearBreach(row.id, stateKey);
}
continue;
}
// Open the dwell window on the first breaching sample.
if (!state?.breachStartedAt) {
await repository.upsertTriggerState({
automationId: row.id,
stateKey,
breachStartedAt: new Date(now).toISOString(),
lastValue: value,
});
if (trigger.forSeconds) continue;
}
const breachStartedAt =
state?.breachStartedAt ?? new Date(now).toISOString();
if (!hasDwelled(breachStartedAt, trigger.forSeconds, now)) continue;
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"metric_threshold",
{
hostId: event.hostId,
value,
threshold: trigger.value,
operator: trigger.operator,
metric: trigger.metric.path,
mount: "mount" in trigger.metric ? trigger.metric.mount : undefined,
severity: severityForValue(value, trigger.severity),
},
event.hostId,
);
}
}
/** Host reachability transitions. Only edges fire, never steady state. */
export async function onStatus(event: StatusEvent): Promise<void> {
const automations = await loadEnabledFor(event.ownerUserId);
const repository = createCurrentAutomationRepository();
const now = Date.now();
const observed = event.online ? "online" : "offline";
/**
* Metric samples. Called for every polled host, including hosts polled only
* because an automation asked for them.
*/
async function onMetrics(event: MetricEvent): Promise<void> {
const automations = await loadEnabledFor(event.ownerUserId);
const now = Date.now();
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "host_status") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "metric_threshold") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
const stateKey = String(event.hostId);
const state = await repository.getTriggerState(row.id, stateKey);
const value = extractMetricValue(event.metrics, trigger.metric);
if (value === null) continue;
if (state?.lastObservedState === observed) continue;
const stateKey = metricStateKey(event.hostId, trigger.metric);
const state = await repository.getTriggerState(row.id, stateKey);
const breaching = compare(value, trigger.operator, trigger.value);
await repository.upsertTriggerState({
automationId: row.id,
stateKey,
lastObservedState: observed,
});
// The first observation establishes a baseline rather than firing, so a
// restart does not announce every host as though it just changed.
if (!state?.lastObservedState) continue;
if (trigger.to !== observed) continue;
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"host_status",
{ hostId: event.hostId, status: observed },
event.hostId,
);
}
}
export async function onHealthCheck(event: HealthEvent): Promise<void> {
const automations = await loadEnabledFor(event.userId);
const repository = createCurrentAutomationRepository();
const now = Date.now();
const observed = event.ok ? "recovered" : "failing";
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "health_check") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
if (trigger.checkId && trigger.checkId !== event.checkId) continue;
const stateKey = `${event.hostId}:${event.checkId}`;
const state = await repository.getTriggerState(row.id, stateKey);
if (state?.lastObservedState === observed) continue;
await repository.upsertTriggerState({
automationId: row.id,
stateKey,
lastObservedState: observed,
});
if (!state?.lastObservedState) continue;
if (trigger.to !== observed) continue;
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"health_check",
{
hostId: event.hostId,
checkId: event.checkId,
state: observed,
detail: event.detail,
},
event.hostId,
);
}
}
export async function onDockerEvent(event: DockerEvent): Promise<void> {
const automations = await loadEnabledFor(event.ownerUserId);
const repository = createCurrentAutomationRepository();
const now = Date.now();
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "docker_event") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
if (trigger.container && trigger.container !== event.container) continue;
if (trigger.event !== event.event) continue;
const stateKey = `${event.hostId}:${event.container}`;
const state = await repository.getTriggerState(row.id, stateKey);
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"docker_event",
{
hostId: event.hostId,
container: event.container,
event: event.event,
},
event.hostId,
);
}
}
export async function onInternalEvent(event: InternalEvent): Promise<void> {
const automations = await loadEnabledFor(event.userId);
const repository = createCurrentAutomationRepository();
const now = Date.now();
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "internal_event") continue;
if (trigger.event !== event.event) continue;
if (
event.hostId !== undefined &&
!selectorCoversHost(trigger.hostSelector, event.hostId)
) {
continue;
}
const stateKey = event.hostId ? String(event.hostId) : "global";
const state = await repository.getTriggerState(row.id, stateKey);
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"internal_event",
{ event: event.event, hostId: event.hostId, ...(event.details ?? {}) },
event.hostId,
);
}
}
let unsubscribers: Array<() => void> = [];
/**
* Subscribes the engine to the event bus.
*
* The hosts modules publish onto the bus and know nothing about automations;
* this is the other half of that arrangement, and it is what keeps the edge
* one-directional (automations may import repositories, hosts modules must not
* import automations).
*
* Handlers are async and the bus is fire-and-forget, so a rejection here is
* logged by the bus rather than surfacing to whichever host feature emitted.
*/
export function subscribeAutomationTriggers(): void {
if (unsubscribers.length > 0) return;
unsubscribers = [
pluginEvents.on("plugin.host-metrics.snapshot", (payload) =>
onMetrics(payload as MetricEvent),
),
pluginEvents.on(TOPICS.hostStatus, (payload) =>
onStatus(payload as StatusEvent),
),
pluginEvents.on("plugin.host-metrics.health-check", (payload) =>
onHealthCheck(payload as HealthEvent),
),
pluginEvents.on(TOPICS.internalEvent, (payload) =>
onInternalEvent(payload as InternalEvent),
),
// An SSH login seen by the terminal, which host-metrics used to relay.
pluginEvents.on(TOPICS.hostLogin, (payload) => {
const login = payload as {
hostId: number;
userId: string;
sshUser?: string;
fromIp?: string;
};
if (!login?.userId) return;
return onInternalEvent({
event: "user_login",
userId: login.userId,
hostId: login.hostId,
details: { sshUser: login.sshUser, fromIp: login.fromIp },
});
}),
];
}
export function unsubscribeAutomationTriggers(): void {
for (const unsubscribe of unsubscribers) unsubscribe();
unsubscribers = [];
}
/**
* Hosts that an enabled automation watches, so the poller knows to collect
* metrics for them even when nobody is looking.
*/
export async function listAutomationWatchedHosts(): Promise<
Map<number, string>
> {
const watched = new Map<number, string>();
try {
const rows = await createCurrentAutomationRepository().listAllEnabled();
for (const row of rows) {
let definition: AutomationDefinition;
try {
definition = JSON.parse(row.definition) as AutomationDefinition;
} catch {
if (!breaching) {
if (state?.breachStartedAt) {
await repository.clearBreach(row.id, stateKey);
}
continue;
}
const trigger: Trigger | undefined = definition.trigger;
// Only metric thresholds need heavy collection; status triggers are
// already served by the cheap reachability probe.
if (trigger?.kind !== "metric_threshold") continue;
const selector = trigger.hostSelector;
if (selector?.kind === "host") {
watched.set(selector.hostId, row.userId);
} else if (selector?.kind === "hosts") {
for (const hostId of selector.hostIds) watched.set(hostId, row.userId);
// Open the dwell window on the first breaching sample.
if (!state?.breachStartedAt) {
await repository.upsertTriggerState({
automationId: row.id,
stateKey,
breachStartedAt: new Date(now).toISOString(),
lastValue: value,
});
if (trigger.forSeconds) continue;
}
// Fleet and "all" selectors are resolved by the scheduler, which can
// expand them without blocking this call.
const breachStartedAt =
state?.breachStartedAt ?? new Date(now).toISOString();
if (!hasDwelled(breachStartedAt, trigger.forSeconds, now)) continue;
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"metric_threshold",
{
hostId: event.hostId,
value,
threshold: trigger.value,
operator: trigger.operator,
metric: trigger.metric.path,
mount: "mount" in trigger.metric ? trigger.metric.mount : undefined,
severity: severityForValue(value, trigger.severity),
},
event.hostId,
);
}
}
/** Host reachability transitions. Only edges fire, never steady state. */
async function onStatus(event: StatusEvent): Promise<void> {
const automations = await loadEnabledFor(event.ownerUserId);
const now = Date.now();
const observed = event.online ? "online" : "offline";
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "host_status") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
const stateKey = String(event.hostId);
const state = await repository.getTriggerState(row.id, stateKey);
if (state?.lastObservedState === observed) continue;
await repository.upsertTriggerState({
automationId: row.id,
stateKey,
lastObservedState: observed,
});
// The first observation establishes a baseline rather than firing, so a
// restart does not announce every host as though it just changed.
if (!state?.lastObservedState) continue;
if (trigger.to !== observed) continue;
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"host_status",
{ hostId: event.hostId, status: observed },
event.hostId,
);
}
}
async function onHealthCheck(event: HealthEvent): Promise<void> {
const automations = await loadEnabledFor(event.userId);
const now = Date.now();
const observed = event.ok ? "recovered" : "failing";
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "health_check") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
if (trigger.checkId && trigger.checkId !== event.checkId) continue;
const stateKey = `${event.hostId}:${event.checkId}`;
const state = await repository.getTriggerState(row.id, stateKey);
if (state?.lastObservedState === observed) continue;
await repository.upsertTriggerState({
automationId: row.id,
stateKey,
lastObservedState: observed,
});
if (!state?.lastObservedState) continue;
if (trigger.to !== observed) continue;
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"health_check",
{
hostId: event.hostId,
checkId: event.checkId,
state: observed,
detail: event.detail,
},
event.hostId,
);
}
}
async function onDockerEvent(event: DockerEvent): Promise<void> {
const automations = await loadEnabledFor(event.ownerUserId);
const now = Date.now();
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "docker_event") continue;
if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
if (trigger.container && trigger.container !== event.container) continue;
if (trigger.event !== event.event) continue;
const stateKey = `${event.hostId}:${event.container}`;
const state = await repository.getTriggerState(row.id, stateKey);
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"docker_event",
{
hostId: event.hostId,
container: event.container,
event: event.event,
},
event.hostId,
);
}
}
async function onInternalEvent(event: InternalEvent): Promise<void> {
const automations = await loadEnabledFor(event.userId);
const now = Date.now();
for (const { row, definition } of automations) {
const trigger = definition.trigger;
if (trigger?.kind !== "internal_event") continue;
if (trigger.event !== event.event) continue;
if (
event.hostId !== undefined &&
!selectorCoversHost(trigger.hostSelector, event.hostId)
) {
continue;
}
const stateKey = event.hostId ? String(event.hostId) : "global";
const state = await repository.getTriggerState(row.id, stateKey);
if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
continue;
await fire(
row,
stateKey,
"internal_event",
{ event: event.event, hostId: event.hostId, ...(event.details ?? {}) },
event.hostId,
);
}
}
/**
* Hosts an enabled automation watches with the given trigger kind, with
* the owning user. Fleet and "all" selectors are not expanded: collecting
* for every host a user owns is far too costly.
*/
async function watchedHosts(
kind: "metric_threshold" | "docker_event",
): Promise<Map<number, string>> {
const watched = new Map<number, string>();
try {
for (const row of await repository.listAllEnabled()) {
let definition: AutomationDefinition;
try {
definition = JSON.parse(row.definition) as AutomationDefinition;
} catch {
continue;
}
const trigger: Trigger | undefined = definition.trigger;
if (trigger?.kind !== kind) continue;
const selector = trigger.hostSelector;
if (selector?.kind === "host") {
watched.set(selector.hostId, row.userId);
} else if (selector?.kind === "hosts") {
for (const hostId of selector.hostIds)
watched.set(hostId, row.userId);
}
}
} catch {
return watched;
}
} catch {
return watched;
}
return watched;
/**
* Subscribes to the topics the triggers read. Every listener is disposed
* with the plugin. Handlers are async and the bus is fire-and-forget, so a
* rejection is logged here rather than reaching whoever emitted.
*/
function subscribe(events: PluginEvents): void {
const on = <T>(
topic: string,
handler: (payload: T) => Promise<void> | undefined,
) =>
events.on(topic, (payload) => {
void Promise.resolve(handler(payload as T)).catch((error: unknown) =>
log.warn(
`Automation trigger for ${topic} failed: ${
error instanceof Error ? error.message : String(error)
}`,
),
);
});
on<MetricEvent>("plugin.host-metrics.snapshot", onMetrics);
on<HealthEvent>("plugin.host-metrics.health-check", onHealthCheck);
on<StatusEvent>("host.status", onStatus);
on<InternalEvent>("internal.event", (event) =>
event?.userId ? onInternalEvent(event) : undefined,
);
// An SSH login seen by the terminal.
on<{ hostId: number; userId: string; sshUser?: string; fromIp?: string }>(
"host.login",
(login) =>
login?.userId
? onInternalEvent({
event: "user_login",
userId: login.userId,
hostId: login.hostId,
details: { sshUser: login.sshUser, fromIp: login.fromIp },
})
: undefined,
);
// Nothing arrives here while the tunnels plugin is off, which is the
// whole of that optional dependency.
on<{ userId?: string; hostId?: number; tunnelName?: string }>(
"plugin.tunnels.tunnel_disconnected",
(event) =>
event?.userId
? onInternalEvent({
event: "tunnel_disconnected",
userId: event.userId,
hostId: event.hostId,
details: { tunnelName: event.tunnelName },
})
: undefined,
);
on<{
userId?: string;
automationId?: number;
automationName?: string;
runId?: number;
error?: string | null;
}>(TOPIC_AUTOMATION_FAILED, (event) =>
event?.userId
? onInternalEvent({
event: "automation_failed",
userId: event.userId,
details: {
automationId: event.automationId,
automationName: event.automationName,
runId: event.runId,
error: event.error ?? null,
},
})
: undefined,
);
}
return {
onMetrics,
onStatus,
onHealthCheck,
onDockerEvent,
onInternalEvent,
watchedHosts,
subscribe,
};
}
plugins/automations/src/backend/tunnels.ts
-63
View File
@@ -1,63 +0,0 @@
import type { PluginServices } from "@termix/plugin-sdk/backend";
let current: PluginServices | null = null;
/** Set in activate, cleared on deactivate. */
export function setTunnelServices(services: PluginServices | null): void {
current = services;
}
/** The tunnels plugin, as ctx.services.get("tunnels.access") returns it. */
interface TunnelsAccess {
start: (name: string) => Promise<void>;
stop: (name: string) => Promise<void>;
}
export interface TunnelActionResult {
ok: boolean;
/** True when the tunnels plugin is off or the user may not use it. */
unavailable?: boolean;
error?: string;
}
/**
* Connects or disconnects a saved tunnel as userId. Optional: without the
* tunnels plugin the step fails with a clear message rather than the plugin
* failing to activate.
*/
export async function runTunnelAction(
userId: string,
action: "connect" | "disconnect",
name: string,
): Promise<TunnelActionResult> {
if (!current) {
return {
ok: false,
unavailable: true,
error: "The tunnels plugin is not available",
};
}
let tunnels: TunnelsAccess;
try {
tunnels = current.get<TunnelsAccess>("tunnels.access", { userId });
} catch {
return {
ok: false,
unavailable: true,
error: "The tunnels plugin is not available",
};
}
try {
if (action === "connect") await tunnels.start(name);
else await tunnels.stop(name);
return { ok: true };
} catch (error) {
return {
ok: false,
unavailable: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
plugins/automations/src/backend/validate.ts
+153
View File
@@ -0,0 +1,153 @@
import type { AutomationDefinition, Step, Trigger } from "../types.js";
import { AUTOMATION_DEFINITION_VERSION } from "../types.js";
import { isValidCron, isValidTimezone } from "./cron.js";
const TRIGGER_KINDS = new Set([
"metric_threshold",
"host_status",
"health_check",
"schedule",
"docker_event",
"internal_event",
"webhook",
]);
const STEP_TYPES = new Set([
"notify",
"http",
"run_snippet",
"run_command",
"docker",
"tunnel",
"wol",
"wait",
"set_var",
"if",
"run_automation",
"stop",
]);
const OPERATORS = new Set([
">",
"<",
">=",
"<=",
"==",
"!=",
"contains",
"not_contains",
"changed",
]);
export function isNonEmptyString(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
/**
* Validates a definition before it is stored. The engine treats the stored
* blob as trusted, so everything it relies on is checked once here.
*/
export function validateDefinition(value: unknown): {
ok: boolean;
error?: string;
definition?: AutomationDefinition;
} {
if (typeof value !== "object" || value === null) {
return { ok: false, error: "Definition must be an object" };
}
const candidate = value as Partial<AutomationDefinition>;
const trigger = candidate.trigger as Trigger | undefined;
if (!trigger || !TRIGGER_KINDS.has(trigger.kind)) {
return { ok: false, error: "Unknown or missing trigger kind" };
}
if (trigger.kind === "metric_threshold") {
if (!OPERATORS.has(trigger.operator)) {
return { ok: false, error: "Unknown comparison operator" };
}
if (typeof trigger.value !== "number" || !Number.isFinite(trigger.value)) {
return { ok: false, error: "Threshold value must be a number" };
}
if (!trigger.metric?.path) {
return { ok: false, error: "Trigger is missing a metric" };
}
}
if (trigger.kind === "schedule") {
const hasInterval =
typeof trigger.intervalSeconds === "number" &&
trigger.intervalSeconds > 0;
const hasCron = isNonEmptyString(trigger.cron);
if (!hasInterval && !hasCron) {
return { ok: false, error: "Schedule needs an interval or a cron" };
}
if (hasCron && !isValidCron(trigger.cron as string)) {
return { ok: false, error: "Cron expression is not valid" };
}
if (hasInterval && (trigger.intervalSeconds as number) < 60) {
return { ok: false, error: "Interval must be at least 60 seconds" };
}
if (
isNonEmptyString(trigger.timezone) &&
!isValidTimezone(trigger.timezone)
) {
return { ok: false, error: "Time zone is not valid" };
}
}
const steps = candidate.steps;
if (!Array.isArray(steps)) {
return { ok: false, error: "Definition must include a steps array" };
}
const seen = new Set<string>();
const stepError = validateSteps(steps as Step[], seen);
if (stepError) return { ok: false, error: stepError };
return {
ok: true,
definition: {
version: candidate.version ?? AUTOMATION_DEFINITION_VERSION,
trigger,
steps: steps as Step[],
},
};
}
function validateSteps(steps: Step[], seen: Set<string>): string | null {
for (const step of steps) {
if (!step || typeof step !== "object") return "Step must be an object";
if (!isNonEmptyString(step.id)) return "Every step needs an id";
if (seen.has(step.id)) return `Duplicate step id: ${step.id}`;
seen.add(step.id);
if (!STEP_TYPES.has(step.type)) {
return `Unknown step type: ${step.type}`;
}
if (step.type === "if") {
if (!step.condition || !OPERATORS.has(step.condition.operator)) {
return "Condition needs a valid operator";
}
const thenError = validateSteps(step.then ?? [], seen);
if (thenError) return thenError;
const elseError = validateSteps(step.else ?? [], seen);
if (elseError) return elseError;
}
if (step.type === "http" && !isNonEmptyString(step.url)) {
return "HTTP steps need a URL";
}
if (step.type === "run_command" && !isNonEmptyString(step.command)) {
return "Command steps need a command";
}
if (step.type === "wait" && typeof step.seconds !== "number") {
return "Wait steps need a number of seconds";
}
if (step.type === "set_var" && !isNonEmptyString(step.name)) {
return "Variable steps need a name";
}
}
return null;
}
plugins/automations/src/frontend/AutomationsPanel.tsx
+55 -79
View File
@@ -1,4 +1,3 @@
import { getErrorMessage } from "@/lib/error-message.js";
import { useCallback, useEffect, useState } from "react";
import { useTranslation } from "@termix/plugin-sdk/frontend";
import { toast } from "sonner";
@@ -12,30 +11,22 @@ import {
Trash2,
Workflow,
} from "lucide-react";
import { Button } from "@/components/button";
import { Badge } from "@/components/badge";
import {
createAutomation,
deleteAutomation,
listAutomationRunSteps,
listAutomationRuns,
listAutomations,
runAutomation,
updateAutomation,
Badge,
Button,
NotificationChannelDialog,
deleteNotificationChannel,
getBasePath,
getNotificationChannels,
testNotificationChannel,
type NotificationChannel,
} from "@termix/plugin-sdk/ui";
import {
useAutomationsApi,
type AutomationRow,
type AutomationRunRow,
type AutomationRunStepRow,
} from "./automations-api";
import {
deleteNotificationChannel,
getNotificationChannels,
testNotificationChannel,
type NotificationChannel,
} from "@/api/notification-channels-api";
import { NotificationChannelDialog } from "@/sidebar/NotificationChannelDialog";
import { getSnippets } from "@/api/snippets-api";
import { getSSHHosts } from "@/api/ssh-host-management-api";
import { invokeAction } from "@termix/plugin-sdk/frontend";
import {
AutomationEditor,
@@ -47,6 +38,10 @@ import {
type AutomationEditorOptions,
} from "./automations/editor-types";
function getErrorMessage(error: unknown, fallback: string): string {
return error instanceof Error ? error.message : fallback;
}
type PanelTab = "automations" | "runs" | "channels";
const STATUS_CLASS: Record<string, string> = {
@@ -58,25 +53,20 @@ const STATUS_CLASS: Record<string, string> = {
cancelled: "text-muted-foreground",
};
function timeAgo(iso: string | null): string {
type Translate = (key: string, options?: Record<string, unknown>) => string;
function timeAgo(iso: string | null, t: Translate): string {
if (!iso) return "";
const ms = Date.now() - new Date(iso).getTime();
if (!Number.isFinite(ms)) return "";
const base = "newUi.sidebar.automations.ago";
const sec = Math.floor(ms / 1000);
if (sec < 60) return `${sec}s ago`;
if (sec < 60) return t(`${base}.seconds`, { count: sec });
const min = Math.floor(sec / 60);
if (min < 60) return `${min}m ago`;
if (min < 60) return t(`${base}.minutes`, { count: min });
const hr = Math.floor(min / 60);
if (hr < 24) return `${hr}h ago`;
return `${Math.floor(hr / 24)}d ago`;
}
/** Fleets from the fleets plugin, or none when it is not running. */
async function listFleets(): Promise<{ id: number; name: string }[]> {
const result = await invokeAction("fleets.list");
return Array.isArray(result)
? (result as { id: number; name: string }[])
: [];
if (hr < 24) return t(`${base}.hours`, { count: hr });
return t(`${base}.days`, { count: Math.floor(hr / 24) });
}
export function AutomationsPanel({
@@ -87,6 +77,7 @@ export function AutomationsPanel({
onEditingChange?: (editing: boolean) => void;
}) {
const { t } = useTranslation();
const api = useAutomationsApi();
const base = "newUi.sidebar.automations";
const [tab, setTab] = useState<PanelTab>("automations");
@@ -113,60 +104,34 @@ export function AutomationsPanel({
const loadAutomations = useCallback(async () => {
setLoading(true);
try {
setAutomations(await listAutomations());
setAutomations(await api.list());
} catch {
// A failed load leaves the previous list on screen.
} finally {
setLoading(false);
}
}, []);
}, [api]);
const loadRuns = useCallback(async () => {
try {
setRuns(await listAutomationRuns({ limit: 100 }));
setRuns(await api.runs({ limit: 100 }));
} catch {
// Non-fatal.
}
}, []);
}, [api]);
const loadOptions = useCallback(async () => {
const [hosts, snippets, channelsResult, fleets] = await Promise.allSettled([
getSSHHosts({ includeStatus: false }),
getSnippets(),
const [editorOptions, channelsResult] = await Promise.allSettled([
api.editorOptions(),
getNotificationChannels(),
listFleets(),
]);
if (channelsResult.status === "fulfilled") {
setChannels(channelsResult.value);
}
setOptions({
hosts:
hosts.status === "fulfilled"
? hosts.value.map((host) => ({
id: host.id,
name: host.name || host.ip,
}))
: [],
snippets:
snippets.status === "fulfilled"
? snippets.value.map((s) => ({
id: s.id,
// SnippetRow only types its id; the rest is narrowed by callers.
name: typeof s.name === "string" ? s.name : `#${s.id}`,
}))
: [],
channels:
channelsResult.status === "fulfilled"
? channelsResult.value.map((c) => ({ id: c.id, name: c.name }))
: [],
fleets:
fleets.status === "fulfilled"
? fleets.value.map((f) => ({ id: f.id, name: f.name }))
: [],
});
}, []);
if (editorOptions.status === "fulfilled") {
setOptions(editorOptions.value);
}
}, [api]);
useEffect(() => {
if (!active) return;
@@ -222,7 +187,7 @@ export function AutomationsPanel({
setSaving(true);
try {
if (editingId === null) {
const created = await createAutomation({
const created = await api.create({
name: draft.name.trim(),
description: draft.description || null,
enabled: draft.enabled,
@@ -236,7 +201,7 @@ export function AutomationsPanel({
setEditorOpen(false);
}
} else {
await updateAutomation(editingId, {
await api.update(editingId, {
name: draft.name.trim(),
description: draft.description || null,
enabled: draft.enabled,
@@ -257,7 +222,7 @@ export function AutomationsPanel({
async function remove(row: AutomationRow) {
if (!confirm(t(`${base}.deleteConfirm`))) return;
try {
await deleteAutomation(row.id);
await api.remove(row.id);
toast.success(t(`${base}.deleted`));
await loadAutomations();
} catch (error) {
@@ -267,7 +232,7 @@ export function AutomationsPanel({
async function run(row: AutomationRow, dryRun: boolean) {
try {
const outcome = await runAutomation(row.id, { dryRun });
const outcome = await api.run(row.id, { dryRun });
toast.success(t(`${base}.runFinished`, { status: outcome.status }));
await loadAutomations();
if (tab === "runs") await loadRuns();
@@ -304,7 +269,7 @@ export function AutomationsPanel({
}
setExpandedRun(runId);
try {
setRunSteps(await listAutomationRunSteps(runId));
setRunSteps(await api.runSteps(runId));
} catch {
setRunSteps([]);
}
@@ -314,7 +279,7 @@ export function AutomationsPanel({
// host manager does, rather than opening a dialog over the app.
if (editorOpen) {
const webhookUrl = webhookToken
? `${window.location.origin}/automations/webhook/${webhookToken}`
? `${window.location.origin}${getBasePath()}/plugin-api/automations/webhook/${webhookToken}`
: "";
return (
@@ -432,7 +397,7 @@ export function AutomationsPanel({
rel="noreferrer"
className="ml-auto shrink-0 text-[10px] text-accent-brand hover:underline"
>
{t("hosts.docsLink")}
{t(`${base}.docsLink`)}
</a>
</div>
@@ -475,9 +440,20 @@ export function AutomationsPanel({
>
{row.name}
</button>
{!!row.missingPlugins?.length && (
<Badge
variant="outline"
className="text-[10px] border-destructive/50 text-destructive"
title={t(`${base}.needsPluginHint`)}
>
{t(`${base}.needsPlugin`, {
plugin: row.missingPlugins.join(", "),
})}
</Badge>
)}
{!row.enabled && (
<Badge variant="outline" className="text-[10px]">
off
{t(`${base}.offBadge`)}
</Badge>
)}
</div>
@@ -490,7 +466,7 @@ export function AutomationsPanel({
)
: ""}
{row.last_run_status
? ` · ${timeAgo(row.last_run_at)}`
? ` · ${timeAgo(row.last_run_at, t)}`
: ""}
</span>
<Button
@@ -555,7 +531,7 @@ export function AutomationsPanel({
</span>
</div>
<div className="text-[11px] text-muted-foreground">
{timeAgo(entry.started_at)}
{timeAgo(entry.started_at, t)}
{entry.duration_ms !== null
? ` · ${Math.round(entry.duration_ms / 100) / 10}s`
: ""}
plugins/automations/src/frontend/automations-api.ts
+78 -66
View File
@@ -1,4 +1,8 @@
import { authApi, handleApiError } from "@/main-axios";
import { useMemo } from "react";
import {
usePluginApi,
type PluginApiClient,
} from "@termix/plugin-sdk/frontend";
import type {
AutomationDefinition,
ConcurrencyPolicy,
@@ -6,7 +10,7 @@ import type {
StepStatus,
StepType,
TriggerKind,
} from "@/types/automations";
} from "../types";
export interface AutomationRow {
id: number;
@@ -24,6 +28,8 @@ export interface AutomationRow {
created_at: string;
updated_at: string;
channels: number[];
/** Plugins this automation needs that are not running. */
missingPlugins?: string[];
/** Returned once, only when a webhook automation is created. */
webhookToken?: string;
}
@@ -73,76 +79,82 @@ export interface AutomationInput {
channels?: number[];
}
export async function listAutomations(): Promise<AutomationRow[]> {
/** Which optional plugins are running, keyed like the backend's deps. */
export interface AutomationProviders {
snippets: boolean;
fleets: boolean;
tunnels: boolean;
docker: boolean;
"docker-events": boolean;
"host-metrics": boolean;
"wake-on-lan": boolean;
}
export interface AutomationEditorOptionsResponse {
hosts: Array<{ id: number; name: string }>;
snippets: Array<{ id: number; name: string }>;
fleets: Array<{ id: number; name: string }>;
channels: Array<{ id: number; name: string }>;
providers: AutomationProviders;
}
function apiError(error: unknown, action: string): Error {
const data = (
error as { response?: { data?: { error?: string; message?: string } } }
)?.response?.data;
return new Error(
data?.error ||
data?.message ||
(error instanceof Error ? error.message : `Failed to ${action}`),
);
}
async function call<T>(
action: string,
request: () => Promise<{ data: T }>,
): Promise<T> {
try {
return (await authApi.get("/plugin-api/automations")).data;
return (await request()).data;
} catch (error) {
throw handleApiError(error, "fetch automations");
throw apiError(error, action);
}
}
export async function createAutomation(
input: AutomationInput,
): Promise<AutomationRow> {
try {
return (await authApi.post("/plugin-api/automations", input)).data;
} catch (error) {
throw handleApiError(error, "create automation");
}
export function createAutomationsApi(api: PluginApiClient) {
return {
list: () => call<AutomationRow[]>("fetch automations", () => api.get("/")),
editorOptions: () =>
call<AutomationEditorOptionsResponse>("fetch editor options", () =>
api.get("/editor-options"),
),
create: (input: AutomationInput) =>
call<AutomationRow>("create automation", () => api.post("/", input)),
update: (id: number, input: Partial<AutomationInput>) =>
call<AutomationRow>("update automation", () => api.put(`/${id}`, input)),
remove: (id: number) =>
call<{ success: boolean }>("delete automation", () =>
api.delete(`/${id}`),
),
run: (id: number, options: { dryRun?: boolean } = {}) =>
call<AutomationRunOutcome>("run automation", () =>
api.post(`/${id}/run`, options),
),
runs: (
options: { automationId?: number; limit?: number; offset?: number } = {},
) =>
call<AutomationRunRow[]>("fetch automation runs", () =>
api.get("/runs/history", { params: options }),
),
runSteps: (runId: number) =>
call<AutomationRunStepRow[]>("fetch run steps", () =>
api.get(`/runs/${runId}/steps`),
),
};
}
export async function updateAutomation(
id: number,
input: Partial<AutomationInput>,
): Promise<AutomationRow> {
try {
return (await authApi.put(`/plugin-api/automations/${id}`, input)).data;
} catch (error) {
throw handleApiError(error, "update automation");
}
}
export type AutomationsApi = ReturnType<typeof createAutomationsApi>;
export async function deleteAutomation(id: number): Promise<void> {
try {
await authApi.delete(`/plugin-api/automations/${id}`);
} catch (error) {
throw handleApiError(error, "delete automation");
}
}
export async function runAutomation(
id: number,
options: { dryRun?: boolean } = {},
): Promise<AutomationRunOutcome> {
try {
return (await authApi.post(`/plugin-api/automations/${id}/run`, options))
.data;
} catch (error) {
throw handleApiError(error, "run automation");
}
}
export async function listAutomationRuns(
options: { automationId?: number; limit?: number; offset?: number } = {},
): Promise<AutomationRunRow[]> {
try {
return (
await authApi.get("/plugin-api/automations/runs/history", {
params: options,
})
).data;
} catch (error) {
throw handleApiError(error, "fetch automation runs");
}
}
export async function listAutomationRunSteps(
runId: number,
): Promise<AutomationRunStepRow[]> {
try {
return (await authApi.get(`/plugin-api/automations/runs/${runId}/steps`))
.data;
} catch (error) {
throw handleApiError(error, "fetch run steps");
}
export function useAutomationsApi(): AutomationsApi {
const api = usePluginApi();
return useMemo(() => createAutomationsApi(api), [api]);
}
plugins/automations/src/frontend/automations/AutomationEditor.tsx
+11 -10
View File
@@ -1,24 +1,23 @@
import { getErrorMessage } from "@/lib/error-message.js";
import { useState } from "react";
import { useTranslation } from "@termix/plugin-sdk/frontend";
import { Plus } from "lucide-react";
import { Button } from "@/components/button";
import { Input } from "@/components/input";
import { Label } from "@/components/label";
import { Switch } from "@/components/switch";
import {
Button,
Input,
Label,
Switch,
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/select";
} from "@termix/plugin-sdk/ui";
import { useTranslation } from "@termix/plugin-sdk/frontend";
import { Plus } from "lucide-react";
import {
AUTOMATION_DEFINITION_VERSION,
type AutomationDefinition,
type ConcurrencyPolicy,
type Step,
} from "@/types/automations";
} from "../../types";
import { StepBlock } from "./StepBlock";
import { TriggerCard, defaultTrigger } from "./TriggerCard";
import { newStepId, type AutomationEditorOptions } from "./editor-types";
@@ -246,7 +245,9 @@ export function AutomationEditor({
setDefinition(parsed as AutomationDefinition);
setJsonError(null);
} catch (error) {
setJsonError(getErrorMessage(error, String(error)));
setJsonError(
error instanceof Error ? error.message : String(error),
);
}
}}
/>
plugins/automations/src/frontend/automations/HostSelectorField.tsx
+8 -6
View File
@@ -1,13 +1,13 @@
import { useTranslation } from "@termix/plugin-sdk/frontend";
import { Label } from "@/components/label";
import {
Label,
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/select";
import type { HostSelector } from "@/types/automations";
} from "@termix/plugin-sdk/ui";
import type { HostSelector } from "../../types";
import type { AutomationEditorOptions } from "./editor-types";
/**
@@ -56,9 +56,11 @@ export function HostSelectorField({
<SelectItem value="host">
{t(`${base}.hostSelector.host`)}
</SelectItem>
<SelectItem value="fleet">
{t(`${base}.hostSelector.fleet`)}
</SelectItem>
{(options.providers.fleets || kind === "fleet") && (
<SelectItem value="fleet">
{t(`${base}.hostSelector.fleet`)}
</SelectItem>
)}
<SelectItem value="all">{t(`${base}.hostSelector.all`)}</SelectItem>
</SelectContent>
</Select>
plugins/automations/src/frontend/automations/StepBlock.tsx
+26 -14
View File
@@ -1,20 +1,23 @@
import { useTranslation } from "@termix/plugin-sdk/frontend";
import { ChevronDown, ChevronUp, Trash2 } from "lucide-react";
import { Button } from "@/components/button";
import { Input } from "@/components/input";
import { Label } from "@/components/label";
import { Textarea } from "@/components/textarea";
import { Switch } from "@/components/switch";
import {
Button,
Input,
Label,
Textarea,
Switch,
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/select";
import type { Step, StepType } from "@/types/automations";
} from "@termix/plugin-sdk/ui";
import { ChevronDown, ChevronUp, Trash2 } from "lucide-react";
import type { Step, StepType } from "../../types";
import { HostSelectorField } from "./HostSelectorField";
import type { AutomationEditorOptions } from "./editor-types";
import {
missingStepProvider,
type AutomationEditorOptions,
} from "./editor-types";
const STEP_TYPES: StepType[] = [
"notify",
@@ -86,11 +89,20 @@ export function StepBlock({
<SelectValue />
</SelectTrigger>
<SelectContent>
{STEP_TYPES.map((type) => (
<SelectItem key={type} value={type}>
{t(`${base}.stepTypes.${type}`)}
</SelectItem>
))}
{STEP_TYPES.map((type) => {
const needs = missingStepProvider(type, options.providers);
// A type whose plugin is off is only listed when this step
// already uses it, marked as needing that plugin.
if (needs && type !== step.type) return null;
return (
<SelectItem key={type} value={type}>
{t(`${base}.stepTypes.${type}`)}
{needs
? ` (${t(`${base}.needsPlugin`, { plugin: needs })})`
: ""}
</SelectItem>
);
})}
</SelectContent>
</Select>
plugins/automations/src/frontend/automations/TriggerCard.tsx
+22 -10
View File
@@ -1,16 +1,19 @@
import { useTranslation } from "@termix/plugin-sdk/frontend";
import { Input } from "@/components/input";
import { Label } from "@/components/label";
import {
Input,
Label,
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/select";
import type { Trigger, TriggerKind } from "@/types/automations";
} from "@termix/plugin-sdk/ui";
import type { Trigger, TriggerKind } from "../../types";
import { HostSelectorField } from "./HostSelectorField";
import type { AutomationEditorOptions } from "./editor-types";
import {
missingTriggerProvider,
type AutomationEditorOptions,
} from "./editor-types";
const TRIGGER_KINDS: TriggerKind[] = [
"metric_threshold",
@@ -98,11 +101,20 @@ export function TriggerCard({
<SelectValue />
</SelectTrigger>
<SelectContent>
{TRIGGER_KINDS.map((kind) => (
<SelectItem key={kind} value={kind}>
{t(`${base}.triggerKinds.${kind}`)}
</SelectItem>
))}
{TRIGGER_KINDS.map((kind) => {
const needs = missingTriggerProvider(kind, options.providers);
// A kind whose plugin is off is only listed when this trigger
// already uses it, marked as needing that plugin.
if (needs && kind !== trigger.kind) return null;
return (
<SelectItem key={kind} value={kind}>
{t(`${base}.triggerKinds.${kind}`)}
{needs
? ` (${t(`${base}.needsPlugin`, { plugin: needs })})`
: ""}
</SelectItem>
);
})}
</SelectContent>
</Select>
</div>
plugins/automations/src/frontend/automations/editor-types.ts
+53
View File
@@ -1,9 +1,14 @@
import type { StepType, TriggerKind } from "../../types";
import type { AutomationProviders } from "../automations-api";
/** Lookup data the editor needs to turn ids into names. */
export interface AutomationEditorOptions {
hosts: Array<{ id: number; name: string }>;
snippets: Array<{ id: number; name: string }>;
channels: Array<{ id: number; name: string }>;
fleets: Array<{ id: number; name: string }>;
/** Which optional plugins are running. */
providers: AutomationProviders;
}
export const EMPTY_EDITOR_OPTIONS: AutomationEditorOptions = {
@@ -11,9 +16,57 @@ export const EMPTY_EDITOR_OPTIONS: AutomationEditorOptions = {
snippets: [],
channels: [],
fleets: [],
providers: {
snippets: false,
fleets: false,
tunnels: false,
docker: false,
"docker-events": false,
"host-metrics": false,
"wake-on-lan": false,
},
};
/** Short unique id for a new step, stable for the life of the automation. */
export function newStepId(): string {
return `s${Math.random().toString(36).slice(2, 8)}`;
}
/** The plugin a trigger kind needs, when it is not core. */
const TRIGGER_PROVIDERS: Partial<
Record<TriggerKind, keyof AutomationProviders>
> = {
metric_threshold: "host-metrics",
health_check: "host-metrics",
docker_event: "docker-events",
};
/** The plugin a step type needs, when it is not core. */
const STEP_PROVIDERS: Partial<Record<StepType, keyof AutomationProviders>> = {
run_snippet: "snippets",
docker: "docker",
tunnel: "tunnels",
wol: "wake-on-lan",
};
/** The plugin id shown in "needs <plugin>", for a provider key. */
export function providerPlugin(key: keyof AutomationProviders): string {
return key === "docker-events" ? "docker" : key;
}
/** The provider a kind needs that is not running, or null when it can run. */
export function missingTriggerProvider(
kind: TriggerKind,
providers: AutomationProviders,
): string | null {
const key = TRIGGER_PROVIDERS[kind];
return key && !providers[key] ? providerPlugin(key) : null;
}
export function missingStepProvider(
type: StepType,
providers: AutomationProviders,
): string | null {
const key = STEP_PROVIDERS[type];
return key && !providers[key] ? providerPlugin(key) : null;
}
plugins/automations/src/frontend/index.tsx
+3 -2
View File
@@ -5,7 +5,7 @@ import type {
TermixApp,
} from "@termix/plugin-sdk/frontend";
import { AutomationsPanel } from "./AutomationsPanel";
import { listAutomations } from "./automations-api";
import { createAutomationsApi } from "./automations-api";
const VIEW_ID = "automations";
@@ -40,7 +40,8 @@ export function activate(app: TermixApp): void {
});
// The assistant's @-mentions list automations through this.
app.registerAction("automations.list", () => listAutomations());
const api = createAutomationsApi(app.api);
app.registerAction("automations.list", () => api.list());
app.registerSlotContribution("onboarding.features", {
actionId: "automations.feature",
plugins/automations/src/types.ts
+286
View File
@@ -0,0 +1,286 @@
/**
* Shared automation model used by both the backend engine and the editor UI.
* An automation is one trigger plus an ordered list of steps. The whole
* definition is stored as JSON on the automations row, so this file is the
* only contract describing that blob.
*/
export const AUTOMATION_DEFINITION_VERSION = 1;
/** Which hosts a trigger watches or a step acts on. */
export type HostSelector =
| { kind: "host"; hostId: number }
| { kind: "hosts"; hostIds: number[] }
| { kind: "fleet"; fleetId: number }
| { kind: "all" }
/** The host that produced the event. Only valid for host-scoped triggers. */
| { kind: "trigger" };
/**
* A metric to compare against. Paths mirror the shape returned by
* collectMetrics(). mount/iface pick one entry out of a per-instance list so a
* rule can watch a single filesystem rather than the aggregate.
*/
export type MetricPath =
| { path: "cpu.percent" }
| { path: "cpu.load1" }
| { path: "cpu.load5" }
| { path: "cpu.load15" }
| { path: "memory.percent" }
| { path: "memory.usedGiB" }
| { path: "disk.percent"; mount?: string }
| { path: "disk.availableBytes"; mount?: string }
| { path: "temperature.highestCelsius" }
| { path: "uptime.seconds" }
| { path: "processes.total" }
| { path: "network.rxBytes"; iface?: string }
| { path: "network.txBytes"; iface?: string }
| { path: "network.rxRateBps"; iface?: string }
| { path: "network.txRateBps"; iface?: string };
export type Operator =
| ">"
| "<"
| ">="
| "<="
| "=="
| "!="
| "contains"
| "not_contains"
| "changed";
export type Severity = "info" | "warning" | "critical";
export type HostStatusState = "online" | "offline";
export type HealthCheckState = "failing" | "recovered";
export type DockerEventKind = "exited" | "started" | "unhealthy" | "restarting";
export type InternalEventKind =
| "user_login"
| "host_added"
| "host_deleted"
| "tunnel_disconnected"
| "automation_failed";
export type Trigger =
| {
kind: "metric_threshold";
hostSelector: HostSelector;
metric: MetricPath;
operator: Operator;
value: number;
/** Sustained breach window before firing. 0 fires immediately. */
forSeconds?: number;
cooldownMinutes: number;
severity?: Severity;
}
| {
kind: "host_status";
hostSelector: HostSelector;
to: HostStatusState;
cooldownMinutes: number;
}
| {
kind: "health_check";
hostSelector: HostSelector;
checkId?: string;
to: HealthCheckState;
cooldownMinutes: number;
}
| {
kind: "schedule";
/** Five field cron. Ignored when intervalSeconds is set. */
cron?: string;
intervalSeconds?: number;
timezone?: string;
}
| {
kind: "docker_event";
hostSelector: HostSelector;
container?: string;
event: DockerEventKind;
cooldownMinutes: number;
}
| {
kind: "internal_event";
event: InternalEventKind;
hostSelector?: HostSelector;
cooldownMinutes: number;
}
| {
kind: "webhook";
/** Only the hash is persisted. The raw token is shown once on creation. */
tokenHash: string;
};
export type TriggerKind = Trigger["kind"];
/** A comparison used by `if` steps, evaluated against the run context. */
export interface Condition {
/** Template string, e.g. "{{steps.check.stdout}}" or "{{trigger.value}}". */
left: string;
operator: Operator;
/** Template string. Compared numerically when both sides parse as numbers. */
right?: string;
}
export type StepErrorPolicy = "stop" | "continue" | "branch";
interface StepBase {
/** Stable across edits so run history survives reordering. */
id: string;
name?: string;
enabled?: boolean;
onError?: StepErrorPolicy;
timeoutMs?: number;
}
export type Step =
| (StepBase & {
type: "notify";
channelIds: number[];
title?: string;
body?: string;
severity?: Severity;
})
| (StepBase & {
type: "http";
method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
url: string;
headers?: Record<string, string>;
body?: string;
/** Opt in to LAN/private addresses. Off by default, see safe-outbound-fetch. */
allowPrivateNetwork?: boolean;
})
| (StepBase & {
type: "run_snippet";
snippetId: number;
hostSelector: HostSelector;
inputValues?: Record<string, string>;
elevated?: boolean;
})
| (StepBase & {
type: "run_command";
command: string;
hostSelector: HostSelector;
elevated?: boolean;
})
| (StepBase & {
type: "docker";
action: "start" | "stop" | "restart";
container: string;
hostSelector: HostSelector;
})
| (StepBase & {
type: "tunnel";
action: "connect" | "disconnect";
tunnelName: string;
})
| (StepBase & { type: "wol"; hostId: number })
| (StepBase & { type: "wait"; seconds: number })
| (StepBase & { type: "set_var"; name: string; value: string })
| (StepBase & {
type: "if";
condition: Condition;
then: Step[];
else?: Step[];
})
| (StepBase & { type: "run_automation"; automationId: number })
| (StepBase & { type: "stop"; status?: "success" | "failed" });
export type StepType = Step["type"];
export interface AutomationDefinition {
version: number;
trigger: Trigger;
steps: Step[];
}
export type ConcurrencyPolicy = "skip" | "queue" | "allow";
export type RunStatus =
"running" | "success" | "failed" | "timeout" | "skipped" | "cancelled";
export type StepStatus =
"pending" | "running" | "success" | "failed" | "skipped";
export interface Automation {
id: number;
userId: string;
name: string;
description: string | null;
enabled: boolean;
definition: AutomationDefinition;
concurrencyPolicy: ConcurrencyPolicy;
maxRunSeconds: number;
dryRun: boolean;
lastRunAt: string | null;
lastRunStatus: RunStatus | null;
createdAt: string;
updatedAt: string;
}
export interface AutomationRun {
id: number;
automationId: number;
userId: string;
triggerType: TriggerKind | "manual";
triggerContext: Record<string, unknown> | null;
status: RunStatus;
startedAt: string;
finishedAt: string | null;
durationMs: number | null;
error: string | null;
dryRun: boolean;
parentRunId: number | null;
}
export interface AutomationRunStep {
id: number;
runId: number;
stepIndex: number;
stepId: string;
stepType: StepType;
status: StepStatus;
startedAt: string;
finishedAt: string | null;
output: string | null;
error: string | null;
truncated: boolean;
}
/** Defaults applied when an automation does not override them. */
export const DEFAULT_MAX_RUN_SECONDS = 300;
export const DEFAULT_STEP_TIMEOUT_MS = 60_000;
export const DEFAULT_COOLDOWN_MINUTES = 15;
/** Guards run_automation against direct and mutual recursion. */
export const MAX_AUTOMATION_DEPTH = 5;
/** Step output beyond this is truncated before it reaches the database. */
export const MAX_STEP_OUTPUT_BYTES = 32_768;
export const OPERATOR_LABELS: Record<Operator, string> = {
">": "greater than",
"<": "less than",
">=": "greater than or equal to",
"<=": "less than or equal to",
"==": "equals",
"!=": "does not equal",
contains: "contains",
not_contains: "does not contain",
changed: "changed",
};
/** Steps that reach outside Termix and are therefore stubbed in a dry run. */
export const SIDE_EFFECTING_STEP_TYPES: readonly StepType[] = [
"notify",
"http",
"run_snippet",
"run_command",
"docker",
"tunnel",
"wol",
];
export function isSideEffectingStep(type: StepType): boolean {
return SIDE_EFFECTING_STEP_TYPES.includes(type);
}