feat: make each plugin an npm workspace built by a termix-plugin CLI

plugins/automations/src/backend/actions/host-targets.ts

@@ -0,0 +1,90 @@
+import type { HostSelector } from "../../../../../src/types/automations.js";
+import { resolveHostById } from "../../../../../src/backend/hosts/host-resolver.js";
+import { createCurrentFleetRepository } from "../../../../../src/backend/database/repositories/factory.js";
+import { PermissionManager } from "../../../../../src/backend/utils/permission-manager.js";
+import type { StepExecutionContext } from "./types.js";
+
+/** A host the caller is allowed to act on. `host` is always resolved. */
+export interface ResolvedTarget {
+  id: number;
+  name: string;
+  host: NonNullable<Awaited<ReturnType<typeof resolveHostById>>>;
+}
+
+/**
+ * Turns a selector into the hosts a step may actually act on.
+ *
+ * Access is checked here, at execution time rather than when the automation
+ * was saved, so a permission revoked after the fact takes effect on the next
+ * run. resolveHostById performs its own connect-level check and returns null
+ * when the owner can no longer reach the host.
+ */
+export async function resolveTargets(
+  selector: HostSelector,
+  context: StepExecutionContext,
+): Promise<{ targets: ResolvedTarget[]; skipped: number[] }> {
+  const ids = await selectorHostIds(selector, context);
+  const targets: ResolvedTarget[] = [];
+  const skipped: number[] = [];
+
+  for (const id of ids) {
+    const host = await resolveHostById(id, context.userId);
+    if (!host) {
+      skipped.push(id);
+      continue;
+    }
+    targets.push({ id, name: host.name || host.ip, host });
+  }
+
+  return { targets, skipped };
+}
+
+async function selectorHostIds(
+  selector: HostSelector,
+  context: StepExecutionContext,
+): Promise<number[]> {
+  switch (selector.kind) {
+    case "host":
+      return [selector.hostId];
+    case "hosts":
+      return selector.hostIds;
+    case "trigger":
+      return context.triggerHostId ? [context.triggerHostId] : [];
+    case "fleet":
+      return fleetHostIds(selector.fleetId, context.userId);
+    case "all":
+      return allAccessibleHostIds(context.userId);
+    default:
+      return [];
+  }
+}
+
+async function fleetHostIds(
+  fleetId: number,
+  userId: string,
+): Promise<number[]> {
+  try {
+    const repository = createCurrentFleetRepository();
+    const members = await repository.listEffectiveMembers(userId, fleetId);
+    return members.map((member) => member.id);
+  } catch {
+    return [];
+  }
+}
+
+async function allAccessibleHostIds(userId: string): Promise<number[]> {
+  try {
+    const { createCurrentHostRepository } =
+      await import("../../../../../src/backend/database/repositories/factory.js");
+    const hosts = await createCurrentHostRepository().listByUserId(userId);
+    const ids = hosts.map((host) => host.id);
+    const allowed =
+      await PermissionManager.getInstance().filterAccessibleHostIds(
+        userId,
+        ids,
+      );
+    return ids.filter((id) => allowed.has(id));
+  } catch {
+    return [];
+  }
+}

plugins/automations/src/backend/actions/index.ts

@@ -0,0 +1,407 @@
+import {
+  DEFAULT_STEP_TIMEOUT_MS,
+  type Step,
+} from "../../../../../src/types/automations.js";
+import { execCommand } from "../../../../../src/backend/hosts/metrics-shared/common-utils.js";
+import {
+  execElevated,
+  shellSingleQuote,
+} from "../../../../../src/backend/hosts/metrics-shared/exec-elevated.js";
+import {
+  createFleetSshFactory,
+  getFleetPoolKey,
+} from "../../../../../src/backend/hosts/ssh-client-factory.js";
+import { withConnection } from "../../../../../src/backend/hosts/ssh-connection-pool.js";
+import { resolveSnippetCommand } from "../../../../../src/backend/database/routes/snippets-execution.js";
+import {
+  createCurrentNotificationChannelRepository,
+  createCurrentSnippetRepository,
+} from "../../../../../src/backend/database/repositories/factory.js";
+import { sendAutomationNotification } from "../notify.js";
+import { automationFetch } from "../http.js";
+import { renderRecord, renderTemplate } from "../template.js";
+import { resolveTargets, type ResolvedTarget } from "./host-targets.js";
+import {
+  fail,
+  ok,
+  stepTimeout,
+  type StepExecutionContext,
+  type StepResult,
+} from "./types.js";
+
+/**
+ * One executor per step type.
+ *
+ * Anything that leaves Termix checks context.dryRun first and reports what it
+ * would have done instead of doing it, so an automation can be exercised
+ * safely while it is being built.
+ */
+export async function executeStep(
+  step: Step,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  switch (step.type) {
+    case "notify":
+      return runNotify(step, context);
+    case "http":
+      return runHttp(step, context);
+    case "run_command":
+      return runCommand(step, context);
+    case "run_snippet":
+      return runSnippet(step, context);
+    case "docker":
+      return runDocker(step, context);
+    case "tunnel":
+      return runTunnel(step, context);
+    case "wol":
+      return runWol(step, context);
+    case "wait":
+      return runWait(step, context);
+    case "set_var":
+      return runSetVar(step, context);
+    case "stop":
+      return {
+        success: true,
+        halt: { status: step.status ?? "success" },
+        output: `Stopped with status ${step.status ?? "success"}`,
+      };
+    default:
+      return fail(`Unsupported step type: ${(step as Step).type}`);
+  }
+}
+
+async function runNotify(
+  step: Extract<Step, { type: "notify" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const title = renderTemplate(step.title ?? "", context.template);
+  const body = renderTemplate(step.body ?? "", context.template);
+
+  if (step.channelIds.length === 0) {
+    return fail("No notification channels selected");
+  }
+  if (context.dryRun) {
+    return ok(
+      `Would notify ${step.channelIds.length} channel(s): ${title || body}`,
+    );
+  }
+
+  const repository = createCurrentNotificationChannelRepository();
+  const channels = await repository.listNotificationChannels(context.userId);
+  const selected = channels.filter((channel) =>
+    step.channelIds.includes(channel.id),
+  );
+
+  if (selected.length === 0) {
+    return fail("Selected notification channels no longer exist");
+  }
+
+  let delivered = 0;
+  const errors: string[] = [];
+  for (const channel of selected) {
+    if (!channel.enabled) continue;
+    try {
+      await sendAutomationNotification(
+        { id: channel.id, type: channel.type, config: channel.config },
+        {
+          title,
+          body,
+          severity: step.severity ?? "warning",
+          context: context.template,
+        },
+      );
+      delivered++;
+    } catch (error) {
+      errors.push(
+        `${channel.name}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  if (delivered === 0) {
+    return fail(errors.join("; ") || "No enabled channels to notify");
+  }
+  return ok(
+    `Notified ${delivered} channel(s)${errors.length ? `; ${errors.join("; ")}` : ""}`,
+  );
+}
+
+async function runHttp(
+  step: Extract<Step, { type: "http" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const url = renderTemplate(step.url, context.template);
+  const headers = renderRecord(step.headers, context.template);
+  const body = step.body
+    ? renderTemplate(step.body, context.template)
+    : undefined;
+
+  if (context.dryRun) {
+    return ok(`Would ${step.method} ${url}`);
+  }
+
+  try {
+    const response = await automationFetch(url, {
+      method: step.method,
+      headers,
+      body,
+      allowPrivateNetwork: step.allowPrivateNetwork,
+      timeoutMs: stepTimeout(context, step.timeoutMs, DEFAULT_STEP_TIMEOUT_MS),
+    });
+
+    const text = await response.text();
+    const summary = `HTTP ${response.status} ${response.statusText}\n${text}`;
+    return response.ok ? ok(summary) : fail(`HTTP ${response.status}`, summary);
+  } catch (error) {
+    return fail(error instanceof Error ? error.message : String(error));
+  }
+}
+
+async function runCommand(
+  step: Extract<Step, { type: "run_command" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const command = renderTemplate(step.command, context.template);
+  return runOnTargets(step.hostSelector, context, async (target) => {
+    if (context.dryRun) {
+      return { output: `Would run on ${target.name}: ${command}` };
+    }
+    return execOnHost(
+      target.host,
+      command,
+      step.elevated,
+      context,
+      step.timeoutMs,
+    );
+  });
+}
+
+async function runSnippet(
+  step: Extract<Step, { type: "run_snippet" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const snippet = await createCurrentSnippetRepository()
+    .findOwnedById(context.userId, step.snippetId)
+    .catch(() => null);
+
+  if (!snippet) return fail("Snippet not found");
+  if (snippet.isNote) return fail("Notes cannot be executed on a host");
+
+  // Template values only ever reach the snippet through inputValues, never by
+  // rewriting the snippet body, so automation variables cannot inject snippet
+  // syntax of their own.
+  const inputValues = renderRecord(step.inputValues, context.template) ?? {};
+
+  return runOnTargets(step.hostSelector, context, async (target) => {
+    const command = resolveSnippetCommand(
+      snippet.content,
+      {
+        ip: target.host.ip,
+        username: target.host.username,
+        port: target.host.port,
+        name: target.host.name,
+      },
+      inputValues,
+    );
+
+    if (context.dryRun) {
+      return { output: `Would run snippet on ${target.name}: ${command}` };
+    }
+    return execOnHost(
+      target.host,
+      command,
+      step.elevated,
+      context,
+      step.timeoutMs,
+    );
+  });
+}
+
+async function runDocker(
+  step: Extract<Step, { type: "docker" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const container = renderTemplate(step.container, context.template);
+  if (!container) return fail("Container name is required");
+
+  return runOnTargets(step.hostSelector, context, async (target) => {
+    if (context.dryRun) {
+      return {
+        output: `Would ${step.action} container ${container} on ${target.name}`,
+      };
+    }
+
+    // The Docker HTTP routes are tied to an interactive session, so run the
+    // equivalent command over the same pooled SSH connection everything else
+    // uses. The container name is quoted because it comes from a template.
+    const command = `docker ${step.action} ${shellSingleQuote(container)}`;
+    return execOnHost(target.host, command, false, context, step.timeoutMs);
+  });
+}
+
+async function runTunnel(
+  step: Extract<Step, { type: "tunnel" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const name = renderTemplate(step.tunnelName, context.template);
+  if (context.dryRun) return ok(`Would ${step.action} tunnel ${name}`);
+
+  try {
+    const manager =
+      await import("../../../../../src/backend/hosts/tunnel/manager.js");
+    const config = manager.tunnelConfigs?.get(name);
+    if (!config) return fail(`Tunnel "${name}" is not configured`);
+
+    if (step.action === "connect") {
+      await manager.connectSSHTunnel(config);
+      return ok(`Tunnel ${name} connected`);
+    }
+
+    // shouldRetry false, otherwise the manager immediately reconnects the
+    // tunnel the automation just asked it to drop.
+    manager.manualDisconnects.add(name);
+    await manager.handleDisconnect(name, config, false);
+    return ok(`Tunnel ${name} disconnected`);
+  } catch (error) {
+    return fail(error instanceof Error ? error.message : String(error));
+  }
+}
+
+async function runWol(
+  step: Extract<Step, { type: "wol" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const host = await resolveTargets(
+    { kind: "host", hostId: step.hostId },
+    context,
+  );
+  const target = host.targets[0];
+  if (!target?.host) return fail("Host not found or not accessible");
+
+  const mac = (target.host as { macAddress?: string }).macAddress;
+  if (!mac) return fail("Host has no MAC address configured");
+  if (context.dryRun) return ok(`Would wake ${target.name} (${mac})`);
+
+  try {
+    const { sendWakeOnLan, isValidMac } =
+      await import("../../../../../src/backend/utils/wake-on-lan.js");
+    if (!isValidMac(mac)) return fail(`Invalid MAC address: ${mac}`);
+    await sendWakeOnLan(mac);
+    return ok(`Sent magic packet to ${target.name}`);
+  } catch (error) {
+    return fail(error instanceof Error ? error.message : String(error));
+  }
+}
+
+async function runWait(
+  step: Extract<Step, { type: "wait" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const requested = Math.max(step.seconds, 0) * 1000;
+  const waitMs = Math.min(
+    requested,
+    stepTimeout(context, undefined, requested),
+  );
+  if (context.dryRun) return ok(`Would wait ${step.seconds}s`);
+
+  await new Promise((resolve) => setTimeout(resolve, waitMs));
+  return ok(`Waited ${Math.round(waitMs / 1000)}s`);
+}
+
+async function runSetVar(
+  step: Extract<Step, { type: "set_var" }>,
+  context: StepExecutionContext,
+): Promise<StepResult> {
+  const value = renderTemplate(step.value, context.template);
+  return ok(`${step.name} = ${value}`, { [step.name]: value });
+}
+
+/**
+ * Runs a per-host action across a selector's targets. One host failing does
+ * not stop the others, matching how fleet execution behaves.
+ */
+async function runOnTargets(
+  selector: Extract<Step, { type: "run_command" }>["hostSelector"],
+  context: StepExecutionContext,
+  run: (target: ResolvedTarget) => Promise<{ output?: string; error?: string }>,
+): Promise<StepResult> {
+  const { targets, skipped } = await resolveTargets(selector, context);
+
+  if (targets.length === 0) {
+    return fail(
+      skipped.length > 0
+        ? `No accessible hosts (${skipped.length} skipped)`
+        : "No hosts matched the selector",
+    );
+  }
+
+  const results = await Promise.allSettled(
+    targets.map(async (target) => ({
+      target,
+      result: await run(target),
+    })),
+  );
+
+  const lines: string[] = [];
+  let failures = 0;
+
+  for (const [index, settled] of results.entries()) {
+    const name = targets[index].name;
+    if (settled.status === "rejected") {
+      failures++;
+      lines.push(`${name}: ${String(settled.reason)}`);
+      continue;
+    }
+    const { result } = settled.value;
+    if (result.error) {
+      failures++;
+      lines.push(`${name}: ${result.error}`);
+    } else {
+      lines.push(`${name}: ${result.output ?? "ok"}`);
+    }
+  }
+
+  if (skipped.length > 0) {
+    lines.push(`${skipped.length} host(s) skipped: no access`);
+  }
+
+  const output = lines.join("\n");
+  return failures > 0 && failures === targets.length
+    ? fail(`All ${failures} host(s) failed`, output)
+    : ok(output);
+}
+
+async function execOnHost(
+  host: ResolvedTarget["host"],
+  command: string,
+  elevated: boolean | undefined,
+  context: StepExecutionContext,
+  timeoutMs: number | undefined,
+): Promise<{ output?: string; error?: string }> {
+  const sshHost = host as ResolvedTarget["host"] & { sudoPassword?: string };
+  const timeout = stepTimeout(context, timeoutMs, DEFAULT_STEP_TIMEOUT_MS);
+  if (timeout <= 0) return { error: "Run deadline exceeded" };
+
+  try {
+    const result = await withConnection(
+      getFleetPoolKey(sshHost),
+      createFleetSshFactory(sshHost),
+      async (client) => {
+        if (elevated) {
+          return execElevated(client, command, sshHost.sudoPassword, {
+            timeoutMs: timeout,
+          });
+        }
+        return execCommand(client, command, timeout);
+      },
+    );
+
+    const output = [result.stdout, result.stderr].filter(Boolean).join("\n");
+    if (result.code === 0 || result.code === null) {
+      return { output: output || "(no output)" };
+    }
+    return { error: `Exited with code ${result.code}`, output };
+  } catch (error) {
+    return { error: error instanceof Error ? error.message : String(error) };
+  }
+}

plugins/automations/src/backend/actions/types.ts

@@ -0,0 +1,58 @@
+import type { Step } from "../../../../../src/types/automations.js";
+import type { TemplateContext } from "../template.js";
+
+/** What every executor returns. `output` is what later steps can read. */
+export interface StepResult {
+  success: boolean;
+  output?: string;
+  error?: string;
+  /** Merged into the run's variables, for set_var and friends. */
+  vars?: Record<string, string>;
+  /** Set by the stop step to end the run early. */
+  halt?: { status: "success" | "failed" };
+}
+
+export interface StepExecutionContext {
+  userId: string;
+  automationId: number;
+  runId: number;
+  /** Nothing that leaves Termix may actually happen when this is set. */
+  dryRun: boolean;
+  template: TemplateContext;
+  /** Host the trigger fired for, when there was one. */
+  triggerHostId?: number;
+  /** Automations already on the stack, to refuse recursion. */
+  ancestry: number[];
+  depth: number;
+  /** Wall-clock deadline for the whole run. */
+  deadlineAt: number;
+  signal?: AbortSignal;
+}
+
+export type StepExecutor<T extends Step = Step> = (
+  step: T,
+  context: StepExecutionContext,
+) => Promise<StepResult>;
+
+export function ok(output?: string, vars?: Record<string, string>): StepResult {
+  return { success: true, output, vars };
+}
+
+export function fail(error: string, output?: string): StepResult {
+  return { success: false, error, output };
+}
+
+/** Milliseconds left before the run's overall deadline. */
+export function remainingMs(context: StepExecutionContext): number {
+  return Math.max(context.deadlineAt - Date.now(), 0);
+}
+
+/** A step's timeout, clamped so it can never outlive the run. */
+export function stepTimeout(
+  context: StepExecutionContext,
+  requested: number | undefined,
+  fallback: number,
+): number {
+  const wanted = requested && requested > 0 ? requested : fallback;
+  return Math.max(Math.min(wanted, remainingMs(context)), 0);
+}

plugins/automations/src/backend/conditions.ts

@@ -0,0 +1,206 @@
+import type {
+  MetricPath,
+  Operator,
+} from "../../../../src/types/automations.js";
+
+/**
+ * Operator evaluation and metric extraction.
+ *
+ * Pure: no database, no SSH, no clock. Everything here is driven by values the
+ * caller already has, which keeps the comparison rules testable on their own.
+ */
+
+/** Shape of the metrics snapshot this module reads. Mirrors collectMetrics(). */
+export interface MetricsSnapshot {
+  cpu?: {
+    percent?: number | null;
+    load?: [number, number, number] | null;
+  } | null;
+  memory?: { percent?: number | null; usedGiB?: number | null } | null;
+  disk?: {
+    percent?: number | null;
+    filesystems?: Array<{
+      mount?: string;
+      percent?: number | null;
+      availableBytes?: number | null;
+    }> | null;
+  } | null;
+  network?: {
+    interfaces?: Array<{
+      name?: string;
+      rxBytes?: string | number | null;
+      txBytes?: string | number | null;
+      rxRateBps?: number | null;
+      txRateBps?: number | null;
+    }> | null;
+  } | null;
+  temperature?: { highestCelsius?: number | null } | null;
+  uptime?: { seconds?: number | null } | null;
+  processes?: { total?: number | null } | null;
+}
+
+function toNumber(value: unknown): number | null {
+  if (value === null || value === undefined || value === "") return null;
+  const parsed = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+/**
+ * Pulls the value a trigger watches out of a metrics snapshot.
+ *
+ * The mount and iface selectors are what let a rule watch one filesystem or
+ * one interface. Without a selector the aggregate is used, which for disk is
+ * the primary (root) mount, matching what the metrics UI shows.
+ */
+export function extractMetricValue(
+  metrics: MetricsSnapshot | null | undefined,
+  metric: MetricPath,
+): number | null {
+  if (!metrics) return null;
+
+  switch (metric.path) {
+    case "cpu.percent":
+      return toNumber(metrics.cpu?.percent);
+    case "cpu.load1":
+      return toNumber(metrics.cpu?.load?.[0]);
+    case "cpu.load5":
+      return toNumber(metrics.cpu?.load?.[1]);
+    case "cpu.load15":
+      return toNumber(metrics.cpu?.load?.[2]);
+    case "memory.percent":
+      return toNumber(metrics.memory?.percent);
+    case "memory.usedGiB":
+      return toNumber(metrics.memory?.usedGiB);
+    case "disk.percent": {
+      if (!metric.mount) return toNumber(metrics.disk?.percent);
+      const fs = findMount(metrics, metric.mount);
+      return fs ? toNumber(fs.percent) : null;
+    }
+    case "disk.availableBytes": {
+      const fs = metric.mount ? findMount(metrics, metric.mount) : null;
+      if (metric.mount) return fs ? toNumber(fs.availableBytes) : null;
+      const first = metrics.disk?.filesystems?.[0];
+      return first ? toNumber(first.availableBytes) : null;
+    }
+    case "temperature.highestCelsius":
+      return toNumber(metrics.temperature?.highestCelsius);
+    case "uptime.seconds":
+      return toNumber(metrics.uptime?.seconds);
+    case "processes.total":
+      return toNumber(metrics.processes?.total);
+    case "network.rxBytes":
+      return toNumber(findInterface(metrics, metric.iface)?.rxBytes);
+    case "network.txBytes":
+      return toNumber(findInterface(metrics, metric.iface)?.txBytes);
+    case "network.rxRateBps":
+      return toNumber(findInterface(metrics, metric.iface)?.rxRateBps);
+    case "network.txRateBps":
+      return toNumber(findInterface(metrics, metric.iface)?.txRateBps);
+    default:
+      return null;
+  }
+}
+
+function findMount(metrics: MetricsSnapshot, mount: string) {
+  return (
+    metrics.disk?.filesystems?.find((entry) => entry.mount === mount) ?? null
+  );
+}
+
+function findInterface(metrics: MetricsSnapshot, iface?: string) {
+  const interfaces = metrics.network?.interfaces;
+  if (!interfaces || interfaces.length === 0) return null;
+  if (!iface) return interfaces[0];
+  return interfaces.find((entry) => entry.name === iface) ?? null;
+}
+
+/**
+ * The key a trigger's durable state is stored under. Including the mount or
+ * container is what allows a sustained-breach window per filesystem rather
+ * than per host.
+ */
+export function metricStateKey(hostId: number, metric: MetricPath): string {
+  if ("mount" in metric && metric.mount) return `${hostId}:${metric.mount}`;
+  if ("iface" in metric && metric.iface) return `${hostId}:${metric.iface}`;
+  return String(hostId);
+}
+
+/**
+ * Compares two values. Numeric when both sides look numeric, so "90" and 90
+ * behave the same; string comparison otherwise. `changed` is handled by the
+ * caller, which is the only place that knows the previous value.
+ */
+export function compare(
+  left: unknown,
+  operator: Operator,
+  right: unknown,
+): boolean {
+  if (operator === "contains" || operator === "not_contains") {
+    const haystack = String(left ?? "");
+    const needle = String(right ?? "");
+    const found = haystack.includes(needle);
+    return operator === "contains" ? found : !found;
+  }
+
+  const leftNumber = toNumber(left);
+  const rightNumber = toNumber(right);
+  const numeric = leftNumber !== null && rightNumber !== null;
+
+  switch (operator) {
+    case ">":
+      return numeric && leftNumber > rightNumber;
+    case "<":
+      return numeric && leftNumber < rightNumber;
+    case ">=":
+      return numeric && leftNumber >= rightNumber;
+    case "<=":
+      return numeric && leftNumber <= rightNumber;
+    case "==":
+      return numeric
+        ? leftNumber === rightNumber
+        : String(left ?? "") === String(right ?? "");
+    case "!=":
+      return numeric
+        ? leftNumber !== rightNumber
+        : String(left ?? "") !== String(right ?? "");
+    case "changed":
+      return String(left ?? "") !== String(right ?? "");
+    default:
+      return false;
+  }
+}
+
+/** Whether a cooldown window is still open. */
+export function isCoolingDown(
+  lastFiredAt: string | null | undefined,
+  cooldownMinutes: number,
+  now: number = Date.now(),
+): boolean {
+  if (!lastFiredAt) return false;
+  const last = Date.parse(lastFiredAt);
+  if (Number.isNaN(last)) return false;
+  return now - last < Math.max(cooldownMinutes, 0) * 60_000;
+}
+
+/** Whether a sustained breach has been held long enough to fire. */
+export function hasDwelled(
+  breachStartedAt: string | null | undefined,
+  forSeconds: number | undefined,
+  now: number = Date.now(),
+): boolean {
+  if (!forSeconds) return true;
+  if (!breachStartedAt) return false;
+  const started = Date.parse(breachStartedAt);
+  if (Number.isNaN(started)) return false;
+  return now - started >= forSeconds * 1000;
+}
+
+/** Severity for a threshold breach, matching the old engine's behaviour. */
+export function severityForValue(
+  value: number | null,
+  explicit?: "info" | "warning" | "critical",
+): "info" | "warning" | "critical" {
+  if (explicit) return explicit;
+  if (value !== null && value >= 95) return "critical";
+  return "warning";
+}

plugins/automations/src/backend/cron.ts

@@ -0,0 +1,291 @@
+/**
+ * A small five field cron parser, used only to work out when a schedule is
+ * next due.
+ *
+ * Deliberately not a dependency: the engine needs "when is this next due?" and
+ * nothing else, and a pure function is far easier to test than a scheduler
+ * library. Fields are the standard minute, hour, day-of-month, month and
+ * day-of-week, supporting *, lists (1,2), ranges (1-5) and steps (a slash).
+ *
+ * Day-of-month and day-of-week follow cron's union rule: when both are
+ * restricted, a date matching either one matches.
+ */
+
+interface CronFields {
+  minutes: Set<number>;
+  hours: Set<number>;
+  daysOfMonth: Set<number>;
+  months: Set<number>;
+  daysOfWeek: Set<number>;
+  domRestricted: boolean;
+  dowRestricted: boolean;
+}
+
+const RANGES: Record<string, [number, number]> = {
+  minute: [0, 59],
+  hour: [0, 23],
+  dayOfMonth: [1, 31],
+  month: [1, 12],
+  // 7 is accepted as an alias for Sunday and folded to 0 once parsed.
+  dayOfWeek: [0, 7],
+};
+
+const NAMED_MONTHS: Record<string, number> = {
+  jan: 1,
+  feb: 2,
+  mar: 3,
+  apr: 4,
+  may: 5,
+  jun: 6,
+  jul: 7,
+  aug: 8,
+  sep: 9,
+  oct: 10,
+  nov: 11,
+  dec: 12,
+};
+
+const NAMED_DAYS: Record<string, number> = {
+  sun: 0,
+  mon: 1,
+  tue: 2,
+  wed: 3,
+  thu: 4,
+  fri: 5,
+  sat: 6,
+};
+
+function normalize(token: string, kind: string): string {
+  const lower = token.toLowerCase();
+  if (kind === "month" && lower in NAMED_MONTHS) {
+    return String(NAMED_MONTHS[lower]);
+  }
+  if (kind === "dayOfWeek" && lower in NAMED_DAYS) {
+    return String(NAMED_DAYS[lower]);
+  }
+  return token;
+}
+
+function parseField(field: string, kind: keyof typeof RANGES): Set<number> {
+  const [min, max] = RANGES[kind];
+  const values = new Set<number>();
+
+  for (const part of field.split(",")) {
+    const trimmed = part.trim();
+    if (!trimmed) throw new Error(`Empty value in ${kind} field`);
+
+    const [rangePart, stepPart] = trimmed.split("/");
+    const step = stepPart === undefined ? 1 : Number(stepPart);
+    if (!Number.isInteger(step) || step < 1) {
+      throw new Error(`Invalid step in ${kind} field`);
+    }
+
+    let start: number;
+    let end: number;
+
+    if (rangePart === "*" || rangePart === "") {
+      start = min;
+      end = max;
+    } else if (rangePart.includes("-")) {
+      const [from, to] = rangePart.split("-");
+      start = Number(normalize(from, kind));
+      end = Number(normalize(to, kind));
+    } else {
+      start = Number(normalize(rangePart, kind));
+      end = stepPart === undefined ? start : max;
+    }
+
+    if (!Number.isInteger(start) || !Number.isInteger(end)) {
+      throw new Error(`Invalid value in ${kind} field`);
+    }
+    if (start < min || end > max || start > end) {
+      throw new Error(`Value out of range in ${kind} field`);
+    }
+
+    for (let value = start; value <= end; value += step) {
+      // Sunday can be written as 7; cron treats it as 0.
+      values.add(kind === "dayOfWeek" && value === 7 ? 0 : value);
+    }
+  }
+
+  return values;
+}
+
+export function parseCron(expression: string): CronFields {
+  const fields = expression.trim().split(/\s+/);
+  if (fields.length !== 5) {
+    throw new Error("A cron expression needs five fields");
+  }
+
+  const [minute, hour, dayOfMonth, month, dayOfWeek] = fields;
+  return {
+    minutes: parseField(minute, "minute"),
+    hours: parseField(hour, "hour"),
+    daysOfMonth: parseField(dayOfMonth, "dayOfMonth"),
+    months: parseField(month, "month"),
+    daysOfWeek: parseField(dayOfWeek, "dayOfWeek"),
+    domRestricted: dayOfMonth.trim() !== "*",
+    dowRestricted: dayOfWeek.trim() !== "*",
+  };
+}
+
+export function isValidCron(expression: string): boolean {
+  try {
+    parseCron(expression);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Wall-clock fields for a moment, in a given zone or server local time. */
+interface WallClock {
+  minute: number;
+  hour: number;
+  dayOfMonth: number;
+  month: number;
+  dayOfWeek: number;
+}
+
+const WEEKDAYS: Record<string, number> = {
+  Sun: 0,
+  Mon: 1,
+  Tue: 2,
+  Wed: 3,
+  Thu: 4,
+  Fri: 5,
+  Sat: 6,
+};
+
+const formatterCache = new Map<string, Intl.DateTimeFormat>();
+
+/**
+ * Whether a zone name is one this runtime actually knows. An unknown zone
+ * falls back to server local time rather than throwing, so a bad value saved
+ * against a schedule cannot stop it from ever running.
+ */
+export function isValidTimezone(timezone: string): boolean {
+  try {
+    new Intl.DateTimeFormat("en-US", { timeZone: timezone });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function formatterFor(timezone: string): Intl.DateTimeFormat | null {
+  const cached = formatterCache.get(timezone);
+  if (cached) return cached;
+
+  try {
+    const formatter = new Intl.DateTimeFormat("en-US", {
+      timeZone: timezone,
+      hour12: false,
+      weekday: "short",
+      month: "numeric",
+      day: "numeric",
+      hour: "numeric",
+      minute: "numeric",
+    });
+    formatterCache.set(timezone, formatter);
+    return formatter;
+  } catch {
+    return null;
+  }
+}
+
+function wallClock(date: Date, timezone?: string | null): WallClock {
+  const formatter = timezone ? formatterFor(timezone) : null;
+  if (!formatter) {
+    return {
+      minute: date.getMinutes(),
+      hour: date.getHours(),
+      dayOfMonth: date.getDate(),
+      month: date.getMonth() + 1,
+      dayOfWeek: date.getDay(),
+    };
+  }
+
+  const parts: Record<string, string> = {};
+  for (const part of formatter.formatToParts(date)) {
+    parts[part.type] = part.value;
+  }
+
+  return {
+    // Midnight formats as 24 in some locales' hour-cycle handling.
+    minute: Number(parts.minute),
+    hour: Number(parts.hour) % 24,
+    dayOfMonth: Number(parts.day),
+    month: Number(parts.month),
+    dayOfWeek: WEEKDAYS[parts.weekday] ?? date.getDay(),
+  };
+}
+
+function matches(
+  fields: CronFields,
+  date: Date,
+  timezone?: string | null,
+): boolean {
+  const clock = wallClock(date, timezone);
+  if (!fields.months.has(clock.month)) return false;
+  if (!fields.minutes.has(clock.minute)) return false;
+  if (!fields.hours.has(clock.hour)) return false;
+
+  const domMatch = fields.daysOfMonth.has(clock.dayOfMonth);
+  const dowMatch = fields.daysOfWeek.has(clock.dayOfWeek);
+
+  // Both restricted means either may match, which is how cron behaves.
+  if (fields.domRestricted && fields.dowRestricted) return domMatch || dowMatch;
+  if (fields.domRestricted) return domMatch;
+  if (fields.dowRestricted) return dowMatch;
+  return true;
+}
+
+/**
+ * The next time on or after `from` that the expression matches, or null when
+ * nothing matches within a four year window (e.g. Feb 30).
+ */
+export function nextCronRun(
+  expression: string,
+  from: Date = new Date(),
+  timezone?: string | null,
+): Date | null {
+  const fields = parseCron(expression);
+
+  const candidate = new Date(from.getTime());
+  candidate.setSeconds(0, 0);
+  candidate.setMinutes(candidate.getMinutes() + 1);
+
+  // Four years covers every leap year cycle, so a date that never matches
+  // gives up rather than looping.
+  const limit = 366 * 4 * 24 * 60;
+  for (let i = 0; i < limit; i++) {
+    if (matches(fields, candidate, timezone)) return candidate;
+    candidate.setMinutes(candidate.getMinutes() + 1);
+  }
+  return null;
+}
+
+/**
+ * Next due time for a schedule trigger, as an ISO string. Interval wins over
+ * cron when both are set, matching the editor which offers one or the other.
+ */
+export function computeNextDueAt(
+  schedule: {
+    cron?: string | null;
+    intervalSeconds?: number | null;
+    timezone?: string | null;
+  },
+  from: Date = new Date(),
+): string | null {
+  if (schedule.intervalSeconds && schedule.intervalSeconds > 0) {
+    return new Date(
+      from.getTime() + schedule.intervalSeconds * 1000,
+    ).toISOString();
+  }
+  if (schedule.cron) {
+    const next = nextCronRun(schedule.cron, from, schedule.timezone);
+    return next ? next.toISOString() : null;
+  }
+  return null;
+}

plugins/automations/src/backend/docker-watcher.ts

@@ -0,0 +1,238 @@
+import type { AutomationDefinition } from "../../../../src/types/automations.js";
+import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
+import { resolveHostById } from "../../../../src/backend/hosts/host-resolver.js";
+import { DataCrypto } from "../../../../src/backend/utils/data-crypto.js";
+import { execCommand } from "../../../../src/backend/hosts/metrics-shared/common-utils.js";
+import {
+  createFleetSshFactory,
+  getFleetPoolKey,
+} from "../../../../src/backend/hosts/ssh-client-factory.js";
+import { withConnection } from "../../../../src/backend/hosts/ssh-connection-pool.js";
+import { statsLogger } from "../../../../src/backend/utils/logger.js";
+import { onDockerEvent } from "./triggers.js";
+
+/**
+ * Container state polling for docker_event triggers.
+ *
+ * Everything else Docker-related in the backend hangs off an interactive
+ * session that only exists while somebody has the UI open, so a trigger built
+ * on it would only ever fire while being watched. This polls over the same
+ * pooled SSH connection the other automation steps use, and only for hosts a
+ * docker_event trigger actually names, so an install with no such automation
+ * does no extra work at all.
+ *
+ * Events are derived by diffing successive snapshots: the poll interval is the
+ * resolution, so a container that stops and starts between two polls is not
+ * reported. That is the tradeoff for not holding a `docker events` stream open
+ * against every host.
+ */
+
+const POLL_INTERVAL_MS = 60_000;
+const EXEC_TIMEOUT_MS = 15_000;
+
+interface ContainerState {
+  /** Docker's own state word: running, exited, restarting, ... */
+  state: string;
+  /** Health from the status text, when the image declares a healthcheck. */
+  unhealthy: boolean;
+}
+
+/** Last snapshot per host, so transitions can be spotted. */
+const snapshots = new Map<number, Map<string, ContainerState>>();
+const lastPolledAt = new Map<number, number>();
+
+/** Hosts named by an enabled docker_event trigger, with the owning user. */
+export async function listDockerWatchedHosts(): Promise<Map<number, string>> {
+  const watched = new Map<number, string>();
+
+  try {
+    const rows = await createCurrentAutomationRepository().listAllEnabled();
+    for (const row of rows) {
+      let definition: AutomationDefinition;
+      try {
+        definition = JSON.parse(row.definition) as AutomationDefinition;
+      } catch {
+        continue;
+      }
+
+      const trigger = definition.trigger;
+      if (trigger?.kind !== "docker_event") continue;
+
+      const selector = trigger.hostSelector;
+      if (selector?.kind === "host") {
+        watched.set(selector.hostId, row.userId);
+      } else if (selector?.kind === "hosts") {
+        for (const hostId of selector.hostIds) watched.set(hostId, row.userId);
+      }
+      // Fleet and "all" selectors are deliberately not expanded: polling every
+      // host a user owns for container state is far too costly to do blindly.
+    }
+  } catch {
+    return watched;
+  }
+
+  return watched;
+}
+
+/**
+ * Parses `docker ps -a` output. One JSON object per line, matching the format
+ * string the container routes use.
+ */
+export function parseContainerStates(
+  output: string,
+): Map<string, ContainerState> {
+  const states = new Map<string, ContainerState>();
+
+  for (const line of output.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    try {
+      const parsed = JSON.parse(trimmed) as {
+        name?: string;
+        state?: string;
+        status?: string;
+      };
+      if (!parsed.name) continue;
+
+      states.set(parsed.name, {
+        state: (parsed.state ?? "").toLowerCase(),
+        unhealthy: /\(unhealthy\)/i.test(parsed.status ?? ""),
+      });
+    } catch {
+      // A partial line is not worth failing the whole poll over.
+    }
+  }
+
+  return states;
+}
+
+/**
+ * Works out which events a pair of snapshots implies.
+ *
+ * A container missing from the previous snapshot is treated as newly seen
+ * rather than started, so the first poll after a restart does not replay every
+ * running container as a fresh start event.
+ */
+export function diffContainerStates(
+  previous: Map<string, ContainerState>,
+  current: Map<string, ContainerState>,
+): Array<{ container: string; event: DockerEventName }> {
+  const events: Array<{ container: string; event: DockerEventName }> = [];
+
+  for (const [name, now] of current) {
+    const before = previous.get(name);
+    if (!before) continue;
+
+    if (before.state !== now.state) {
+      if (now.state === "exited")
+        events.push({ container: name, event: "exited" });
+      else if (now.state === "running")
+        events.push({ container: name, event: "started" });
+      else if (now.state === "restarting")
+        events.push({ container: name, event: "restarting" });
+    }
+
+    // Health is independent of state: a container can go unhealthy while it
+    // stays up, which is exactly the case worth alerting on.
+    if (!before.unhealthy && now.unhealthy) {
+      events.push({ container: name, event: "unhealthy" });
+    }
+  }
+
+  return events;
+}
+
+export type DockerEventName = "exited" | "started" | "unhealthy" | "restarting";
+
+const PS_FORMAT = `'{"name":"{{.Names}}","state":"{{.State}}","status":"{{.Status}}"}'`;
+
+async function pollHost(hostId: number, userId: string): Promise<void> {
+  const host = await resolveHostById(hostId, userId);
+  if (!host) {
+    snapshots.delete(hostId);
+    return;
+  }
+
+  const result = await withConnection(
+    getFleetPoolKey(host as never),
+    createFleetSshFactory(host as never),
+    (client) =>
+      execCommand(
+        client,
+        `docker ps -a --format ${PS_FORMAT}`,
+        EXEC_TIMEOUT_MS,
+      ),
+  );
+
+  if (result.code !== 0 && result.code !== null) {
+    // Docker missing or not permitted on this host. Drop the snapshot so a
+    // later success is treated as a first observation rather than a diff.
+    snapshots.delete(hostId);
+    return;
+  }
+
+  const current = parseContainerStates(result.stdout);
+  const previous = snapshots.get(hostId);
+  snapshots.set(hostId, current);
+
+  if (!previous) return;
+
+  for (const { container, event } of diffContainerStates(previous, current)) {
+    await onDockerEvent({
+      hostId,
+      ownerUserId: userId,
+      container,
+      event,
+    }).catch(() => undefined);
+  }
+}
+
+/**
+ * Polls every watched host whose interval has elapsed. Called from the
+ * automation scheduler tick rather than owning a timer of its own.
+ */
+export async function pollDockerEvents(
+  now: number = Date.now(),
+): Promise<void> {
+  const watched = await listDockerWatchedHosts();
+
+  for (const hostId of [...snapshots.keys()]) {
+    if (!watched.has(hostId)) {
+      snapshots.delete(hostId);
+      lastPolledAt.delete(hostId);
+    }
+  }
+
+  for (const [hostId, userId] of watched) {
+    const last = lastPolledAt.get(hostId) ?? 0;
+    if (now - last < POLL_INTERVAL_MS) continue;
+    // Host credentials cannot be decrypted while the owner's key is locked.
+    if (!canAccess(userId)) continue;
+    lastPolledAt.set(hostId, now);
+
+    try {
+      await pollHost(hostId, userId);
+    } catch (error) {
+      statsLogger.warn("Docker event poll failed", {
+        operation: "automation_docker_poll_error",
+        hostId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+}
+
+function canAccess(userId: string): boolean {
+  try {
+    return DataCrypto.canUserAccessData(userId);
+  } catch {
+    return false;
+  }
+}
+
+/** Clears cached state, for shutdown and tests. */
+export function resetDockerWatcher(): void {
+  snapshots.clear();
+  lastPolledAt.clear();
+}

plugins/automations/src/backend/engine.ts

@@ -0,0 +1,448 @@
+import type {
+  AutomationDefinition,
+  RunStatus,
+  Step,
+} from "../../../../src/types/automations.js";
+import {
+  DEFAULT_MAX_RUN_SECONDS,
+  MAX_AUTOMATION_DEPTH,
+  MAX_STEP_OUTPUT_BYTES,
+} from "../../../../src/types/automations.js";
+import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
+import { statsLogger } from "../../../../src/backend/utils/logger.js";
+import { resolveHostById } from "../../../../src/backend/hosts/host-resolver.js";
+import { notifyAutomationInternalEvent } from "../../../../src/backend/hosts/automation-events.js";
+import { executeStep } from "./actions/index.js";
+import type { StepExecutionContext, StepResult } from "./actions/types.js";
+import { compare } from "./conditions.js";
+import { renderTemplate, type TemplateContext } from "./template.js";
+
+export interface RunRequest {
+  automationId: number;
+  triggerType: string;
+  triggerContext?: Record<string, unknown>;
+  triggerHostId?: number;
+  /** Overrides the automation's own dry-run flag, for "test run". */
+  dryRun?: boolean;
+  parentRunId?: number;
+  ancestry?: number[];
+  depth?: number;
+}
+
+export interface RunOutcome {
+  runId: number | null;
+  status: RunStatus;
+  error?: string;
+}
+
+/**
+ * Executes automations.
+ *
+ * Both HTTP handlers and the metrics hooks live in this same process, so this
+ * is a plain singleton rather than anything cross-process. State that has to
+ * survive a restart (cooldowns, dwell windows) lives in the database; the only
+ * thing held in memory is the set of runs currently in flight, which is
+ * meaningless after a restart anyway.
+ */
+export class AutomationEngine {
+  private static instance: AutomationEngine;
+
+  private readonly running = new Set<number>();
+  private readonly queued = new Map<number, number>();
+
+  static getInstance(): AutomationEngine {
+    if (!AutomationEngine.instance) {
+      AutomationEngine.instance = new AutomationEngine();
+    }
+    return AutomationEngine.instance;
+  }
+
+  isRunning(automationId: number): boolean {
+    return this.running.has(automationId);
+  }
+
+  async run(request: RunRequest): Promise<RunOutcome> {
+    const repository = createCurrentAutomationRepository();
+    const automation = await repository.findById(request.automationId);
+
+    if (!automation) {
+      return { runId: null, status: "failed", error: "Automation not found" };
+    }
+
+    const depth = request.depth ?? 0;
+    const ancestry = request.ancestry ?? [];
+
+    // Refuse recursion before anything is recorded, so a cycle cannot spin.
+    if (depth > MAX_AUTOMATION_DEPTH) {
+      return {
+        runId: null,
+        status: "failed",
+        error: `Maximum automation depth of ${MAX_AUTOMATION_DEPTH} exceeded`,
+      };
+    }
+    if (ancestry.includes(automation.id)) {
+      return {
+        runId: null,
+        status: "failed",
+        error: `Automation ${automation.id} is already running in this chain`,
+      };
+    }
+
+    let definition: AutomationDefinition;
+    try {
+      definition = JSON.parse(automation.definition) as AutomationDefinition;
+    } catch {
+      return {
+        runId: null,
+        status: "failed",
+        error: "Automation definition is not valid JSON",
+      };
+    }
+
+    // A second trigger while a run is in flight is recorded as skipped rather
+    // than dropped silently, so the history explains what happened.
+    //
+    // The slot has to be claimed in the same tick as the check. It used to be
+    // claimed several awaits later, so two triggers arriving together both
+    // passed this test and both ran.
+    let claimed = false;
+    if (this.running.has(automation.id)) {
+      const policy = automation.concurrencyPolicy;
+      if (policy === "skip") {
+        const run = await repository.createRun({
+          automationId: automation.id,
+          userId: automation.userId,
+          triggerType: request.triggerType,
+          triggerContext: JSON.stringify(request.triggerContext ?? {}),
+          status: "skipped",
+        });
+        await repository.finishRun(run.id, {
+          status: "skipped",
+          error: "A previous run was still in progress",
+          durationMs: 0,
+        });
+        return { runId: run.id, status: "skipped" };
+      }
+      if (policy === "queue") {
+        const depthNow = this.queued.get(automation.id) ?? 0;
+        if (depthNow >= 5) {
+          return { runId: null, status: "skipped", error: "Queue is full" };
+        }
+        this.queued.set(automation.id, depthNow + 1);
+        try {
+          await this.waitUntilFree(automation.id);
+        } finally {
+          this.queued.set(
+            automation.id,
+            (this.queued.get(automation.id) ?? 1) - 1,
+          );
+        }
+        // waitUntilFree gives up on its own deadline, so the slot may still be
+        // taken. Only claim it when it is genuinely free.
+        if (!this.running.has(automation.id)) {
+          this.running.add(automation.id);
+          claimed = true;
+        }
+      }
+    } else {
+      this.running.add(automation.id);
+      claimed = true;
+    }
+
+    const dryRun = request.dryRun ?? automation.dryRun;
+    const maxRunSeconds = automation.maxRunSeconds || DEFAULT_MAX_RUN_SECONDS;
+    const startedAt = Date.now();
+
+    let run: { id: number };
+    try {
+      run = await repository.createRun({
+        automationId: automation.id,
+        userId: automation.userId,
+        triggerType: request.triggerType,
+        triggerContext: JSON.stringify(request.triggerContext ?? {}),
+        status: "running",
+        dryRun,
+        parentRunId: request.parentRunId ?? null,
+      });
+    } catch (err) {
+      // The slot is already claimed at this point, so it has to be given back
+      // here; the finally below is only reached once a run row exists.
+      if (claimed) this.running.delete(automation.id);
+      return {
+        runId: null,
+        status: "failed",
+        error: err instanceof Error ? err.message : String(err),
+      };
+    }
+
+    if (!claimed) this.running.add(automation.id);
+
+    const trigger = { ...(request.triggerContext ?? {}) };
+    trigger.type ??= request.triggerType;
+    let host: TemplateContext["host"];
+    if (request.triggerHostId) {
+      try {
+        const resolved = await resolveHostById(
+          request.triggerHostId,
+          automation.userId,
+        );
+        if (resolved) {
+          host = {
+            id: request.triggerHostId,
+            name: resolved.name || resolved.ip,
+            ip: resolved.ip,
+            username: resolved.username,
+            port: resolved.port,
+          };
+          trigger.hostName ??= host.name;
+        }
+      } catch {
+        // A notification should still run with its numeric host id.
+      }
+    }
+
+    const template: TemplateContext = {
+      host,
+      trigger,
+      steps: {},
+      vars: {},
+      run: {
+        id: run.id,
+        automationId: automation.id,
+        startedAt: new Date(startedAt).toISOString(),
+      },
+    };
+
+    const context: StepExecutionContext = {
+      userId: automation.userId,
+      automationId: automation.id,
+      runId: run.id,
+      dryRun,
+      template,
+      triggerHostId: request.triggerHostId,
+      ancestry: [...ancestry, automation.id],
+      depth,
+      deadlineAt: startedAt + maxRunSeconds * 1000,
+    };
+
+    let status: RunStatus = "success";
+    let error: string | undefined;
+
+    try {
+      const result = await this.runSteps(definition.steps ?? [], context, {
+        index: 0,
+      });
+      if (result.halted?.status === "failed") {
+        status = "failed";
+        error = "Stopped by a stop step";
+      } else if (result.failed) {
+        status = "failed";
+        error = result.error;
+      }
+      if (Date.now() >= context.deadlineAt) {
+        status = "timeout";
+        error = `Run exceeded ${maxRunSeconds}s`;
+      }
+    } catch (err) {
+      status = "failed";
+      error = err instanceof Error ? err.message : String(err);
+    } finally {
+      this.running.delete(automation.id);
+    }
+
+    await repository.finishRun(run.id, {
+      status,
+      error: error ?? null,
+      durationMs: Date.now() - startedAt,
+    });
+
+    if (status === "failed") {
+      statsLogger.warn(`Automation "${automation.name}" failed`, {
+        operation: "automation_run_failed",
+        automationId: automation.id,
+        runId: run.id,
+        error,
+      });
+
+      // An automation_failed handler that itself fails must not re-announce
+      // its own failure, so the event is not emitted for runs that this event
+      // already started.
+      if (request.triggerType !== "internal_event") {
+        // Emitted onto the bus rather than imported from hosts/: the bus is
+        // fire-and-forget, so this no longer needs the lazy-import dance that
+        // was working around the repositories cycle.
+        notifyAutomationInternalEvent(
+          "automation_failed",
+          automation.userId,
+          undefined,
+          {
+            automationId: automation.id,
+            automationName: automation.name,
+            runId: run.id,
+            error: error ?? null,
+          },
+        );
+      }
+    }
+
+    return { runId: run.id, status, error };
+  }
+
+  /**
+   * Runs a list of steps in order, descending into if/else. Returns as soon as
+   * a stop step halts the run or a failing step's policy says to stop.
+   */
+  private async runSteps(
+    steps: Step[],
+    context: StepExecutionContext,
+    cursor: { index: number },
+  ): Promise<{
+    failed: boolean;
+    error?: string;
+    halted?: { status: "success" | "failed" };
+  }> {
+    const repository = createCurrentAutomationRepository();
+
+    for (const step of steps) {
+      if (step.enabled === false) continue;
+
+      if (Date.now() >= context.deadlineAt) {
+        return { failed: true, error: "Run deadline exceeded" };
+      }
+
+      const stepIndex = cursor.index++;
+
+      if (step.type === "if") {
+        const left = renderTemplate(step.condition.left, context.template);
+        const right = renderTemplate(
+          step.condition.right ?? "",
+          context.template,
+        );
+        const matched = compare(left, step.condition.operator, right);
+
+        const rowId = await repository.createRunStep({
+          runId: context.runId,
+          stepIndex,
+          stepId: step.id,
+          stepType: "if",
+          status: "running",
+        });
+        await repository.finishRunStep(rowId, {
+          status: "success",
+          output: `Condition ${matched ? "matched" : "did not match"}: ${left} ${step.condition.operator} ${right}`,
+        });
+
+        const branch = matched ? step.then : (step.else ?? []);
+        const result = await this.runSteps(branch, context, cursor);
+        if (result.halted) return result;
+        if (result.failed) return result;
+        continue;
+      }
+
+      if (step.type === "run_automation") {
+        const rowId = await repository.createRunStep({
+          runId: context.runId,
+          stepIndex,
+          stepId: step.id,
+          stepType: step.type,
+          status: "running",
+        });
+
+        const nested = await this.run({
+          automationId: step.automationId,
+          triggerType: "run_automation",
+          triggerContext: { parentAutomationId: context.automationId },
+          triggerHostId: context.triggerHostId,
+          dryRun: context.dryRun,
+          parentRunId: context.runId,
+          ancestry: context.ancestry,
+          depth: context.depth + 1,
+        });
+
+        const nestedOk = nested.status === "success";
+        await repository.finishRunStep(rowId, {
+          status: nestedOk ? "success" : "failed",
+          output: `Nested run ${nested.runId ?? "not started"}: ${nested.status}`,
+          error: nested.error ?? null,
+        });
+
+        if (!nestedOk && (step.onError ?? "stop") === "stop") {
+          return { failed: true, error: nested.error ?? "Nested run failed" };
+        }
+        continue;
+      }
+
+      const rowId = await repository.createRunStep({
+        runId: context.runId,
+        stepIndex,
+        stepId: step.id,
+        stepType: step.type,
+        status: "running",
+      });
+
+      let result: StepResult;
+      try {
+        result = await executeStep(step, context);
+      } catch (err) {
+        result = {
+          success: false,
+          error: err instanceof Error ? err.message : String(err),
+        };
+      }
+
+      const { text, truncated } = truncate(result.output);
+      await repository.finishRunStep(rowId, {
+        status: result.success ? "success" : "failed",
+        output: text,
+        error: result.error ?? null,
+        truncated,
+      });
+
+      // Later steps read earlier output through {{steps.<id>.stdout}}.
+      context.template.steps = {
+        ...context.template.steps,
+        [step.id]: {
+          stdout: result.output ?? "",
+          code: result.success ? 0 : 1,
+        },
+      };
+      if (result.vars) {
+        context.template.vars = { ...context.template.vars, ...result.vars };
+      }
+
+      if (result.halt) return { failed: false, halted: result.halt };
+
+      if (!result.success) {
+        const policy = step.onError ?? "stop";
+        if (policy === "stop") {
+          return { failed: true, error: result.error };
+        }
+      }
+    }
+
+    return { failed: false };
+  }
+
+  private async waitUntilFree(automationId: number): Promise<void> {
+    const started = Date.now();
+    while (this.running.has(automationId)) {
+      if (Date.now() - started > 60_000) return;
+      await new Promise((resolve) => setTimeout(resolve, 250));
+    }
+  }
+}
+
+/** Keeps a single step's output from bloating the database. */
+function truncate(output: string | undefined): {
+  text: string | null;
+  truncated: boolean;
+} {
+  if (!output) return { text: null, truncated: false };
+  if (Buffer.byteLength(output, "utf8") <= MAX_STEP_OUTPUT_BYTES) {
+    return { text: output, truncated: false };
+  }
+  return {
+    text: output.slice(0, MAX_STEP_OUTPUT_BYTES) + "\n... (truncated)",
+    truncated: true,
+  };
+}

plugins/automations/src/backend/headless-viewer.ts

@@ -0,0 +1,105 @@
+import { statsLogger } from "../../../../src/backend/utils/logger.js";
+import { listAutomationWatchedHosts } from "./triggers.js";
+
+/**
+ * Keeps metric collection running for hosts an automation watches.
+ *
+ * Heavy metric collection is normally started by a UI viewer and stops when
+ * the last one leaves, which means threshold rules only ever evaluated while
+ * somebody had the host open. Automations register a synthetic viewer instead
+ * of bypassing that mechanism, so a real viewer arriving or leaving still
+ * behaves exactly as before.
+ *
+ * The catch is `cleanupInactiveViewers`, which drops any viewer whose
+ * heartbeat is older than 120s. Without the heartbeat below, headless polling
+ * would quietly stop two minutes after it started.
+ */
+
+export interface ViewerRegistry {
+  registerViewer(hostId: number, sessionId: string, userId: string): void;
+  unregisterViewer(hostId: number, sessionId: string): void;
+  updateHeartbeat(sessionId: string): boolean;
+}
+
+const SESSION_PREFIX = "automation:";
+
+let registry: ViewerRegistry | null = null;
+const registered = new Map<number, string>();
+
+export function setViewerRegistry(next: ViewerRegistry | null): void {
+  registry = next;
+}
+
+export function automationSessionId(hostId: number): string {
+  return `${SESSION_PREFIX}${hostId}`;
+}
+
+/**
+ * Brings the set of synthetic viewers in line with what the enabled
+ * automations currently watch, and heartbeats the ones that stay.
+ */
+export async function reconcileHeadlessViewers(): Promise<{
+  added: number;
+  removed: number;
+  active: number;
+}> {
+  if (!registry) return { added: 0, removed: 0, active: 0 };
+
+  let watched: Map<number, string>;
+  try {
+    watched = await listAutomationWatchedHosts();
+  } catch {
+    return { added: 0, removed: 0, active: registered.size };
+  }
+
+  let added = 0;
+  let removed = 0;
+
+  for (const [hostId, userId] of watched) {
+    const sessionId = automationSessionId(hostId);
+    if (registered.has(hostId)) {
+      // Refresh before the 120s reaper would take it.
+      registry.updateHeartbeat(sessionId);
+      continue;
+    }
+
+    try {
+      registry.registerViewer(hostId, sessionId, userId);
+      registered.set(hostId, userId);
+      added++;
+    } catch (error) {
+      statsLogger.warn("Could not start headless metrics for a host", {
+        operation: "automation_headless_register_error",
+        hostId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  for (const hostId of [...registered.keys()]) {
+    if (watched.has(hostId)) continue;
+    try {
+      registry.unregisterViewer(hostId, automationSessionId(hostId));
+    } catch {
+      // Already gone; drop it either way.
+    }
+    registered.delete(hostId);
+    removed++;
+  }
+
+  return { added, removed, active: registered.size };
+}
+
+/** Drops every synthetic viewer, for shutdown and tests. */
+export function releaseHeadlessViewers(): void {
+  if (registry) {
+    for (const hostId of registered.keys()) {
+      try {
+        registry.unregisterViewer(hostId, automationSessionId(hostId));
+      } catch {
+        // Nothing useful to do during teardown.
+      }
+    }
+  }
+  registered.clear();
+}

plugins/automations/src/backend/http.ts

@@ -0,0 +1,79 @@
+import { safeOutboundFetch } from "../../../../src/backend/utils/safe-outbound-fetch.js";
+import { readNotificationPrivateAllowlist } from "../../../../src/backend/utils/notification-egress.js";
+
+/**
+ * Outbound HTTP for automation steps and notification channels.
+ *
+ * safeOutboundFetch refuses private and loopback addresses, which is the right
+ * default against SSRF but also blocks the self-hosted ntfy or Gotify sitting
+ * on a LAN that many installs actually use. Private delivery therefore needs
+ * both a channel opt-in and an exact host in the administrator allowlist;
+ * scheme validation, DNS pinning and redirect refusal remain in force.
+ */
+export interface AutomationFetchOptions {
+  method?: string;
+  headers?: Record<string, string>;
+  body?: string;
+  allowPrivateNetwork?: boolean;
+  timeoutMs?: number;
+}
+
+export async function automationFetch(
+  url: string,
+  options: AutomationFetchOptions = {},
+): Promise<Response> {
+  const {
+    method = "GET",
+    headers,
+    body,
+    allowPrivateNetwork,
+    timeoutMs = 30_000,
+  } = options;
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, 1));
+
+  const init: RequestInit = {
+    method,
+    headers: body
+      ? { "Content-Type": "application/json", ...(headers ?? {}) }
+      : headers,
+    body,
+    signal: controller.signal,
+  };
+
+  try {
+    if (allowPrivateNetwork) {
+      return await privateNetworkFetch(url, init);
+    }
+    return await safeOutboundFetch(url, init);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * The opt-in path still goes through the guarded resolver. Only an exact host
+ * authorized by an administrator may resolve to a private address.
+ */
+async function privateNetworkFetch(
+  rawUrl: string,
+  init: RequestInit,
+): Promise<Response> {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    throw new Error("Invalid URL");
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error("Only http and https URLs are allowed");
+  }
+  if (parsed.username || parsed.password) {
+    throw new Error("URLs with embedded credentials are not allowed");
+  }
+
+  const allowlist = await readNotificationPrivateAllowlist();
+  return safeOutboundFetch(rawUrl, init, allowlist);
+}

plugins/automations/src/backend/index.ts

@@ -0,0 +1,19 @@
+import type { PluginContext } from "@termix/plugin-sdk/backend";
+import { startAutomationsService, stopAutomationsService } from "./routes.js";
+import {
+  startAutomationScheduler,
+  stopAutomationScheduler,
+} from "./scheduler.js";
+
+export async function activate(ctx: PluginContext) {
+  startAutomationsService();
+  startAutomationScheduler();
+  ctx.log.info(
+    "Automations router registered at /automations; scheduler started",
+  );
+}
+
+export async function deactivate() {
+  stopAutomationScheduler();
+  stopAutomationsService();
+}

plugins/automations/src/backend/notify.ts

@@ -0,0 +1,193 @@
+import { statsLogger } from "../../../../src/backend/utils/logger.js";
+import { automationFetch } from "./http.js";
+import type { TemplateContext } from "./template.js";
+
+/**
+ * Notification delivery for automations.
+ *
+ * The alert engine special-cased Discord because its dispatcher only knew
+ * webhook and ntfy; here every transport goes through one switch, so adding a
+ * channel type is a single edit.
+ */
+export interface AutomationChannel {
+  id: number;
+  type: string;
+  config: string;
+}
+
+export interface AutomationNotification {
+  title: string;
+  body: string;
+  severity: "info" | "warning" | "critical";
+  context?: TemplateContext;
+}
+
+const NTFY_PRIORITY: Record<string, number> = {
+  info: 2,
+  warning: 3,
+  critical: 5,
+};
+
+const NTFY_TAGS: Record<string, string> = {
+  info: "information_source",
+  warning: "warning",
+  critical: "rotating_light",
+};
+
+const DISCORD_COLORS: Record<string, number> = {
+  info: 3066993,
+  warning: 16753920,
+  critical: 15158332,
+};
+
+export async function sendAutomationNotification(
+  channel: AutomationChannel,
+  notification: AutomationNotification,
+): Promise<void> {
+  let config: Record<string, unknown>;
+  try {
+    config = JSON.parse(channel.config) as Record<string, unknown>;
+  } catch {
+    throw new Error("Channel configuration is not valid JSON");
+  }
+
+  const allowPrivateNetwork = config.allowPrivateNetwork === true;
+
+  switch (channel.type) {
+    case "webhook":
+      return sendWebhook(config, notification, allowPrivateNetwork);
+    case "ntfy":
+      return sendNtfy(config, notification, allowPrivateNetwork);
+    case "discord":
+      return sendDiscord(config, notification, allowPrivateNetwork);
+    default:
+      throw new Error(`Unsupported channel type: ${channel.type}`);
+  }
+}
+
+function requireUrl(config: Record<string, unknown>): string {
+  const url = typeof config.url === "string" ? config.url.trim() : "";
+  if (!url) throw new Error("Channel is missing a URL");
+  return url;
+}
+
+async function sendWebhook(
+  config: Record<string, unknown>,
+  notification: AutomationNotification,
+  allowPrivateNetwork: boolean,
+): Promise<void> {
+  const url = requireUrl(config);
+  const method = config.method === "PUT" ? "PUT" : "POST";
+  const headers =
+    config.headers && typeof config.headers === "object"
+      ? (config.headers as Record<string, string>)
+      : {};
+
+  const response = await automationFetch(url, {
+    method,
+    headers,
+    body: JSON.stringify({
+      title: notification.title,
+      hostName:
+        notification.context?.host?.name ??
+        notification.context?.trigger?.hostName,
+      hostId:
+        notification.context?.host?.id ?? notification.context?.trigger?.hostId,
+      ruleName: notification.title,
+      ruleId: notification.context?.run?.automationId,
+      triggerType: notification.context?.trigger?.type,
+      value: notification.context?.trigger?.value,
+      threshold: notification.context?.trigger?.threshold,
+      message: notification.body,
+      severity: notification.severity,
+      timestamp: new Date().toISOString(),
+    }),
+    allowPrivateNetwork,
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText}`);
+  }
+}
+
+async function sendNtfy(
+  config: Record<string, unknown>,
+  notification: AutomationNotification,
+  allowPrivateNetwork: boolean,
+): Promise<void> {
+  const base = requireUrl(config).replace(/\/$/, "");
+  const topic = typeof config.topic === "string" ? config.topic.trim() : "";
+  if (!topic) throw new Error("ntfy channel is missing a topic");
+
+  const headers: Record<string, string> = {
+    Title: notification.title || "Termix automation",
+    Priority: String(NTFY_PRIORITY[notification.severity] ?? 3),
+    Tags: NTFY_TAGS[notification.severity] ?? "information_source",
+  };
+  if (typeof config.token === "string" && config.token) {
+    headers.Authorization = `Bearer ${config.token}`;
+  }
+
+  const response = await automationFetch(`${base}/${topic}`, {
+    method: "POST",
+    headers,
+    body: notification.body || notification.title,
+    allowPrivateNetwork,
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText}`);
+  }
+}
+
+async function sendDiscord(
+  config: Record<string, unknown>,
+  notification: AutomationNotification,
+  allowPrivateNetwork: boolean,
+): Promise<void> {
+  const url = requireUrl(config);
+  const payload: Record<string, unknown> = {
+    embeds: [
+      {
+        title: notification.title || "Termix automation",
+        description: notification.body || undefined,
+        color: DISCORD_COLORS[notification.severity] ?? 3447003,
+        timestamp: new Date().toISOString(),
+      },
+    ],
+  };
+  if (typeof config.username === "string" && config.username) {
+    payload.username = config.username;
+  }
+  if (typeof config.avatar_url === "string" && config.avatar_url) {
+    payload.avatar_url = config.avatar_url;
+  }
+
+  const response = await automationFetch(url, {
+    method: "POST",
+    body: JSON.stringify(payload),
+    allowPrivateNetwork,
+  });
+
+  if (!response.ok) {
+    const detail = await response.text().catch(() => "");
+    throw new Error(
+      `HTTP ${response.status} ${response.statusText}${detail ? `: ${detail}` : ""}`,
+    );
+  }
+}
+
+/** Fire-and-forget wrapper for callers that must not block on delivery. */
+export function sendAutomationNotificationSafely(
+  channel: AutomationChannel,
+  notification: AutomationNotification,
+): void {
+  sendAutomationNotification(channel, notification).catch((error) => {
+    statsLogger.warn("Automation notification failed", {
+      operation: "automation_notification_error",
+      channelId: channel.id,
+      type: channel.type,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  });
+}

plugins/automations/src/backend/routes.ts

@@ -0,0 +1,832 @@
+import crypto from "node:crypto";
+import express, { type Request, type Response } from "express";
+import type { AuthenticatedRequest } from "../../../../src/types/index.js";
+import type {
+  AutomationDefinition,
+  Step,
+  Trigger,
+} from "../../../../src/types/automations.js";
+import { AUTOMATION_DEFINITION_VERSION } from "../../../../src/types/automations.js";
+import { PermissionManager } from "../../../../src/backend/utils/permission-manager.js";
+import { AuthManager } from "../../../../src/backend/utils/auth-manager.js";
+import { databaseLogger } from "../../../../src/backend/utils/logger.js";
+import {
+  getAuditUsername,
+  getRequestMeta,
+  logAudit,
+} from "../../../../src/backend/utils/audit-logger.js";
+import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
+import type { AutomationRow } from "../../../../src/backend/database/repositories/automation-repository.js";
+import { AutomationEngine } from "./engine.js";
+import { computeNextDueAt, isValidCron, isValidTimezone } from "./cron.js";
+import {
+  registerAutomationsRouter,
+  unregisterAutomationsRouter,
+} from "../../../../src/backend/database/routes/automation-dispatch.js";
+
+export const router = express.Router();
+
+const authManager = AuthManager.getInstance();
+const permissionManager = PermissionManager.getInstance();
+const authenticateJWT = authManager.createAuthMiddleware();
+const requireDataAccess = authManager.createDataAccessMiddleware();
+
+const TRIGGER_KINDS = new Set([
+  "metric_threshold",
+  "host_status",
+  "health_check",
+  "schedule",
+  "docker_event",
+  "internal_event",
+  "webhook",
+]);
+
+const STEP_TYPES = new Set([
+  "notify",
+  "http",
+  "run_snippet",
+  "run_command",
+  "docker",
+  "tunnel",
+  "wol",
+  "wait",
+  "set_var",
+  "if",
+  "run_automation",
+  "stop",
+]);
+
+const OPERATORS = new Set([
+  ">",
+  "<",
+  ">=",
+  "<=",
+  "==",
+  "!=",
+  "contains",
+  "not_contains",
+  "changed",
+]);
+
+function parseId(raw: unknown): number | null {
+  const id = typeof raw === "string" ? parseInt(raw, 10) : NaN;
+  return Number.isInteger(id) && id > 0 ? id : null;
+}
+
+function isNonEmptyString(value: unknown): value is string {
+  return typeof value === "string" && value.trim().length > 0;
+}
+
+/**
+ * Validates a definition before it is stored. The engine treats the stored
+ * blob as trusted, so everything it relies on is checked once here.
+ */
+export function validateDefinition(value: unknown): {
+  ok: boolean;
+  error?: string;
+  definition?: AutomationDefinition;
+} {
+  if (typeof value !== "object" || value === null) {
+    return { ok: false, error: "Definition must be an object" };
+  }
+
+  const candidate = value as Partial<AutomationDefinition>;
+  const trigger = candidate.trigger as Trigger | undefined;
+
+  if (!trigger || !TRIGGER_KINDS.has(trigger.kind)) {
+    return { ok: false, error: "Unknown or missing trigger kind" };
+  }
+
+  if (trigger.kind === "metric_threshold") {
+    if (!OPERATORS.has(trigger.operator)) {
+      return { ok: false, error: "Unknown comparison operator" };
+    }
+    if (typeof trigger.value !== "number" || !Number.isFinite(trigger.value)) {
+      return { ok: false, error: "Threshold value must be a number" };
+    }
+    if (!trigger.metric?.path) {
+      return { ok: false, error: "Trigger is missing a metric" };
+    }
+  }
+
+  if (trigger.kind === "schedule") {
+    const hasInterval =
+      typeof trigger.intervalSeconds === "number" &&
+      trigger.intervalSeconds > 0;
+    const hasCron = isNonEmptyString(trigger.cron);
+    if (!hasInterval && !hasCron) {
+      return { ok: false, error: "Schedule needs an interval or a cron" };
+    }
+    if (hasCron && !isValidCron(trigger.cron as string)) {
+      return { ok: false, error: "Cron expression is not valid" };
+    }
+    if (hasInterval && (trigger.intervalSeconds as number) < 60) {
+      return { ok: false, error: "Interval must be at least 60 seconds" };
+    }
+    if (
+      isNonEmptyString(trigger.timezone) &&
+      !isValidTimezone(trigger.timezone)
+    ) {
+      return { ok: false, error: "Time zone is not valid" };
+    }
+  }
+
+  const steps = candidate.steps;
+  if (!Array.isArray(steps)) {
+    return { ok: false, error: "Definition must include a steps array" };
+  }
+
+  const seen = new Set<string>();
+  const stepError = validateSteps(steps as Step[], seen);
+  if (stepError) return { ok: false, error: stepError };
+
+  return {
+    ok: true,
+    definition: {
+      version: candidate.version ?? AUTOMATION_DEFINITION_VERSION,
+      trigger,
+      steps: steps as Step[],
+    },
+  };
+}
+
+function validateSteps(steps: Step[], seen: Set<string>): string | null {
+  for (const step of steps) {
+    if (!step || typeof step !== "object") return "Step must be an object";
+    if (!isNonEmptyString(step.id)) return "Every step needs an id";
+    if (seen.has(step.id)) return `Duplicate step id: ${step.id}`;
+    seen.add(step.id);
+    if (!STEP_TYPES.has(step.type)) {
+      return `Unknown step type: ${step.type}`;
+    }
+
+    if (step.type === "if") {
+      if (!step.condition || !OPERATORS.has(step.condition.operator)) {
+        return "Condition needs a valid operator";
+      }
+      const thenError = validateSteps(step.then ?? [], seen);
+      if (thenError) return thenError;
+      const elseError = validateSteps(step.else ?? [], seen);
+      if (elseError) return elseError;
+    }
+
+    if (step.type === "http" && !isNonEmptyString(step.url)) {
+      return "HTTP steps need a URL";
+    }
+    if (step.type === "run_command" && !isNonEmptyString(step.command)) {
+      return "Command steps need a command";
+    }
+    if (step.type === "wait" && typeof step.seconds !== "number") {
+      return "Wait steps need a number of seconds";
+    }
+    if (step.type === "set_var" && !isNonEmptyString(step.name)) {
+      return "Variable steps need a name";
+    }
+  }
+  return null;
+}
+
+/** Never leak a webhook token hash to the client. */
+function serialize(row: AutomationRow) {
+  let definition: AutomationDefinition | null = null;
+  try {
+    definition = JSON.parse(row.definition) as AutomationDefinition;
+  } catch {
+    definition = null;
+  }
+
+  if (definition?.trigger?.kind === "webhook") {
+    definition = {
+      ...definition,
+      trigger: { ...definition.trigger, tokenHash: "" },
+    };
+  }
+
+  return { ...row, definition };
+}
+
+async function syncSchedule(
+  automationId: number,
+  definition: AutomationDefinition,
+): Promise<void> {
+  const repository = createCurrentAutomationRepository();
+  if (definition.trigger?.kind !== "schedule") {
+    await repository.deleteSchedule(automationId);
+    return;
+  }
+
+  const trigger = definition.trigger;
+  await repository.upsertSchedule({
+    automationId,
+    cron: trigger.cron ?? null,
+    intervalSeconds: trigger.intervalSeconds ?? null,
+    timezone: trigger.timezone ?? null,
+    nextDueAt: computeNextDueAt({
+      cron: trigger.cron,
+      intervalSeconds: trigger.intervalSeconds,
+      timezone: trigger.timezone,
+    }),
+  });
+}
+
+/**
+ * @openapi
+ * /automations:
+ *   get:
+ *     summary: List the current user's automations
+ *     description: Returns every automation the caller owns, with its parsed definition and linked notification channels.
+ *     tags:
+ *       - Automations
+ *     responses:
+ *       200:
+ *         description: List of automations.
+ *       403:
+ *         description: Missing the automations.view permission.
+ */
+router.get(
+  "/",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.view"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    try {
+      const rows = await createCurrentAutomationRepository().list(userId);
+      res.json(rows.map(serialize));
+    } catch (error) {
+      databaseLogger.error("Failed to list automations", error, {
+        operation: "automation_list_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to list automations" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/{id}:
+ *   get:
+ *     summary: Fetch a single automation
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: path
+ *         name: id
+ *         required: true
+ *         schema:
+ *           type: integer
+ *     responses:
+ *       200:
+ *         description: The automation.
+ *       404:
+ *         description: Automation not found.
+ */
+router.get(
+  "/:id",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.view"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    const id = parseId(req.params.id);
+    if (id === null) return res.status(400).json({ error: "Invalid id" });
+
+    try {
+      const row = await createCurrentAutomationRepository().findForUser(
+        id,
+        userId,
+      );
+      if (!row) return res.status(404).json({ error: "Automation not found" });
+      res.json(serialize(row));
+    } catch (error) {
+      databaseLogger.error("Failed to fetch automation", error, {
+        operation: "automation_get_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to fetch automation" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations:
+ *   post:
+ *     summary: Create an automation
+ *     description: Validates the trigger and every step before storing the definition. A schedule trigger also registers its next due time.
+ *     tags:
+ *       - Automations
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             type: object
+ *             properties:
+ *               name:
+ *                 type: string
+ *               definition:
+ *                 type: object
+ *     responses:
+ *       201:
+ *         description: The created automation.
+ *       400:
+ *         description: Validation failed.
+ */
+router.post(
+  "/",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.create"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    const { name, description, enabled, definition, concurrencyPolicy } =
+      req.body ?? {};
+
+    if (!isNonEmptyString(name)) {
+      return res.status(400).json({ error: "Name is required" });
+    }
+
+    const validated = validateDefinition(definition);
+    if (!validated.ok || !validated.definition) {
+      return res.status(400).json({ error: validated.error });
+    }
+
+    // A webhook trigger's token is shown once here and only stored hashed.
+    let webhookToken: string | undefined;
+    if (validated.definition.trigger.kind === "webhook") {
+      webhookToken = crypto.randomBytes(32).toString("hex");
+      validated.definition = {
+        ...validated.definition,
+        trigger: {
+          kind: "webhook",
+          tokenHash: hashToken(webhookToken),
+        },
+      };
+    }
+
+    try {
+      const repository = createCurrentAutomationRepository();
+      const created = await repository.create({
+        userId,
+        name: name.trim(),
+        description: description ?? null,
+        enabled: enabled !== false,
+        definition: JSON.stringify(validated.definition),
+        concurrencyPolicy,
+        channels: Array.isArray(req.body?.channels) ? req.body.channels : [],
+      });
+
+      await syncSchedule(created.id, validated.definition);
+
+      const { ipAddress, userAgent } = getRequestMeta(req);
+      await logAudit({
+        userId,
+        username: await getAuditUsername(userId),
+        action: "create_automation",
+        resourceType: "automation",
+        resourceId: String(created.id),
+        resourceName: created.name,
+        ipAddress,
+        userAgent,
+        success: true,
+      });
+
+      res.status(201).json({ ...serialize(created), webhookToken });
+    } catch (error) {
+      databaseLogger.error("Failed to create automation", error, {
+        operation: "automation_create_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to create automation" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/{id}:
+ *   put:
+ *     summary: Update an automation
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: path
+ *         name: id
+ *         required: true
+ *         schema:
+ *           type: integer
+ *     responses:
+ *       200:
+ *         description: The updated automation.
+ *       400:
+ *         description: Validation failed.
+ *       404:
+ *         description: Automation not found.
+ */
+router.put(
+  "/:id",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.edit"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    const id = parseId(req.params.id);
+    if (id === null) return res.status(400).json({ error: "Invalid id" });
+
+    const repository = createCurrentAutomationRepository();
+    const existing = await repository.findForUser(id, userId);
+    if (!existing) {
+      return res.status(404).json({ error: "Automation not found" });
+    }
+
+    const update: Record<string, unknown> = {};
+    if (req.body?.name !== undefined) {
+      if (!isNonEmptyString(req.body.name)) {
+        return res.status(400).json({ error: "Name cannot be empty" });
+      }
+      update.name = req.body.name.trim();
+    }
+    if (req.body?.description !== undefined) {
+      update.description = req.body.description;
+    }
+    if (req.body?.enabled !== undefined) update.enabled = !!req.body.enabled;
+    if (req.body?.concurrencyPolicy !== undefined) {
+      update.concurrencyPolicy = req.body.concurrencyPolicy;
+    }
+    if (Array.isArray(req.body?.channels)) update.channels = req.body.channels;
+
+    let parsedDefinition: AutomationDefinition | null = null;
+    if (req.body?.definition !== undefined) {
+      const validated = validateDefinition(req.body.definition);
+      if (!validated.ok || !validated.definition) {
+        return res.status(400).json({ error: validated.error });
+      }
+
+      // Keep the stored token hash: the raw token is only ever shown once.
+      if (validated.definition.trigger.kind === "webhook") {
+        const previous = JSON.parse(
+          existing.definition,
+        ) as AutomationDefinition;
+        const previousHash =
+          previous.trigger?.kind === "webhook"
+            ? previous.trigger.tokenHash
+            : "";
+        validated.definition = {
+          ...validated.definition,
+          trigger: { kind: "webhook", tokenHash: previousHash },
+        };
+      }
+
+      parsedDefinition = validated.definition;
+      update.definition = JSON.stringify(validated.definition);
+    }
+
+    try {
+      const updated = await repository.update(id, userId, update);
+      if (!updated) {
+        return res.status(404).json({ error: "Automation not found" });
+      }
+
+      if (parsedDefinition) await syncSchedule(id, parsedDefinition);
+
+      const { ipAddress, userAgent } = getRequestMeta(req);
+      await logAudit({
+        userId,
+        username: await getAuditUsername(userId),
+        action: "update_automation",
+        resourceType: "automation",
+        resourceId: String(id),
+        resourceName: updated.name,
+        ipAddress,
+        userAgent,
+        success: true,
+      });
+
+      res.json(serialize(updated));
+    } catch (error) {
+      databaseLogger.error("Failed to update automation", error, {
+        operation: "automation_update_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to update automation" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/{id}:
+ *   delete:
+ *     summary: Delete an automation
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: path
+ *         name: id
+ *         required: true
+ *         schema:
+ *           type: integer
+ *     responses:
+ *       200:
+ *         description: Deleted.
+ *       404:
+ *         description: Automation not found.
+ */
+router.delete(
+  "/:id",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.delete"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    const id = parseId(req.params.id);
+    if (id === null) return res.status(400).json({ error: "Invalid id" });
+
+    try {
+      const deleted = await createCurrentAutomationRepository().delete(
+        id,
+        userId,
+      );
+      if (!deleted) {
+        return res.status(404).json({ error: "Automation not found" });
+      }
+
+      const { ipAddress, userAgent } = getRequestMeta(req);
+      await logAudit({
+        userId,
+        username: await getAuditUsername(userId),
+        action: "delete_automation",
+        resourceType: "automation",
+        resourceId: String(id),
+        ipAddress,
+        userAgent,
+        success: true,
+      });
+
+      res.json({ success: true });
+    } catch (error) {
+      databaseLogger.error("Failed to delete automation", error, {
+        operation: "automation_delete_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to delete automation" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/{id}/run:
+ *   post:
+ *     summary: Run an automation now
+ *     description: Runs immediately, as the automation's owner. Pass dryRun to record what each step would do without touching anything outside Termix.
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: path
+ *         name: id
+ *         required: true
+ *         schema:
+ *           type: integer
+ *     requestBody:
+ *       content:
+ *         application/json:
+ *           schema:
+ *             type: object
+ *             properties:
+ *               dryRun:
+ *                 type: boolean
+ *     responses:
+ *       200:
+ *         description: The run outcome.
+ *       404:
+ *         description: Automation not found.
+ */
+router.post(
+  "/:id/run",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.run"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    const id = parseId(req.params.id);
+    if (id === null) return res.status(400).json({ error: "Invalid id" });
+
+    const existing = await createCurrentAutomationRepository().findForUser(
+      id,
+      userId,
+    );
+    if (!existing) {
+      return res.status(404).json({ error: "Automation not found" });
+    }
+
+    try {
+      const outcome = await AutomationEngine.getInstance().run({
+        automationId: id,
+        triggerType: "manual",
+        triggerContext: { manual: true, requestedBy: userId },
+        dryRun: req.body?.dryRun === true,
+      });
+
+      const { ipAddress, userAgent } = getRequestMeta(req);
+      await logAudit({
+        userId,
+        username: await getAuditUsername(userId),
+        action: "run_automation",
+        resourceType: "automation",
+        resourceId: String(id),
+        resourceName: existing.name,
+        details: JSON.stringify({
+          status: outcome.status,
+          dryRun: req.body?.dryRun === true,
+        }),
+        ipAddress,
+        userAgent,
+        success: outcome.status === "success",
+      });
+
+      res.json(outcome);
+    } catch (error) {
+      databaseLogger.error("Failed to run automation", error, {
+        operation: "automation_run_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to run automation" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/runs:
+ *   get:
+ *     summary: List automation runs
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: query
+ *         name: automationId
+ *         schema:
+ *           type: integer
+ *       - in: query
+ *         name: limit
+ *         schema:
+ *           type: integer
+ *     responses:
+ *       200:
+ *         description: Recent runs, newest first.
+ */
+router.get(
+  "/runs/history",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.view"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    try {
+      const runs = await createCurrentAutomationRepository().listRuns(userId, {
+        automationId: parseId(req.query.automationId) ?? undefined,
+        limit: Number(req.query.limit) || 50,
+        offset: Number(req.query.offset) || 0,
+      });
+      res.json(runs);
+    } catch (error) {
+      databaseLogger.error("Failed to list automation runs", error, {
+        operation: "automation_runs_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to list runs" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/runs/{runId}/steps:
+ *   get:
+ *     summary: Step-by-step results for a run
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: path
+ *         name: runId
+ *         required: true
+ *         schema:
+ *           type: integer
+ *     responses:
+ *       200:
+ *         description: The run's steps in order.
+ *       404:
+ *         description: Run not found.
+ */
+router.get(
+  "/runs/:runId/steps",
+  authenticateJWT,
+  permissionManager.requirePermission("automations.view"),
+  requireDataAccess,
+  async (req: Request, res: Response) => {
+    const userId = (req as AuthenticatedRequest).userId;
+    const runId = parseId(req.params.runId);
+    if (runId === null) return res.status(400).json({ error: "Invalid id" });
+
+    try {
+      const repository = createCurrentAutomationRepository();
+      const run = await repository.findRunForUser(runId, userId);
+      if (!run) return res.status(404).json({ error: "Run not found" });
+
+      res.json(await repository.listRunSteps(runId));
+    } catch (error) {
+      databaseLogger.error("Failed to list run steps", error, {
+        operation: "automation_run_steps_error",
+        userId,
+      });
+      res.status(500).json({ error: "Failed to list run steps" });
+    }
+  },
+);
+
+/**
+ * @openapi
+ * /automations/webhook/{token}:
+ *   post:
+ *     summary: Trigger an automation from an external system
+ *     description: Unauthenticated by design; the 32-byte token in the path is the credential and is compared against a stored hash in constant time.
+ *     tags:
+ *       - Automations
+ *     parameters:
+ *       - in: path
+ *         name: token
+ *         required: true
+ *         schema:
+ *           type: string
+ *     responses:
+ *       202:
+ *         description: The run was accepted.
+ *       404:
+ *         description: No automation matches that token.
+ */
+router.post("/webhook/:token", async (req: Request, res: Response) => {
+  const token = req.params.token;
+  if (!isNonEmptyString(token) || token.length < 32) {
+    return res.status(404).json({ error: "Not found" });
+  }
+
+  try {
+    const repository = createCurrentAutomationRepository();
+    const candidates = await repository.listAllEnabled();
+    const wanted = hashToken(token);
+
+    const match = candidates.find((row) => {
+      try {
+        const definition = JSON.parse(row.definition) as AutomationDefinition;
+        if (definition.trigger?.kind !== "webhook") return false;
+        return timingSafeEqual(definition.trigger.tokenHash, wanted);
+      } catch {
+        return false;
+      }
+    });
+
+    if (!match) return res.status(404).json({ error: "Not found" });
+
+    const outcome = await AutomationEngine.getInstance().run({
+      automationId: match.id,
+      triggerType: "webhook",
+      triggerContext: {
+        body: req.body ?? {},
+        receivedAt: new Date().toISOString(),
+      },
+    });
+
+    res.status(202).json({ runId: outcome.runId, status: outcome.status });
+  } catch (error) {
+    databaseLogger.error("Webhook automation failed", error, {
+      operation: "automation_webhook_error",
+    });
+    res.status(500).json({ error: "Failed to run automation" });
+  }
+});
+
+function hashToken(token: string): string {
+  return crypto.createHash("sha256").update(token).digest("hex");
+}
+
+function timingSafeEqual(a: string, b: string): boolean {
+  const left = Buffer.from(a || "", "utf8");
+  const right = Buffer.from(b || "", "utf8");
+  if (left.length !== right.length) return false;
+  return crypto.timingSafeEqual(left, right);
+}
+
+/** Called from activate(). Mounts this router at /automations via the shared
+ * dispatcher. */
+export function startAutomationsService(): void {
+  registerAutomationsRouter(router);
+}
+
+/** Called from deactivate(). /automations/* falls back to 404 until
+ * reactivated. */
+export function stopAutomationsService(): void {
+  unregisterAutomationsRouter();
+}

plugins/automations/src/backend/scheduler.ts

@@ -0,0 +1,216 @@
+import type { AutomationDefinition } from "../../../../src/types/automations.js";
+import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
+import { DataCrypto } from "../../../../src/backend/utils/data-crypto.js";
+import { statsLogger } from "../../../../src/backend/utils/logger.js";
+import { computeNextDueAt } from "./cron.js";
+import { hasDwelled, isCoolingDown } from "./conditions.js";
+import { pollDockerEvents } from "./docker-watcher.js";
+import { AutomationEngine } from "./engine.js";
+import { reconcileHeadlessViewers } from "./headless-viewer.js";
+import {
+  subscribeAutomationTriggers,
+  unsubscribeAutomationTriggers,
+} from "./triggers.js";
+
+/**
+ * The one timer the automations feature owns.
+ *
+ * Every other periodic job in the backend is its own module-level setInterval;
+ * this deliberately is not one per automation. A single tick handles due
+ * schedules, dwell windows that need re-checking without a fresh sample, the
+ * synthetic viewer heartbeat, and history pruning.
+ */
+
+const TICK_MS = 15_000;
+const STARTUP_DELAY_MS = 30_000;
+const PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
+const RUN_RETENTION_DAYS = 30;
+/** A "running" row older than this belongs to a process that is gone. */
+const STALE_RUN_MS = 6 * 60 * 60 * 1000;
+
+let tickTimer: NodeJS.Timeout | null = null;
+let startupTimer: NodeJS.Timeout | null = null;
+let lastPruneAt = 0;
+let ticking = false;
+
+export function startAutomationScheduler(): void {
+  if (tickTimer) return;
+
+  // Event-driven triggers come through the bus; the interval below only drives
+  // the schedule-based ones.
+  subscribeAutomationTriggers();
+
+  startupTimer = setTimeout(() => {
+    void tick();
+  }, STARTUP_DELAY_MS);
+  startupTimer.unref?.();
+
+  tickTimer = setInterval(() => {
+    void tick();
+  }, TICK_MS);
+  tickTimer.unref?.();
+}
+
+export function stopAutomationScheduler(): void {
+  if (tickTimer) clearInterval(tickTimer);
+  if (startupTimer) clearTimeout(startupTimer);
+  tickTimer = null;
+  startupTimer = null;
+  unsubscribeAutomationTriggers();
+}
+
+/** Exposed for tests; the interval calls this. */
+export async function tick(now: Date = new Date()): Promise<void> {
+  // A slow tick must not overlap the next one.
+  if (ticking) return;
+  ticking = true;
+
+  try {
+    await reconcileHeadlessViewers().catch(() => undefined);
+    await runDueSchedules(now);
+    await recheckOpenBreaches(now);
+    await pollDockerEvents(now.getTime()).catch(() => undefined);
+    await pruneIfDue(now);
+  } catch (error) {
+    statsLogger.warn("Automation scheduler tick failed", {
+      operation: "automation_scheduler_tick_error",
+      error: error instanceof Error ? error.message : String(error),
+    });
+  } finally {
+    ticking = false;
+  }
+}
+
+async function runDueSchedules(now: Date): Promise<void> {
+  const repository = createCurrentAutomationRepository();
+  const due = await repository.listDueSchedules(now.toISOString());
+
+  for (const schedule of due) {
+    const automation = await repository.findById(schedule.automationId);
+    if (!automation) continue;
+
+    // Background work can only touch a user's data while their key resolves.
+    if (!canAccess(automation.userId)) {
+      statsLogger.warn("Skipping scheduled automation: data key unavailable", {
+        operation: "automation_schedule_locked",
+        automationId: automation.id,
+      });
+      continue;
+    }
+
+    const nextDueAt = computeNextDueAt(
+      {
+        cron: schedule.cron,
+        intervalSeconds: schedule.intervalSeconds,
+        timezone: schedule.timezone,
+      },
+      now,
+    );
+    await repository.markScheduleTicked(
+      schedule.automationId,
+      nextDueAt,
+      now.toISOString(),
+    );
+
+    AutomationEngine.getInstance()
+      .run({
+        automationId: schedule.automationId,
+        triggerType: "schedule",
+        triggerContext: { scheduledFor: now.toISOString() },
+      })
+      .catch((error) => {
+        statsLogger.warn("Scheduled automation failed to start", {
+          operation: "automation_schedule_error",
+          automationId: schedule.automationId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      });
+  }
+}
+
+/**
+ * Fires sustained breaches whose window has elapsed.
+ *
+ * Without this a dwell window only completes when another sample happens to
+ * arrive, so a breach that starts just before polling stops would never fire.
+ */
+async function recheckOpenBreaches(now: Date): Promise<void> {
+  const repository = createCurrentAutomationRepository();
+  const open = await repository.listOpenBreaches();
+  const nowMs = now.getTime();
+
+  for (const state of open) {
+    const automation = await repository.findById(state.automationId);
+    if (!automation || !automation.enabled) continue;
+    if (!canAccess(automation.userId)) continue;
+
+    let definition: AutomationDefinition;
+    try {
+      definition = JSON.parse(automation.definition) as AutomationDefinition;
+    } catch {
+      continue;
+    }
+
+    const trigger = definition.trigger;
+    if (trigger?.kind !== "metric_threshold") continue;
+    if (!trigger.forSeconds) continue;
+    if (!hasDwelled(state.breachStartedAt, trigger.forSeconds, nowMs)) continue;
+    if (isCoolingDown(state.lastFiredAt, trigger.cooldownMinutes, nowMs)) {
+      continue;
+    }
+
+    const hostId = Number(state.stateKey.split(":")[0]);
+    await repository.upsertTriggerState({
+      automationId: automation.id,
+      stateKey: state.stateKey,
+      lastFiredAt: now.toISOString(),
+    });
+
+    AutomationEngine.getInstance()
+      .run({
+        automationId: automation.id,
+        triggerType: "metric_threshold",
+        triggerContext: {
+          hostId,
+          value: state.lastValue,
+          threshold: trigger.value,
+          metric: trigger.metric.path,
+          sustained: true,
+        },
+        triggerHostId: Number.isFinite(hostId) ? hostId : undefined,
+      })
+      .catch(() => undefined);
+  }
+}
+
+async function pruneIfDue(now: Date): Promise<void> {
+  if (now.getTime() - lastPruneAt < PRUNE_INTERVAL_MS) return;
+  lastPruneAt = now.getTime();
+
+  const repository = createCurrentAutomationRepository();
+  try {
+    await repository.failStaleRunningRuns(
+      new Date(now.getTime() - STALE_RUN_MS).toISOString(),
+    );
+    const deleted = await repository.pruneRunsOlderThan(RUN_RETENTION_DAYS);
+    if (deleted > 0) {
+      statsLogger.info(`Pruned ${deleted} old automation run(s)`, {
+        operation: "automation_run_prune",
+        deleted,
+      });
+    }
+  } catch (error) {
+    statsLogger.warn("Automation run pruning failed", {
+      operation: "automation_run_prune_error",
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+function canAccess(userId: string): boolean {
+  try {
+    return DataCrypto.canUserAccessData(userId);
+  } catch {
+    return false;
+  }
+}

plugins/automations/src/backend/template.ts

@@ -0,0 +1,101 @@
+/**
+ * Variable substitution for automation steps.
+ *
+ * Templates read from the run context: {{host.name}}, {{trigger.value}},
+ * {{steps.<stepId>.stdout}}, {{vars.myVar}}. Resolution always produces a
+ * plain string and never shell syntax; callers that build a command are
+ * responsible for quoting the result (see shellSingleQuote in
+ * hosts/metrics/managers/exec-elevated.ts). Nothing here escapes anything,
+ * precisely so there is one obvious place where quoting happens.
+ */
+
+export interface TemplateContext {
+  host?: {
+    id?: number;
+    name?: string;
+    ip?: string;
+    username?: string;
+    port?: number;
+  };
+  trigger?: Record<string, unknown>;
+  steps?: Record<string, { stdout?: string; stderr?: string; code?: number }>;
+  vars?: Record<string, string>;
+  run?: { id?: number; automationId?: number; startedAt?: string };
+}
+
+const TOKEN = /\{\{\s*([a-zA-Z0-9_.-]+)\s*\}\}/g;
+
+function readPath(context: TemplateContext, path: string): unknown {
+  const parts = path.split(".");
+  let current: unknown = context;
+
+  for (const part of parts) {
+    if (current === null || current === undefined) return undefined;
+    if (typeof current !== "object") return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}
+
+function stringify(value: unknown): string {
+  if (value === null || value === undefined) return "";
+  if (typeof value === "string") return value;
+  if (typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+  return JSON.stringify(value);
+}
+
+/**
+ * Replaces every {{token}} it can resolve. An unresolvable token is left
+ * as-is so a typo shows up in the run output rather than silently becoming an
+ * empty string, which is the difference between a visible mistake and a
+ * command that quietly does the wrong thing.
+ */
+export function renderTemplate(
+  input: string,
+  context: TemplateContext,
+): string {
+  if (!input || !input.includes("{{")) return input;
+
+  return input.replace(TOKEN, (match, path: string) => {
+    const value = readPath(context, path);
+    return value === undefined ? match : stringify(value);
+  });
+}
+
+/** Renders every string in a flat record, leaving keys untouched. */
+export function renderRecord(
+  input: Record<string, string> | undefined,
+  context: TemplateContext,
+): Record<string, string> | undefined {
+  if (!input) return undefined;
+  const output: Record<string, string> = {};
+  for (const [key, value] of Object.entries(input)) {
+    output[key] = renderTemplate(value, context);
+  }
+  return output;
+}
+
+/** True when a template still has unresolved tokens after rendering. */
+export function hasUnresolvedTokens(rendered: string): boolean {
+  TOKEN.lastIndex = 0;
+  return TOKEN.test(rendered);
+}
+
+const SECRET_KEY = /(authorization|token|password|secret|api[-_]?key|cookie)/i;
+
+/**
+ * Masks values whose key looks like a credential, for anything written to run
+ * history or returned by the API.
+ */
+export function redactSecrets(
+  input: Record<string, string> | undefined,
+): Record<string, string> | undefined {
+  if (!input) return undefined;
+  const output: Record<string, string> = {};
+  for (const [key, value] of Object.entries(input)) {
+    output[key] = SECRET_KEY.test(key) ? "***" : value;
+  }
+  return output;
+}

plugins/automations/src/backend/triggers.ts

@@ -0,0 +1,428 @@
+import type {
+  AutomationDefinition,
+  HostSelector,
+  Trigger,
+} from "../../../../src/types/automations.js";
+import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
+import type { AutomationEngineRow } from "../../../../src/backend/database/repositories/automation-repository.js";
+import { statsLogger } from "../../../../src/backend/utils/logger.js";
+import {
+  compare,
+  extractMetricValue,
+  hasDwelled,
+  isCoolingDown,
+  metricStateKey,
+  severityForValue,
+  type MetricsSnapshot,
+} from "./conditions.js";
+import { AutomationEngine } from "./engine.js";
+import {
+  pluginEvents,
+  TOPICS,
+} from "../../../../src/backend/plugins/events.js";
+
+/**
+ * Matches events against automation triggers and decides what fires.
+ *
+ * Dwell windows and cooldowns live in automation_trigger_state rather than in
+ * memory, so a restart mid-breach neither loses the window nor re-fires an
+ * alert that already went out.
+ */
+
+export interface MetricEvent {
+  hostId: number;
+  ownerUserId: string;
+  metrics: MetricsSnapshot;
+}
+
+export interface StatusEvent {
+  hostId: number;
+  ownerUserId: string;
+  online: boolean;
+}
+
+export interface HealthEvent {
+  hostId: number;
+  userId: string;
+  checkId: string;
+  ok: boolean;
+  detail?: string;
+}
+
+export interface DockerEvent {
+  hostId: number;
+  ownerUserId: string;
+  container: string;
+  event: "exited" | "started" | "unhealthy" | "restarting";
+}
+
+export interface InternalEvent {
+  event: string;
+  userId: string;
+  hostId?: number;
+  details?: Record<string, unknown>;
+}
+
+interface LoadedAutomation {
+  row: AutomationEngineRow;
+  definition: AutomationDefinition;
+}
+
+async function loadEnabledFor(userId: string): Promise<LoadedAutomation[]> {
+  try {
+    const rows =
+      await createCurrentAutomationRepository().listEnabledForUser(userId);
+    const loaded: LoadedAutomation[] = [];
+    for (const row of rows) {
+      try {
+        loaded.push({
+          row,
+          definition: JSON.parse(row.definition) as AutomationDefinition,
+        });
+      } catch {
+        // A malformed definition should not stop the others from evaluating.
+      }
+    }
+    return loaded;
+  } catch {
+    return [];
+  }
+}
+
+/** Whether a selector covers a host. Ownership is checked by the caller. */
+function selectorCoversHost(
+  selector: HostSelector | undefined,
+  hostId: number,
+): boolean {
+  if (!selector) return true;
+  switch (selector.kind) {
+    case "all":
+    case "trigger":
+      return true;
+    case "host":
+      return selector.hostId === hostId;
+    case "hosts":
+      return selector.hostIds.includes(hostId);
+    case "fleet":
+      // Fleet membership is resolved at execution time; evaluate optimistically
+      // so a fleet-scoped trigger still reaches the engine.
+      return true;
+    default:
+      return false;
+  }
+}
+
+async function fire(
+  automation: AutomationEngineRow,
+  stateKey: string,
+  triggerType: string,
+  triggerContext: Record<string, unknown>,
+  hostId?: number,
+): Promise<void> {
+  const repository = createCurrentAutomationRepository();
+  await repository.upsertTriggerState({
+    automationId: automation.id,
+    stateKey,
+    lastFiredAt: new Date().toISOString(),
+  });
+
+  AutomationEngine.getInstance()
+    .run({
+      automationId: automation.id,
+      triggerType,
+      triggerContext,
+      triggerHostId: hostId,
+    })
+    .catch((error) => {
+      statsLogger.warn("Automation run failed to start", {
+        operation: "automation_trigger_error",
+        automationId: automation.id,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    });
+}
+
+/**
+ * Metric samples. Called for every polled host, including hosts polled only
+ * because an automation asked for them.
+ */
+export async function onMetrics(event: MetricEvent): Promise<void> {
+  const automations = await loadEnabledFor(event.ownerUserId);
+  const repository = createCurrentAutomationRepository();
+  const now = Date.now();
+
+  for (const { row, definition } of automations) {
+    const trigger = definition.trigger;
+    if (trigger?.kind !== "metric_threshold") continue;
+    if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
+
+    const value = extractMetricValue(event.metrics, trigger.metric);
+    if (value === null) continue;
+
+    const stateKey = metricStateKey(event.hostId, trigger.metric);
+    const state = await repository.getTriggerState(row.id, stateKey);
+    const breaching = compare(value, trigger.operator, trigger.value);
+
+    if (!breaching) {
+      if (state?.breachStartedAt) {
+        await repository.clearBreach(row.id, stateKey);
+      }
+      continue;
+    }
+
+    // Open the dwell window on the first breaching sample.
+    if (!state?.breachStartedAt) {
+      await repository.upsertTriggerState({
+        automationId: row.id,
+        stateKey,
+        breachStartedAt: new Date(now).toISOString(),
+        lastValue: value,
+      });
+      if (trigger.forSeconds) continue;
+    }
+
+    const breachStartedAt =
+      state?.breachStartedAt ?? new Date(now).toISOString();
+    if (!hasDwelled(breachStartedAt, trigger.forSeconds, now)) continue;
+    if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
+      continue;
+
+    await fire(
+      row,
+      stateKey,
+      "metric_threshold",
+      {
+        hostId: event.hostId,
+        value,
+        threshold: trigger.value,
+        operator: trigger.operator,
+        metric: trigger.metric.path,
+        mount: "mount" in trigger.metric ? trigger.metric.mount : undefined,
+        severity: severityForValue(value, trigger.severity),
+      },
+      event.hostId,
+    );
+  }
+}
+
+/** Host reachability transitions. Only edges fire, never steady state. */
+export async function onStatus(event: StatusEvent): Promise<void> {
+  const automations = await loadEnabledFor(event.ownerUserId);
+  const repository = createCurrentAutomationRepository();
+  const now = Date.now();
+  const observed = event.online ? "online" : "offline";
+
+  for (const { row, definition } of automations) {
+    const trigger = definition.trigger;
+    if (trigger?.kind !== "host_status") continue;
+    if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
+
+    const stateKey = String(event.hostId);
+    const state = await repository.getTriggerState(row.id, stateKey);
+
+    if (state?.lastObservedState === observed) continue;
+
+    await repository.upsertTriggerState({
+      automationId: row.id,
+      stateKey,
+      lastObservedState: observed,
+    });
+
+    // The first observation establishes a baseline rather than firing, so a
+    // restart does not announce every host as though it just changed.
+    if (!state?.lastObservedState) continue;
+    if (trigger.to !== observed) continue;
+    if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
+      continue;
+
+    await fire(
+      row,
+      stateKey,
+      "host_status",
+      { hostId: event.hostId, status: observed },
+      event.hostId,
+    );
+  }
+}
+
+export async function onHealthCheck(event: HealthEvent): Promise<void> {
+  const automations = await loadEnabledFor(event.userId);
+  const repository = createCurrentAutomationRepository();
+  const now = Date.now();
+  const observed = event.ok ? "recovered" : "failing";
+
+  for (const { row, definition } of automations) {
+    const trigger = definition.trigger;
+    if (trigger?.kind !== "health_check") continue;
+    if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
+    if (trigger.checkId && trigger.checkId !== event.checkId) continue;
+
+    const stateKey = `${event.hostId}:${event.checkId}`;
+    const state = await repository.getTriggerState(row.id, stateKey);
+
+    if (state?.lastObservedState === observed) continue;
+
+    await repository.upsertTriggerState({
+      automationId: row.id,
+      stateKey,
+      lastObservedState: observed,
+    });
+
+    if (!state?.lastObservedState) continue;
+    if (trigger.to !== observed) continue;
+    if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
+      continue;
+
+    await fire(
+      row,
+      stateKey,
+      "health_check",
+      {
+        hostId: event.hostId,
+        checkId: event.checkId,
+        state: observed,
+        detail: event.detail,
+      },
+      event.hostId,
+    );
+  }
+}
+
+export async function onDockerEvent(event: DockerEvent): Promise<void> {
+  const automations = await loadEnabledFor(event.ownerUserId);
+  const repository = createCurrentAutomationRepository();
+  const now = Date.now();
+
+  for (const { row, definition } of automations) {
+    const trigger = definition.trigger;
+    if (trigger?.kind !== "docker_event") continue;
+    if (!selectorCoversHost(trigger.hostSelector, event.hostId)) continue;
+    if (trigger.container && trigger.container !== event.container) continue;
+    if (trigger.event !== event.event) continue;
+
+    const stateKey = `${event.hostId}:${event.container}`;
+    const state = await repository.getTriggerState(row.id, stateKey);
+    if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
+      continue;
+
+    await fire(
+      row,
+      stateKey,
+      "docker_event",
+      {
+        hostId: event.hostId,
+        container: event.container,
+        event: event.event,
+      },
+      event.hostId,
+    );
+  }
+}
+
+export async function onInternalEvent(event: InternalEvent): Promise<void> {
+  const automations = await loadEnabledFor(event.userId);
+  const repository = createCurrentAutomationRepository();
+  const now = Date.now();
+
+  for (const { row, definition } of automations) {
+    const trigger = definition.trigger;
+    if (trigger?.kind !== "internal_event") continue;
+    if (trigger.event !== event.event) continue;
+    if (
+      event.hostId !== undefined &&
+      !selectorCoversHost(trigger.hostSelector, event.hostId)
+    ) {
+      continue;
+    }
+
+    const stateKey = event.hostId ? String(event.hostId) : "global";
+    const state = await repository.getTriggerState(row.id, stateKey);
+    if (isCoolingDown(state?.lastFiredAt, trigger.cooldownMinutes, now))
+      continue;
+
+    await fire(
+      row,
+      stateKey,
+      "internal_event",
+      { event: event.event, hostId: event.hostId, ...(event.details ?? {}) },
+      event.hostId,
+    );
+  }
+}
+
+let unsubscribers: Array<() => void> = [];
+
+/**
+ * Subscribes the engine to the event bus.
+ *
+ * The hosts modules publish onto the bus and know nothing about automations;
+ * this is the other half of that arrangement, and it is what keeps the edge
+ * one-directional (automations may import repositories, hosts modules must not
+ * import automations).
+ *
+ * Handlers are async and the bus is fire-and-forget, so a rejection here is
+ * logged by the bus rather than surfacing to whichever host feature emitted.
+ */
+export function subscribeAutomationTriggers(): void {
+  if (unsubscribers.length > 0) return;
+
+  unsubscribers = [
+    pluginEvents.on(TOPICS.hostMetrics, (payload) =>
+      onMetrics(payload as MetricEvent),
+    ),
+    pluginEvents.on(TOPICS.hostStatus, (payload) =>
+      onStatus(payload as StatusEvent),
+    ),
+    pluginEvents.on(TOPICS.hostHealthCheck, (payload) =>
+      onHealthCheck(payload as HealthEvent),
+    ),
+    pluginEvents.on(TOPICS.internalEvent, (payload) =>
+      onInternalEvent(payload as InternalEvent),
+    ),
+  ];
+}
+
+export function unsubscribeAutomationTriggers(): void {
+  for (const unsubscribe of unsubscribers) unsubscribe();
+  unsubscribers = [];
+}
+
+/**
+ * Hosts that an enabled automation watches, so the poller knows to collect
+ * metrics for them even when nobody is looking.
+ */
+export async function listAutomationWatchedHosts(): Promise<
+  Map<number, string>
+> {
+  const watched = new Map<number, string>();
+
+  try {
+    const rows = await createCurrentAutomationRepository().listAllEnabled();
+    for (const row of rows) {
+      let definition: AutomationDefinition;
+      try {
+        definition = JSON.parse(row.definition) as AutomationDefinition;
+      } catch {
+        continue;
+      }
+
+      const trigger: Trigger | undefined = definition.trigger;
+      // Only metric thresholds need heavy collection; status triggers are
+      // already served by the cheap reachability probe.
+      if (trigger?.kind !== "metric_threshold") continue;
+
+      const selector = trigger.hostSelector;
+      if (selector?.kind === "host") {
+        watched.set(selector.hostId, row.userId);
+      } else if (selector?.kind === "hosts") {
+        for (const hostId of selector.hostIds) watched.set(hostId, row.userId);
+      }
+      // Fleet and "all" selectors are resolved by the scheduler, which can
+      // expand them without blocking this call.
+    }
+  } catch {
+    return watched;
+  }
+
+  return watched;
+}