feat: remove legacy alert rules, superseded by automations

Alert rules already auto-migrate into automations on boot and the old
engine stands down after. Deletes the rule/firing tables, routes, UI,
and AI tools; keeps notification channels since automations still use
them. Also fixes automations' notify step, which still called the
deleted repository.
This commit is contained in:
LukeGus committed 2026-09-22 10:45:12 -05:00
1 parent e0f14f2ecf
commit e372b0eda7
101 files changed
+29118 -5539

No files matched your search

+2 -2
View File
@@ -13,7 +13,7 @@ export function buildSystemPrompt(options: {
}): string {
const lines: string[] = [
"You are the assistant built into Termix, a self-hosted server management app.",
"You help the user manage their servers, snippets, automations, fleets and alerts.",
"You help the user manage their servers, snippets, automations and fleets.",
"",
"How you work:",
"- You start with no knowledge of this user's setup. Call a read tool to find out anything you need.",
@@ -24,7 +24,7 @@ export function buildSystemPrompt(options: {
"- Do exactly what was asked, and nothing beyond it. A question is a request for an answer, not for changes.",
"- Questions like 'what is running on this server', 'check this host' or 'why is this slow' are answered with information. Read, then report. Do not propose anything.",
"- Only propose a change when the user asked for one, in words like 'add', 'create', 'set up', 'fix' or 'change'.",
"- Do not propose follow-up work you thought of yourself: no monitoring, no alert rules, no scripts, no snippets, no cleanup, unless that is what was requested.",
"- Do not propose follow-up work you thought of yourself: no monitoring automations, no scripts, no snippets, no cleanup, unless that is what was requested.",
"- If you think something is worth doing, say so in one sentence and stop. Let the user ask.",
"- One request means one proposal at most. Do not bundle extras alongside it.",
"",
+2 -2
View File
@@ -4,11 +4,11 @@
* This plugin runs on the main thread rather than in a worker for the same
* reason ssh-terminal, docker and host-metrics do (see
* src/backend/plugins/first-party.ts): it writes directly across hosts,
* snippets, fleets, alert rules and automations through their repositories,
* snippets, fleets and automations through their repositories,
* runs approved commands over the shared SSH pool via resolveHostById and
* withConnection, and streams its chat replies as server-sent events. None
* of that fits the worker ctx -- ctx.hosts is a 13-field read-only view with
* no write methods and no snippets/fleets/alerts/automations surface at all,
* no write methods and no snippets/fleets/automations surface at all,
* and ctx.http.route replies with a single postMessage value rather than a
* stream.
*
-30
View File
@@ -1,5 +1,4 @@
import {
createCurrentAlertRepository,
createCurrentAutomationRepository,
createCurrentFleetRepository,
createCurrentHostRepository,
@@ -217,35 +216,6 @@ export async function applyProposal(
};
}
case "propose_create_alert_rule": {
const created = await createCurrentAlertRepository().createAlertRule({
userId,
name: requireString(payload.name, "name"),
hostId:
payload.hostId === null || payload.hostId === undefined
? null
: requireNumber(payload.hostId, "hostId"),
enabled: true,
triggerType: requireString(payload.triggerType, "triggerType"),
thresholdValue:
payload.thresholdValue === null ||
payload.thresholdValue === undefined
? null
: Number(payload.thresholdValue),
thresholdDurationSeconds:
payload.thresholdDurationSeconds === null ||
payload.thresholdDurationSeconds === undefined
? null
: Number(payload.thresholdDurationSeconds),
cooldownMinutes: Number(payload.cooldownMinutes) || 15,
channelIds: [],
} as any);
return {
ok: true,
summary: `Created alert rule ${(created as any)?.name ?? ""}`.trim(),
};
}
case "propose_create_automation": {
// Reuses the same validator the automations route runs, so an
// LLM-authored definition is held to exactly the human standard.
-34
View File
@@ -216,40 +216,6 @@ export const proposeTools: AiTool[] = [
hostIds: args.hostIds ?? [],
}),
},
{
name: "propose_create_alert_rule",
description:
"Propose a new alert rule that fires when a host metric crosses a threshold.",
category: "propose",
parameters: objectSchema(
{
name: str("Rule name"),
hostId: num("Host id to watch, or omit to watch all hosts"),
triggerType: str(
"What to watch, for example cpu, memory, disk or host_status",
),
thresholdValue: num("Threshold to compare against"),
thresholdDurationSeconds: num(
"How long the breach must persist before firing",
),
cooldownMinutes: num("Minimum minutes between repeat firings"),
},
["name", "triggerType"],
),
handler: async (args) =>
proposal(
"propose_create_alert_rule",
`Create alert rule ${String(args.name)}`,
{
name: args.name,
hostId: args.hostId ?? null,
triggerType: args.triggerType,
thresholdValue: args.thresholdValue ?? null,
thresholdDurationSeconds: args.thresholdDurationSeconds ?? null,
cooldownMinutes: args.cooldownMinutes ?? 15,
},
),
},
{
name: "propose_run_command",
description:
+2 -55
View File
@@ -1,5 +1,5 @@
import {
createCurrentAlertRepository,
createCurrentNotificationChannelRepository,
createCurrentAutomationRepository,
createCurrentCommandHistoryRepository,
createCurrentFleetRepository,
@@ -190,30 +190,6 @@ export const readTools: AiTool[] = [
};
},
},
{
name: "list_alert_rules",
description:
"List the user's alert rules with their thresholds and enabled state.",
category: "read",
parameters: objectSchema({}),
handler: async (_args, context) => {
const rules = await createCurrentAlertRepository().listAlertRules(
context.userId,
);
return {
rules: rules.map((rule) => ({
id: rule.id,
name: rule.name,
hostId: rule.host_id,
enabled: rule.enabled === 1,
triggerType: rule.trigger_type,
thresholdValue: rule.threshold_value,
thresholdDurationSeconds: rule.threshold_duration_seconds,
cooldownMinutes: rule.cooldown_minutes,
})),
};
},
},
{
name: "list_notification_channels",
description:
@@ -222,7 +198,7 @@ export const readTools: AiTool[] = [
parameters: objectSchema({}),
handler: async (_args, context) => {
const channels =
await createCurrentAlertRepository().listNotificationChannels(
await createCurrentNotificationChannelRepository().listNotificationChannels(
context.userId,
);
return {
@@ -235,35 +211,6 @@ export const readTools: AiTool[] = [
};
},
},
{
name: "get_alert_firings",
description:
"Recent alert firings, newest first. Use this to answer questions about what has been alerting.",
category: "read",
parameters: objectSchema({
limit: num("How many firings to return (default 25, max 100)"),
}),
handler: async (args, context) => {
const limit = Math.min(Math.max(Number(args.limit) || 25, 1), 100);
const result = await createCurrentAlertRepository().listAlertFirings({
userId: context.userId,
limit,
offset: 0,
});
return {
firings: (result.firings ?? []).map((firing) => ({
id: firing.id,
ruleName: firing.rule_name,
hostName: firing.host_name,
firedAt: firing.fired_at,
resolvedAt: firing.resolved_at,
severity: firing.severity,
message: firing.message,
acknowledged: firing.acknowledged === 1,
})),
};
},
},
{
name: "list_homepage_items",
description: "List the user's homepage service-link tiles.",
+2 -2
View File
@@ -14,7 +14,7 @@ import {
import { withConnection } from "../../../../src/backend/hosts/ssh-connection-pool.js";
import { resolveSnippetCommand } from "../../../../src/backend/database/routes/snippets-execution.js";
import {
createCurrentAlertRepository,
createCurrentNotificationChannelRepository,
createCurrentSnippetRepository,
} from "../../../../src/backend/database/repositories/factory.js";
import { sendAutomationNotification } from "../notify.js";
@@ -86,7 +86,7 @@ async function runNotify(
);
}
const repository = createCurrentAlertRepository();
const repository = createCurrentNotificationChannelRepository();
const channels = await repository.listNotificationChannels(context.userId);
const selected = channels.filter((channel) =>
step.channelIds.includes(channel.id),
@@ -31,7 +31,7 @@ import {
getNotificationChannels,
testNotificationChannel,
type NotificationChannel,
} from "@/api/alerts-api";
} from "@/api/notification-channels-api";
import { NotificationChannelDialog } from "@/sidebar/NotificationChannelDialog";
import { getSnippets } from "@/api/snippets-api";
import { getSSHHosts } from "@/api/ssh-host-management-api";
@@ -1,373 +0,0 @@
import { createCurrentAlertRepository } from "../../../src/backend/database/repositories/factory.js";
import { statsLogger } from "../../../src/backend/utils/logger.js";
import {
sendNotification,
type AlertPayload,
type NotificationChannel,
} from "../../../src/backend/utils/notification-sender.js";
import { sendDiscord } from "../../../src/backend/utils/discord-sender.js";
type AlertTriggerType =
| "host_offline"
| "host_online"
| "cpu_threshold"
| "memory_threshold"
| "disk_threshold"
| "health_check_failure"
| "health_check_recovery"
| "user_login";
interface AlertRule {
id: number;
userId: string;
hostId: number | null;
name: string;
enabled: boolean;
triggerType: AlertTriggerType;
thresholdValue: number | null;
thresholdDurationSeconds: number | null;
cooldownMinutes: number;
}
/**
* Set once the alert rules have been copied into automations. From then on the
* automations engine owns evaluation, and this one stands down rather than
* sending a second notification for every rule.
*
* The class is left in place so an install that has not migrated yet, or one
* rolled back, still alerts exactly as before.
*/
let supersededByAutomations = false;
export function markAlertEngineSuperseded(): void {
supersededByAutomations = true;
}
export function isAlertEngineSuperseded(): boolean {
return supersededByAutomations;
}
export class AlertEngine {
private static instance: AlertEngine;
// "${ruleId}:${hostId}" → lastFiredAt ms
private cooldownMap = new Map<string, number>();
// "${ruleId}:${hostId}" → breachStartTime ms
private breachStartMap = new Map<string, number>();
// hostId → last known status
private lastStatusMap = new Map<number, "online" | "offline">();
// "${hostId}:${checkId}" → last known ok state
private healthCheckStateMap = new Map<string, boolean>();
static getInstance(): AlertEngine {
if (!AlertEngine.instance) {
AlertEngine.instance = new AlertEngine();
}
return AlertEngine.instance;
}
async evaluateMetrics(
hostId: number,
metrics: {
cpu?: { percent: number | null } | null;
memory?: { percent: number | null } | null;
disk?: { percent: number | null } | null;
},
): Promise<void> {
if (supersededByAutomations) return;
const rules = (await this.loadRulesForHost(hostId)).filter((r) =>
["cpu_threshold", "memory_threshold", "disk_threshold"].includes(
r.triggerType,
),
);
if (rules.length === 0) return;
const now = Date.now();
for (const rule of rules) {
let currentValue: number | null | undefined;
if (rule.triggerType === "cpu_threshold")
currentValue = metrics.cpu?.percent;
else if (rule.triggerType === "memory_threshold")
currentValue = metrics.memory?.percent;
else if (rule.triggerType === "disk_threshold")
currentValue = metrics.disk?.percent;
if (currentValue == null || rule.thresholdValue == null) continue;
const key = `${rule.id}:${hostId}`;
const isBreaching = currentValue >= rule.thresholdValue;
if (isBreaching) {
if (!this.breachStartMap.has(key)) {
this.breachStartMap.set(key, now);
}
const breachStart = this.breachStartMap.get(key)!;
const durationMs = (rule.thresholdDurationSeconds ?? 0) * 1000;
if (
now - breachStart >= durationMs &&
!(await this.isCoolingDown(rule.id, hostId))
) {
const hostName = await this.getHostName(hostId);
await this.fireAlert(rule, hostId, hostName, {
value: currentValue,
message: `${rule.triggerType.replace("_threshold", "").toUpperCase()} usage at ${currentValue.toFixed(1)}% (threshold: ${rule.thresholdValue}%)`,
severity: currentValue >= 95 ? "critical" : "warning",
});
}
} else {
this.breachStartMap.delete(key);
}
}
}
async evaluateStatus(hostId: number, isOnline: boolean): Promise<void> {
if (supersededByAutomations) return;
const currentStatus = isOnline ? "online" : "offline";
const lastStatus = this.lastStatusMap.get(hostId);
if (lastStatus === currentStatus) return;
this.lastStatusMap.set(hostId, currentStatus);
if (lastStatus === undefined) return;
const triggerType: AlertTriggerType = isOnline
? "host_online"
: "host_offline";
const rules = (await this.loadRulesForHost(hostId)).filter(
(r) => r.triggerType === triggerType,
);
for (const rule of rules) {
if (!(await this.isCoolingDown(rule.id, hostId))) {
const hostName = await this.getHostName(hostId);
await this.fireAlert(rule, hostId, hostName, {
message: isOnline
? `Host "${hostName}" is back online`
: `Host "${hostName}" is offline`,
severity: isOnline ? "info" : "critical",
});
}
}
}
async evaluateHealthCheck(
hostId: number,
userId: string,
checkId: string,
ok: boolean,
detail?: string,
): Promise<void> {
if (supersededByAutomations) return;
const stateKey = `${hostId}:${checkId}`;
const lastOk = this.healthCheckStateMap.get(stateKey);
this.healthCheckStateMap.set(stateKey, ok);
if (lastOk === undefined) return;
let triggerType: AlertTriggerType | null = null;
if (!ok && lastOk) triggerType = "health_check_failure";
else if (ok && !lastOk) triggerType = "health_check_recovery";
if (!triggerType) return;
const rules = (await this.loadRulesForHostUser(hostId, userId)).filter(
(r) => r.triggerType === triggerType,
);
for (const rule of rules) {
if (!(await this.isCoolingDown(rule.id, hostId))) {
const hostName = await this.getHostName(hostId);
await this.fireAlert(rule, hostId, hostName, {
message: ok
? `Health check recovered on "${hostName}"${detail ? `: ${detail}` : ""}`
: `Health check failed on "${hostName}"${detail ? `: ${detail}` : ""}`,
severity: ok ? "info" : "warning",
});
}
}
}
async evaluateUserLogin(
hostId: number,
userId: string,
sshUser: string,
fromIp: string,
): Promise<void> {
if (supersededByAutomations) return;
const rules = (await this.loadRulesForHostUser(hostId, userId)).filter(
(r) => r.triggerType === "user_login",
);
for (const rule of rules) {
if (!(await this.isCoolingDown(rule.id, hostId))) {
const hostName = await this.getHostName(hostId);
await this.fireAlert(rule, hostId, hostName, {
message: `User "${sshUser}" logged in to "${hostName}" from ${fromIp}`,
severity: "info",
});
}
}
}
private async fireAlert(
rule: AlertRule,
hostId: number,
hostName: string,
context: {
value?: number;
message: string;
severity: "info" | "warning" | "critical";
},
): Promise<void> {
this.markCooldown(rule.id, hostId);
const payload: AlertPayload = {
hostName,
hostId,
triggerType: rule.triggerType,
value: context.value,
threshold: rule.thresholdValue ?? undefined,
message: context.message,
severity: context.severity,
timestamp: new Date().toISOString(),
ruleId: rule.id,
ruleName: rule.name,
};
try {
const repository = createCurrentAlertRepository();
await repository.createFiring({
userId: rule.userId,
ruleId: rule.id,
hostId,
hostName,
value: context.value ?? null,
message: context.message,
severity: context.severity,
});
await repository.pruneFiringsOlderThan(rule.userId, 30);
} catch (err) {
statsLogger.warn("Failed to write alert firing", {
operation: "alert_firing_insert_error",
ruleId: rule.id,
error: err instanceof Error ? err.message : String(err),
});
}
const channels = await this.loadChannelsForRule(rule.id);
for (const channel of channels) {
if (channel.type === "discord") {
// parse config and send via Discord sender directly
try {
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(channel.config) as Record<string, unknown>;
} catch {
statsLogger.warn("Failed to parse discord channel config", {
operation: "discord_config_parse_error",
channelId: channel.id,
});
continue;
}
sendDiscord(parsed as any, payload).catch((err) => {
statsLogger.warn("Failed to send discord notification", {
operation: "discord_notification_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
});
});
} catch (err) {
statsLogger.warn("Failed to send discord notification", {
operation: "discord_notification_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
});
}
} else {
sendNotification(channel, payload).catch((err) => {
statsLogger.warn("Failed to send notification", {
operation: "notification_delivery_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
});
});
}
}
}
private async isCoolingDown(
ruleId: number,
hostId: number,
): Promise<boolean> {
const key = `${ruleId}:${hostId}`;
const lastFired = this.cooldownMap.get(key);
if (!lastFired) return false;
const rule = await this.loadRuleById(ruleId);
const cooldownMs = (rule?.cooldownMinutes ?? 15) * 60 * 1000;
return Date.now() - lastFired < cooldownMs;
}
private markCooldown(ruleId: number, hostId: number): void {
this.cooldownMap.set(`${ruleId}:${hostId}`, Date.now());
}
private async loadRulesForHost(hostId: number): Promise<AlertRule[]> {
try {
return (await createCurrentAlertRepository().listEnabledRulesForHost(
hostId,
)) as AlertRule[];
} catch {
return [];
}
}
private async loadRulesForHostUser(
hostId: number,
userId: string,
): Promise<AlertRule[]> {
try {
return (await createCurrentAlertRepository().listEnabledRulesForHostUser(
hostId,
userId,
)) as AlertRule[];
} catch {
return [];
}
}
private async loadRuleById(ruleId: number): Promise<AlertRule | null> {
try {
return (await createCurrentAlertRepository().findRuleById(
ruleId,
)) as AlertRule | null;
} catch {
return null;
}
}
private async loadChannelsForRule(
ruleId: number,
): Promise<NotificationChannel[]> {
try {
return await createCurrentAlertRepository().listEnabledChannelsForRule(
ruleId,
);
} catch {
return [];
}
}
private async getHostName(hostId: number): Promise<string> {
try {
return (
(await createCurrentAlertRepository().getHostDisplayName(hostId)) ??
`Host #${hostId}`
);
} catch {
return `Host #${hostId}`;
}
}
}
@@ -7,7 +7,6 @@ import { managerHandler, ManagerInputError } from "./route-helpers.js";
import { shellSingleQuote } from "../../../../src/backend/hosts/metrics-shared/exec-elevated.js";
import { isValidPort } from "../../../../src/backend/hosts/metrics-shared/validation.js";
import type { ManagerRoutesDeps } from "./types.js";
import { AlertEngine } from "../alert-engine.js";
import { notifyAutomationHealthCheck } from "../automation-bridge.js";
export interface HealthCheck {
@@ -157,15 +156,6 @@ export function registerHealthRoutes(
if (results.length) {
await recordHistory(userId, host.id, results);
for (const r of results) {
AlertEngine.getInstance()
.evaluateHealthCheck(
host.id,
userId,
r.checkId,
r.ok,
r.detail ?? undefined,
)
.catch(() => {});
notifyAutomationHealthCheck(
host.id,
userId,
@@ -243,15 +233,6 @@ export function registerHealthRoutes(
if (results.length) {
await recordHistory(userId, host.id, results);
for (const r of results) {
AlertEngine.getInstance()
.evaluateHealthCheck(
host.id,
userId,
r.checkId,
r.ok,
r.detail ?? undefined,
)
.catch(() => {});
notifyAutomationHealthCheck(
host.id,
userId,
-13
View File
@@ -69,7 +69,6 @@ import { registerProxmoxStatsHistoryRoutes } from "../../../src/backend/hosts/me
import { ProxmoxPollingManager } from "../../../src/backend/hosts/metrics/proxmox-stats-polling.js";
import type { HostSessionStatusPayload } from "../../../src/backend/hosts/host-session-status.js";
import { pluginEvents, TOPICS } from "../../../src/backend/plugins/events.js";
import { AlertEngine } from "./alert-engine.js";
import {
notifyAutomationMetrics,
notifyAutomationStatus,
@@ -662,9 +661,6 @@ class PollingManager {
this.scheduleInitialMetricsPoll(config.host, config.viewerUserId);
}
}
AlertEngine.getInstance()
.evaluateStatus(refreshedHost.id, isOnline)
.catch(() => {});
notifyAutomationStatus(refreshedHost.id, refreshedHost.userId, isOnline);
} catch {
const statusEntry: StatusEntry = {
@@ -672,9 +668,6 @@ class PollingManager {
lastChecked: new Date().toISOString(),
};
this.statusStore.set(refreshedHost.id, statusEntry);
AlertEngine.getInstance()
.evaluateStatus(refreshedHost.id, false)
.catch(() => {});
notifyAutomationStatus(refreshedHost.id, refreshedHost.userId, false);
}
}
@@ -730,9 +723,6 @@ class PollingManager {
timestamp: Date.now(),
});
await this.insertMetricsHistory(refreshedHost.id, metrics);
AlertEngine.getInstance()
.evaluateMetrics(refreshedHost.id, metrics)
.catch(() => {});
notifyAutomationMetrics(refreshedHost.id, refreshedHost.userId, metrics);
pollingBackoff.reset(refreshedHost.id);
authFailureTracker.reset(refreshedHost.id);
@@ -1124,9 +1114,6 @@ app.post("/internal/login-alert", async (req, res) => {
sshUser,
fromIp,
});
AlertEngine.getInstance()
.evaluateUserLogin(hostId, userId, sshUser, fromIp)
.catch(() => {});
res.json({ ok: true });
});