import { createCurrentAlertRepository } from "../../../src/backend/database/repositories/factory.js"; import { statsLogger } from "../../../src/backend/utils/logger.js"; import { sendNotification, type AlertPayload, type NotificationChannel, } from "../../../src/backend/utils/notification-sender.js"; import { sendDiscord } from "../../../src/backend/utils/discord-sender.js"; type AlertTriggerType = | "host_offline" | "host_online" | "cpu_threshold" | "memory_threshold" | "disk_threshold" | "health_check_failure" | "health_check_recovery" | "user_login"; interface AlertRule { id: number; userId: string; hostId: number | null; name: string; enabled: boolean; triggerType: AlertTriggerType; thresholdValue: number | null; thresholdDurationSeconds: number | null; cooldownMinutes: number; } /** * Set once the alert rules have been copied into automations. From then on the * automations engine owns evaluation, and this one stands down rather than * sending a second notification for every rule. * * The class is left in place so an install that has not migrated yet, or one * rolled back, still alerts exactly as before. */ let supersededByAutomations = false; export function markAlertEngineSuperseded(): void { supersededByAutomations = true; } export function isAlertEngineSuperseded(): boolean { return supersededByAutomations; } export class AlertEngine { private static instance: AlertEngine; // "${ruleId}:${hostId}" → lastFiredAt ms private cooldownMap = new Map(); // "${ruleId}:${hostId}" → breachStartTime ms private breachStartMap = new Map(); // hostId → last known status private lastStatusMap = new Map(); // "${hostId}:${checkId}" → last known ok state private healthCheckStateMap = new Map(); static getInstance(): AlertEngine { if (!AlertEngine.instance) { AlertEngine.instance = new AlertEngine(); } return AlertEngine.instance; } async evaluateMetrics( hostId: number, metrics: { cpu?: { percent: number | null } | null; memory?: { percent: number | null } | null; disk?: { percent: number | null } | null; }, ): Promise { if (supersededByAutomations) return; const rules = (await this.loadRulesForHost(hostId)).filter((r) => ["cpu_threshold", "memory_threshold", "disk_threshold"].includes( r.triggerType, ), ); if (rules.length === 0) return; const now = Date.now(); for (const rule of rules) { let currentValue: number | null | undefined; if (rule.triggerType === "cpu_threshold") currentValue = metrics.cpu?.percent; else if (rule.triggerType === "memory_threshold") currentValue = metrics.memory?.percent; else if (rule.triggerType === "disk_threshold") currentValue = metrics.disk?.percent; if (currentValue == null || rule.thresholdValue == null) continue; const key = `${rule.id}:${hostId}`; const isBreaching = currentValue >= rule.thresholdValue; if (isBreaching) { if (!this.breachStartMap.has(key)) { this.breachStartMap.set(key, now); } const breachStart = this.breachStartMap.get(key)!; const durationMs = (rule.thresholdDurationSeconds ?? 0) * 1000; if ( now - breachStart >= durationMs && !(await this.isCoolingDown(rule.id, hostId)) ) { const hostName = await this.getHostName(hostId); await this.fireAlert(rule, hostId, hostName, { value: currentValue, message: `${rule.triggerType.replace("_threshold", "").toUpperCase()} usage at ${currentValue.toFixed(1)}% (threshold: ${rule.thresholdValue}%)`, severity: currentValue >= 95 ? "critical" : "warning", }); } } else { this.breachStartMap.delete(key); } } } async evaluateStatus(hostId: number, isOnline: boolean): Promise { if (supersededByAutomations) return; const currentStatus = isOnline ? "online" : "offline"; const lastStatus = this.lastStatusMap.get(hostId); if (lastStatus === currentStatus) return; this.lastStatusMap.set(hostId, currentStatus); if (lastStatus === undefined) return; const triggerType: AlertTriggerType = isOnline ? "host_online" : "host_offline"; const rules = (await this.loadRulesForHost(hostId)).filter( (r) => r.triggerType === triggerType, ); for (const rule of rules) { if (!(await this.isCoolingDown(rule.id, hostId))) { const hostName = await this.getHostName(hostId); await this.fireAlert(rule, hostId, hostName, { message: isOnline ? `Host "${hostName}" is back online` : `Host "${hostName}" is offline`, severity: isOnline ? "info" : "critical", }); } } } async evaluateHealthCheck( hostId: number, userId: string, checkId: string, ok: boolean, detail?: string, ): Promise { if (supersededByAutomations) return; const stateKey = `${hostId}:${checkId}`; const lastOk = this.healthCheckStateMap.get(stateKey); this.healthCheckStateMap.set(stateKey, ok); if (lastOk === undefined) return; let triggerType: AlertTriggerType | null = null; if (!ok && lastOk) triggerType = "health_check_failure"; else if (ok && !lastOk) triggerType = "health_check_recovery"; if (!triggerType) return; const rules = (await this.loadRulesForHostUser(hostId, userId)).filter( (r) => r.triggerType === triggerType, ); for (const rule of rules) { if (!(await this.isCoolingDown(rule.id, hostId))) { const hostName = await this.getHostName(hostId); await this.fireAlert(rule, hostId, hostName, { message: ok ? `Health check recovered on "${hostName}"${detail ? `: ${detail}` : ""}` : `Health check failed on "${hostName}"${detail ? `: ${detail}` : ""}`, severity: ok ? "info" : "warning", }); } } } async evaluateUserLogin( hostId: number, userId: string, sshUser: string, fromIp: string, ): Promise { if (supersededByAutomations) return; const rules = (await this.loadRulesForHostUser(hostId, userId)).filter( (r) => r.triggerType === "user_login", ); for (const rule of rules) { if (!(await this.isCoolingDown(rule.id, hostId))) { const hostName = await this.getHostName(hostId); await this.fireAlert(rule, hostId, hostName, { message: `User "${sshUser}" logged in to "${hostName}" from ${fromIp}`, severity: "info", }); } } } private async fireAlert( rule: AlertRule, hostId: number, hostName: string, context: { value?: number; message: string; severity: "info" | "warning" | "critical"; }, ): Promise { this.markCooldown(rule.id, hostId); const payload: AlertPayload = { hostName, hostId, triggerType: rule.triggerType, value: context.value, threshold: rule.thresholdValue ?? undefined, message: context.message, severity: context.severity, timestamp: new Date().toISOString(), ruleId: rule.id, ruleName: rule.name, }; try { const repository = createCurrentAlertRepository(); await repository.createFiring({ userId: rule.userId, ruleId: rule.id, hostId, hostName, value: context.value ?? null, message: context.message, severity: context.severity, }); await repository.pruneFiringsOlderThan(rule.userId, 30); } catch (err) { statsLogger.warn("Failed to write alert firing", { operation: "alert_firing_insert_error", ruleId: rule.id, error: err instanceof Error ? err.message : String(err), }); } const channels = await this.loadChannelsForRule(rule.id); for (const channel of channels) { if (channel.type === "discord") { // parse config and send via Discord sender directly try { let parsed: Record; try { parsed = JSON.parse(channel.config) as Record; } catch { statsLogger.warn("Failed to parse discord channel config", { operation: "discord_config_parse_error", channelId: channel.id, }); continue; } sendDiscord(parsed as any, payload).catch((err) => { statsLogger.warn("Failed to send discord notification", { operation: "discord_notification_error", channelId: channel.id, error: err instanceof Error ? err.message : String(err), }); }); } catch (err) { statsLogger.warn("Failed to send discord notification", { operation: "discord_notification_error", channelId: channel.id, error: err instanceof Error ? err.message : String(err), }); } } else { sendNotification(channel, payload).catch((err) => { statsLogger.warn("Failed to send notification", { operation: "notification_delivery_error", channelId: channel.id, error: err instanceof Error ? err.message : String(err), }); }); } } } private async isCoolingDown( ruleId: number, hostId: number, ): Promise { const key = `${ruleId}:${hostId}`; const lastFired = this.cooldownMap.get(key); if (!lastFired) return false; const rule = await this.loadRuleById(ruleId); const cooldownMs = (rule?.cooldownMinutes ?? 15) * 60 * 1000; return Date.now() - lastFired < cooldownMs; } private markCooldown(ruleId: number, hostId: number): void { this.cooldownMap.set(`${ruleId}:${hostId}`, Date.now()); } private async loadRulesForHost(hostId: number): Promise { try { return (await createCurrentAlertRepository().listEnabledRulesForHost( hostId, )) as AlertRule[]; } catch { return []; } } private async loadRulesForHostUser( hostId: number, userId: string, ): Promise { try { return (await createCurrentAlertRepository().listEnabledRulesForHostUser( hostId, userId, )) as AlertRule[]; } catch { return []; } } private async loadRuleById(ruleId: number): Promise { try { return (await createCurrentAlertRepository().findRuleById( ruleId, )) as AlertRule | null; } catch { return null; } } private async loadChannelsForRule( ruleId: number, ): Promise { try { return await createCurrentAlertRepository().listEnabledChannelsForRule( ruleId, ); } catch { return []; } } private async getHostName(hostId: number): Promise { try { return ( (await createCurrentAlertRepository().getHostDisplayName(hostId)) ?? `Host #${hostId}` ); } catch { return `Host #${hostId}`; } } }