import type { Express } from "express"; import type { Client } from "ssh2"; import type { AuthenticatedRequest } from "../../../../src/types/index.js"; import { execCommand } from "../../../../src/backend/hosts/metrics-shared/common-utils.js"; import { createCurrentHostHealthRepository } from "../../../../src/backend/database/repositories/factory.js"; import { managerHandler, ManagerInputError } from "./route-helpers.js"; import { shellSingleQuote } from "../../../../src/backend/hosts/metrics-shared/exec-elevated.js"; import { isValidPort } from "../../../../src/backend/hosts/metrics-shared/validation.js"; import type { ManagerRoutesDeps } from "./types.js"; import { AlertEngine } from "../alert-engine.js"; import { notifyAutomationHealthCheck } from "../automation-bridge.js"; export interface HealthCheck { id: string; name: string; type: "tcp" | "http"; target: string; port?: number; path?: string; } export interface HealthResult { checkId: string; ok: boolean; latencyMs: number | null; detail: string; } const TARGET_RE = /^[A-Za-z0-9.\-_:]+$/; const PATH_RE = /^\/[A-Za-z0-9._~!$&'()*+,;=:@/%-]*$/; export function isValidHealthCheck(c: unknown): c is HealthCheck { if (!c || typeof c !== "object") return false; const o = c as Record; if (typeof o.id !== "string" || !o.id) return false; if (typeof o.name !== "string") return false; if (o.type !== "tcp" && o.type !== "http") return false; if (typeof o.target !== "string" || !TARGET_RE.test(o.target)) return false; if (o.type === "tcp" && !isValidPort(o.port)) return false; if ( o.path !== undefined && (typeof o.path !== "string" || !PATH_RE.test(o.path)) ) return false; return true; } /** Build a command that runs the check from the host and prints "ok latency". */ export function buildHealthCheckCommand(check: HealthCheck): string { if (check.type === "tcp") { const host = shellSingleQuote(check.target); const port = check.port; // Prefer bash /dev/tcp; time it with date in ms. return `start=$(date +%s%3N); if timeout 3 bash -c '/dev/null; then echo "ok $(( $(date +%s%3N) - start ))"; else echo "fail $(( $(date +%s%3N) - start ))"; fi`; } // http const scheme = check.target.includes("://") ? "" : "http://"; const url = shellSingleQuote(`${scheme}${check.target}${check.path ?? ""}`); return `curl -s -o /dev/null -m 5 -w '%{http_code} %{time_total}' ${url} || echo '000 0'`; } export function parseHealthResult( check: HealthCheck, output: string, ): HealthResult { const line = output.trim().split("\n").pop() ?? ""; if (check.type === "tcp") { const [status, ms] = line.split(/\s+/); return { checkId: check.id, ok: status === "ok", latencyMs: Number(ms) || null, detail: status === "ok" ? "open" : "closed/timeout", }; } const m = line.match(/^(\d{3})\s+([\d.]+)/); if (!m) return { checkId: check.id, ok: false, latencyMs: null, detail: line }; const code = Number(m[1]); return { checkId: check.id, ok: code >= 200 && code < 400, latencyMs: Math.round(Number(m[2]) * 1000), detail: `HTTP ${code}`, }; } async function runChecks( client: Client, checks: HealthCheck[], ): Promise { return Promise.all( checks.map(async (check) => { try { const { stdout } = await execCommand( client, buildHealthCheckCommand(check), 8000, ); return parseHealthResult(check, stdout); } catch { return { checkId: check.id, ok: false, latencyMs: null, detail: "error", }; } }), ); } const HISTORY_KEEP = 500; function recordHistory( userId: string, hostId: number, results: HealthResult[], ): Promise { return createCurrentHostHealthRepository() .recordHistory(userId, hostId, results, HISTORY_KEEP) .then(() => undefined); } async function loadChecks( userId: string, hostId: number, ): Promise { const row = await createCurrentHostHealthRepository().findChecksByUserAndHost( userId, hostId, ); if (!row?.checks) return []; try { const parsed = JSON.parse(row.checks); return Array.isArray(parsed) ? parsed.filter(isValidHealthCheck) : []; } catch { return []; } } export function registerHealthRoutes( app: Express, { validateHostId, runOnHost }: ManagerRoutesDeps, ): void { app.get( "/host-metrics/managers/health/:id", validateHostId, managerHandler( runOnHost, "connect", "health_get", async (client, host, req) => { const userId = (req as AuthenticatedRequest).userId; const checks = await loadChecks(userId, host.id); const results = checks.length ? await runChecks(client, checks) : []; if (results.length) { await recordHistory(userId, host.id, results); for (const r of results) { AlertEngine.getInstance() .evaluateHealthCheck( host.id, userId, r.checkId, r.ok, r.detail ?? undefined, ) .catch(() => {}); notifyAutomationHealthCheck( host.id, userId, r.checkId, r.ok, r.detail ?? undefined, ); } } const history = ( await createCurrentHostHealthRepository().listHistory( userId, host.id, 200, ) ).map((row) => ({ checkId: row.checkId, ts: row.ts, ok: row.ok, latencyMs: row.latencyMs, detail: row.detail, })); return { checks, results, history }; }, ), ); app.post( "/host-metrics/managers/health/:id/config", validateHostId, managerHandler( runOnHost, "connect", "health_config", async (_client, host, req) => { const userId = (req as AuthenticatedRequest).userId; const { checks, intervalSeconds } = req.body as { checks?: unknown; intervalSeconds?: number; }; if (!Array.isArray(checks) || !checks.every(isValidHealthCheck)) { throw new ManagerInputError("Invalid checks"); } const interval = typeof intervalSeconds === "number" && intervalSeconds >= 30 && intervalSeconds <= 86400 ? Math.round(intervalSeconds) : 300; const now = new Date().toISOString(); await createCurrentHostHealthRepository().upsertChecks( userId, host.id, JSON.stringify(checks), interval, now, ); return { success: true }; }, ), ); app.post( "/host-metrics/managers/health/:id/run", validateHostId, managerHandler( runOnHost, "connect", "health_run", async (client, host, req) => { const userId = (req as AuthenticatedRequest).userId; const checks = await loadChecks(userId, host.id); const results = await runChecks(client, checks); if (results.length) { await recordHistory(userId, host.id, results); for (const r of results) { AlertEngine.getInstance() .evaluateHealthCheck( host.id, userId, r.checkId, r.ok, r.detail ?? undefined, ) .catch(() => {}); notifyAutomationHealthCheck( host.id, userId, r.checkId, r.ok, r.detail ?? undefined, ); } } return { results }; }, ), ); }