import express from "express"; import net from "net"; import { createCorsMiddleware } from "../utils/cors-config.js"; import cookieParser from "cookie-parser"; import { Client, type ConnectConfig } from "ssh2"; import { SSH_ALGORITHMS } from "../utils/ssh-algorithms.js"; import { pickResolvedUsername } from "./credential-username.js"; import { getDb } from "../database/db/index.js"; import { hosts, sshCredentials } from "../database/db/schema.js"; import { eq } from "drizzle-orm"; import { statsLogger } from "../utils/logger.js"; import { SimpleDBOps } from "../utils/simple-db-ops.js"; import { AuthManager } from "../utils/auth-manager.js"; import { PermissionManager } from "../utils/permission-manager.js"; import type { AuthenticatedRequest, ProxyNode } from "../../types/index.js"; import type { LogEntry, ConnectionStage } from "../../types/connection-log.js"; import { collectCpuMetrics } from "./widgets/cpu-collector.js"; import { collectMemoryMetrics } from "./widgets/memory-collector.js"; import { collectDiskMetrics } from "./widgets/disk-collector.js"; import { collectNetworkMetrics } from "./widgets/network-collector.js"; import { collectUptimeMetrics } from "./widgets/uptime-collector.js"; import { collectProcessesMetrics } from "./widgets/processes-collector.js"; import { collectSystemMetrics } from "./widgets/system-collector.js"; import { collectLoginStats } from "./widgets/login-stats-collector.js"; import { collectPortsMetrics } from "./widgets/ports-collector.js"; import { collectFirewallMetrics } from "./widgets/firewall-collector.js"; import { collectTemperatureMetrics } from "./widgets/temperature-collector.js"; import { createSocks5Connection, type SOCKS5Config, } from "../utils/socks5-helper.js"; import { preparePrivateKeyForSSH2 } from "../utils/ssh-key-utils.js"; import { SSHHostKeyVerifier } from "./host-key-verifier.js"; import { connectionPool, withConnection } from "./ssh-connection-pool.js"; import { registerHostMetricsSettingsRoutes } from "./host-metrics-settings-routes.js"; import { registerHostMetricsViewerRoutes } from "./host-metrics-viewer-routes.js"; import { registerHostMetricsPreferencesRoutes } from "./host-metrics-preferences-routes.js"; import { registerHostMetricsHistoryRoutes } from "./host-metrics-history-routes.js"; import { AlertEngine } from "./alert-engine.js"; import { registerManagerRoutes } from "./managers/index.js"; import { AccessDeniedError } from "./managers/route-helpers.js"; import type { ManagerHost } from "./managers/types.js"; import { createJumpHostChain } from "./host-metrics-jump-hosts.js"; import { createConnectionLog, isTcpPingEnabled, supportsMetrics, } from "./host-metrics-helpers.js"; import { applyAgentAuth } from "./terminal-auth-helpers.js"; import { cleanupMetricsSession, getSessionKey, metricsSessions, pendingTOTPSessions, scheduleMetricsSessionCleanup, type MetricsViewer, } from "./host-metrics-sessions.js"; import { authFailureTracker, metricsCache, pollingBackoff, requestQueue, } from "./host-metrics-state.js"; const authManager = AuthManager.getInstance(); const permissionManager = PermissionManager.getInstance(); type HostStatus = "online" | "offline"; interface SSHHostWithCredentials { id: number; name: string; ip: string; port: number; username: string; folder: string; tags: string[]; pin: boolean; authType: string; password?: string; key?: string; keyPassword?: string; keyType?: string; sudoPassword?: string; credentialId?: number; enableTerminal: boolean; enableTunnel: boolean; enableFileManager: boolean; defaultPath: string; tunnelConnections: unknown[]; jumpHosts?: Array<{ hostId: number }>; statsConfig?: string | StatsConfig; createdAt: string; updatedAt: string; userId: string; useSocks5?: boolean; socks5Host?: string; socks5Port?: number; socks5Username?: string; socks5Password?: string; socks5ProxyChain?: ProxyNode[]; connectionType?: "ssh" | "rdp" | "vnc" | "telnet"; rdpPort?: number; vncPort?: number; telnetPort?: number; vaultProfile?: { id?: number | null } | null; } type StatusEntry = { status: HostStatus; lastChecked: string; }; interface StatsConfig { enabledWidgets: string[]; statusCheckEnabled: boolean; statusCheckInterval: number; useGlobalStatusInterval?: boolean; metricsEnabled: boolean; metricsInterval: number; useGlobalMetricsInterval?: boolean; disableTcpPing?: boolean; } const DEFAULT_STATS_CONFIG: StatsConfig = { enabledWidgets: [ "cpu", "memory", "disk", "network", "uptime", "system", "login_stats", "processes", "ports", "firewall", "temperature", ], statusCheckEnabled: true, statusCheckInterval: 60, metricsEnabled: true, metricsInterval: 30, }; interface HostPollingConfig { host: SSHHostWithCredentials; statsConfig: StatsConfig; statusTimer?: NodeJS.Timeout; metricsTimer?: NodeJS.Timeout; viewerUserId?: string; } class PollingManager { private pollingConfigs = new Map(); private statusStore = new Map(); private metricsStore = new Map< number, { data: Awaited>; timestamp: number; } >(); private activeViewers = new Map>(); private viewerDetails = new Map(); private viewerCleanupInterval: NodeJS.Timeout; constructor() { this.viewerCleanupInterval = setInterval(() => { this.cleanupInactiveViewers(); }, 60000); } private getGlobalDefaults(): { statusCheckInterval: number; metricsInterval: number; } { try { const db = getDb(); const statusRow = db.$client .prepare( "SELECT value FROM settings WHERE key = 'global_status_check_interval'", ) .get() as { value: string } | undefined; const metricsRow = db.$client .prepare( "SELECT value FROM settings WHERE key = 'global_metrics_interval'", ) .get() as { value: string } | undefined; return { statusCheckInterval: statusRow ? parseInt(statusRow.value, 10) || DEFAULT_STATS_CONFIG.statusCheckInterval : DEFAULT_STATS_CONFIG.statusCheckInterval, metricsInterval: metricsRow ? parseInt(metricsRow.value, 10) || DEFAULT_STATS_CONFIG.metricsInterval : DEFAULT_STATS_CONFIG.metricsInterval, }; } catch { return { statusCheckInterval: DEFAULT_STATS_CONFIG.statusCheckInterval, metricsInterval: DEFAULT_STATS_CONFIG.metricsInterval, }; } } parseStatsConfig(statsConfigStr?: string | StatsConfig): StatsConfig { if (!statsConfigStr) { return DEFAULT_STATS_CONFIG; } let parsed: StatsConfig; if (typeof statsConfigStr === "object") { parsed = statsConfigStr; } else { try { let temp: unknown = JSON.parse(statsConfigStr); if (typeof temp === "string") { temp = JSON.parse(temp); } parsed = temp as StatsConfig; } catch (error) { statsLogger.warn( `Failed to parse statsConfig: ${error instanceof Error ? error.message : "Unknown error"}`, { operation: "parse_stats_config_error", statsConfigStr, }, ); return DEFAULT_STATS_CONFIG; } } const result = { ...DEFAULT_STATS_CONFIG, ...parsed }; const globalDefaults = this.getGlobalDefaults(); if (result.useGlobalStatusInterval !== false) { result.statusCheckInterval = globalDefaults.statusCheckInterval; } if (result.useGlobalMetricsInterval !== false) { result.metricsInterval = globalDefaults.metricsInterval; } return result; } async startPollingForHost( host: SSHHostWithCredentials, options?: { statusOnly?: boolean; viewerUserId?: string }, ): Promise { const statsConfig = this.parseStatsConfig(host.statsConfig); const statusOnly = options?.statusOnly ?? false; const viewerUserId = options?.viewerUserId; const canCollectMetrics = supportsMetrics(host); const enabledCollectors: string[] = []; if (isTcpPingEnabled(statsConfig)) { enabledCollectors.push("status"); } if (!statusOnly && statsConfig.metricsEnabled && canCollectMetrics) { enabledCollectors.push( "cpu", "memory", "disk", "network", "uptime", "processes", "system", ); } const existingConfig = this.pollingConfigs.get(host.id); if (existingConfig) { if (existingConfig.statusTimer) { clearInterval(existingConfig.statusTimer); existingConfig.statusTimer = undefined; } if (existingConfig.metricsTimer) { clearInterval(existingConfig.metricsTimer); existingConfig.metricsTimer = undefined; } } if (!isTcpPingEnabled(statsConfig) && !statsConfig.metricsEnabled) { this.pollingConfigs.delete(host.id); this.statusStore.delete(host.id); this.metricsStore.delete(host.id); return; } const config: HostPollingConfig = { host, statsConfig, viewerUserId, }; this.pollingConfigs.set(host.id, config); if (isTcpPingEnabled(statsConfig)) { const intervalMs = statsConfig.statusCheckInterval * 1000; this.pollHostStatus(host, viewerUserId); config.statusTimer = setInterval(() => { const latestConfig = this.pollingConfigs.get(host.id); if (latestConfig && isTcpPingEnabled(latestConfig.statsConfig)) { this.pollHostStatus(latestConfig.host, latestConfig.viewerUserId); } }, intervalMs); } else { this.statusStore.delete(host.id); } if (!statusOnly && statsConfig.metricsEnabled && canCollectMetrics) { const intervalMs = statsConfig.metricsInterval * 1000; await this.pollHostMetrics(host, viewerUserId); config.metricsTimer = setInterval(() => { const latestConfig = this.pollingConfigs.get(host.id); if ( latestConfig && latestConfig.statsConfig.metricsEnabled && supportsMetrics(latestConfig.host) ) { this.pollHostMetrics( latestConfig.host, latestConfig.viewerUserId, ).catch((err) => { statsLogger.error("Metrics polling failed", err, { operation: "metrics_poll_unhandled", hostId: host.id, }); }); } }, intervalMs); } else { this.metricsStore.delete(host.id); } } private async pollHostStatus( host: SSHHostWithCredentials, viewerUserId?: string, ): Promise { const userId = viewerUserId || host.userId; const refreshedHost = await fetchHostById(host.id, userId); if (!refreshedHost) { return; } try { let pingHost = refreshedHost.ip; const ct = refreshedHost.connectionType || "ssh"; let pingPort: number; if (ct === "rdp") pingPort = refreshedHost.rdpPort ?? 3389; else if (ct === "vnc") pingPort = refreshedHost.vncPort ?? 5900; else if (ct === "telnet") pingPort = refreshedHost.telnetPort ?? 23; else pingPort = refreshedHost.port; if (refreshedHost.jumpHosts && refreshedHost.jumpHosts.length > 0) { const firstJump = await fetchHostById( refreshedHost.jumpHosts[0].hostId, userId, ); if (firstJump) { pingHost = firstJump.ip; pingPort = firstJump.port; } } const isOnline = await tcpPing(pingHost, pingPort, 5000); const statusEntry: StatusEntry = { status: isOnline ? "online" : "offline", lastChecked: new Date().toISOString(), }; this.statusStore.set(refreshedHost.id, statusEntry); AlertEngine.getInstance() .evaluateStatus(refreshedHost.id, isOnline) .catch(() => {}); } catch { const statusEntry: StatusEntry = { status: "offline", lastChecked: new Date().toISOString(), }; this.statusStore.set(refreshedHost.id, statusEntry); AlertEngine.getInstance() .evaluateStatus(refreshedHost.id, false) .catch(() => {}); } } private async pollHostMetrics( host: SSHHostWithCredentials, viewerUserId?: string, ): Promise { const userId = viewerUserId || host.userId; const refreshedHost = await fetchHostById(host.id, userId); if (!refreshedHost) { return; } if (!supportsMetrics(refreshedHost)) { statsLogger.debug("Skipping metrics collection for non-SSH host", { operation: "poll_host_metrics_skipped", hostId: refreshedHost.id, connectionType: refreshedHost.connectionType || "ssh", }); return; } const config = this.pollingConfigs.get(refreshedHost.id); if (!config || !config.statsConfig.metricsEnabled) { return; } if (authFailureTracker.shouldSkip(host.id)) { return; } if (pollingBackoff.shouldSkip(host.id)) { return; } try { const metrics = await collectMetrics(refreshedHost); this.metricsStore.set(refreshedHost.id, { data: metrics, timestamp: Date.now(), }); this.insertMetricsHistory(refreshedHost.id, metrics); AlertEngine.getInstance() .evaluateMetrics(refreshedHost.id, metrics) .catch(() => {}); pollingBackoff.reset(refreshedHost.id); authFailureTracker.reset(refreshedHost.id); } catch (error) { const isAuthError = error instanceof Error && (error.message.includes("authentication") || error.message.includes("Authentication") || error.message.includes("permission denied") || error.message.includes("Permission denied")); if (isAuthError) { // authFailureTracker already handles auth errors inside collectMetrics; // only log on the first occurrence to avoid repeated spam const alreadyTracked = authFailureTracker.shouldSkip(host.id); if (!alreadyTracked) { statsLogger.error("Stats collector connection failed", error, { operation: "stats_connect_failed", hostId: refreshedHost.id, }); } return; } pollingBackoff.recordFailure(refreshedHost.id); // Only log when a new retry window opens, not on every skipped poll const backoff = pollingBackoff.getBackoffInfo(refreshedHost.id); const isNewFailure = backoff !== null && !backoff.includes("polling suspended"); if (isNewFailure) { statsLogger.error("Stats collector connection failed", error, { operation: "stats_connect_failed", hostId: refreshedHost.id, }); } } } private getRetentionDays(): number { try { const db = getDb(); const row = db.$client .prepare( "SELECT value FROM settings WHERE key = 'metrics_history_retention_days'", ) .get() as { value: string } | undefined; const days = row ? parseInt(row.value, 10) : 7; return isNaN(days) || days < 1 ? 7 : Math.min(days, 90); } catch { return 7; } } private insertMetricsHistory( hostId: number, metrics: { cpu: { percent: number | null }; memory: { percent: number | null }; disk: { percent: number | null }; network: { interfaces: Array<{ rxBytes: string | null; txBytes: string | null }>; }; }, ): void { try { const db = getDb(); const iface = metrics.network?.interfaces?.[0]; const rxRaw = iface?.rxBytes ? parseInt(iface.rxBytes, 10) : null; const txRaw = iface?.txBytes ? parseInt(iface.txBytes, 10) : null; db.$client .prepare( `INSERT INTO host_metrics_history (host_id, cpu_percent, mem_percent, disk_percent, net_rx_bytes, net_tx_bytes) VALUES (?, ?, ?, ?, ?, ?)`, ) .run( hostId, metrics.cpu?.percent ?? null, metrics.memory?.percent ?? null, metrics.disk?.percent ?? null, isNaN(rxRaw as number) ? null : rxRaw, isNaN(txRaw as number) ? null : txRaw, ); const retentionDays = this.getRetentionDays(); db.$client .prepare( `DELETE FROM host_metrics_history WHERE host_id = ? AND ts < datetime('now', ?)`, ) .run(hostId, `-${retentionDays} days`); } catch (err) { statsLogger.warn("Failed to write metrics history", { operation: "insert_metrics_history", hostId, error: err instanceof Error ? err.message : String(err), }); } } stopPollingForHost(hostId: number, clearData = true): void { const config = this.pollingConfigs.get(hostId); if (config) { if (config.statusTimer) { clearInterval(config.statusTimer); config.statusTimer = undefined; } if (config.metricsTimer) { clearInterval(config.metricsTimer); config.metricsTimer = undefined; } this.pollingConfigs.delete(hostId); if (clearData) { this.statusStore.delete(hostId); this.metricsStore.delete(hostId); } } } stopMetricsOnly(hostId: number): void { const config = this.pollingConfigs.get(hostId); if (config?.metricsTimer) { clearInterval(config.metricsTimer); config.metricsTimer = undefined; } } getStatus(hostId: number): StatusEntry | undefined { return this.statusStore.get(hostId); } getAllStatuses(): Map { return this.statusStore; } getMetrics( hostId: number, ): | { data: Awaited>; timestamp: number } | undefined { return this.metricsStore.get(hostId); } async initializePolling(userId: string): Promise { const hosts = await fetchAllHosts(userId); for (const host of hosts) { await this.startPollingForHost(host, { statusOnly: true }); } } async refreshHostPolling(userId: string): Promise { const hosts = await fetchAllHosts(userId); const currentHostIds = new Set(hosts.map((h) => h.id)); for (const hostId of this.pollingConfigs.keys()) { this.stopPollingForHost(hostId, false); } for (const hostId of this.statusStore.keys()) { if (!currentHostIds.has(hostId)) { this.statusStore.delete(hostId); this.metricsStore.delete(hostId); } } for (const host of hosts) { await this.startPollingForHost(host, { statusOnly: true }); } } async refreshAllPolling(): Promise { const hostsToRefresh: Array<{ host: SSHHostWithCredentials; viewerUserId?: string; }> = []; for (const [hostId, config] of this.pollingConfigs.entries()) { const status = this.statusStore.get(hostId); if (!status || status.status === "online") { hostsToRefresh.push({ host: config.host, viewerUserId: config.viewerUserId, }); } } for (const hostId of this.pollingConfigs.keys()) { this.stopPollingForHost(hostId, false); } for (const { host, viewerUserId } of hostsToRefresh) { await this.startPollingForHost(host, { statusOnly: true, viewerUserId }); } } registerViewer(hostId: number, sessionId: string, userId: string): void { if (!this.activeViewers.has(hostId)) { this.activeViewers.set(hostId, new Set()); } this.activeViewers.get(hostId)!.add(sessionId); this.viewerDetails.set(sessionId, { sessionId, userId, hostId, lastHeartbeat: Date.now(), }); if (this.activeViewers.get(hostId)!.size === 1) { // Fire-and-forget: never let background metrics start-up failures // propagate up to the HTTP handler that registered the viewer. Promise.resolve() .then(() => this.startMetricsForHost(hostId, userId)) .catch((err) => { statsLogger.warn("startMetricsForHost rejected (non-fatal)", { operation: "start_metrics_unhandled", hostId, userId, error: err instanceof Error ? err.message : String(err), }); }); } } updateHeartbeat(sessionId: string): boolean { const viewer = this.viewerDetails.get(sessionId); if (viewer) { viewer.lastHeartbeat = Date.now(); return true; } return false; } unregisterViewer(hostId: number, sessionId: string): void { const viewers = this.activeViewers.get(hostId); if (viewers) { viewers.delete(sessionId); if (viewers.size === 0) { this.activeViewers.delete(hostId); this.stopMetricsForHost(hostId); } } this.viewerDetails.delete(sessionId); } private async startMetricsForHost( hostId: number, userId: string, ): Promise { try { const host = await fetchHostById(hostId, userId); if (host) { await this.startPollingForHost(host, { viewerUserId: userId }); } } catch (error) { statsLogger.error("Failed to start metrics polling", { operation: "start_metrics_error", hostId, error: error instanceof Error ? error.message : String(error), }); } } private stopMetricsForHost(hostId: number): void { this.stopMetricsOnly(hostId); } private cleanupInactiveViewers(): void { const now = Date.now(); const maxInactivity = 120000; for (const [sessionId, viewer] of this.viewerDetails.entries()) { if (now - viewer.lastHeartbeat > maxInactivity) { this.unregisterViewer(viewer.hostId, sessionId); } } } destroy(): void { clearInterval(this.viewerCleanupInterval); for (const hostId of this.pollingConfigs.keys()) { this.stopPollingForHost(hostId); } } } const pollingManager = new PollingManager(); function validateHostId( req: express.Request, res: express.Response, next: express.NextFunction, ) { const id = Number(req.params.id); if (!id || !Number.isInteger(id) || id <= 0) { return res.status(400).json({ error: "Invalid host ID" }); } next(); } const app = express(); app.use(createCorsMiddleware()); app.use(cookieParser()); app.use(express.json({ limit: "1mb" })); app.use((_req, res, next) => { res.setHeader("Cache-Control", "no-store"); next(); }); app.use(authManager.createAuthMiddleware()); const requireAdmin = authManager.createAdminMiddleware(); async function fetchAllHosts( userId: string, ): Promise { try { const hostResults = await SimpleDBOps.select( getDb().select().from(hosts).where(eq(hosts.userId, userId)), "ssh_data", userId, ); const hostsWithCredentials: SSHHostWithCredentials[] = []; for (const host of hostResults) { try { const hostWithCreds = await resolveHostCredentials(host, userId); if (hostWithCreds) { hostsWithCredentials.push(hostWithCreds); } } catch (err) { statsLogger.warn( `Failed to resolve credentials for host ${host.id}: ${err instanceof Error ? err.message : "Unknown error"}`, ); } } return hostsWithCredentials.filter((h) => !!h.id && !!h.ip && !!h.port); } catch (err) { statsLogger.error("Failed to fetch hosts from database", err); return []; } } async function fetchHostById( id: number, userId: string, ): Promise { try { if (!SimpleDBOps.isUserDataUnlocked(userId)) { return undefined; } const accessInfo = await permissionManager.canAccessHost( userId, id, "read", ); if (!accessInfo.hasAccess) { statsLogger.warn(`User ${userId} cannot access host ${id}`, { operation: "fetch_host_access_denied", userId, hostId: id, }); return undefined; } const hostResults = await SimpleDBOps.select( getDb().select().from(hosts).where(eq(hosts.id, id)), "ssh_data", userId, ); if (hostResults.length === 0) { return undefined; } const host = hostResults[0]; return await resolveHostCredentials(host, userId); } catch (err) { statsLogger.error(`Failed to fetch host ${id}`, err); return undefined; } } async function resolveHostCredentials( host: Record, userId: string, ): Promise { try { const baseHost: Record = { id: host.id, name: host.name, ip: host.ip, port: host.port, username: host.username, folder: host.folder || "", tags: typeof host.tags === "string" ? host.tags ? host.tags.split(",").filter(Boolean) : [] : [], pin: !!host.pin, authType: host.authType, terminalConfig: (() => { try { return typeof host.terminalConfig === "string" ? JSON.parse(host.terminalConfig as string) : host.terminalConfig || undefined; } catch { return undefined; } })(), enableTerminal: !!host.enableTerminal, enableTunnel: !!host.enableTunnel, enableFileManager: !!host.enableFileManager, defaultPath: host.defaultPath || "/", tunnelConnections: host.tunnelConnections ? JSON.parse(host.tunnelConnections as string) : [], jumpHosts: host.jumpHosts ? JSON.parse(host.jumpHosts as string) : [], statsConfig: host.statsConfig || undefined, sudoPassword: (() => { const top = host.sudoPassword as string | null | undefined; if (top) return top; try { const tc = typeof host.terminalConfig === "string" ? JSON.parse(host.terminalConfig as string) : host.terminalConfig; return (tc?.sudoPassword as string) || undefined; } catch { return undefined; } })(), createdAt: host.createdAt, updatedAt: host.updatedAt, userId: host.userId, useSocks5: !!host.useSocks5, socks5Host: host.socks5Host || undefined, socks5Port: host.socks5Port || undefined, socks5Username: host.socks5Username || undefined, socks5Password: host.socks5Password || undefined, socks5ProxyChain: host.socks5ProxyChain ? JSON.parse(host.socks5ProxyChain as string) : undefined, connectionType: (host.connectionType as SSHHostWithCredentials["connectionType"]) || "ssh", rdpPort: (host.rdpPort as number | undefined) ?? undefined, vncPort: (host.vncPort as number | undefined) ?? undefined, telnetPort: (host.telnetPort as number | undefined) ?? undefined, }; if (host.credentialId) { try { const ownerId = host.userId; const isSharedHost = userId !== ownerId; if (isSharedHost) { const { SharedCredentialManager } = await import("../utils/shared-credential-manager.js"); const sharedCredManager = SharedCredentialManager.getInstance(); const sharedCred = await sharedCredManager.getSharedCredentialForUser( host.id as number, userId, ); if (sharedCred) { baseHost.credentialId = host.credentialId; baseHost.authType = sharedCred.authType; baseHost.username = pickResolvedUsername( host.username, sharedCred.username, host.overrideCredentialUsername, ); if (sharedCred.password) { baseHost.password = sharedCred.password; } if (sharedCred.key) { baseHost.key = sharedCred.key; } if (sharedCred.keyPassword) { baseHost.keyPassword = sharedCred.keyPassword; } if (sharedCred.keyType) { baseHost.keyType = sharedCred.keyType; } } } else { const credentials = await SimpleDBOps.select( getDb() .select() .from(sshCredentials) .where(eq(sshCredentials.id, host.credentialId as number)), "ssh_credentials", userId, ); if (credentials.length > 0) { const credential = credentials[0]; baseHost.credentialId = credential.id; baseHost.authType = credential.authType || (credential.password ? "password" : credential.key || (credential as Record).privateKey ? "key" : "none"); baseHost.username = pickResolvedUsername( host.username, credential.username, host.overrideCredentialUsername, ); if (credential.password) { baseHost.password = credential.password; } if ( credential.key || (credential as Record).privateKey ) { baseHost.key = credential.key || ((credential as Record).privateKey as string); } if (credential.keyPassword) { baseHost.keyPassword = credential.keyPassword; } if (credential.keyType) { baseHost.keyType = credential.keyType; } } else { addLegacyCredentials(baseHost, host); if (baseHost.authType === "credential") { baseHost.authType = baseHost.password ? "password" : baseHost.key ? "key" : "none"; } } } } catch (error) { statsLogger.warn( `Failed to resolve credential ${host.credentialId} for host ${host.id}: ${error instanceof Error ? error.message : "Unknown error"}`, ); addLegacyCredentials(baseHost, host); if (baseHost.authType === "credential") { baseHost.authType = baseHost.password ? "password" : baseHost.key ? "key" : "none"; } } } else { addLegacyCredentials(baseHost, host); } return baseHost as unknown as SSHHostWithCredentials; } catch (error) { statsLogger.error( `Failed to resolve host credentials for host ${host.id}: ${error instanceof Error ? error.message : "Unknown error"}`, ); return undefined; } } function addLegacyCredentials( baseHost: Record, host: Record, ): void { baseHost.password = host.password || null; baseHost.key = host.key || null; baseHost.keyPassword = host.keyPassword || null; baseHost.keyType = host.keyType; } async function buildSshConfig( host: SSHHostWithCredentials, ): Promise { const base: ConnectConfig = { host: host.ip?.replace(/^\[|\]$/g, "") || host.ip, port: host.port, username: host.username, tryKeyboard: true, keepaliveInterval: 30000, keepaliveCountMax: 3, readyTimeout: 60000, tcpKeepAlive: true, tcpKeepAliveInitialDelay: 30000, hostVerifier: await SSHHostKeyVerifier.createHostVerifier( host.id, host.ip, host.port, null, host.userId || "", false, ), env: { TERM: "xterm-256color", LANG: "en_US.UTF-8", LC_ALL: "en_US.UTF-8", LC_CTYPE: "en_US.UTF-8", LC_MESSAGES: "en_US.UTF-8", LC_MONETARY: "en_US.UTF-8", LC_NUMERIC: "en_US.UTF-8", LC_TIME: "en_US.UTF-8", LC_COLLATE: "en_US.UTF-8", COLORTERM: "truecolor", }, algorithms: { kex: [ "curve25519-sha256", "curve25519-sha256@libssh.org", "ecdh-sha2-nistp521", "ecdh-sha2-nistp384", "ecdh-sha2-nistp256", "diffie-hellman-group-exchange-sha256", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1", "diffie-hellman-group-exchange-sha1", "diffie-hellman-group1-sha1", ], serverHostKey: [ "ssh-ed25519", "ecdsa-sha2-nistp521", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp256", "rsa-sha2-512", "rsa-sha2-256", "ssh-rsa", "ssh-dss", ], cipher: SSH_ALGORITHMS.cipher, hmac: [ "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512", "hmac-sha2-256", "hmac-sha1", "hmac-md5", ], compress: ["none", "zlib@openssh.com", "zlib"], }, } as ConnectConfig; if (host.authType === "password") { if (!host.password) { throw new Error(`No password available for host ${host.ip}`); } base.password = host.password; } else if (host.authType === "key") { if (!host.key) { throw new Error(`No SSH key available for host ${host.ip}`); } try { (base as Record).privateKey = preparePrivateKeyForSSH2( host.key, host.keyPassword, ); if (host.keyPassword) { (base as Record).passphrase = host.keyPassword; } } catch (keyError) { statsLogger.error( `SSH key format error for host ${host.ip}: ${keyError instanceof Error ? keyError.message : "Unknown error"}`, ); throw new Error(`Invalid SSH key format for host ${host.ip}`, { cause: keyError, }); } } else if ( host.authType === "none" || host.authType === "tailscale" || host.authType === "warpgate" ) { // no credentials needed } else if (host.authType === "opkssh") { // cert auth setup happens in createSshFactory (needs client instance) } else if (host.authType === "vault") { // cert auth setup happens in createSshFactory (needs client instance) } else if (host.authType === "credential") { if (host.password) { base.password = host.password; } else if (host.key) { const cleanKey = host.key .trim() .replace(/\r\n/g, "\n") .replace(/\r/g, "\n"); (base as Record).privateKey = Buffer.from( cleanKey, "utf8", ); if (host.keyPassword) { (base as Record).passphrase = host.keyPassword; } } else { throw new Error(`Credential for host ${host.ip} could not be resolved`); } } else if (host.authType === "agent") { const result = await applyAgentAuth( base as Record, (host as { terminalConfig?: Record }).terminalConfig, ); if ("error" in result) { throw new Error(result.error); } } else { throw new Error( `Unsupported authentication type '${host.authType}' for host ${host.ip}`, ); } return base; } function getPoolKey(host: SSHHostWithCredentials): string { const socks5Key = host.useSocks5 ? `:socks5:${host.socks5Host}:${host.socks5Port}` : ""; return `stats:${host.userId}:${host.ip}:${host.port}:${host.username}${socks5Key}`; } function createSshFactory(host: SSHHostWithCredentials): () => Promise { return async () => { const config = await buildSshConfig(host); const client = new Client(); // Set up OPKSSH cert auth if needed (requires client instance) if (host.authType === "opkssh" && host.userId) { const { getOPKSSHToken } = await import("./opkssh-auth.js"); const token = await getOPKSSHToken(host.userId, host.id); if (!token) { throw new Error( "OPKSSH authentication required. Please open a Terminal connection first.", ); } const { setupOPKSSHCertAuth } = await import("./opkssh-cert-auth.js"); await setupOPKSSHCertAuth(config, client, token, host.username); } else if (host.authType === "vault") { const { setupVaultSshSignerAuth } = await import("./vault-ssh-connect.js"); await setupVaultSshSignerAuth(config, client, host); } const proxyConfig: SOCKS5Config | null = host.useSocks5 && (host.socks5Host || (host.socks5ProxyChain && host.socks5ProxyChain.length > 0)) ? { useSocks5: host.useSocks5, socks5Host: host.socks5Host, socks5Port: host.socks5Port, socks5Username: host.socks5Username, socks5Password: host.socks5Password, socks5ProxyChain: host.socks5ProxyChain, } : null; const hasJumpHosts = host.jumpHosts && host.jumpHosts.length > 0 && host.userId; let jumpClient: Client | null = null; if (hasJumpHosts) { jumpClient = await createJumpHostChain( host.jumpHosts!, host.userId!, proxyConfig, ); if (!jumpClient) { throw new Error("Failed to establish jump host chain"); } } else if (proxyConfig) { try { const proxySocket = await createSocks5Connection( host.ip, host.port, proxyConfig, ); if (proxySocket) { config.sock = proxySocket; } } catch (proxyError) { throw new Error( "Proxy connection failed: " + (proxyError instanceof Error ? proxyError.message : "Unknown error"), { cause: proxyError }, ); } } return new Promise((resolve, reject) => { const timeout = setTimeout(() => { client.end(); jumpClient?.end(); reject(new Error("SSH connection timeout")); }, 30000); client.on("ready", () => { clearTimeout(timeout); resolve(client); }); client.on("close", () => { jumpClient?.end(); }); client.on("error", (err) => { clearTimeout(timeout); jumpClient?.end(); reject(err); }); client.on( "keyboard-interactive", ( _name: string, _instructions: string, _instructionsLang: string, prompts: Array<{ prompt: string; echo: boolean }>, finish: (responses: string[]) => void, ) => { const totpPromptIndex = prompts.findIndex((p) => /verification code|verification_code|token|otp|2fa|authenticator|google.*auth/i.test( p.prompt, ), ); if (totpPromptIndex !== -1) { const sessionId = `totp-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pendingTOTPSessions[sessionId] = { client, finish, config, createdAt: Date.now(), sessionId, hostId: host.id, userId: host.userId!, prompts: prompts.map((p) => ({ prompt: p.prompt, echo: p.echo ?? false, })), totpPromptIndex, resolvedPassword: host.password, totpAttempts: 0, }; return; } else if (host.password) { const responses = prompts.map((p) => { if (/password/i.test(p.prompt)) { return host.password || ""; } return ""; }); finish(responses); } else { finish(prompts.map(() => "")); } }, ); if (jumpClient) { jumpClient.forwardOut( "127.0.0.1", 0, host.ip, host.port, (err, stream) => { if (err) { clearTimeout(timeout); jumpClient!.end(); reject( new Error( "Failed to forward through jump host: " + err.message, ), ); return; } config.sock = stream; client.connect(config); }, ); } else { client.connect(config); } }); }; } async function withSshConnection( host: SSHHostWithCredentials, fn: (client: Client) => Promise, ): Promise { const key = getPoolKey(host); const factory = createSshFactory(host); return withConnection(key, factory, fn); } async function collectMetrics(host: SSHHostWithCredentials): Promise<{ cpu: { percent: number | null; cores: number | null; load: [number, number, number] | null; }; memory: { percent: number | null; usedGiB: number | null; totalGiB: number | null; }; disk: { percent: number | null; usedHuman: string | null; totalHuman: string | null; availableHuman: string | null; }; network: { interfaces: Array<{ name: string; ip: string; state: string; rxBytes: string | null; txBytes: string | null; }>; }; uptime: { seconds: number | null; formatted: string | null; }; processes: { total: number | null; running: number | null; top: Array<{ pid: string; user: string; cpu: string; mem: string; command: string; }>; }; system: { hostname: string | null; kernel: string | null; os: string | null; }; temperature: { source: "sysfs" | "sensors" | "none"; highestCelsius: number | null; sensors: Array<{ label: string; celsius: number; }>; }; }> { if (!supportsMetrics(host)) { throw new Error("Metrics collection only supported for SSH hosts"); } if (authFailureTracker.shouldSkip(host.id)) { const reason = authFailureTracker.getSkipReason(host.id); throw new Error(reason || "Authentication failed"); } const cached = metricsCache.get(host.id); if (cached) { return cached as ReturnType extends Promise ? T : never; } return requestQueue.queueRequest(host.id, async () => { const sessionKey = getSessionKey(host.id, host.userId!); const existingSession = metricsSessions[sessionKey]; try { const collectFn = async (client: Client) => { const cpu = await collectCpuMetrics(client); const memory = await collectMemoryMetrics(client); const disk = await collectDiskMetrics(client); const network = await collectNetworkMetrics(client); const uptime = await collectUptimeMetrics(client); const processes = await collectProcessesMetrics(client); const system = await collectSystemMetrics(client); const temperature = await collectTemperatureMetrics(client); let login_stats = { recentLogins: [], failedLogins: [], totalLogins: 0, uniqueIPs: 0, }; try { login_stats = await collectLoginStats(client); } catch { // expected } let ports: { source: "ss" | "netstat" | "none"; ports: Array<{ protocol: "tcp" | "udp"; localAddress: string; localPort: number; state?: string; pid?: number; process?: string; }>; } = { source: "none", ports: [], }; try { ports = await collectPortsMetrics(client); } catch { // expected } let firewall: { type: "iptables" | "nftables" | "none"; status: "active" | "inactive" | "unknown"; chains: Array<{ name: string; policy: string; rules: Array<{ chain: string; target: string; protocol: string; source: string; destination: string; dport?: string; sport?: string; state?: string; interface?: string; extra?: string; }>; }>; } = { type: "none", status: "unknown", chains: [], }; try { firewall = await collectFirewallMetrics(client); } catch { // expected } const result = { cpu, memory, disk, network, uptime, processes, system, temperature, login_stats, ports, firewall, }; metricsCache.set(host.id, result); return result; }; if (existingSession && existingSession.isConnected) { existingSession.activeOperations++; try { const result = await collectFn(existingSession.client); existingSession.lastActive = Date.now(); return result; } finally { existingSession.activeOperations--; } } else { return await withSshConnection(host, collectFn); } } catch (error) { if (error instanceof Error) { if (error.message.includes("TOTP authentication required")) { throw error; } else if ( error.message.includes("No password available") || error.message.includes("Unsupported authentication type") || error.message.includes("No SSH key available") || error.message.includes("Invalid SSH key format") ) { authFailureTracker.recordFailure(host.id, "AUTH", true); } else if ( error.message.includes("authentication") || error.message.includes("Permission denied") || error.message.includes("All configured authentication methods failed") ) { authFailureTracker.recordFailure(host.id, "AUTH"); } else if ( error.message.includes("timeout") || error.message.includes("ETIMEDOUT") ) { authFailureTracker.recordFailure(host.id, "TIMEOUT"); } } throw error; } }); } function tcpPing( host: string, port: number, timeoutMs = 5000, ): Promise { return new Promise((resolve) => { const socket = new net.Socket(); let settled = false; const finish = (result: boolean) => { if (settled) return; settled = true; resolve(result); }; const cleanup = () => { try { socket.destroy(); } catch { // expected } }; socket.setTimeout(timeoutMs); socket.once("connect", () => { const dataTimeout = setTimeout(() => { cleanup(); finish(true); }, 2000); socket.once("data", (data) => { clearTimeout(dataTimeout); const dataStr = data.toString("utf8"); if (dataStr.startsWith("SSH-")) { try { socket.end("SSH-2.0-TermixHealthCheck\r\n"); } catch { // expected } setTimeout(cleanup, 200); } else { cleanup(); } finish(true); }); }); socket.once("timeout", () => { cleanup(); finish(false); }); socket.once("error", () => { cleanup(); finish(false); }); socket.connect(port, host); }); } /** * @openapi * /status: * get: * summary: Get all host statuses * description: Retrieves the status of all hosts for the authenticated user. * tags: * - Host Metrics * responses: * 200: * description: A map of host IDs to their status entries. * 401: * description: Session expired - please log in again. */ app.get("/status", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } const statuses = pollingManager.getAllStatuses(); if (statuses.size === 0) { await pollingManager.initializePolling(userId); } const result: Record = {}; for (const [id, entry] of pollingManager.getAllStatuses().entries()) { result[id] = entry; } res.json(result); }); /** * @openapi * /status/{id}: * get: * summary: Get host status by ID * description: Retrieves the status of a specific host by its ID. * tags: * - Host Metrics * parameters: * - in: path * name: id * required: true * schema: * type: integer * responses: * 200: * description: Host status entry. * 401: * description: Session expired - please log in again. * 404: * description: Status not available. */ app.get("/status/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } const statuses = pollingManager.getAllStatuses(); if (statuses.size === 0) { await pollingManager.initializePolling(userId); } const statusEntry = pollingManager.getStatus(id); if (!statusEntry) { return res.status(404).json({ error: "Status not available" }); } res.json(statusEntry); }); /** * @openapi * /clear-connections: * post: * summary: Clear all SSH connections * description: Clears all SSH connections from the connection pool. * tags: * - Host Metrics * responses: * 200: * description: All SSH connections cleared. * 401: * description: Session expired - please log in again. */ app.post("/clear-connections", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } connectionPool.clearAllConnections(); res.json({ message: "All SSH connections cleared" }); }); /** * @openapi * /refresh: * post: * summary: Refresh polling * description: Clears all SSH connections and refreshes host polling. * tags: * - Host Metrics * responses: * 200: * description: Polling refreshed. * 401: * description: Session expired - please log in again. */ app.post("/refresh", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } connectionPool.clearAllConnections(); await pollingManager.refreshHostPolling(userId); res.json({ message: "Polling refreshed" }); }); /** * @openapi * /host-updated: * post: * summary: Start polling for updated host * description: Starts polling for a specific host after it has been updated. * tags: * - Host Metrics * requestBody: * required: true * content: * application/json: * schema: * type: object * properties: * hostId: * type: integer * responses: * 200: * description: Host polling started. * 400: * description: Invalid hostId. * 401: * description: Session expired - please log in again. * 404: * description: Host not found. * 500: * description: Failed to start polling. */ app.post("/host-updated", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; const { hostId } = req.body; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } if (!hostId || typeof hostId !== "number") { return res.status(400).json({ error: "Invalid hostId" }); } try { const host = await fetchHostById(hostId, userId); if (host) { connectionPool.clearKeyConnections(getPoolKey(host)); await pollingManager.startPollingForHost(host); res.json({ message: "Host polling started" }); } else { res.status(404).json({ error: "Host not found" }); } } catch (error) { statsLogger.error("Failed to start polling for host", error, { operation: "host_updated", hostId, userId, }); res.status(500).json({ error: "Failed to start polling" }); } }); /** * @openapi * /host-deleted: * post: * summary: Stop polling for deleted host * description: Stops polling for a specific host after it has been deleted. * tags: * - Host Metrics * requestBody: * required: true * content: * application/json: * schema: * type: object * properties: * hostId: * type: integer * responses: * 200: * description: Host polling stopped. * 400: * description: Invalid hostId. * 401: * description: Session expired - please log in again. * 500: * description: Failed to stop polling. */ app.post("/host-deleted", async (req, res) => { const userId = (req as AuthenticatedRequest).userId; const { hostId } = req.body; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } if (!hostId || typeof hostId !== "number") { return res.status(400).json({ error: "Invalid hostId" }); } try { pollingManager.stopPollingForHost(hostId, true); res.json({ message: "Host polling stopped" }); } catch (error) { statsLogger.error("Failed to stop polling for host", error, { operation: "host_deleted", hostId, userId, }); res.status(500).json({ error: "Failed to stop polling" }); } }); /** * @openapi * /metrics/{id}: * get: * summary: Get host metrics * description: Retrieves current metrics for a specific host including CPU, memory, disk, network, processes, and system information. * tags: * - Host Metrics * parameters: * - in: path * name: id * required: true * schema: * type: integer * responses: * 200: * description: Host metrics data. * 401: * description: Session expired - please log in again. * 404: * description: Metrics not available. */ app.get("/metrics/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } const metricsData = pollingManager.getMetrics(id); if (!metricsData) { return res.status(404).json({ error: "Metrics not available", cpu: { percent: null, cores: null, load: null }, memory: { percent: null, usedGiB: null, totalGiB: null }, disk: { percent: null, usedHuman: null, totalHuman: null, availableHuman: null, }, network: { interfaces: [] }, uptime: { seconds: null, formatted: null }, processes: { total: null, running: null, top: [] }, system: { hostname: null, kernel: null, os: null }, lastChecked: new Date().toISOString(), }); } res.json({ ...metricsData.data, lastChecked: new Date(metricsData.timestamp).toISOString(), }); }); /** * @openapi * /metrics/start/{id}: * post: * summary: Start metrics collection * description: Establishes an SSH connection and starts collecting metrics for a specific host. * tags: * - Host Metrics * parameters: * - in: path * name: id * required: true * schema: * type: integer * responses: * 200: * description: Metrics collection started successfully, or TOTP required. * 401: * description: Session expired - please log in again. * 404: * description: Host not found. * 500: * description: Failed to start metrics collection. */ app.post("/metrics/start/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; const connectionLogs: Array> = []; if (!SimpleDBOps.isUserDataUnlocked(userId)) { connectionLogs.push( createConnectionLog("error", "stats_connecting", "Session expired"), ); return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", connectionLogs, }); } try { const host = await fetchHostById(id, userId); if (!host) { connectionLogs.push( createConnectionLog("error", "stats_connecting", "Host not found"), ); return res.status(404).json({ error: "Host not found", connectionLogs }); } if (!supportsMetrics(host)) { connectionLogs.push( createConnectionLog( "info", "stats_connecting", "Metrics collection is only supported for SSH hosts", { connectionType: host.connectionType || "ssh", }, ), ); return res.json({ success: true, skipped: true, connectionLogs }); } connectionLogs.push( createConnectionLog( "info", "stats_connecting", "Starting metrics collection", ), ); connectionLogs.push( createConnectionLog("info", "dns", `Resolving DNS for ${host.ip}`), ); connectionLogs.push( createConnectionLog( "info", "tcp", `Connecting to ${host.ip}:${host.port}`, ), ); connectionLogs.push( createConnectionLog("info", "handshake", "Initiating SSH handshake"), ); if (host.authType === "password") { connectionLogs.push( createConnectionLog("info", "auth", "Authenticating with password"), ); } else if (host.authType === "key") { connectionLogs.push( createConnectionLog("info", "auth", "Authenticating with SSH key"), ); } const sessionKey = getSessionKey(host.id, userId); const existingSession = metricsSessions[sessionKey]; if (existingSession && existingSession.isConnected) { connectionLogs.push( createConnectionLog( "success", "stats_polling", "Using existing metrics session", ), ); const viewerSessionId = `viewer-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pollingManager.registerViewer(host.id, viewerSessionId, userId); return res.json({ success: true, viewerSessionId, connectionLogs }); } const config = await buildSshConfig(host); const client = new Client(); if (host.authType === "opkssh" && host.userId) { const { getOPKSSHToken } = await import("./opkssh-auth.js"); const token = await getOPKSSHToken(host.userId, host.id); if (!token) { connectionLogs.push( createConnectionLog( "error", "auth", "OPKSSH authentication required. Please open a Terminal connection first.", ), ); return res.status(401).json({ error: "OPKSSH authentication required", requiresOPKSSHAuth: true, connectionLogs, }); } const { setupOPKSSHCertAuth } = await import("./opkssh-cert-auth.js"); await setupOPKSSHCertAuth(config, client, token, host.username); } else if (host.authType === "vault") { const { setupVaultSshSignerAuth } = await import("./vault-ssh-connect.js"); await setupVaultSshSignerAuth(config, client, host); } const connectionPromise = new Promise<{ success: boolean; requires_totp?: boolean; sessionId?: string; prompt?: string; viewerSessionId?: string; }>((resolve, reject) => { let isResolved = false; const timeout = setTimeout(() => { if (!isResolved) { isResolved = true; client.end(); reject(new Error("Connection timeout")); } }, 60000); client.on( "keyboard-interactive", (name, instructions, instructionsLang, prompts, finish) => { const totpPromptIndex = prompts.findIndex((p) => /verification code|verification_code|token|otp|2fa|authenticator|google.*auth/i.test( p.prompt, ), ); if (totpPromptIndex !== -1) { const sessionId = `totp-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pendingTOTPSessions[sessionId] = { client, finish, config, createdAt: Date.now(), sessionId, hostId: host.id, userId: host.userId!, prompts: prompts.map((p) => ({ prompt: p.prompt, echo: p.echo ?? false, })), totpPromptIndex, resolvedPassword: host.password, totpAttempts: 0, }; connectionLogs.push( createConnectionLog( "info", "stats_totp", "TOTP verification required", ), ); clearTimeout(timeout); if (!isResolved) { isResolved = true; resolve({ success: false, requires_totp: true, sessionId, prompt: prompts[totpPromptIndex].prompt, }); } return; } else { const responses = prompts.map((p) => { if (/password/i.test(p.prompt) && host.password) { return host.password; } return ""; }); finish(responses); } }, ); client.on("ready", () => { clearTimeout(timeout); if (!isResolved) { isResolved = true; connectionLogs.push( createConnectionLog( "success", "connected", "SSH connection established successfully", ), ); connectionLogs.push( createConnectionLog( "success", "stats_polling", "Metrics session established", ), ); metricsSessions[sessionKey] = { client, isConnected: true, lastActive: Date.now(), activeOperations: 0, hostId: host.id, userId, }; scheduleMetricsSessionCleanup(sessionKey); const viewerSessionId = `viewer-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pollingManager.registerViewer(host.id, viewerSessionId, userId); resolve({ success: true, viewerSessionId }); } }); client.on("error", (error) => { clearTimeout(timeout); if (!isResolved) { isResolved = true; const errorMessage = error instanceof Error ? error.message : String(error); let errorStage: ConnectionStage; if ( errorMessage.includes("ENOTFOUND") || errorMessage.includes("getaddrinfo") ) { errorStage = "dns"; connectionLogs.push( createConnectionLog( "error", errorStage, `DNS resolution failed: ${errorMessage}`, ), ); } else if ( errorMessage.includes("ECONNREFUSED") || errorMessage.includes("ETIMEDOUT") ) { errorStage = "tcp"; connectionLogs.push( createConnectionLog( "error", errorStage, `TCP connection failed: ${errorMessage}`, ), ); } else if ( errorMessage.includes("handshake") || errorMessage.includes("key exchange") ) { errorStage = "handshake"; connectionLogs.push( createConnectionLog( "error", errorStage, `SSH handshake failed: ${errorMessage}`, ), ); } else if ( errorMessage.includes("authentication") || errorMessage.includes("Authentication") ) { errorStage = "auth"; connectionLogs.push( createConnectionLog( "error", errorStage, `Authentication failed: ${errorMessage}`, ), ); } else if (errorMessage.includes("verification failed")) { errorStage = "handshake"; connectionLogs.push( createConnectionLog( "error", errorStage, `SSH host key has changed. For security, please open a Terminal connection to this host first to verify and accept the new key fingerprint.`, ), ); } else { connectionLogs.push( createConnectionLog( "error", "error", `SSH connection failed: ${errorMessage}`, ), ); } statsLogger.error("SSH connection error in metrics/start", { operation: "metrics_start_ssh_error", hostId: host.id, error: errorMessage, }); reject(error); } }); const hasJumpHosts = host.jumpHosts && host.jumpHosts.length > 0 && host.userId; if (hasJumpHosts) { connectionLogs.push( createConnectionLog( "info", "proxy", "Connecting via jump host chain", ), ); createJumpHostChain(host.jumpHosts!, host.userId!) .then((jumpClient) => { jumpClient.forwardOut( "127.0.0.1", 0, host.ip, host.port, (err, stream) => { if (err || !stream) { if (!isResolved) { isResolved = true; clearTimeout(timeout); reject(err || new Error("Jump host forward failed")); } return; } config.sock = stream; delete config.host; delete config.port; client.connect(config); }, ); }) .catch((error) => { if (!isResolved) { isResolved = true; clearTimeout(timeout); connectionLogs.push( createConnectionLog( "error", "proxy", `Jump host connection failed: ${error instanceof Error ? error.message : "Unknown error"}`, ), ); reject(error); } }); } else if ( host.useSocks5 && (host.socks5Host || (host.socks5ProxyChain && host.socks5ProxyChain.length > 0)) ) { connectionLogs.push( createConnectionLog("info", "proxy", "Connecting via SOCKS5 proxy"), ); createSocks5Connection(host.ip, host.port, { useSocks5: host.useSocks5, socks5Host: host.socks5Host, socks5Port: host.socks5Port, socks5Username: host.socks5Username, socks5Password: host.socks5Password, socks5ProxyChain: host.socks5ProxyChain, }) .then((socks5Socket) => { if (socks5Socket) { config.sock = socks5Socket; } client.connect(config); }) .catch((error) => { if (!isResolved) { isResolved = true; clearTimeout(timeout); connectionLogs.push( createConnectionLog( "error", "proxy", `SOCKS5 proxy connection failed: ${error instanceof Error ? error.message : "Unknown error"}`, ), ); reject(error); } }); } else { client.connect(config); } }); const result = await connectionPromise; res.json({ ...result, connectionLogs }); } catch (error) { statsLogger.error("Failed to start metrics collection", { operation: "metrics_start_error", hostId: id, error: error instanceof Error ? error.message : String(error), }); connectionLogs.push( createConnectionLog( "error", "stats_connecting", `Failed to start metrics: ${error instanceof Error ? error.message : "Unknown error"}`, ), ); res.status(500).json({ error: error instanceof Error ? error.message : "Failed to start metrics collection", connectionLogs, }); } }); /** * @openapi * /metrics/stop/{id}: * post: * summary: Stop metrics collection * description: Stops metrics collection for a specific host and cleans up the SSH session. * tags: * - Host Metrics * parameters: * - in: path * name: id * required: true * schema: * type: integer * requestBody: * required: false * content: * application/json: * schema: * type: object * properties: * viewerSessionId: * type: string * responses: * 200: * description: Metrics collection stopped successfully. * 401: * description: Session expired - please log in again. * 500: * description: Failed to stop metrics collection. */ app.post("/metrics/stop/:id", validateHostId, async (req, res) => { const id = Number(req.params.id); const userId = (req as AuthenticatedRequest).userId; const { viewerSessionId } = req.body; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } try { const sessionKey = getSessionKey(id, userId); const session = metricsSessions[sessionKey]; if (session) { cleanupMetricsSession(sessionKey); } if (viewerSessionId && typeof viewerSessionId === "string") { pollingManager.unregisterViewer(id, viewerSessionId); } else { pollingManager.stopMetricsOnly(id); } res.json({ success: true }); } catch (error) { statsLogger.error("Failed to stop metrics collection", { operation: "metrics_stop_error", hostId: id, error: error instanceof Error ? error.message : String(error), }); res.status(500).json({ error: error instanceof Error ? error.message : "Failed to stop metrics collection", }); } }); /** * @openapi * /metrics/connect-totp: * post: * summary: Complete TOTP verification for metrics * description: Verifies the TOTP code and completes the metrics SSH connection. * tags: * - Host Metrics * requestBody: * required: true * content: * application/json: * schema: * type: object * properties: * sessionId: * type: string * totpCode: * type: string * responses: * 200: * description: TOTP verified, metrics connection established. * 400: * description: Missing sessionId or totpCode. * 401: * description: Session expired or invalid TOTP code. * 404: * description: TOTP session not found or expired. * 500: * description: Failed to verify TOTP. */ app.post("/metrics/connect-totp", async (req, res) => { const { sessionId, totpCode } = req.body; const userId = (req as AuthenticatedRequest).userId; if (!SimpleDBOps.isUserDataUnlocked(userId)) { return res.status(401).json({ error: "Session expired - please log in again", code: "SESSION_EXPIRED", }); } if (!sessionId || !totpCode) { return res.status(400).json({ error: "Missing sessionId or totpCode" }); } const session = pendingTOTPSessions[sessionId]; if (!session) { return res.status(404).json({ error: "TOTP session not found or expired" }); } if (Date.now() - session.createdAt > 180000) { delete pendingTOTPSessions[sessionId]; try { session.client.end(); } catch { // expected } return res.status(408).json({ error: "TOTP session timeout" }); } if (session.userId !== userId) { return res.status(403).json({ error: "Unauthorized" }); } session.totpAttempts++; if (session.totpAttempts > 3) { delete pendingTOTPSessions[sessionId]; try { session.client.end(); } catch { // expected } return res.status(429).json({ error: "Too many TOTP attempts" }); } try { const responses = (session.prompts || []).map((p, idx) => { if (idx === session.totpPromptIndex) { return totpCode.trim(); } else if (/password/i.test(p.prompt) && session.resolvedPassword) { return session.resolvedPassword; } return ""; }); const connectionPromise = new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error("TOTP verification timeout")); }, 30000); session.client.once( "keyboard-interactive", (name, instructions, instructionsLang, prompts, finish) => { statsLogger.warn("Second keyboard-interactive received after TOTP", { operation: "totp_second_keyboard_interactive", hostId: session.hostId, sessionId, prompts: prompts.map((p) => p.prompt), }); const secondResponses = prompts.map((p) => { if (/password/i.test(p.prompt) && session.resolvedPassword) { return session.resolvedPassword; } return ""; }); finish(secondResponses); }, ); session.client.once("ready", () => { clearTimeout(timeout); resolve(); }); session.client.once("error", (error) => { clearTimeout(timeout); statsLogger.error("SSH client error after TOTP", { operation: "totp_client_error", hostId: session.hostId, sessionId, error: error instanceof Error ? error.message : String(error), }); reject(error); }); }); session.finish(responses); await connectionPromise; const sessionKey = getSessionKey(session.hostId, userId); metricsSessions[sessionKey] = { client: session.client, isConnected: true, lastActive: Date.now(), activeOperations: 0, hostId: session.hostId, userId, }; scheduleMetricsSessionCleanup(sessionKey); delete pendingTOTPSessions[sessionId]; const viewerSessionId = `viewer-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; pollingManager.registerViewer(session.hostId, viewerSessionId, userId); res.json({ success: true, viewerSessionId }); } catch (error) { statsLogger.error("TOTP verification failed", { operation: "totp_verification_failed", hostId: session.hostId, sessionId, error: error instanceof Error ? error.message : String(error), }); if (session.totpAttempts >= 3) { delete pendingTOTPSessions[sessionId]; try { session.client.end(); } catch { // expected } } res.status(401).json({ error: "TOTP verification failed", attemptsRemaining: Math.max(0, 3 - session.totpAttempts), }); } }); registerHostMetricsViewerRoutes(app, { fetchHostById, supportsMetrics: (host: SSHHostWithCredentials) => supportsMetrics(host), parseStatsConfig: (statsConfig: SSHHostWithCredentials["statsConfig"]) => pollingManager.parseStatsConfig(statsConfig), updateHeartbeat: (viewerSessionId) => pollingManager.updateHeartbeat(viewerSessionId), registerViewer: (hostId, viewerSessionId, userId) => pollingManager.registerViewer(hostId, viewerSessionId, userId), unregisterViewer: (hostId, viewerSessionId) => pollingManager.unregisterViewer(hostId, viewerSessionId), }); registerHostMetricsSettingsRoutes(app, { requireAdmin, defaultStatsConfig: DEFAULT_STATS_CONFIG, refreshAllPolling: () => pollingManager.refreshAllPolling(), }); registerHostMetricsPreferencesRoutes(app, { validateHostId, fetchHostById: async (hostId, userId) => (await fetchHostById(hostId, userId)) ?? null, parseStatsConfig: (statsConfig) => pollingManager.parseStatsConfig( statsConfig as string | StatsConfig | undefined, ), canAccessHost: async (userId, hostId, level) => (await permissionManager.canAccessHost(userId, hostId, level)).hasAccess, }); registerHostMetricsHistoryRoutes(app, { validateHostId, canAccessHost: async (userId, hostId, level) => (await permissionManager.canAccessHost(userId, hostId, level)).hasAccess, }); registerManagerRoutes(app, { validateHostId, runOnHost: async (hostId, userId, level, fn) => { const access = await permissionManager.canAccessHost(userId, hostId, level); if (!access.hasAccess) { throw new AccessDeniedError(); } const host = await fetchHostById(hostId, userId); if (!host) { throw new AccessDeniedError("Host not found"); } // sudoPassword is encrypted with the owner's key; when the viewer is a // different user, re-fetch under the owner's key to decrypt it correctly. let sudoPassword = host.sudoPassword; if (host.userId && host.userId !== userId) { const ownerHost = await fetchHostById(hostId, host.userId); if (ownerHost) sudoPassword = ownerHost.sudoPassword; } const managerHost: ManagerHost = { id: host.id, userId: host.userId, sudoPassword, enableDocker: !!(host as { enableDocker?: boolean }).enableDocker, }; return withSshConnection(host, (client) => fn(client, managerHost)); }, }); // Internal endpoint — only accepts calls from localhost. // Used by the main backend to notify the metrics service of SSH login events. app.post("/internal/login-alert", async (req, res) => { const remoteIp = req.socket.remoteAddress; if ( remoteIp !== "127.0.0.1" && remoteIp !== "::1" && remoteIp !== "::ffff:127.0.0.1" ) { return res.status(403).json({ error: "Forbidden" }); } const systemCrypto = (await import("../utils/system-crypto.js")).SystemCrypto; const expectedToken = await systemCrypto.getInstance().getInternalAuthToken(); const token = req.headers["x-internal-auth"]; if (!token || token !== expectedToken) { return res.status(403).json({ error: "Forbidden" }); } const { hostId, userId, sshUser, fromIp } = req.body as { hostId: number; userId: string; sshUser: string; fromIp: string; }; AlertEngine.getInstance() .evaluateUserLogin(hostId, userId, sshUser, fromIp) .catch(() => {}); res.json({ ok: true }); }); process.on("SIGINT", () => { pollingManager.destroy(); connectionPool.destroy(); process.exit(0); }); process.on("SIGTERM", () => { pollingManager.destroy(); connectionPool.destroy(); process.exit(0); }); const PORT = 30005; app.listen(PORT, async () => { try { await authManager.initialize(); } catch (err) { statsLogger.error("Failed to initialize AuthManager", err, { operation: "auth_init_error", }); } setInterval( () => { authFailureTracker.cleanup(); pollingBackoff.cleanup(); }, 10 * 60 * 1000, ); });