import express, { type Response } from "express"; import { createServer } from "http"; import { createServer as createTcpServer, Socket as TcpSocket, type Server as TcpServer, } from "net"; import { createCorsMiddleware } from "../utils/cors-config.js"; import cookieParser from "cookie-parser"; import { Client, type ClientChannel } from "ssh2"; import { WebSocketServer } from "ws"; import { SSH_ALGORITHMS } from "../utils/ssh-algorithms.js"; import { ChildProcess } from "child_process"; import axios from "axios"; import { getDb } from "../database/db/index.js"; import { sshCredentials } from "../database/db/schema.js"; import { eq } from "drizzle-orm"; import type { SSHHost, TunnelConfig, TunnelStatus, VerificationData, AuthenticatedRequest, } from "../../types/index.js"; import { CONNECTION_STATES } from "../../types/index.js"; import { tunnelLogger } from "../utils/logger.js"; import { logAudit } from "../utils/audit-logger.js"; import { SystemCrypto } from "../utils/system-crypto.js"; import { SimpleDBOps } from "../utils/simple-db-ops.js"; import { DataCrypto } from "../utils/data-crypto.js"; import { createSocks5Connection } from "../utils/socks5-helper.js"; import { AuthManager } from "../utils/auth-manager.js"; import { PermissionManager } from "../utils/permission-manager.js"; import { withConnection } from "./ssh-connection-pool.js"; import { applyAuthOptions, bindForwardIn, connectClient, forwardOut, getManagedTunnelAlgorithms, pipeTunnelStreams, unbindForwardIn, } from "./tunnel-ssh-primitives.js"; import { classifyTunnelError, getTunnelBindHost, getTunnelMarker, getTunnelMode, getTunnelScope, normalizeTunnelName, validateTunnelConfig, } from "./tunnel-utils.js"; import { describeC2SRelayError, extractRequestToken, sendC2SError, } from "./tunnel-c2s-relay-utils.js"; import { handleC2SRelayOpen, handleC2SRelayTest, type C2SOpenMessage, } from "./tunnel-c2s-relay.js"; import { handleSocks5Connect } from "./tunnel-socks5-relay.js"; const app = express(); app.use(createCorsMiddleware(["GET", "POST", "PUT", "DELETE", "OPTIONS"])); app.use(cookieParser()); app.use(express.json()); app.use((_req, res, next) => { res.setHeader("Cache-Control", "no-store"); next(); }); const authManager = AuthManager.getInstance(); const permissionManager = PermissionManager.getInstance(); const authenticateJWT = authManager.createAuthMiddleware(); const activeTunnels = new Map(); const retryCounters = new Map(); const connectionStatus = new Map(); const tunnelVerifications = new Map(); const manualDisconnects = new Set(); const verificationTimers = new Map(); const activeRetryTimers = new Map(); const countdownIntervals = new Map(); const retryExhaustedTunnels = new Set(); const cleanupInProgress = new Set(); const tunnelConnecting = new Set(); const lastTunnelErrors = new Map(); const lastTunnelErrorTypes = new Map(); const tunnelConfigs = new Map(); const activeTunnelProcesses = new Map(); const pendingTunnelOperations = new Map>(); const tunnelStatusClients = new Set(); type ActiveTunnelRuntime = { sourceClient: Client; endpointClient?: Client; bindClient?: Client; bindHost?: string; bindPort?: number; tcpServer?: TcpServer; close: () => void; }; const activeTunnelRuntimes = new Map(); function broadcastTunnelStatus(tunnelName: string, status: TunnelStatus): void { if ( status.status === CONNECTION_STATES.CONNECTED && activeRetryTimers.has(tunnelName) ) { return; } const nextStatus = { ...status }; if ( retryExhaustedTunnels.has(tunnelName) && nextStatus.status === CONNECTION_STATES.FAILED ) { const previousReason = lastTunnelErrors.get(tunnelName); nextStatus.reason = previousReason ? `Max retries exhausted: ${previousReason}` : "Max retries exhausted"; } if (nextStatus.status === CONNECTION_STATES.FAILED && nextStatus.reason) { lastTunnelErrors.set(tunnelName, nextStatus.reason); if (nextStatus.errorType) { lastTunnelErrorTypes.set(tunnelName, nextStatus.errorType); } } else if ( (nextStatus.status === CONNECTION_STATES.CONNECTING || nextStatus.status === CONNECTION_STATES.RETRYING || nextStatus.status === CONNECTION_STATES.WAITING) && !nextStatus.reason ) { nextStatus.reason = lastTunnelErrors.get(tunnelName); nextStatus.errorType = lastTunnelErrorTypes.get(tunnelName); } else if ( nextStatus.status === CONNECTION_STATES.CONNECTED || (nextStatus.status === CONNECTION_STATES.DISCONNECTED && nextStatus.manualDisconnect) ) { lastTunnelErrors.delete(tunnelName); lastTunnelErrorTypes.delete(tunnelName); } connectionStatus.set(tunnelName, nextStatus); broadcastTunnelStatusSnapshot(); } function getAllTunnelStatus(): Record { const tunnelStatus: Record = {}; connectionStatus.forEach((status, key) => { tunnelStatus[key] = status; }); return tunnelStatus; } function sendTunnelStatusSnapshot(res: Response): void { try { res.write( `event: statuses\ndata: ${JSON.stringify(getAllTunnelStatus())}\n\n`, ); } catch { tunnelStatusClients.delete(res); } } function broadcastTunnelStatusSnapshot(): void { for (const client of tunnelStatusClients) { sendTunnelStatusSnapshot(client); } } async function cleanupTunnelResources( tunnelName: string, forceCleanup = false, ): Promise { if (cleanupInProgress.has(tunnelName)) { return; } if (!forceCleanup && tunnelConnecting.has(tunnelName)) { return; } cleanupInProgress.add(tunnelName); const tunnelConfig = tunnelConfigs.get(tunnelName); const runtime = activeTunnelRuntimes.get(tunnelName); if (runtime) { try { runtime.close(); } catch (error) { tunnelLogger.error("Error while closing managed tunnel runtime", error, { operation: "managed_tunnel_cleanup", tunnelName, }); } activeTunnelRuntimes.delete(tunnelName); cleanupInProgress.delete(tunnelName); } else if (tunnelConfig) { await new Promise((resolve) => { killRemoteTunnelByMarker(tunnelConfig, tunnelName, (err) => { cleanupInProgress.delete(tunnelName); if (err) { tunnelLogger.error( `Failed to kill remote tunnel for '${tunnelName}': ${err.message}`, ); } resolve(); }); }); } else { cleanupInProgress.delete(tunnelName); } if (activeTunnelProcesses.has(tunnelName)) { try { const proc = activeTunnelProcesses.get(tunnelName); if (proc) { proc.kill("SIGTERM"); } } catch (e) { tunnelLogger.error( `Error while killing local ssh process for tunnel '${tunnelName}'`, e, ); } activeTunnelProcesses.delete(tunnelName); } if (activeTunnels.has(tunnelName)) { try { const conn = activeTunnels.get(tunnelName); if (conn) { conn.end(); } } catch (e) { tunnelLogger.error( `Error while closing SSH2 Client for tunnel '${tunnelName}'`, e, ); } activeTunnels.delete(tunnelName); } if (tunnelVerifications.has(tunnelName)) { const verification = tunnelVerifications.get(tunnelName); if (verification?.timeout) clearTimeout(verification.timeout); try { verification?.conn.end(); } catch (error) { tunnelLogger.error("Error during tunnel cleanup", error, { operation: "tunnel_cleanup_error", tunnelName, }); } tunnelVerifications.delete(tunnelName); } const timerKeys = [ tunnelName, `${tunnelName}_confirm`, `${tunnelName}_retry`, `${tunnelName}_verify_retry`, `${tunnelName}_ping`, ]; timerKeys.forEach((key) => { if (verificationTimers.has(key)) { clearTimeout(verificationTimers.get(key)!); verificationTimers.delete(key); } }); if (activeRetryTimers.has(tunnelName)) { clearTimeout(activeRetryTimers.get(tunnelName)!); activeRetryTimers.delete(tunnelName); } if (countdownIntervals.has(tunnelName)) { clearInterval(countdownIntervals.get(tunnelName)!); countdownIntervals.delete(tunnelName); } } function resetRetryState(tunnelName: string): void { retryCounters.delete(tunnelName); retryExhaustedTunnels.delete(tunnelName); lastTunnelErrors.delete(tunnelName); lastTunnelErrorTypes.delete(tunnelName); cleanupInProgress.delete(tunnelName); tunnelConnecting.delete(tunnelName); if (activeRetryTimers.has(tunnelName)) { clearTimeout(activeRetryTimers.get(tunnelName)!); activeRetryTimers.delete(tunnelName); } if (countdownIntervals.has(tunnelName)) { clearInterval(countdownIntervals.get(tunnelName)!); countdownIntervals.delete(tunnelName); } ["", "_confirm", "_retry", "_verify_retry", "_ping"].forEach((suffix) => { const timerKey = `${tunnelName}${suffix}`; if (verificationTimers.has(timerKey)) { clearTimeout(verificationTimers.get(timerKey)!); verificationTimers.delete(timerKey); } }); } async function handleDisconnect( tunnelName: string, tunnelConfig: TunnelConfig | null, shouldRetry = true, ): Promise { if (tunnelVerifications.has(tunnelName)) { try { const verification = tunnelVerifications.get(tunnelName); if (verification?.timeout) clearTimeout(verification.timeout); verification?.conn.end(); } catch (error) { tunnelLogger.error("Error during tunnel cleanup", error, { operation: "tunnel_cleanup_error", tunnelName, }); } tunnelVerifications.delete(tunnelName); } while (cleanupInProgress.has(tunnelName)) { await new Promise((resolve) => setTimeout(resolve, 100)); } await cleanupTunnelResources(tunnelName); if (manualDisconnects.has(tunnelName)) { resetRetryState(tunnelName); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.DISCONNECTED, manualDisconnect: true, }); return; } if (retryExhaustedTunnels.has(tunnelName)) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: "Max retries already exhausted", }); return; } if (activeRetryTimers.has(tunnelName)) { return; } if (shouldRetry && tunnelConfig) { const maxRetries = tunnelConfig.maxRetries || 3; const retryInterval = tunnelConfig.retryInterval || 5000; let retryCount = retryCounters.get(tunnelName) || 0; retryCount = retryCount + 1; if (retryCount > maxRetries) { tunnelLogger.error(`All ${maxRetries} retries failed for ${tunnelName}`); retryExhaustedTunnels.add(tunnelName); activeTunnels.delete(tunnelName); retryCounters.delete(tunnelName); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, retryExhausted: true, reason: `Max retries exhausted`, }); return; } retryCounters.set(tunnelName, retryCount); if (retryCount <= maxRetries) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.RETRYING, retryCount: retryCount, maxRetries: maxRetries, nextRetryIn: retryInterval / 1000, }); if (activeRetryTimers.has(tunnelName)) { clearTimeout(activeRetryTimers.get(tunnelName)!); activeRetryTimers.delete(tunnelName); } const initialNextRetryIn = Math.ceil(retryInterval / 1000); let currentNextRetryIn = initialNextRetryIn; broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.WAITING, retryCount: retryCount, maxRetries: maxRetries, nextRetryIn: currentNextRetryIn, }); const countdownInterval = setInterval(() => { currentNextRetryIn--; if (currentNextRetryIn > 0) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.WAITING, retryCount: retryCount, maxRetries: maxRetries, nextRetryIn: currentNextRetryIn, }); } }, 1000); countdownIntervals.set(tunnelName, countdownInterval); const timer = setTimeout(() => { clearInterval(countdownInterval); countdownIntervals.delete(tunnelName); activeRetryTimers.delete(tunnelName); if (!manualDisconnects.has(tunnelName)) { activeTunnels.delete(tunnelName); connectSSHTunnel(tunnelConfig, retryCount).catch((error) => { tunnelLogger.error( `Failed to connect tunnel ${tunnelConfig.name}: ${error instanceof Error ? error.message : "Unknown error"}`, ); }); } }, retryInterval); activeRetryTimers.set(tunnelName, timer); } } else { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, }); activeTunnels.delete(tunnelName); } } function setupPingInterval(tunnelName: string): void { const pingKey = `${tunnelName}_ping`; if (verificationTimers.has(pingKey)) { clearInterval(verificationTimers.get(pingKey)!); verificationTimers.delete(pingKey); } const pingInterval = setInterval(() => { const currentStatus = connectionStatus.get(tunnelName); if (currentStatus?.status === CONNECTION_STATES.CONNECTED) { if (!activeTunnels.has(tunnelName)) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.DISCONNECTED, reason: "Tunnel connection lost", }); clearInterval(pingInterval); verificationTimers.delete(pingKey); } } else { clearInterval(pingInterval); verificationTimers.delete(pingKey); } }, 120000); verificationTimers.set(pingKey, pingInterval); } async function connectEndpointThroughSource( sourceClient: Client, tunnelConfig: TunnelConfig, endpointCredentials: { password?: string; sshKey?: string; keyPassword?: string; keyType?: string; authMethod?: string; }, ): Promise { const endpointSock = await forwardOut( sourceClient, tunnelConfig.endpointIP, tunnelConfig.endpointSSHPort, tunnelConfig.name, ); const endpointOptions: Record = { sock: endpointSock, username: tunnelConfig.endpointUsername, tryKeyboard: true, keepaliveInterval: tunnelConfig.keepaliveInterval ?? 30000, keepaliveCountMax: tunnelConfig.keepaliveCountMax ?? 3, readyTimeout: 60000, tcpKeepAlive: true, tcpKeepAliveInitialDelay: 30000, algorithms: getManagedTunnelAlgorithms(), }; applyAuthOptions(endpointOptions, endpointCredentials); return connectClient(endpointOptions, tunnelConfig.name, "endpoint"); } function resolveS2SLocalTargetHost(tunnelConfig: TunnelConfig): string { const targetHost = tunnelConfig.targetHost?.trim(); if ( !targetHost || targetHost === tunnelConfig.endpointHost || targetHost === tunnelConfig.hostName ) { return "127.0.0.1"; } return targetHost; } function isSingleHostTunnel(tunnelConfig: TunnelConfig): boolean { if (!tunnelConfig.endpointHost && !tunnelConfig.endpointIP) return true; if ( tunnelConfig.endpointHost === "127.0.0.1" || tunnelConfig.endpointHost === "localhost" ) { return true; } if ( tunnelConfig.endpointIP && tunnelConfig.endpointIP === tunnelConfig.sourceIP && tunnelConfig.endpointSSHPort === tunnelConfig.sourceSSHPort ) { return true; } return false; } async function establishDirectTunnel( sourceClient: Client, tunnelConfig: TunnelConfig, ): Promise { const tunnelName = tunnelConfig.name; const mode = getTunnelMode(tunnelConfig); const bindHost = getTunnelBindHost(tunnelConfig); const sourcePort = tunnelConfig.sourcePort; const targetHost = tunnelConfig.targetHost || "127.0.0.1"; const targetPort = tunnelConfig.endpointPort; if (mode === "remote") { const remoteBindPort = await bindForwardIn( sourceClient, targetHost, sourcePort, ); const sockets = new Set(); sourceClient.on("tcp connection", (info, accept, reject) => { if (info.destPort !== remoteBindPort) { reject(); return; } const inbound = accept(); const local = new TcpSocket(); sockets.add(local); local.connect(targetPort, bindHost, () => { pipeTunnelStreams(inbound, Promise.resolve(local), tunnelName); }); local.on("error", () => { inbound.destroy(); sockets.delete(local); }); local.on("close", () => sockets.delete(local)); }); const close = () => { unbindForwardIn(sourceClient, targetHost, remoteBindPort); for (const s of sockets) s.destroy(); sockets.clear(); try { sourceClient.end(); } catch { // expected } }; activeTunnelRuntimes.set(tunnelName, { sourceClient, bindHost: targetHost, bindPort: remoteBindPort, close, }); activeTunnels.set(tunnelName, sourceClient); return; } // Local and dynamic modes: listen locally, forward through SSH const sockets = new Set(); const tcpServer = createTcpServer((socket) => { sockets.add(socket); socket.on("close", () => sockets.delete(socket)); socket.on("error", () => { sockets.delete(socket); socket.destroy(); }); if (mode === "dynamic") { handleSocks5Connect( socket, (host, port) => forwardOut(sourceClient, host, port), tunnelName, ); return; } forwardOut(sourceClient, targetHost, targetPort, tunnelName) .then((outbound) => pipeTunnelStreams(socket, Promise.resolve(outbound), tunnelName), ) .catch(() => socket.destroy()); }); await new Promise((resolve, reject) => { tcpServer.once("error", reject); tcpServer.listen({ host: bindHost, port: sourcePort }, () => { tcpServer.removeListener("error", reject); resolve(); }); }); tunnelLogger.info("Direct tunnel listener started", { operation: "direct_tunnel_listen", tunnelName, mode, bindHost, sourcePort, targetHost, targetPort, }); const close = () => { for (const s of sockets) s.destroy(); sockets.clear(); tcpServer.close(); try { sourceClient.end(); } catch { // expected } }; sourceClient.on("close", () => { close(); }); activeTunnelRuntimes.set(tunnelName, { sourceClient, tcpServer, bindHost, bindPort: sourcePort, close, }); activeTunnels.set(tunnelName, sourceClient); } async function establishManagedS2STunnel( sourceClient: Client, tunnelConfig: TunnelConfig, endpointCredentials: { password?: string; sshKey?: string; keyPassword?: string; keyType?: string; authMethod?: string; }, ): Promise { const tunnelName = tunnelConfig.name; const mode = getTunnelMode(tunnelConfig); const bindHost = getTunnelBindHost(tunnelConfig); const endpointClient = await connectEndpointThroughSource( sourceClient, tunnelConfig, endpointCredentials, ); const bindClient = mode === "remote" ? endpointClient : sourceClient; const outboundClient = mode === "remote" ? sourceClient : endpointClient; const bindPort = mode === "remote" ? tunnelConfig.endpointPort : tunnelConfig.sourcePort; const staticTargetHost = mode === "remote" ? tunnelConfig.targetHost || "127.0.0.1" : resolveS2SLocalTargetHost(tunnelConfig); const staticTargetPort = mode === "remote" ? tunnelConfig.sourcePort : tunnelConfig.endpointPort; tunnelLogger.info("Managed S2S tunnel route resolved", { operation: "managed_tunnel_route_resolved", tunnelName, mode, bindHost, bindPort, targetHost: staticTargetHost, targetPort: staticTargetPort, endpointHost: tunnelConfig.endpointHost, endpointIP: tunnelConfig.endpointIP, }); const actualPort = await bindForwardIn(bindClient, bindHost, bindPort); const tcpHandler = ( info: { destIP: string; destPort: number; srcIP: string; srcPort: number; }, accept: () => ClientChannel, reject: () => void, ) => { if (info.destPort !== actualPort) { reject(); return; } const inbound = accept(); if (mode === "dynamic") { handleSocks5Connect( inbound, (host, port) => forwardOut(outboundClient, host, port), tunnelName, ); return; } pipeTunnelStreams( inbound, forwardOut( outboundClient, staticTargetHost, staticTargetPort, tunnelName, ), tunnelName, ); }; bindClient.on("tcp connection", tcpHandler); const close = () => { bindClient.off("tcp connection", tcpHandler); unbindForwardIn(bindClient, bindHost, actualPort); try { endpointClient.end(); } catch { // expected during shutdown } try { sourceClient.end(); } catch { // expected during shutdown } }; activeTunnelRuntimes.set(tunnelName, { sourceClient, endpointClient, bindClient, bindHost, bindPort: actualPort, close, }); activeTunnels.set(tunnelName, sourceClient); } async function connectSSHTunnel( tunnelConfig: TunnelConfig, retryAttempt = 0, ): Promise { const tunnelName = tunnelConfig.name; tunnelLogger.info("Tunnel creation request received", { operation: "tunnel_create_request", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, tunnelType: tunnelConfig.tunnelType || "remote", sourcePort: tunnelConfig.sourcePort, endpointHost: tunnelConfig.endpointHost, endpointPort: tunnelConfig.endpointPort, }); if (manualDisconnects.has(tunnelName)) { return; } tunnelConnecting.add(tunnelName); await cleanupTunnelResources(tunnelName, true); if (retryAttempt === 0) { retryExhaustedTunnels.delete(tunnelName); retryCounters.delete(tunnelName); } const currentStatus = connectionStatus.get(tunnelName); if (!currentStatus || currentStatus.status !== CONNECTION_STATES.WAITING) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.CONNECTING, retryCount: retryAttempt > 0 ? retryAttempt : undefined, }); } if ( !tunnelConfig || !tunnelConfig.sourceIP || !tunnelConfig.sourceUsername || !tunnelConfig.sourceSSHPort ) { const missingFields = []; if (!tunnelConfig) missingFields.push("tunnelConfig"); if (!tunnelConfig?.sourceIP) missingFields.push("sourceIP"); if (!tunnelConfig?.sourceUsername) missingFields.push("sourceUsername"); if (!tunnelConfig?.sourceSSHPort) missingFields.push("sourceSSHPort"); tunnelLogger.error("Invalid tunnel connection details", undefined, { operation: "tunnel_connect_validation_failed", tunnelName, missingFields: missingFields.join(", "), hasSourceIP: !!tunnelConfig?.sourceIP, hasSourceUsername: !!tunnelConfig?.sourceUsername, hasSourceSSHPort: !!tunnelConfig?.sourceSSHPort, }); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: "Missing required connection details", }); tunnelConnecting.delete(tunnelName); return; } let resolvedSourceCredentials = { password: tunnelConfig.sourcePassword, sshKey: tunnelConfig.sourceSSHKey, keyPassword: tunnelConfig.sourceKeyPassword, keyType: tunnelConfig.sourceKeyType, authMethod: tunnelConfig.sourceAuthMethod, }; const effectiveUserId = tunnelConfig.requestingUserId || tunnelConfig.sourceUserId; // Resolve source credentials server-side when not provided by frontend if ( tunnelConfig.sourceHostId && effectiveUserId && !tunnelConfig.sourcePassword && !tunnelConfig.sourceSSHKey ) { try { const { resolveHostById } = await import("./host-resolver.js"); const resolvedHost = await resolveHostById( tunnelConfig.sourceHostId, effectiveUserId, ); if (resolvedHost) { resolvedSourceCredentials = { password: resolvedHost.password, sshKey: resolvedHost.key, keyPassword: resolvedHost.keyPassword, keyType: resolvedHost.keyType, authMethod: resolvedHost.authType, }; if (tunnelConfig.keepaliveInterval === undefined) { tunnelConfig.keepaliveInterval = typeof resolvedHost.terminalConfig?.keepaliveInterval === "number" ? resolvedHost.terminalConfig.keepaliveInterval * 1000 : 60000; } if (tunnelConfig.keepaliveCountMax === undefined) { tunnelConfig.keepaliveCountMax = typeof resolvedHost.terminalConfig?.keepaliveCountMax === "number" ? resolvedHost.terminalConfig.keepaliveCountMax : 5; } } } catch (error) { tunnelLogger.warn("Failed to resolve source host credentials", { operation: "tunnel_connect", tunnelName, sourceHostId: tunnelConfig.sourceHostId, error: error instanceof Error ? error.message : "Unknown error", }); } } else if (tunnelConfig.sourceCredentialId && effectiveUserId) { // Legacy: credential resolution from credentialId try { if (tunnelConfig.sourceHostId) { const { resolveHostById } = await import("./host-resolver.js"); const resolvedHost = await resolveHostById( tunnelConfig.sourceHostId, effectiveUserId, ); if (resolvedHost) { resolvedSourceCredentials = { password: resolvedHost.password, sshKey: resolvedHost.key, keyPassword: resolvedHost.keyPassword, keyType: resolvedHost.keyType, authMethod: resolvedHost.authType, }; } } } catch (error) { tunnelLogger.warn("Failed to resolve source credentials", { operation: "tunnel_connect", tunnelName, credentialId: tunnelConfig.sourceCredentialId, error: error instanceof Error ? error.message : "Unknown error", }); } } let resolvedEndpointCredentials = { password: tunnelConfig.endpointPassword, sshKey: tunnelConfig.endpointSSHKey, keyPassword: tunnelConfig.endpointKeyPassword, keyType: tunnelConfig.endpointKeyType, authMethod: tunnelConfig.endpointAuthMethod, }; if (tunnelConfig.endpointCredentialId && tunnelConfig.endpointUserId) { try { const userDataKey = DataCrypto.getUserDataKey( tunnelConfig.endpointUserId, ); if (userDataKey) { const credentials = await SimpleDBOps.select( getDb() .select() .from(sshCredentials) .where(eq(sshCredentials.id, tunnelConfig.endpointCredentialId)), "ssh_credentials", tunnelConfig.endpointUserId, ); if (credentials.length > 0) { const credential = credentials[0]; resolvedEndpointCredentials = { password: credential.password as string | undefined, sshKey: (credential.key || credential.privateKey) as | string | undefined, keyPassword: credential.keyPassword as string | undefined, keyType: credential.keyType as string | undefined, authMethod: credential.authType as string, }; } else { tunnelLogger.warn("No endpoint credentials found in database", { operation: "tunnel_connect", tunnelName, credentialId: tunnelConfig.endpointCredentialId, }); } } } catch (error) { tunnelLogger.warn( `Failed to resolve endpoint credentials for tunnel ${tunnelName}: ${error instanceof Error ? error.message : "Unknown error"}`, ); } } else if (tunnelConfig.endpointCredentialId) { tunnelLogger.warn("Missing userId for endpoint credential resolution", { operation: "tunnel_connect", tunnelName, credentialId: tunnelConfig.endpointCredentialId, hasUserId: !!tunnelConfig.endpointUserId, }); } if ( resolvedEndpointCredentials.authMethod === "password" && !resolvedEndpointCredentials.password ) { const errorMessage = `Cannot connect tunnel '${tunnelName}': endpoint host requires password authentication but no plaintext password available. Enable autostart for endpoint host or configure credentials in tunnel connection.`; tunnelLogger.error(errorMessage, undefined, { operation: "tunnel_endpoint_password_unavailable", tunnelName, endpointHost: `${tunnelConfig.endpointUsername}@${tunnelConfig.endpointIP}:${tunnelConfig.endpointPort}`, endpointAuthMethod: resolvedEndpointCredentials.authMethod, }); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: errorMessage, }); tunnelConnecting.delete(tunnelName); return; } if ( resolvedEndpointCredentials.authMethod === "key" && !resolvedEndpointCredentials.sshKey ) { const errorMessage = `Cannot connect tunnel '${tunnelName}': endpoint host requires key authentication but no plaintext key available. Enable autostart for endpoint host or configure credentials in tunnel connection.`; tunnelLogger.error(errorMessage, undefined, { operation: "tunnel_endpoint_key_unavailable", tunnelName, endpointHost: `${tunnelConfig.endpointUsername}@${tunnelConfig.endpointIP}:${tunnelConfig.endpointPort}`, endpointAuthMethod: resolvedEndpointCredentials.authMethod, }); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: errorMessage, }); tunnelConnecting.delete(tunnelName); return; } const conn = new Client(); const connectionTimeout = setTimeout(() => { if (conn) { if (activeRetryTimers.has(tunnelName)) { return; } tunnelLogger.error( `Tunnel connection timeout after 60 seconds for '${tunnelName}'`, undefined, { operation: "tunnel_connection_timeout", tunnelName, sourceHost: `${tunnelConfig.sourceUsername}@${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`, endpointHost: `${tunnelConfig.endpointUsername}@${tunnelConfig.endpointIP}:${tunnelConfig.endpointPort}`, retryAttempt, usingSocks5: tunnelConfig.useSocks5 || false, }, ); try { conn.end(); } catch { // expected } activeTunnels.delete(tunnelName); if (!activeRetryTimers.has(tunnelName)) { handleDisconnect( tunnelName, tunnelConfig, !manualDisconnects.has(tunnelName), ); } } }, 60000); conn.on("error", (err) => { clearTimeout(connectionTimeout); const errorType = classifyTunnelError(err.message); tunnelLogger.error(`Tunnel connection failed for '${tunnelName}'`, err, { operation: "tunnel_connect_error", tunnelName, errorType, errorMessage: err.message, sourceHost: `${tunnelConfig.sourceUsername}@${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`, endpointHost: `${tunnelConfig.endpointUsername}@${tunnelConfig.endpointIP}:${tunnelConfig.endpointPort}`, tunnelType: tunnelConfig.tunnelType || "remote", sourcePort: tunnelConfig.sourcePort, retryAttempt, usingSocks5: tunnelConfig.useSocks5 || false, authMethod: tunnelConfig.sourceAuthMethod, }); tunnelConnecting.delete(tunnelName); if (activeRetryTimers.has(tunnelName)) { return; } if (!manualDisconnects.has(tunnelName)) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, errorType: errorType, reason: err.message, }); } activeTunnels.delete(tunnelName); const shouldNotRetry = errorType === "AUTHENTICATION_FAILED" || errorType === "CONNECTION_FAILED" || manualDisconnects.has(tunnelName); handleDisconnect(tunnelName, tunnelConfig, !shouldNotRetry); }); conn.on("close", () => { clearTimeout(connectionTimeout); tunnelConnecting.delete(tunnelName); if (activeRetryTimers.has(tunnelName)) { return; } if (!manualDisconnects.has(tunnelName)) { const currentStatus = connectionStatus.get(tunnelName); if (!currentStatus || currentStatus.status !== CONNECTION_STATES.FAILED) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.DISCONNECTED, }); } if (!activeRetryTimers.has(tunnelName)) { handleDisconnect( tunnelName, tunnelConfig, !manualDisconnects.has(tunnelName), ); } } }); conn.on("ready", async () => { clearTimeout(connectionTimeout); tunnelLogger.info("Creating managed SSH tunnel", { operation: "managed_tunnel_connection_create", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, scope: getTunnelScope(tunnelConfig), mode: getTunnelMode(tunnelConfig), }); const isAlreadyVerifying = tunnelVerifications.has(tunnelName); if (isAlreadyVerifying) { return; } try { if (getTunnelScope(tunnelConfig) !== "s2s") { throw new Error( "C2S tunnels must be started from the desktop client local configuration", ); } if (isSingleHostTunnel(tunnelConfig)) { await establishDirectTunnel(conn, tunnelConfig); } else { await establishManagedS2STunnel( conn, tunnelConfig, resolvedEndpointCredentials, ); } tunnelConnecting.delete(tunnelName); tunnelLogger.success("Managed tunnel creation complete", { operation: "managed_tunnel_create_complete", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, mode: getTunnelMode(tunnelConfig), sourcePort: tunnelConfig.sourcePort, endpointPort: tunnelConfig.endpointPort, }); logAudit({ userId: tunnelConfig.sourceUserId, username: tunnelConfig.sourceUserId, action: "tunnel_connect", resourceType: "tunnel", resourceId: String(tunnelConfig.sourceHostId), resourceName: tunnelName, details: JSON.stringify({ mode: getTunnelMode(tunnelConfig), sourcePort: tunnelConfig.sourcePort, }), success: true, }); broadcastTunnelStatus(tunnelName, { connected: true, status: CONNECTION_STATES.CONNECTED, }); setupPingInterval(tunnelName); } catch (error) { const message = error instanceof Error ? error.message : "Failed to create tunnel"; const errorType = classifyTunnelError(message); tunnelLogger.error("Failed to create managed tunnel", error, { operation: "managed_tunnel_create_failed", tunnelName, errorType, retryAttempt, }); tunnelConnecting.delete(tunnelName); activeTunnels.delete(tunnelName); activeTunnelRuntimes.delete(tunnelName); try { conn.end(); } catch { // expected } broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, errorType, reason: message, }); const shouldNotRetry = errorType === "AUTHENTICATION_FAILED" || errorType === "CONNECTION_FAILED"; handleDisconnect(tunnelName, tunnelConfig, !shouldNotRetry); } }); const connOptions: Record = { host: tunnelConfig.sourceIP?.replace(/^\[|\]$/g, "") || tunnelConfig.sourceIP, port: tunnelConfig.sourceSSHPort, username: tunnelConfig.sourceUsername, tryKeyboard: true, keepaliveInterval: tunnelConfig.keepaliveInterval ?? 30000, keepaliveCountMax: tunnelConfig.keepaliveCountMax ?? 3, readyTimeout: 60000, tcpKeepAlive: true, tcpKeepAliveInitialDelay: 30000, env: { TERM: "xterm-256color", LANG: "en_US.UTF-8", LC_ALL: "en_US.UTF-8", LC_CTYPE: "en_US.UTF-8", LC_MESSAGES: "en_US.UTF-8", LC_MONETARY: "en_US.UTF-8", LC_NUMERIC: "en_US.UTF-8", LC_TIME: "en_US.UTF-8", LC_COLLATE: "en_US.UTF-8", COLORTERM: "truecolor", }, algorithms: { kex: [ "curve25519-sha256", "curve25519-sha256@libssh.org", "ecdh-sha2-nistp521", "ecdh-sha2-nistp384", "ecdh-sha2-nistp256", "diffie-hellman-group-exchange-sha256", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1", "diffie-hellman-group-exchange-sha1", "diffie-hellman-group1-sha1", ], serverHostKey: [ "ssh-ed25519", "ecdsa-sha2-nistp521", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp256", "rsa-sha2-512", "rsa-sha2-256", "ssh-rsa", "ssh-dss", ], cipher: SSH_ALGORITHMS.cipher, hmac: [ "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512", "hmac-sha2-256", "hmac-sha1", "hmac-md5", ], compress: ["none", "zlib@openssh.com", "zlib"], }, }; if ( resolvedSourceCredentials.authMethod === "key" && resolvedSourceCredentials.sshKey ) { if ( !resolvedSourceCredentials.sshKey.includes("-----BEGIN") || !resolvedSourceCredentials.sshKey.includes("-----END") ) { tunnelLogger.error( `Invalid SSH key format for tunnel '${tunnelName}'. Key should contain both BEGIN and END markers`, undefined, { operation: "tunnel_invalid_ssh_key_format", tunnelName, sourceHost: `${tunnelConfig.sourceUsername}@${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`, keyType: resolvedSourceCredentials.keyType, hasBeginMarker: resolvedSourceCredentials.sshKey.includes("-----BEGIN"), hasEndMarker: resolvedSourceCredentials.sshKey.includes("-----END"), }, ); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: "Invalid SSH key format", }); tunnelConnecting.delete(tunnelName); return; } const cleanKey = resolvedSourceCredentials.sshKey .trim() .replace(/\r\n/g, "\n") .replace(/\r/g, "\n"); connOptions.privateKey = Buffer.from(cleanKey, "utf8"); if (resolvedSourceCredentials.keyPassword) { connOptions.passphrase = resolvedSourceCredentials.keyPassword; } if ( resolvedSourceCredentials.keyType && resolvedSourceCredentials.keyType !== "auto" ) { connOptions.privateKeyType = resolvedSourceCredentials.keyType; } } else if (resolvedSourceCredentials.authMethod === "key") { tunnelLogger.error( `SSH key authentication requested but no key provided for tunnel '${tunnelName}'`, undefined, { operation: "tunnel_ssh_key_missing", tunnelName, sourceHost: `${tunnelConfig.sourceUsername}@${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`, authMethod: resolvedSourceCredentials.authMethod, }, ); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: "SSH key authentication requested but no key provided", }); tunnelConnecting.delete(tunnelName); return; } else { connOptions.password = resolvedSourceCredentials.password; } const finalStatus = connectionStatus.get(tunnelName); if (!finalStatus || finalStatus.status !== CONNECTION_STATES.WAITING) { broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.CONNECTING, retryCount: retryAttempt > 0 ? retryAttempt : undefined, }); } if ( tunnelConfig.useSocks5 && (tunnelConfig.socks5Host || (tunnelConfig.socks5ProxyChain && tunnelConfig.socks5ProxyChain.length > 0)) ) { try { const socks5Socket = await createSocks5Connection( tunnelConfig.sourceIP, tunnelConfig.sourceSSHPort, { useSocks5: tunnelConfig.useSocks5, socks5Host: tunnelConfig.socks5Host, socks5Port: tunnelConfig.socks5Port, socks5Username: tunnelConfig.socks5Username, socks5Password: tunnelConfig.socks5Password, socks5ProxyChain: tunnelConfig.socks5ProxyChain, }, ); if (socks5Socket) { connOptions.sock = socks5Socket; conn.connect(connOptions); return; } } catch (socks5Error) { tunnelLogger.error("SOCKS5 connection failed for tunnel", socks5Error, { operation: "tunnel_socks5_connection_failed", tunnelName, sourceHost: `${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`, proxyHost: tunnelConfig.socks5Host, proxyPort: tunnelConfig.socks5Port || 1080, hasProxyAuth: !!( tunnelConfig.socks5Username && tunnelConfig.socks5Password ), errorMessage: socks5Error instanceof Error ? socks5Error.message : "Unknown error", }); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: "SOCKS5 proxy connection failed: " + (socks5Error instanceof Error ? socks5Error.message : "Unknown error"), }); tunnelConnecting.delete(tunnelName); return; } } conn.connect(connOptions); } async function killRemoteTunnelByMarker( tunnelConfig: TunnelConfig, tunnelName: string, callback: (err?: Error) => void, ) { const tunnelMarker = getTunnelMarker(tunnelName); tunnelLogger.info("Killing remote tunnel process", { operation: "tunnel_remote_kill", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, marker: tunnelMarker, }); let resolvedSourceCredentials = { password: tunnelConfig.sourcePassword, sshKey: tunnelConfig.sourceSSHKey, keyPassword: tunnelConfig.sourceKeyPassword, keyType: tunnelConfig.sourceKeyType, authMethod: tunnelConfig.sourceAuthMethod, }; if ( tunnelConfig.sourceHostId && tunnelConfig.sourceUserId && !tunnelConfig.sourcePassword && !tunnelConfig.sourceSSHKey ) { try { const { resolveHostById } = await import("./host-resolver.js"); const resolvedHost = await resolveHostById( tunnelConfig.sourceHostId, tunnelConfig.sourceUserId, ); if (resolvedHost) { resolvedSourceCredentials = { password: resolvedHost.password, sshKey: resolvedHost.key, keyPassword: resolvedHost.keyPassword, keyType: resolvedHost.keyType, authMethod: resolvedHost.authType, }; } } catch (error) { tunnelLogger.warn("Failed to resolve source credentials for cleanup", { tunnelName, sourceHostId: tunnelConfig.sourceHostId, error: error instanceof Error ? error.message : "Unknown error", }); } } if ( resolvedSourceCredentials.authMethod === "key" && resolvedSourceCredentials.sshKey ) { if ( !resolvedSourceCredentials.sshKey.includes("-----BEGIN") || !resolvedSourceCredentials.sshKey.includes("-----END") ) { callback(new Error("Invalid SSH key format")); return; } } const poolKey = `tunnel:${tunnelConfig.sourceUserId}:${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}:${tunnelConfig.sourceUsername}`; const factory = async (): Promise => { const connOptions: Record = { host: tunnelConfig.sourceIP?.replace(/^\[|\]$/g, "") || tunnelConfig.sourceIP, port: tunnelConfig.sourceSSHPort, username: tunnelConfig.sourceUsername, keepaliveInterval: tunnelConfig.keepaliveInterval ?? 60000, keepaliveCountMax: tunnelConfig.keepaliveCountMax ?? 5, readyTimeout: 60000, tcpKeepAlive: true, tcpKeepAliveInitialDelay: 30000, algorithms: { kex: [ "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1", "diffie-hellman-group1-sha1", "diffie-hellman-group-exchange-sha256", "diffie-hellman-group-exchange-sha1", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", ], cipher: [ "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com", "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc", ], hmac: [ "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1", "hmac-md5", ], compress: ["none", "zlib@openssh.com", "zlib"], }, }; if ( resolvedSourceCredentials.authMethod === "key" && resolvedSourceCredentials.sshKey ) { const cleanKey = resolvedSourceCredentials.sshKey .trim() .replace(/\r\n/g, "\n") .replace(/\r/g, "\n"); connOptions.privateKey = Buffer.from(cleanKey, "utf8"); if (resolvedSourceCredentials.keyPassword) { connOptions.passphrase = resolvedSourceCredentials.keyPassword; } if ( resolvedSourceCredentials.keyType && resolvedSourceCredentials.keyType !== "auto" ) { connOptions.privateKeyType = resolvedSourceCredentials.keyType; } } else { connOptions.password = resolvedSourceCredentials.password; } if ( tunnelConfig.useSocks5 && (tunnelConfig.socks5Host || (tunnelConfig.socks5ProxyChain && tunnelConfig.socks5ProxyChain.length > 0)) ) { try { const socks5Socket = await createSocks5Connection( tunnelConfig.sourceIP, tunnelConfig.sourceSSHPort, { useSocks5: tunnelConfig.useSocks5, socks5Host: tunnelConfig.socks5Host, socks5Port: tunnelConfig.socks5Port, socks5Username: tunnelConfig.socks5Username, socks5Password: tunnelConfig.socks5Password, socks5ProxyChain: tunnelConfig.socks5ProxyChain, }, ); if (socks5Socket) { connOptions.sock = socks5Socket; } else { throw new Error("Failed to create SOCKS5 connection"); } } catch (socks5Error) { tunnelLogger.error( "SOCKS5 connection failed for killing tunnel", socks5Error, { operation: "socks5_connect_kill", tunnelName, proxyHost: tunnelConfig.socks5Host, proxyPort: tunnelConfig.socks5Port || 1080, }, ); throw new Error( "SOCKS5 proxy connection failed: " + (socks5Error instanceof Error ? socks5Error.message : "Unknown error"), { cause: socks5Error }, ); } } return new Promise((resolve, reject) => { const conn = new Client(); conn.on("ready", () => resolve(conn)); conn.on("error", (err) => reject(err)); conn.connect(connOptions); }); }; const execCommand = (client: Client, cmd: string): Promise => new Promise((resolve, reject) => { client.exec(cmd, (err, stream) => { if (err) { reject(err); return; } let output = ""; stream.on("data", (data: Buffer) => { output += data.toString(); }); stream.stderr.on("data", (data: Buffer) => { const stderr = data.toString().trim(); if (stderr && !stderr.includes("debug1")) { tunnelLogger.warn( `Kill command stderr for '${tunnelName}': ${stderr}`, ); } }); stream.on("close", () => resolve(output.trim())); }); }); try { await withConnection(poolKey, factory, async (client) => { const checkCmd = `ps aux | grep -F '${tunnelMarker}' | grep -v grep`; const checkOutput = await execCommand(client, checkCmd); if (!checkOutput) { tunnelLogger.warn("Remote tunnel process not found", { operation: "tunnel_remote_not_found", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, marker: tunnelMarker, }); return; } tunnelLogger.info("Remote tunnel process found, proceeding to kill", { operation: "tunnel_remote_found", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, marker: tunnelMarker, }); const killCmds = [ `pkill -TERM -f '${tunnelMarker}'`, `sleep 2 && pkill -9 -f '${tunnelMarker}'`, ]; for (const killCmd of killCmds) { try { await execCommand(client, killCmd); } catch (err) { tunnelLogger.warn( `Kill command failed for '${tunnelName}': ${(err as Error).message}`, ); } } const verifyOutput = await execCommand(client, checkCmd); if (verifyOutput) { tunnelLogger.warn( `Some tunnel processes may still be running for '${tunnelName}'`, ); } else { tunnelLogger.success("Remote tunnel process killed", { operation: "tunnel_remote_killed", userId: tunnelConfig.sourceUserId, hostId: tunnelConfig.sourceHostId, tunnelName, }); } }); callback(); } catch (err) { tunnelLogger.error( `Failed to connect to source host for killing tunnel '${tunnelName}': ${(err as Error).message}`, ); callback(err as Error); } } /** * @openapi * /ssh/tunnel/status: * get: * summary: Get all tunnel statuses * description: Retrieves the status of all SSH tunnels. * tags: * - SSH Tunnels * responses: * 200: * description: A list of all tunnel statuses. */ app.get( "/ssh/tunnel/status", authenticateJWT, (req: AuthenticatedRequest, res: Response) => { if (!req.userId) { return res.status(401).json({ error: "Authentication required" }); } res.json(getAllTunnelStatus()); }, ); app.get( "/ssh/tunnel/status/stream", authenticateJWT, (req: AuthenticatedRequest, res: Response) => { if (!req.userId) { return res.status(401).json({ error: "Authentication required" }); } res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-store, no-transform", Connection: "keep-alive", "X-Accel-Buffering": "no", }); res.flushHeaders?.(); tunnelStatusClients.add(res); sendTunnelStatusSnapshot(res); const heartbeat = setInterval(() => { try { res.write(": keepalive\n\n"); } catch { closeStream(); } }, 30000); const closeStream = () => { clearInterval(heartbeat); tunnelStatusClients.delete(res); }; req.on("close", closeStream); }, ); /** * @openapi * /ssh/tunnel/status/{tunnelName}: * get: * summary: Get tunnel status by name * description: Retrieves the status of a specific SSH tunnel by its name. * tags: * - SSH Tunnels * parameters: * - in: path * name: tunnelName * required: true * schema: * type: string * responses: * 200: * description: Tunnel status. * 404: * description: Tunnel not found. */ app.get( "/ssh/tunnel/status/:tunnelName", authenticateJWT, (req: AuthenticatedRequest, res: Response) => { if (!req.userId) { return res.status(401).json({ error: "Authentication required" }); } const tunnelNameParam = req.params.tunnelName; const tunnelName = Array.isArray(tunnelNameParam) ? tunnelNameParam[0] : tunnelNameParam; const status = connectionStatus.get(tunnelName); if (!status) { return res.status(404).json({ error: "Tunnel not found" }); } res.json({ name: tunnelName, status }); }, ); /** * @openapi * /ssh/tunnel/connect: * post: * summary: Connect SSH tunnel * description: Establishes an SSH tunnel connection with the specified configuration. * tags: * - SSH Tunnels * requestBody: * required: true * content: * application/json: * schema: * type: object * properties: * name: * type: string * sourceHostId: * type: integer * tunnelIndex: * type: integer * responses: * 200: * description: Connection request received. * 400: * description: Invalid tunnel configuration. * 401: * description: Authentication required. * 403: * description: Access denied to this host. * 500: * description: Failed to connect tunnel. */ app.post( "/ssh/tunnel/connect", authenticateJWT, async (req: AuthenticatedRequest, res: Response) => { const tunnelConfig: TunnelConfig = req.body; const userId = req.userId; if (!userId) { return res.status(401).json({ error: "Authentication required" }); } if (!tunnelConfig || !tunnelConfig.name) { return res.status(400).json({ error: "Invalid tunnel configuration" }); } const tunnelName = tunnelConfig.name; try { if (!validateTunnelConfig(tunnelName, tunnelConfig)) { tunnelLogger.error(`Tunnel config validation failed`, { operation: "tunnel_connect", tunnelName, configHostId: tunnelConfig.sourceHostId, configTunnelIndex: tunnelConfig.tunnelIndex, }); return res.status(400).json({ error: "Tunnel configuration does not match tunnel name", }); } if (tunnelConfig.sourceHostId) { const accessInfo = await permissionManager.canAccessHost( userId, tunnelConfig.sourceHostId, "read", ); if (!accessInfo.hasAccess) { tunnelLogger.warn("User attempted tunnel connect without access", { operation: "tunnel_connect_unauthorized", userId, hostId: tunnelConfig.sourceHostId, tunnelName, }); return res.status(403).json({ error: "Access denied to this host" }); } if (accessInfo.isShared && !accessInfo.isOwner) { tunnelConfig.requestingUserId = userId; } } if (pendingTunnelOperations.has(tunnelName)) { try { await pendingTunnelOperations.get(tunnelName); } catch { tunnelLogger.warn(`Previous tunnel operation failed`, { tunnelName }); } } const operation = (async () => { manualDisconnects.delete(tunnelName); retryCounters.delete(tunnelName); retryExhaustedTunnels.delete(tunnelName); await cleanupTunnelResources(tunnelName); if (tunnelConfigs.has(tunnelName)) { const existingConfig = tunnelConfigs.get(tunnelName); if ( existingConfig && (existingConfig.sourceHostId !== tunnelConfig.sourceHostId || existingConfig.tunnelIndex !== tunnelConfig.tunnelIndex) ) { throw new Error(`Tunnel name collision detected: ${tunnelName}`); } } if ( !isSingleHostTunnel(tunnelConfig) && (!tunnelConfig.endpointIP || !tunnelConfig.endpointUsername) ) { try { const systemCrypto = SystemCrypto.getInstance(); const internalAuthToken = await systemCrypto.getInternalAuthToken(); const allHostsResponse = await axios.get( "http://localhost:30001/host/db/host/internal/all", { headers: { "Content-Type": "application/json", "X-Internal-Auth-Token": internalAuthToken, }, }, ); const allHosts: SSHHost[] = allHostsResponse.data || []; const endpointHost = allHosts.find( (h) => h.name === tunnelConfig.endpointHost || `${h.username}@${h.ip}` === tunnelConfig.endpointHost, ); if (!endpointHost) { throw new Error( `Endpoint host '${tunnelConfig.endpointHost}' not found in database`, ); } tunnelConfig.endpointIP = endpointHost.ip; tunnelConfig.endpointSSHPort = endpointHost.port; tunnelConfig.endpointUsername = endpointHost.username; tunnelConfig.endpointAuthMethod = endpointHost.authType; tunnelConfig.endpointKeyType = endpointHost.keyType; tunnelConfig.endpointCredentialId = endpointHost.credentialId; tunnelConfig.endpointUserId = endpointHost.userId; // Resolve credentials server-side instead of from HTTP response if (endpointHost.id && endpointHost.userId) { try { const { resolveHostById } = await import("./host-resolver.js"); const resolved = await resolveHostById( endpointHost.id, endpointHost.userId, ); if (resolved) { tunnelConfig.endpointPassword = resolved.password; tunnelConfig.endpointSSHKey = resolved.key; tunnelConfig.endpointKeyPassword = resolved.keyPassword; } } catch (credError) { tunnelLogger.warn( "Failed to resolve endpoint credentials from DB", { operation: "tunnel_endpoint_credential_resolve", endpointHostId: endpointHost.id, error: credError instanceof Error ? credError.message : "Unknown", }, ); } } } catch (resolveError) { tunnelLogger.error( "Failed to resolve endpoint host", resolveError, { operation: "tunnel_connect_resolve_endpoint_failed", tunnelName, endpointHost: tunnelConfig.endpointHost, }, ); throw new Error( `Failed to resolve endpoint host: ${resolveError instanceof Error ? resolveError.message : "Unknown error"}`, { cause: resolveError }, ); } } tunnelConfigs.set(tunnelName, tunnelConfig); await connectSSHTunnel(tunnelConfig, 0); })(); pendingTunnelOperations.set(tunnelName, operation); res.json({ message: "Connection request received", tunnelName }); operation .catch((err) => { tunnelLogger.error("Tunnel operation failed", err, { operation: "tunnel_operation_failed", tunnelName, }); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.FAILED, reason: err instanceof Error ? err.message : "Unknown error", }); tunnelConnecting.delete(tunnelName); }) .finally(() => { pendingTunnelOperations.delete(tunnelName); }); } catch (error) { tunnelLogger.error("Failed to process tunnel connect", error, { operation: "tunnel_connect", tunnelName, userId, }); res.status(500).json({ error: "Failed to connect tunnel" }); } }, ); /** * @openapi * /ssh/tunnel/disconnect: * post: * summary: Disconnect SSH tunnel * description: Disconnects an active SSH tunnel. * tags: * - SSH Tunnels * requestBody: * required: true * content: * application/json: * schema: * type: object * properties: * tunnelName: * type: string * responses: * 200: * description: Disconnect request received. * 400: * description: Tunnel name required. * 401: * description: Authentication required. * 403: * description: Access denied. * 500: * description: Failed to disconnect tunnel. */ app.post( "/ssh/tunnel/disconnect", authenticateJWT, async (req: AuthenticatedRequest, res: Response) => { const { tunnelName } = req.body; const userId = req.userId; if (!userId) { return res.status(401).json({ error: "Authentication required" }); } if (!tunnelName) { return res.status(400).json({ error: "Tunnel name required" }); } try { const config = tunnelConfigs.get(tunnelName); if (config && config.sourceHostId) { const accessInfo = await permissionManager.canAccessHost( userId, config.sourceHostId, "read", ); if (!accessInfo.hasAccess) { return res.status(403).json({ error: "Access denied" }); } } tunnelLogger.info("Tunnel stop request received", { operation: "tunnel_stop_request", userId, hostId: config?.sourceHostId, tunnelName, }); manualDisconnects.add(tunnelName); retryCounters.delete(tunnelName); retryExhaustedTunnels.delete(tunnelName); if (activeRetryTimers.has(tunnelName)) { clearTimeout(activeRetryTimers.get(tunnelName)!); activeRetryTimers.delete(tunnelName); } await cleanupTunnelResources(tunnelName, true); tunnelLogger.info("Tunnel cleanup completed", { operation: "tunnel_cleanup_complete", userId, hostId: config?.sourceHostId, tunnelName, }); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.DISCONNECTED, manualDisconnect: true, }); const tunnelConfig = tunnelConfigs.get(tunnelName) || null; handleDisconnect(tunnelName, tunnelConfig, false); setTimeout(() => { manualDisconnects.delete(tunnelName); }, 5000); res.json({ message: "Disconnect request received", tunnelName }); } catch (error) { tunnelLogger.error("Failed to disconnect tunnel", error, { operation: "tunnel_disconnect", tunnelName, userId, }); res.status(500).json({ error: "Failed to disconnect tunnel" }); } }, ); /** * @openapi * /ssh/tunnel/cancel: * post: * summary: Cancel tunnel retry * description: Cancels the retry mechanism for a failed SSH tunnel connection. * tags: * - SSH Tunnels * requestBody: * required: true * content: * application/json: * schema: * type: object * properties: * tunnelName: * type: string * responses: * 200: * description: Cancel request received. * 400: * description: Tunnel name required. * 401: * description: Authentication required. * 403: * description: Access denied. * 500: * description: Failed to cancel tunnel retry. */ app.post( "/ssh/tunnel/cancel", authenticateJWT, async (req: AuthenticatedRequest, res: Response) => { const { tunnelName } = req.body; const userId = req.userId; if (!userId) { return res.status(401).json({ error: "Authentication required" }); } if (!tunnelName) { return res.status(400).json({ error: "Tunnel name required" }); } try { const config = tunnelConfigs.get(tunnelName); if (config && config.sourceHostId) { const accessInfo = await permissionManager.canAccessHost( userId, config.sourceHostId, "read", ); if (!accessInfo.hasAccess) { return res.status(403).json({ error: "Access denied" }); } } retryCounters.delete(tunnelName); retryExhaustedTunnels.delete(tunnelName); if (activeRetryTimers.has(tunnelName)) { clearTimeout(activeRetryTimers.get(tunnelName)!); activeRetryTimers.delete(tunnelName); } if (countdownIntervals.has(tunnelName)) { clearInterval(countdownIntervals.get(tunnelName)!); countdownIntervals.delete(tunnelName); } await cleanupTunnelResources(tunnelName, true); broadcastTunnelStatus(tunnelName, { connected: false, status: CONNECTION_STATES.DISCONNECTED, manualDisconnect: true, }); const tunnelConfig = tunnelConfigs.get(tunnelName) || null; handleDisconnect(tunnelName, tunnelConfig, false); setTimeout(() => { manualDisconnects.delete(tunnelName); }, 5000); res.json({ message: "Cancel request received", tunnelName }); } catch (error) { tunnelLogger.error("Failed to cancel tunnel retry", error, { operation: "tunnel_cancel", tunnelName, userId, }); res.status(500).json({ error: "Failed to cancel tunnel retry" }); } }, ); async function initializeAutoStartTunnels(): Promise { try { const systemCrypto = SystemCrypto.getInstance(); const internalAuthToken = await systemCrypto.getInternalAuthToken(); const autostartResponse = await axios.get( "http://localhost:30001/host/db/host/internal", { headers: { "Content-Type": "application/json", "X-Internal-Auth-Token": internalAuthToken, }, }, ); const allHostsResponse = await axios.get( "http://localhost:30001/host/db/host/internal/all", { headers: { "Content-Type": "application/json", "X-Internal-Auth-Token": internalAuthToken, }, }, ); const autostartHosts: SSHHost[] = autostartResponse.data || []; const allHosts: SSHHost[] = allHostsResponse.data || []; const autoStartTunnels: TunnelConfig[] = []; tunnelLogger.info( `Found ${autostartHosts.length} autostart hosts and ${allHosts.length} total hosts for endpointHost resolution`, ); for (const host of autostartHosts) { if (host.enableTunnel && host.tunnelConnections) { for (const tunnelConnection of host.tunnelConnections) { if (tunnelConnection.autoStart) { const endpointHost = allHosts.find( (h) => h.name === tunnelConnection.endpointHost || `${h.username}@${h.ip}` === tunnelConnection.endpointHost, ); if (endpointHost) { const tunnelIndex = host.tunnelConnections.indexOf(tunnelConnection); const tunnelConfig: TunnelConfig = { name: normalizeTunnelName( host.id, tunnelIndex, host.name || `${host.username}@${host.ip}`, tunnelConnection.sourcePort, tunnelConnection.endpointHost, tunnelConnection.endpointPort, ), scope: tunnelConnection.scope || "s2s", mode: tunnelConnection.mode || tunnelConnection.tunnelType || "remote", bindHost: tunnelConnection.bindHost, targetHost: tunnelConnection.targetHost, tunnelType: tunnelConnection.tunnelType || "remote", sourceHostId: host.id, tunnelIndex: tunnelIndex, hostName: host.name || `${host.username}@${host.ip}`, sourceIP: host.ip, sourceSSHPort: host.port, sourceUsername: host.username, sourceAuthMethod: host.authType, sourceKeyType: host.keyType, sourceCredentialId: host.credentialId, sourceUserId: host.userId, endpointIP: endpointHost.ip, endpointSSHPort: endpointHost.port, endpointUsername: endpointHost.username, endpointHost: tunnelConnection.endpointHost, endpointAuthMethod: tunnelConnection.endpointAuthType || endpointHost.authType, endpointKeyType: tunnelConnection.endpointKeyType || endpointHost.keyType, endpointCredentialId: endpointHost.credentialId, endpointUserId: endpointHost.userId, sourcePort: tunnelConnection.sourcePort, endpointPort: tunnelConnection.endpointPort, maxRetries: tunnelConnection.maxRetries, retryInterval: tunnelConnection.retryInterval * 1000, autoStart: tunnelConnection.autoStart, isPinned: host.pin, useSocks5: host.useSocks5, socks5Host: host.socks5Host, socks5Port: host.socks5Port, socks5Username: host.socks5Username, socks5Password: host.socks5Password, }; autoStartTunnels.push(tunnelConfig); } else { tunnelLogger.error( `Failed to find endpointHost '${tunnelConnection.endpointHost}' for tunnel from ${host.name || `${host.username}@${host.ip}`}. Available hosts: ${allHosts.map((h) => h.name || `${h.username}@${h.ip}`).join(", ")}`, ); } } } } } for (const tunnelConfig of autoStartTunnels) { tunnelConfigs.set(tunnelConfig.name, tunnelConfig); setTimeout(() => { connectSSHTunnel(tunnelConfig, 0).catch((error) => { tunnelLogger.error( `Failed to connect tunnel ${tunnelConfig.name}: ${error instanceof Error ? error.message : "Unknown error"}`, ); }); }, 1000); } } catch (error) { tunnelLogger.error( "Failed to initialize auto-start tunnels:", error instanceof Error ? error.message : "Unknown error", ); } } const PORT = 30003; const server = createServer(app); const c2sRelayWss = new WebSocketServer({ server, path: "/ssh/tunnel/c2s/stream", }); c2sRelayWss.on("connection", (ws, req) => { let opened = false; ws.once("message", async (raw) => { try { const token = extractRequestToken(req); const payload = token ? await authManager.verifyJWTToken(token) : null; if (!payload?.userId || payload.pendingTOTP) { sendC2SError(ws, "Authentication required"); ws.close(); return; } const message = JSON.parse(raw.toString()) as C2SOpenMessage; if (message.type !== "open" && message.type !== "test") { throw new Error("Invalid client tunnel relay request"); } opened = true; if (message.type === "test") { await handleC2SRelayTest(ws, message, payload.userId); } else { await handleC2SRelayOpen(ws, message, payload.userId); } } catch (error) { const message = describeC2SRelayError(error); tunnelLogger.error("Failed to open C2S relay", error, { operation: "c2s_relay_open_failed", }); sendC2SError(ws, message); ws.close(); } }); ws.on("close", () => { if (!opened) { tunnelLogger.info("C2S relay closed before opening", { operation: "c2s_relay_closed_before_open", }); } }); }); server.listen(PORT, () => { setTimeout(() => { initializeAutoStartTunnels(); }, 2000); });