From 4f05bd4fbd0bf5309183b85638fc002ec10d8bea Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Fri, 10 Jul 2026 08:03:19 +0800 Subject: [PATCH] Retry transient terminal DNS lookups (#1011) * Retry transient terminal DNS lookups * Apply DNS retry to SSH entry points --- src/backend/ssh/docker-console.ts | 9 ++++ src/backend/ssh/docker.ts | 2 + src/backend/ssh/file-manager.ts | 3 ++ src/backend/ssh/host-metrics.ts | 25 +++++++++- src/backend/ssh/ssh-dns.test.ts | 79 +++++++++++++++++++++++++++++ src/backend/ssh/ssh-dns.ts | 82 +++++++++++++++++++++++++++++++ src/backend/ssh/terminal.ts | 36 +++++++++++++- src/backend/ssh/tmux-monitor.ts | 12 ++++- src/backend/ssh/tunnel.ts | 26 ++++++++++ 9 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 src/backend/ssh/ssh-dns.test.ts create mode 100644 src/backend/ssh/ssh-dns.ts diff --git a/src/backend/ssh/docker-console.ts b/src/backend/ssh/docker-console.ts index b6e80845..d845e268 100644 --- a/src/backend/ssh/docker-console.ts +++ b/src/backend/ssh/docker-console.ts @@ -14,6 +14,7 @@ import { getContainerRuntimeConfig, type ContainerRuntime, } from "./container-runtime.js"; +import { resolveSshConnectConfigHost } from "./ssh-dns.js"; const sshLogger = systemLogger; @@ -276,6 +277,10 @@ async function createJumpHostChain( }); } + if (!config.sock) { + await resolveSshConnectConfigHost(config); + } + await new Promise((resolve, reject) => { client.on("ready", () => resolve()); client.on("error", reject); @@ -489,6 +494,10 @@ wss.on("connection", async (ws: WebSocket, req) => { } } + if (!config.sock) { + await resolveSshConnectConfigHost(config); + } + await new Promise((resolve, reject) => { client.on("ready", () => resolve()); client.on("error", reject); diff --git a/src/backend/ssh/docker.ts b/src/backend/ssh/docker.ts index a3cefe33..3ff4066b 100644 --- a/src/backend/ssh/docker.ts +++ b/src/backend/ssh/docker.ts @@ -28,6 +28,7 @@ import { getRuntimeLabel, type ContainerRuntime, } from "./container-runtime.js"; +import { resolveSshConnectConfigHost } from "./ssh-dns.js"; const sshLogger = logger; @@ -1617,6 +1618,7 @@ app.post("/docker/ssh/connect", async (req, res) => { return; } } else { + await resolveSshConnectConfigHost(config); client.connect(config); } } catch (error) { diff --git a/src/backend/ssh/file-manager.ts b/src/backend/ssh/file-manager.ts index 1a799d9b..96610d10 100644 --- a/src/backend/ssh/file-manager.ts +++ b/src/backend/ssh/file-manager.ts @@ -43,6 +43,7 @@ import { } from "./file-manager-session.js"; import { registerFileListingRoutes } from "./file-manager-list-routes.js"; import { registerFileOperationRoutes } from "./file-manager-operation-routes.js"; +import { resolveSshConnectConfigHost } from "./ssh-dns.js"; import { registerFileDownloadRoutes } from "./file-manager-download-routes.js"; import { registerFileActionRoutes } from "./file-manager-action-routes.js"; import { applyAgentAuth } from "./terminal-auth-helpers.js"; @@ -194,6 +195,7 @@ async function buildDedicatedTransferConnectConfig( compress: ["none", "zlib@openssh.com", "zlib"], }, }; + await resolveSshConnectConfigHost(config); const authType = host.authType; @@ -1759,6 +1761,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => { }); } } else { + await resolveSshConnectConfigHost(config); client.connect(config); } }); diff --git a/src/backend/ssh/host-metrics.ts b/src/backend/ssh/host-metrics.ts index d10a3334..532c9307 100644 --- a/src/backend/ssh/host-metrics.ts +++ b/src/backend/ssh/host-metrics.ts @@ -41,6 +41,7 @@ import { registerHostMetricsPreferencesRoutes } from "./host-metrics-preferences import { registerHostMetricsHistoryRoutes } from "./host-metrics-history-routes.js"; import { AlertEngine } from "./alert-engine.js"; import { registerManagerRoutes } from "./managers/index.js"; +import { resolveSshConnectConfigHost } from "./ssh-dns.js"; import { AccessDeniedError } from "./managers/route-helpers.js"; import type { ManagerHost } from "./managers/types.js"; import { createJumpHostChain } from "./host-metrics-jump-hosts.js"; @@ -1342,8 +1343,18 @@ function createSshFactory(host: SSHHostWithCredentials): () => Promise { client.connect(config); }, ); - } else { + } else if (config.sock) { client.connect(config); + } else { + resolveSshConnectConfigHost(config) + .then(() => { + client.connect(config); + }) + .catch((error) => { + clearTimeout(timeout); + reject(error); + }); + return; } }); }; @@ -2372,7 +2383,17 @@ app.post("/metrics/start/:id", validateHostId, async (req, res) => { } }); } else { - client.connect(config); + resolveSshConnectConfigHost(config) + .then(() => { + client.connect(config); + }) + .catch((error) => { + if (!isResolved) { + isResolved = true; + clearTimeout(timeout); + reject(error); + } + }); } }); diff --git a/src/backend/ssh/ssh-dns.test.ts b/src/backend/ssh/ssh-dns.test.ts new file mode 100644 index 00000000..aa647bfd --- /dev/null +++ b/src/backend/ssh/ssh-dns.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from "vitest"; +import { + isRetriableDnsError, + resolveHostForSshConnect, + resolveSshConnectConfigHost, + shouldResolveBeforeSshConnect, +} from "./ssh-dns.js"; + +describe("SSH DNS resolution", () => { + it("retries transient EAI_AGAIN errors before returning an address", async () => { + const lookup = vi + .fn() + .mockRejectedValueOnce( + Object.assign(new Error("try again"), { code: "EAI_AGAIN" }), + ) + .mockResolvedValueOnce({ address: "10.0.0.5", family: 4 }); + const wait = vi.fn().mockResolvedValue(undefined); + + await expect( + resolveHostForSshConnect("alp", lookup, [10], wait), + ).resolves.toEqual({ + host: "10.0.0.5", + resolvedAddress: "10.0.0.5", + attempts: 2, + }); + expect(wait).toHaveBeenCalledWith(10); + }); + + it("does not retry permanent DNS failures", async () => { + const error = Object.assign(new Error("not found"), { code: "ENOTFOUND" }); + const lookup = vi.fn().mockRejectedValue(error); + const wait = vi.fn().mockResolvedValue(undefined); + + await expect( + resolveHostForSshConnect("missing", lookup, [10], wait), + ).rejects.toBe(error); + expect(wait).not.toHaveBeenCalled(); + }); + + it("skips DNS lookup for literal IP addresses", async () => { + const lookup = vi.fn(); + + await expect( + resolveHostForSshConnect("192.0.2.1", lookup), + ).resolves.toEqual({ + host: "192.0.2.1", + attempts: 0, + }); + expect(lookup).not.toHaveBeenCalled(); + }); + + it("detects retryable DNS errors by code or message", () => { + expect(isRetriableDnsError({ code: "EAI_AGAIN" })).toBe(true); + expect(isRetriableDnsError(new Error("getaddrinfo EAI_AGAIN alp"))).toBe( + true, + ); + expect(isRetriableDnsError({ code: "ENOTFOUND" })).toBe(false); + }); + + it("only pre-resolves hostnames", () => { + expect(shouldResolveBeforeSshConnect("alp")).toBe(true); + expect(shouldResolveBeforeSshConnect("127.0.0.1")).toBe(false); + expect(shouldResolveBeforeSshConnect("[2001:db8::1]")).toBe(false); + }); + + it("updates SSH connect config hosts in place", async () => { + const lookup = vi + .fn() + .mockResolvedValue({ address: "10.0.0.6", family: 4 }); + const config = { host: "alp", port: 22 }; + + await expect(resolveSshConnectConfigHost(config, lookup)).resolves.toEqual({ + host: "10.0.0.6", + port: 22, + originalHost: "alp", + resolvedHost: "10.0.0.6", + }); + }); +}); diff --git a/src/backend/ssh/ssh-dns.ts b/src/backend/ssh/ssh-dns.ts new file mode 100644 index 00000000..8b016ee9 --- /dev/null +++ b/src/backend/ssh/ssh-dns.ts @@ -0,0 +1,82 @@ +import dns from "dns/promises"; +import net from "net"; + +export const SSH_DNS_RETRY_DELAYS_MS = [250, 750, 1500]; + +type Lookup = typeof dns.lookup; +type Sleep = (ms: number) => Promise; +type SshConnectConfigHost = { + host?: unknown; +}; + +const sleep: Sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +export function isRetriableDnsError(error: unknown): boolean { + const err = error as { code?: unknown; message?: unknown }; + return ( + err.code === "EAI_AGAIN" || + (typeof err.message === "string" && err.message.includes("EAI_AGAIN")) + ); +} + +export function shouldResolveBeforeSshConnect(host: string): boolean { + const normalized = host.replace(/^\[|\]$/g, "").trim(); + if (!normalized) return false; + return net.isIP(normalized) === 0; +} + +export async function resolveHostForSshConnect( + host: string, + lookup: Lookup = dns.lookup, + retryDelaysMs = SSH_DNS_RETRY_DELAYS_MS, + wait: Sleep = sleep, +): Promise<{ host: string; resolvedAddress?: string; attempts: number }> { + const normalized = host.replace(/^\[|\]$/g, "").trim(); + if (!shouldResolveBeforeSshConnect(normalized)) { + return { host: normalized || host, attempts: 0 }; + } + + for (let attempt = 0; ; attempt += 1) { + try { + const result = await lookup(normalized); + return { + host: result.address, + resolvedAddress: result.address, + attempts: attempt + 1, + }; + } catch (error) { + if (!isRetriableDnsError(error) || attempt >= retryDelaysMs.length) { + throw error; + } + await wait(retryDelaysMs[attempt]); + } + } +} + +export async function resolveSshConnectConfigHost< + T extends SshConnectConfigHost, +>( + config: T, + lookup: Lookup = dns.lookup, + retryDelaysMs = SSH_DNS_RETRY_DELAYS_MS, + wait: Sleep = sleep, +): Promise< + T & { host?: unknown; resolvedHost?: string; originalHost?: string } +> { + if (typeof config.host !== "string") return config; + + const originalHost = config.host; + const resolution = await resolveHostForSshConnect( + originalHost, + lookup, + retryDelaysMs, + wait, + ); + if (!resolution.resolvedAddress) return config; + + config.host = resolution.host; + return Object.assign(config, { + originalHost, + resolvedHost: resolution.resolvedAddress, + }); +} diff --git a/src/backend/ssh/terminal.ts b/src/backend/ssh/terminal.ts index cb6674c0..2c351f0b 100644 --- a/src/backend/ssh/terminal.ts +++ b/src/backend/ssh/terminal.ts @@ -37,6 +37,7 @@ import { import { isWindowsSftpPath, sftpPathToLocalPath } from "./transfer-paths.js"; import { preparePrivateKeyForSSH2 } from "../utils/ssh-key-utils.js"; import { triggerLoginAlert } from "../utils/alert-trigger.js"; +import { isRetriableDnsError, resolveHostForSshConnect } from "./ssh-dns.js"; interface ConnectToHostData { cols: number; @@ -1282,6 +1283,39 @@ wss.on("connection", async (ws: WebSocket, req) => { } sendLog("dns", "info", `Starting address resolution of ${ip}`); + let connectHost = ip; + try { + const resolution = await resolveHostForSshConnect(ip); + connectHost = resolution.host; + if (resolution.resolvedAddress && resolution.resolvedAddress !== ip) { + sendLog( + "dns", + "success", + `Resolved ${ip} to ${resolution.resolvedAddress}`, + { attempts: resolution.attempts }, + ); + } + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown error"; + sshLogger.error("SSH hostname resolution failed", error, { + operation: "terminal_dns_resolve", + hostId: id, + ip, + port, + transient: isRetriableDnsError(error), + }); + sendLog("dns", "error", `DNS resolution failed for ${ip}: ${message}`); + ws.send( + JSON.stringify({ + type: "error", + message: isRetriableDnsError(error) + ? "SSH error: DNS lookup temporarily failed. Check the Docker/container DNS configuration or try again." + : "SSH error: Could not resolve hostname from the Termix server container.", + }), + ); + cleanupAuthState(connectionTimeout); + return; + } sendLog("tcp", "info", `Connecting to ${ip} port ${port}`); sshConn.on("ready", () => { @@ -2320,7 +2354,7 @@ wss.on("connection", async (ws: WebSocket, req) => { const preloadedHostData = await SSHHostKeyVerifier.preloadHostData(id); const connectConfig: Record = { - host: ip, + host: connectHost, port, username, tryKeyboard: resolvedCredentials.authType !== "tailscale", diff --git a/src/backend/ssh/tmux-monitor.ts b/src/backend/ssh/tmux-monitor.ts index d504efcf..33626991 100644 --- a/src/backend/ssh/tmux-monitor.ts +++ b/src/backend/ssh/tmux-monitor.ts @@ -20,6 +20,7 @@ import { type SOCKS5Config, } from "../utils/socks5-helper.js"; import { withConnection } from "./ssh-connection-pool.js"; +import { resolveSshConnectConfigHost } from "./ssh-dns.js"; import { execCommand, tmuxCommand, withTmuxPath } from "./tmux-helper.js"; import { SEP, @@ -209,8 +210,17 @@ export function connectToHost(host: SSHHost): () => Promise { client.connect(config); }, ); - } else { + } else if (config.sock) { client.connect(config); + } else { + resolveSshConnectConfigHost(config) + .then(() => { + client.connect(config); + }) + .catch((error) => { + clearTimeout(timeout); + reject(error); + }); } }); }; diff --git a/src/backend/ssh/tunnel.ts b/src/backend/ssh/tunnel.ts index 205278b0..f5e08e1e 100644 --- a/src/backend/ssh/tunnel.ts +++ b/src/backend/ssh/tunnel.ts @@ -61,6 +61,7 @@ import { handleC2SRelayTest, type C2SOpenMessage, } from "./tunnel-c2s-relay.js"; +import { resolveSshConnectConfigHost } from "./ssh-dns.js"; import { handleSocks5Connect } from "./tunnel-socks5-relay.js"; const app = express(); @@ -1507,6 +1508,27 @@ async function connectSSHTunnel( } } + try { + await resolveSshConnectConfigHost(connOptions); + } catch (error) { + tunnelLogger.error("Tunnel source hostname resolution failed", error, { + operation: "tunnel_dns_resolve", + tunnelName, + sourceHost: `${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`, + retryAttempt, + }); + broadcastTunnelStatus(tunnelName, { + connected: false, + status: CONNECTION_STATES.FAILED, + reason: + error instanceof Error + ? error.message + : "Failed to resolve tunnel source hostname", + }); + tunnelConnecting.delete(tunnelName); + return; + } + conn.connect(connOptions); } @@ -1693,6 +1715,10 @@ async function killRemoteTunnelByMarker( } } + if (!connOptions.sock) { + await resolveSshConnectConfigHost(connOptions); + } + return new Promise((resolve, reject) => { const conn = new Client(); conn.on("ready", () => resolve(conn));