Retry transient terminal DNS lookups (#1011)

* Retry transient terminal DNS lookups

* Apply DNS retry to SSH entry points

src/backend/ssh/docker-console.ts

@@ -14,6 +14,7 @@ import {
   getContainerRuntimeConfig,
   type ContainerRuntime,
 } from "./container-runtime.js";
+import { resolveSshConnectConfigHost } from "./ssh-dns.js";
 
 const sshLogger = systemLogger;
 
@@ -276,6 +277,10 @@ async function createJumpHostChain(
       });
     }
 
+    if (!config.sock) {
+      await resolveSshConnectConfigHost(config);
+    }
+
     await new Promise<void>((resolve, reject) => {
       client.on("ready", () => resolve());
       client.on("error", reject);
@@ -489,6 +494,10 @@ wss.on("connection", async (ws: WebSocket, req) => {
               }
             }
 
+            if (!config.sock) {
+              await resolveSshConnectConfigHost(config);
+            }
+
             await new Promise<void>((resolve, reject) => {
               client.on("ready", () => resolve());
               client.on("error", reject);

src/backend/ssh/docker.ts

@@ -28,6 +28,7 @@ import {
   getRuntimeLabel,
   type ContainerRuntime,
 } from "./container-runtime.js";
+import { resolveSshConnectConfigHost } from "./ssh-dns.js";
 
 const sshLogger = logger;
 
@@ -1617,6 +1618,7 @@ app.post("/docker/ssh/connect", async (req, res) => {
         return;
       }
     } else {
+      await resolveSshConnectConfigHost(config);
       client.connect(config);
     }
   } catch (error) {

src/backend/ssh/file-manager.ts

@@ -43,6 +43,7 @@ import {
 } from "./file-manager-session.js";
 import { registerFileListingRoutes } from "./file-manager-list-routes.js";
 import { registerFileOperationRoutes } from "./file-manager-operation-routes.js";
+import { resolveSshConnectConfigHost } from "./ssh-dns.js";
 import { registerFileDownloadRoutes } from "./file-manager-download-routes.js";
 import { registerFileActionRoutes } from "./file-manager-action-routes.js";
 import { applyAgentAuth } from "./terminal-auth-helpers.js";
@@ -194,6 +195,7 @@ async function buildDedicatedTransferConnectConfig(
       compress: ["none", "zlib@openssh.com", "zlib"],
     },
   };
+  await resolveSshConnectConfigHost(config);
 
   const authType = host.authType;
 
@@ -1759,6 +1761,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
       });
     }
   } else {
+    await resolveSshConnectConfigHost(config);
     client.connect(config);
   }
 });

src/backend/ssh/host-metrics.ts

@@ -41,6 +41,7 @@ import { registerHostMetricsPreferencesRoutes } from "./host-metrics-preferences
 import { registerHostMetricsHistoryRoutes } from "./host-metrics-history-routes.js";
 import { AlertEngine } from "./alert-engine.js";
 import { registerManagerRoutes } from "./managers/index.js";
+import { resolveSshConnectConfigHost } from "./ssh-dns.js";
 import { AccessDeniedError } from "./managers/route-helpers.js";
 import type { ManagerHost } from "./managers/types.js";
 import { createJumpHostChain } from "./host-metrics-jump-hosts.js";
@@ -1342,8 +1343,18 @@ function createSshFactory(host: SSHHostWithCredentials): () => Promise<Client> {
             client.connect(config);
           },
         );
-      } else {
+      } else if (config.sock) {
         client.connect(config);
+      } else {
+        resolveSshConnectConfigHost(config)
+          .then(() => {
+            client.connect(config);
+          })
+          .catch((error) => {
+            clearTimeout(timeout);
+            reject(error);
+          });
+        return;
       }
     });
   };
@@ -2372,7 +2383,17 @@ app.post("/metrics/start/:id", validateHostId, async (req, res) => {
             }
           });
       } else {
-        client.connect(config);
+        resolveSshConnectConfigHost(config)
+          .then(() => {
+            client.connect(config);
+          })
+          .catch((error) => {
+            if (!isResolved) {
+              isResolved = true;
+              clearTimeout(timeout);
+              reject(error);
+            }
+          });
       }
     });
 

src/backend/ssh/ssh-dns.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  isRetriableDnsError,
+  resolveHostForSshConnect,
+  resolveSshConnectConfigHost,
+  shouldResolveBeforeSshConnect,
+} from "./ssh-dns.js";
+
+describe("SSH DNS resolution", () => {
+  it("retries transient EAI_AGAIN errors before returning an address", async () => {
+    const lookup = vi
+      .fn()
+      .mockRejectedValueOnce(
+        Object.assign(new Error("try again"), { code: "EAI_AGAIN" }),
+      )
+      .mockResolvedValueOnce({ address: "10.0.0.5", family: 4 });
+    const wait = vi.fn().mockResolvedValue(undefined);
+
+    await expect(
+      resolveHostForSshConnect("alp", lookup, [10], wait),
+    ).resolves.toEqual({
+      host: "10.0.0.5",
+      resolvedAddress: "10.0.0.5",
+      attempts: 2,
+    });
+    expect(wait).toHaveBeenCalledWith(10);
+  });
+
+  it("does not retry permanent DNS failures", async () => {
+    const error = Object.assign(new Error("not found"), { code: "ENOTFOUND" });
+    const lookup = vi.fn().mockRejectedValue(error);
+    const wait = vi.fn().mockResolvedValue(undefined);
+
+    await expect(
+      resolveHostForSshConnect("missing", lookup, [10], wait),
+    ).rejects.toBe(error);
+    expect(wait).not.toHaveBeenCalled();
+  });
+
+  it("skips DNS lookup for literal IP addresses", async () => {
+    const lookup = vi.fn();
+
+    await expect(
+      resolveHostForSshConnect("192.0.2.1", lookup),
+    ).resolves.toEqual({
+      host: "192.0.2.1",
+      attempts: 0,
+    });
+    expect(lookup).not.toHaveBeenCalled();
+  });
+
+  it("detects retryable DNS errors by code or message", () => {
+    expect(isRetriableDnsError({ code: "EAI_AGAIN" })).toBe(true);
+    expect(isRetriableDnsError(new Error("getaddrinfo EAI_AGAIN alp"))).toBe(
+      true,
+    );
+    expect(isRetriableDnsError({ code: "ENOTFOUND" })).toBe(false);
+  });
+
+  it("only pre-resolves hostnames", () => {
+    expect(shouldResolveBeforeSshConnect("alp")).toBe(true);
+    expect(shouldResolveBeforeSshConnect("127.0.0.1")).toBe(false);
+    expect(shouldResolveBeforeSshConnect("[2001:db8::1]")).toBe(false);
+  });
+
+  it("updates SSH connect config hosts in place", async () => {
+    const lookup = vi
+      .fn()
+      .mockResolvedValue({ address: "10.0.0.6", family: 4 });
+    const config = { host: "alp", port: 22 };
+
+    await expect(resolveSshConnectConfigHost(config, lookup)).resolves.toEqual({
+      host: "10.0.0.6",
+      port: 22,
+      originalHost: "alp",
+      resolvedHost: "10.0.0.6",
+    });
+  });
+});

src/backend/ssh/ssh-dns.ts

@@ -0,0 +1,82 @@
+import dns from "dns/promises";
+import net from "net";
+
+export const SSH_DNS_RETRY_DELAYS_MS = [250, 750, 1500];
+
+type Lookup = typeof dns.lookup;
+type Sleep = (ms: number) => Promise<void>;
+type SshConnectConfigHost = {
+  host?: unknown;
+};
+
+const sleep: Sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export function isRetriableDnsError(error: unknown): boolean {
+  const err = error as { code?: unknown; message?: unknown };
+  return (
+    err.code === "EAI_AGAIN" ||
+    (typeof err.message === "string" && err.message.includes("EAI_AGAIN"))
+  );
+}
+
+export function shouldResolveBeforeSshConnect(host: string): boolean {
+  const normalized = host.replace(/^\[|\]$/g, "").trim();
+  if (!normalized) return false;
+  return net.isIP(normalized) === 0;
+}
+
+export async function resolveHostForSshConnect(
+  host: string,
+  lookup: Lookup = dns.lookup,
+  retryDelaysMs = SSH_DNS_RETRY_DELAYS_MS,
+  wait: Sleep = sleep,
+): Promise<{ host: string; resolvedAddress?: string; attempts: number }> {
+  const normalized = host.replace(/^\[|\]$/g, "").trim();
+  if (!shouldResolveBeforeSshConnect(normalized)) {
+    return { host: normalized || host, attempts: 0 };
+  }
+
+  for (let attempt = 0; ; attempt += 1) {
+    try {
+      const result = await lookup(normalized);
+      return {
+        host: result.address,
+        resolvedAddress: result.address,
+        attempts: attempt + 1,
+      };
+    } catch (error) {
+      if (!isRetriableDnsError(error) || attempt >= retryDelaysMs.length) {
+        throw error;
+      }
+      await wait(retryDelaysMs[attempt]);
+    }
+  }
+}
+
+export async function resolveSshConnectConfigHost<
+  T extends SshConnectConfigHost,
+>(
+  config: T,
+  lookup: Lookup = dns.lookup,
+  retryDelaysMs = SSH_DNS_RETRY_DELAYS_MS,
+  wait: Sleep = sleep,
+): Promise<
+  T & { host?: unknown; resolvedHost?: string; originalHost?: string }
+> {
+  if (typeof config.host !== "string") return config;
+
+  const originalHost = config.host;
+  const resolution = await resolveHostForSshConnect(
+    originalHost,
+    lookup,
+    retryDelaysMs,
+    wait,
+  );
+  if (!resolution.resolvedAddress) return config;
+
+  config.host = resolution.host;
+  return Object.assign(config, {
+    originalHost,
+    resolvedHost: resolution.resolvedAddress,
+  });
+}

src/backend/ssh/terminal.ts

@@ -37,6 +37,7 @@ import {
 import { isWindowsSftpPath, sftpPathToLocalPath } from "./transfer-paths.js";
 import { preparePrivateKeyForSSH2 } from "../utils/ssh-key-utils.js";
 import { triggerLoginAlert } from "../utils/alert-trigger.js";
+import { isRetriableDnsError, resolveHostForSshConnect } from "./ssh-dns.js";
 
 interface ConnectToHostData {
   cols: number;
@@ -1282,6 +1283,39 @@ wss.on("connection", async (ws: WebSocket, req) => {
     }
 
     sendLog("dns", "info", `Starting address resolution of ${ip}`);
+    let connectHost = ip;
+    try {
+      const resolution = await resolveHostForSshConnect(ip);
+      connectHost = resolution.host;
+      if (resolution.resolvedAddress && resolution.resolvedAddress !== ip) {
+        sendLog(
+          "dns",
+          "success",
+          `Resolved ${ip} to ${resolution.resolvedAddress}`,
+          { attempts: resolution.attempts },
+        );
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      sshLogger.error("SSH hostname resolution failed", error, {
+        operation: "terminal_dns_resolve",
+        hostId: id,
+        ip,
+        port,
+        transient: isRetriableDnsError(error),
+      });
+      sendLog("dns", "error", `DNS resolution failed for ${ip}: ${message}`);
+      ws.send(
+        JSON.stringify({
+          type: "error",
+          message: isRetriableDnsError(error)
+            ? "SSH error: DNS lookup temporarily failed. Check the Docker/container DNS configuration or try again."
+            : "SSH error: Could not resolve hostname from the Termix server container.",
+        }),
+      );
+      cleanupAuthState(connectionTimeout);
+      return;
+    }
     sendLog("tcp", "info", `Connecting to ${ip} port ${port}`);
 
     sshConn.on("ready", () => {
@@ -2320,7 +2354,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
     const preloadedHostData = await SSHHostKeyVerifier.preloadHostData(id);
 
     const connectConfig: Record<string, unknown> = {
-      host: ip,
+      host: connectHost,
       port,
       username,
       tryKeyboard: resolvedCredentials.authType !== "tailscale",

src/backend/ssh/tmux-monitor.ts

@@ -20,6 +20,7 @@ import {
   type SOCKS5Config,
 } from "../utils/socks5-helper.js";
 import { withConnection } from "./ssh-connection-pool.js";
+import { resolveSshConnectConfigHost } from "./ssh-dns.js";
 import { execCommand, tmuxCommand, withTmuxPath } from "./tmux-helper.js";
 import {
   SEP,
@@ -209,8 +210,17 @@ export function connectToHost(host: SSHHost): () => Promise<Client> {
             client.connect(config);
           },
         );
-      } else {
+      } else if (config.sock) {
         client.connect(config);
+      } else {
+        resolveSshConnectConfigHost(config)
+          .then(() => {
+            client.connect(config);
+          })
+          .catch((error) => {
+            clearTimeout(timeout);
+            reject(error);
+          });
       }
     });
   };

src/backend/ssh/tunnel.ts

@@ -61,6 +61,7 @@ import {
   handleC2SRelayTest,
   type C2SOpenMessage,
 } from "./tunnel-c2s-relay.js";
+import { resolveSshConnectConfigHost } from "./ssh-dns.js";
 import { handleSocks5Connect } from "./tunnel-socks5-relay.js";
 
 const app = express();
@@ -1507,6 +1508,27 @@ async function connectSSHTunnel(
     }
   }
 
+  try {
+    await resolveSshConnectConfigHost(connOptions);
+  } catch (error) {
+    tunnelLogger.error("Tunnel source hostname resolution failed", error, {
+      operation: "tunnel_dns_resolve",
+      tunnelName,
+      sourceHost: `${tunnelConfig.sourceIP}:${tunnelConfig.sourceSSHPort}`,
+      retryAttempt,
+    });
+    broadcastTunnelStatus(tunnelName, {
+      connected: false,
+      status: CONNECTION_STATES.FAILED,
+      reason:
+        error instanceof Error
+          ? error.message
+          : "Failed to resolve tunnel source hostname",
+    });
+    tunnelConnecting.delete(tunnelName);
+    return;
+  }
+
   conn.connect(connOptions);
 }
 
@@ -1693,6 +1715,10 @@ async function killRemoteTunnelByMarker(
       }
     }
 
+    if (!connOptions.sock) {
+      await resolveSshConnectConfigHost(connOptions);
+    }
+
     return new Promise<Client>((resolve, reject) => {
       const conn = new Client();
       conn.on("ready", () => resolve(conn));