fix: high cpu from status probes and full database saves on every sample (#1300)

src/backend/hosts/status/host-status-service.ts

@@ -7,8 +7,10 @@
  * failed (a session, a ctx.ssh connection or a plugin's
  * ctx.hosts.status.reportLogin said so). It clears on the next working login,
  * when the host is edited, or when a changed host key is accepted. Probing
- * never logs in: on RADIUS or Duo backed devices a login fires a real 2FA
- * push, every interval.
+ * never logs in to the host itself: on RADIUS or Duo backed devices a login
+ * fires a real 2FA push, every interval. A host behind jump hosts is probed
+ * through a chain, which does log in to the hops, so hosts that share a hop
+ * list share one chain (jump-probe-pool.ts).
  *
  * Polling is demand driven. A user asking for statuses starts polling their
  * own hosts; nothing is probed for a user who never opened the app.
@@ -28,7 +30,8 @@ import {
   type HostStatus,
 } from "./host-status.js";
 import { ConcurrentLimiter } from "./limiter.js";
-import { tcpPing, tcpPingThroughJumpHost } from "./tcp-ping.js";
+import { tcpPing } from "./tcp-ping.js";
+import { JumpProbePool } from "./jump-probe-pool.js";
 
 export const GLOBAL_STATUS_INTERVAL_KEY = "global_status_check_interval";
 export const DEFAULT_STATUS_INTERVAL = 60;
@@ -111,6 +114,11 @@ export function toStatusTarget(row: HostStatusTargetRow): StatusTarget {
   };
 }
 
+const jumpProbes = new JumpProbePool(async (jumpHosts, userId) => {
+  const { createJumpHostChain } = await import("../jump-host-chain.js");
+  return createJumpHostChain(jumpHosts, userId);
+});
+
 const defaultDeps: HostStatusDeps = {
   loadTargets: async (filter) =>
     (
@@ -129,16 +137,8 @@ const defaultDeps: HostStatusDeps = {
   },
   hasActiveSession: (hostId) => hostSessionStatus.hasActiveSession(hostId),
   ping: (host, port) => tcpPing(host, port, 5000),
-  pingThroughJumpHosts: async (target, port) => {
-    const { createJumpHostChain } = await import("../jump-host-chain.js");
-    const client = await createJumpHostChain(
-      target.jumpHosts,
-      target.userId,
-    ).catch(() => null);
-    return client
-      ? tcpPingThroughJumpHost(client, target.ip, port, 5000)
-      : false;
-  },
+  pingThroughJumpHosts: (target, port) =>
+    jumpProbes.ping(target.jumpHosts, target.userId, target.ip, port, 5000),
   globalInterval: () => {
     const value = Number(getCurrentSettingValue(GLOBAL_STATUS_INTERVAL_KEY));
     return Number.isInteger(value) && value >= 5

src/backend/hosts/status/jump-probe-pool.ts

@@ -0,0 +1,87 @@
+import type { Client } from "ssh2";
+import { tcpPingThroughJumpHost } from "./tcp-ping.js";
+
+type OpenChain = (
+  jumpHosts: Array<{ hostId: number }>,
+  userId: string,
+) => Promise<Client | null>;
+
+interface Entry {
+  client: Promise<Client | null>;
+  idle?: NodeJS.Timeout;
+}
+
+/**
+ * Status probes for hosts behind jump hosts share one chain per hop list.
+ * Opening a chain logs in to every hop, so doing it for every host on every
+ * interval made an import of jump-hosted hosts open dozens of SSH sessions
+ * each minute. A chain closes after it sits idle.
+ */
+export class JumpProbePool {
+  private readonly entries = new Map<string, Entry>();
+
+  constructor(
+    private readonly open: OpenChain,
+    private readonly idleMs = 120_000,
+  ) {}
+
+  async ping(
+    jumpHosts: Array<{ hostId: number }>,
+    userId: string,
+    host: string,
+    port: number,
+    timeoutMs = 5000,
+  ): Promise<boolean> {
+    const key = `${userId}:${jumpHosts.map((hop) => hop.hostId).join(">")}`;
+    let entry = this.entries.get(key);
+    if (!entry) {
+      entry = { client: this.connect(key, jumpHosts, userId) };
+      this.entries.set(key, entry);
+    }
+    const client = await entry.client;
+    if (!client) {
+      this.drop(key, entry);
+      return false;
+    }
+    this.touch(key, entry, client);
+    return tcpPingThroughJumpHost(client, host, port, timeoutMs, true);
+  }
+
+  closeAll(): void {
+    for (const [key, entry] of [...this.entries]) {
+      this.drop(key, entry);
+      void entry.client.then((client) => client?.end()).catch(() => {});
+    }
+  }
+
+  private async connect(
+    key: string,
+    jumpHosts: Array<{ hostId: number }>,
+    userId: string,
+  ): Promise<Client | null> {
+    const client = await this.open(jumpHosts, userId).catch(() => null);
+    if (client) {
+      const forget = () => {
+        const current = this.entries.get(key);
+        if (current) this.drop(key, current);
+      };
+      client.once("close", forget);
+      client.once("error", forget);
+    }
+    return client;
+  }
+
+  private touch(key: string, entry: Entry, client: Client): void {
+    if (entry.idle) clearTimeout(entry.idle);
+    entry.idle = setTimeout(() => {
+      this.drop(key, entry);
+      client.end();
+    }, this.idleMs);
+    entry.idle.unref?.();
+  }
+
+  private drop(key: string, entry: Entry): void {
+    if (entry.idle) clearTimeout(entry.idle);
+    if (this.entries.get(key) === entry) this.entries.delete(key);
+  }
+}

src/backend/hosts/status/tcp-ping.ts

@@ -70,6 +70,7 @@ export function tcpPingThroughJumpHost(
   host: string,
   port: number,
   timeoutMs = 5000,
+  keepOpen = false,
 ): Promise<boolean> {
   return new Promise((resolve) => {
     let settled = false;
@@ -78,7 +79,7 @@ export function tcpPingThroughJumpHost(
       if (settled) return;
       settled = true;
       clearTimeout(timeout);
-      jumpClient.end();
+      if (!keepOpen) jumpClient.end();
       resolve(result);
     };