feat: finish the host-metrics plugin, move host status into core (B16)

src/backend/tests/hosts/host-key-recovery.test.ts

@@ -1,8 +1,8 @@
 import { EventEmitter } from "node:events";
 import type { WebSocket } from "ws";
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
 import { SSHHostKeyVerifier } from "../../hosts/host-key-verifier.js";
-import { authFailureTracker } from "../../hosts/metrics-shared/state.js";
+import { pluginEvents, TOPICS } from "../../plugins/events.js";
 
 const { updateHostKey } = vi.hoisted(() => ({ updateHostKey: vi.fn() }));
 vi.mock("../../database/repositories/factory.js", () => ({
@@ -12,10 +12,15 @@ vi.mock("../../utils/logger.js", () => ({
   sshLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
 }));
 const hostId = 991267;
+const accepted: unknown[] = [];
+const unsubscribe = pluginEvents.on(TOPICS.hostKeyUpdated, (payload) =>
+  accepted.push(payload),
+);
 afterEach(() => {
-  authFailureTracker.reset(hostId);
+  accepted.length = 0;
   vi.resetAllMocks();
 });
+afterAll(() => unsubscribe());
 
 async function verifyChangedKey(action: "accept" | "reject") {
   const socket = new EventEmitter() as EventEmitter & {
@@ -52,35 +57,22 @@ async function verifyChangedKey(action: "accept" | "reject") {
   );
 }
 
-describe("metrics recovery after host key verification", () => {
-  it("allows polling after accepting and saving the replacement key", async () => {
-    authFailureTracker.recordFailure(hostId, "HOST_KEY", true);
-    expect(authFailureTracker.shouldSkip(hostId)).toBe(true);
+describe("host key recovery", () => {
+  it("announces an accepted and saved replacement key", async () => {
     expect(await verifyChangedKey("accept")).toBe(true);
     expect(updateHostKey).toHaveBeenCalledOnce();
-    expect(authFailureTracker.shouldSkip(hostId)).toBe(false);
+    expect(accepted).toEqual([{ hostId }]);
   });
 
-  it("keeps polling blocked when the new key is rejected", async () => {
-    authFailureTracker.recordFailure(hostId, "HOST_KEY", true);
+  it("announces nothing when the new key is rejected", async () => {
     expect(await verifyChangedKey("reject")).toBe(false);
     expect(updateHostKey).not.toHaveBeenCalled();
-    expect(authFailureTracker.shouldSkip(hostId)).toBe(true);
+    expect(accepted).toEqual([]);
   });
 
-  it("keeps polling blocked if persisting the accepted key fails", async () => {
-    authFailureTracker.recordFailure(hostId, "HOST_KEY", true);
+  it("announces nothing if persisting the accepted key fails", async () => {
     updateHostKey.mockRejectedValueOnce(new Error("database unavailable"));
     expect(await verifyChangedKey("accept")).toBe(false);
-    expect(authFailureTracker.shouldSkip(hostId)).toBe(true);
+    expect(accepted).toEqual([]);
   });
-
-  it.each(["AUTH", "TOTP"] as const)(
-    "preserves unrelated %s failures",
-    async (reason) => {
-      authFailureTracker.recordFailure(hostId, reason, true);
-      expect(await verifyChangedKey("accept")).toBe(true);
-      expect(authFailureTracker.shouldSkip(hostId)).toBe(true);
-    },
-  );
 });

src/backend/tests/hosts/host-resolver.test.ts

@@ -122,7 +122,6 @@ function baseHost(overrides: Record<string, unknown> = {}) {
     autostartKeyPassword: null,
     jumpHosts: null,
     tunnelConnections: null,
-    statsConfig: null,
     terminalConfig: null,
     socks5ProxyChain: null,
     quickActions: null,

src/backend/tests/hosts/status/host-status-service.test.ts

@@ -0,0 +1,259 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../../../database/repositories/factory.js", () => ({
+  createCurrentHostResolutionRepository: vi.fn(),
+  getCurrentSettingValue: vi.fn(() => null),
+}));
+vi.mock("../../../utils/logger.js", () => ({
+  sshLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+  pluginLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+const { HostStatusService, toStatusTarget } =
+  await import("../../../hosts/status/host-status-service.js");
+const { hostSessionStatus } =
+  await import("../../../hosts/host-session-status.js");
+const { pluginEvents, TOPICS } = await import("../../../plugins/events.js");
+
+type Target = ReturnType<typeof toStatusTarget>;
+
+function target(id: number, extra: Partial<Target> = {}): Target {
+  return {
+    id,
+    userId: "owner",
+    ip: `10.0.0.${id}`,
+    port: 22,
+    connectionType: "ssh",
+    jumpHosts: [],
+    statusCheckEnabled: true,
+    statusCheckInterval: null,
+    ...extra,
+  };
+}
+
+function setup(targets: Target[], reachable = true) {
+  const emitted: unknown[] = [];
+  const ping = vi.fn(async () => reachable);
+  const pingThroughJumpHosts = vi.fn(async () => reachable);
+  const loadTargets = vi.fn(
+    async (filter: { userId?: string; hostIds?: number[] }) =>
+      targets.filter(
+        (t) =>
+          (!filter.userId || t.userId === filter.userId) &&
+          (!filter.hostIds || filter.hostIds.includes(t.id)),
+      ),
+  );
+  const service = new HostStatusService({
+    loadTargets,
+    ping,
+    pingThroughJumpHosts,
+    globalInterval: () => 60,
+    emit: (payload) => emitted.push(payload),
+  });
+  return { service, emitted, ping, pingThroughJumpHosts, loadTargets };
+}
+
+const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+let active: InstanceType<typeof HostStatusService> | null = null;
+
+beforeEach(() => {
+  vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] });
+});
+
+afterEach(() => {
+  active?.stop();
+  active = null;
+  pluginEvents.clear();
+  vi.useRealTimers();
+});
+
+describe("HostStatusService", () => {
+  it("starts a user's own hosts and reports them reachable", async () => {
+    const { service, emitted, ping } = setup([target(1), target(2)]);
+    active = service;
+
+    await service.statusesFor("owner", null);
+    await flush();
+
+    expect(ping).toHaveBeenCalledWith("10.0.0.1", 22);
+    expect(service.get(1)?.status).toBe("reachable");
+    expect(emitted).toContainEqual({
+      hostId: 1,
+      ownerUserId: "owner",
+      status: "reachable",
+      previous: null,
+      online: true,
+    });
+  });
+
+  it("reports offline for a host that does not answer", async () => {
+    const { service } = setup([target(1)], false);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(service.get(1)?.status).toBe("offline");
+  });
+
+  it("never checks a host with status checks off", async () => {
+    const { service, ping } = setup([target(1, { statusCheckEnabled: false })]);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(ping).not.toHaveBeenCalled();
+    expect(service.get(1)).toBeNull();
+  });
+
+  it("goes through the jump hosts when a host has them", async () => {
+    const { service, ping, pingThroughJumpHosts } = setup([
+      target(1, { jumpHosts: [{ hostId: 9 }] }),
+    ]);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(ping).not.toHaveBeenCalled();
+    expect(pingThroughJumpHosts).toHaveBeenCalledOnce();
+  });
+
+  it("asks a registered plugin for a protocol's port", async () => {
+    const { service, ping } = setup([target(1, { connectionType: "rdp" })]);
+    active = service;
+    service.registerPort("rdp", async () => 3390);
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(ping).toHaveBeenCalledWith("10.0.0.1", 3390);
+  });
+
+  it("falls back to the host's port when no plugin knows the protocol", async () => {
+    const { service, ping } = setup([
+      target(1, { connectionType: "rdp", port: 3389 }),
+    ]);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(ping).toHaveBeenCalledWith("10.0.0.1", 3389);
+  });
+
+  it("turns reachable into online when a plugin reports a login", async () => {
+    const { service, emitted } = setup([target(1)]);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+
+    service.reportLogin(1, { ok: true });
+    await flush();
+    expect(service.get(1)?.status).toBe("online");
+    expect(emitted.at(-1)).toMatchObject({
+      status: "online",
+      previous: "reachable",
+    });
+
+    service.reportLogin(1, { ok: false, hostKeyChanged: true });
+    expect(service.get(1)).toMatchObject({
+      status: "reachable",
+      reason: "host_key_changed",
+    });
+  });
+
+  it("clears the host key reason once the new key is accepted", async () => {
+    const { service } = setup([target(1)]);
+    active = service;
+    service.start();
+    await service.statusesFor("owner", null);
+    await flush();
+    service.reportLogin(1, { ok: false, hostKeyChanged: true });
+
+    pluginEvents.emit(TOPICS.hostKeyUpdated, { hostId: 1 });
+    expect(service.get(1)?.reason).toBeUndefined();
+  });
+
+  it("shows a host with an open terminal session as online", async () => {
+    const { service } = setup([target(1)]);
+    active = service;
+    service.start();
+    await service.statusesFor("owner", null);
+    await flush();
+
+    const release = hostSessionStatus.register(1);
+    expect(service.get(1)?.status).toBe("online");
+    release();
+    expect(service.get(1)?.status).toBe("reachable");
+  });
+
+  it("only checks the requested hosts on the desktop app", async () => {
+    const { service, ping } = setup([target(1), target(2)]);
+    active = service;
+    await service.statusesFor("owner", new Set([2]));
+    await flush();
+    expect(ping).toHaveBeenCalledTimes(1);
+    expect(ping).toHaveBeenCalledWith("10.0.0.2", 22);
+  });
+
+  it("checks again on its interval", async () => {
+    const { service, ping } = setup([target(1, { statusCheckInterval: 10 })]);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(ping).toHaveBeenCalledTimes(1);
+
+    vi.advanceTimersByTime(15_000);
+    await flush();
+    expect(ping).toHaveBeenCalledTimes(2);
+  });
+
+  it("checks a host on demand unless the last result is fresh", async () => {
+    const { service, ping } = setup([target(1)]);
+    active = service;
+    expect(await service.check(1)).toMatchObject({ status: "reachable" });
+    expect(await service.check(1)).toMatchObject({ status: "reachable" });
+    expect(ping).toHaveBeenCalledTimes(1);
+  });
+
+  it("forgets a deleted host", async () => {
+    const { service } = setup([target(1)]);
+    active = service;
+    service.start();
+    await service.statusesFor("owner", null);
+    await flush();
+
+    pluginEvents.emit(TOPICS.hostDeleted, { hostId: 1 });
+    expect(service.get(1)).toBeNull();
+  });
+
+  it("never logs in to check a host", async () => {
+    // The only transports are the two ping functions: nothing here can
+    // authenticate, so a RADIUS or Duo backed host gets no 2FA push.
+    const { service, ping, pingThroughJumpHosts } = setup([target(1)]);
+    active = service;
+    await service.statusesFor("owner", null);
+    await flush();
+    expect(ping.mock.calls[0]).toEqual(["10.0.0.1", 22]);
+    expect(pingThroughJumpHosts).not.toHaveBeenCalled();
+  });
+});
+
+describe("toStatusTarget", () => {
+  it("parses the jump hosts and defaults the connection type", () => {
+    expect(
+      toStatusTarget({
+        id: 1,
+        userId: "u",
+        ip: "h",
+        port: 22,
+        connectionType: "",
+        jumpHosts: '[{"hostId":3},{"bad":1}]',
+        statusCheckEnabled: false,
+        statusCheckInterval: 30,
+      }),
+    ).toEqual({
+      id: 1,
+      userId: "u",
+      ip: "h",
+      port: 22,
+      connectionType: "ssh",
+      jumpHosts: [{ hostId: 3 }],
+      statusCheckEnabled: false,
+      statusCheckInterval: 30,
+    });
+  });
+});

src/backend/tests/hosts/status/host-status.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, it } from "vitest";
+import {
+  isHostKeyVerificationError,
+  statusAfterAuthentication,
+  statusAfterReachabilityCheck,
+} from "../../../hosts/status/host-status.js";
+
+describe("isHostKeyVerificationError", () => {
+  it.each([
+    "Host denied (verification failed)",
+    "Host key changed - please connect via Terminal to verify the new key",
+  ])("classifies %s", (message) => {
+    expect(isHostKeyVerificationError(new Error(message))).toBe(true);
+  });
+
+  it("does not classify ordinary authentication failures", () => {
+    expect(isHostKeyVerificationError(new Error("Permission denied"))).toBe(
+      false,
+    );
+  });
+});
+
+describe("host availability status", () => {
+  it("does not call a TCP-reachable host online before authentication", () => {
+    expect(statusAfterReachabilityCheck(true)).toBe("reachable");
+  });
+
+  it("keeps a verified host online across later reachability checks", () => {
+    expect(statusAfterReachabilityCheck(true, "online")).toBe("online");
+  });
+
+  it("marks successful authentication online", () => {
+    expect(statusAfterAuthentication(true, "reachable")).toBe("online");
+  });
+
+  it("downgrades failed authentication without hiding reachability", () => {
+    expect(statusAfterAuthentication(false, "online")).toBe("reachable");
+    expect(statusAfterAuthentication(false, "offline")).toBe("offline");
+  });
+});

src/backend/tests/hosts/status/limiter.test.ts

@@ -0,0 +1,162 @@
+import { describe, expect, it } from "vitest";
+import { ConcurrentLimiter } from "../../../hosts/status/limiter.js";
+
+describe("ConcurrentLimiter", () => {
+  it("never exceeds max concurrent runners", async () => {
+    const limiter = new ConcurrentLimiter(2);
+    let peak = 0;
+    let current = 0;
+
+    const job = async () => {
+      current += 1;
+      peak = Math.max(peak, current);
+      await new Promise((r) => setTimeout(r, 30));
+      current -= 1;
+    };
+
+    await Promise.all([
+      limiter.run(job),
+      limiter.run(job),
+      limiter.run(job),
+      limiter.run(job),
+    ]);
+
+    expect(peak).toBeLessThanOrEqual(2);
+    expect(limiter.activeCount).toBe(0);
+    expect(limiter.pendingCount).toBe(0);
+  });
+
+  it("runs waiters in FIFO order after a slot frees", async () => {
+    const limiter = new ConcurrentLimiter(1);
+    const order: number[] = [];
+
+    const first = limiter.run(async () => {
+      order.push(1);
+      await new Promise((r) => setTimeout(r, 20));
+    });
+    const second = limiter.run(async () => {
+      order.push(2);
+    });
+    const third = limiter.run(async () => {
+      order.push(3);
+    });
+
+    await Promise.all([first, second, third]);
+    expect(order).toEqual([1, 2, 3]);
+  });
+
+  it("rejects invalid maxConcurrent", () => {
+    expect(() => new ConcurrentLimiter(0)).toThrow(/maxConcurrent/);
+  });
+
+  describe("setLimit", () => {
+    it("releases queued waiters as soon as the ceiling is raised", async () => {
+      const limiter = new ConcurrentLimiter(1);
+      let running = 0;
+      let peak = 0;
+      const release: Array<() => void> = [];
+
+      const job = () =>
+        limiter.run(async () => {
+          running += 1;
+          peak = Math.max(peak, running);
+          await new Promise<void>((r) => release.push(r));
+          running -= 1;
+        });
+
+      const jobs = [job(), job(), job(), job()];
+      await new Promise((r) => setTimeout(r, 10));
+      expect(peak).toBe(1);
+      expect(limiter.pendingCount).toBe(3);
+
+      // Widening must drain the backlog without waiting for the running job.
+      limiter.setLimit(4);
+      await new Promise((r) => setTimeout(r, 10));
+      expect(peak).toBe(4);
+      expect(limiter.pendingCount).toBe(0);
+
+      release.forEach((fn) => fn());
+      await Promise.all(jobs);
+      expect(limiter.activeCount).toBe(0);
+    });
+
+    it("does not over-release beyond the new ceiling", async () => {
+      const limiter = new ConcurrentLimiter(1);
+      let running = 0;
+      let peak = 0;
+      const release: Array<() => void> = [];
+      // Later waves of woken jobs enqueue their own resolvers, so draining has
+      // to keep going until nothing is left rather than flushing a snapshot.
+      const drain = async () => {
+        while (release.length > 0) {
+          release.splice(0).forEach((fn) => fn());
+          await new Promise((r) => setTimeout(r, 5));
+        }
+      };
+
+      const job = () =>
+        limiter.run(async () => {
+          running += 1;
+          peak = Math.max(peak, running);
+          await new Promise<void>((r) => release.push(r));
+          running -= 1;
+        });
+
+      const jobs = [job(), job(), job(), job(), job()];
+      await new Promise((r) => setTimeout(r, 10));
+
+      limiter.setLimit(3);
+      await new Promise((r) => setTimeout(r, 10));
+      expect(peak).toBe(3);
+      expect(limiter.pendingCount).toBe(2);
+
+      await drain();
+      await Promise.all(jobs);
+      expect(limiter.activeCount).toBe(0);
+      expect(limiter.pendingCount).toBe(0);
+    });
+
+    it("lets running work finish when the ceiling shrinks", async () => {
+      const limiter = new ConcurrentLimiter(4);
+      let running = 0;
+      let peak = 0;
+      const release: Array<() => void> = [];
+      const drain = async () => {
+        while (release.length > 0) {
+          release.splice(0).forEach((fn) => fn());
+          await new Promise((r) => setTimeout(r, 5));
+        }
+      };
+
+      const job = () =>
+        limiter.run(async () => {
+          running += 1;
+          peak = Math.max(peak, running);
+          await new Promise<void>((r) => release.push(r));
+          running -= 1;
+        });
+
+      const jobs = [job(), job(), job(), job(), job(), job()];
+      await new Promise((r) => setTimeout(r, 10));
+      expect(peak).toBe(4);
+
+      // Shrinking never kills in-flight work; it applies to later releases.
+      limiter.setLimit(2);
+      expect(limiter.activeCount).toBe(4);
+
+      await drain();
+      await Promise.all(jobs);
+      expect(limiter.activeCount).toBe(0);
+      expect(limiter.pendingCount).toBe(0);
+      // The two queued jobs ran only after the shrink, so they never pushed
+      // occupancy back up to the old width.
+      expect(peak).toBe(4);
+    });
+
+    it("rejects an invalid new limit", () => {
+      const limiter = new ConcurrentLimiter(2);
+      expect(() => limiter.setLimit(0)).toThrow(/maxConcurrent/);
+      expect(limiter.limit).toBe(2);
+    });
+  });
+});

src/backend/tests/hosts/status/tcp-ping.test.ts

@@ -0,0 +1,57 @@
+import type { Client } from "ssh2";
+import { describe, expect, it, vi } from "vitest";
+import { tcpPingThroughJumpHost } from "../../../hosts/status/tcp-ping.js";
+
+describe("tcpPingThroughJumpHost", () => {
+  it("reports the final destination online when forwarding succeeds", async () => {
+    const stream = { destroy: vi.fn() };
+    const jumpClient = {
+      end: vi.fn(),
+      forwardOut: vi.fn((_src, _srcPort, host, port, callback) => {
+        expect(host).toBe("private.example");
+        expect(port).toBe(22);
+        callback(undefined, stream);
+      }),
+    } as unknown as Pick<Client, "forwardOut" | "end">;
+
+    await expect(
+      tcpPingThroughJumpHost(jumpClient, "private.example", 22),
+    ).resolves.toBe(true);
+    expect(stream.destroy).toHaveBeenCalledOnce();
+    expect(jumpClient.end).toHaveBeenCalledOnce();
+  });
+
+  it("reports the final destination offline when forwarding fails", async () => {
+    const jumpClient = {
+      end: vi.fn(),
+      forwardOut: vi.fn((_src, _srcPort, _host, _port, callback) => {
+        callback(new Error("Connection refused"));
+      }),
+    } as unknown as Pick<Client, "forwardOut" | "end">;
+
+    await expect(
+      tcpPingThroughJumpHost(jumpClient, "private.example", 22),
+    ).resolves.toBe(false);
+    expect(jumpClient.end).toHaveBeenCalledOnce();
+  });
+
+  it("reports the final destination offline when forwarding times out", async () => {
+    vi.useFakeTimers();
+    const jumpClient = {
+      end: vi.fn(),
+      forwardOut: vi.fn(),
+    } as unknown as Pick<Client, "forwardOut" | "end">;
+
+    const result = tcpPingThroughJumpHost(
+      jumpClient,
+      "private.example",
+      22,
+      5000,
+    );
+    await vi.advanceTimersByTimeAsync(5000);
+
+    await expect(result).resolves.toBe(false);
+    expect(jumpClient.end).toHaveBeenCalledOnce();
+    vi.useRealTimers();
+  });
+});