fix: host status shows online when the port answers and only warns on a failed login

This commit is contained in:
LukeGus committed 2026-09-30 18:18:38 -05:00
1 parent c28cb1bfc6
commit 49acec4770
14 files changed
+141 -141

No files matched your search

@@ -24,7 +24,7 @@ function StatusDot({ status }: { status: string }) {
status === "online"
? getAccentColor()
: status === "reachable"
? `color-mix(in srgb, ${getAccentColor()} 40%, transparent)`
? "#fbbf24"
: status === "offline"
? "#ef4444"
: "#6b7280";
@@ -60,14 +60,14 @@ function HostStatusWidget({
: online
? getAccentColor()
: reachable
? `color-mix(in srgb, ${getAccentColor()} 40%, transparent)`
? "#fbbf24"
: "#ef4444";
const onlineLabel = !statusInfo
? t("common.unknown")
: online
? t("common.online")
: reachable
? t("common.reachable", { defaultValue: "Reachable" })
? t("common.signInFailed")
: t("common.offline");
return (
+2 -1
View File
@@ -368,7 +368,8 @@ export class MetricsPoller {
const message = error instanceof Error ? error.message : "";
const isAuthError = /authentication|permission denied/i.test(message);
if (!authenticated) {
// A timeout or a dropped connection says nothing about the login.
if (!authenticated && (isAuthError || hostKeyChanged)) {
this.ctx.hosts.status.reportLogin(host.id, {
ok: false,
hostKeyChanged,
@@ -154,6 +154,19 @@ describe("MetricsPoller", () => {
});
});
it("does not report a login for a connection that timed out", async () => {
collectMetrics.mockRejectedValue(new Error("Timed out while waiting"));
const { fake, poller } = setup();
poller.registerViewer(7, "viewer-a", "user-1");
await flush();
expect(fake.statusReports).toEqual([]);
expect(fake.emitted.at(-1)?.payload).toMatchObject({
state: "unreachable",
});
});
it("marks a changed host key", async () => {
collectMetrics.mockRejectedValue(new Error("Host key changed"));
const { fake, poller, state } = setup();
@@ -142,9 +142,7 @@ function buildNodeSvg(
? "rgb(16,185,129)"
: resolveCssVar("--accent-brand", "rgb(16,185,129)");
} else if (status === "reachable") {
statusColor = useRealColors
? "rgba(16,185,129,0.4)"
: `color-mix(in srgb, ${resolveCssVar("--accent-brand", "rgb(16,185,129)")} 40%, transparent)`;
statusColor = "rgb(251,191,36)";
} else if (isOffline) {
statusColor = useRealColors ? "rgb(239,68,68)" : "rgba(16,185,129,0.2)";
} else {
+23 -51
View File
@@ -2,12 +2,13 @@
* The host list's status dot.
*
* Every host with status checks on gets a TCP connection to its port on a
* timer. That alone says "reachable". A host becomes "online" when a login
* worked: an open terminal session, any plugin connection through ctx.ssh, or
* a plugin reporting one through ctx.hosts.status.reportLogin. It stays online
* for LOGIN_FRESH_MS after the last login while the port still answers.
* Probing never logs in: on RADIUS or Duo backed devices a login fires a real
* 2FA push, every interval.
* timer. A port that answers is "online", one that does not is "offline".
* "reachable" is the warning in between: the port answers but the last login
* failed (a session, a ctx.ssh connection or a plugin's
* ctx.hosts.status.reportLogin said so). It clears on the next working login,
* when the host is edited, or when a changed host key is accepted. Probing
* never logs in: on RADIUS or Duo backed devices a login fires a real 2FA
* push, every interval.
*
* Polling is demand driven. A user asking for statuses starts polling their
* own hosts; nothing is probed for a user who never opened the app.
@@ -33,8 +34,6 @@ export const GLOBAL_STATUS_INTERVAL_KEY = "global_status_check_interval";
export const DEFAULT_STATUS_INTERVAL = 60;
/** A status younger than this is fresh enough for check(). */
const FRESH_MS = 30_000;
/** How long a successful login keeps a reachable host online. */
export const LOGIN_FRESH_MS = 60 * 60_000;
export interface HostStatusEntry {
status: HostStatus;
@@ -81,7 +80,6 @@ export interface HostStatusDeps {
) => Promise<boolean>;
globalInterval: () => number;
emit: (payload: HostStatusPayload) => void;
now?: () => number;
}
function parseJumpHosts(raw: string | null): Array<{ hostId: number }> {
@@ -158,8 +156,7 @@ export class HostStatusService {
private readonly store = new Map<number, HostStatusEntry>();
private readonly owners = new Map<number, string>();
private readonly inFlight = new Map<number, Promise<void>>();
private readonly sessionOnline = new Set<number>();
private readonly lastLogin = new Map<number, number>();
private readonly loginFailed = new Set<number>();
private readonly startedUsers = new Set<string>();
private readonly portResolvers = new Map<string, Set<PortResolver>>();
private readonly limiter = new ConcurrentLimiter(20);
@@ -171,12 +168,15 @@ export class HostStatusService {
start(): void {
if (this.unsubscribers.length > 0) return;
this.unsubscribers = [
hostSessionStatus.subscribe((hostId, online) =>
this.setSessionOnline(hostId, online),
),
hostSessionStatus.subscribe((hostId, online) => {
if (online) this.reportLogin(hostId, { ok: true });
}),
pluginEvents.on(TOPICS.hostUpdated, (payload) => {
const { hostId } = payload as { hostId?: number };
if (hostId) void this.reload(hostId);
if (!hostId) return;
// An edit may have fixed the login, so the next attempt decides.
this.loginFailed.delete(hostId);
void this.reload(hostId);
}),
pluginEvents.on(TOPICS.hostDeleted, (payload) => {
const { hostId } = payload as { hostId?: number };
@@ -252,15 +252,13 @@ export class HostStatusService {
outcome: { ok: boolean; hostKeyChanged?: boolean },
): void {
if (outcome.ok) {
this.lastLogin.set(hostId, this.now());
this.loginFailed.delete(hostId);
this.set(hostId, { status: statusAfterAuthentication(true) });
return;
}
this.lastLogin.delete(hostId);
this.loginFailed.add(hostId);
this.set(hostId, {
status: this.sessionOnline.has(hostId)
? "online"
: statusAfterAuthentication(false, this.store.get(hostId)?.status),
status: statusAfterAuthentication(false, this.store.get(hostId)?.status),
...(outcome.hostKeyChanged ? { reason: "host_key_changed" } : {}),
});
}
@@ -351,44 +349,18 @@ export class HostStatusService {
this.stopPolling(hostId);
this.store.delete(hostId);
this.owners.delete(hostId);
this.lastLogin.delete(hostId);
this.loginFailed.delete(hostId);
}
private clearHostKeyReason(hostId: number): void {
const current = this.store.get(hostId);
if (current?.reason !== "host_key_changed") return;
this.store.set(hostId, {
status: current.status,
lastChecked: current.lastChecked,
this.loginFailed.delete(hostId);
this.set(hostId, {
status: current.status === "offline" ? "offline" : "online",
});
}
private setSessionOnline(hostId: number, online: boolean): void {
if (online) {
this.sessionOnline.add(hostId);
this.lastLogin.set(hostId, this.now());
if (this.polled.has(hostId)) this.set(hostId, { status: "online" });
return;
}
this.sessionOnline.delete(hostId);
if (
!this.isLoggedIn(hostId) &&
this.store.get(hostId)?.status === "online"
) {
this.set(hostId, { status: "reachable" });
}
}
private now(): number {
return this.deps.now?.() ?? Date.now();
}
private isLoggedIn(hostId: number): boolean {
if (this.sessionOnline.has(hostId)) return true;
const at = this.lastLogin.get(hostId);
return at !== undefined && this.now() - at < LOGIN_FRESH_MS;
}
private intervalMs(target: StatusTarget): number {
const seconds = target.statusCheckInterval ?? this.deps.globalInterval();
const intervalMs = Math.max(5, seconds) * 1000;
@@ -470,7 +442,7 @@ export class HostStatusService {
this.set(target.id, {
status: statusAfterReachabilityCheck(
reachable,
this.isLoggedIn(target.id),
this.loginFailed.has(target.id),
),
...(reachable && current?.reason === "host_key_changed"
? { reason: "host_key_changed" as const }
+2 -2
View File
@@ -10,10 +10,10 @@ export function isHostKeyVerificationError(error: unknown): boolean {
export function statusAfterReachabilityCheck(
reachable: boolean,
loggedIn: boolean,
loginFailed: boolean,
): HostStatus {
if (!reachable) return "offline";
return loggedIn ? "online" : "reachable";
return loginFailed ? "reachable" : "online";
}
export function statusAfterAuthentication(
+18 -3
View File
@@ -33,6 +33,7 @@ import {
} from "../hosts/connect/auth-provider-registry.js";
import { classifyKeyboardInteractive } from "../hosts/connect/keyboard-interactive.js";
import { ensureCoreSshAuthProviders } from "../hosts/connect/core-providers.js";
import { isHostKeyVerificationError } from "../hosts/status/host-status.js";
import {
getLoginMethod,
registerLoginMethod,
@@ -95,6 +96,9 @@ function describeHost(host: number | PluginSshHost): string {
return typeof host === "number" ? `host ${host}` : `host ${host.id}`;
}
const LOGIN_FAILED =
/all configured authentication methods failed|permission denied|authentication failed/i;
export function createPluginSsh({ manifest, bag, audit }: Deps): PluginSsh {
const pluginId = manifest.id;
const declared = manifest.capabilities;
@@ -164,14 +168,24 @@ export function createPluginSsh({ manifest, bag, audit }: Deps): PluginSsh {
return { ...(host as unknown as SshConnectHost), userId };
};
/** A login to a saved host core resolved counts toward its status dot. */
const reportLogin = (host: number | PluginSshHost) => {
/**
* A login to a saved host core resolved feeds its status dot. Only a clear
* auth or host key failure counts: a timeout says nothing about the login.
*/
const reportLogin = (host: number | PluginSshHost, error?: unknown) => {
const hostId =
typeof host === "number" ? host : resolvedHosts.get(host)?.host.id;
if (!Number.isInteger(hostId) || hostId <= 0) return;
let outcome: { ok: boolean; hostKeyChanged?: boolean } = { ok: true };
if (error !== undefined) {
const message = error instanceof Error ? error.message : String(error);
const hostKeyChanged = isHostKeyVerificationError(error);
if (!hostKeyChanged && !LOGIN_FAILED.test(message)) return;
outcome = { ok: false, hostKeyChanged };
}
void import("../hosts/status/host-status-service.js")
.then(({ hostStatusService }) =>
hostStatusService.reportLogin(hostId, { ok: true }),
hostStatusService.reportLogin(hostId, outcome),
)
.catch(() => {});
};
@@ -231,6 +245,7 @@ export function createPluginSsh({ manifest, bag, audit }: Deps): PluginSsh {
success: false,
errorMessage: error instanceof Error ? error.message : String(error),
});
reportLogin(host, error);
throw error;
}
};
@@ -9,7 +9,7 @@ vi.mock("../../../utils/logger.js", () => ({
pluginLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
const { HostStatusService, toStatusTarget, LOGIN_FRESH_MS } =
const { HostStatusService, toStatusTarget } =
await import("../../../hosts/status/host-status-service.js");
const { hostSessionStatus } =
await import("../../../hosts/host-session-status.js");
@@ -35,7 +35,6 @@ function setup(
targets: Target[],
reachable = true,
shared: Record<string, number[]> = {},
clock = { now: 1_000_000 },
) {
const emitted: unknown[] = [];
const ping = vi.fn(async () => reachable);
@@ -55,7 +54,6 @@ function setup(
pingThroughJumpHosts,
globalInterval: () => 60,
emit: (payload) => emitted.push(payload),
now: () => clock.now,
});
return { service, emitted, ping, pingThroughJumpHosts, loadTargets };
}
@@ -76,7 +74,7 @@ afterEach(() => {
});
describe("HostStatusService", () => {
it("starts a user's own hosts and reports them reachable", async () => {
it("starts a user's own hosts and reports them online", async () => {
const { service, emitted, ping } = setup([target(1), target(2)]);
active = service;
@@ -84,11 +82,11 @@ describe("HostStatusService", () => {
await flush();
expect(ping).toHaveBeenCalledWith("10.0.0.1", 22);
expect(service.get(1)?.status).toBe("reachable");
expect(service.get(1)?.status).toBe("online");
expect(emitted).toContainEqual({
hostId: 1,
ownerUserId: "owner",
status: "reachable",
status: "online",
previous: null,
online: true,
});
@@ -102,7 +100,7 @@ describe("HostStatusService", () => {
await flush();
expect(ping).toHaveBeenCalledWith("10.0.0.5", 22);
expect(service.get(5)?.status).toBe("reachable");
expect(service.get(5)?.status).toBe("online");
});
it("only starts the shared hosts a desktop request names", async () => {
@@ -165,90 +163,82 @@ describe("HostStatusService", () => {
expect(ping).toHaveBeenCalledWith("10.0.0.1", 3389);
});
it("turns reachable into online when a plugin reports a login", async () => {
it("warns about a failed login until a login works again", async () => {
const { service, emitted } = setup([target(1)]);
active = service;
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: true });
await flush();
expect(service.get(1)?.status).toBe("online");
service.reportLogin(1, { ok: false });
expect(service.get(1)?.status).toBe("reachable");
expect(emitted.at(-1)).toMatchObject({
status: "online",
previous: "reachable",
status: "reachable",
previous: "online",
});
vi.advanceTimersByTime(75_000);
await flush();
expect(service.get(1)?.status).toBe("reachable");
service.reportLogin(1, { ok: true });
expect(service.get(1)?.status).toBe("online");
vi.advanceTimersByTime(75_000);
await flush();
expect(service.get(1)?.status).toBe("online");
});
it("marks a changed host key and clears it once the new key is accepted", async () => {
const { service } = setup([target(1)]);
active = service;
service.start();
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: false, hostKeyChanged: true });
expect(service.get(1)).toMatchObject({
status: "reachable",
reason: "host_key_changed",
});
});
it("clears the host key reason once the new key is accepted", async () => {
const { service } = setup([target(1)]);
active = service;
service.start();
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: false, hostKeyChanged: true });
pluginEvents.emit(TOPICS.hostKeyUpdated, { hostId: 1 });
expect(service.get(1)?.reason).toBeUndefined();
expect(service.get(1)?.status).toBe("online");
});
it("shows a host with an open terminal session as online", async () => {
it("clears a failed login when the host is edited", async () => {
const { service } = setup([target(1)]);
active = service;
service.start();
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: false });
const release = hostSessionStatus.register(1);
expect(service.get(1)?.status).toBe("online");
release();
pluginEvents.emit(TOPICS.hostUpdated, { hostId: 1 });
await flush();
await flush();
expect(service.get(1)?.status).toBe("online");
});
it("keeps a host online for a while after its last login", async () => {
const clock = { now: 1_000_000 };
const { service } = setup([target(1)], true, {}, clock);
active = service;
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: true });
clock.now += LOGIN_FRESH_MS - 1;
vi.advanceTimersByTime(75_000);
await flush();
expect(service.get(1)?.status).toBe("online");
clock.now += 2;
vi.advanceTimersByTime(75_000);
await flush();
expect(service.get(1)?.status).toBe("reachable");
});
it("drops an ended session back to reachable once the login is stale", async () => {
const clock = { now: 1_000_000 };
const { service } = setup([target(1)], true, {}, clock);
it("clears a failed login when a terminal session opens", async () => {
const { service } = setup([target(1)]);
active = service;
service.start();
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: false });
const release = hostSessionStatus.register(1);
clock.now += LOGIN_FRESH_MS;
expect(service.get(1)?.status).toBe("online");
release();
expect(service.get(1)?.status).toBe("reachable");
expect(service.get(1)?.status).toBe("online");
});
it("shows an offline host as offline even after a login", async () => {
it("shows an offline host as offline whatever the login said", async () => {
const { service } = setup([target(1)], false);
active = service;
await service.statusesFor("owner", null);
await flush();
service.reportLogin(1, { ok: false });
expect(service.get(1)?.status).toBe("offline");
service.reportLogin(1, { ok: true });
vi.advanceTimersByTime(75_000);
await flush();
@@ -279,8 +269,8 @@ describe("HostStatusService", () => {
it("checks a host on demand unless the last result is fresh", async () => {
const { service, ping } = setup([target(1)]);
active = service;
expect(await service.check(1)).toMatchObject({ status: "reachable" });
expect(await service.check(1)).toMatchObject({ status: "reachable" });
expect(await service.check(1)).toMatchObject({ status: "online" });
expect(await service.check(1)).toMatchObject({ status: "online" });
expect(ping).toHaveBeenCalledTimes(1);
});
@@ -21,12 +21,13 @@ describe("isHostKeyVerificationError", () => {
});
describe("host availability status", () => {
it("does not call a TCP-reachable host online before authentication", () => {
expect(statusAfterReachabilityCheck(true, false)).toBe("reachable");
it("calls a host whose port answers online", () => {
expect(statusAfterReachabilityCheck(true, false)).toBe("online");
expect(statusAfterReachabilityCheck(false, false)).toBe("offline");
});
it("keeps a recently verified host online across reachability checks", () => {
expect(statusAfterReachabilityCheck(true, true)).toBe("online");
it("warns about a reachable host whose last login failed", () => {
expect(statusAfterReachabilityCheck(true, true)).toBe("reachable");
expect(statusAfterReachabilityCheck(false, true)).toBe("offline");
});
@@ -14,6 +14,7 @@ const h = vi.hoisted(() => ({
trustedCleared: [] as string[],
policyError: null as Error | null,
logins: [] as Array<{ hostId: number; outcome: unknown }>,
connectError: null as Error | null,
}));
vi.mock("../../utils/logger.js", () => {
@@ -85,6 +86,7 @@ vi.mock("../../hosts/connect/connect-host.js", () => ({
: target,
connectHost: async (target: unknown, options: Record<string, unknown>) => {
h.connects.push({ target, options });
if (h.connectError) throw h.connectError;
const client = Object.assign(new EventEmitter(), { end: vi.fn() });
return { client, jumpClient: null, dispose: vi.fn(() => client.end()) };
},
@@ -149,6 +151,7 @@ beforeEach(() => {
h.clearedAll = 0;
h.pooled = [];
h.logins = [];
h.connectError = null;
});
describe("ctx.ssh", () => {
@@ -205,6 +208,26 @@ describe("ctx.ssh", () => {
expect(h.logins).toEqual([{ hostId: 7, outcome: { ok: true } }]);
});
it("reports an auth or host key failure, but not a timeout", async () => {
h.granted = new Set(["ssh:connect", "credentials:use"]);
const ssh = createPluginSsh({
manifest: manifest(["ssh:connect", "credentials:use"]),
bag: new DisposableBag("fixture"),
audit: vi.fn(async () => {}),
});
h.connectError = new Error("Timed out while waiting for handshake");
await expect(ssh.connect(7)).rejects.toThrow();
h.connectError = new Error("All configured authentication methods failed");
await expect(ssh.connect(7)).rejects.toThrow();
h.connectError = new Error("Host key changed");
await expect(ssh.connect(8)).rejects.toThrow();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(h.logins).toEqual([
{ hostId: 7, outcome: { ok: false, hostKeyChanged: false } },
{ hostId: 8, outcome: { ok: false, hostKeyChanged: true } },
]);
});
it("never believes a host object's own userId", async () => {
h.actor = undefined;
h.granted = new Set(["ssh:connect", "credentials:use"]);
+3 -10
View File
@@ -65,17 +65,10 @@ export function getStatusClasses(
if (variant === "stripe") return "bg-muted-foreground/20 animate-pulse";
return "border-border/50 text-muted-foreground/50 bg-muted/20 animate-pulse";
}
// Reachable is a faded online, not a warning: the port answers, nobody has
// logged in lately.
if (reachable) {
if (scheme === "status") {
if (variant === "dot") return "bg-emerald-500/40";
if (variant === "stripe") return "bg-emerald-500/30";
return "border-emerald-500/30 text-emerald-500/70 bg-emerald-500/5";
}
if (variant === "dot") return "bg-accent-brand/40";
if (variant === "stripe") return "bg-accent-brand/30";
return "border-accent-brand/30 text-accent-brand/70 bg-accent-brand/5";
if (variant === "dot") return "bg-amber-400";
if (variant === "stripe") return "bg-amber-400/50";
return "border-amber-400/40 text-amber-400 bg-amber-400/10";
}
if (scheme === "status") {
if (variant === "dot") return online ? "bg-emerald-500" : "bg-red-500";
+2 -1
View File
@@ -120,6 +120,7 @@
"minimize": "Minimize",
"online": "Online",
"offline": "Offline",
"signInFailed": "Sign in failed",
"unknown": "Unknown",
"continue": "Continue",
"maintenance": "Maintenance",
@@ -349,7 +350,7 @@
"serverStats": "Host Metrics",
"status": {
"available": "Available",
"reachable": "Up, not signed in lately",
"reachable": "Up, but the last sign in failed",
"offline": "Offline",
"monitoringDisabled": "Monitoring disabled",
"label": "Status"
@@ -66,15 +66,8 @@ describe("getStatusClasses", () => {
expect(getStatusClasses(true, "accent", "dot")).toContain("accent-brand");
});
it("draws reachable as a faded online, never a warning color", () => {
expect(getStatusClasses("reachable", "accent", "dot")).toBe(
"bg-accent-brand/40",
);
expect(getStatusClasses("reachable", "status", "dot")).toBe(
"bg-emerald-500/40",
);
expect(getStatusClasses("reachable", "status", "badge")).not.toMatch(
/amber|yellow|red/,
);
it("draws reachable as a warning in both schemes", () => {
expect(getStatusClasses("reachable", "accent", "dot")).toContain("amber");
expect(getStatusClasses("reachable", "status", "dot")).toContain("amber");
});
});