mirror of
https://github.com/Termix-SSH/Termix.git
synced 2026-10-29 14:45:24 +00:00
feat: finish the host-metrics plugin, move host status into core (B16)
This commit is contained in:
LukeGus committed
1 parent
14eb558b85
commit
0a3f05d99d
308 files changed
+53073
-8985
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { authFailureTracker } from "./metrics-shared/state.js";
|
||||
import { pluginEvents, TOPICS } from "../plugins/events.js";
|
||||
import type { WebSocket } from "ws";
|
||||
import { createCurrentHostResolutionRepository } from "../database/repositories/factory.js";
|
||||
import { sshLogger } from "../utils/logger.js";
|
||||
@@ -328,7 +328,7 @@ export class SSHHostKeyVerifier {
|
||||
algorithm,
|
||||
currentChangeCount,
|
||||
);
|
||||
authFailureTracker.resetHostKeyFailure(hostId);
|
||||
pluginEvents.emit(TOPICS.hostKeyUpdated, { hostId });
|
||||
}
|
||||
|
||||
private static async promptUserForNewKey(
|
||||
|
||||
@@ -117,13 +117,6 @@ export async function resolveHostById(
|
||||
host.tunnelConnections = [];
|
||||
}
|
||||
}
|
||||
if (typeof host.statsConfig === "string" && host.statsConfig) {
|
||||
try {
|
||||
host.statsConfig = JSON.parse(host.statsConfig as string);
|
||||
} catch {
|
||||
host.statsConfig = undefined;
|
||||
}
|
||||
}
|
||||
if (typeof host.terminalConfig === "string" && host.terminalConfig) {
|
||||
try {
|
||||
host.terminalConfig = JSON.parse(host.terminalConfig as string);
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
import type { Client, ClientChannel } from "ssh2";
|
||||
|
||||
export function execCommand(
|
||||
client: Client,
|
||||
command: string,
|
||||
timeoutMs = 30000,
|
||||
): Promise<{
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
code: number | null;
|
||||
}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let stream: ClientChannel | null = null;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(new Error(`Command timeout after ${timeoutMs}ms: ${command}`));
|
||||
}
|
||||
}, timeoutMs);
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
if (stream) {
|
||||
try {
|
||||
stream.removeAllListeners();
|
||||
if (stream.stderr) {
|
||||
stream.stderr.removeAllListeners();
|
||||
}
|
||||
stream.destroy();
|
||||
} catch {
|
||||
// expected - cleanup errors ignored
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
client.exec(command, { pty: false }, (err, _stream) => {
|
||||
if (err) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(err);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
stream = _stream;
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let exitCode: number | null = null;
|
||||
|
||||
stream
|
||||
.on("close", (code: number | undefined) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
exitCode = typeof code === "number" ? code : null;
|
||||
cleanup();
|
||||
resolve({ stdout, stderr, code: exitCode });
|
||||
}
|
||||
})
|
||||
.on("data", (data: Buffer) => {
|
||||
stdout += data.toString("utf8");
|
||||
})
|
||||
.on("error", (streamErr: Error) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(streamErr);
|
||||
}
|
||||
});
|
||||
|
||||
if (stream.stderr) {
|
||||
stream.stderr
|
||||
.on("data", (data: Buffer) => {
|
||||
stderr += data.toString("utf8");
|
||||
})
|
||||
.on("error", (stderrErr: Error) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(stderrErr);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export type HostPlatform = "darwin" | "linux" | "windows" | "other";
|
||||
|
||||
// Windows OpenSSH's default shell is usually cmd.exe (occasionally
|
||||
// PowerShell), so scripts are sent base64-encoded via -EncodedCommand -
|
||||
// that avoids every cmd.exe quoting/escaping pitfall entirely, since the
|
||||
// argument ends up being plain alphanumeric text with no shell metacharacters.
|
||||
export function encodePowerShellCommand(script: string): string {
|
||||
return Buffer.from(script, "utf16le").toString("base64");
|
||||
}
|
||||
|
||||
export function execPowerShell(
|
||||
client: Client,
|
||||
script: string,
|
||||
timeoutMs = 30000,
|
||||
): Promise<{ stdout: string; stderr: string; code: number | null }> {
|
||||
const encoded = encodePowerShellCommand(script);
|
||||
return execCommand(
|
||||
client,
|
||||
`powershell -NoProfile -NonInteractive -EncodedCommand ${encoded}`,
|
||||
timeoutMs,
|
||||
);
|
||||
}
|
||||
|
||||
export async function detectPlatform(client: Client): Promise<HostPlatform> {
|
||||
try {
|
||||
const { stdout, code } = await execCommand(client, "uname -s", 10000);
|
||||
const kernel = stdout.trim().toLowerCase();
|
||||
if (kernel === "darwin") return "darwin";
|
||||
if (kernel === "linux") return "linux";
|
||||
if (code === 0 && kernel) return "other";
|
||||
} catch {
|
||||
// expected on hosts with no POSIX shell (e.g. Windows via cmd.exe)
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout } = await execPowerShell(client, "'WIN_OK'", 10000);
|
||||
if (stdout.includes("WIN_OK")) return "windows";
|
||||
} catch {
|
||||
// expected on hosts with no PowerShell available
|
||||
}
|
||||
|
||||
return "other";
|
||||
}
|
||||
|
||||
export function toFixedNum(
|
||||
n: number | null | undefined,
|
||||
digits = 2,
|
||||
): number | null {
|
||||
if (typeof n !== "number" || !Number.isFinite(n)) return null;
|
||||
return Number(n.toFixed(digits));
|
||||
}
|
||||
|
||||
export function kibToGiB(kib: number): number {
|
||||
return kib / (1024 * 1024);
|
||||
}
|
||||
@@ -1,169 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export type ElevationErrorCode = "SUDO_REQUIRED" | "SUDO_FAILED" | "NOT_SUDOER";
|
||||
|
||||
export class ElevationError extends Error {
|
||||
code: ElevationErrorCode;
|
||||
constructor(code: ElevationErrorCode, message: string) {
|
||||
super(message);
|
||||
this.name = "ElevationError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ElevatedResult {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
code: number | null;
|
||||
usedSudo: boolean;
|
||||
}
|
||||
|
||||
const PERMISSION_DENIED = [
|
||||
"permission denied",
|
||||
"operation not permitted",
|
||||
"must be run as root",
|
||||
"must be superuser",
|
||||
"you need to be root",
|
||||
"are you root",
|
||||
"access denied",
|
||||
];
|
||||
|
||||
/**
|
||||
* Phrases sudo itself prints to STDERR when authentication or authorization
|
||||
* fails. We only match these against sudo's stderr (never command output), so a
|
||||
* command that happens to print "permission denied" on stdout is not mistaken
|
||||
* for a sudo failure.
|
||||
*/
|
||||
const SUDO_AUTH_FAILED = [
|
||||
"incorrect password",
|
||||
"a password is required",
|
||||
"a terminal is required",
|
||||
"no tty present",
|
||||
"sorry, try again",
|
||||
"no password was provided",
|
||||
"1 incorrect password attempt",
|
||||
];
|
||||
|
||||
const SUDO_NOT_SUDOER = [
|
||||
"is not in the sudoers file",
|
||||
"not allowed to run sudo",
|
||||
"not allowed to execute",
|
||||
];
|
||||
|
||||
/**
|
||||
* Marker printed only after sudo has successfully authenticated and started the
|
||||
* inner shell. Its presence on stdout is the authoritative "elevation worked"
|
||||
* signal; its absence (together with sudo stderr) means auth failed.
|
||||
*/
|
||||
const SUDO_OK_MARKER = "__TX_SUDO_OK__";
|
||||
const SUDO_OK_LINE_RE = new RegExp(`^${SUDO_OK_MARKER}\\r?\\n?`);
|
||||
|
||||
/** Escape a value for single-quoted shell context. */
|
||||
export function shellSingleQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, "'\"'\"'")}'`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the elevated command string:
|
||||
* echo '<pw>' | sudo -S -p '' sh -c 'echo __TX_SUDO_OK__; <command>'
|
||||
*
|
||||
* `-p ''` suppresses the prompt. stderr is NOT merged into stdout, so the
|
||||
* command's own output stays clean and sudo's auth errors stay on stderr. The
|
||||
* marker is echoed by the inner shell once sudo authenticates, letting us tell a
|
||||
* genuine auth failure from command output that merely contains scary words.
|
||||
*/
|
||||
export function buildSudoCommand(
|
||||
command: string,
|
||||
sudoPassword: string,
|
||||
): string {
|
||||
const pw = shellSingleQuote(sudoPassword);
|
||||
const inner = shellSingleQuote(`echo ${SUDO_OK_MARKER}; ${command}`);
|
||||
return `echo ${pw} | sudo -S -p '' sh -c ${inner}`;
|
||||
}
|
||||
|
||||
function includesAny(text: string, needles: string[]): boolean {
|
||||
const lower = text.toLowerCase();
|
||||
return needles.some((n) => lower.includes(n));
|
||||
}
|
||||
|
||||
function looksLikePermissionDenied(text: string): boolean {
|
||||
return includesAny(text, PERMISSION_DENIED);
|
||||
}
|
||||
|
||||
/** Strip the success marker line from the front of stdout. */
|
||||
function stripMarker(stdout: string): string {
|
||||
return stdout.replace(SUDO_OK_LINE_RE, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a command on a pooled client, elevating with the host's stored sudo
|
||||
* password only when needed (or when forced). Throws a typed `ElevationError`
|
||||
* when elevation is required but unavailable/incorrect.
|
||||
*/
|
||||
export async function execElevated(
|
||||
client: Client,
|
||||
command: string,
|
||||
sudoPassword: string | undefined,
|
||||
opts: { forceSudo?: boolean; timeoutMs?: number } = {},
|
||||
): Promise<ElevatedResult> {
|
||||
const timeoutMs = opts.timeoutMs ?? 30000;
|
||||
|
||||
if (!opts.forceSudo) {
|
||||
const direct = await execCommand(client, command, timeoutMs);
|
||||
if (direct.code === 0) {
|
||||
return { ...direct, usedSudo: false };
|
||||
}
|
||||
// Only escalate when the failure looks like a privilege problem. A
|
||||
// permission-denied phrase on stderr (not arbitrary stdout) is the signal.
|
||||
if (!looksLikePermissionDenied(direct.stderr)) {
|
||||
return { ...direct, usedSudo: false };
|
||||
}
|
||||
if (!sudoPassword) {
|
||||
throw new ElevationError(
|
||||
"SUDO_REQUIRED",
|
||||
"This action requires elevated privileges. Set a sudo password for this host to continue.",
|
||||
);
|
||||
}
|
||||
} else if (!sudoPassword) {
|
||||
throw new ElevationError(
|
||||
"SUDO_REQUIRED",
|
||||
"This action requires elevated privileges. Set a sudo password for this host to continue.",
|
||||
);
|
||||
}
|
||||
|
||||
const sudoCmd = buildSudoCommand(command, sudoPassword as string);
|
||||
const result = await execCommand(client, sudoCmd, timeoutMs);
|
||||
const authenticated = result.stdout.includes(SUDO_OK_MARKER);
|
||||
|
||||
if (!authenticated) {
|
||||
// Elevation never started: diagnose from sudo's stderr only.
|
||||
const notSudoer = includesAny(result.stderr, SUDO_NOT_SUDOER);
|
||||
const authFailed = includesAny(result.stderr, SUDO_AUTH_FAILED);
|
||||
if (notSudoer) {
|
||||
throw new ElevationError(
|
||||
"NOT_SUDOER",
|
||||
"The connected user is not permitted to use sudo on this host.",
|
||||
);
|
||||
}
|
||||
if (authFailed) {
|
||||
throw new ElevationError(
|
||||
"SUDO_FAILED",
|
||||
"Elevation failed. Check the host's sudo password.",
|
||||
);
|
||||
}
|
||||
// No marker and no recognizable sudo error: treat as a generic failure but
|
||||
// keep the original output so the caller can surface it.
|
||||
throw new ElevationError(
|
||||
"SUDO_FAILED",
|
||||
"Elevation failed. Check the host's sudo password.",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
stdout: stripMarker(result.stdout),
|
||||
stderr: result.stderr,
|
||||
code: result.code,
|
||||
usedSudo: true,
|
||||
};
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
import type { PackageManager } from "./platform.js";
|
||||
|
||||
export interface UpgradablePackage {
|
||||
name: string;
|
||||
currentVersion?: string;
|
||||
newVersion?: string;
|
||||
}
|
||||
|
||||
export function buildListUpgradableCommand(pkg: PackageManager): string | null {
|
||||
switch (pkg) {
|
||||
case "apt":
|
||||
return "apt list --upgradable 2>/dev/null | tail -n +2";
|
||||
case "dnf":
|
||||
return "dnf -q check-update 2>/dev/null || true";
|
||||
case "yum":
|
||||
return "yum -q check-update 2>/dev/null || true";
|
||||
case "pacman":
|
||||
return "pacman -Qu 2>/dev/null || true";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseUpgradable(
|
||||
pkg: PackageManager,
|
||||
output: string,
|
||||
): UpgradablePackage[] {
|
||||
const out: UpgradablePackage[] = [];
|
||||
const lines = output
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
if (pkg === "apt") {
|
||||
for (const line of lines) {
|
||||
// name/suite newver arch [upgradable from: oldver]
|
||||
const m = line.match(
|
||||
/^([^/\s]+)\/\S+\s+(\S+)\s+\S+(?:\s+\[upgradable from:\s+(\S+)\])?/,
|
||||
);
|
||||
if (m) out.push({ name: m[1], newVersion: m[2], currentVersion: m[3] });
|
||||
}
|
||||
} else if (pkg === "dnf" || pkg === "yum") {
|
||||
for (const line of lines) {
|
||||
if (/^(Last metadata|Obsoleting|Security:)/i.test(line)) continue;
|
||||
const m = line.match(/^(\S+)\s+(\S+)\s+\S+$/);
|
||||
if (m && m[1].includes(".")) out.push({ name: m[1], newVersion: m[2] });
|
||||
}
|
||||
} else if (pkg === "pacman") {
|
||||
for (const line of lines) {
|
||||
const m = line.match(/^(\S+)\s+(\S+)\s+->\s+(\S+)$/);
|
||||
if (m) out.push({ name: m[1], currentVersion: m[2], newVersion: m[3] });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export type PackageAction = "upgrade-all" | "install" | "upgrade";
|
||||
|
||||
export function buildPackageActionCommand(
|
||||
pkg: PackageManager,
|
||||
action: PackageAction,
|
||||
name?: string,
|
||||
): string | null {
|
||||
const target = name ? ` ${name}` : "";
|
||||
switch (pkg) {
|
||||
case "apt":
|
||||
if (action === "upgrade-all")
|
||||
return "DEBIAN_FRONTEND=noninteractive apt-get -y upgrade";
|
||||
return `DEBIAN_FRONTEND=noninteractive apt-get -y install${target}`;
|
||||
case "dnf":
|
||||
return action === "upgrade-all"
|
||||
? "dnf -y upgrade"
|
||||
: `dnf -y install${target}`;
|
||||
case "yum":
|
||||
return action === "upgrade-all"
|
||||
? "yum -y update"
|
||||
: `yum -y install${target}`;
|
||||
case "pacman":
|
||||
return action === "upgrade-all"
|
||||
? "pacman -Syu --noconfirm"
|
||||
: `pacman -S --noconfirm${target}`;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export type PackageManager = "apt" | "dnf" | "yum" | "pacman" | null;
|
||||
|
||||
export interface PlatformInfo {
|
||||
hasSystemd: boolean;
|
||||
pkg: PackageManager;
|
||||
hasCertbot: boolean;
|
||||
hasAcmeSh: boolean;
|
||||
hasDocker: boolean;
|
||||
osPrettyName: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-interactive SSH shells don't source ~/.zprofile or ~/.bash_profile,
|
||||
* so PATH additions from installers like Homebrew or OrbStack are missing.
|
||||
* Extend PATH with their common install locations before probing.
|
||||
*/
|
||||
export const EXTRA_PATH_DIRS =
|
||||
'/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:"$HOME/.orbstack/bin"';
|
||||
|
||||
const DOCKER_PROBE =
|
||||
`command -v docker >/dev/null 2>&1` +
|
||||
` || [ -S /var/run/docker.sock ]` +
|
||||
` || [ -S "$HOME/.orbstack/run/docker.sock" ]`;
|
||||
|
||||
/**
|
||||
* Single probe that reports which tooling is available. Each line is
|
||||
* "key=value" so the parser is trivial and order-independent.
|
||||
*/
|
||||
export const PLATFORM_PROBE_COMMAND = [
|
||||
`PATH="${EXTRA_PATH_DIRS}:$PATH"`,
|
||||
"echo systemd=$(command -v systemctl >/dev/null 2>&1 && echo 1 || echo 0)",
|
||||
"echo apt=$(command -v apt-get >/dev/null 2>&1 && echo 1 || echo 0)",
|
||||
"echo dnf=$(command -v dnf >/dev/null 2>&1 && echo 1 || echo 0)",
|
||||
"echo yum=$(command -v yum >/dev/null 2>&1 && echo 1 || echo 0)",
|
||||
"echo pacman=$(command -v pacman >/dev/null 2>&1 && echo 1 || echo 0)",
|
||||
"echo certbot=$(command -v certbot >/dev/null 2>&1 && echo 1 || echo 0)",
|
||||
'echo acmesh=$( { command -v acme.sh >/dev/null 2>&1 || [ -x "$HOME/.acme.sh/acme.sh" ]; } && echo 1 || echo 0)',
|
||||
`echo docker=$(${DOCKER_PROBE} && echo 1 || echo 0)`,
|
||||
'echo os=$(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME")',
|
||||
].join("; ");
|
||||
|
||||
export function parsePlatformProbe(output: string): PlatformInfo {
|
||||
const map = new Map<string, string>();
|
||||
for (const line of output.split("\n")) {
|
||||
const idx = line.indexOf("=");
|
||||
if (idx === -1) continue;
|
||||
map.set(line.slice(0, idx).trim(), line.slice(idx + 1).trim());
|
||||
}
|
||||
const on = (k: string) => map.get(k) === "1";
|
||||
|
||||
// Prefer dnf over yum when both exist (dnf is the modern front-end).
|
||||
let pkg: PackageManager = null;
|
||||
if (on("apt")) pkg = "apt";
|
||||
else if (on("dnf")) pkg = "dnf";
|
||||
else if (on("yum")) pkg = "yum";
|
||||
else if (on("pacman")) pkg = "pacman";
|
||||
|
||||
const os = map.get("os");
|
||||
return {
|
||||
hasSystemd: on("systemd"),
|
||||
pkg,
|
||||
hasCertbot: on("certbot"),
|
||||
hasAcmeSh: on("acmesh"),
|
||||
hasDocker: on("docker"),
|
||||
osPrettyName: os && os.length > 0 ? os : null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function detectPlatform(client: Client): Promise<PlatformInfo> {
|
||||
const { stdout } = await execCommand(client, PLATFORM_PROBE_COMMAND, 15000);
|
||||
return parsePlatformProbe(stdout);
|
||||
}
|
||||
@@ -1,434 +0,0 @@
|
||||
class RequestQueue {
|
||||
private queues = new Map<number, Array<() => Promise<unknown>>>();
|
||||
private processing = new Set<number>();
|
||||
private requestTimeout = 60000;
|
||||
|
||||
async queueRequest<T>(hostId: number, request: () => Promise<T>): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const wrappedRequest = async () => {
|
||||
try {
|
||||
const result = await Promise.race<T>([
|
||||
request(),
|
||||
new Promise<never>((_, rej) =>
|
||||
setTimeout(
|
||||
() =>
|
||||
rej(
|
||||
new Error(
|
||||
`Request timeout after ${this.requestTimeout}ms for host ${hostId}`,
|
||||
),
|
||||
),
|
||||
this.requestTimeout,
|
||||
),
|
||||
),
|
||||
]);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
|
||||
const queue = this.queues.get(hostId) || [];
|
||||
queue.push(wrappedRequest);
|
||||
this.queues.set(hostId, queue);
|
||||
this.processQueue(hostId);
|
||||
});
|
||||
}
|
||||
|
||||
private async processQueue(hostId: number): Promise<void> {
|
||||
if (this.processing.has(hostId)) return;
|
||||
|
||||
this.processing.add(hostId);
|
||||
const queue = this.queues.get(hostId) || [];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const request = queue.shift();
|
||||
if (request) {
|
||||
try {
|
||||
await request();
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.processing.delete(hostId);
|
||||
const currentQueue = this.queues.get(hostId);
|
||||
if (currentQueue && currentQueue.length > 0) {
|
||||
this.processQueue(hostId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface CachedMetrics {
|
||||
data: unknown;
|
||||
timestamp: number;
|
||||
hostId: number;
|
||||
}
|
||||
|
||||
class MetricsCache {
|
||||
private cache = new Map<number, CachedMetrics>();
|
||||
private ttl = 30000;
|
||||
|
||||
get(hostId: number): unknown | null {
|
||||
const cached = this.cache.get(hostId);
|
||||
if (cached && Date.now() - cached.timestamp < this.ttl) {
|
||||
return cached.data;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
set(hostId: number, data: unknown): void {
|
||||
this.cache.set(hostId, {
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
hostId,
|
||||
});
|
||||
}
|
||||
|
||||
clear(hostId?: number): void {
|
||||
if (hostId) {
|
||||
this.cache.delete(hostId);
|
||||
} else {
|
||||
this.cache.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface AuthFailureRecord {
|
||||
count: number;
|
||||
lastFailure: number;
|
||||
reason: "TOTP" | "AUTH" | "TIMEOUT" | "HOST_KEY";
|
||||
permanent: boolean;
|
||||
}
|
||||
|
||||
class AuthFailureTracker {
|
||||
private failures = new Map<number, AuthFailureRecord>();
|
||||
private maxRetries = 3;
|
||||
private backoffBase = 5000;
|
||||
|
||||
recordFailure(
|
||||
hostId: number,
|
||||
reason: "TOTP" | "AUTH" | "TIMEOUT" | "HOST_KEY",
|
||||
permanent = false,
|
||||
): void {
|
||||
const existing = this.failures.get(hostId);
|
||||
if (existing) {
|
||||
existing.count++;
|
||||
existing.lastFailure = Date.now();
|
||||
existing.reason = reason;
|
||||
if (permanent) existing.permanent = true;
|
||||
} else {
|
||||
this.failures.set(hostId, {
|
||||
count: 1,
|
||||
lastFailure: Date.now(),
|
||||
reason,
|
||||
permanent,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
shouldSkip(hostId: number): boolean {
|
||||
const record = this.failures.get(hostId);
|
||||
if (!record) return false;
|
||||
|
||||
if (record.reason === "TOTP" || record.permanent) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (record.count >= this.maxRetries) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const backoffTime = this.backoffBase * Math.pow(2, record.count - 1);
|
||||
const timeSinceFailure = Date.now() - record.lastFailure;
|
||||
|
||||
return timeSinceFailure < backoffTime;
|
||||
}
|
||||
|
||||
getSkipReason(hostId: number): string | null {
|
||||
const record = this.failures.get(hostId);
|
||||
if (!record) return null;
|
||||
|
||||
if (record.reason === "TOTP") {
|
||||
return "TOTP authentication required (metrics unavailable)";
|
||||
}
|
||||
|
||||
if (record.reason === "HOST_KEY") {
|
||||
return "SSH host key changed (verification required)";
|
||||
}
|
||||
|
||||
if (record.permanent) {
|
||||
return "Authentication permanently failed";
|
||||
}
|
||||
|
||||
if (record.count >= this.maxRetries) {
|
||||
return `Too many authentication failures (${record.count} attempts)`;
|
||||
}
|
||||
|
||||
const backoffTime = this.backoffBase * Math.pow(2, record.count - 1);
|
||||
const timeSinceFailure = Date.now() - record.lastFailure;
|
||||
const remainingTime = Math.ceil((backoffTime - timeSinceFailure) / 1000);
|
||||
|
||||
if (timeSinceFailure < backoffTime) {
|
||||
return `Retry in ${remainingTime}s (attempt ${record.count}/${this.maxRetries})`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
resetHostKeyFailure(hostId: number): void {
|
||||
if (this.failures.get(hostId)?.reason === "HOST_KEY") this.reset(hostId);
|
||||
}
|
||||
|
||||
reset(hostId: number): void {
|
||||
this.failures.delete(hostId);
|
||||
}
|
||||
|
||||
cleanup(): void {
|
||||
const maxAge = 60 * 60 * 1000;
|
||||
const now = Date.now();
|
||||
|
||||
for (const [hostId, record] of this.failures.entries()) {
|
||||
if (!record.permanent && now - record.lastFailure > maxAge) {
|
||||
this.failures.delete(hostId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class PollingBackoff {
|
||||
private failures = new Map<number, { count: number; nextRetry: number }>();
|
||||
private baseDelay = 30000;
|
||||
private maxDelay = 600000;
|
||||
private maxRetries = 5;
|
||||
|
||||
recordFailure(hostId: number): void {
|
||||
const existing = this.failures.get(hostId) || { count: 0, nextRetry: 0 };
|
||||
const delay = Math.min(
|
||||
this.baseDelay * Math.pow(2, existing.count),
|
||||
this.maxDelay,
|
||||
);
|
||||
this.failures.set(hostId, {
|
||||
count: existing.count + 1,
|
||||
nextRetry: Date.now() + delay,
|
||||
});
|
||||
}
|
||||
|
||||
shouldSkip(hostId: number): boolean {
|
||||
const backoff = this.failures.get(hostId);
|
||||
if (!backoff) return false;
|
||||
|
||||
if (backoff.count >= this.maxRetries) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return Date.now() < backoff.nextRetry;
|
||||
}
|
||||
|
||||
getBackoffInfo(hostId: number): string | null {
|
||||
const backoff = this.failures.get(hostId);
|
||||
if (!backoff) return null;
|
||||
|
||||
if (backoff.count >= this.maxRetries) {
|
||||
return `Max retries exceeded (${backoff.count} failures) - polling suspended`;
|
||||
}
|
||||
|
||||
const remainingMs = backoff.nextRetry - Date.now();
|
||||
if (remainingMs > 0) {
|
||||
const remainingSec = Math.ceil(remainingMs / 1000);
|
||||
return `Retry in ${remainingSec}s (attempt ${backoff.count}/${this.maxRetries})`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
reset(hostId: number): void {
|
||||
this.failures.delete(hostId);
|
||||
}
|
||||
|
||||
cleanup(): void {
|
||||
const maxAge = 60 * 60 * 1000;
|
||||
const now = Date.now();
|
||||
|
||||
for (const [hostId, backoff] of this.failures.entries()) {
|
||||
if (backoff.count < this.maxRetries && now - backoff.nextRetry > maxAge) {
|
||||
this.failures.delete(hostId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Limits how many async jobs run at once. Extra callers wait in FIFO order.
|
||||
* Used to stop host-metrics status/metrics polls from stampeding under load.
|
||||
*/
|
||||
export class ConcurrentLimiter {
|
||||
private active = 0;
|
||||
private readonly waiters: Array<() => void> = [];
|
||||
|
||||
constructor(private maxConcurrent: number) {
|
||||
if (maxConcurrent < 1) {
|
||||
throw new Error("maxConcurrent must be >= 1");
|
||||
}
|
||||
}
|
||||
|
||||
get activeCount(): number {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
get pendingCount(): number {
|
||||
return this.waiters.length;
|
||||
}
|
||||
|
||||
get limit(): number {
|
||||
return this.maxConcurrent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Changes the ceiling at runtime.
|
||||
*
|
||||
* Raising it wakes the waiters the new headroom allows, so a queue that built
|
||||
* up under the old limit drains at once instead of one job at a time as
|
||||
* running work finishes. Lowering it never interrupts work already running;
|
||||
* the new limit simply applies from the next release onward.
|
||||
*/
|
||||
setLimit(maxConcurrent: number): void {
|
||||
if (maxConcurrent < 1) {
|
||||
throw new Error("maxConcurrent must be >= 1");
|
||||
}
|
||||
this.maxConcurrent = maxConcurrent;
|
||||
this.releaseWaiters();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wakes as many queued callers as there is now room for.
|
||||
*
|
||||
* `woken` counts callers that have been resumed but have not yet reached the
|
||||
* `active += 1` on the far side of their await. Without it the occupancy
|
||||
* looks lower than it really is for a microtask, and the loop would release
|
||||
* past the ceiling.
|
||||
*/
|
||||
private releaseWaiters(): void {
|
||||
while (
|
||||
this.active + this.woken < this.maxConcurrent &&
|
||||
this.waiters.length > 0
|
||||
) {
|
||||
this.woken += 1;
|
||||
this.waiters.shift()!();
|
||||
}
|
||||
}
|
||||
|
||||
private woken = 0;
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
// Queue behind anything already woken, so a caller arriving mid-handoff
|
||||
// cannot jump the queue and push occupancy past the ceiling.
|
||||
if (this.active + this.woken >= this.maxConcurrent) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.waiters.push(resolve);
|
||||
});
|
||||
// Resumed by a slot handoff; that reservation is now consumed.
|
||||
this.woken = Math.max(0, this.woken - 1);
|
||||
}
|
||||
|
||||
this.active += 1;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.active -= 1;
|
||||
this.releaseWaiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* How many metrics polls may run at once for a given number of polled hosts.
|
||||
*
|
||||
* A metrics poll is an SSH exec, so this cannot simply be unbounded — but the
|
||||
* old fixed ceiling of 5 meant a sweep of 500 hosts took ~40s against a 30s
|
||||
* interval, so polling fell permanently behind and a host could wait over a
|
||||
* minute for a "30 second" metric. Scaling with the fleet keeps a sweep inside
|
||||
* its interval; the cap keeps file descriptors and CPU bounded.
|
||||
*/
|
||||
export const METRICS_CONCURRENCY_ENV = "METRICS_POLL_CONCURRENCY";
|
||||
const MIN_METRICS_CONCURRENCY = 5;
|
||||
const MAX_METRICS_CONCURRENCY = 50;
|
||||
/** Aim to spend about a twentieth of the interval per sweep wave. */
|
||||
const HOSTS_PER_WORKER = 20;
|
||||
|
||||
export function metricsConcurrencyFor(
|
||||
hostCount: number,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): number {
|
||||
const override = Number(env[METRICS_CONCURRENCY_ENV]);
|
||||
if (Number.isFinite(override) && override >= 1) {
|
||||
return Math.min(Math.floor(override), MAX_METRICS_CONCURRENCY);
|
||||
}
|
||||
|
||||
const scaled = Math.ceil(Math.max(0, hostCount) / HOSTS_PER_WORKER);
|
||||
return Math.min(
|
||||
Math.max(scaled, MIN_METRICS_CONCURRENCY),
|
||||
MAX_METRICS_CONCURRENCY,
|
||||
);
|
||||
}
|
||||
|
||||
/** Short-lived host snapshots for polling — avoids decrypting host rows every tick. */
|
||||
export class HostPollCache<THost extends { id: number } = { id: number }> {
|
||||
private cache = new Map<
|
||||
number,
|
||||
{ host: THost; userId: string; expiresAt: number }
|
||||
>();
|
||||
|
||||
constructor(private readonly ttlMs = 30_000) {}
|
||||
|
||||
get(hostId: number, userId: string): THost | null {
|
||||
const entry = this.cache.get(hostId);
|
||||
if (!entry) return null;
|
||||
if (entry.userId !== userId || Date.now() >= entry.expiresAt) {
|
||||
this.cache.delete(hostId);
|
||||
return null;
|
||||
}
|
||||
return entry.host;
|
||||
}
|
||||
|
||||
set(hostId: number, userId: string, host: THost): void {
|
||||
this.cache.set(hostId, {
|
||||
host,
|
||||
userId,
|
||||
expiresAt: Date.now() + this.ttlMs,
|
||||
});
|
||||
}
|
||||
|
||||
invalidate(hostId?: number): void {
|
||||
if (hostId === undefined) {
|
||||
this.cache.clear();
|
||||
return;
|
||||
}
|
||||
this.cache.delete(hostId);
|
||||
}
|
||||
}
|
||||
|
||||
/** TCP status checks are cheap relative to SSH metrics collection. */
|
||||
export const statusPollLimiter = new ConcurrentLimiter(20);
|
||||
/** SSH metrics execs are expensive; keep concurrency tight. */
|
||||
export const metricsPollLimiter = new ConcurrentLimiter(5);
|
||||
/** Viewer registration is bursty; admit only two first samples at a time. */
|
||||
export const initialMetricsPollLimiter = new ConcurrentLimiter(2);
|
||||
|
||||
export function canStartInitialMetrics(
|
||||
status: HostStatus | undefined,
|
||||
hasViewers: boolean,
|
||||
statusCheckEnabled = true,
|
||||
): boolean {
|
||||
return (
|
||||
hasViewers &&
|
||||
(!statusCheckEnabled || status === "reachable" || status === "online")
|
||||
);
|
||||
}
|
||||
export const hostPollCache = new HostPollCache(30_000);
|
||||
|
||||
export const requestQueue = new RequestQueue();
|
||||
export const metricsCache = new MetricsCache();
|
||||
export const authFailureTracker = new AuthFailureTracker();
|
||||
export const pollingBackoff = new PollingBackoff();
|
||||
import type { HostStatus } from "./host-status.js";
|
||||
@@ -1,127 +0,0 @@
|
||||
/**
|
||||
* Strict allowlist validators for every dynamic value that reaches a shell
|
||||
* command. Managers MUST validate inputs through these before interpolation;
|
||||
* never pass raw user text to the shell.
|
||||
*/
|
||||
|
||||
const SYSTEMD_UNIT_RE =
|
||||
/^[A-Za-z0-9@._:\\-]+\.(service|socket|timer|target|path|mount)$/;
|
||||
const PACKAGE_RE = /^[A-Za-z0-9][A-Za-z0-9.+_-]*$/;
|
||||
const USERNAME_RE = /^[a-z_][a-z0-9_-]*\$?$/;
|
||||
const GROUP_RE = /^[a-z_][a-z0-9_-]*$/;
|
||||
const DOMAIN_RE =
|
||||
/^(\*\.)?(?!-)[A-Za-z0-9-]{1,63}(?<!-)(\.(?!-)[A-Za-z0-9-]{1,63}(?<!-))*$/;
|
||||
const PROVIDER_RE = /^[a-z0-9_]+$/;
|
||||
|
||||
export function isValidSystemdUnit(unit: unknown): unit is string {
|
||||
return (
|
||||
typeof unit === "string" && unit.length <= 256 && SYSTEMD_UNIT_RE.test(unit)
|
||||
);
|
||||
}
|
||||
|
||||
export function isValidPid(pid: unknown): pid is number {
|
||||
const n = typeof pid === "string" ? Number(pid) : pid;
|
||||
return typeof n === "number" && Number.isInteger(n) && n > 0 && n < 2 ** 31;
|
||||
}
|
||||
|
||||
export function isValidPort(port: unknown): port is number {
|
||||
const n = typeof port === "string" ? Number(port) : port;
|
||||
return typeof n === "number" && Number.isInteger(n) && n >= 1 && n <= 65535;
|
||||
}
|
||||
|
||||
export function isValidPackageName(pkg: unknown): pkg is string {
|
||||
return typeof pkg === "string" && pkg.length <= 128 && PACKAGE_RE.test(pkg);
|
||||
}
|
||||
|
||||
export function isValidUsername(name: unknown): name is string {
|
||||
return (
|
||||
typeof name === "string" && name.length <= 32 && USERNAME_RE.test(name)
|
||||
);
|
||||
}
|
||||
|
||||
export function isValidGroupName(name: unknown): name is string {
|
||||
return typeof name === "string" && name.length <= 32 && GROUP_RE.test(name);
|
||||
}
|
||||
|
||||
export function isValidDomain(domain: unknown): domain is string {
|
||||
return (
|
||||
typeof domain === "string" && domain.length <= 253 && DOMAIN_RE.test(domain)
|
||||
);
|
||||
}
|
||||
|
||||
export function isValidDnsProvider(provider: unknown): provider is string {
|
||||
return (
|
||||
typeof provider === "string" &&
|
||||
provider.length <= 64 &&
|
||||
PROVIDER_RE.test(provider)
|
||||
);
|
||||
}
|
||||
|
||||
export type Signal = "TERM" | "KILL" | "HUP" | "INT";
|
||||
const SIGNALS: Signal[] = ["TERM", "KILL", "HUP", "INT"];
|
||||
export function isValidSignal(sig: unknown): sig is Signal {
|
||||
return typeof sig === "string" && (SIGNALS as string[]).includes(sig);
|
||||
}
|
||||
|
||||
export type ServiceAction =
|
||||
"start" | "stop" | "restart" | "reload" | "enable" | "disable";
|
||||
const SERVICE_ACTIONS: ServiceAction[] = [
|
||||
"start",
|
||||
"stop",
|
||||
"restart",
|
||||
"reload",
|
||||
"enable",
|
||||
"disable",
|
||||
];
|
||||
export function isValidServiceAction(a: unknown): a is ServiceAction {
|
||||
return typeof a === "string" && (SERVICE_ACTIONS as string[]).includes(a);
|
||||
}
|
||||
|
||||
export type IpProtocol = "tcp" | "udp";
|
||||
export function isValidIpProtocol(p: unknown): p is IpProtocol {
|
||||
return p === "tcp" || p === "udp";
|
||||
}
|
||||
|
||||
export type FirewallTarget = "ACCEPT" | "DROP" | "REJECT";
|
||||
const FW_TARGETS: FirewallTarget[] = ["ACCEPT", "DROP", "REJECT"];
|
||||
export function isValidFirewallTarget(t: unknown): t is FirewallTarget {
|
||||
return typeof t === "string" && (FW_TARGETS as string[]).includes(t);
|
||||
}
|
||||
|
||||
const WG_IFACE_RE = /^[A-Za-z0-9_-]{1,15}$/;
|
||||
export function isValidWireGuardInterface(name: unknown): name is string {
|
||||
return typeof name === "string" && WG_IFACE_RE.test(name);
|
||||
}
|
||||
|
||||
export type WireGuardAction = "up" | "down";
|
||||
const WG_ACTIONS: WireGuardAction[] = ["up", "down"];
|
||||
export function isValidWireGuardAction(a: unknown): a is WireGuardAction {
|
||||
return typeof a === "string" && (WG_ACTIONS as string[]).includes(a);
|
||||
}
|
||||
|
||||
export type TailscaleAction = "up" | "down";
|
||||
const TAILSCALE_ACTIONS: TailscaleAction[] = ["up", "down"];
|
||||
export function isValidTailscaleAction(a: unknown): a is TailscaleAction {
|
||||
return typeof a === "string" && (TAILSCALE_ACTIONS as string[]).includes(a);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an absolute file path against an allowlist of permitted prefixes and
|
||||
* reject traversal. Used by the log viewer (e.g. only under /var/log).
|
||||
*/
|
||||
export function isAllowedPath(
|
||||
path: unknown,
|
||||
allowedPrefixes: string[],
|
||||
): path is string {
|
||||
if (typeof path !== "string" || path.length === 0 || path.length > 4096) {
|
||||
return false;
|
||||
}
|
||||
if (!path.startsWith("/")) return false;
|
||||
if (path.includes("\0")) return false;
|
||||
if (path.split("/").some((seg) => seg === "..")) return false;
|
||||
return allowedPrefixes.some(
|
||||
(prefix) =>
|
||||
path === prefix ||
|
||||
path.startsWith(prefix.endsWith("/") ? prefix : `${prefix}/`),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,458 @@
|
||||
/**
|
||||
* The host list's status dot.
|
||||
*
|
||||
* Every host with status checks on gets a TCP connection to its port on a
|
||||
* timer. That alone says "reachable". A host becomes "online" when a login
|
||||
* worked: an open terminal session, or a plugin reporting one through
|
||||
* ctx.hosts.status.reportLogin. Probing never logs in: on RADIUS or Duo backed
|
||||
* devices a login fires a real 2FA push, every interval.
|
||||
*
|
||||
* Polling is demand driven. A user asking for statuses starts polling their
|
||||
* own hosts; nothing is probed for a user who never opened the app.
|
||||
*/
|
||||
|
||||
import { pluginEvents, TOPICS } from "../../plugins/events.js";
|
||||
import { hostSessionStatus } from "../host-session-status.js";
|
||||
import { sshLogger } from "../../utils/logger.js";
|
||||
import {
|
||||
createCurrentHostResolutionRepository,
|
||||
getCurrentSettingValue,
|
||||
} from "../../database/repositories/factory.js";
|
||||
import type { HostStatusTargetRow } from "../../database/repositories/host-resolution-repository.js";
|
||||
import {
|
||||
statusAfterAuthentication,
|
||||
statusAfterReachabilityCheck,
|
||||
type HostStatus,
|
||||
} from "./host-status.js";
|
||||
import { ConcurrentLimiter } from "./limiter.js";
|
||||
import { tcpPing, tcpPingThroughJumpHost } from "./tcp-ping.js";
|
||||
|
||||
export const GLOBAL_STATUS_INTERVAL_KEY = "global_status_check_interval";
|
||||
export const DEFAULT_STATUS_INTERVAL = 60;
|
||||
/** A status younger than this is fresh enough for check(). */
|
||||
const FRESH_MS = 30_000;
|
||||
|
||||
export interface HostStatusEntry {
|
||||
status: HostStatus;
|
||||
lastChecked: string;
|
||||
reason?: "host_key_changed";
|
||||
}
|
||||
|
||||
export interface HostStatusPayload {
|
||||
hostId: number;
|
||||
ownerUserId: string;
|
||||
status: HostStatus;
|
||||
previous: HostStatus | null;
|
||||
/** Anything but offline, the shape automations' trigger already reads. */
|
||||
online: boolean;
|
||||
reason?: "host_key_changed";
|
||||
}
|
||||
|
||||
export interface StatusTarget {
|
||||
id: number;
|
||||
userId: string;
|
||||
ip: string;
|
||||
port: number;
|
||||
connectionType: string;
|
||||
jumpHosts: Array<{ hostId: number }>;
|
||||
statusCheckEnabled: boolean;
|
||||
statusCheckInterval: number | null;
|
||||
}
|
||||
|
||||
type PortResolver = (
|
||||
hostId: number,
|
||||
) => number | undefined | Promise<number | undefined>;
|
||||
|
||||
export interface HostStatusDeps {
|
||||
loadTargets: (filter: {
|
||||
userId?: string;
|
||||
hostIds?: number[];
|
||||
}) => Promise<StatusTarget[]>;
|
||||
ping: (host: string, port: number) => Promise<boolean>;
|
||||
pingThroughJumpHosts: (
|
||||
target: StatusTarget,
|
||||
port: number,
|
||||
) => Promise<boolean>;
|
||||
globalInterval: () => number;
|
||||
emit: (payload: HostStatusPayload) => void;
|
||||
}
|
||||
|
||||
function parseJumpHosts(raw: string | null): Array<{ hostId: number }> {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter(
|
||||
(hop): hop is { hostId: number } =>
|
||||
!!hop && Number.isInteger((hop as { hostId?: unknown }).hostId),
|
||||
)
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function toStatusTarget(row: HostStatusTargetRow): StatusTarget {
|
||||
return {
|
||||
id: row.id,
|
||||
userId: row.userId,
|
||||
ip: row.ip,
|
||||
port: row.port,
|
||||
connectionType: row.connectionType || "ssh",
|
||||
jumpHosts: parseJumpHosts(row.jumpHosts),
|
||||
statusCheckEnabled: row.statusCheckEnabled !== false,
|
||||
statusCheckInterval: row.statusCheckInterval ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
const defaultDeps: HostStatusDeps = {
|
||||
loadTargets: async (filter) =>
|
||||
(
|
||||
await createCurrentHostResolutionRepository().listStatusTargets(filter)
|
||||
).map(toStatusTarget),
|
||||
ping: (host, port) => tcpPing(host, port, 5000),
|
||||
pingThroughJumpHosts: async (target, port) => {
|
||||
const { createJumpHostChain } = await import("../jump-host-chain.js");
|
||||
const client = await createJumpHostChain(
|
||||
target.jumpHosts,
|
||||
target.userId,
|
||||
).catch(() => null);
|
||||
return client
|
||||
? tcpPingThroughJumpHost(client, target.ip, port, 5000)
|
||||
: false;
|
||||
},
|
||||
globalInterval: () => {
|
||||
const value = Number(getCurrentSettingValue(GLOBAL_STATUS_INTERVAL_KEY));
|
||||
return Number.isInteger(value) && value >= 5
|
||||
? value
|
||||
: DEFAULT_STATUS_INTERVAL;
|
||||
},
|
||||
emit: (payload) => pluginEvents.emit(TOPICS.hostStatus, payload),
|
||||
};
|
||||
|
||||
interface Polled {
|
||||
target: StatusTarget;
|
||||
timer?: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
export class HostStatusService {
|
||||
private readonly polled = new Map<number, Polled>();
|
||||
private readonly store = new Map<number, HostStatusEntry>();
|
||||
private readonly owners = new Map<number, string>();
|
||||
private readonly inFlight = new Map<number, Promise<void>>();
|
||||
private readonly sessionOnline = new Set<number>();
|
||||
private readonly loggedIn = new Set<number>();
|
||||
private readonly startedUsers = new Set<string>();
|
||||
private readonly portResolvers = new Map<string, Set<PortResolver>>();
|
||||
private readonly limiter = new ConcurrentLimiter(20);
|
||||
private unsubscribers: Array<() => void> = [];
|
||||
|
||||
constructor(private readonly deps: HostStatusDeps = defaultDeps) {}
|
||||
|
||||
/** Subscribes to the core events the status depends on. */
|
||||
start(): void {
|
||||
if (this.unsubscribers.length > 0) return;
|
||||
this.unsubscribers = [
|
||||
hostSessionStatus.subscribe((hostId, online) =>
|
||||
this.setSessionOnline(hostId, online),
|
||||
),
|
||||
pluginEvents.on(TOPICS.hostUpdated, (payload) => {
|
||||
const { hostId } = payload as { hostId?: number };
|
||||
if (hostId) void this.reload(hostId);
|
||||
}),
|
||||
pluginEvents.on(TOPICS.hostDeleted, (payload) => {
|
||||
const { hostId } = payload as { hostId?: number };
|
||||
if (hostId) this.forget(hostId);
|
||||
}),
|
||||
pluginEvents.on(TOPICS.hostKeyUpdated, (payload) => {
|
||||
const { hostId } = payload as { hostId?: number };
|
||||
if (hostId) this.clearHostKeyReason(hostId);
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
for (const unsubscribe of this.unsubscribers) unsubscribe();
|
||||
this.unsubscribers = [];
|
||||
for (const hostId of [...this.polled.keys()]) this.stopPolling(hostId);
|
||||
this.startedUsers.clear();
|
||||
}
|
||||
|
||||
get(hostId: number): HostStatusEntry | null {
|
||||
return this.store.get(hostId) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Statuses for a user's request. With hostIds, polling is brought in line
|
||||
* with exactly those of their own hosts (the desktop app sends only the
|
||||
* hosts that live on this backend). Without, their own hosts are started
|
||||
* once.
|
||||
*/
|
||||
async statusesFor(
|
||||
userId: string,
|
||||
requestedHostIds: Set<number> | null,
|
||||
): Promise<Map<number, HostStatusEntry>> {
|
||||
if (requestedHostIds !== null) {
|
||||
await this.reconcile(userId, requestedHostIds);
|
||||
} else if (!this.startedUsers.has(userId)) {
|
||||
await this.startUser(userId);
|
||||
}
|
||||
return this.store;
|
||||
}
|
||||
|
||||
/** Drops every timer for a user's hosts and starts them again. */
|
||||
async refresh(userId: string): Promise<void> {
|
||||
const targets = await this.deps.loadTargets({ userId });
|
||||
for (const target of targets) this.stopPolling(target.id);
|
||||
for (const target of targets) this.startPolling(target);
|
||||
this.startedUsers.add(userId);
|
||||
}
|
||||
|
||||
/** Re-times every polled host, after the global interval changed. */
|
||||
retimeAll(): void {
|
||||
for (const { target } of [...this.polled.values()]) {
|
||||
this.startPolling(target, false);
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks now unless the last result is fresh. */
|
||||
async check(hostId: number): Promise<HostStatusEntry | null> {
|
||||
const current = this.store.get(hostId);
|
||||
if (current && Date.now() - Date.parse(current.lastChecked) < FRESH_MS) {
|
||||
return current;
|
||||
}
|
||||
const target =
|
||||
this.polled.get(hostId)?.target ??
|
||||
(await this.deps.loadTargets({ hostIds: [hostId] }))[0];
|
||||
if (!target || !target.statusCheckEnabled) return current ?? null;
|
||||
await this.probe(target);
|
||||
return this.store.get(hostId) ?? null;
|
||||
}
|
||||
|
||||
reportLogin(
|
||||
hostId: number,
|
||||
outcome: { ok: boolean; hostKeyChanged?: boolean },
|
||||
): void {
|
||||
if (outcome.ok) {
|
||||
this.loggedIn.add(hostId);
|
||||
this.set(hostId, { status: statusAfterAuthentication(true) });
|
||||
return;
|
||||
}
|
||||
this.loggedIn.delete(hostId);
|
||||
this.set(hostId, {
|
||||
status: this.sessionOnline.has(hostId)
|
||||
? "online"
|
||||
: statusAfterAuthentication(false, this.store.get(hostId)?.status),
|
||||
...(outcome.hostKeyChanged ? { reason: "host_key_changed" } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
registerPort(connectionType: string, resolve: PortResolver): () => void {
|
||||
let set = this.portResolvers.get(connectionType);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
this.portResolvers.set(connectionType, set);
|
||||
}
|
||||
set.add(resolve);
|
||||
return () => {
|
||||
set!.delete(resolve);
|
||||
if (set!.size === 0) this.portResolvers.delete(connectionType);
|
||||
};
|
||||
}
|
||||
|
||||
private async startUser(userId: string): Promise<void> {
|
||||
this.startedUsers.add(userId);
|
||||
const targets = await this.deps.loadTargets({ userId });
|
||||
for (const target of targets) {
|
||||
if (!this.polled.has(target.id)) this.startPolling(target);
|
||||
}
|
||||
}
|
||||
|
||||
private async reconcile(userId: string, allowed: Set<number>): Promise<void> {
|
||||
const targets = await this.deps.loadTargets({ userId });
|
||||
for (const target of targets) {
|
||||
if (!allowed.has(target.id)) {
|
||||
this.stopPolling(target.id);
|
||||
this.store.delete(target.id);
|
||||
continue;
|
||||
}
|
||||
if (!this.polled.has(target.id)) this.startPolling(target);
|
||||
}
|
||||
}
|
||||
|
||||
private async reload(hostId: number): Promise<void> {
|
||||
try {
|
||||
const [target] = await this.deps.loadTargets({ hostIds: [hostId] });
|
||||
if (!target) {
|
||||
this.forget(hostId);
|
||||
return;
|
||||
}
|
||||
if (this.polled.has(hostId) || this.startedUsers.has(target.userId)) {
|
||||
this.startPolling(target);
|
||||
}
|
||||
} catch (error) {
|
||||
sshLogger.warn("Could not reload host for status checks", {
|
||||
operation: "host_status_reload",
|
||||
hostId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private forget(hostId: number): void {
|
||||
this.stopPolling(hostId);
|
||||
this.store.delete(hostId);
|
||||
this.owners.delete(hostId);
|
||||
this.loggedIn.delete(hostId);
|
||||
}
|
||||
|
||||
private clearHostKeyReason(hostId: number): void {
|
||||
const current = this.store.get(hostId);
|
||||
if (current?.reason !== "host_key_changed") return;
|
||||
this.store.set(hostId, {
|
||||
status: current.status,
|
||||
lastChecked: current.lastChecked,
|
||||
});
|
||||
}
|
||||
|
||||
private setSessionOnline(hostId: number, online: boolean): void {
|
||||
if (online) {
|
||||
this.sessionOnline.add(hostId);
|
||||
if (this.polled.has(hostId)) this.set(hostId, { status: "online" });
|
||||
return;
|
||||
}
|
||||
this.sessionOnline.delete(hostId);
|
||||
if (
|
||||
!this.loggedIn.has(hostId) &&
|
||||
this.store.get(hostId)?.status === "online"
|
||||
) {
|
||||
this.set(hostId, { status: "reachable" });
|
||||
}
|
||||
}
|
||||
|
||||
private intervalMs(target: StatusTarget): number {
|
||||
const seconds = target.statusCheckInterval ?? this.deps.globalInterval();
|
||||
const intervalMs = Math.max(5, seconds) * 1000;
|
||||
// Spread hosts so a fleet does not fire on the same second.
|
||||
const spread = Math.min(intervalMs * 0.2, 15_000);
|
||||
return (
|
||||
intervalMs + ((target.id * 1103515245) % Math.max(1, Math.floor(spread)))
|
||||
);
|
||||
}
|
||||
|
||||
private startPolling(target: StatusTarget, probeNow = true): void {
|
||||
this.stopPolling(target.id);
|
||||
this.owners.set(target.id, target.userId);
|
||||
if (!target.statusCheckEnabled) {
|
||||
this.store.delete(target.id);
|
||||
return;
|
||||
}
|
||||
const polled: Polled = { target };
|
||||
this.polled.set(target.id, polled);
|
||||
if (probeNow) void this.probe(target);
|
||||
polled.timer = setInterval(() => {
|
||||
const latest = this.polled.get(target.id);
|
||||
if (latest) void this.probe(latest.target);
|
||||
}, this.intervalMs(target));
|
||||
polled.timer.unref?.();
|
||||
}
|
||||
|
||||
private stopPolling(hostId: number): void {
|
||||
const polled = this.polled.get(hostId);
|
||||
if (polled?.timer) clearInterval(polled.timer);
|
||||
this.polled.delete(hostId);
|
||||
}
|
||||
|
||||
private probe(target: StatusTarget): Promise<void> {
|
||||
const running = this.inFlight.get(target.id);
|
||||
if (running) return running;
|
||||
const job = this.limiter
|
||||
.run(() => this.probeNow(target))
|
||||
.catch((error) => {
|
||||
sshLogger.error("Status check failed", error, {
|
||||
operation: "host_status_probe",
|
||||
hostId: target.id,
|
||||
});
|
||||
})
|
||||
.finally(() => this.inFlight.delete(target.id));
|
||||
this.inFlight.set(target.id, job);
|
||||
return job;
|
||||
}
|
||||
|
||||
private async portFor(target: StatusTarget): Promise<number> {
|
||||
if (target.connectionType !== "ssh") {
|
||||
for (const resolve of this.portResolvers.get(target.connectionType) ??
|
||||
[]) {
|
||||
try {
|
||||
const port = await resolve(target.id);
|
||||
if (Number.isInteger(port) && port! > 0) return port!;
|
||||
} catch {
|
||||
// fall through to the next resolver
|
||||
}
|
||||
}
|
||||
}
|
||||
return target.port;
|
||||
}
|
||||
|
||||
private async probeNow(target: StatusTarget): Promise<void> {
|
||||
this.owners.set(target.id, target.userId);
|
||||
let reachable = false;
|
||||
try {
|
||||
const port = await this.portFor(target);
|
||||
reachable =
|
||||
target.jumpHosts.length > 0
|
||||
? await this.deps.pingThroughJumpHosts(target, port)
|
||||
: await this.deps.ping(target.ip, port);
|
||||
} catch {
|
||||
reachable = false;
|
||||
}
|
||||
|
||||
const current = this.store.get(target.id);
|
||||
this.set(target.id, {
|
||||
status:
|
||||
reachable && this.sessionOnline.has(target.id)
|
||||
? "online"
|
||||
: statusAfterReachabilityCheck(reachable, current?.status),
|
||||
...(reachable && current?.reason === "host_key_changed"
|
||||
? { reason: "host_key_changed" as const }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
private set(hostId: number, entry: Omit<HostStatusEntry, "lastChecked">) {
|
||||
const previous = this.store.get(hostId)?.status ?? null;
|
||||
const next: HostStatusEntry = {
|
||||
...entry,
|
||||
lastChecked: new Date().toISOString(),
|
||||
};
|
||||
this.store.set(hostId, next);
|
||||
if (previous === next.status) return;
|
||||
void this.emitChange(hostId, next, previous);
|
||||
}
|
||||
|
||||
private async emitChange(
|
||||
hostId: number,
|
||||
entry: HostStatusEntry,
|
||||
previous: HostStatus | null,
|
||||
): Promise<void> {
|
||||
let ownerUserId = this.owners.get(hostId);
|
||||
if (!ownerUserId) {
|
||||
const [target] = await this.deps
|
||||
.loadTargets({ hostIds: [hostId] })
|
||||
.catch(() => []);
|
||||
if (!target) return;
|
||||
ownerUserId = target.userId;
|
||||
this.owners.set(hostId, ownerUserId);
|
||||
}
|
||||
this.deps.emit({
|
||||
hostId,
|
||||
ownerUserId,
|
||||
status: entry.status,
|
||||
previous,
|
||||
online: entry.status !== "offline",
|
||||
...(entry.reason ? { reason: entry.reason } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const hostStatusService = new HostStatusService();
|
||||
File renamed without changes.
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* Limits how many async jobs run at once. Extra callers wait in FIFO order.
|
||||
*/
|
||||
export class ConcurrentLimiter {
|
||||
private active = 0;
|
||||
private readonly waiters: Array<() => void> = [];
|
||||
|
||||
constructor(private maxConcurrent: number) {
|
||||
if (maxConcurrent < 1) {
|
||||
throw new Error("maxConcurrent must be >= 1");
|
||||
}
|
||||
}
|
||||
|
||||
get activeCount(): number {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
get pendingCount(): number {
|
||||
return this.waiters.length;
|
||||
}
|
||||
|
||||
get limit(): number {
|
||||
return this.maxConcurrent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Changes the ceiling at runtime.
|
||||
*
|
||||
* Raising it wakes the waiters the new headroom allows, so a queue that built
|
||||
* up under the old limit drains at once instead of one job at a time as
|
||||
* running work finishes. Lowering it never interrupts work already running;
|
||||
* the new limit simply applies from the next release onward.
|
||||
*/
|
||||
setLimit(maxConcurrent: number): void {
|
||||
if (maxConcurrent < 1) {
|
||||
throw new Error("maxConcurrent must be >= 1");
|
||||
}
|
||||
this.maxConcurrent = maxConcurrent;
|
||||
this.releaseWaiters();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wakes as many queued callers as there is now room for.
|
||||
*
|
||||
* `woken` counts callers that have been resumed but have not yet reached the
|
||||
* `active += 1` on the far side of their await. Without it the occupancy
|
||||
* looks lower than it really is for a microtask, and the loop would release
|
||||
* past the ceiling.
|
||||
*/
|
||||
private releaseWaiters(): void {
|
||||
while (
|
||||
this.active + this.woken < this.maxConcurrent &&
|
||||
this.waiters.length > 0
|
||||
) {
|
||||
this.woken += 1;
|
||||
this.waiters.shift()!();
|
||||
}
|
||||
}
|
||||
|
||||
private woken = 0;
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
// Queue behind anything already woken, so a caller arriving mid-handoff
|
||||
// cannot jump the queue and push occupancy past the ceiling.
|
||||
if (this.active + this.woken >= this.maxConcurrent) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.waiters.push(resolve);
|
||||
});
|
||||
// Resumed by a slot handoff; that reservation is now consumed.
|
||||
this.woken = Math.max(0, this.woken - 1);
|
||||
}
|
||||
|
||||
this.active += 1;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.active -= 1;
|
||||
this.releaseWaiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import net from "net";
|
||||
import type { Client } from "ssh2";
|
||||
|
||||
/**
|
||||
* Opens a TCP connection and closes it again. An SSH server gets a polite
|
||||
* banner back so it does not log a failed handshake.
|
||||
*/
|
||||
export function tcpPing(
|
||||
host: string,
|
||||
port: number,
|
||||
timeoutMs = 5000,
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const socket = new net.Socket();
|
||||
let settled = false;
|
||||
|
||||
const finish = (result: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
try {
|
||||
socket.destroy();
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
};
|
||||
|
||||
socket.setTimeout(timeoutMs);
|
||||
|
||||
socket.once("connect", () => {
|
||||
const dataTimeout = setTimeout(() => {
|
||||
cleanup();
|
||||
finish(true);
|
||||
}, 2000);
|
||||
|
||||
socket.once("data", (data) => {
|
||||
clearTimeout(dataTimeout);
|
||||
if (data.toString("utf8").startsWith("SSH-")) {
|
||||
try {
|
||||
socket.end("SSH-2.0-TermixHealthCheck\r\n");
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
setTimeout(cleanup, 200);
|
||||
} else {
|
||||
cleanup();
|
||||
}
|
||||
finish(true);
|
||||
});
|
||||
});
|
||||
|
||||
socket.once("timeout", () => {
|
||||
cleanup();
|
||||
finish(false);
|
||||
});
|
||||
socket.once("error", () => {
|
||||
cleanup();
|
||||
finish(false);
|
||||
});
|
||||
socket.connect(port, host);
|
||||
});
|
||||
}
|
||||
|
||||
/** The same check from the far end of a jump host chain. Ends the chain. */
|
||||
export function tcpPingThroughJumpHost(
|
||||
jumpClient: Pick<Client, "forwardOut" | "end">,
|
||||
host: string,
|
||||
port: number,
|
||||
timeoutMs = 5000,
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
|
||||
const finish = (result: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
jumpClient.end();
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
const timeout = setTimeout(() => finish(false), timeoutMs);
|
||||
|
||||
jumpClient.forwardOut("127.0.0.1", 0, host, port, (error, stream) => {
|
||||
stream?.destroy();
|
||||
finish(!error && !!stream);
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user