mirror of
https://github.com/Termix-SSH/Termix.git
synced 2026-10-29 22:55:24 +00:00
feat: ship host metrics as a first-party in-process plugin
This commit is contained in:
LukeGus committed
1 parent
a2479c19b0
commit
19b70430a1
145 files changed
+675
-349
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { authFailureTracker } from "./metrics/state.js";
|
||||
import { authFailureTracker } from "./metrics-shared/state.js";
|
||||
import type { WebSocket } from "ws";
|
||||
import { createCurrentHostResolutionRepository } from "../database/repositories/factory.js";
|
||||
import { sshLogger } from "../utils/logger.js";
|
||||
|
||||
File renamed without changes.
+1
-1
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export type ElevationErrorCode = "SUDO_REQUIRED" | "SUDO_FAILED" | "NOT_SUDOER";
|
||||
|
||||
File renamed without changes.
+1
-65
@@ -1,10 +1,4 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import { isValidPackageName } from "./validation.js";
|
||||
import { detectPlatform, type PackageManager } from "./platform.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
import type { PackageManager } from "./platform.js";
|
||||
|
||||
export interface UpgradablePackage {
|
||||
name: string;
|
||||
@@ -88,61 +82,3 @@ export function buildPackageActionCommand(
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerPackageRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/packages/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "packages_list", async (client) => {
|
||||
const platform = await detectPlatform(client);
|
||||
const cmd = buildListUpgradableCommand(platform.pkg);
|
||||
if (!cmd) return { pkg: platform.pkg, upgradable: [] };
|
||||
const { stdout } = await execCommand(client, cmd, 60000);
|
||||
return {
|
||||
pkg: platform.pkg,
|
||||
upgradable: parseUpgradable(platform.pkg, stdout),
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/packages/:id/action",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"packages_action",
|
||||
async (client, host, req) => {
|
||||
const { action, pkg: name } = req.body as {
|
||||
action?: PackageAction;
|
||||
pkg?: string;
|
||||
};
|
||||
if (
|
||||
action !== "upgrade-all" &&
|
||||
action !== "install" &&
|
||||
action !== "upgrade"
|
||||
) {
|
||||
throw new ManagerInputError("Invalid action");
|
||||
}
|
||||
if (action !== "upgrade-all" && !isValidPackageName(name)) {
|
||||
throw new ManagerInputError("Invalid package name");
|
||||
}
|
||||
const platform = await detectPlatform(client);
|
||||
const cmd = buildPackageActionCommand(platform.pkg, action, name);
|
||||
if (!cmd) throw new ManagerInputError("No supported package manager");
|
||||
// Package operations can be slow; allow up to 10 minutes.
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
timeoutMs: 600000,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: (result.stdout || result.stderr).slice(-8000),
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export type PackageManager = "apt" | "dnf" | "yum" | "pacman" | null;
|
||||
|
||||
File renamed without changes.
File renamed without changes.
@@ -1,373 +0,0 @@
|
||||
import { createCurrentAlertRepository } from "../../database/repositories/factory.js";
|
||||
import { statsLogger } from "../../utils/logger.js";
|
||||
import {
|
||||
sendNotification,
|
||||
type AlertPayload,
|
||||
type NotificationChannel,
|
||||
} from "../../utils/notification-sender.js";
|
||||
import { sendDiscord } from "../../utils/discord-sender.js";
|
||||
|
||||
type AlertTriggerType =
|
||||
| "host_offline"
|
||||
| "host_online"
|
||||
| "cpu_threshold"
|
||||
| "memory_threshold"
|
||||
| "disk_threshold"
|
||||
| "health_check_failure"
|
||||
| "health_check_recovery"
|
||||
| "user_login";
|
||||
|
||||
interface AlertRule {
|
||||
id: number;
|
||||
userId: string;
|
||||
hostId: number | null;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
triggerType: AlertTriggerType;
|
||||
thresholdValue: number | null;
|
||||
thresholdDurationSeconds: number | null;
|
||||
cooldownMinutes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set once the alert rules have been copied into automations. From then on the
|
||||
* automations engine owns evaluation, and this one stands down rather than
|
||||
* sending a second notification for every rule.
|
||||
*
|
||||
* The class is left in place so an install that has not migrated yet, or one
|
||||
* rolled back, still alerts exactly as before.
|
||||
*/
|
||||
let supersededByAutomations = false;
|
||||
|
||||
export function markAlertEngineSuperseded(): void {
|
||||
supersededByAutomations = true;
|
||||
}
|
||||
|
||||
export function isAlertEngineSuperseded(): boolean {
|
||||
return supersededByAutomations;
|
||||
}
|
||||
|
||||
export class AlertEngine {
|
||||
private static instance: AlertEngine;
|
||||
|
||||
// "${ruleId}:${hostId}" → lastFiredAt ms
|
||||
private cooldownMap = new Map<string, number>();
|
||||
// "${ruleId}:${hostId}" → breachStartTime ms
|
||||
private breachStartMap = new Map<string, number>();
|
||||
// hostId → last known status
|
||||
private lastStatusMap = new Map<number, "online" | "offline">();
|
||||
// "${hostId}:${checkId}" → last known ok state
|
||||
private healthCheckStateMap = new Map<string, boolean>();
|
||||
|
||||
static getInstance(): AlertEngine {
|
||||
if (!AlertEngine.instance) {
|
||||
AlertEngine.instance = new AlertEngine();
|
||||
}
|
||||
return AlertEngine.instance;
|
||||
}
|
||||
|
||||
async evaluateMetrics(
|
||||
hostId: number,
|
||||
metrics: {
|
||||
cpu?: { percent: number | null } | null;
|
||||
memory?: { percent: number | null } | null;
|
||||
disk?: { percent: number | null } | null;
|
||||
},
|
||||
): Promise<void> {
|
||||
if (supersededByAutomations) return;
|
||||
const rules = (await this.loadRulesForHost(hostId)).filter((r) =>
|
||||
["cpu_threshold", "memory_threshold", "disk_threshold"].includes(
|
||||
r.triggerType,
|
||||
),
|
||||
);
|
||||
|
||||
if (rules.length === 0) return;
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
for (const rule of rules) {
|
||||
let currentValue: number | null | undefined;
|
||||
if (rule.triggerType === "cpu_threshold")
|
||||
currentValue = metrics.cpu?.percent;
|
||||
else if (rule.triggerType === "memory_threshold")
|
||||
currentValue = metrics.memory?.percent;
|
||||
else if (rule.triggerType === "disk_threshold")
|
||||
currentValue = metrics.disk?.percent;
|
||||
|
||||
if (currentValue == null || rule.thresholdValue == null) continue;
|
||||
|
||||
const key = `${rule.id}:${hostId}`;
|
||||
const isBreaching = currentValue >= rule.thresholdValue;
|
||||
|
||||
if (isBreaching) {
|
||||
if (!this.breachStartMap.has(key)) {
|
||||
this.breachStartMap.set(key, now);
|
||||
}
|
||||
const breachStart = this.breachStartMap.get(key)!;
|
||||
const durationMs = (rule.thresholdDurationSeconds ?? 0) * 1000;
|
||||
if (
|
||||
now - breachStart >= durationMs &&
|
||||
!(await this.isCoolingDown(rule.id, hostId))
|
||||
) {
|
||||
const hostName = await this.getHostName(hostId);
|
||||
await this.fireAlert(rule, hostId, hostName, {
|
||||
value: currentValue,
|
||||
message: `${rule.triggerType.replace("_threshold", "").toUpperCase()} usage at ${currentValue.toFixed(1)}% (threshold: ${rule.thresholdValue}%)`,
|
||||
severity: currentValue >= 95 ? "critical" : "warning",
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.breachStartMap.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async evaluateStatus(hostId: number, isOnline: boolean): Promise<void> {
|
||||
if (supersededByAutomations) return;
|
||||
const currentStatus = isOnline ? "online" : "offline";
|
||||
const lastStatus = this.lastStatusMap.get(hostId);
|
||||
|
||||
if (lastStatus === currentStatus) return;
|
||||
this.lastStatusMap.set(hostId, currentStatus);
|
||||
|
||||
if (lastStatus === undefined) return;
|
||||
|
||||
const triggerType: AlertTriggerType = isOnline
|
||||
? "host_online"
|
||||
: "host_offline";
|
||||
const rules = (await this.loadRulesForHost(hostId)).filter(
|
||||
(r) => r.triggerType === triggerType,
|
||||
);
|
||||
|
||||
for (const rule of rules) {
|
||||
if (!(await this.isCoolingDown(rule.id, hostId))) {
|
||||
const hostName = await this.getHostName(hostId);
|
||||
await this.fireAlert(rule, hostId, hostName, {
|
||||
message: isOnline
|
||||
? `Host "${hostName}" is back online`
|
||||
: `Host "${hostName}" is offline`,
|
||||
severity: isOnline ? "info" : "critical",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async evaluateHealthCheck(
|
||||
hostId: number,
|
||||
userId: string,
|
||||
checkId: string,
|
||||
ok: boolean,
|
||||
detail?: string,
|
||||
): Promise<void> {
|
||||
if (supersededByAutomations) return;
|
||||
const stateKey = `${hostId}:${checkId}`;
|
||||
const lastOk = this.healthCheckStateMap.get(stateKey);
|
||||
this.healthCheckStateMap.set(stateKey, ok);
|
||||
|
||||
if (lastOk === undefined) return;
|
||||
|
||||
let triggerType: AlertTriggerType | null = null;
|
||||
if (!ok && lastOk) triggerType = "health_check_failure";
|
||||
else if (ok && !lastOk) triggerType = "health_check_recovery";
|
||||
|
||||
if (!triggerType) return;
|
||||
|
||||
const rules = (await this.loadRulesForHostUser(hostId, userId)).filter(
|
||||
(r) => r.triggerType === triggerType,
|
||||
);
|
||||
|
||||
for (const rule of rules) {
|
||||
if (!(await this.isCoolingDown(rule.id, hostId))) {
|
||||
const hostName = await this.getHostName(hostId);
|
||||
await this.fireAlert(rule, hostId, hostName, {
|
||||
message: ok
|
||||
? `Health check recovered on "${hostName}"${detail ? `: ${detail}` : ""}`
|
||||
: `Health check failed on "${hostName}"${detail ? `: ${detail}` : ""}`,
|
||||
severity: ok ? "info" : "warning",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async evaluateUserLogin(
|
||||
hostId: number,
|
||||
userId: string,
|
||||
sshUser: string,
|
||||
fromIp: string,
|
||||
): Promise<void> {
|
||||
if (supersededByAutomations) return;
|
||||
const rules = (await this.loadRulesForHostUser(hostId, userId)).filter(
|
||||
(r) => r.triggerType === "user_login",
|
||||
);
|
||||
|
||||
for (const rule of rules) {
|
||||
if (!(await this.isCoolingDown(rule.id, hostId))) {
|
||||
const hostName = await this.getHostName(hostId);
|
||||
await this.fireAlert(rule, hostId, hostName, {
|
||||
message: `User "${sshUser}" logged in to "${hostName}" from ${fromIp}`,
|
||||
severity: "info",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async fireAlert(
|
||||
rule: AlertRule,
|
||||
hostId: number,
|
||||
hostName: string,
|
||||
context: {
|
||||
value?: number;
|
||||
message: string;
|
||||
severity: "info" | "warning" | "critical";
|
||||
},
|
||||
): Promise<void> {
|
||||
this.markCooldown(rule.id, hostId);
|
||||
|
||||
const payload: AlertPayload = {
|
||||
hostName,
|
||||
hostId,
|
||||
triggerType: rule.triggerType,
|
||||
value: context.value,
|
||||
threshold: rule.thresholdValue ?? undefined,
|
||||
message: context.message,
|
||||
severity: context.severity,
|
||||
timestamp: new Date().toISOString(),
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
};
|
||||
|
||||
try {
|
||||
const repository = createCurrentAlertRepository();
|
||||
await repository.createFiring({
|
||||
userId: rule.userId,
|
||||
ruleId: rule.id,
|
||||
hostId,
|
||||
hostName,
|
||||
value: context.value ?? null,
|
||||
message: context.message,
|
||||
severity: context.severity,
|
||||
});
|
||||
|
||||
await repository.pruneFiringsOlderThan(rule.userId, 30);
|
||||
} catch (err) {
|
||||
statsLogger.warn("Failed to write alert firing", {
|
||||
operation: "alert_firing_insert_error",
|
||||
ruleId: rule.id,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
const channels = await this.loadChannelsForRule(rule.id);
|
||||
for (const channel of channels) {
|
||||
if (channel.type === "discord") {
|
||||
// parse config and send via Discord sender directly
|
||||
try {
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(channel.config) as Record<string, unknown>;
|
||||
} catch {
|
||||
statsLogger.warn("Failed to parse discord channel config", {
|
||||
operation: "discord_config_parse_error",
|
||||
channelId: channel.id,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
sendDiscord(parsed as any, payload).catch((err) => {
|
||||
statsLogger.warn("Failed to send discord notification", {
|
||||
operation: "discord_notification_error",
|
||||
channelId: channel.id,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
});
|
||||
} catch (err) {
|
||||
statsLogger.warn("Failed to send discord notification", {
|
||||
operation: "discord_notification_error",
|
||||
channelId: channel.id,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
sendNotification(channel, payload).catch((err) => {
|
||||
statsLogger.warn("Failed to send notification", {
|
||||
operation: "notification_delivery_error",
|
||||
channelId: channel.id,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async isCoolingDown(
|
||||
ruleId: number,
|
||||
hostId: number,
|
||||
): Promise<boolean> {
|
||||
const key = `${ruleId}:${hostId}`;
|
||||
const lastFired = this.cooldownMap.get(key);
|
||||
if (!lastFired) return false;
|
||||
const rule = await this.loadRuleById(ruleId);
|
||||
const cooldownMs = (rule?.cooldownMinutes ?? 15) * 60 * 1000;
|
||||
return Date.now() - lastFired < cooldownMs;
|
||||
}
|
||||
|
||||
private markCooldown(ruleId: number, hostId: number): void {
|
||||
this.cooldownMap.set(`${ruleId}:${hostId}`, Date.now());
|
||||
}
|
||||
|
||||
private async loadRulesForHost(hostId: number): Promise<AlertRule[]> {
|
||||
try {
|
||||
return (await createCurrentAlertRepository().listEnabledRulesForHost(
|
||||
hostId,
|
||||
)) as AlertRule[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async loadRulesForHostUser(
|
||||
hostId: number,
|
||||
userId: string,
|
||||
): Promise<AlertRule[]> {
|
||||
try {
|
||||
return (await createCurrentAlertRepository().listEnabledRulesForHostUser(
|
||||
hostId,
|
||||
userId,
|
||||
)) as AlertRule[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async loadRuleById(ruleId: number): Promise<AlertRule | null> {
|
||||
try {
|
||||
return (await createCurrentAlertRepository().findRuleById(
|
||||
ruleId,
|
||||
)) as AlertRule | null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async loadChannelsForRule(
|
||||
ruleId: number,
|
||||
): Promise<NotificationChannel[]> {
|
||||
try {
|
||||
return await createCurrentAlertRepository().listEnabledChannelsForRule(
|
||||
ruleId,
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async getHostName(hostId: number): Promise<string> {
|
||||
try {
|
||||
return (
|
||||
(await createCurrentAlertRepository().getHostDisplayName(hostId)) ??
|
||||
`Host #${hostId}`
|
||||
);
|
||||
} catch {
|
||||
return `Host #${hostId}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
/**
|
||||
* One-line hand-off from the metrics poller onto the ctx.events bus.
|
||||
*
|
||||
* The poller is already a very large module, so the hooks it calls live here
|
||||
* instead. Everything is fire-and-forget: a failure in a subscriber must never
|
||||
* disturb metric collection, and publishing onto the bus creates no static edge
|
||||
* to the automations layer (which reads repositories the metrics module also
|
||||
* pulls in).
|
||||
*
|
||||
* For the generic (non-metrics) internal event notifier, see
|
||||
* `hosts/automation-events.ts` -- that one is shared across features.
|
||||
*/
|
||||
|
||||
import { pluginEvents, TOPICS } from "../../plugins/events.js";
|
||||
import type { MetricsSnapshot } from "../../automations/conditions.js";
|
||||
|
||||
export interface HostMetricsPayload {
|
||||
hostId: number;
|
||||
ownerUserId: string;
|
||||
metrics: MetricsSnapshot;
|
||||
}
|
||||
|
||||
export interface HostStatusPayload {
|
||||
hostId: number;
|
||||
ownerUserId: string;
|
||||
online: boolean;
|
||||
}
|
||||
|
||||
export interface HostHealthCheckPayload {
|
||||
hostId: number;
|
||||
userId: string;
|
||||
checkId: string;
|
||||
ok: boolean;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export function notifyAutomationMetrics(
|
||||
hostId: number,
|
||||
ownerUserId: string,
|
||||
metrics: MetricsSnapshot,
|
||||
): void {
|
||||
if (!ownerUserId) return;
|
||||
pluginEvents.emit(TOPICS.hostMetrics, {
|
||||
hostId,
|
||||
ownerUserId,
|
||||
metrics,
|
||||
} satisfies HostMetricsPayload);
|
||||
}
|
||||
|
||||
export function notifyAutomationStatus(
|
||||
hostId: number,
|
||||
ownerUserId: string,
|
||||
online: boolean,
|
||||
): void {
|
||||
if (!ownerUserId) return;
|
||||
pluginEvents.emit(TOPICS.hostStatus, {
|
||||
hostId,
|
||||
ownerUserId,
|
||||
online,
|
||||
} satisfies HostStatusPayload);
|
||||
}
|
||||
|
||||
export function notifyAutomationHealthCheck(
|
||||
hostId: number,
|
||||
userId: string,
|
||||
checkId: string,
|
||||
ok: boolean,
|
||||
detail?: string,
|
||||
): void {
|
||||
if (!userId) return;
|
||||
pluginEvents.emit(TOPICS.hostHealthCheck, {
|
||||
hostId,
|
||||
userId,
|
||||
checkId,
|
||||
ok,
|
||||
detail,
|
||||
} satisfies HostHealthCheckPayload);
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { usesIssuedCertificate } from "../issued-certificate-auth.js";
|
||||
|
||||
export type StatsCapableHost = {
|
||||
connectionType?: string;
|
||||
authType?: string;
|
||||
};
|
||||
|
||||
export type TcpPingStatsConfig = {
|
||||
statusCheckEnabled: boolean;
|
||||
disableTcpPing?: boolean;
|
||||
};
|
||||
|
||||
export function supportsMetrics(host: StatsCapableHost): boolean {
|
||||
const connectionType = host.connectionType || "ssh";
|
||||
if (connectionType !== "ssh") return false;
|
||||
if (host.authType === "none" || usesIssuedCertificate(host.authType))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function isTcpPingEnabled(statsConfig: TcpPingStatsConfig): boolean {
|
||||
return statsConfig.statusCheckEnabled && !statsConfig.disableTcpPing;
|
||||
}
|
||||
|
||||
export function parseStatusHostIds(value: unknown): Set<number> | null {
|
||||
if (value === undefined) return null;
|
||||
if (typeof value !== "string") return new Set();
|
||||
|
||||
return new Set(
|
||||
value
|
||||
.split(",")
|
||||
.map(Number)
|
||||
.filter((id) => Number.isSafeInteger(id) && id > 0),
|
||||
);
|
||||
}
|
||||
|
||||
export function tcpPingThroughJumpHost(
|
||||
jumpClient: Pick<Client, "forwardOut" | "end">,
|
||||
host: string,
|
||||
port: number,
|
||||
timeoutMs = 5000,
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
|
||||
const finish = (result: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
jumpClient.end();
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
const timeout = setTimeout(() => finish(false), timeoutMs);
|
||||
|
||||
jumpClient.forwardOut("127.0.0.1", 0, host, port, (error, stream) => {
|
||||
stream?.destroy();
|
||||
finish(!error && !!stream);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
import type { Express, RequestHandler } from "express";
|
||||
import type { AuthenticatedRequest } from "../../../types/index.js";
|
||||
import { createCurrentHostMetricsHistoryRepository } from "../../database/repositories/factory.js";
|
||||
import { statsLogger } from "../../utils/logger.js";
|
||||
import type { HostAction } from "../../utils/permission-manager.js";
|
||||
|
||||
type HistoryRoutesDeps = {
|
||||
validateHostId: RequestHandler;
|
||||
canAccessHost: (
|
||||
userId: string,
|
||||
hostId: number,
|
||||
level: HostAction,
|
||||
) => Promise<boolean>;
|
||||
};
|
||||
|
||||
const RANGE_OFFSETS: Record<string, number> = {
|
||||
"1h": 1 * 60 * 60 * 1000,
|
||||
"6h": 6 * 60 * 60 * 1000,
|
||||
"24h": 24 * 60 * 60 * 1000,
|
||||
"7d": 7 * 24 * 60 * 60 * 1000,
|
||||
"30d": 30 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
|
||||
export function registerHostMetricsHistoryRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, canAccessHost }: HistoryRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /metrics/history/{id}:
|
||||
* get:
|
||||
* summary: Get historical metrics for a host
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* - in: query
|
||||
* name: range
|
||||
* schema:
|
||||
* type: string
|
||||
* enum: [1h, 6h, 24h, 7d, 30d]
|
||||
* - in: query
|
||||
* name: from
|
||||
* schema:
|
||||
* type: string
|
||||
* format: date-time
|
||||
* - in: query
|
||||
* name: to
|
||||
* schema:
|
||||
* type: string
|
||||
* format: date-time
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Array of metric history rows.
|
||||
* 403:
|
||||
* description: Access denied.
|
||||
* 404:
|
||||
* description: Host not found or no access.
|
||||
*/
|
||||
app.get("/metrics/history/:id", validateHostId, async (req, res) => {
|
||||
const hostId = Number(req.params.id);
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
|
||||
try {
|
||||
const hasAccess = await canAccessHost(userId, hostId, "connect");
|
||||
if (!hasAccess) {
|
||||
return res.status(403).json({ error: "Access denied" });
|
||||
}
|
||||
|
||||
const { range, from, to } = req.query as Record<
|
||||
string,
|
||||
string | undefined
|
||||
>;
|
||||
|
||||
let fromTs: string;
|
||||
let toTs: string = new Date().toISOString();
|
||||
|
||||
if (range) {
|
||||
const offsetMs = RANGE_OFFSETS[range];
|
||||
if (!offsetMs) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: "Invalid range. Use 1h, 6h, 24h, 7d, or 30d" });
|
||||
}
|
||||
fromTs = new Date(Date.now() - offsetMs).toISOString();
|
||||
} else if (from && to) {
|
||||
const fromDate = new Date(from);
|
||||
const toDate = new Date(to);
|
||||
if (isNaN(fromDate.getTime()) || isNaN(toDate.getTime())) {
|
||||
return res.status(400).json({ error: "Invalid from/to date format" });
|
||||
}
|
||||
fromTs = fromDate.toISOString();
|
||||
toTs = toDate.toISOString();
|
||||
} else {
|
||||
fromTs = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
// SQLite CURRENT_TIMESTAMP stores 'YYYY-MM-DD HH:MM:SS' (no T/Z).
|
||||
// Normalize both sides to that format for reliable string comparison.
|
||||
const toSqlite = (iso: string) =>
|
||||
iso.replace("T", " ").replace(/\.\d{3}Z$/, "");
|
||||
const rows = (
|
||||
await createCurrentHostMetricsHistoryRepository().listRange(
|
||||
hostId,
|
||||
toSqlite(fromTs),
|
||||
toSqlite(toTs),
|
||||
)
|
||||
).map((row) => ({
|
||||
ts: row.ts,
|
||||
cpu_percent: row.cpuPercent,
|
||||
mem_percent: row.memPercent,
|
||||
disk_percent: row.diskPercent,
|
||||
net_rx_bytes: row.netRxBytes,
|
||||
net_tx_bytes: row.netTxBytes,
|
||||
}));
|
||||
|
||||
res.json({ rows, fromTs, toTs });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to fetch metrics history", {
|
||||
operation: "metrics_history_fetch_error",
|
||||
hostId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res.status(500).json({ error: "Failed to fetch metrics history" });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isHostKeyVerificationError,
|
||||
statusAfterAuthentication,
|
||||
statusAfterReachabilityCheck,
|
||||
} from "./host-status.js";
|
||||
|
||||
describe("isHostKeyVerificationError", () => {
|
||||
it.each([
|
||||
"Host denied (verification failed)",
|
||||
"Host key changed - please connect via Terminal to verify the new key",
|
||||
])("classifies %s", (message) => {
|
||||
expect(isHostKeyVerificationError(new Error(message))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not classify ordinary authentication failures", () => {
|
||||
expect(isHostKeyVerificationError(new Error("Permission denied"))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("host availability status", () => {
|
||||
it("does not call a TCP-reachable host online before authentication", () => {
|
||||
expect(statusAfterReachabilityCheck(true)).toBe("reachable");
|
||||
});
|
||||
|
||||
it("keeps a verified host online across later reachability checks", () => {
|
||||
expect(statusAfterReachabilityCheck(true, "online")).toBe("online");
|
||||
});
|
||||
|
||||
it("marks successful authentication online", () => {
|
||||
expect(statusAfterAuthentication(true, "reachable")).toBe("online");
|
||||
});
|
||||
|
||||
it("downgrades failed authentication without hiding reachability", () => {
|
||||
expect(statusAfterAuthentication(false, "online")).toBe("reachable");
|
||||
expect(statusAfterAuthentication(false, "offline")).toBe("offline");
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,127 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import { shellSingleQuote } from "./exec-elevated.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
export interface CronEntry {
|
||||
raw: string;
|
||||
enabled: boolean;
|
||||
schedule: string;
|
||||
command: string;
|
||||
}
|
||||
|
||||
const READ_CRONTAB_CMD = "crontab -l 2>/dev/null";
|
||||
|
||||
/** Parse a crontab into entries (comments/blank lines are dropped except as toggles). */
|
||||
export function parseCrontab(output: string): CronEntry[] {
|
||||
const entries: CronEntry[] = [];
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.replace(/\s+$/, "");
|
||||
if (!line.trim()) continue;
|
||||
// A commented-out job: "# <schedule> <command>" (our toggle convention).
|
||||
const disabled = line.match(
|
||||
/^#\s*((?:@\w+|\S+\s+\S+\s+\S+\s+\S+\s+\S+)\s+.+)$/,
|
||||
);
|
||||
if (disabled) {
|
||||
const { schedule, command } = splitScheduleCommand(disabled[1]);
|
||||
if (command) {
|
||||
entries.push({ raw: line, enabled: false, schedule, command });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Skip pure comments / env assignments.
|
||||
if (line.trimStart().startsWith("#")) continue;
|
||||
if (/^\s*[A-Z_]+=/.test(line)) continue;
|
||||
const { schedule, command } = splitScheduleCommand(line.trim());
|
||||
if (!command) continue;
|
||||
entries.push({ raw: line, enabled: true, schedule, command });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function splitScheduleCommand(line: string): {
|
||||
schedule: string;
|
||||
command: string;
|
||||
} {
|
||||
if (line.startsWith("@")) {
|
||||
const [sched, ...rest] = line.split(/\s+/);
|
||||
return { schedule: sched, command: rest.join(" ") };
|
||||
}
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length < 6) return { schedule: "", command: "" };
|
||||
return {
|
||||
schedule: parts.slice(0, 5).join(" "),
|
||||
command: parts.slice(5).join(" "),
|
||||
};
|
||||
}
|
||||
|
||||
const CRON_SCHEDULE_RE =
|
||||
/^(@(reboot|yearly|annually|monthly|weekly|daily|midnight|hourly)|[\d*/,-]+\s+[\d*/,-]+\s+[\d*/,-]+\s+[\d*/,A-Za-z-]+\s+[\d*/,A-Za-z-]+)$/;
|
||||
|
||||
export function isValidCronSchedule(schedule: string): boolean {
|
||||
return CRON_SCHEDULE_RE.test(schedule.trim());
|
||||
}
|
||||
|
||||
/** Serialize entries into a full crontab body (toggled entries are commented). */
|
||||
export function serializeCrontab(entries: CronEntry[]): string {
|
||||
const lines = entries.map((e) => {
|
||||
const body = `${e.schedule} ${e.command}`.trim();
|
||||
return e.enabled ? body : `# ${body}`;
|
||||
});
|
||||
return lines.join("\n") + (lines.length ? "\n" : "");
|
||||
}
|
||||
|
||||
/** Build a command that atomically replaces the user crontab from the given body. */
|
||||
export function buildApplyCrontabCommand(body: string): string {
|
||||
// printf the exact bytes into `crontab -` (reads new crontab from stdin).
|
||||
return `printf '%s' ${shellSingleQuote(body)} | crontab -`;
|
||||
}
|
||||
|
||||
export function registerCronRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/cron/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "cron_list", async (client) => {
|
||||
const { stdout } = await execCommand(client, READ_CRONTAB_CMD, 15000);
|
||||
return { entries: parseCrontab(stdout) };
|
||||
}),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/cron/:id",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"cron_replace",
|
||||
async (client, _host, req) => {
|
||||
const { entries } = req.body as { entries?: CronEntry[] };
|
||||
if (!Array.isArray(entries)) {
|
||||
throw new ManagerInputError("entries must be an array");
|
||||
}
|
||||
for (const e of entries) {
|
||||
if (typeof e?.command !== "string" || !e.command.trim()) {
|
||||
throw new ManagerInputError("Each entry needs a command");
|
||||
}
|
||||
if (e.command.includes("\n")) {
|
||||
throw new ManagerInputError("Commands cannot contain newlines");
|
||||
}
|
||||
if (!isValidCronSchedule(String(e.schedule))) {
|
||||
throw new ManagerInputError(`Invalid schedule: ${e.schedule}`);
|
||||
}
|
||||
}
|
||||
const body = serializeCrontab(entries);
|
||||
const { stdout, stderr, code } = await execCommand(
|
||||
client,
|
||||
buildApplyCrontabCommand(body),
|
||||
15000,
|
||||
);
|
||||
return { success: code === 0, output: stdout || stderr };
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { collectFirewallMetrics } from "../widgets/firewall-collector.js";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import {
|
||||
isValidPort,
|
||||
isValidIpProtocol,
|
||||
isValidFirewallTarget,
|
||||
type IpProtocol,
|
||||
type FirewallTarget,
|
||||
} from "./validation.js";
|
||||
import { detectPlatform } from "./platform.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
export interface FirewallRuleSpec {
|
||||
protocol: IpProtocol;
|
||||
port: number;
|
||||
target: FirewallTarget;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an iptables add/delete for an INPUT rule. We only ever touch INPUT for a
|
||||
* specific dport with an explicit target, and never the chain policy, so an
|
||||
* existing ESTABLISHED/SSH rule is left intact.
|
||||
*/
|
||||
export function buildIptablesRuleCommand(
|
||||
op: "add" | "delete",
|
||||
spec: FirewallRuleSpec,
|
||||
): string {
|
||||
const flag = op === "add" ? "-A" : "-D";
|
||||
return `iptables ${flag} INPUT -p ${spec.protocol} --dport ${spec.port} -j ${spec.target}`;
|
||||
}
|
||||
|
||||
export function buildNftRuleCommand(
|
||||
op: "add" | "delete",
|
||||
spec: FirewallRuleSpec,
|
||||
): string {
|
||||
// nftables uses the inet filter table's input chain by convention.
|
||||
const verb = op === "add" ? "add" : "delete";
|
||||
const action =
|
||||
spec.target.toLowerCase() === "reject"
|
||||
? "reject"
|
||||
: spec.target.toLowerCase();
|
||||
return `nft ${verb} rule inet filter input ${spec.protocol} dport ${spec.port} ${action}`;
|
||||
}
|
||||
|
||||
export function registerFirewallRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/firewall/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "firewall_read", async (client) => {
|
||||
return await collectFirewallMetrics(client);
|
||||
}),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/firewall/:id/rule",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"firewall_rule",
|
||||
async (client, host, req) => {
|
||||
const { op, protocol, port, target } = req.body as {
|
||||
op?: "add" | "delete";
|
||||
protocol?: string;
|
||||
port?: number;
|
||||
target?: string;
|
||||
};
|
||||
if (op !== "add" && op !== "delete") {
|
||||
throw new ManagerInputError("Invalid op");
|
||||
}
|
||||
if (!isValidIpProtocol(protocol))
|
||||
throw new ManagerInputError("Invalid protocol");
|
||||
if (!isValidPort(port)) throw new ManagerInputError("Invalid port");
|
||||
if (!isValidFirewallTarget(target))
|
||||
throw new ManagerInputError("Invalid target");
|
||||
|
||||
const spec: FirewallRuleSpec = {
|
||||
protocol,
|
||||
port: Number(port),
|
||||
target,
|
||||
};
|
||||
const fw = await collectFirewallMetrics(client);
|
||||
const cmd =
|
||||
fw.type === "nftables"
|
||||
? buildNftRuleCommand(op, spec)
|
||||
: buildIptablesRuleCommand(op, spec);
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: result.stdout || result.stderr,
|
||||
backend: fw.type,
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/firewall/:id/persist",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"firewall_persist",
|
||||
async (client, host) => {
|
||||
const platform = await detectPlatform(client);
|
||||
// Best-effort persistence across common tools.
|
||||
const cmd =
|
||||
"(command -v netfilter-persistent >/dev/null 2>&1 && netfilter-persistent save) || " +
|
||||
"(command -v service >/dev/null 2>&1 && service iptables save) || " +
|
||||
"(command -v nft >/dev/null 2>&1 && nft list ruleset > /etc/nftables.conf) || true";
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: result.stdout || result.stderr,
|
||||
pkg: platform.pkg,
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import type { Client } from "ssh2";
|
||||
import type { AuthenticatedRequest } from "../../../../types/index.js";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { createCurrentHostHealthRepository } from "../../../database/repositories/factory.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import { shellSingleQuote } from "./exec-elevated.js";
|
||||
import { isValidPort } from "./validation.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
import { AlertEngine } from "../alert-engine.js";
|
||||
import { notifyAutomationHealthCheck } from "../automation-bridge.js";
|
||||
|
||||
export interface HealthCheck {
|
||||
id: string;
|
||||
name: string;
|
||||
type: "tcp" | "http";
|
||||
target: string;
|
||||
port?: number;
|
||||
path?: string;
|
||||
}
|
||||
|
||||
export interface HealthResult {
|
||||
checkId: string;
|
||||
ok: boolean;
|
||||
latencyMs: number | null;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
const TARGET_RE = /^[A-Za-z0-9.\-_:]+$/;
|
||||
const PATH_RE = /^\/[A-Za-z0-9._~!$&'()*+,;=:@/%-]*$/;
|
||||
|
||||
export function isValidHealthCheck(c: unknown): c is HealthCheck {
|
||||
if (!c || typeof c !== "object") return false;
|
||||
const o = c as Record<string, unknown>;
|
||||
if (typeof o.id !== "string" || !o.id) return false;
|
||||
if (typeof o.name !== "string") return false;
|
||||
if (o.type !== "tcp" && o.type !== "http") return false;
|
||||
if (typeof o.target !== "string" || !TARGET_RE.test(o.target)) return false;
|
||||
if (o.type === "tcp" && !isValidPort(o.port)) return false;
|
||||
if (
|
||||
o.path !== undefined &&
|
||||
(typeof o.path !== "string" || !PATH_RE.test(o.path))
|
||||
)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Build a command that runs the check from the host and prints "ok latency". */
|
||||
export function buildHealthCheckCommand(check: HealthCheck): string {
|
||||
if (check.type === "tcp") {
|
||||
const host = shellSingleQuote(check.target);
|
||||
const port = check.port;
|
||||
// Prefer bash /dev/tcp; time it with date in ms.
|
||||
return `start=$(date +%s%3N); if timeout 3 bash -c '</dev/tcp/'${host}'/'${port} 2>/dev/null; then echo "ok $(( $(date +%s%3N) - start ))"; else echo "fail $(( $(date +%s%3N) - start ))"; fi`;
|
||||
}
|
||||
// http
|
||||
const scheme = check.target.includes("://") ? "" : "http://";
|
||||
const url = shellSingleQuote(`${scheme}${check.target}${check.path ?? ""}`);
|
||||
return `curl -s -o /dev/null -m 5 -w '%{http_code} %{time_total}' ${url} || echo '000 0'`;
|
||||
}
|
||||
|
||||
export function parseHealthResult(
|
||||
check: HealthCheck,
|
||||
output: string,
|
||||
): HealthResult {
|
||||
const line = output.trim().split("\n").pop() ?? "";
|
||||
if (check.type === "tcp") {
|
||||
const [status, ms] = line.split(/\s+/);
|
||||
return {
|
||||
checkId: check.id,
|
||||
ok: status === "ok",
|
||||
latencyMs: Number(ms) || null,
|
||||
detail: status === "ok" ? "open" : "closed/timeout",
|
||||
};
|
||||
}
|
||||
const m = line.match(/^(\d{3})\s+([\d.]+)/);
|
||||
if (!m)
|
||||
return { checkId: check.id, ok: false, latencyMs: null, detail: line };
|
||||
const code = Number(m[1]);
|
||||
return {
|
||||
checkId: check.id,
|
||||
ok: code >= 200 && code < 400,
|
||||
latencyMs: Math.round(Number(m[2]) * 1000),
|
||||
detail: `HTTP ${code}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function runChecks(
|
||||
client: Client,
|
||||
checks: HealthCheck[],
|
||||
): Promise<HealthResult[]> {
|
||||
return Promise.all(
|
||||
checks.map(async (check) => {
|
||||
try {
|
||||
const { stdout } = await execCommand(
|
||||
client,
|
||||
buildHealthCheckCommand(check),
|
||||
8000,
|
||||
);
|
||||
return parseHealthResult(check, stdout);
|
||||
} catch {
|
||||
return {
|
||||
checkId: check.id,
|
||||
ok: false,
|
||||
latencyMs: null,
|
||||
detail: "error",
|
||||
};
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const HISTORY_KEEP = 500;
|
||||
|
||||
function recordHistory(
|
||||
userId: string,
|
||||
hostId: number,
|
||||
results: HealthResult[],
|
||||
): Promise<void> {
|
||||
return createCurrentHostHealthRepository()
|
||||
.recordHistory(userId, hostId, results, HISTORY_KEEP)
|
||||
.then(() => undefined);
|
||||
}
|
||||
|
||||
async function loadChecks(
|
||||
userId: string,
|
||||
hostId: number,
|
||||
): Promise<HealthCheck[]> {
|
||||
const row = await createCurrentHostHealthRepository().findChecksByUserAndHost(
|
||||
userId,
|
||||
hostId,
|
||||
);
|
||||
if (!row?.checks) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(row.checks);
|
||||
return Array.isArray(parsed) ? parsed.filter(isValidHealthCheck) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function registerHealthRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/health/:id",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"health_get",
|
||||
async (client, host, req) => {
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
const checks = await loadChecks(userId, host.id);
|
||||
const results = checks.length ? await runChecks(client, checks) : [];
|
||||
if (results.length) {
|
||||
await recordHistory(userId, host.id, results);
|
||||
for (const r of results) {
|
||||
AlertEngine.getInstance()
|
||||
.evaluateHealthCheck(
|
||||
host.id,
|
||||
userId,
|
||||
r.checkId,
|
||||
r.ok,
|
||||
r.detail ?? undefined,
|
||||
)
|
||||
.catch(() => {});
|
||||
notifyAutomationHealthCheck(
|
||||
host.id,
|
||||
userId,
|
||||
r.checkId,
|
||||
r.ok,
|
||||
r.detail ?? undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const history = (
|
||||
await createCurrentHostHealthRepository().listHistory(
|
||||
userId,
|
||||
host.id,
|
||||
200,
|
||||
)
|
||||
).map((row) => ({
|
||||
checkId: row.checkId,
|
||||
ts: row.ts,
|
||||
ok: row.ok,
|
||||
latencyMs: row.latencyMs,
|
||||
detail: row.detail,
|
||||
}));
|
||||
return { checks, results, history };
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/health/:id/config",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"health_config",
|
||||
async (_client, host, req) => {
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
const { checks, intervalSeconds } = req.body as {
|
||||
checks?: unknown;
|
||||
intervalSeconds?: number;
|
||||
};
|
||||
if (!Array.isArray(checks) || !checks.every(isValidHealthCheck)) {
|
||||
throw new ManagerInputError("Invalid checks");
|
||||
}
|
||||
const interval =
|
||||
typeof intervalSeconds === "number" &&
|
||||
intervalSeconds >= 30 &&
|
||||
intervalSeconds <= 86400
|
||||
? Math.round(intervalSeconds)
|
||||
: 300;
|
||||
const now = new Date().toISOString();
|
||||
await createCurrentHostHealthRepository().upsertChecks(
|
||||
userId,
|
||||
host.id,
|
||||
JSON.stringify(checks),
|
||||
interval,
|
||||
now,
|
||||
);
|
||||
return { success: true };
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/health/:id/run",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"health_run",
|
||||
async (client, host, req) => {
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
const checks = await loadChecks(userId, host.id);
|
||||
const results = await runChecks(client, checks);
|
||||
if (results.length) {
|
||||
await recordHistory(userId, host.id, results);
|
||||
for (const r of results) {
|
||||
AlertEngine.getInstance()
|
||||
.evaluateHealthCheck(
|
||||
host.id,
|
||||
userId,
|
||||
r.checkId,
|
||||
r.ok,
|
||||
r.detail ?? undefined,
|
||||
)
|
||||
.catch(() => {});
|
||||
notifyAutomationHealthCheck(
|
||||
host.id,
|
||||
userId,
|
||||
r.checkId,
|
||||
r.ok,
|
||||
r.detail ?? undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { results };
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { detectPlatform } from "./platform.js";
|
||||
import { managerHandler } from "./route-helpers.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
import { registerServiceRoutes } from "./services.js";
|
||||
import { registerProcessRoutes } from "./processes.js";
|
||||
import { registerSimpleReadRoutes } from "./simple-reads.js";
|
||||
import { registerCronRoutes } from "./cron.js";
|
||||
import { registerPackageRoutes } from "./packages.js";
|
||||
import { registerSslRoutes } from "./ssl.js";
|
||||
import { registerFirewallRoutes } from "./firewall.js";
|
||||
import { registerUserRoutes } from "./users.js";
|
||||
import { registerHealthRoutes } from "./health.js";
|
||||
import { registerLogRoutes } from "./logs.js";
|
||||
import { registerWireGuardRoutes } from "./wireguard.js";
|
||||
import { registerTailscaleRoutes } from "./tailscale.js";
|
||||
|
||||
/**
|
||||
* Registers every Host Metrics manager route under the `/host-metrics/managers`
|
||||
* prefix on the stats app. All routes are on-demand (not polled).
|
||||
*/
|
||||
export function registerManagerRoutes(
|
||||
app: Express,
|
||||
deps: ManagerRoutesDeps,
|
||||
): void {
|
||||
const { validateHostId, runOnHost } = deps;
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/platform/{id}:
|
||||
* get:
|
||||
* summary: Detect available management tooling on a host
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Platform capabilities (systemd, package manager, certbot, docker).
|
||||
*/
|
||||
app.get(
|
||||
"/host-metrics/platform/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "platform_detect", (client) =>
|
||||
detectPlatform(client),
|
||||
),
|
||||
);
|
||||
|
||||
registerServiceRoutes(app, deps);
|
||||
registerProcessRoutes(app, deps);
|
||||
registerSimpleReadRoutes(app, deps);
|
||||
registerCronRoutes(app, deps);
|
||||
registerPackageRoutes(app, deps);
|
||||
registerSslRoutes(app, deps);
|
||||
registerFirewallRoutes(app, deps);
|
||||
registerUserRoutes(app, deps);
|
||||
registerHealthRoutes(app, deps);
|
||||
registerLogRoutes(app, deps);
|
||||
registerWireGuardRoutes(app, deps);
|
||||
registerTailscaleRoutes(app, deps);
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated, shellSingleQuote } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import { isAllowedPath, isValidSystemdUnit } from "./validation.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
/** Directories from which arbitrary log files may be tailed. */
|
||||
export const LOG_PATH_ALLOWLIST = ["/var/log"];
|
||||
|
||||
const COMMON_LOGS = [
|
||||
"/var/log/syslog",
|
||||
"/var/log/messages",
|
||||
"/var/log/auth.log",
|
||||
"/var/log/secure",
|
||||
"/var/log/kern.log",
|
||||
"/var/log/dpkg.log",
|
||||
"/var/log/nginx/access.log",
|
||||
"/var/log/nginx/error.log",
|
||||
];
|
||||
|
||||
const LIST_LOGS_CMD = `ls -1 ${LOG_PATH_ALLOWLIST.map(shellSingleQuote).join(" ")} 2>/dev/null`;
|
||||
|
||||
export function clampLines(n: unknown): number {
|
||||
const v = typeof n === "string" ? Number(n) : n;
|
||||
if (typeof v !== "number" || !Number.isFinite(v)) return 200;
|
||||
return Math.min(2000, Math.max(1, Math.round(v)));
|
||||
}
|
||||
|
||||
export function buildTailCommand(path: string, lines: number): string {
|
||||
// Keep stderr intact so execElevated can detect a permission error and
|
||||
// escalate; suppressing it (2>/dev/null) would hide the denial and return an
|
||||
// empty log with no chance to retry under sudo.
|
||||
return `tail -n ${lines} ${shellSingleQuote(path)}`;
|
||||
}
|
||||
|
||||
export function buildJournalCommand(unit: string, lines: number): string {
|
||||
return `journalctl -u ${shellSingleQuote(unit)} -n ${lines} --no-pager`;
|
||||
}
|
||||
|
||||
export function registerLogRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/logs/:id/files",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "logs_list", async (client) => {
|
||||
const { stdout } = await execCommand(client, LIST_LOGS_CMD, 10000);
|
||||
const found = stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
.map((name) => `/var/log/${name}`);
|
||||
return { common: COMMON_LOGS, files: found };
|
||||
}),
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/host-metrics/managers/logs/:id",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"logs_tail",
|
||||
async (client, host, req) => {
|
||||
const path = req.query.path as string | undefined;
|
||||
const unit = req.query.unit as string | undefined;
|
||||
const lines = clampLines(req.query.lines);
|
||||
|
||||
let cmd: string;
|
||||
if (unit) {
|
||||
if (!isValidSystemdUnit(unit))
|
||||
throw new ManagerInputError("Invalid unit");
|
||||
cmd = buildJournalCommand(unit, lines);
|
||||
} else if (path) {
|
||||
if (!isAllowedPath(path, LOG_PATH_ALLOWLIST)) {
|
||||
throw new ManagerInputError("Path not allowed");
|
||||
}
|
||||
cmd = buildTailCommand(path, lines);
|
||||
} else {
|
||||
throw new ManagerInputError("Provide a path or unit");
|
||||
}
|
||||
|
||||
// Try unprivileged; many logs need root (auth.log, etc.).
|
||||
const result = await execElevated(client, cmd, host.sudoPassword);
|
||||
return { content: result.stdout, lines };
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import { isValidPid, isValidSignal, type Signal } from "./validation.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
export interface ProcessRow {
|
||||
pid: number;
|
||||
ppid: number;
|
||||
user: string;
|
||||
cpu: number;
|
||||
mem: number;
|
||||
rss: number;
|
||||
stat: string;
|
||||
command: string;
|
||||
args: string;
|
||||
}
|
||||
|
||||
const LIST_PROCESSES_CMD =
|
||||
"ps -eo pid,ppid,user:20,pcpu,pmem,rss,stat,comm,args --sort=-pcpu --no-headers 2>/dev/null | head -n 300";
|
||||
|
||||
/** Parse `ps -eo pid,ppid,user,pcpu,pmem,rss,stat,comm,args` output. */
|
||||
export function parseProcessList(output: string): ProcessRow[] {
|
||||
const rows: ProcessRow[] = [];
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
const m = line.match(
|
||||
/^(\d+)\s+(\d+)\s+(\S+)\s+([\d.]+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.*)$/,
|
||||
);
|
||||
if (!m) continue;
|
||||
rows.push({
|
||||
pid: Number(m[1]),
|
||||
ppid: Number(m[2]),
|
||||
user: m[3],
|
||||
cpu: Number(m[4]),
|
||||
mem: Number(m[5]),
|
||||
rss: Number(m[6]),
|
||||
stat: m[7],
|
||||
command: m[8],
|
||||
args: m[9],
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
export function buildKillCommand(pid: number, signal: Signal): string {
|
||||
return `kill -${signal} ${pid}`;
|
||||
}
|
||||
|
||||
export function registerProcessRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/processes/{id}:
|
||||
* get:
|
||||
* summary: List processes (rich, sortable, filterable client-side)
|
||||
* tags: [Host Metrics]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema: { type: integer }
|
||||
* responses:
|
||||
* 200: { description: Process list. }
|
||||
*/
|
||||
app.get(
|
||||
"/host-metrics/managers/processes/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "processes_list", async (client) => {
|
||||
const { stdout } = await execCommand(client, LIST_PROCESSES_CMD, 20000);
|
||||
return { processes: parseProcessList(stdout) };
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/processes/{id}/signal:
|
||||
* post:
|
||||
* summary: Send a signal to a process (TERM/KILL/HUP/INT)
|
||||
* tags: [Host Metrics]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema: { type: integer }
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* pid: { type: integer }
|
||||
* signal: { type: string }
|
||||
* responses:
|
||||
* 200: { description: Signal result. }
|
||||
* 400: { description: Invalid pid or signal. }
|
||||
* 403: { description: Elevation required or denied. }
|
||||
*/
|
||||
app.post(
|
||||
"/host-metrics/managers/processes/:id/signal",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"processes_signal",
|
||||
async (client, host, req) => {
|
||||
const { pid, signal } = req.body as {
|
||||
pid?: number;
|
||||
signal?: string;
|
||||
};
|
||||
if (!isValidPid(pid)) throw new ManagerInputError("Invalid pid");
|
||||
if (!isValidSignal(signal))
|
||||
throw new ManagerInputError("Invalid signal");
|
||||
const cmd = buildKillCommand(Number(pid), signal);
|
||||
// Try unprivileged first; elevate only if the process isn't owned.
|
||||
const result = await execElevated(client, cmd, host.sudoPassword);
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: result.stdout || result.stderr,
|
||||
usedSudo: result.usedSudo,
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import { getErrorMessage } from "../../../utils/error-message.js";
|
||||
import type { Request, Response } from "express";
|
||||
import type { Client } from "ssh2";
|
||||
import type { AuthenticatedRequest } from "../../../../types/index.js";
|
||||
import { statsLogger } from "../../../utils/logger.js";
|
||||
import { ElevationError } from "./exec-elevated.js";
|
||||
import type { ManagerHost, RunOnHost } from "./types.js";
|
||||
import type { HostAction } from "../../../utils/permission-manager.js";
|
||||
|
||||
export class AccessDeniedError extends Error {
|
||||
constructor(message = "No access to this host") {
|
||||
super(message);
|
||||
this.name = "AccessDeniedError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ManagerInputError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ManagerInputError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a manager handler: parses hostId, runs `fn` on a pooled connection at the
|
||||
* given access level, and maps known errors to clean HTTP responses.
|
||||
*/
|
||||
export function managerHandler(
|
||||
runOnHost: RunOnHost,
|
||||
level: HostAction,
|
||||
operation: string,
|
||||
fn: (client: Client, host: ManagerHost, req: Request) => Promise<unknown>,
|
||||
) {
|
||||
return async (req: Request, res: Response) => {
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
const hostId = parseInt(String(req.params.id), 10);
|
||||
try {
|
||||
const result = await runOnHost(hostId, userId, level, (client, host) =>
|
||||
fn(client, host, req),
|
||||
);
|
||||
return res.json(result);
|
||||
} catch (error) {
|
||||
if (error instanceof ManagerInputError) {
|
||||
return res.status(400).json({ error: error.message });
|
||||
}
|
||||
if (error instanceof AccessDeniedError) {
|
||||
return res.status(403).json({ error: error.message });
|
||||
}
|
||||
if (error instanceof ElevationError) {
|
||||
return res.status(403).json({ error: error.message, code: error.code });
|
||||
}
|
||||
statsLogger.error(`Manager operation failed: ${operation}`, {
|
||||
operation,
|
||||
hostId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return res.status(500).json({
|
||||
error: getErrorMessage(error, "Operation failed"),
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import {
|
||||
isValidSystemdUnit,
|
||||
isValidServiceAction,
|
||||
type ServiceAction,
|
||||
} from "./validation.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
export interface SystemdService {
|
||||
unit: string;
|
||||
load: string;
|
||||
active: string;
|
||||
sub: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
const LIST_SERVICES_CMD =
|
||||
"systemctl list-units --type=service --all --no-legend --no-pager --plain 2>/dev/null";
|
||||
|
||||
/** Parse `systemctl list-units --plain` output into structured rows. */
|
||||
export function parseServiceList(output: string): SystemdService[] {
|
||||
const services: SystemdService[] = [];
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
// Columns: UNIT LOAD ACTIVE SUB DESCRIPTION (description may contain spaces)
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length < 4) continue;
|
||||
const [unit, load, active, sub, ...rest] = parts;
|
||||
if (!unit.endsWith(".service")) continue;
|
||||
services.push({
|
||||
unit,
|
||||
load,
|
||||
active,
|
||||
sub,
|
||||
description: rest.join(" "),
|
||||
});
|
||||
}
|
||||
return services;
|
||||
}
|
||||
|
||||
export function buildServiceActionCommand(
|
||||
unit: string,
|
||||
action: ServiceAction,
|
||||
): string {
|
||||
return `systemctl ${action} ${unit}`;
|
||||
}
|
||||
|
||||
export function registerServiceRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/services/{id}:
|
||||
* get:
|
||||
* summary: List systemd services
|
||||
* tags: [Host Metrics]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema: { type: integer }
|
||||
* responses:
|
||||
* 200: { description: List of services. }
|
||||
*/
|
||||
app.get(
|
||||
"/host-metrics/managers/services/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "services_list", async (client) => {
|
||||
const { stdout } = await execCommand(client, LIST_SERVICES_CMD, 20000);
|
||||
return { services: parseServiceList(stdout) };
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/services/{id}/action:
|
||||
* post:
|
||||
* summary: Start/stop/restart/enable/disable a systemd service
|
||||
* tags: [Host Metrics]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema: { type: integer }
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* unit: { type: string }
|
||||
* action: { type: string }
|
||||
* responses:
|
||||
* 200: { description: Action result. }
|
||||
* 400: { description: Invalid unit or action. }
|
||||
* 403: { description: Elevation required or denied. }
|
||||
*/
|
||||
app.post(
|
||||
"/host-metrics/managers/services/:id/action",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"services_action",
|
||||
async (client, host, req) => {
|
||||
const { unit, action } = req.body as {
|
||||
unit?: string;
|
||||
action?: string;
|
||||
};
|
||||
if (!isValidSystemdUnit(unit)) {
|
||||
throw new ManagerInputError("Invalid unit name");
|
||||
}
|
||||
if (!isValidServiceAction(action)) {
|
||||
throw new ManagerInputError("Invalid action");
|
||||
}
|
||||
const result = await execElevated(
|
||||
client,
|
||||
buildServiceActionCommand(unit, action),
|
||||
host.sudoPassword,
|
||||
{ forceSudo: true, timeoutMs: 30000 },
|
||||
);
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: result.stdout || result.stderr,
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { managerHandler } from "./route-helpers.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
// ─── Top by memory ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface MemProcessRow {
|
||||
pid: number;
|
||||
user: string;
|
||||
mem: number;
|
||||
rss: number;
|
||||
command: string;
|
||||
}
|
||||
|
||||
const TOP_MEM_CMD =
|
||||
"ps -eo pid,user:20,pmem,rss,comm --sort=-pmem --no-headers 2>/dev/null | head -n 20";
|
||||
|
||||
export function parseTopMemory(output: string): MemProcessRow[] {
|
||||
const rows: MemProcessRow[] = [];
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
const m = line.match(/^(\d+)\s+(\S+)\s+([\d.]+)\s+(\d+)\s+(.*)$/);
|
||||
if (!m) continue;
|
||||
rows.push({
|
||||
pid: Number(m[1]),
|
||||
user: m[2],
|
||||
mem: Number(m[3]),
|
||||
rss: Number(m[4]),
|
||||
command: m[5],
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ─── Systemd timers ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface TimerRow {
|
||||
next: string;
|
||||
left: string;
|
||||
last: string;
|
||||
unit: string;
|
||||
activates: string;
|
||||
}
|
||||
|
||||
const TIMERS_CMD =
|
||||
"systemctl list-timers --all --no-legend --no-pager 2>/dev/null";
|
||||
|
||||
export function parseTimers(output: string): TimerRow[] {
|
||||
const rows: TimerRow[] = [];
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (!line || line.startsWith("NEXT")) continue;
|
||||
// NEXT(3) LEFT(2) LAST(3) PASSED(2) UNIT ACTIVATES -> columns vary; grab
|
||||
// the trailing UNIT + ACTIVATES which always end the line.
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length < 2) continue;
|
||||
const activates = parts[parts.length - 1];
|
||||
const unit = parts[parts.length - 2];
|
||||
if (!unit.endsWith(".timer")) continue;
|
||||
rows.push({
|
||||
next: parts.slice(0, 3).join(" "),
|
||||
left: "",
|
||||
last: "",
|
||||
unit,
|
||||
activates,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ─── Disk breakdown (per-mount) ─────────────────────────────────────────────
|
||||
|
||||
export interface MountUsage {
|
||||
filesystem: string;
|
||||
sizeKb: number;
|
||||
usedKb: number;
|
||||
availKb: number;
|
||||
usePct: number;
|
||||
mount: string;
|
||||
}
|
||||
|
||||
const DF_CMD = "df -Pk 2>/dev/null | tail -n +2";
|
||||
|
||||
export function parseDfMounts(output: string): MountUsage[] {
|
||||
const mounts: MountUsage[] = [];
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
const m = line.match(/^(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.+)$/);
|
||||
if (!m) continue;
|
||||
const fs = m[1];
|
||||
// Skip pseudo/virtual filesystems that clutter the view.
|
||||
if (/^(tmpfs|devtmpfs|overlay|udev|none|shm)$/.test(fs)) continue;
|
||||
mounts.push({
|
||||
filesystem: fs,
|
||||
sizeKb: Number(m[2]),
|
||||
usedKb: Number(m[3]),
|
||||
availKb: Number(m[4]),
|
||||
usePct: Number(m[5]),
|
||||
mount: m[6],
|
||||
});
|
||||
}
|
||||
return mounts;
|
||||
}
|
||||
|
||||
export function registerSimpleReadRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/top-memory/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "top_memory", async (client) => {
|
||||
const { stdout } = await execCommand(client, TOP_MEM_CMD, 15000);
|
||||
return { processes: parseTopMemory(stdout) };
|
||||
}),
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/host-metrics/managers/timers/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "systemd_timers", async (client) => {
|
||||
const { stdout } = await execCommand(client, TIMERS_CMD, 15000);
|
||||
return { timers: parseTimers(stdout) };
|
||||
}),
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/host-metrics/managers/disk-breakdown/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "disk_breakdown", async (client) => {
|
||||
const { stdout } = await execCommand(client, DF_CMD, 15000);
|
||||
return { mounts: parseDfMounts(stdout) };
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -1,317 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated, shellSingleQuote } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import {
|
||||
isValidDomain,
|
||||
isValidDnsProvider,
|
||||
isAllowedPath,
|
||||
} from "./validation.js";
|
||||
import { detectPlatform } from "./platform.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
export type AcmeClient = "certbot" | "acme.sh";
|
||||
export type ChallengeType = "http-standalone" | "http-webroot" | "dns";
|
||||
|
||||
export interface CertInfo {
|
||||
client: AcmeClient | "other";
|
||||
name: string;
|
||||
domains: string[];
|
||||
expiry: string | null;
|
||||
path?: string;
|
||||
}
|
||||
|
||||
const CERTBOT_LIST_CMD = "certbot certificates 2>/dev/null";
|
||||
const ACMESH_BIN = '"$(command -v acme.sh || echo "$HOME/.acme.sh/acme.sh")"';
|
||||
const ACMESH_LIST_CMD = `${ACMESH_BIN} --list 2>/dev/null`;
|
||||
|
||||
/** Parse `certbot certificates` output. */
|
||||
export function parseCertbotCertificates(output: string): CertInfo[] {
|
||||
const certs: CertInfo[] = [];
|
||||
let current: CertInfo | null = null;
|
||||
for (const raw of output.split("\n")) {
|
||||
const line = raw.trim();
|
||||
const nameM = line.match(/^Certificate Name:\s+(.+)$/);
|
||||
if (nameM) {
|
||||
if (current) certs.push(current);
|
||||
current = {
|
||||
client: "certbot",
|
||||
name: nameM[1],
|
||||
domains: [],
|
||||
expiry: null,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
if (!current) continue;
|
||||
const domM = line.match(/^Domains:\s+(.+)$/);
|
||||
if (domM) current.domains = domM[1].split(/\s+/).filter(Boolean);
|
||||
const expM = line.match(/^Expiry Date:\s+(\S+\s+\S+)/);
|
||||
if (expM) current.expiry = expM[1];
|
||||
const pathM = line.match(/^Certificate Path:\s+(.+)$/);
|
||||
if (pathM) current.path = pathM[1];
|
||||
}
|
||||
if (current) certs.push(current);
|
||||
return certs;
|
||||
}
|
||||
|
||||
/** Parse `acme.sh --list` (tab/space separated columns with a header). */
|
||||
export function parseAcmeShList(output: string): CertInfo[] {
|
||||
const certs: CertInfo[] = [];
|
||||
const lines = output.split("\n").filter((l) => l.trim());
|
||||
if (lines.length < 2) return certs;
|
||||
for (const line of lines.slice(1)) {
|
||||
const cols = line
|
||||
.split(/\s{2,}|\t/)
|
||||
.map((c) => c.trim())
|
||||
.filter(Boolean);
|
||||
if (cols.length < 1) continue;
|
||||
certs.push({
|
||||
client: "acme.sh",
|
||||
name: cols[0],
|
||||
domains: [cols[0]],
|
||||
expiry: cols[cols.length - 1] || null,
|
||||
});
|
||||
}
|
||||
return certs;
|
||||
}
|
||||
|
||||
/**
|
||||
* A certbot cert name or acme.sh primary domain. Allows letters, digits, dots,
|
||||
* hyphens, underscores and the wildcard `*` (acme.sh), but no shell metachars.
|
||||
*/
|
||||
const CERT_NAME_RE = /^[A-Za-z0-9._*-]+$/;
|
||||
export function isValidCertName(name: unknown): name is string {
|
||||
return typeof name === "string" && name.length > 0 && CERT_NAME_RE.test(name);
|
||||
}
|
||||
|
||||
export interface IssueRequest {
|
||||
client: AcmeClient;
|
||||
domains: string[];
|
||||
challenge: ChallengeType;
|
||||
webroot?: string;
|
||||
dnsProvider?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the certificate issuance command for the chosen client/challenge.
|
||||
* DNS-01 provider credentials are expected to already be present in the
|
||||
* environment / provider config file; we never put secrets in argv.
|
||||
*/
|
||||
export function buildIssueCommand(req: IssueRequest): string {
|
||||
const domains = req.domains;
|
||||
if (req.client === "certbot") {
|
||||
const dFlags = domains.map((d) => `-d ${shellSingleQuote(d)}`).join(" ");
|
||||
if (req.challenge === "dns") {
|
||||
return `certbot certonly --non-interactive --agree-tos --dns-${req.dnsProvider} ${dFlags}`;
|
||||
}
|
||||
if (req.challenge === "http-webroot") {
|
||||
return `certbot certonly --non-interactive --agree-tos --webroot -w ${shellSingleQuote(
|
||||
req.webroot as string,
|
||||
)} ${dFlags}`;
|
||||
}
|
||||
return `certbot certonly --non-interactive --agree-tos --standalone ${dFlags}`;
|
||||
}
|
||||
// acme.sh
|
||||
const dFlags = domains.map((d) => `-d ${shellSingleQuote(d)}`).join(" ");
|
||||
if (req.challenge === "dns") {
|
||||
return `${ACMESH_BIN} --issue --dns dns_${req.dnsProvider} ${dFlags}`;
|
||||
}
|
||||
if (req.challenge === "http-webroot") {
|
||||
return `${ACMESH_BIN} --issue -w ${shellSingleQuote(
|
||||
req.webroot as string,
|
||||
)} ${dFlags}`;
|
||||
}
|
||||
return `${ACMESH_BIN} --issue --standalone ${dFlags}`;
|
||||
}
|
||||
|
||||
export function buildRenewCommand(client: AcmeClient, dryRun: boolean): string {
|
||||
if (client === "certbot") {
|
||||
return `certbot renew${dryRun ? " --dry-run" : ""}`;
|
||||
}
|
||||
return `${ACMESH_BIN} --renew-all${dryRun ? " --staging" : ""}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Revoke (and remove) a certificate. certbot revokes by its certificate name and
|
||||
* deletes the lineage afterwards; acme.sh revokes by primary domain then removes
|
||||
* it from management. `name` is the certbot cert name or the acme.sh domain.
|
||||
*/
|
||||
export function buildRevokeCommand(client: AcmeClient, name: string): string {
|
||||
if (client === "certbot") {
|
||||
return `certbot revoke --non-interactive --cert-name ${shellSingleQuote(
|
||||
name,
|
||||
)} --delete-after-revoke`;
|
||||
}
|
||||
const d = shellSingleQuote(name);
|
||||
return `${ACMESH_BIN} --revoke -d ${d} && ${ACMESH_BIN} --remove -d ${d}`;
|
||||
}
|
||||
|
||||
export function registerSslRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/ssl/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "ssl_list", async (client, host) => {
|
||||
const platform = await detectPlatform(client);
|
||||
const certs: CertInfo[] = [];
|
||||
if (platform.hasCertbot) {
|
||||
const r = await execElevated(
|
||||
client,
|
||||
CERTBOT_LIST_CMD,
|
||||
host.sudoPassword,
|
||||
).catch(() => null);
|
||||
if (r) certs.push(...parseCertbotCertificates(r.stdout));
|
||||
}
|
||||
if (platform.hasAcmeSh) {
|
||||
const { stdout } = await execCommand(
|
||||
client,
|
||||
ACMESH_LIST_CMD,
|
||||
15000,
|
||||
).catch(() => ({ stdout: "" }) as { stdout: string });
|
||||
certs.push(...parseAcmeShList(stdout));
|
||||
}
|
||||
return {
|
||||
clients: {
|
||||
certbot: platform.hasCertbot,
|
||||
acmeSh: platform.hasAcmeSh,
|
||||
},
|
||||
certs,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/ssl/:id/issue",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"ssl_issue",
|
||||
async (client, host, req) => {
|
||||
const body = req.body as Partial<IssueRequest>;
|
||||
if (body.client !== "certbot" && body.client !== "acme.sh") {
|
||||
throw new ManagerInputError("Invalid ACME client");
|
||||
}
|
||||
if (!Array.isArray(body.domains) || body.domains.length === 0) {
|
||||
throw new ManagerInputError("At least one domain is required");
|
||||
}
|
||||
for (const d of body.domains) {
|
||||
if (!isValidDomain(d))
|
||||
throw new ManagerInputError(`Invalid domain: ${d}`);
|
||||
}
|
||||
const challenge = body.challenge;
|
||||
if (
|
||||
challenge !== "http-standalone" &&
|
||||
challenge !== "http-webroot" &&
|
||||
challenge !== "dns"
|
||||
) {
|
||||
throw new ManagerInputError("Invalid challenge type");
|
||||
}
|
||||
if (challenge === "dns" && !isValidDnsProvider(body.dnsProvider)) {
|
||||
throw new ManagerInputError("Invalid DNS provider");
|
||||
}
|
||||
if (
|
||||
challenge === "http-webroot" &&
|
||||
!isAllowedPath(body.webroot, ["/var/www", "/srv", "/usr/share/nginx"])
|
||||
) {
|
||||
throw new ManagerInputError("Invalid or disallowed webroot path");
|
||||
}
|
||||
const cmd = buildIssueCommand(body as IssueRequest);
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
timeoutMs: 300000,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: (result.stdout || result.stderr).slice(-8000),
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/ssl/:id/renew",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"ssl_renew",
|
||||
async (client, host, req) => {
|
||||
const { client: acmeClient, dryRun } = req.body as {
|
||||
client?: AcmeClient;
|
||||
dryRun?: boolean;
|
||||
};
|
||||
if (acmeClient !== "certbot" && acmeClient !== "acme.sh") {
|
||||
throw new ManagerInputError("Invalid ACME client");
|
||||
}
|
||||
const cmd = buildRenewCommand(acmeClient, !!dryRun);
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
timeoutMs: 300000,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: (result.stdout || result.stderr).slice(-8000),
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/ssl/{id}/revoke:
|
||||
* post:
|
||||
* summary: Revoke and remove an issued certificate (certbot or acme.sh)
|
||||
* tags: [Host Metrics]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema: { type: integer }
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* client: { type: string, enum: [certbot, acme.sh] }
|
||||
* name: { type: string, description: certbot cert name or acme.sh domain }
|
||||
* responses:
|
||||
* 200: { description: Revoke result. }
|
||||
* 400: { description: Invalid client or certificate name. }
|
||||
* 403: { description: Elevation required or denied. }
|
||||
*/
|
||||
app.post(
|
||||
"/host-metrics/managers/ssl/:id/revoke",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"ssl_revoke",
|
||||
async (client, host, req) => {
|
||||
const { client: acmeClient, name } = req.body as {
|
||||
client?: AcmeClient;
|
||||
name?: string;
|
||||
};
|
||||
if (acmeClient !== "certbot" && acmeClient !== "acme.sh") {
|
||||
throw new ManagerInputError("Invalid ACME client");
|
||||
}
|
||||
if (!isValidCertName(name)) {
|
||||
throw new ManagerInputError("Invalid certificate name");
|
||||
}
|
||||
const cmd = buildRevokeCommand(acmeClient, name);
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
timeoutMs: 120000,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: (result.stdout || result.stderr).slice(-8000),
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
import { isValidTailscaleAction } from "./validation.js";
|
||||
|
||||
export interface TailscalePeer {
|
||||
hostname: string;
|
||||
tailscaleIPs: string[];
|
||||
online: boolean;
|
||||
isExitNode: boolean;
|
||||
}
|
||||
|
||||
export interface TailscaleData {
|
||||
installed: boolean;
|
||||
running: boolean;
|
||||
tailscaleIPs: string[];
|
||||
hostname: string | null;
|
||||
peers: TailscalePeer[];
|
||||
exitNodeInUse: boolean;
|
||||
}
|
||||
|
||||
const PROBE_CMD = [
|
||||
"command -v tailscale >/dev/null 2>&1 && echo ts_installed=1 || echo ts_installed=0",
|
||||
"tailscale status --json 2>/dev/null",
|
||||
].join("; ");
|
||||
|
||||
export function parseTailscaleData(output: string): TailscaleData {
|
||||
const notInstalled: TailscaleData = {
|
||||
installed: false,
|
||||
running: false,
|
||||
tailscaleIPs: [],
|
||||
hostname: null,
|
||||
peers: [],
|
||||
exitNodeInUse: false,
|
||||
};
|
||||
|
||||
if (output.includes("ts_installed=0")) return notInstalled;
|
||||
|
||||
// Strip the installation probe line to get the raw JSON
|
||||
const lines = output.split("\n");
|
||||
const jsonLines = lines.filter(
|
||||
(l) => !l.startsWith("ts_installed=") && l.trim() !== "",
|
||||
);
|
||||
const jsonStr = jsonLines.join("\n");
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(jsonStr) as {
|
||||
BackendState?: string;
|
||||
Self?: { HostName?: string; TailscaleIPs?: string[] };
|
||||
Peer?: Record<
|
||||
string,
|
||||
{
|
||||
HostName?: string;
|
||||
TailscaleIPs?: string[];
|
||||
Online?: boolean;
|
||||
ExitNode?: boolean;
|
||||
}
|
||||
>;
|
||||
CurrentExitNode?: string;
|
||||
};
|
||||
|
||||
const peers: TailscalePeer[] = Object.values(parsed.Peer ?? {}).map(
|
||||
(p) => ({
|
||||
hostname: p.HostName ?? "",
|
||||
tailscaleIPs: p.TailscaleIPs ?? [],
|
||||
online: p.Online ?? false,
|
||||
isExitNode: p.ExitNode ?? false,
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
installed: true,
|
||||
running: parsed.BackendState === "Running",
|
||||
tailscaleIPs: parsed.Self?.TailscaleIPs ?? [],
|
||||
hostname: parsed.Self?.HostName ?? null,
|
||||
peers,
|
||||
exitNodeInUse:
|
||||
typeof parsed.CurrentExitNode === "string" &&
|
||||
parsed.CurrentExitNode !== "",
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
installed: true,
|
||||
running: false,
|
||||
tailscaleIPs: [],
|
||||
hostname: null,
|
||||
peers: [],
|
||||
exitNodeInUse: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function registerTailscaleRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/tailscale/{id}:
|
||||
* get:
|
||||
* summary: Get Tailscale status and IPs
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Tailscale installation status, running state, IPs, and peer count.
|
||||
*/
|
||||
app.get(
|
||||
"/host-metrics/managers/tailscale/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "tailscale_read", async (client) => {
|
||||
const { stdout } = await execCommand(client, PROBE_CMD, 15000);
|
||||
return parseTailscaleData(stdout);
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/tailscale/{id}/action:
|
||||
* post:
|
||||
* summary: Connect or disconnect Tailscale
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* action:
|
||||
* type: string
|
||||
* enum: [up, down]
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Action result.
|
||||
*/
|
||||
app.post(
|
||||
"/host-metrics/managers/tailscale/:id/action",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"tailscale_action",
|
||||
async (client, host, req) => {
|
||||
const { action } = req.body as { action: unknown };
|
||||
if (!isValidTailscaleAction(action)) {
|
||||
throw new ManagerInputError("Invalid action, must be 'up' or 'down'");
|
||||
}
|
||||
const result = await execElevated(
|
||||
client,
|
||||
`tailscale ${action}`,
|
||||
host.sudoPassword,
|
||||
{ forceSudo: false, timeoutMs: 30000 },
|
||||
);
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: (result.stdout + result.stderr).trim(),
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import type { RequestHandler } from "express";
|
||||
import type { HostAction } from "../../../utils/permission-manager.js";
|
||||
|
||||
/** Minimal host shape managers need (includes the decrypted sudo password). */
|
||||
export interface ManagerHost {
|
||||
id: number;
|
||||
userId: string;
|
||||
sudoPassword?: string;
|
||||
enableDocker?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `fn` against a pooled SSH connection for the host, after verifying the
|
||||
* user has at least `level` access. Resolves the host (with sudoPassword) so
|
||||
* managers can elevate. Rejects with an access error if not permitted.
|
||||
*/
|
||||
export type RunOnHost = <T>(
|
||||
hostId: number,
|
||||
userId: string,
|
||||
level: HostAction,
|
||||
fn: (client: Client, host: ManagerHost) => Promise<T>,
|
||||
) => Promise<T>;
|
||||
|
||||
export interface ManagerRoutesDeps {
|
||||
validateHostId: RequestHandler;
|
||||
runOnHost: RunOnHost;
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import { isValidUsername, isValidGroupName } from "./validation.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
|
||||
export interface SystemUser {
|
||||
name: string;
|
||||
uid: number;
|
||||
gid: number;
|
||||
home: string;
|
||||
shell: string;
|
||||
}
|
||||
|
||||
export interface SystemGroup {
|
||||
name: string;
|
||||
gid: number;
|
||||
members: string[];
|
||||
}
|
||||
|
||||
// Human users only (uid >= 1000, excluding nobody at 65534).
|
||||
const READ_USERS_CMD = "getent passwd 2>/dev/null";
|
||||
const READ_GROUPS_CMD = "getent group 2>/dev/null";
|
||||
const READ_SUDOERS_CMD = "getent group sudo wheel 2>/dev/null";
|
||||
|
||||
export function parsePasswd(output: string): SystemUser[] {
|
||||
const users: SystemUser[] = [];
|
||||
for (const line of output.split("\n")) {
|
||||
const parts = line.split(":");
|
||||
if (parts.length < 7) continue;
|
||||
const uid = Number(parts[2]);
|
||||
if (!Number.isFinite(uid)) continue;
|
||||
if (uid < 1000 || uid === 65534) continue;
|
||||
users.push({
|
||||
name: parts[0],
|
||||
uid,
|
||||
gid: Number(parts[3]),
|
||||
home: parts[5],
|
||||
shell: parts[6],
|
||||
});
|
||||
}
|
||||
return users;
|
||||
}
|
||||
|
||||
export function parseGroups(output: string): SystemGroup[] {
|
||||
const groups: SystemGroup[] = [];
|
||||
for (const line of output.split("\n")) {
|
||||
const parts = line.split(":");
|
||||
if (parts.length < 4) continue;
|
||||
groups.push({
|
||||
name: parts[0],
|
||||
gid: Number(parts[2]),
|
||||
members: parts[3].split(",").filter(Boolean),
|
||||
});
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
export function parseSudoers(output: string): string[] {
|
||||
const members = new Set<string>();
|
||||
for (const line of output.split("\n")) {
|
||||
const parts = line.split(":");
|
||||
if (parts.length < 4) continue;
|
||||
parts[3]
|
||||
.split(",")
|
||||
.filter(Boolean)
|
||||
.forEach((m) => members.add(m));
|
||||
}
|
||||
return [...members];
|
||||
}
|
||||
|
||||
export type UserAction = "create" | "delete" | "addToGroup" | "removeFromGroup";
|
||||
|
||||
export function registerUserRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
app.get(
|
||||
"/host-metrics/managers/users/:id",
|
||||
validateHostId,
|
||||
managerHandler(runOnHost, "connect", "users_list", async (client) => {
|
||||
const [passwd, groups, sudoers] = await Promise.all([
|
||||
execCommand(client, READ_USERS_CMD, 15000),
|
||||
execCommand(client, READ_GROUPS_CMD, 15000),
|
||||
execCommand(client, READ_SUDOERS_CMD, 15000),
|
||||
]);
|
||||
return {
|
||||
users: parsePasswd(passwd.stdout),
|
||||
groups: parseGroups(groups.stdout),
|
||||
sudoers: parseSudoers(sudoers.stdout),
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/host-metrics/managers/users/:id/action",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"users_action",
|
||||
async (client, host, req) => {
|
||||
const { action, username, group } = req.body as {
|
||||
action?: UserAction;
|
||||
username?: string;
|
||||
group?: string;
|
||||
};
|
||||
if (!isValidUsername(username))
|
||||
throw new ManagerInputError("Invalid username");
|
||||
|
||||
// Never modify/delete the user we're connected as, or root.
|
||||
const who = (await execCommand(client, "id -un", 8000)).stdout.trim();
|
||||
if (username === who || username === "root") {
|
||||
throw new ManagerInputError(
|
||||
"Refusing to modify the connected user or root",
|
||||
);
|
||||
}
|
||||
|
||||
let cmd: string;
|
||||
switch (action) {
|
||||
case "create":
|
||||
cmd = `useradd -m ${username}`;
|
||||
break;
|
||||
case "delete":
|
||||
cmd = `userdel -r ${username}`;
|
||||
break;
|
||||
case "addToGroup":
|
||||
case "removeFromGroup":
|
||||
if (!isValidGroupName(group))
|
||||
throw new ManagerInputError("Invalid group");
|
||||
cmd =
|
||||
action === "addToGroup"
|
||||
? `usermod -aG ${group} ${username}`
|
||||
: `gpasswd -d ${username} ${group}`;
|
||||
break;
|
||||
default:
|
||||
throw new ManagerInputError("Invalid action");
|
||||
}
|
||||
|
||||
const result = await execElevated(client, cmd, host.sudoPassword, {
|
||||
forceSudo: true,
|
||||
});
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: result.stdout || result.stderr,
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
import type { Express } from "express";
|
||||
import { execElevated } from "./exec-elevated.js";
|
||||
import { managerHandler, ManagerInputError } from "./route-helpers.js";
|
||||
import type { ManagerRoutesDeps } from "./types.js";
|
||||
import {
|
||||
isValidWireGuardInterface,
|
||||
isValidWireGuardAction,
|
||||
} from "./validation.js";
|
||||
|
||||
export interface WireGuardPeer {
|
||||
publicKey: string;
|
||||
endpoint: string | null;
|
||||
allowedIPs: string[];
|
||||
latestHandshake: number | null;
|
||||
rxBytes: number;
|
||||
txBytes: number;
|
||||
}
|
||||
|
||||
export interface WireGuardInterface {
|
||||
name: string;
|
||||
publicKey: string | null;
|
||||
listenPort: number | null;
|
||||
up: boolean;
|
||||
peers: WireGuardPeer[];
|
||||
}
|
||||
|
||||
export interface WireGuardData {
|
||||
installed: boolean;
|
||||
interfaces: WireGuardInterface[];
|
||||
}
|
||||
|
||||
const PROBE_CMD = [
|
||||
"command -v wg >/dev/null 2>&1 && echo wg_installed=1 || echo wg_installed=0",
|
||||
"ip link show type wireguard 2>/dev/null",
|
||||
"echo __DUMP__",
|
||||
"wg show all dump 2>/dev/null",
|
||||
].join("; ");
|
||||
|
||||
export function parseWireGuardData(output: string): WireGuardData {
|
||||
if (output.includes("wg_installed=0")) {
|
||||
return { installed: false, interfaces: [] };
|
||||
}
|
||||
|
||||
const dumpIdx = output.indexOf("__DUMP__");
|
||||
const ipLinkPart = dumpIdx >= 0 ? output.slice(0, dumpIdx) : "";
|
||||
const dumpPart =
|
||||
dumpIdx >= 0 ? output.slice(dumpIdx + "__DUMP__".length) : "";
|
||||
|
||||
// Collect up interface names from `ip link show type wireguard`
|
||||
const upSet = new Set<string>();
|
||||
for (const line of ipLinkPart.split("\n")) {
|
||||
const m = line.match(/^\d+:\s+([A-Za-z0-9_-]+):/);
|
||||
if (m) upSet.add(m[1]);
|
||||
}
|
||||
|
||||
const ifaceMap = new Map<string, WireGuardInterface>();
|
||||
|
||||
for (const raw of dumpPart.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
const cols = line.split("\t");
|
||||
|
||||
if (cols.length === 5) {
|
||||
// Interface row: iface private_key public_key listen_port fwmark
|
||||
const [name, , publicKey, listenPortStr] = cols;
|
||||
if (!name) continue;
|
||||
ifaceMap.set(name, {
|
||||
name,
|
||||
publicKey: publicKey && publicKey !== "(none)" ? publicKey : null,
|
||||
listenPort:
|
||||
listenPortStr && listenPortStr !== "(none)"
|
||||
? Number(listenPortStr)
|
||||
: null,
|
||||
up: upSet.has(name),
|
||||
peers: [],
|
||||
});
|
||||
} else if (cols.length === 9) {
|
||||
// Peer row: iface public_key preshared_key endpoint allowed_ips latest_handshake rx_bytes tx_bytes persistent_keepalive
|
||||
const [
|
||||
name,
|
||||
publicKey,
|
||||
,
|
||||
endpoint,
|
||||
allowedIPsStr,
|
||||
handshakeStr,
|
||||
rxStr,
|
||||
txStr,
|
||||
] = cols;
|
||||
if (!name) continue;
|
||||
|
||||
if (!ifaceMap.has(name)) {
|
||||
ifaceMap.set(name, {
|
||||
name,
|
||||
publicKey: null,
|
||||
listenPort: null,
|
||||
up: upSet.has(name),
|
||||
peers: [],
|
||||
});
|
||||
}
|
||||
|
||||
const handshake = Number(handshakeStr);
|
||||
ifaceMap.get(name)!.peers.push({
|
||||
publicKey: publicKey ?? "",
|
||||
endpoint: endpoint && endpoint !== "(none)" ? endpoint : null,
|
||||
allowedIPs:
|
||||
allowedIPsStr && allowedIPsStr !== "(none)"
|
||||
? allowedIPsStr.split(",").map((s) => s.trim())
|
||||
: [],
|
||||
latestHandshake: handshake > 0 ? handshake : null,
|
||||
rxBytes: Number(rxStr) || 0,
|
||||
txBytes: Number(txStr) || 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { installed: true, interfaces: Array.from(ifaceMap.values()) };
|
||||
}
|
||||
|
||||
export function registerWireGuardRoutes(
|
||||
app: Express,
|
||||
{ validateHostId, runOnHost }: ManagerRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/wireguard/{id}:
|
||||
* get:
|
||||
* summary: Get WireGuard interfaces and peers
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: WireGuard installation status, interfaces, and peer details.
|
||||
*/
|
||||
app.get(
|
||||
"/host-metrics/managers/wireguard/:id",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"wireguard_read",
|
||||
async (client, host) => {
|
||||
const result = await execElevated(
|
||||
client,
|
||||
PROBE_CMD,
|
||||
host.sudoPassword,
|
||||
{
|
||||
forceSudo: false,
|
||||
timeoutMs: 15000,
|
||||
},
|
||||
);
|
||||
return parseWireGuardData(result.stdout);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/managers/wireguard/{id}/action:
|
||||
* post:
|
||||
* summary: Bring a WireGuard interface up or down
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* interface:
|
||||
* type: string
|
||||
* action:
|
||||
* type: string
|
||||
* enum: [up, down]
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Action result.
|
||||
*/
|
||||
app.post(
|
||||
"/host-metrics/managers/wireguard/:id/action",
|
||||
validateHostId,
|
||||
managerHandler(
|
||||
runOnHost,
|
||||
"connect",
|
||||
"wireguard_action",
|
||||
async (client, host, req) => {
|
||||
const { interface: iface, action } = req.body as {
|
||||
interface: unknown;
|
||||
action: unknown;
|
||||
};
|
||||
if (!isValidWireGuardInterface(iface)) {
|
||||
throw new ManagerInputError("Invalid WireGuard interface name");
|
||||
}
|
||||
if (!isValidWireGuardAction(action)) {
|
||||
throw new ManagerInputError("Invalid action, must be 'up' or 'down'");
|
||||
}
|
||||
const result = await execElevated(
|
||||
client,
|
||||
`wg-quick ${action} ${iface}`,
|
||||
host.sudoPassword,
|
||||
{ forceSudo: true, timeoutMs: 30000 },
|
||||
);
|
||||
return {
|
||||
success: result.code === 0,
|
||||
output: (result.stdout + result.stderr).trim(),
|
||||
};
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
import type { Express, RequestHandler } from "express";
|
||||
import type { AuthenticatedRequest } from "../../../types/index.js";
|
||||
import { createCurrentHostMetricsPreferenceRepository } from "../../database/repositories/factory.js";
|
||||
import { statsLogger } from "../../utils/logger.js";
|
||||
import {
|
||||
deriveEnabledWidgets,
|
||||
defaultLayoutFromWidgets,
|
||||
type HostMetricsLayout,
|
||||
} from "../../../types/host-metrics.js";
|
||||
import type { HostAction } from "../../utils/permission-manager.js";
|
||||
|
||||
interface PrefStatsConfig {
|
||||
enabledWidgets?: string[];
|
||||
}
|
||||
|
||||
interface PrefHost {
|
||||
id: number;
|
||||
userId: string;
|
||||
statsConfig?: unknown;
|
||||
}
|
||||
|
||||
type HostMetricsPreferencesRoutesDeps = {
|
||||
validateHostId: RequestHandler;
|
||||
fetchHostById: (hostId: number, userId: string) => Promise<PrefHost | null>;
|
||||
parseStatsConfig: (statsConfig: unknown) => PrefStatsConfig;
|
||||
canAccessHost: (
|
||||
userId: string,
|
||||
hostId: number,
|
||||
level: HostAction,
|
||||
) => Promise<boolean>;
|
||||
};
|
||||
|
||||
function sanitizeLayout(input: unknown): HostMetricsLayout | null {
|
||||
if (!input || typeof input !== "object") return null;
|
||||
const obj = input as Record<string, unknown>;
|
||||
if (!Array.isArray(obj.slots)) return null;
|
||||
const columns =
|
||||
typeof obj.columns === "number" && obj.columns >= 1 && obj.columns <= 4
|
||||
? Math.round(obj.columns)
|
||||
: 3;
|
||||
const slots = obj.slots
|
||||
.filter((s): s is Record<string, unknown> => !!s && typeof s === "object")
|
||||
.map((s, i) => ({
|
||||
id: String(s.id),
|
||||
order: typeof s.order === "number" ? s.order : i,
|
||||
colSpan:
|
||||
s.colSpan === 1 || s.colSpan === 2 || s.colSpan === 3 ? s.colSpan : 1,
|
||||
height:
|
||||
typeof s.height === "number" && s.height > 0
|
||||
? Math.round(s.height)
|
||||
: null,
|
||||
}))
|
||||
.filter((s) => s.id && s.id !== "undefined");
|
||||
return { slots, columns } as HostMetricsLayout;
|
||||
}
|
||||
|
||||
export function registerHostMetricsPreferencesRoutes(
|
||||
app: Express,
|
||||
{
|
||||
validateHostId,
|
||||
fetchHostById,
|
||||
parseStatsConfig,
|
||||
canAccessHost,
|
||||
}: HostMetricsPreferencesRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/preferences/{id}:
|
||||
* get:
|
||||
* summary: Get the Host Metrics layout for a host
|
||||
* description: Returns the current user's saved card layout for the host, or a default layout derived from the host's enabled widgets when none is saved.
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: The layout for this host.
|
||||
* 403:
|
||||
* description: No access to this host.
|
||||
* 404:
|
||||
* description: Host not found.
|
||||
*/
|
||||
app.get("/host-metrics/preferences/:id", validateHostId, async (req, res) => {
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
const hostId = parseInt(String(req.params.id), 10);
|
||||
try {
|
||||
if (!(await canAccessHost(userId, hostId, "connect"))) {
|
||||
return res.status(403).json({ error: "No access to this host" });
|
||||
}
|
||||
const host = await fetchHostById(hostId, userId);
|
||||
if (!host) {
|
||||
return res.status(404).json({ error: "Host not found" });
|
||||
}
|
||||
|
||||
const row =
|
||||
await createCurrentHostMetricsPreferenceRepository().findByUserAndHost(
|
||||
userId,
|
||||
hostId,
|
||||
);
|
||||
|
||||
if (row?.layout) {
|
||||
try {
|
||||
const parsed = sanitizeLayout(JSON.parse(row.layout));
|
||||
if (parsed) return res.json({ layout: parsed });
|
||||
} catch {
|
||||
// fall through to default
|
||||
}
|
||||
}
|
||||
|
||||
const statsConfig = parseStatsConfig(host.statsConfig);
|
||||
const layout = defaultLayoutFromWidgets(statsConfig.enabledWidgets ?? []);
|
||||
return res.json({ layout });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to fetch host metrics preferences", {
|
||||
operation: "host_metrics_prefs_fetch_error",
|
||||
hostId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return res
|
||||
.status(500)
|
||||
.json({ error: "Failed to fetch host metrics preferences" });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /host-metrics/preferences/{id}:
|
||||
* post:
|
||||
* summary: Save the Host Metrics layout for a host
|
||||
* description: Persists the current user's card layout for the host and keeps statsConfig.enabledWidgets in sync (for hosts the user owns) so the mobile app keeps working.
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* slots:
|
||||
* type: array
|
||||
* columns:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Layout saved.
|
||||
* 400:
|
||||
* description: Invalid layout.
|
||||
* 403:
|
||||
* description: No access to this host.
|
||||
*/
|
||||
app.post(
|
||||
"/host-metrics/preferences/:id",
|
||||
validateHostId,
|
||||
async (req, res) => {
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
const hostId = parseInt(String(req.params.id), 10);
|
||||
try {
|
||||
if (!(await canAccessHost(userId, hostId, "connect"))) {
|
||||
return res.status(403).json({ error: "No access to this host" });
|
||||
}
|
||||
const host = await fetchHostById(hostId, userId);
|
||||
if (!host) {
|
||||
return res.status(404).json({ error: "Host not found" });
|
||||
}
|
||||
|
||||
const layout = sanitizeLayout(req.body);
|
||||
if (!layout) {
|
||||
return res.status(400).json({ error: "Invalid layout" });
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const layoutJson = JSON.stringify(layout);
|
||||
|
||||
await createCurrentHostMetricsPreferenceRepository().upsertLayout(
|
||||
userId,
|
||||
hostId,
|
||||
layoutJson,
|
||||
now,
|
||||
);
|
||||
|
||||
// Keep statsConfig.enabledWidgets in sync (mobile contract) for hosts the
|
||||
// user owns. stats_config is plain JSON text (not an encrypted field).
|
||||
if (host.userId === userId) {
|
||||
try {
|
||||
const current = parseStatsConfig(host.statsConfig);
|
||||
const merged = {
|
||||
...current,
|
||||
enabledWidgets: deriveEnabledWidgets(layout.slots),
|
||||
};
|
||||
await createCurrentHostMetricsPreferenceRepository().updateHostStatsConfig(
|
||||
userId,
|
||||
hostId,
|
||||
JSON.stringify(merged),
|
||||
);
|
||||
} catch (syncErr) {
|
||||
statsLogger.warn("Failed to sync enabledWidgets from layout", {
|
||||
operation: "host_metrics_prefs_sync_widgets",
|
||||
hostId,
|
||||
error:
|
||||
syncErr instanceof Error ? syncErr.message : String(syncErr),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return res.json({ success: true });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to save host metrics preferences", {
|
||||
operation: "host_metrics_prefs_save_error",
|
||||
hostId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return res
|
||||
.status(500)
|
||||
.json({ error: "Failed to save host metrics preferences" });
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import {
|
||||
createCurrentProxmoxNodeHistoryRepository,
|
||||
getCurrentSettingValue,
|
||||
} from "../../database/repositories/factory.js";
|
||||
import { ConcurrentLimiter } from "./state.js";
|
||||
import { ConcurrentLimiter } from "../metrics-shared/state.js";
|
||||
import {
|
||||
collectProxmoxStats,
|
||||
type ProxmoxStatsSnapshot,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execCommand } from "../../metrics-shared/common-utils.js";
|
||||
|
||||
export interface ProxmoxClusterNodeEntry {
|
||||
name: string;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execCommand } from "../../metrics-shared/common-utils.js";
|
||||
import { isSafeNodeName } from "../../proxmox-shared.js";
|
||||
import {
|
||||
collectProxmoxNodeStatus,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "../widgets/common-utils.js";
|
||||
import { execCommand, toFixedNum } from "../../metrics-shared/common-utils.js";
|
||||
import { isSafeNodeName } from "../../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxGuestSummaryEntry {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "../widgets/common-utils.js";
|
||||
import { execCommand } from "../../metrics-shared/common-utils.js";
|
||||
import { isSafeNodeName } from "../../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxNodeNetworkInterface {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "../widgets/common-utils.js";
|
||||
import { execCommand, toFixedNum } from "../../metrics-shared/common-utils.js";
|
||||
import { isSafeNodeName } from "../../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxNodeStatusResult {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "../widgets/common-utils.js";
|
||||
import { execCommand, toFixedNum } from "../../metrics-shared/common-utils.js";
|
||||
import { isSafeNodeName } from "../../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxStoragePoolEntry {
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
import type { Client, ConnectConfig } from "ssh2";
|
||||
import { statsLogger } from "../../utils/logger.js";
|
||||
|
||||
export interface MetricsSession {
|
||||
client: Client;
|
||||
isConnected: boolean;
|
||||
lastActive: number;
|
||||
timeout?: NodeJS.Timeout;
|
||||
activeOperations: number;
|
||||
hostId: number;
|
||||
userId: string;
|
||||
}
|
||||
|
||||
export interface PendingTOTPSession {
|
||||
client: Client;
|
||||
finish: (responses: string[]) => void;
|
||||
config: ConnectConfig;
|
||||
createdAt: number;
|
||||
sessionId: string;
|
||||
hostId: number;
|
||||
userId: string;
|
||||
prompts?: Array<{ prompt: string; echo: boolean }>;
|
||||
totpPromptIndex?: number;
|
||||
resolvedPassword?: string;
|
||||
totpAttempts: number;
|
||||
}
|
||||
|
||||
export interface MetricsViewer {
|
||||
sessionId: string;
|
||||
userId: string;
|
||||
hostId: number;
|
||||
lastHeartbeat: number;
|
||||
}
|
||||
|
||||
export const metricsSessions: Record<string, MetricsSession> = {};
|
||||
export const pendingTOTPSessions: Record<string, PendingTOTPSession> = {};
|
||||
|
||||
export function cleanupMetricsSession(sessionId: string): void {
|
||||
const session = metricsSessions[sessionId];
|
||||
if (session) {
|
||||
if (session.activeOperations > 0) {
|
||||
statsLogger.warn(
|
||||
`Deferring metrics session cleanup - ${session.activeOperations} active operations`,
|
||||
{
|
||||
operation: "cleanup_deferred",
|
||||
sessionId,
|
||||
activeOperations: session.activeOperations,
|
||||
},
|
||||
);
|
||||
scheduleMetricsSessionCleanup(sessionId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
session.client.end();
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
clearTimeout(session.timeout);
|
||||
delete metricsSessions[sessionId];
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleMetricsSessionCleanup(sessionId: string): void {
|
||||
const session = metricsSessions[sessionId];
|
||||
if (session) {
|
||||
if (session.timeout) clearTimeout(session.timeout);
|
||||
|
||||
session.timeout = setTimeout(
|
||||
() => {
|
||||
cleanupMetricsSession(sessionId);
|
||||
},
|
||||
30 * 60 * 1000,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function getSessionKey(hostId: number, userId: string): string {
|
||||
return `${userId}:${hostId}`;
|
||||
}
|
||||
@@ -1,256 +0,0 @@
|
||||
import type { Express, RequestHandler } from "express";
|
||||
import { statsLogger } from "../../utils/logger.js";
|
||||
import { createCurrentSettingsRepository } from "../../database/repositories/factory.js";
|
||||
|
||||
type HostMetricsSettingsConfig = {
|
||||
statusCheckInterval: number;
|
||||
metricsInterval: number;
|
||||
};
|
||||
|
||||
type HostMetricsSettingsRoutesDeps = {
|
||||
requireAdmin: RequestHandler;
|
||||
defaultStatsConfig: HostMetricsSettingsConfig;
|
||||
refreshAllPolling: () => Promise<void>;
|
||||
};
|
||||
|
||||
function isMissingSettingsTable(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error && error.message.includes("no such table: settings")
|
||||
);
|
||||
}
|
||||
|
||||
export function registerHostMetricsSettingsRoutes(
|
||||
app: Express,
|
||||
{
|
||||
requireAdmin,
|
||||
defaultStatsConfig: DEFAULT_STATS_CONFIG,
|
||||
refreshAllPolling,
|
||||
}: HostMetricsSettingsRoutesDeps,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /global-settings:
|
||||
* get:
|
||||
* summary: Get global monitoring defaults
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Global monitoring settings.
|
||||
* 403:
|
||||
* description: Requires admin privileges.
|
||||
*/
|
||||
app.get("/global-settings", requireAdmin, async (_req, res) => {
|
||||
try {
|
||||
const settings = createCurrentSettingsRepository();
|
||||
const statusValue = await settings.get("global_status_check_interval");
|
||||
const metricsValue = await settings.get("global_metrics_interval");
|
||||
|
||||
res.json({
|
||||
statusCheckInterval: statusValue
|
||||
? parseInt(statusValue, 10)
|
||||
: DEFAULT_STATS_CONFIG.statusCheckInterval,
|
||||
metricsInterval: metricsValue
|
||||
? parseInt(metricsValue, 10)
|
||||
: DEFAULT_STATS_CONFIG.metricsInterval,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isMissingSettingsTable(error)) {
|
||||
statsLogger.warn("Settings table does not exist, using defaults", {
|
||||
operation: "global_settings_table_check",
|
||||
error: error.message,
|
||||
});
|
||||
return res.json({
|
||||
statusCheckInterval: DEFAULT_STATS_CONFIG.statusCheckInterval,
|
||||
metricsInterval: DEFAULT_STATS_CONFIG.metricsInterval,
|
||||
});
|
||||
}
|
||||
|
||||
statsLogger.error("Failed to fetch global settings", {
|
||||
operation: "global_settings_fetch_error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res.status(500).json({ error: "Failed to fetch global settings" });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /global-settings:
|
||||
* post:
|
||||
* summary: Update global monitoring defaults
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* statusCheckInterval:
|
||||
* type: integer
|
||||
* metricsInterval:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Settings saved.
|
||||
* 400:
|
||||
* description: Invalid parameters.
|
||||
* 403:
|
||||
* description: Requires admin privileges.
|
||||
*/
|
||||
app.post("/global-settings", requireAdmin, async (req, res) => {
|
||||
const { statusCheckInterval, metricsInterval } = req.body;
|
||||
|
||||
if (
|
||||
statusCheckInterval !== undefined &&
|
||||
(typeof statusCheckInterval !== "number" ||
|
||||
statusCheckInterval < 5 ||
|
||||
statusCheckInterval > 3600)
|
||||
) {
|
||||
return res.status(400).json({
|
||||
error: "statusCheckInterval must be between 5 and 3600 seconds",
|
||||
});
|
||||
}
|
||||
if (
|
||||
metricsInterval !== undefined &&
|
||||
(typeof metricsInterval !== "number" ||
|
||||
metricsInterval < 5 ||
|
||||
metricsInterval > 3600)
|
||||
) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: "metricsInterval must be between 5 and 3600 seconds" });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = createCurrentSettingsRepository();
|
||||
|
||||
if (statusCheckInterval !== undefined) {
|
||||
await settings.set(
|
||||
"global_status_check_interval",
|
||||
String(statusCheckInterval),
|
||||
);
|
||||
}
|
||||
if (metricsInterval !== undefined) {
|
||||
await settings.set("global_metrics_interval", String(metricsInterval));
|
||||
}
|
||||
|
||||
await refreshAllPolling();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: "Settings updated and polling refreshed",
|
||||
});
|
||||
} catch (error) {
|
||||
if (isMissingSettingsTable(error)) {
|
||||
statsLogger.error(
|
||||
"Settings table does not exist, cannot save settings",
|
||||
{
|
||||
operation: "global_settings_table_check",
|
||||
error: error.message,
|
||||
},
|
||||
);
|
||||
return res.status(500).json({
|
||||
error:
|
||||
"Database settings table is missing. Please check database initialization.",
|
||||
});
|
||||
}
|
||||
|
||||
statsLogger.error("Failed to save global settings", {
|
||||
operation: "global_settings_save_error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res.status(500).json({
|
||||
error: "Failed to save global settings",
|
||||
details: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /global-settings/history:
|
||||
* get:
|
||||
* summary: Get metrics history retention setting
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Retention setting in days.
|
||||
* 403:
|
||||
* description: Requires admin privileges.
|
||||
*/
|
||||
app.get("/global-settings/history", requireAdmin, async (_req, res) => {
|
||||
try {
|
||||
const value = await createCurrentSettingsRepository().get(
|
||||
"metrics_history_retention_days",
|
||||
);
|
||||
const days = value ? parseInt(value, 10) : 7;
|
||||
res.json({ metricsHistoryRetentionDays: isNaN(days) ? 7 : days });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to fetch history retention setting", {
|
||||
operation: "history_retention_fetch_error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res
|
||||
.status(500)
|
||||
.json({ error: "Failed to fetch history retention setting" });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /global-settings/history:
|
||||
* post:
|
||||
* summary: Update metrics history retention setting
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* metricsHistoryRetentionDays:
|
||||
* type: integer
|
||||
* minimum: 1
|
||||
* maximum: 90
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Setting saved.
|
||||
* 400:
|
||||
* description: Invalid value.
|
||||
* 403:
|
||||
* description: Requires admin privileges.
|
||||
*/
|
||||
app.post("/global-settings/history", requireAdmin, async (req, res) => {
|
||||
const { metricsHistoryRetentionDays } = req.body;
|
||||
if (
|
||||
typeof metricsHistoryRetentionDays !== "number" ||
|
||||
metricsHistoryRetentionDays < 1 ||
|
||||
metricsHistoryRetentionDays > 90
|
||||
) {
|
||||
return res.status(400).json({
|
||||
error: "metricsHistoryRetentionDays must be between 1 and 90",
|
||||
});
|
||||
}
|
||||
try {
|
||||
await createCurrentSettingsRepository().set(
|
||||
"metrics_history_retention_days",
|
||||
String(Math.floor(metricsHistoryRetentionDays)),
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to save history retention setting", {
|
||||
operation: "history_retention_save_error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res
|
||||
.status(500)
|
||||
.json({ error: "Failed to save history retention setting" });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
import { getErrorMessage } from "../../utils/error-message.js";
|
||||
import type { Express } from "express";
|
||||
import type { AuthenticatedRequest } from "../../../types/index.js";
|
||||
import { statsLogger } from "../../utils/logger.js";
|
||||
import { DataCrypto } from "../../utils/data-crypto.js";
|
||||
|
||||
type ViewerStatsConfig = {
|
||||
metricsEnabled: boolean;
|
||||
};
|
||||
|
||||
type HostMetricsViewerRoutesDeps<
|
||||
THost extends { statsConfig?: string | TStatsConfig },
|
||||
TStatsConfig extends ViewerStatsConfig,
|
||||
> = {
|
||||
fetchHostById: (hostId: number, userId: string) => Promise<THost>;
|
||||
supportsMetrics: (host: THost) => boolean;
|
||||
parseStatsConfig: (statsConfig: THost["statsConfig"]) => TStatsConfig;
|
||||
updateHeartbeat: (viewerSessionId: string) => boolean;
|
||||
registerViewer: (
|
||||
hostId: number,
|
||||
viewerSessionId: string,
|
||||
userId: string,
|
||||
) => void;
|
||||
unregisterViewer: (hostId: number, viewerSessionId: string) => void;
|
||||
/** Route path segment, e.g. "metrics" -> /metrics/heartbeat. Defaults to "metrics". */
|
||||
pathPrefix?: string;
|
||||
};
|
||||
|
||||
export function registerHostMetricsViewerRoutes<
|
||||
THost extends { statsConfig?: string | TStatsConfig },
|
||||
TStatsConfig extends ViewerStatsConfig,
|
||||
>(
|
||||
app: Express,
|
||||
{
|
||||
fetchHostById,
|
||||
supportsMetrics,
|
||||
parseStatsConfig,
|
||||
updateHeartbeat,
|
||||
registerViewer,
|
||||
unregisterViewer,
|
||||
pathPrefix = "metrics",
|
||||
}: HostMetricsViewerRoutesDeps<THost, TStatsConfig>,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /metrics/heartbeat:
|
||||
* post:
|
||||
* summary: Update viewer heartbeat
|
||||
* description: Updates the heartbeat timestamp for a metrics viewer session to keep it alive.
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* viewerSessionId:
|
||||
* type: string
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Heartbeat updated successfully.
|
||||
* 400:
|
||||
* description: Invalid viewerSessionId.
|
||||
* 401:
|
||||
* description: Session expired - please log in again.
|
||||
* 404:
|
||||
* description: Viewer session not found.
|
||||
* 500:
|
||||
* description: Failed to update heartbeat.
|
||||
*/
|
||||
app.post(`/${pathPrefix}/heartbeat`, async (req, res) => {
|
||||
const { viewerSessionId } = req.body;
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
|
||||
if (DataCrypto.getUserDataKey(userId) === null) {
|
||||
return res.status(401).json({
|
||||
error: "Session expired - please log in again",
|
||||
code: "SESSION_EXPIRED",
|
||||
});
|
||||
}
|
||||
|
||||
if (!viewerSessionId || typeof viewerSessionId !== "string") {
|
||||
return res.status(400).json({ error: "Invalid viewerSessionId" });
|
||||
}
|
||||
|
||||
try {
|
||||
const success = updateHeartbeat(viewerSessionId);
|
||||
if (success) {
|
||||
res.json({ success: true });
|
||||
} else {
|
||||
res.status(404).json({ error: "Viewer session not found" });
|
||||
}
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to update heartbeat", {
|
||||
operation: "heartbeat_error",
|
||||
viewerSessionId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res.status(500).json({ error: "Failed to update heartbeat" });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /metrics/register-viewer:
|
||||
* post:
|
||||
* summary: Register metrics viewer
|
||||
* description: Registers a new viewer session for a host to track who is viewing metrics.
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* hostId:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Viewer registered successfully.
|
||||
* 400:
|
||||
* description: Invalid hostId.
|
||||
* 401:
|
||||
* description: Session expired - please log in again.
|
||||
* 500:
|
||||
* description: Failed to register viewer.
|
||||
*/
|
||||
app.post(`/${pathPrefix}/register-viewer`, async (req, res) => {
|
||||
const { hostId } = req.body;
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
|
||||
if (DataCrypto.getUserDataKey(userId) === null) {
|
||||
return res.status(401).json({
|
||||
error: "Session expired - please log in again",
|
||||
code: "SESSION_EXPIRED",
|
||||
});
|
||||
}
|
||||
|
||||
if (!hostId || typeof hostId !== "number") {
|
||||
return res.status(400).json({ error: "Invalid hostId" });
|
||||
}
|
||||
|
||||
try {
|
||||
// Graceful no-op if host is inaccessible, metrics disabled, or host type
|
||||
// does not support metrics. The client may call this speculatively, so
|
||||
// avoid returning 5xx for expected "no metrics available" scenarios.
|
||||
let host: THost | undefined;
|
||||
try {
|
||||
host = await fetchHostById(hostId, userId);
|
||||
} catch (lookupErr) {
|
||||
statsLogger.warn(
|
||||
"register-viewer host lookup failed (treating as no-op)",
|
||||
{
|
||||
operation: "register_viewer_lookup",
|
||||
hostId,
|
||||
userId,
|
||||
error: getErrorMessage(lookupErr, String(lookupErr)),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (!host) {
|
||||
return res.json({
|
||||
success: true,
|
||||
skipped: true,
|
||||
reason: "host_not_found",
|
||||
});
|
||||
}
|
||||
|
||||
if (!supportsMetrics(host)) {
|
||||
return res.json({
|
||||
success: true,
|
||||
skipped: true,
|
||||
reason: "metrics_unsupported",
|
||||
});
|
||||
}
|
||||
|
||||
const statsConfig = parseStatsConfig(host.statsConfig);
|
||||
if (!statsConfig.metricsEnabled) {
|
||||
return res.json({
|
||||
success: true,
|
||||
skipped: true,
|
||||
reason: "metrics_disabled",
|
||||
});
|
||||
}
|
||||
|
||||
const viewerSessionId = `viewer-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
try {
|
||||
registerViewer(hostId, viewerSessionId, userId);
|
||||
} catch (regErr) {
|
||||
statsLogger.warn(
|
||||
"pollingManager.registerViewer threw (treating as no-op)",
|
||||
{
|
||||
operation: "register_viewer_internal",
|
||||
hostId,
|
||||
userId,
|
||||
error: regErr instanceof Error ? regErr.message : String(regErr),
|
||||
},
|
||||
);
|
||||
return res.json({
|
||||
success: true,
|
||||
skipped: true,
|
||||
reason: "register_failed_noop",
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ success: true, viewerSessionId });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to register viewer", {
|
||||
operation: "register_viewer_error",
|
||||
hostId,
|
||||
userId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
// Even on unexpected errors we prefer a graceful client experience: the
|
||||
// viewer-registration is purely an optimization and should never break
|
||||
// the UI. Report success:false but HTTP 200 so the client can decide.
|
||||
res.status(200).json({
|
||||
success: false,
|
||||
skipped: true,
|
||||
reason: "internal_error",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /metrics/unregister-viewer:
|
||||
* post:
|
||||
* summary: Unregister metrics viewer
|
||||
* description: Unregisters a viewer session when they stop viewing metrics for a host.
|
||||
* tags:
|
||||
* - Host Metrics
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* hostId:
|
||||
* type: integer
|
||||
* viewerSessionId:
|
||||
* type: string
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Viewer unregistered successfully.
|
||||
* 400:
|
||||
* description: Invalid hostId or viewerSessionId.
|
||||
* 401:
|
||||
* description: Session expired - please log in again.
|
||||
* 500:
|
||||
* description: Failed to unregister viewer.
|
||||
*/
|
||||
app.post(`/${pathPrefix}/unregister-viewer`, async (req, res) => {
|
||||
const { hostId, viewerSessionId } = req.body;
|
||||
const userId = (req as AuthenticatedRequest).userId;
|
||||
|
||||
if (DataCrypto.getUserDataKey(userId) === null) {
|
||||
return res.status(401).json({
|
||||
error: "Session expired - please log in again",
|
||||
code: "SESSION_EXPIRED",
|
||||
});
|
||||
}
|
||||
|
||||
if (!hostId || typeof hostId !== "number") {
|
||||
return res.status(400).json({ error: "Invalid hostId" });
|
||||
}
|
||||
|
||||
if (!viewerSessionId || typeof viewerSessionId !== "string") {
|
||||
return res.status(400).json({ error: "Invalid viewerSessionId" });
|
||||
}
|
||||
|
||||
try {
|
||||
unregisterViewer(hostId, viewerSessionId);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
statsLogger.error("Failed to unregister viewer", {
|
||||
operation: "unregister_viewer_error",
|
||||
hostId,
|
||||
viewerSessionId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
res.status(500).json({ error: "Failed to unregister viewer" });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,229 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import {
|
||||
execCommand,
|
||||
execPowerShell,
|
||||
toFixedNum,
|
||||
type HostPlatform,
|
||||
} from "./common-utils.js";
|
||||
|
||||
/**
|
||||
* CPU percent is derived from the delta against the previous poll's
|
||||
* /proc/stat sample rather than a same-poll double-read, so a poll no longer
|
||||
* has to block for a fixed settle window. Keyed by host.
|
||||
*/
|
||||
const previousStat = new Map<
|
||||
number,
|
||||
{ total: number; idle: number; timestamp: number }
|
||||
>();
|
||||
|
||||
export function clearCpuSampleCache(hostId?: number): void {
|
||||
if (hostId === undefined) {
|
||||
previousStat.clear();
|
||||
return;
|
||||
}
|
||||
previousStat.delete(hostId);
|
||||
}
|
||||
|
||||
export function parseCpuLine(
|
||||
cpuLine: string,
|
||||
): { total: number; idle: number } | undefined {
|
||||
const parts = cpuLine.trim().split(/\s+/);
|
||||
if (parts[0] !== "cpu") return undefined;
|
||||
const nums = parts
|
||||
.slice(1)
|
||||
.map((n) => Number(n))
|
||||
.filter((n) => Number.isFinite(n));
|
||||
if (nums.length < 4) return undefined;
|
||||
const idle = (nums[3] ?? 0) + (nums[4] ?? 0);
|
||||
const total = nums.reduce((a, b) => a + b, 0);
|
||||
return { total, idle };
|
||||
}
|
||||
|
||||
// Parses macOS `top -l 1 -n 0` output, e.g.
|
||||
// "CPU usage: 12.34% user, 5.67% sys, 81.99% idle"
|
||||
export function parseDarwinCpuUsageLine(
|
||||
line: string,
|
||||
): { percent: number } | undefined {
|
||||
const match = line.match(
|
||||
/([\d.]+)%\s*user,\s*([\d.]+)%\s*sys,\s*([\d.]+)%\s*idle/i,
|
||||
);
|
||||
if (!match) return undefined;
|
||||
const user = Number(match[1]);
|
||||
const sys = Number(match[2]);
|
||||
if (!Number.isFinite(user) || !Number.isFinite(sys)) return undefined;
|
||||
return { percent: Math.max(0, Math.min(100, user + sys)) };
|
||||
}
|
||||
|
||||
async function collectDarwinCpuMetrics(client: Client): Promise<{
|
||||
percent: number | null;
|
||||
cores: number | null;
|
||||
load: [number, number, number] | null;
|
||||
}> {
|
||||
let cpuPercent: number | null = null;
|
||||
let cores: number | null = null;
|
||||
let loadTriplet: [number, number, number] | null = null;
|
||||
|
||||
try {
|
||||
const [topOut, coresOut, loadOut] = await Promise.race([
|
||||
Promise.all([
|
||||
execCommand(client, "top -l 1 -n 0"),
|
||||
execCommand(client, "sysctl -n hw.ncpu"),
|
||||
execCommand(client, "sysctl -n vm.loadavg"),
|
||||
]),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(
|
||||
() => reject(new Error("CPU metrics collection timeout")),
|
||||
25000,
|
||||
),
|
||||
),
|
||||
]);
|
||||
|
||||
const usageLine = topOut.stdout
|
||||
.split("\n")
|
||||
.find((l) => l.includes("CPU usage:"));
|
||||
const parsed = usageLine ? parseDarwinCpuUsageLine(usageLine) : undefined;
|
||||
if (parsed) cpuPercent = parsed.percent;
|
||||
|
||||
const coresNum = Number((coresOut.stdout || "").trim());
|
||||
cores = Number.isFinite(coresNum) && coresNum > 0 ? coresNum : null;
|
||||
|
||||
// sysctl -n vm.loadavg -> "{ 1.23 1.45 1.67 }"
|
||||
const loadMatch = loadOut.stdout.match(/([\d.]+)\s+([\d.]+)\s+([\d.]+)/);
|
||||
if (loadMatch) {
|
||||
loadTriplet = [
|
||||
Number(loadMatch[1]),
|
||||
Number(loadMatch[2]),
|
||||
Number(loadMatch[3]),
|
||||
].map((v) => (Number.isFinite(v) ? Number(v) : 0)) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
}
|
||||
} catch {
|
||||
cpuPercent = null;
|
||||
loadTriplet = null;
|
||||
}
|
||||
|
||||
return {
|
||||
percent: toFixedNum(cpuPercent, 0),
|
||||
cores,
|
||||
load: loadTriplet,
|
||||
};
|
||||
}
|
||||
|
||||
// No POSIX-style load average exists on Windows, so `load` always stays null.
|
||||
async function collectWindowsCpuMetrics(client: Client): Promise<{
|
||||
percent: number | null;
|
||||
cores: number | null;
|
||||
load: [number, number, number] | null;
|
||||
}> {
|
||||
let cpuPercent: number | null = null;
|
||||
let cores: number | null = null;
|
||||
|
||||
try {
|
||||
const { stdout } = await execPowerShell(
|
||||
client,
|
||||
"$cpu=(Get-CimInstance Win32_Processor | Measure-Object -Property LoadPercentage -Average).Average; $cores=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors; [PSCustomObject]@{cpu=$cpu; cores=$cores} | ConvertTo-Json -Compress",
|
||||
25000,
|
||||
);
|
||||
const parsed = JSON.parse(stdout.trim());
|
||||
const cpuNum = Number(parsed?.cpu);
|
||||
const coresNum = Number(parsed?.cores);
|
||||
cpuPercent = Number.isFinite(cpuNum)
|
||||
? Math.max(0, Math.min(100, cpuNum))
|
||||
: null;
|
||||
cores = Number.isFinite(coresNum) && coresNum > 0 ? coresNum : null;
|
||||
} catch {
|
||||
cpuPercent = null;
|
||||
}
|
||||
|
||||
return {
|
||||
percent: toFixedNum(cpuPercent, 0),
|
||||
cores,
|
||||
load: null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectCpuMetrics(
|
||||
client: Client,
|
||||
platform?: HostPlatform,
|
||||
hostId?: number,
|
||||
): Promise<{
|
||||
percent: number | null;
|
||||
cores: number | null;
|
||||
load: [number, number, number] | null;
|
||||
}> {
|
||||
if (platform === "darwin") {
|
||||
return collectDarwinCpuMetrics(client);
|
||||
}
|
||||
if (platform === "windows") {
|
||||
return collectWindowsCpuMetrics(client);
|
||||
}
|
||||
|
||||
let cpuPercent: number | null = null;
|
||||
let cores: number | null = null;
|
||||
let loadTriplet: [number, number, number] | null = null;
|
||||
|
||||
try {
|
||||
const readAt = Date.now();
|
||||
const [statOut, loadAvgOut, coresOut] = await Promise.race([
|
||||
Promise.all([
|
||||
execCommand(client, "cat /proc/stat"),
|
||||
execCommand(client, "cat /proc/loadavg"),
|
||||
execCommand(
|
||||
client,
|
||||
"nproc 2>/dev/null || grep -c ^processor /proc/cpuinfo",
|
||||
),
|
||||
]),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(
|
||||
() => reject(new Error("CPU metrics collection timeout")),
|
||||
25000,
|
||||
),
|
||||
),
|
||||
]);
|
||||
|
||||
const cpuLine = (
|
||||
statOut.stdout.split("\n").find((l) => l.startsWith("cpu ")) || ""
|
||||
).trim();
|
||||
const current = parseCpuLine(cpuLine);
|
||||
const previous =
|
||||
hostId !== undefined ? previousStat.get(hostId) : undefined;
|
||||
if (current && previous) {
|
||||
const totalDiff = current.total - previous.total;
|
||||
const idleDiff = current.idle - previous.idle;
|
||||
const used = totalDiff - idleDiff;
|
||||
if (totalDiff > 0)
|
||||
cpuPercent = Math.max(0, Math.min(100, (used / totalDiff) * 100));
|
||||
}
|
||||
if (current && hostId !== undefined) {
|
||||
previousStat.set(hostId, { ...current, timestamp: readAt });
|
||||
}
|
||||
|
||||
const laParts = loadAvgOut.stdout.trim().split(/\s+/);
|
||||
if (laParts.length >= 3) {
|
||||
loadTriplet = [
|
||||
Number(laParts[0]),
|
||||
Number(laParts[1]),
|
||||
Number(laParts[2]),
|
||||
].map((v) => (Number.isFinite(v) ? Number(v) : 0)) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
}
|
||||
|
||||
const coresNum = Number((coresOut.stdout || "").trim());
|
||||
cores = Number.isFinite(coresNum) && coresNum > 0 ? coresNum : null;
|
||||
} catch {
|
||||
cpuPercent = null;
|
||||
loadTriplet = null;
|
||||
}
|
||||
|
||||
return {
|
||||
percent: toFixedNum(cpuPercent, 0),
|
||||
cores,
|
||||
load: loadTriplet,
|
||||
};
|
||||
}
|
||||
@@ -1,530 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import {
|
||||
execCommand,
|
||||
execPowerShell,
|
||||
toFixedNum,
|
||||
type HostPlatform,
|
||||
} from "./common-utils.js";
|
||||
|
||||
const PSEUDO_FS_RE = /^(tmpfs|devtmpfs|overlay|udev|none|shm)$/;
|
||||
|
||||
export interface DfRow {
|
||||
filesystem: string;
|
||||
type: string;
|
||||
mount: string;
|
||||
parts: string[];
|
||||
}
|
||||
|
||||
export interface DiskFilesystem {
|
||||
filesystem: string;
|
||||
type: string;
|
||||
mount: string;
|
||||
percent: number | null;
|
||||
usedHuman: string | null;
|
||||
totalHuman: string | null;
|
||||
availableHuman: string | null;
|
||||
usedBytes: number | null;
|
||||
totalBytes: number | null;
|
||||
availableBytes: number | null;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface MonitoredMount {
|
||||
path: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
// Parses `df -T -P`-style output: Filesystem, Type, then the size columns,
|
||||
// with Mounted-on last. The Type column (e.g. ext4, nfs4, cifs) is what lets
|
||||
// callers filter network shares out by filesystem type rather than guessing
|
||||
// from the source path.
|
||||
export function parseDfLines(output: string): DfRow[] {
|
||||
return output
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const parts = line.split(/\s+/);
|
||||
return {
|
||||
filesystem: parts[0] || "",
|
||||
type: parts[1] || "",
|
||||
mount: parts[6] || "",
|
||||
parts,
|
||||
};
|
||||
})
|
||||
.filter((row) => row.parts.length >= 7 && !PSEUDO_FS_RE.test(row.type));
|
||||
}
|
||||
|
||||
// Finds the index of the most-utilized real filesystem in a GNU `df -TkP`-style
|
||||
// row set (parts[2] = total KB, parts[3] = used KB), so a nearly-full
|
||||
// secondary mount (e.g. /data) isn't hidden behind a healthy root filesystem.
|
||||
export function findWorstMountIndex(bytesRows: DfRow[]): {
|
||||
index: number;
|
||||
usedBytes: number;
|
||||
totalBytes: number;
|
||||
} {
|
||||
let worstIndex = -1;
|
||||
let worstUsedBytes = -1;
|
||||
let worstTotalBytes = 0;
|
||||
|
||||
bytesRows.forEach((row, index) => {
|
||||
const totalBytes = Number(row.parts[2]);
|
||||
const usedBytes = Number(row.parts[3]);
|
||||
if (
|
||||
!Number.isFinite(totalBytes) ||
|
||||
!Number.isFinite(usedBytes) ||
|
||||
totalBytes <= 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const usedRatio = usedBytes / totalBytes;
|
||||
const worstRatio =
|
||||
worstTotalBytes > 0 ? worstUsedBytes / worstTotalBytes : -1;
|
||||
if (usedRatio > worstRatio) {
|
||||
worstIndex = index;
|
||||
worstUsedBytes = usedBytes;
|
||||
worstTotalBytes = totalBytes;
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
index: worstIndex,
|
||||
usedBytes: worstUsedBytes,
|
||||
totalBytes: worstTotalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
// Merges a GNU `df -TkP` size-row set (parts[2..4] in `blockSizeBytes` units)
|
||||
// and a `df -hT -P` human-row set into one filesystem list. Byte rows drive
|
||||
// the maths; human rows only supply the display strings, matched by mount
|
||||
// point so a mismatched row count can't shift the columns.
|
||||
export function buildFilesystemList(
|
||||
bytesRows: DfRow[],
|
||||
humanRows: DfRow[],
|
||||
blockSizeBytes = 1,
|
||||
): DiskFilesystem[] {
|
||||
const aligned = humanRows.length === bytesRows.length;
|
||||
|
||||
return bytesRows
|
||||
.map((row, index) => {
|
||||
const totalBytes = Number(row.parts[2]) * blockSizeBytes;
|
||||
const usedBytes = Number(row.parts[3]) * blockSizeBytes;
|
||||
const availableBytes = Number(row.parts[4]) * blockSizeBytes;
|
||||
if (!Number.isFinite(totalBytes) || totalBytes <= 0) return null;
|
||||
|
||||
const humanRow = aligned
|
||||
? humanRows[index]
|
||||
: humanRows.find((h) => h.mount === row.mount);
|
||||
|
||||
const percent = Number.isFinite(usedBytes)
|
||||
? Math.max(0, Math.min(100, (usedBytes / totalBytes) * 100))
|
||||
: null;
|
||||
|
||||
return {
|
||||
filesystem: row.filesystem,
|
||||
type: row.type,
|
||||
mount: row.mount,
|
||||
percent: toFixedNum(percent, 0),
|
||||
usedHuman: humanRow?.parts[3] || null,
|
||||
totalHuman: humanRow?.parts[2] || null,
|
||||
availableHuman: humanRow?.parts[4] || null,
|
||||
usedBytes: Number.isFinite(usedBytes) ? usedBytes : null,
|
||||
totalBytes,
|
||||
availableBytes: Number.isFinite(availableBytes) ? availableBytes : null,
|
||||
};
|
||||
})
|
||||
.filter((fs): fs is DiskFilesystem => fs !== null);
|
||||
}
|
||||
|
||||
// The headline disk figure should be the root filesystem - that is what users
|
||||
// mean by "the server's disk". Only when there is no root mount (containers,
|
||||
// chroots) do we fall back to the most-utilized mount.
|
||||
export function selectPrimaryFilesystem(
|
||||
filesystems: DiskFilesystem[],
|
||||
): DiskFilesystem | null {
|
||||
if (filesystems.length === 0) return null;
|
||||
|
||||
const root = filesystems.find((fs) => fs.mount === "/");
|
||||
if (root) return root;
|
||||
|
||||
let best = filesystems[0];
|
||||
for (const fs of filesystems) {
|
||||
const ratio = (fs.usedBytes ?? 0) / (fs.totalBytes || 1);
|
||||
const bestRatio = (best.usedBytes ?? 0) / (best.totalBytes || 1);
|
||||
if (ratio > bestRatio) best = fs;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// Excluded mounts are user-configured per host: an exact mount-path match
|
||||
// (e.g. "/mnt/nas") or a filesystem-type substring match (e.g. "nfs" matches
|
||||
// nfs/nfs4, "cifs" matches cifs/smb3), so network shares can be dropped from
|
||||
// the headline percent and the filesystem list without hiding local disks.
|
||||
export function filterExcludedFilesystems(
|
||||
filesystems: DiskFilesystem[],
|
||||
excludedMounts?: string[] | null,
|
||||
): DiskFilesystem[] {
|
||||
if (!excludedMounts || excludedMounts.length === 0) return filesystems;
|
||||
|
||||
const normalized = excludedMounts
|
||||
.map((entry) => entry.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
if (normalized.length === 0) return filesystems;
|
||||
|
||||
return filesystems.filter((fs) => {
|
||||
const mount = fs.mount.toLowerCase();
|
||||
const type = fs.type.toLowerCase();
|
||||
return !normalized.some(
|
||||
(entry) => mount === entry || (type && type.includes(entry)),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function mergeMonitoredFilesystems(
|
||||
filesystems: DiskFilesystem[],
|
||||
monitored: MonitoredMount[],
|
||||
customFilesystems: Array<DiskFilesystem | null>,
|
||||
): DiskFilesystem[] {
|
||||
const result = [...filesystems];
|
||||
monitored.forEach((entry, index) => {
|
||||
const path = entry.path.trim();
|
||||
const custom = customFilesystems[index];
|
||||
if (!path || !custom) return;
|
||||
|
||||
const existing = result.find((fs) => fs.mount === path);
|
||||
if (existing) {
|
||||
existing.label = entry.label?.trim() || undefined;
|
||||
return;
|
||||
}
|
||||
result.push({
|
||||
...custom,
|
||||
mount: path,
|
||||
label: entry.label?.trim() || undefined,
|
||||
});
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
function humanizeBytes(bytes: number): string {
|
||||
const units = ["B", "K", "M", "G", "T", "P"];
|
||||
let value = Math.max(0, bytes);
|
||||
let unitIndex = 0;
|
||||
while (value >= 1024 && unitIndex < units.length - 1) {
|
||||
value /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
const rounded =
|
||||
unitIndex > 0 && value < 10 ? value.toFixed(1) : Math.round(value);
|
||||
return `${rounded}${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
// BSD/macOS `df -T` takes a filesystem-type argument to *filter* by, unlike
|
||||
// GNU `df -T` which *prints* a type column - so macOS needs its own command
|
||||
// (`df -Pk`, no type column) and gets the type separately from `mount`.
|
||||
const DARWIN_SKIP_MOUNT_RE =
|
||||
/^\/System\/Volumes\/(VM|Preboot|Update|xarts|iSCPreboot|Hardware)(\/|$)/;
|
||||
const DARWIN_SKIP_TYPE_RE = /^(devfs|autofs)$/;
|
||||
|
||||
export function parseDarwinMountTypes(output: string): Map<string, string> {
|
||||
const map = new Map<string, string>();
|
||||
for (const line of output.split("\n")) {
|
||||
// Device names can contain spaces (e.g. autofs's "map auto_home"), so
|
||||
// match the literal " on " / " (" separators rather than a single token.
|
||||
const match = line.match(/^.+? on (.+?) \(([^,)]+)/);
|
||||
if (match) map.set(match[1], match[2].trim());
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
export function parseDarwinDfRows(
|
||||
output: string,
|
||||
typeByMount: Map<string, string>,
|
||||
): DiskFilesystem[] {
|
||||
return output
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length < 6) return null;
|
||||
|
||||
const filesystem = parts[0];
|
||||
const totalKb = Number(parts[1]);
|
||||
const usedKb = Number(parts[2]);
|
||||
const availableKb = Number(parts[3]);
|
||||
const mount = parts.slice(5).join(" ");
|
||||
if (!Number.isFinite(totalKb) || totalKb <= 0) return null;
|
||||
if (DARWIN_SKIP_MOUNT_RE.test(mount)) return null;
|
||||
|
||||
const type = typeByMount.get(mount) || "";
|
||||
if (DARWIN_SKIP_TYPE_RE.test(type)) return null;
|
||||
|
||||
const totalBytes = totalKb * 1024;
|
||||
const usedBytes = Number.isFinite(usedKb) ? usedKb * 1024 : null;
|
||||
const availableBytes = Number.isFinite(availableKb)
|
||||
? availableKb * 1024
|
||||
: null;
|
||||
const percent =
|
||||
usedBytes !== null
|
||||
? Math.max(0, Math.min(100, (usedBytes / totalBytes) * 100))
|
||||
: null;
|
||||
|
||||
return {
|
||||
filesystem,
|
||||
type,
|
||||
mount,
|
||||
percent: toFixedNum(percent, 0),
|
||||
usedHuman: usedBytes !== null ? humanizeBytes(usedBytes) : null,
|
||||
totalHuman: humanizeBytes(totalBytes),
|
||||
availableHuman:
|
||||
availableBytes !== null ? humanizeBytes(availableBytes) : null,
|
||||
usedBytes,
|
||||
totalBytes,
|
||||
availableBytes,
|
||||
};
|
||||
})
|
||||
.filter((fs): fs is DiskFilesystem => fs !== null);
|
||||
}
|
||||
|
||||
async function collectDarwinDiskMetrics(
|
||||
client: Client,
|
||||
excludedMounts?: string[] | null,
|
||||
monitoredMounts?: MonitoredMount[] | null,
|
||||
): Promise<{
|
||||
percent: number | null;
|
||||
usedHuman: string | null;
|
||||
totalHuman: string | null;
|
||||
availableHuman: string | null;
|
||||
mount: string | null;
|
||||
filesystems: DiskFilesystem[];
|
||||
}> {
|
||||
try {
|
||||
const [dfOut, mountOut] = await Promise.all([
|
||||
execCommand(client, "df -Pk | tail -n +2"),
|
||||
execCommand(client, "mount"),
|
||||
]);
|
||||
|
||||
const typeByMount = parseDarwinMountTypes(mountOut.stdout);
|
||||
let detected = parseDarwinDfRows(dfOut.stdout, typeByMount);
|
||||
|
||||
const monitored = (monitoredMounts ?? []).filter((entry) =>
|
||||
Boolean(entry.path.trim()),
|
||||
);
|
||||
if (monitored.length > 0) {
|
||||
const customFilesystems = await Promise.all(
|
||||
monitored.map(async (entry) => {
|
||||
const path = shellQuote(entry.path.trim());
|
||||
try {
|
||||
const custom = await execCommand(
|
||||
client,
|
||||
`df -Pk -- ${path} | tail -n +2`,
|
||||
);
|
||||
return parseDarwinDfRows(custom.stdout, typeByMount)[0] ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
detected = mergeMonitoredFilesystems(
|
||||
detected,
|
||||
monitored,
|
||||
customFilesystems,
|
||||
);
|
||||
}
|
||||
|
||||
const filesystems = filterExcludedFilesystems(detected, excludedMounts);
|
||||
const primary = selectPrimaryFilesystem(filesystems);
|
||||
|
||||
return {
|
||||
percent: primary?.percent ?? null,
|
||||
usedHuman: primary?.usedHuman ?? null,
|
||||
totalHuman: primary?.totalHuman ?? null,
|
||||
availableHuman: primary?.availableHuman ?? null,
|
||||
mount: primary?.mount ?? null,
|
||||
filesystems,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
percent: null,
|
||||
usedHuman: null,
|
||||
totalHuman: null,
|
||||
availableHuman: null,
|
||||
mount: null,
|
||||
filesystems: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
interface WindowsDiskRow {
|
||||
drive: string;
|
||||
total: number;
|
||||
free: number;
|
||||
}
|
||||
|
||||
const WINDOWS_DISK_SCRIPT =
|
||||
'Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" | ForEach-Object {' +
|
||||
" [PSCustomObject]@{drive=$_.DeviceID; total=$_.Size; free=$_.FreeSpace}" +
|
||||
" } | ConvertTo-Json -Compress";
|
||||
|
||||
export function parseWindowsDiskJson(output: string): WindowsDiskRow[] {
|
||||
const trimmed = output.trim();
|
||||
if (!trimmed) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
const rows = Array.isArray(parsed) ? parsed : [parsed];
|
||||
return rows
|
||||
.filter((row): row is Record<string, unknown> => Boolean(row?.drive))
|
||||
.map((row) => ({
|
||||
drive: String(row.drive),
|
||||
total: Number(row.total),
|
||||
free: Number(row.free),
|
||||
}))
|
||||
.filter((row) => Number.isFinite(row.total) && row.total > 0);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function collectWindowsDiskMetrics(
|
||||
client: Client,
|
||||
excludedMounts?: string[] | null,
|
||||
): Promise<{
|
||||
percent: number | null;
|
||||
usedHuman: string | null;
|
||||
totalHuman: string | null;
|
||||
availableHuman: string | null;
|
||||
mount: string | null;
|
||||
filesystems: DiskFilesystem[];
|
||||
}> {
|
||||
try {
|
||||
const { stdout } = await execPowerShell(client, WINDOWS_DISK_SCRIPT);
|
||||
const rows = parseWindowsDiskJson(stdout);
|
||||
|
||||
const filesystems: DiskFilesystem[] = rows.map((row) => {
|
||||
const usedBytes = row.total - row.free;
|
||||
const percent = Math.max(0, Math.min(100, (usedBytes / row.total) * 100));
|
||||
return {
|
||||
filesystem: row.drive,
|
||||
type: "NTFS",
|
||||
mount: row.drive,
|
||||
percent: toFixedNum(percent, 0),
|
||||
usedHuman: humanizeBytes(usedBytes),
|
||||
totalHuman: humanizeBytes(row.total),
|
||||
availableHuman: humanizeBytes(row.free),
|
||||
usedBytes,
|
||||
totalBytes: row.total,
|
||||
availableBytes: row.free,
|
||||
};
|
||||
});
|
||||
|
||||
const filtered = filterExcludedFilesystems(filesystems, excludedMounts);
|
||||
const primary =
|
||||
filtered.find((fs) => fs.mount.toUpperCase().startsWith("C")) ??
|
||||
filtered[0] ??
|
||||
null;
|
||||
|
||||
return {
|
||||
percent: primary?.percent ?? null,
|
||||
usedHuman: primary?.usedHuman ?? null,
|
||||
totalHuman: primary?.totalHuman ?? null,
|
||||
availableHuman: primary?.availableHuman ?? null,
|
||||
mount: primary?.mount ?? null,
|
||||
filesystems: filtered,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
percent: null,
|
||||
usedHuman: null,
|
||||
totalHuman: null,
|
||||
availableHuman: null,
|
||||
mount: null,
|
||||
filesystems: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function collectDiskMetrics(
|
||||
client: Client,
|
||||
excludedMounts?: string[] | null,
|
||||
monitoredMounts?: MonitoredMount[] | null,
|
||||
platform?: HostPlatform,
|
||||
): Promise<{
|
||||
percent: number | null;
|
||||
usedHuman: string | null;
|
||||
totalHuman: string | null;
|
||||
availableHuman: string | null;
|
||||
mount: string | null;
|
||||
filesystems: DiskFilesystem[];
|
||||
}> {
|
||||
if (platform === "windows") {
|
||||
return collectWindowsDiskMetrics(client, excludedMounts);
|
||||
}
|
||||
if (platform === "darwin") {
|
||||
return collectDarwinDiskMetrics(client, excludedMounts, monitoredMounts);
|
||||
}
|
||||
|
||||
try {
|
||||
const [diskOutHuman, diskOutBytes] = await Promise.all([
|
||||
execCommand(client, "df -hT -P | tail -n +2"),
|
||||
execCommand(client, "df -TkP | tail -n +2"),
|
||||
]);
|
||||
|
||||
const humanRows = parseDfLines(diskOutHuman.stdout);
|
||||
const bytesRows = parseDfLines(diskOutBytes.stdout);
|
||||
let detected = buildFilesystemList(bytesRows, humanRows, 1024);
|
||||
const monitored = (monitoredMounts ?? []).filter((entry) =>
|
||||
Boolean(entry.path.trim()),
|
||||
);
|
||||
if (monitored.length > 0) {
|
||||
const customFilesystems = await Promise.all(
|
||||
monitored.map(async (entry) => {
|
||||
const path = shellQuote(entry.path.trim());
|
||||
try {
|
||||
const [customHuman, customBytes] = await Promise.all([
|
||||
execCommand(client, `df -hT -P -- ${path} | tail -n +2`),
|
||||
execCommand(client, `df -TkP -- ${path} | tail -n +2`),
|
||||
]);
|
||||
return (
|
||||
buildFilesystemList(
|
||||
parseDfLines(customBytes.stdout),
|
||||
parseDfLines(customHuman.stdout),
|
||||
1024,
|
||||
)[0] ?? null
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
detected = mergeMonitoredFilesystems(
|
||||
detected,
|
||||
monitored,
|
||||
customFilesystems,
|
||||
);
|
||||
}
|
||||
const filesystems = filterExcludedFilesystems(detected, excludedMounts);
|
||||
const primary = selectPrimaryFilesystem(filesystems);
|
||||
|
||||
return {
|
||||
percent: primary?.percent ?? null,
|
||||
usedHuman: primary?.usedHuman ?? null,
|
||||
totalHuman: primary?.totalHuman ?? null,
|
||||
availableHuman: primary?.availableHuman ?? null,
|
||||
mount: primary?.mount ?? null,
|
||||
filesystems,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
percent: null,
|
||||
usedHuman: null,
|
||||
totalHuman: null,
|
||||
availableHuman: null,
|
||||
mount: null,
|
||||
filesystems: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
import type {
|
||||
FirewallMetrics,
|
||||
FirewallChain,
|
||||
FirewallRule,
|
||||
} from "../../../../types/stats-widgets.js";
|
||||
|
||||
function parseIptablesRule(line: string): FirewallRule | null {
|
||||
if (!line.startsWith("-A ")) return null;
|
||||
|
||||
const rule: FirewallRule = {
|
||||
chain: "",
|
||||
target: "",
|
||||
protocol: "all",
|
||||
source: "0.0.0.0/0",
|
||||
destination: "0.0.0.0/0",
|
||||
};
|
||||
|
||||
const chainMatch = line.match(/^-A\s+(\S+)/);
|
||||
if (chainMatch) {
|
||||
rule.chain = chainMatch[1];
|
||||
}
|
||||
|
||||
const targetMatch = line.match(/-j\s+(\S+)/);
|
||||
if (targetMatch) {
|
||||
rule.target = targetMatch[1];
|
||||
}
|
||||
|
||||
const protocolMatch = line.match(/-p\s+(\S+)/);
|
||||
if (protocolMatch) {
|
||||
rule.protocol = protocolMatch[1];
|
||||
}
|
||||
|
||||
const sourceMatch = line.match(/-s\s+(\S+)/);
|
||||
if (sourceMatch) {
|
||||
rule.source = sourceMatch[1];
|
||||
}
|
||||
|
||||
const destMatch = line.match(/-d\s+(\S+)/);
|
||||
if (destMatch) {
|
||||
rule.destination = destMatch[1];
|
||||
}
|
||||
|
||||
const dportMatch = line.match(/--dport\s+(\S+)/);
|
||||
if (dportMatch) {
|
||||
rule.dport = dportMatch[1];
|
||||
}
|
||||
|
||||
const sportMatch = line.match(/--sport\s+(\S+)/);
|
||||
if (sportMatch) {
|
||||
rule.sport = sportMatch[1];
|
||||
}
|
||||
|
||||
const stateMatch = line.match(/--state\s+(\S+)/);
|
||||
if (stateMatch) {
|
||||
rule.state = stateMatch[1];
|
||||
}
|
||||
|
||||
const interfaceMatch = line.match(/-i\s+(\S+)/);
|
||||
if (interfaceMatch) {
|
||||
rule.interface = interfaceMatch[1];
|
||||
}
|
||||
|
||||
return rule;
|
||||
}
|
||||
|
||||
function parseIptablesOutput(output: string): FirewallChain[] {
|
||||
const chains: Map<string, FirewallChain> = new Map();
|
||||
const lines = output.split("\n");
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
const policyMatch = trimmed.match(/^:(\S+)\s+(\S+)/);
|
||||
if (policyMatch) {
|
||||
const [, chainName, policy] = policyMatch;
|
||||
chains.set(chainName, {
|
||||
name: chainName,
|
||||
policy: policy,
|
||||
rules: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const rule = parseIptablesRule(trimmed);
|
||||
if (rule) {
|
||||
let chain = chains.get(rule.chain);
|
||||
if (!chain) {
|
||||
chain = {
|
||||
name: rule.chain,
|
||||
policy: "ACCEPT",
|
||||
rules: [],
|
||||
};
|
||||
chains.set(rule.chain, chain);
|
||||
}
|
||||
chain.rules.push(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(chains.values());
|
||||
}
|
||||
|
||||
function parseNftablesOutput(output: string): FirewallChain[] {
|
||||
const chains: FirewallChain[] = [];
|
||||
let currentChain: FirewallChain | null = null;
|
||||
|
||||
const lines = output.split("\n");
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
const chainMatch = trimmed.match(
|
||||
/chain\s+(\S+)\s*\{?\s*(?:type\s+\S+\s+hook\s+(\S+))?/,
|
||||
);
|
||||
if (chainMatch) {
|
||||
if (currentChain) {
|
||||
chains.push(currentChain);
|
||||
}
|
||||
currentChain = {
|
||||
name: chainMatch[1].toUpperCase(),
|
||||
policy: "ACCEPT",
|
||||
rules: [],
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentChain && trimmed.startsWith("policy ")) {
|
||||
const policyMatch = trimmed.match(/policy\s+(\S+)/);
|
||||
if (policyMatch) {
|
||||
currentChain.policy = policyMatch[1].toUpperCase();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentChain && trimmed && !trimmed.startsWith("}")) {
|
||||
const rule: FirewallRule = {
|
||||
chain: currentChain.name,
|
||||
target: "",
|
||||
protocol: "all",
|
||||
source: "0.0.0.0/0",
|
||||
destination: "0.0.0.0/0",
|
||||
};
|
||||
|
||||
if (trimmed.includes("accept")) rule.target = "ACCEPT";
|
||||
else if (trimmed.includes("drop")) rule.target = "DROP";
|
||||
else if (trimmed.includes("reject")) rule.target = "REJECT";
|
||||
|
||||
const tcpMatch = trimmed.match(/tcp\s+dport\s+(\S+)/);
|
||||
if (tcpMatch) {
|
||||
rule.protocol = "tcp";
|
||||
rule.dport = tcpMatch[1];
|
||||
}
|
||||
|
||||
const udpMatch = trimmed.match(/udp\s+dport\s+(\S+)/);
|
||||
if (udpMatch) {
|
||||
rule.protocol = "udp";
|
||||
rule.dport = udpMatch[1];
|
||||
}
|
||||
|
||||
const saddrMatch = trimmed.match(/saddr\s+(\S+)/);
|
||||
if (saddrMatch) {
|
||||
rule.source = saddrMatch[1];
|
||||
}
|
||||
|
||||
const daddrMatch = trimmed.match(/daddr\s+(\S+)/);
|
||||
if (daddrMatch) {
|
||||
rule.destination = daddrMatch[1];
|
||||
}
|
||||
|
||||
const iifMatch = trimmed.match(/iif\s+"?(\S+)"?/);
|
||||
if (iifMatch) {
|
||||
rule.interface = iifMatch[1].replace(/"/g, "");
|
||||
}
|
||||
|
||||
const ctStateMatch = trimmed.match(/ct\s+state\s+(\S+)/);
|
||||
if (ctStateMatch) {
|
||||
rule.state = ctStateMatch[1].toUpperCase();
|
||||
}
|
||||
|
||||
if (rule.target) {
|
||||
currentChain.rules.push(rule);
|
||||
}
|
||||
}
|
||||
|
||||
if (trimmed === "}") {
|
||||
if (currentChain) {
|
||||
chains.push(currentChain);
|
||||
currentChain = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (currentChain) {
|
||||
chains.push(currentChain);
|
||||
}
|
||||
|
||||
return chains;
|
||||
}
|
||||
|
||||
export async function collectFirewallMetrics(
|
||||
client: Client,
|
||||
): Promise<FirewallMetrics> {
|
||||
try {
|
||||
const iptablesResult = await execCommand(
|
||||
client,
|
||||
"iptables-save 2>/dev/null",
|
||||
15000,
|
||||
);
|
||||
|
||||
if (iptablesResult.stdout && iptablesResult.stdout.includes("*filter")) {
|
||||
const chains = parseIptablesOutput(iptablesResult.stdout);
|
||||
const hasRules = chains.some((c) => c.rules.length > 0);
|
||||
|
||||
return {
|
||||
type: "iptables",
|
||||
status: hasRules ? "active" : "inactive",
|
||||
chains: chains.filter(
|
||||
(c) =>
|
||||
c.name === "INPUT" || c.name === "OUTPUT" || c.name === "FORWARD",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const nftResult = await execCommand(
|
||||
client,
|
||||
"nft list ruleset 2>/dev/null",
|
||||
15000,
|
||||
);
|
||||
|
||||
if (nftResult.stdout && nftResult.stdout.trim()) {
|
||||
const chains = parseNftablesOutput(nftResult.stdout);
|
||||
const hasRules = chains.some((c) => c.rules.length > 0);
|
||||
|
||||
return {
|
||||
type: "nftables",
|
||||
status: hasRules ? "active" : "inactive",
|
||||
chains,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: "none",
|
||||
status: "unknown",
|
||||
chains: [],
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
type: "none",
|
||||
status: "unknown",
|
||||
chains: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "./common-utils.js";
|
||||
|
||||
export interface GpuDevice {
|
||||
index: number;
|
||||
uuid: string;
|
||||
name: string;
|
||||
driverVersion: string | null;
|
||||
utilizationPercent: number | null;
|
||||
memoryUsedMiB: number | null;
|
||||
memoryTotalMiB: number | null;
|
||||
memoryPercent: number | null;
|
||||
temperatureCelsius: number | null;
|
||||
powerDrawWatts: number | null;
|
||||
powerLimitWatts: number | null;
|
||||
fanPercent: number | null;
|
||||
}
|
||||
|
||||
export interface GpuProcess {
|
||||
/** Null when nvidia-smi reports a uuid that matches none of the listed GPUs. */
|
||||
gpuIndex: number | null;
|
||||
pid: number;
|
||||
name: string;
|
||||
memoryUsedMiB: number | null;
|
||||
}
|
||||
|
||||
export interface GpuMetrics {
|
||||
source: "nvidia-smi" | "none";
|
||||
gpus: GpuDevice[];
|
||||
processes: GpuProcess[];
|
||||
}
|
||||
|
||||
const SECTION_SEPARATOR = "---";
|
||||
|
||||
// The free-text name is queried last so commas inside it cannot shift the
|
||||
// numeric columns; the parsers rejoin everything past the fixed fields.
|
||||
const GPU_QUERY_FIELDS = [
|
||||
"index",
|
||||
"uuid",
|
||||
"driver_version",
|
||||
"utilization.gpu",
|
||||
"memory.used",
|
||||
"memory.total",
|
||||
"temperature.gpu",
|
||||
"power.draw",
|
||||
"power.limit",
|
||||
"fan.speed",
|
||||
"name",
|
||||
];
|
||||
const APPS_QUERY_FIELDS = ["gpu_uuid", "pid", "used_memory", "process_name"];
|
||||
|
||||
const NVIDIA_SMI_COMMAND = [
|
||||
"command -v nvidia-smi >/dev/null 2>&1 || exit 0;",
|
||||
`nvidia-smi --query-gpu=${GPU_QUERY_FIELDS.join(",")} --format=csv,noheader,nounits 2>/dev/null;`,
|
||||
`echo ${SECTION_SEPARATOR};`,
|
||||
`nvidia-smi --query-compute-apps=${APPS_QUERY_FIELDS.join(",")} --format=csv,noheader,nounits 2>/dev/null`,
|
||||
].join(" ");
|
||||
|
||||
const NO_GPU: GpuMetrics = { source: "none", gpus: [], processes: [] };
|
||||
|
||||
/** Reads a numeric field; nvidia-smi writes "[N/A]", "[Not Supported]" etc. for missing ones. */
|
||||
function parseReading(raw: string | undefined, digits = 1): number | null {
|
||||
const value = raw?.trim();
|
||||
if (!value || value.startsWith("[")) return null;
|
||||
return toFixedNum(Number(value), digits);
|
||||
}
|
||||
|
||||
function parseInteger(raw: string | undefined): number | null {
|
||||
const value = raw?.trim();
|
||||
return value && /^\d+$/.test(value) ? Number(value) : null;
|
||||
}
|
||||
|
||||
function splitCsvRow(line: string, fixedFields: number): string[] | null {
|
||||
const parts = line.split(",").map((part) => part.trim());
|
||||
if (parts.length <= fixedFields) return null;
|
||||
return [...parts.slice(0, fixedFields), parts.slice(fixedFields).join(", ")];
|
||||
}
|
||||
|
||||
export function parseNvidiaSmiGpus(output: string): GpuDevice[] {
|
||||
const fixedFields = GPU_QUERY_FIELDS.length - 1;
|
||||
const gpus: GpuDevice[] = [];
|
||||
|
||||
for (const line of output.split("\n")) {
|
||||
const row = splitCsvRow(line, fixedFields);
|
||||
if (!row) continue;
|
||||
|
||||
const [
|
||||
rawIndex,
|
||||
uuid,
|
||||
driverVersion,
|
||||
utilization,
|
||||
memoryUsed,
|
||||
memoryTotal,
|
||||
temperature,
|
||||
powerDraw,
|
||||
powerLimit,
|
||||
fan,
|
||||
name,
|
||||
] = row;
|
||||
const index = parseInteger(rawIndex);
|
||||
if (index === null || !uuid) continue;
|
||||
|
||||
const memoryUsedMiB = parseReading(memoryUsed, 0);
|
||||
const memoryTotalMiB = parseReading(memoryTotal, 0);
|
||||
|
||||
gpus.push({
|
||||
index,
|
||||
uuid,
|
||||
name,
|
||||
driverVersion:
|
||||
driverVersion && !driverVersion.startsWith("[") ? driverVersion : null,
|
||||
utilizationPercent: parseReading(utilization),
|
||||
memoryUsedMiB,
|
||||
memoryTotalMiB,
|
||||
memoryPercent:
|
||||
memoryUsedMiB !== null &&
|
||||
memoryTotalMiB !== null &&
|
||||
memoryTotalMiB !== 0
|
||||
? toFixedNum((memoryUsedMiB / memoryTotalMiB) * 100, 1)
|
||||
: null,
|
||||
temperatureCelsius: parseReading(temperature),
|
||||
powerDrawWatts: parseReading(powerDraw),
|
||||
powerLimitWatts: parseReading(powerLimit),
|
||||
fanPercent: parseReading(fan),
|
||||
});
|
||||
}
|
||||
|
||||
return gpus;
|
||||
}
|
||||
|
||||
export function parseNvidiaSmiProcesses(
|
||||
output: string,
|
||||
gpus: GpuDevice[],
|
||||
): GpuProcess[] {
|
||||
const indexByUuid = new Map(gpus.map((gpu) => [gpu.uuid, gpu.index]));
|
||||
const processes: GpuProcess[] = [];
|
||||
|
||||
for (const line of output.split("\n")) {
|
||||
const row = splitCsvRow(line, APPS_QUERY_FIELDS.length - 1);
|
||||
if (!row) continue;
|
||||
|
||||
const [uuid, rawPid, memoryUsed, name] = row;
|
||||
const pid = parseInteger(rawPid);
|
||||
if (pid === null) continue;
|
||||
|
||||
processes.push({
|
||||
gpuIndex: indexByUuid.get(uuid) ?? null,
|
||||
pid,
|
||||
name,
|
||||
memoryUsedMiB: parseReading(memoryUsed, 0),
|
||||
});
|
||||
}
|
||||
|
||||
return processes;
|
||||
}
|
||||
|
||||
export function parseNvidiaSmiOutput(stdout: string): GpuMetrics {
|
||||
const lines = stdout.split("\n");
|
||||
const separator = lines.findIndex(
|
||||
(line) => line.trim() === SECTION_SEPARATOR,
|
||||
);
|
||||
const gpuSection =
|
||||
separator === -1 ? stdout : lines.slice(0, separator).join("\n");
|
||||
const appsSection =
|
||||
separator === -1 ? "" : lines.slice(separator + 1).join("\n");
|
||||
|
||||
const gpus = parseNvidiaSmiGpus(gpuSection);
|
||||
if (gpus.length === 0) return NO_GPU;
|
||||
|
||||
return {
|
||||
source: "nvidia-smi",
|
||||
gpus,
|
||||
processes: parseNvidiaSmiProcesses(appsSection, gpus),
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectGpuMetrics(client: Client): Promise<GpuMetrics> {
|
||||
try {
|
||||
// nvidia-smi can take a few seconds on hosts without persistence mode.
|
||||
const result = await execCommand(client, NVIDIA_SMI_COMMAND, 15000);
|
||||
return parseNvidiaSmiOutput(result.stdout);
|
||||
} catch {
|
||||
// expected when the host has no NVIDIA driver or the query times out
|
||||
return NO_GPU;
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export interface LoginRecord {
|
||||
user: string;
|
||||
ip: string;
|
||||
time: string;
|
||||
status: "success" | "failed";
|
||||
}
|
||||
|
||||
export interface LoginStats {
|
||||
recentLogins: LoginRecord[];
|
||||
failedLogins: LoginRecord[];
|
||||
totalLogins: number;
|
||||
uniqueIPs: number;
|
||||
}
|
||||
|
||||
export async function collectLoginStats(client: Client): Promise<LoginStats> {
|
||||
const recentLogins: LoginRecord[] = [];
|
||||
const failedLogins: LoginRecord[] = [];
|
||||
const ipSet = new Set<string>();
|
||||
|
||||
try {
|
||||
const lastOut = await execCommand(
|
||||
client,
|
||||
"last -n 20 -F -w | grep -v 'reboot' | grep -v 'wtmp' | head -20",
|
||||
);
|
||||
|
||||
const lastLines = lastOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const line of lastLines) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 10) {
|
||||
const user = parts[0];
|
||||
const tty = parts[1];
|
||||
const ip =
|
||||
parts[2] === ":" || parts[2].startsWith(":") ? "local" : parts[2];
|
||||
|
||||
const timeStart = parts.indexOf(
|
||||
parts.find((p) => /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)/.test(p)) || "",
|
||||
);
|
||||
if (timeStart > 0 && parts.length > timeStart + 4) {
|
||||
const timeStr = parts.slice(timeStart, timeStart + 5).join(" ");
|
||||
|
||||
if (user && user !== "wtmp" && tty !== "system") {
|
||||
let parsedTime: string;
|
||||
try {
|
||||
const date = new Date(timeStr);
|
||||
parsedTime = isNaN(date.getTime())
|
||||
? timeStr || "unknown"
|
||||
: date.toISOString();
|
||||
} catch {
|
||||
parsedTime = timeStr || "unknown";
|
||||
}
|
||||
|
||||
recentLogins.push({
|
||||
user,
|
||||
ip,
|
||||
time: parsedTime,
|
||||
status: "success",
|
||||
});
|
||||
if (ip !== "local") {
|
||||
ipSet.add(ip);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
try {
|
||||
const failedOut = await execCommand(
|
||||
client,
|
||||
"grep 'Failed password' /var/log/auth.log 2>/dev/null | tail -10 || grep 'authentication failure' /var/log/secure 2>/dev/null | tail -10 || echo ''",
|
||||
);
|
||||
|
||||
const failedLines = failedOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const line of failedLines) {
|
||||
let user = "unknown";
|
||||
let ip = "unknown";
|
||||
let timeStr = "";
|
||||
|
||||
const userMatch = line.match(/for (?:invalid user )?(\S+)/);
|
||||
if (userMatch) {
|
||||
user = userMatch[1];
|
||||
}
|
||||
|
||||
const ipMatch = line.match(/from (\d+\.\d+\.\d+\.\d+)/);
|
||||
if (ipMatch) {
|
||||
ip = ipMatch[1];
|
||||
}
|
||||
|
||||
const dateMatch = line.match(/^(\w+)\s+(\d+)\s+(\d+:\d+:\d+)/);
|
||||
if (dateMatch) {
|
||||
const [, month, day, time] = dateMatch;
|
||||
const now = new Date();
|
||||
const currentYear = now.getFullYear();
|
||||
const candidate = new Date(`${month} ${day}, ${currentYear} ${time}`);
|
||||
if (!isNaN(candidate.getTime()) && candidate > now) {
|
||||
// If parsed date is in the future, it's from last year
|
||||
timeStr = `${month} ${day}, ${currentYear - 1} ${time}`;
|
||||
} else {
|
||||
timeStr = `${month} ${day}, ${currentYear} ${time}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (user && ip) {
|
||||
let parsedTime: string;
|
||||
try {
|
||||
const date = timeStr ? new Date(timeStr) : null;
|
||||
parsedTime =
|
||||
date && !isNaN(date.getTime())
|
||||
? date.toISOString()
|
||||
: timeStr || "unknown";
|
||||
} catch {
|
||||
parsedTime = timeStr || "unknown";
|
||||
}
|
||||
|
||||
failedLogins.push({
|
||||
user,
|
||||
ip,
|
||||
time: parsedTime,
|
||||
status: "failed",
|
||||
});
|
||||
if (ip !== "unknown") {
|
||||
ipSet.add(ip);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return {
|
||||
recentLogins: recentLogins.slice(0, 10),
|
||||
failedLogins: failedLogins.slice(0, 10),
|
||||
totalLogins: recentLogins.length,
|
||||
uniqueIPs: ipSet.size,
|
||||
};
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import {
|
||||
execCommand,
|
||||
execPowerShell,
|
||||
toFixedNum,
|
||||
kibToGiB,
|
||||
type HostPlatform,
|
||||
} from "./common-utils.js";
|
||||
|
||||
async function collectDarwinMemoryMetrics(client: Client): Promise<{
|
||||
percent: number | null;
|
||||
usedGiB: number | null;
|
||||
totalGiB: number | null;
|
||||
}> {
|
||||
let memPercent: number | null = null;
|
||||
let usedGiB: number | null = null;
|
||||
let totalGiB: number | null = null;
|
||||
|
||||
try {
|
||||
const [vmStatOut, memSizeOut] = await Promise.all([
|
||||
execCommand(client, "vm_stat"),
|
||||
execCommand(client, "sysctl -n hw.memsize"),
|
||||
]);
|
||||
|
||||
const totalBytes = Number((memSizeOut.stdout || "").trim());
|
||||
const pageSizeMatch = vmStatOut.stdout.match(/page size of (\d+) bytes/);
|
||||
const pageSize = pageSizeMatch ? Number(pageSizeMatch[1]) : 4096;
|
||||
|
||||
const getPages = (key: string): number | null => {
|
||||
const line = vmStatOut.stdout
|
||||
.split("\n")
|
||||
.find((l) => l.trim().startsWith(key));
|
||||
if (!line) return null;
|
||||
const m = line.match(/(\d+)\./);
|
||||
return m ? Number(m[1]) : null;
|
||||
};
|
||||
|
||||
const active = getPages("Pages active");
|
||||
const wired = getPages("Pages wired down");
|
||||
const compressed = getPages("Pages occupied by compressor");
|
||||
|
||||
if (
|
||||
Number.isFinite(totalBytes) &&
|
||||
totalBytes > 0 &&
|
||||
active !== null &&
|
||||
wired !== null
|
||||
) {
|
||||
const usedPages = active + wired + (compressed ?? 0);
|
||||
const usedBytes = usedPages * pageSize;
|
||||
memPercent = Math.max(0, Math.min(100, (usedBytes / totalBytes) * 100));
|
||||
usedGiB = kibToGiB(usedBytes / 1024);
|
||||
totalGiB = kibToGiB(totalBytes / 1024);
|
||||
}
|
||||
} catch {
|
||||
memPercent = null;
|
||||
usedGiB = null;
|
||||
totalGiB = null;
|
||||
}
|
||||
|
||||
return {
|
||||
percent: toFixedNum(memPercent, 0),
|
||||
usedGiB: usedGiB ? toFixedNum(usedGiB, 2) : null,
|
||||
totalGiB: totalGiB ? toFixedNum(totalGiB, 2) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// Win32_OperatingSystem reports TotalVisibleMemorySize/FreePhysicalMemory in KB.
|
||||
async function collectWindowsMemoryMetrics(client: Client): Promise<{
|
||||
percent: number | null;
|
||||
usedGiB: number | null;
|
||||
totalGiB: number | null;
|
||||
}> {
|
||||
let memPercent: number | null = null;
|
||||
let usedGiB: number | null = null;
|
||||
let totalGiB: number | null = null;
|
||||
|
||||
try {
|
||||
const { stdout } = await execPowerShell(
|
||||
client,
|
||||
"$os=Get-CimInstance Win32_OperatingSystem; [PSCustomObject]@{total=$os.TotalVisibleMemorySize; free=$os.FreePhysicalMemory} | ConvertTo-Json -Compress",
|
||||
);
|
||||
const parsed = JSON.parse(stdout.trim());
|
||||
const totalKb = Number(parsed?.total);
|
||||
const freeKb = Number(parsed?.free);
|
||||
if (Number.isFinite(totalKb) && Number.isFinite(freeKb) && totalKb > 0) {
|
||||
const usedKb = totalKb - freeKb;
|
||||
memPercent = Math.max(0, Math.min(100, (usedKb / totalKb) * 100));
|
||||
usedGiB = kibToGiB(usedKb);
|
||||
totalGiB = kibToGiB(totalKb);
|
||||
}
|
||||
} catch {
|
||||
memPercent = null;
|
||||
usedGiB = null;
|
||||
totalGiB = null;
|
||||
}
|
||||
|
||||
return {
|
||||
percent: toFixedNum(memPercent, 0),
|
||||
usedGiB: usedGiB ? toFixedNum(usedGiB, 2) : null,
|
||||
totalGiB: totalGiB ? toFixedNum(totalGiB, 2) : null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectMemoryMetrics(
|
||||
client: Client,
|
||||
platform?: HostPlatform,
|
||||
): Promise<{
|
||||
percent: number | null;
|
||||
usedGiB: number | null;
|
||||
totalGiB: number | null;
|
||||
}> {
|
||||
if (platform === "darwin") {
|
||||
return collectDarwinMemoryMetrics(client);
|
||||
}
|
||||
if (platform === "windows") {
|
||||
return collectWindowsMemoryMetrics(client);
|
||||
}
|
||||
|
||||
let memPercent: number | null = null;
|
||||
let usedGiB: number | null = null;
|
||||
let totalGiB: number | null = null;
|
||||
|
||||
try {
|
||||
const memInfo = await execCommand(client, "cat /proc/meminfo");
|
||||
const lines = memInfo.stdout.split("\n");
|
||||
const getVal = (key: string) => {
|
||||
const line = lines.find((l) => l.startsWith(key));
|
||||
if (!line) return null;
|
||||
const m = line.match(/\d+/);
|
||||
return m ? Number(m[0]) : null;
|
||||
};
|
||||
const totalKb = getVal("MemTotal:");
|
||||
const availKb = getVal("MemAvailable:");
|
||||
if (totalKb && availKb && totalKb > 0) {
|
||||
const usedKb = totalKb - availKb;
|
||||
memPercent = Math.max(0, Math.min(100, (usedKb / totalKb) * 100));
|
||||
usedGiB = kibToGiB(usedKb);
|
||||
totalGiB = kibToGiB(totalKb);
|
||||
}
|
||||
} catch {
|
||||
memPercent = null;
|
||||
usedGiB = null;
|
||||
totalGiB = null;
|
||||
}
|
||||
|
||||
return {
|
||||
percent: toFixedNum(memPercent, 0),
|
||||
usedGiB: usedGiB ? toFixedNum(usedGiB, 2) : null,
|
||||
totalGiB: totalGiB ? toFixedNum(totalGiB, 2) : null,
|
||||
};
|
||||
}
|
||||
@@ -1,421 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import {
|
||||
execCommand,
|
||||
execPowerShell,
|
||||
type HostPlatform,
|
||||
} from "./common-utils.js";
|
||||
|
||||
export interface NetworkCounters {
|
||||
rx: string;
|
||||
tx: string;
|
||||
}
|
||||
|
||||
export function parseNetworkCounters(
|
||||
output: string,
|
||||
): Map<string, NetworkCounters> {
|
||||
const counters = new Map<string, NetworkCounters>();
|
||||
for (const line of output.split("\n").slice(2)) {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
if (parts.length >= 10) {
|
||||
counters.set(parts[0].replace(":", ""), {
|
||||
rx: parts[1],
|
||||
tx: parts[9],
|
||||
});
|
||||
}
|
||||
}
|
||||
return counters;
|
||||
}
|
||||
|
||||
export function counterRate(
|
||||
before: string | undefined,
|
||||
after: string | undefined,
|
||||
elapsedSeconds: number,
|
||||
): number | null {
|
||||
const first = Number(before);
|
||||
const second = Number(after);
|
||||
if (
|
||||
!Number.isFinite(first) ||
|
||||
!Number.isFinite(second) ||
|
||||
second < first ||
|
||||
elapsedSeconds <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Math.round((second - first) / elapsedSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate is derived from the previous poll's counters rather than a same-poll
|
||||
* double-sample, so a poll no longer has to block for a fixed settle window.
|
||||
* Keyed by host, since counters from one host are meaningless against another.
|
||||
*/
|
||||
const previousSamples = new Map<
|
||||
number,
|
||||
{ counters: Map<string, NetworkCounters>; timestamp: number }
|
||||
>();
|
||||
|
||||
const previousWindowsSamples = new Map<
|
||||
number,
|
||||
{ rows: WindowsAdapterRow[]; timestamp: number }
|
||||
>();
|
||||
|
||||
export function clearNetworkSampleCache(hostId?: number): void {
|
||||
if (hostId === undefined) {
|
||||
previousSamples.clear();
|
||||
previousWindowsSamples.clear();
|
||||
return;
|
||||
}
|
||||
previousSamples.delete(hostId);
|
||||
previousWindowsSamples.delete(hostId);
|
||||
}
|
||||
|
||||
export function parseDarwinIfconfig(
|
||||
output: string,
|
||||
): Map<string, { ip: string; state: string }> {
|
||||
const map = new Map<string, { ip: string; state: string }>();
|
||||
let current: string | null = null;
|
||||
for (const rawLine of output.split("\n")) {
|
||||
const ifaceMatch = rawLine.match(/^(\S+):\s*flags=\d+<([^>]*)>/);
|
||||
if (ifaceMatch) {
|
||||
current = ifaceMatch[1] === "lo0" ? null : ifaceMatch[1];
|
||||
if (current) {
|
||||
const state = ifaceMatch[2].includes("UP") ? "UP" : "DOWN";
|
||||
map.set(current, { ip: "", state });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!current) continue;
|
||||
const inetMatch = rawLine.match(/^\s+inet\s+(\d+\.\d+\.\d+\.\d+)/);
|
||||
if (inetMatch) {
|
||||
const existing = map.get(current);
|
||||
if (existing && !existing.ip) existing.ip = inetMatch[1];
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
export function parseDarwinNetstat(
|
||||
output: string,
|
||||
): Map<string, NetworkCounters> {
|
||||
const counters = new Map<string, NetworkCounters>();
|
||||
for (const line of output.split("\n").slice(1)) {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
if (parts.length < 10) continue;
|
||||
const name = parts[0];
|
||||
if (name === "lo0" || !parts[2]?.startsWith("<Link")) continue;
|
||||
if (!counters.has(name)) {
|
||||
counters.set(name, { rx: parts[6], tx: parts[9] });
|
||||
}
|
||||
}
|
||||
return counters;
|
||||
}
|
||||
|
||||
async function collectDarwinNetworkMetrics(
|
||||
client: Client,
|
||||
hostId?: number,
|
||||
): Promise<{
|
||||
interfaces: Array<{
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
rxRateBps: number | null;
|
||||
txRateBps: number | null;
|
||||
}>;
|
||||
}> {
|
||||
const interfaces: Array<{
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
rxRateBps: number | null;
|
||||
txRateBps: number | null;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
const ifconfigOut = await execCommand(client, "ifconfig -a 2>/dev/null");
|
||||
const ifMap = parseDarwinIfconfig(ifconfigOut.stdout);
|
||||
|
||||
try {
|
||||
const readAt = Date.now();
|
||||
const netstatOut = await execCommand(client, "netstat -ib 2>/dev/null");
|
||||
const current = parseDarwinNetstat(netstatOut.stdout);
|
||||
const previous =
|
||||
hostId !== undefined ? previousSamples.get(hostId) : undefined;
|
||||
const elapsedSeconds = previous
|
||||
? (readAt - previous.timestamp) / 1000
|
||||
: 0;
|
||||
if (ifMap.size === 0) {
|
||||
for (const name of current.keys()) {
|
||||
ifMap.set(name, { ip: "", state: "UNKNOWN" });
|
||||
}
|
||||
}
|
||||
for (const [name, data] of ifMap.entries()) {
|
||||
const c = current.get(name);
|
||||
const p = previous?.counters.get(name);
|
||||
interfaces.push({
|
||||
name,
|
||||
ip: data.ip,
|
||||
state: data.state,
|
||||
rxBytes: c?.rx ?? null,
|
||||
txBytes: c?.tx ?? null,
|
||||
rxRateBps: p ? counterRate(p.rx, c?.rx, elapsedSeconds) : null,
|
||||
txRateBps: p ? counterRate(p.tx, c?.tx, elapsedSeconds) : null,
|
||||
});
|
||||
}
|
||||
if (hostId !== undefined) {
|
||||
previousSamples.set(hostId, { counters: current, timestamp: readAt });
|
||||
}
|
||||
} catch {
|
||||
for (const [name, data] of ifMap.entries()) {
|
||||
interfaces.push({
|
||||
name,
|
||||
ip: data.ip,
|
||||
state: data.state,
|
||||
rxBytes: null,
|
||||
txBytes: null,
|
||||
rxRateBps: null,
|
||||
txRateBps: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return { interfaces };
|
||||
}
|
||||
|
||||
interface WindowsAdapterRow {
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rx: string;
|
||||
tx: string;
|
||||
}
|
||||
|
||||
const WINDOWS_ADAPTER_SCRIPT =
|
||||
"Get-NetAdapter | Where-Object {$_.Status -eq 'Up'} | ForEach-Object {" +
|
||||
" $stats = Get-NetAdapterStatistics -Name $_.Name -ErrorAction SilentlyContinue;" +
|
||||
" $addr = (Get-NetIPAddress -InterfaceIndex $_.ifIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Select-Object -First 1).IPAddress;" +
|
||||
" [PSCustomObject]@{name=$_.Name; ip=$addr; state='UP'; rx=$stats.ReceivedBytes; tx=$stats.SentBytes}" +
|
||||
" } | ConvertTo-Json -Compress";
|
||||
|
||||
export function parseWindowsAdapterJson(output: string): WindowsAdapterRow[] {
|
||||
const trimmed = output.trim();
|
||||
if (!trimmed) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
const rows = Array.isArray(parsed) ? parsed : [parsed];
|
||||
return rows
|
||||
.filter((row): row is Record<string, unknown> => Boolean(row?.name))
|
||||
.map((row) => ({
|
||||
name: String(row.name),
|
||||
ip: row.ip ? String(row.ip) : "",
|
||||
state: String(row.state ?? "UNKNOWN"),
|
||||
rx: String(row.rx ?? ""),
|
||||
tx: String(row.tx ?? ""),
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function collectWindowsNetworkMetrics(
|
||||
client: Client,
|
||||
hostId?: number,
|
||||
): Promise<{
|
||||
interfaces: Array<{
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
rxRateBps: number | null;
|
||||
txRateBps: number | null;
|
||||
}>;
|
||||
}> {
|
||||
const interfaces: Array<{
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
rxRateBps: number | null;
|
||||
txRateBps: number | null;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
const readAt = Date.now();
|
||||
const result = await execPowerShell(client, WINDOWS_ADAPTER_SCRIPT);
|
||||
const currentRows = parseWindowsAdapterJson(result.stdout);
|
||||
const previous =
|
||||
hostId !== undefined ? previousWindowsSamples.get(hostId) : undefined;
|
||||
const elapsedSeconds = previous ? (readAt - previous.timestamp) / 1000 : 0;
|
||||
const previousMap = new Map(
|
||||
(previous?.rows ?? []).map((row) => [row.name, row]),
|
||||
);
|
||||
|
||||
for (const row of currentRows) {
|
||||
const previousRow = previousMap.get(row.name);
|
||||
interfaces.push({
|
||||
name: row.name,
|
||||
ip: row.ip,
|
||||
state: row.state,
|
||||
rxBytes: row.rx || null,
|
||||
txBytes: row.tx || null,
|
||||
rxRateBps: previousRow
|
||||
? counterRate(previousRow.rx, row.rx, elapsedSeconds)
|
||||
: null,
|
||||
txRateBps: previousRow
|
||||
? counterRate(previousRow.tx, row.tx, elapsedSeconds)
|
||||
: null,
|
||||
});
|
||||
}
|
||||
|
||||
if (hostId !== undefined) {
|
||||
previousWindowsSamples.set(hostId, {
|
||||
rows: currentRows,
|
||||
timestamp: readAt,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return { interfaces };
|
||||
}
|
||||
|
||||
export async function collectNetworkMetrics(
|
||||
client: Client,
|
||||
platform?: HostPlatform,
|
||||
hostId?: number,
|
||||
): Promise<{
|
||||
interfaces: Array<{
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
rxRateBps: number | null;
|
||||
txRateBps: number | null;
|
||||
}>;
|
||||
}> {
|
||||
if (platform === "darwin") {
|
||||
return collectDarwinNetworkMetrics(client, hostId);
|
||||
}
|
||||
if (platform === "windows") {
|
||||
return collectWindowsNetworkMetrics(client, hostId);
|
||||
}
|
||||
|
||||
const interfaces: Array<{
|
||||
name: string;
|
||||
ip: string;
|
||||
state: string;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
rxRateBps: number | null;
|
||||
txRateBps: number | null;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
const ifconfigOut = await execCommand(
|
||||
client,
|
||||
"ip -o addr show 2>/dev/null | awk '{print $2,$4}' | grep -v '^lo' || true",
|
||||
);
|
||||
const netStatOut = await execCommand(
|
||||
client,
|
||||
"ip -o link show 2>/dev/null | awk '{gsub(/:/, \"\", $2); print $2,$9}' || true",
|
||||
);
|
||||
|
||||
const addrs = ifconfigOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
const states = netStatOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const ifMap = new Map<string, { ip: string; state: string }>();
|
||||
for (const line of addrs) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const name = parts[0];
|
||||
const ip = parts[1].split("/")[0];
|
||||
if (!ifMap.has(name)) ifMap.set(name, { ip, state: "UNKNOWN" });
|
||||
}
|
||||
}
|
||||
for (const line of states) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const name = parts[0];
|
||||
if (name === "lo") continue;
|
||||
const state = parts[1];
|
||||
const existing = ifMap.get(name);
|
||||
if (existing) {
|
||||
existing.state = state;
|
||||
} else {
|
||||
ifMap.set(name, { ip: "", state });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const readAt = Date.now();
|
||||
const procNet = await execCommand(client, "cat /proc/net/dev");
|
||||
const rxTxMap = parseNetworkCounters(procNet.stdout);
|
||||
const previous =
|
||||
hostId !== undefined ? previousSamples.get(hostId) : undefined;
|
||||
const elapsedSeconds = previous
|
||||
? (readAt - previous.timestamp) / 1000
|
||||
: 0;
|
||||
if (ifMap.size === 0) {
|
||||
for (const name of rxTxMap.keys()) {
|
||||
if (name !== "lo") ifMap.set(name, { ip: "", state: "UNKNOWN" });
|
||||
}
|
||||
}
|
||||
for (const [name, data] of ifMap.entries()) {
|
||||
const rxTx = rxTxMap.get(name);
|
||||
const previousCounters = previous?.counters.get(name);
|
||||
interfaces.push({
|
||||
name,
|
||||
ip: data.ip,
|
||||
state: data.state,
|
||||
rxBytes: rxTx?.rx ?? null,
|
||||
txBytes: rxTx?.tx ?? null,
|
||||
rxRateBps: previousCounters
|
||||
? counterRate(previousCounters.rx, rxTx?.rx, elapsedSeconds)
|
||||
: null,
|
||||
txRateBps: previousCounters
|
||||
? counterRate(previousCounters.tx, rxTx?.tx, elapsedSeconds)
|
||||
: null,
|
||||
});
|
||||
}
|
||||
if (hostId !== undefined) {
|
||||
previousSamples.set(hostId, {
|
||||
counters: rxTxMap,
|
||||
timestamp: readAt,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
for (const [name, data] of ifMap.entries()) {
|
||||
interfaces.push({
|
||||
name,
|
||||
ip: data.ip,
|
||||
state: data.state,
|
||||
rxBytes: null,
|
||||
txBytes: null,
|
||||
rxRateBps: null,
|
||||
txRateBps: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return { interfaces };
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
import type {
|
||||
PortsMetrics,
|
||||
ListeningPort,
|
||||
} from "../../../../types/stats-widgets.js";
|
||||
|
||||
function parseSsOutput(output: string): ListeningPort[] {
|
||||
const ports: ListeningPort[] = [];
|
||||
const lines = output.split("\n").slice(1);
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
const parts = trimmed.split(/\s+/);
|
||||
if (parts.length < 5) continue;
|
||||
|
||||
const protocol = parts[0]?.toLowerCase();
|
||||
if (protocol !== "tcp" && protocol !== "udp") continue;
|
||||
|
||||
const state = parts[1];
|
||||
const localAddr = parts[4];
|
||||
|
||||
if (!localAddr) continue;
|
||||
|
||||
const lastColon = localAddr.lastIndexOf(":");
|
||||
if (lastColon === -1) continue;
|
||||
|
||||
const address = localAddr.substring(0, lastColon);
|
||||
const portStr = localAddr.substring(lastColon + 1);
|
||||
const port = parseInt(portStr, 10);
|
||||
|
||||
if (isNaN(port)) continue;
|
||||
|
||||
const portEntry: ListeningPort = {
|
||||
protocol: protocol as "tcp" | "udp",
|
||||
localAddress: address.replace(/^\[|\]$/g, ""),
|
||||
localPort: port,
|
||||
state: protocol === "tcp" ? state : undefined,
|
||||
};
|
||||
|
||||
const processInfo = parts[6];
|
||||
if (processInfo && processInfo.startsWith("users:")) {
|
||||
const pidMatch = processInfo.match(/pid=(\d+)/);
|
||||
const nameMatch = processInfo.match(/\("([^"]+)"/);
|
||||
if (pidMatch) portEntry.pid = parseInt(pidMatch[1], 10);
|
||||
if (nameMatch) portEntry.process = nameMatch[1];
|
||||
}
|
||||
|
||||
ports.push(portEntry);
|
||||
}
|
||||
|
||||
return ports;
|
||||
}
|
||||
|
||||
function parseNetstatOutput(output: string): ListeningPort[] {
|
||||
const ports: ListeningPort[] = [];
|
||||
const lines = output.split("\n");
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
const parts = trimmed.split(/\s+/);
|
||||
if (parts.length < 4) continue;
|
||||
|
||||
const proto = parts[0]?.toLowerCase();
|
||||
if (!proto) continue;
|
||||
|
||||
let protocol: "tcp" | "udp";
|
||||
if (proto.startsWith("tcp")) {
|
||||
protocol = "tcp";
|
||||
} else if (proto.startsWith("udp")) {
|
||||
protocol = "udp";
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
|
||||
const localAddr = parts[3];
|
||||
if (!localAddr) continue;
|
||||
|
||||
const lastColon = localAddr.lastIndexOf(":");
|
||||
if (lastColon === -1) continue;
|
||||
|
||||
const address = localAddr.substring(0, lastColon);
|
||||
const portStr = localAddr.substring(lastColon + 1);
|
||||
const port = parseInt(portStr, 10);
|
||||
|
||||
if (isNaN(port)) continue;
|
||||
|
||||
const portEntry: ListeningPort = {
|
||||
protocol,
|
||||
localAddress: address,
|
||||
localPort: port,
|
||||
};
|
||||
|
||||
if (protocol === "tcp" && parts.length >= 6) {
|
||||
portEntry.state = parts[5];
|
||||
}
|
||||
|
||||
const pidProgram = parts[parts.length - 1];
|
||||
if (pidProgram && pidProgram.includes("/")) {
|
||||
const [pidStr, process] = pidProgram.split("/");
|
||||
const pid = parseInt(pidStr, 10);
|
||||
if (!isNaN(pid)) portEntry.pid = pid;
|
||||
if (process) portEntry.process = process;
|
||||
}
|
||||
|
||||
ports.push(portEntry);
|
||||
}
|
||||
|
||||
return ports;
|
||||
}
|
||||
|
||||
export async function collectPortsMetrics(
|
||||
client: Client,
|
||||
): Promise<PortsMetrics> {
|
||||
try {
|
||||
const ssResult = await execCommand(client, "ss -tulnp 2>/dev/null", 15000);
|
||||
|
||||
if (ssResult.stdout && ssResult.stdout.includes("Local")) {
|
||||
const ports = parseSsOutput(ssResult.stdout);
|
||||
return {
|
||||
source: "ss",
|
||||
ports: ports.sort((a, b) => a.localPort - b.localPort),
|
||||
};
|
||||
}
|
||||
|
||||
const netstatResult = await execCommand(
|
||||
client,
|
||||
"netstat -tulnp 2>/dev/null",
|
||||
15000,
|
||||
);
|
||||
|
||||
if (netstatResult.stdout && netstatResult.stdout.includes("Local")) {
|
||||
const ports = parseNetstatOutput(netstatResult.stdout);
|
||||
return {
|
||||
source: "netstat",
|
||||
ports: ports.sort((a, b) => a.localPort - b.localPort),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
source: "none",
|
||||
ports: [],
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
source: "none",
|
||||
ports: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export async function collectProcessesMetrics(client: Client): Promise<{
|
||||
total: number | null;
|
||||
running: number | null;
|
||||
top: Array<{
|
||||
pid: string;
|
||||
user: string;
|
||||
cpu: string;
|
||||
mem: string;
|
||||
command: string;
|
||||
}>;
|
||||
}> {
|
||||
let totalProcesses: number | null = null;
|
||||
let runningProcesses: number | null = null;
|
||||
const topProcesses: Array<{
|
||||
pid: string;
|
||||
user: string;
|
||||
cpu: string;
|
||||
mem: string;
|
||||
command: string;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
const psOut = await execCommand(
|
||||
client,
|
||||
"(ps aux --sort=-%cpu 2>/dev/null || ps aux) | head -n 11",
|
||||
);
|
||||
const psLines = psOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
if (psLines.length > 1) {
|
||||
for (let i = 1; i < Math.min(psLines.length, 11); i++) {
|
||||
const parts = psLines[i].split(/\s+/);
|
||||
if (parts.length >= 11) {
|
||||
const cpuVal = Number(parts[2]);
|
||||
const memVal = Number(parts[3]);
|
||||
topProcesses.push({
|
||||
pid: parts[1],
|
||||
user: parts[0],
|
||||
cpu: Number.isFinite(cpuVal) ? cpuVal.toString() : "0",
|
||||
mem: Number.isFinite(memVal) ? memVal.toString() : "0",
|
||||
command: parts.slice(10).join(" ").substring(0, 50),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const procCount = await execCommand(client, "ps aux | wc -l");
|
||||
const runningCount = await execCommand(client, "ps aux | grep -c ' R '");
|
||||
|
||||
const totalCount = Number(procCount.stdout.trim()) - 1;
|
||||
totalProcesses = Number.isFinite(totalCount) ? totalCount : null;
|
||||
|
||||
const runningCount2 = Number(runningCount.stdout.trim());
|
||||
runningProcesses = Number.isFinite(runningCount2) ? runningCount2 : null;
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return {
|
||||
total: totalProcesses,
|
||||
running: runningProcesses,
|
||||
top: topProcesses,
|
||||
};
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import {
|
||||
execCommand,
|
||||
execPowerShell,
|
||||
type HostPlatform,
|
||||
} from "./common-utils.js";
|
||||
|
||||
async function collectWindowsSystemMetrics(client: Client): Promise<{
|
||||
hostname: string | null;
|
||||
kernel: string | null;
|
||||
os: string | null;
|
||||
}> {
|
||||
let hostname: string | null = null;
|
||||
let kernel: string | null = null;
|
||||
let os: string | null = null;
|
||||
|
||||
try {
|
||||
const { stdout } = await execPowerShell(
|
||||
client,
|
||||
"$os=Get-CimInstance Win32_OperatingSystem; [PSCustomObject]@{hostname=$env:COMPUTERNAME; kernel=[System.Environment]::OSVersion.Version.ToString(); os=$os.Caption} | ConvertTo-Json -Compress",
|
||||
);
|
||||
const parsed = JSON.parse(stdout.trim());
|
||||
hostname = parsed?.hostname ? String(parsed.hostname) : null;
|
||||
kernel = parsed?.kernel ? String(parsed.kernel) : null;
|
||||
os = parsed?.os ? String(parsed.os).trim() : null;
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return { hostname, kernel, os };
|
||||
}
|
||||
|
||||
export async function collectSystemMetrics(
|
||||
client: Client,
|
||||
platform?: HostPlatform,
|
||||
): Promise<{
|
||||
hostname: string | null;
|
||||
kernel: string | null;
|
||||
os: string | null;
|
||||
}> {
|
||||
if (platform === "windows") {
|
||||
return collectWindowsSystemMetrics(client);
|
||||
}
|
||||
|
||||
let hostname: string | null = null;
|
||||
let kernel: string | null = null;
|
||||
let os: string | null = null;
|
||||
|
||||
try {
|
||||
const hostnameOut = await execCommand(client, "hostname");
|
||||
const kernelOut = await execCommand(client, "uname -r");
|
||||
const osOut = await execCommand(
|
||||
client,
|
||||
"command -v sw_vers >/dev/null 2>&1 && echo \"$(sw_vers -productName) $(sw_vers -productVersion)\" || (cat /etc/os-release 2>/dev/null | grep '^PRETTY_NAME=' | cut -d'\"' -f2)",
|
||||
);
|
||||
|
||||
hostname = hostnameOut.stdout.trim() || null;
|
||||
kernel = kernelOut.stdout.trim() || null;
|
||||
os = osOut.stdout.trim() || null;
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return {
|
||||
hostname,
|
||||
kernel,
|
||||
os,
|
||||
};
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "./common-utils.js";
|
||||
|
||||
export interface TemperatureSensor {
|
||||
label: string;
|
||||
celsius: number;
|
||||
}
|
||||
|
||||
export interface TemperatureMetrics {
|
||||
source: "sysfs" | "sensors" | "none";
|
||||
highestCelsius: number | null;
|
||||
sensors: TemperatureSensor[];
|
||||
}
|
||||
|
||||
function normalizeSensor(
|
||||
label: string,
|
||||
celsius: number,
|
||||
): TemperatureSensor | null {
|
||||
const cleanLabel = label.trim().replace(/\s+/g, " ");
|
||||
if (!cleanLabel || !Number.isFinite(celsius)) return null;
|
||||
return {
|
||||
label: cleanLabel,
|
||||
celsius: toFixedNum(celsius, 1) ?? celsius,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeSensors(
|
||||
source: TemperatureMetrics["source"],
|
||||
sensors: TemperatureSensor[],
|
||||
): TemperatureMetrics {
|
||||
const highest = sensors.reduce<number | null>(
|
||||
(max, sensor) =>
|
||||
max === null ? sensor.celsius : Math.max(max, sensor.celsius),
|
||||
null,
|
||||
);
|
||||
|
||||
return {
|
||||
source: sensors.length > 0 ? source : "none",
|
||||
highestCelsius:
|
||||
highest === null ? null : (toFixedNum(highest, 1) ?? highest),
|
||||
sensors,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseSysfsThermalOutput(output: string): TemperatureSensor[] {
|
||||
return output
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
const [label, rawValue] = line.split("\t");
|
||||
const value = Number(rawValue);
|
||||
if (!label || !Number.isFinite(value)) return null;
|
||||
const celsius = Math.abs(value) >= 1000 ? value / 1000 : value;
|
||||
return normalizeSensor(label, celsius);
|
||||
})
|
||||
.filter((sensor): sensor is TemperatureSensor => sensor !== null);
|
||||
}
|
||||
|
||||
export function parseSensorsOutput(output: string): TemperatureSensor[] {
|
||||
const seen = new Set<string>();
|
||||
const sensors: TemperatureSensor[] = [];
|
||||
let device = "";
|
||||
|
||||
for (const line of output.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !line.startsWith(" ") && !trimmed.includes(":")) {
|
||||
device = trimmed;
|
||||
continue;
|
||||
}
|
||||
const match = line.match(/^\s*([^:]+):\s*([+-]?\d+(?:\.\d+)?)\s*°?C\b/i);
|
||||
if (!match) continue;
|
||||
|
||||
const label = device ? `${device}: ${match[1]}` : match[1];
|
||||
const sensor = normalizeSensor(label, Number(match[2]));
|
||||
if (!sensor) continue;
|
||||
|
||||
const key = `${sensor.label.toLowerCase()}:${sensor.celsius}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
sensors.push(sensor);
|
||||
}
|
||||
|
||||
return sensors;
|
||||
}
|
||||
|
||||
export async function collectTemperatureMetrics(
|
||||
client: Client,
|
||||
): Promise<TemperatureMetrics> {
|
||||
try {
|
||||
const sysfs = await execCommand(
|
||||
client,
|
||||
[
|
||||
"for zone in /sys/class/thermal/thermal_zone*; do",
|
||||
'[ -r "$zone/temp" ] || continue;',
|
||||
'label="$(cat "$zone/type" 2>/dev/null || basename "$zone")";',
|
||||
'value="$(cat "$zone/temp" 2>/dev/null || true)";',
|
||||
'[ -n "$value" ] && printf "%s\\t%s\\n" "$label" "$value";',
|
||||
"done",
|
||||
].join(" "),
|
||||
10000,
|
||||
);
|
||||
const sensors = parseSysfsThermalOutput(sysfs.stdout);
|
||||
if (sensors.length > 0) {
|
||||
return summarizeSensors("sysfs", sensors);
|
||||
}
|
||||
} catch {
|
||||
// expected on systems without readable thermal zones
|
||||
}
|
||||
|
||||
try {
|
||||
const lmSensors = await execCommand(client, "sensors 2>/dev/null", 10000);
|
||||
const sensors = parseSensorsOutput(lmSensors.stdout);
|
||||
if (sensors.length > 0) {
|
||||
return summarizeSensors("sensors", sensors);
|
||||
}
|
||||
} catch {
|
||||
// expected when lm-sensors is not installed
|
||||
}
|
||||
|
||||
return {
|
||||
source: "none",
|
||||
highestCelsius: null,
|
||||
sensors: [],
|
||||
};
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
import type { Client } from "ssh2";
|
||||
import {
|
||||
execCommand,
|
||||
execPowerShell,
|
||||
type HostPlatform,
|
||||
} from "./common-utils.js";
|
||||
|
||||
function formatUptime(uptimeSeconds: number): string {
|
||||
const days = Math.floor(uptimeSeconds / 86400);
|
||||
const hours = Math.floor((uptimeSeconds % 86400) / 3600);
|
||||
const minutes = Math.floor((uptimeSeconds % 3600) / 60);
|
||||
return `${days}d ${hours}h ${minutes}m`;
|
||||
}
|
||||
|
||||
async function collectDarwinUptimeMetrics(client: Client): Promise<{
|
||||
seconds: number | null;
|
||||
formatted: string | null;
|
||||
}> {
|
||||
let uptimeSeconds: number | null = null;
|
||||
let uptimeFormatted: string | null = null;
|
||||
|
||||
try {
|
||||
const bootTimeOut = await execCommand(client, "sysctl -n kern.boottime");
|
||||
// "{ sec = 1700000000, usec = 123456 } Thu Nov 16 ..."
|
||||
const match = bootTimeOut.stdout.match(/sec\s*=\s*(\d+)/);
|
||||
if (match) {
|
||||
const bootSeconds = Number(match[1]);
|
||||
if (Number.isFinite(bootSeconds)) {
|
||||
uptimeSeconds = Math.max(0, Date.now() / 1000 - bootSeconds);
|
||||
uptimeFormatted = formatUptime(uptimeSeconds);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return {
|
||||
seconds: uptimeSeconds,
|
||||
formatted: uptimeFormatted,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectWindowsUptimeMetrics(client: Client): Promise<{
|
||||
seconds: number | null;
|
||||
formatted: string | null;
|
||||
}> {
|
||||
let uptimeSeconds: number | null = null;
|
||||
let uptimeFormatted: string | null = null;
|
||||
|
||||
try {
|
||||
const { stdout } = await execPowerShell(
|
||||
client,
|
||||
"$os=Get-CimInstance Win32_OperatingSystem; [PSCustomObject]@{seconds=((Get-Date) - $os.LastBootUpTime).TotalSeconds} | ConvertTo-Json -Compress",
|
||||
);
|
||||
const parsed = JSON.parse(stdout.trim());
|
||||
const seconds = Number(parsed?.seconds);
|
||||
if (Number.isFinite(seconds)) {
|
||||
uptimeSeconds = Math.max(0, seconds);
|
||||
uptimeFormatted = formatUptime(uptimeSeconds);
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return {
|
||||
seconds: uptimeSeconds,
|
||||
formatted: uptimeFormatted,
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectUptimeMetrics(
|
||||
client: Client,
|
||||
platform?: HostPlatform,
|
||||
): Promise<{
|
||||
seconds: number | null;
|
||||
formatted: string | null;
|
||||
}> {
|
||||
if (platform === "darwin") {
|
||||
return collectDarwinUptimeMetrics(client);
|
||||
}
|
||||
if (platform === "windows") {
|
||||
return collectWindowsUptimeMetrics(client);
|
||||
}
|
||||
|
||||
let uptimeSeconds: number | null = null;
|
||||
let uptimeFormatted: string | null = null;
|
||||
|
||||
try {
|
||||
const uptimeOut = await execCommand(client, "cat /proc/uptime");
|
||||
const uptimeParts = uptimeOut.stdout.trim().split(/\s+/);
|
||||
if (uptimeParts.length >= 1) {
|
||||
uptimeSeconds = Number(uptimeParts[0]);
|
||||
if (Number.isFinite(uptimeSeconds)) {
|
||||
uptimeFormatted = formatUptime(uptimeSeconds);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
return {
|
||||
seconds: uptimeSeconds,
|
||||
formatted: uptimeFormatted,
|
||||
};
|
||||
}
|
||||
@@ -14,7 +14,8 @@ import { resolveSshConnectConfigHost } from "./ssh-dns.js";
|
||||
/**
|
||||
* Non-interactive SSH connection helpers shared by fleet execution.
|
||||
*
|
||||
* Mirrors buildSshConfig/createSshFactory/getPoolKey in hosts/metrics/index.ts,
|
||||
* Mirrors buildSshConfig/createSshFactory/getPoolKey in
|
||||
* plugins/host-metrics/backend/routes.ts,
|
||||
* minus the keyboard-interactive TOTP prompt handling (metrics/terminal-only)
|
||||
* and OPKSSH/Vault cert setup (not supported as fleet auth for v1 - those flows
|
||||
* need an interactive browser step that has no place in a batch fleet run).
|
||||
|
||||
Reference in new issue
Block a user