feat: move proxmox node stats and collectors into the proxmox plugin

This commit is contained in:
LukeGus committed 2026-09-23 13:06:18 -05:00
1 parent 218c5088e7
commit 37c015a760
37 files changed
+3775 -227

No files matched your search

+23 -1
View File
@@ -1,14 +1,36 @@
import type { Router } from "express";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import { setPluginSsh } from "./ssh.js";
import { setPluginCtx } from "./plugin-ctx.js";
import { startProxmoxService, stopProxmoxService } from "./routes.js";
import {
startProxmoxStatsService,
stopProxmoxStatsService,
} from "./stats-service.js";
import { proxmoxNodeHistory } from "./tables.js";
import { createProxmoxNodeHistoryRepository } from "./proxmox-node-history-repository.js";
export async function activate(ctx: PluginContext) {
setPluginSsh(ctx.ssh);
setPluginCtx(ctx);
ctx.disposables.add(() => setPluginSsh(null));
startProxmoxService(ctx.http.router());
ctx.disposables.add(() => setPluginCtx(null));
const historyTable = await ctx.db.define(proxmoxNodeHistory);
const historyRepository = createProxmoxNodeHistoryRepository(
ctx.db,
historyTable,
);
const router = ctx.http.router<Router>();
startProxmoxService(router);
ctx.disposables.add(stopProxmoxService);
startProxmoxStatsService(ctx, router, historyRepository);
ctx.disposables.add(stopProxmoxStatsService);
ctx.log.info("Proxmox routes mounted at /plugin-api/proxmox");
}
export async function deactivate() {
stopProxmoxService();
stopProxmoxStatsService();
}
+13
View File
@@ -0,0 +1,13 @@
import type { PluginContext } from "@termix/plugin-sdk/backend";
let current: PluginContext | null = null;
/** Set in activate, cleared on deactivate. */
export function setPluginCtx(ctx: PluginContext | null): void {
current = ctx;
}
export function pluginCtx(): PluginContext {
if (!current) throw new Error("The plugin is not active");
return current;
}
@@ -0,0 +1,79 @@
import { and, gte, lte, eq, asc, lt } from "drizzle-orm";
import type { PluginDatabase } from "@termix/plugin-sdk/backend";
export interface ProxmoxNodeHistoryRow {
id: number;
hostId: number;
ts: string;
cpuPercent: number | null;
memPercent: number | null;
diskPercent: number | null;
netRxBytes: number | null;
netTxBytes: number | null;
}
export interface ProxmoxNodeHistoryCreateInput {
hostId: number;
cpuPercent: number | null;
memPercent: number | null;
diskPercent: number | null;
netRxBytes: number | null;
netTxBytes: number | null;
}
/* eslint-disable @typescript-eslint/no-explicit-any */
type Table = any;
type Drizzle = any;
/* eslint-enable @typescript-eslint/no-explicit-any */
export type ProxmoxNodeHistoryRepository = ReturnType<
typeof createProxmoxNodeHistoryRepository
>;
export function createProxmoxNodeHistoryRepository(
db: PluginDatabase,
table: Table,
) {
const client = () => db.client<Drizzle>();
return {
async create(input: ProxmoxNodeHistoryCreateInput): Promise<void> {
const drizzle = await client();
await drizzle.insert(table).values(input);
await db.persist();
},
async listRange(
hostId: number,
fromIso: string,
toIso: string,
): Promise<ProxmoxNodeHistoryRow[]> {
const drizzle = await client();
return drizzle
.select()
.from(table)
.where(
and(
eq(table.hostId, hostId),
gte(table.ts, fromIso),
lte(table.ts, toIso),
),
)
.orderBy(asc(table.ts));
},
async pruneOlderThan(hostId: number, retentionDays: number): Promise<void> {
// ts columns store CURRENT_TIMESTAMP as "YYYY-MM-DD HH:MM:SS" UTC, which
// sorts lexicographically, so the cutoff is a plain string comparison.
const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000)
.toISOString()
.slice(0, 19)
.replace("T", " ");
const drizzle = await client();
await drizzle
.delete(table)
.where(and(eq(table.hostId, hostId), lt(table.ts, cutoff)));
await db.persist();
},
};
}
@@ -0,0 +1,7 @@
// Proxmox node names are restricted to [a-zA-Z0-9-] by PVE itself,
// but we validate defensively before using in a shell command.
const SAFE_NODE_RE = /^[a-zA-Z0-9._-]{1,64}$/;
export function isSafeNodeName(name: string): boolean {
return SAFE_NODE_RE.test(name);
}
@@ -0,0 +1,335 @@
import {
collectProxmoxStats,
type ProxmoxStatsSnapshot,
} from "./proxmox/collect-proxmox-stats.js";
import type { ProxmoxNodeHistoryRepository } from "./proxmox-node-history-repository.js";
import { pluginCtx } from "./plugin-ctx.js";
import type { Client } from "ssh2";
/**
* Proxmox stats polling gets its own concurrency limiter so a burst of polls
* on many hosts cannot starve one host's collection.
*/
class ConcurrentLimiter {
private running = 0;
private queue: Array<() => void> = [];
constructor(private readonly max: number) {}
async run<T>(fn: () => Promise<T>): Promise<T> {
if (this.running >= this.max) {
await new Promise<void>((resolve) => this.queue.push(resolve));
}
this.running++;
try {
return await fn();
} finally {
this.running--;
const next = this.queue.shift();
if (next) next();
}
}
}
const proxmoxStatsPollLimiter = new ConcurrentLimiter(5);
export interface ProxmoxStatsPollableHost {
id: number;
userId: string;
proxmoxStatsConfig?: string | ProxmoxStatsPollConfig | null;
}
export interface ProxmoxStatsPollConfig {
nodeName?: string | null;
pollInterval?: number;
enabledCards?: string[];
}
const DEFAULT_POLL_INTERVAL_SECONDS = 60;
export function parseProxmoxStatsConfig(
raw: string | ProxmoxStatsPollConfig | null | undefined,
): ProxmoxStatsPollConfig {
if (!raw) {
return { nodeName: null, pollInterval: DEFAULT_POLL_INTERVAL_SECONDS };
}
if (typeof raw === "object") {
return {
nodeName: raw.nodeName ?? null,
pollInterval: raw.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS,
enabledCards: raw.enabledCards,
};
}
try {
const parsed = JSON.parse(raw) as ProxmoxStatsPollConfig;
return {
nodeName: parsed.nodeName ?? null,
pollInterval: parsed.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS,
enabledCards: parsed.enabledCards,
};
} catch {
return { nodeName: null, pollInterval: DEFAULT_POLL_INTERVAL_SECONDS };
}
}
interface HostPollingEntry<THost extends ProxmoxStatsPollableHost> {
host: THost;
timer?: NodeJS.Timeout;
viewerUserId?: string;
}
interface ViewerDetail {
sessionId: string;
userId: string;
hostId: number;
lastHeartbeat: number;
}
interface CachedSnapshot {
data: ProxmoxStatsSnapshot;
timestamp: number;
}
interface ErrorSnapshot {
error: string;
timestamp: number;
}
export class ProxmoxPollingManager<
THost extends ProxmoxStatsPollableHost = ProxmoxStatsPollableHost,
> {
private pollingConfigs = new Map<number, HostPollingEntry<THost>>();
private snapshotStore = new Map<number, CachedSnapshot>();
private errorStore = new Map<number, ErrorSnapshot>();
private activeViewers = new Map<number, Set<string>>();
private viewerDetails = new Map<string, ViewerDetail>();
private inFlight = new Set<number>();
private viewerCleanupInterval: NodeJS.Timeout;
constructor(
private readonly deps: {
fetchHostById: (
hostId: number,
userId: string,
) => Promise<THost | undefined>;
withSshConnection: <T>(
host: THost,
fn: (client: Client) => Promise<T>,
) => Promise<T>;
historyRepository: ProxmoxNodeHistoryRepository;
historyEnabled?: () => boolean;
},
) {
this.viewerCleanupInterval = setInterval(() => {
this.cleanupInactiveViewers();
}, 60000);
}
private intervalWithJitter(intervalMs: number, hostId: number): number {
const spread = Math.min(intervalMs * 0.2, 15_000);
const jitter = (hostId * 1103515245) % Math.max(1, Math.floor(spread));
return intervalMs + jitter;
}
// Host Metrics' own retention setting is not on the core settings
// allowlist, so this plugin uses a fixed default rather than reaching for
// it. See FINISH-LIST.md.
private readonly retentionDays = 7;
private async pollHostStats(
host: THost,
viewerUserId?: string,
): Promise<void> {
if (this.inFlight.has(host.id)) return;
this.inFlight.add(host.id);
try {
await proxmoxStatsPollLimiter.run(async () => {
const userId = viewerUserId || host.userId;
const refreshed =
(await this.deps.fetchHostById(host.id, userId)) ?? host;
const config = parseProxmoxStatsConfig(refreshed.proxmoxStatsConfig);
try {
const snapshot = await this.deps.withSshConnection(
refreshed,
(client) => collectProxmoxStats(client, config.nodeName),
);
this.snapshotStore.set(host.id, {
data: snapshot,
timestamp: Date.now(),
});
this.errorStore.delete(host.id);
if (this.deps.historyEnabled?.() ?? true) {
await this.insertHistory(host.id, snapshot);
}
} catch (error) {
const message =
error instanceof Error ? error.message : String(error);
this.errorStore.set(host.id, {
error: message,
timestamp: Date.now(),
});
pluginCtx().log.warn(
`Proxmox stats poll failed for host ${host.id}: ${message}`,
);
}
});
} finally {
this.inFlight.delete(host.id);
}
}
private async insertHistory(
hostId: number,
snapshot: ProxmoxStatsSnapshot,
): Promise<void> {
try {
const iface = snapshot.network?.interfaces?.[0];
const rxRaw = iface?.rxBytes ? parseInt(iface.rxBytes, 10) : null;
const txRaw = iface?.txBytes ? parseInt(iface.txBytes, 10) : null;
await this.deps.historyRepository.create({
hostId,
cpuPercent: snapshot.node?.cpu?.percent ?? null,
memPercent: snapshot.node?.memory?.percent ?? null,
diskPercent: snapshot.node?.disk?.percent ?? null,
netRxBytes: rxRaw !== null && !isNaN(rxRaw) ? rxRaw : null,
netTxBytes: txRaw !== null && !isNaN(txRaw) ? txRaw : null,
});
await this.deps.historyRepository.pruneOlderThan(
hostId,
this.retentionDays,
);
} catch (err) {
pluginCtx().log.warn(
`Failed to write proxmox node history for host ${hostId}: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
private startPollingForHost(host: THost, viewerUserId?: string): void {
const existing = this.pollingConfigs.get(host.id);
if (existing?.timer) {
clearInterval(existing.timer);
}
const config = parseProxmoxStatsConfig(host.proxmoxStatsConfig);
const intervalMs = this.intervalWithJitter(
(config.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS) * 1000,
host.id,
);
void this.pollHostStats(host, viewerUserId);
const timer = setInterval(() => {
const latest = this.pollingConfigs.get(host.id);
if (latest) {
void this.pollHostStats(latest.host, latest.viewerUserId);
}
}, intervalMs);
this.pollingConfigs.set(host.id, { host, timer, viewerUserId });
}
private stopPollingForHost(hostId: number): void {
const config = this.pollingConfigs.get(hostId);
if (config?.timer) {
clearInterval(config.timer);
}
this.pollingConfigs.delete(hostId);
}
getStats(
hostId: number,
): { data: ProxmoxStatsSnapshot; timestamp: number } | undefined {
return this.snapshotStore.get(hostId);
}
getError(hostId: number): ErrorSnapshot | undefined {
return this.errorStore.get(hostId);
}
async ensurePolling(host: THost, viewerUserId?: string): Promise<void> {
if (!this.pollingConfigs.has(host.id)) {
this.startPollingForHost(host, viewerUserId);
}
if (!this.snapshotStore.has(host.id) && !this.inFlight.has(host.id)) {
await this.pollHostStats(host, viewerUserId);
}
}
registerViewer = (
hostId: number,
sessionId: string,
userId: string,
): void => {
if (!this.activeViewers.has(hostId)) {
this.activeViewers.set(hostId, new Set());
}
this.activeViewers.get(hostId)!.add(sessionId);
this.viewerDetails.set(sessionId, {
sessionId,
userId,
hostId,
lastHeartbeat: Date.now(),
});
if (this.activeViewers.get(hostId)!.size === 1) {
Promise.resolve()
.then(async () => {
const host = await this.deps.fetchHostById(hostId, userId);
if (host) {
this.startPollingForHost(host, userId);
}
})
.catch((err) => {
pluginCtx().log.warn(
`Proxmox stats startPollingForHost rejected (non-fatal) for host ${hostId}: ${err instanceof Error ? err.message : String(err)}`,
);
});
}
};
unregisterViewer = (hostId: number, sessionId: string): void => {
const viewers = this.activeViewers.get(hostId);
if (viewers) {
viewers.delete(sessionId);
if (viewers.size === 0) {
this.activeViewers.delete(hostId);
this.stopPollingForHost(hostId);
}
}
this.viewerDetails.delete(sessionId);
};
updateHeartbeat(sessionId: string): boolean {
const viewer = this.viewerDetails.get(sessionId);
if (viewer) {
viewer.lastHeartbeat = Date.now();
return true;
}
return false;
}
private cleanupInactiveViewers(): void {
const now = Date.now();
const maxInactivity = 120000;
for (const [sessionId, viewer] of this.viewerDetails.entries()) {
if (now - viewer.lastHeartbeat > maxInactivity) {
this.unregisterViewer(viewer.hostId, sessionId);
}
}
}
destroy(): void {
clearInterval(this.viewerCleanupInterval);
for (const hostId of this.pollingConfigs.keys()) {
this.stopPollingForHost(hostId);
}
}
}
@@ -0,0 +1,67 @@
import type { Client } from "ssh2";
import { execCommand } from "./common-utils.js";
export interface ProxmoxClusterNodeEntry {
name: string;
online: boolean;
local: boolean;
ip: string | null;
}
export type ProxmoxClusterHealthResult =
| { clustered: false }
| {
clustered: true;
quorate: boolean;
clusterName: string | null;
nodes: ProxmoxClusterNodeEntry[];
};
const EMPTY_RESULT: ProxmoxClusterHealthResult = { clustered: false };
export async function collectProxmoxClusterHealth(
client: Client,
): Promise<ProxmoxClusterHealthResult> {
try {
const { stdout, code } = await execCommand(
client,
"pvesh get /cluster/status --output-format json",
15000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const data = JSON.parse(stdout);
if (!Array.isArray(data)) {
return EMPTY_RESULT;
}
const entries = data as Array<Record<string, unknown>>;
const clusterEntry = entries.find((e) => e.type === "cluster");
if (!clusterEntry) {
return EMPTY_RESULT;
}
const nodes: ProxmoxClusterNodeEntry[] = entries
.filter((e) => e.type === "node")
.map((e) => ({
name: typeof e.name === "string" ? e.name : "",
online: e.online === 1 || e.online === true,
local: e.local === 1 || e.local === true,
ip: typeof e.ip === "string" && e.ip ? e.ip : null,
}));
return {
clustered: true,
quorate: clusterEntry.quorate === 1 || clusterEntry.quorate === true,
clusterName:
typeof clusterEntry.name === "string" && clusterEntry.name
? clusterEntry.name
: null,
nodes,
};
} catch {
return EMPTY_RESULT;
}
}
@@ -0,0 +1,86 @@
import type { Client } from "ssh2";
import { execCommand } from "./common-utils.js";
import { isSafeNodeName } from "../proxmox-shared.js";
import {
collectProxmoxNodeStatus,
type ProxmoxNodeStatusResult,
} from "./node-status-collector.js";
import {
collectProxmoxNodeNetwork,
type ProxmoxNodeNetworkResult,
} from "./node-network-collector.js";
import {
collectProxmoxGuestsSummary,
type ProxmoxGuestsSummaryResult,
} from "./guests-collector.js";
import {
collectProxmoxStorage,
type ProxmoxStorageResult,
} from "./storage-collector.js";
import {
collectProxmoxClusterHealth,
type ProxmoxClusterHealthResult,
} from "./cluster-health-collector.js";
export interface ProxmoxStatsSnapshot {
node: ProxmoxNodeStatusResult;
network: ProxmoxNodeNetworkResult;
guests: ProxmoxGuestsSummaryResult;
storage: ProxmoxStorageResult;
cluster: ProxmoxClusterHealthResult;
lastChecked: string;
}
async function resolveNodeName(
client: Client,
configuredNodeName: string | null | undefined,
): Promise<string> {
if (configuredNodeName) {
// An explicitly configured name is validated and used as-is - an unsafe
// value is rejected outright rather than silently falling back to
// auto-detection, which would mask a misconfigured (or malicious) override.
if (!isSafeNodeName(configuredNodeName)) {
throw new Error("Unable to determine a valid Proxmox node name");
}
return configuredNodeName;
}
const { stdout } = await execCommand(client, "hostname", 10000);
return stdout.trim();
}
export async function collectProxmoxStats(
client: Client,
configuredNodeName: string | null | undefined,
): Promise<ProxmoxStatsSnapshot> {
const pveshCheck = await execCommand(
client,
"command -v pvesh >/dev/null 2>&1 && echo ok || echo missing",
10000,
);
if (pveshCheck.stdout.trim() !== "ok") {
throw new Error("pvesh not found — is this a Proxmox node?");
}
const nodeName = await resolveNodeName(client, configuredNodeName);
if (!isSafeNodeName(nodeName)) {
throw new Error("Unable to determine a valid Proxmox node name");
}
const [node, network, guests, storage, cluster] = await Promise.all([
collectProxmoxNodeStatus(client, nodeName),
collectProxmoxNodeNetwork(client, nodeName),
collectProxmoxGuestsSummary(client, nodeName),
collectProxmoxStorage(client, nodeName),
collectProxmoxClusterHealth(client),
]);
return {
node,
network,
guests,
storage,
cluster,
lastChecked: new Date().toISOString(),
};
}
@@ -0,0 +1,98 @@
import type { Client, ClientChannel } from "ssh2";
/** Runs a command over an already-open ssh2 client and collects its output. */
export function execCommand(
client: Client,
command: string,
timeoutMs = 30000,
): Promise<{
stdout: string;
stderr: string;
code: number | null;
}> {
return new Promise((resolve, reject) => {
let settled = false;
let stream: ClientChannel | null = null;
const timeout = setTimeout(() => {
if (!settled) {
settled = true;
cleanup();
reject(new Error(`Command timeout after ${timeoutMs}ms: ${command}`));
}
}, timeoutMs);
const cleanup = () => {
clearTimeout(timeout);
if (stream) {
try {
stream.removeAllListeners();
if (stream.stderr) {
stream.stderr.removeAllListeners();
}
stream.destroy();
} catch {
// expected - cleanup errors ignored
}
}
};
client.exec(command, { pty: false }, (err, _stream) => {
if (err) {
if (!settled) {
settled = true;
cleanup();
reject(err);
}
return;
}
stream = _stream;
let stdout = "";
let stderr = "";
let exitCode: number | null = null;
stream
.on("close", (code: number | undefined) => {
if (!settled) {
settled = true;
exitCode = typeof code === "number" ? code : null;
cleanup();
resolve({ stdout, stderr, code: exitCode });
}
})
.on("data", (data: Buffer) => {
stdout += data.toString("utf8");
})
.on("error", (streamErr: Error) => {
if (!settled) {
settled = true;
cleanup();
reject(streamErr);
}
});
if (stream.stderr) {
stream.stderr
.on("data", (data: Buffer) => {
stderr += data.toString("utf8");
})
.on("error", (stderrErr: Error) => {
if (!settled) {
settled = true;
cleanup();
reject(stderrErr);
}
});
}
});
});
}
export function toFixedNum(
n: number | null | undefined,
digits = 2,
): number | null {
if (typeof n !== "number" || !Number.isFinite(n)) return null;
return Number(n.toFixed(digits));
}
@@ -0,0 +1,113 @@
import type { Client } from "ssh2";
import { execCommand, toFixedNum } from "./common-utils.js";
import { isSafeNodeName } from "../proxmox-shared.js";
export interface ProxmoxGuestSummaryEntry {
vmid: number;
name: string;
type: "qemu" | "lxc";
status: string;
cpuPercent: number | null;
memPercent: number | null;
memUsedGiB: number | null;
memTotalGiB: number | null;
diskPercent: number | null;
diskUsedGiB: number | null;
diskTotalGiB: number | null;
uptimeSeconds: number | null;
}
export interface ProxmoxGuestsSummaryResult {
guests: ProxmoxGuestSummaryEntry[];
counts: { running: number; stopped: number; total: number };
}
const EMPTY_RESULT: ProxmoxGuestsSummaryResult = {
guests: [],
counts: { running: 0, stopped: 0, total: 0 },
};
function bytesToGiB(bytes: number): number {
return bytes / (1024 * 1024 * 1024);
}
export async function collectProxmoxGuestsSummary(
client: Client,
nodeName: string,
): Promise<ProxmoxGuestsSummaryResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
"pvesh get /cluster/resources --output-format json",
25000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const resources = JSON.parse(stdout);
if (!Array.isArray(resources)) {
return EMPTY_RESULT;
}
const guests: ProxmoxGuestSummaryEntry[] = [];
for (const r of resources as Array<Record<string, unknown>>) {
const type = r.type;
if (type !== "qemu" && type !== "lxc") continue;
if (r.node !== nodeName) continue;
if (r.template === true || r.template === 1) continue;
const cpuFraction = typeof r.cpu === "number" ? r.cpu : null;
const mem = typeof r.mem === "number" ? r.mem : null;
const maxmem = typeof r.maxmem === "number" ? r.maxmem : null;
const memPercent =
mem !== null && maxmem !== null && maxmem > 0
? Math.max(0, Math.min(100, (mem / maxmem) * 100))
: null;
const disk = typeof r.disk === "number" ? r.disk : null;
const maxdisk = typeof r.maxdisk === "number" ? r.maxdisk : null;
// QEMU guests without a running agent report maxdisk: 0 - a false 0%
// is worse than an honest "unknown", so treat that as no data at all.
const hasDiskData = maxdisk !== null && maxdisk > 0;
const diskPercent =
hasDiskData && disk !== null
? Math.max(0, Math.min(100, (disk / maxdisk) * 100))
: null;
guests.push({
vmid: typeof r.vmid === "number" ? r.vmid : Number(r.vmid),
name:
typeof r.name === "string" && r.name ? r.name : String(r.vmid ?? ""),
type,
status: typeof r.status === "string" ? r.status : "unknown",
cpuPercent: toFixedNum(
cpuFraction !== null ? cpuFraction * 100 : null,
0,
),
memPercent: toFixedNum(memPercent, 0),
memUsedGiB: mem !== null ? toFixedNum(bytesToGiB(mem), 2) : null,
memTotalGiB: maxmem !== null ? toFixedNum(bytesToGiB(maxmem), 2) : null,
diskPercent: toFixedNum(diskPercent, 0),
diskUsedGiB:
hasDiskData && disk !== null ? toFixedNum(bytesToGiB(disk), 2) : null,
diskTotalGiB: hasDiskData ? toFixedNum(bytesToGiB(maxdisk), 2) : null,
uptimeSeconds: typeof r.uptime === "number" ? r.uptime : null,
});
}
const running = guests.filter((g) => g.status === "running").length;
const stopped = guests.filter((g) => g.status !== "running").length;
return {
guests,
counts: { running, stopped, total: guests.length },
};
} catch {
return EMPTY_RESULT;
}
}
@@ -0,0 +1,145 @@
import type { Client } from "ssh2";
import { execCommand } from "./common-utils.js";
import { isSafeNodeName } from "../proxmox-shared.js";
export interface ProxmoxNodeNetworkInterface {
name: string;
ip: string | null;
state: string | null;
rxBytes: string | null;
txBytes: string | null;
}
export interface ProxmoxNodeNetworkResult {
interfaces: ProxmoxNodeNetworkInterface[];
}
const EMPTY_RESULT: ProxmoxNodeNetworkResult = { interfaces: [] };
async function collectFromProcNetDev(
client: Client,
): Promise<ProxmoxNodeNetworkResult> {
const interfaces: ProxmoxNodeNetworkInterface[] = [];
try {
const [addrOut, stateOut, procNetOut] = await Promise.all([
execCommand(
client,
"ip -o addr show | awk '{print $2,$4}' | grep -v '^lo'",
),
execCommand(
client,
"ip -o link show | awk '{gsub(/:/, \"\", $2); print $2,$9}'",
),
execCommand(client, "cat /proc/net/dev"),
]);
const ifMap = new Map<
string,
{ ip: string | null; state: string | null }
>();
for (const line of addrOut.stdout
.split("\n")
.map((l) => l.trim())
.filter(Boolean)) {
const parts = line.split(/\s+/);
if (parts.length >= 2) {
const name = parts[0];
const ip = parts[1].split("/")[0];
if (!ifMap.has(name)) ifMap.set(name, { ip, state: null });
}
}
for (const line of stateOut.stdout
.split("\n")
.map((l) => l.trim())
.filter(Boolean)) {
const parts = line.split(/\s+/);
if (parts.length >= 2) {
const existing = ifMap.get(parts[0]);
if (existing) existing.state = parts[1];
}
}
const rxTxMap = new Map<string, { rx: string; tx: string }>();
for (const line of procNetOut.stdout.split("\n").slice(2)) {
const parts = line.trim().split(/\s+/);
if (parts.length >= 10) {
const ifName = parts[0].replace(":", "");
rxTxMap.set(ifName, { rx: parts[1], tx: parts[9] });
}
}
for (const [name, data] of ifMap.entries()) {
const rxTx = rxTxMap.get(name);
interfaces.push({
name,
ip: data.ip,
state: data.state,
rxBytes: rxTx?.rx ?? null,
txBytes: rxTx?.tx ?? null,
});
}
} catch {
return EMPTY_RESULT;
}
return { interfaces };
}
export async function collectProxmoxNodeNetwork(
client: Client,
nodeName: string,
): Promise<ProxmoxNodeNetworkResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
`pvesh get /nodes/${nodeName}/netstat --output-format json`,
15000,
);
if (code !== 0) {
return collectFromProcNetDev(client);
}
const data = JSON.parse(stdout);
if (!Array.isArray(data)) {
return collectFromProcNetDev(client);
}
const interfaces: ProxmoxNodeNetworkInterface[] = data
.filter(
(entry): entry is Record<string, unknown> =>
!!entry && typeof entry === "object",
)
.map((entry) => ({
name:
typeof entry.dev === "string" ? entry.dev : String(entry.dev ?? ""),
ip: null,
state: null,
rxBytes:
typeof entry.in === "number"
? String(entry.in)
: typeof entry.received === "number"
? String(entry.received)
: null,
txBytes:
typeof entry.out === "number"
? String(entry.out)
: typeof entry.transmitted === "number"
? String(entry.transmitted)
: null,
}))
.filter((iface) => iface.name && iface.name !== "lo");
if (interfaces.length === 0) {
return collectFromProcNetDev(client);
}
return { interfaces };
} catch {
return collectFromProcNetDev(client);
}
}
@@ -0,0 +1,142 @@
import type { Client } from "ssh2";
import { execCommand, toFixedNum } from "./common-utils.js";
import { isSafeNodeName } from "../proxmox-shared.js";
export interface ProxmoxNodeStatusResult {
cpu: {
percent: number | null;
cores: number | null;
load: [number, number, number] | null;
};
memory: {
percent: number | null;
usedGiB: number | null;
totalGiB: number | null;
};
disk: {
percent: number | null;
usedGiB: number | null;
totalGiB: number | null;
};
uptime: {
seconds: number | null;
formatted: string | null;
};
system: {
hostname: string | null;
kernel: string | null;
pveVersion: string | null;
};
}
const EMPTY_RESULT: ProxmoxNodeStatusResult = {
cpu: { percent: null, cores: null, load: null },
memory: { percent: null, usedGiB: null, totalGiB: null },
disk: { percent: null, usedGiB: null, totalGiB: null },
uptime: { seconds: null, formatted: null },
system: { hostname: null, kernel: null, pveVersion: null },
};
function bytesToGiB(bytes: number): number {
return bytes / (1024 * 1024 * 1024);
}
function formatUptime(seconds: number): string {
const days = Math.floor(seconds / 86400);
const hours = Math.floor((seconds % 86400) / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
return `${days}d ${hours}h ${minutes}m`;
}
export async function collectProxmoxNodeStatus(
client: Client,
nodeName: string,
): Promise<ProxmoxNodeStatusResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
`pvesh get /nodes/${nodeName}/status --output-format json`,
25000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const data = JSON.parse(stdout) as Record<string, unknown>;
const cpuFraction = typeof data.cpu === "number" ? data.cpu : null;
const cpuinfo = (data.cpuinfo as Record<string, unknown>) || {};
const cores = typeof cpuinfo.cores === "number" ? cpuinfo.cores : null;
const loadavgRaw = data.loadavg;
let load: [number, number, number] | null = null;
if (Array.isArray(loadavgRaw) && loadavgRaw.length >= 3) {
const parsed = loadavgRaw
.slice(0, 3)
.map((v) => Number(v))
.map((v) => (Number.isFinite(v) ? v : 0));
load = parsed as [number, number, number];
}
const memory = (data.memory as Record<string, unknown>) || {};
const memUsed = typeof memory.used === "number" ? memory.used : null;
const memTotal = typeof memory.total === "number" ? memory.total : null;
const memPercent =
memUsed !== null && memTotal !== null && memTotal > 0
? Math.max(0, Math.min(100, (memUsed / memTotal) * 100))
: null;
const rootfs = (data.rootfs as Record<string, unknown>) || {};
const diskUsed = typeof rootfs.used === "number" ? rootfs.used : null;
const diskTotal = typeof rootfs.total === "number" ? rootfs.total : null;
const diskPercent =
diskUsed !== null && diskTotal !== null && diskTotal > 0
? Math.max(0, Math.min(100, (diskUsed / diskTotal) * 100))
: null;
const uptimeSeconds = typeof data.uptime === "number" ? data.uptime : null;
return {
cpu: {
percent: toFixedNum(cpuFraction !== null ? cpuFraction * 100 : null, 0),
cores,
load,
},
memory: {
percent: toFixedNum(memPercent, 0),
usedGiB: memUsed !== null ? toFixedNum(bytesToGiB(memUsed), 2) : null,
totalGiB:
memTotal !== null ? toFixedNum(bytesToGiB(memTotal), 2) : null,
},
disk: {
percent: toFixedNum(diskPercent, 0),
usedGiB: diskUsed !== null ? toFixedNum(bytesToGiB(diskUsed), 2) : null,
totalGiB:
diskTotal !== null ? toFixedNum(bytesToGiB(diskTotal), 2) : null,
},
uptime: {
seconds: uptimeSeconds,
formatted: uptimeSeconds !== null ? formatUptime(uptimeSeconds) : null,
},
system: {
hostname:
typeof data.hostname === "string" && data.hostname
? data.hostname
: null,
kernel:
typeof data.kversion === "string" && data.kversion
? data.kversion
: null,
pveVersion:
typeof data.pveversion === "string" && data.pveversion
? data.pveversion
: null,
},
};
} catch {
return EMPTY_RESULT;
}
}
@@ -0,0 +1,76 @@
import type { Client } from "ssh2";
import { execCommand, toFixedNum } from "./common-utils.js";
import { isSafeNodeName } from "../proxmox-shared.js";
export interface ProxmoxStoragePoolEntry {
name: string;
type: string;
active: boolean;
enabled: boolean;
usedGiB: number | null;
totalGiB: number | null;
availGiB: number | null;
percent: number | null;
}
export interface ProxmoxStorageResult {
pools: ProxmoxStoragePoolEntry[];
}
const EMPTY_RESULT: ProxmoxStorageResult = { pools: [] };
function bytesToGiB(bytes: number): number {
return bytes / (1024 * 1024 * 1024);
}
export async function collectProxmoxStorage(
client: Client,
nodeName: string,
): Promise<ProxmoxStorageResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
`pvesh get /nodes/${nodeName}/storage --output-format json`,
25000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const data = JSON.parse(stdout);
if (!Array.isArray(data)) {
return EMPTY_RESULT;
}
const pools: ProxmoxStoragePoolEntry[] = (
data as Array<Record<string, unknown>>
).map((entry) => {
const used = typeof entry.used === "number" ? entry.used : null;
const total = typeof entry.total === "number" ? entry.total : null;
const avail = typeof entry.avail === "number" ? entry.avail : null;
const percent =
used !== null && total !== null && total > 0
? Math.max(0, Math.min(100, (used / total) * 100))
: null;
return {
name: typeof entry.storage === "string" ? entry.storage : "",
type: typeof entry.type === "string" ? entry.type : "unknown",
active: entry.active === 1 || entry.active === true,
enabled: entry.enabled === 1 || entry.enabled === true,
usedGiB: used !== null ? toFixedNum(bytesToGiB(used), 2) : null,
totalGiB: total !== null ? toFixedNum(bytesToGiB(total), 2) : null,
availGiB: avail !== null ? toFixedNum(bytesToGiB(avail), 2) : null,
percent: toFixedNum(percent, 0),
};
});
return { pools };
} catch {
return EMPTY_RESULT;
}
}
+213 -219
View File
@@ -1,36 +1,27 @@
import { getErrorMessage } from "../../../../src/backend/utils/error-message.js";
import express, { type Router } from "express";
import type { Client as SSHClient } from "ssh2";
import { logger } from "../../../../src/backend/utils/logger.js";
import { DataCrypto } from "../../../../src/backend/utils/data-crypto.js";
import { createCurrentHostRepository } from "../../../../src/backend/database/repositories/factory.js";
import { AuthManager } from "../../../../src/backend/utils/auth-manager.js";
import {
type AuthenticatedRequest,
type SSHHost,
} from "../../../../src/types/index.js";
import { resolveHostById } from "../../../../src/backend/hosts/host-resolver.js";
import type { PluginSshHost } from "@termix/plugin-sdk/backend";
import { execElevated } from "../../../../src/backend/hosts/metrics-shared/exec-elevated.js";
import type {
PluginHostCreateInput,
PluginHostRecord,
PluginHostUpdateInput,
} from "@termix/plugin-sdk/backend";
import { connectSsh } from "./ssh.js";
import { pluginCtx } from "./plugin-ctx.js";
import { resolveProxmoxImportAuth } from "./proxmox-import-auth.js";
import {
parseProxmoxJumpHosts,
serializeProxmoxJumpHosts,
} from "./proxmox-jump-hosts.js";
import { isSafeNodeName } from "../../../../src/backend/hosts/proxmox-shared.js";
import { execElevated } from "../../../../src/backend/hosts/metrics-shared/exec-elevated.js";
import { isSafeNodeName } from "./proxmox-shared.js";
const router = express.Router();
const proxmoxLogger = logger;
const runningSyncs = new Set<string>();
const MIN_SYNC_INTERVAL_MINUTES = 5;
const DEFAULT_SYNC_INTERVAL_MINUTES = 15;
const authManager = AuthManager.getInstance();
const authenticateJWT = authManager.createAuthMiddleware();
const requireDataAccess = authManager.createDataAccessMiddleware();
// Helpers
function execCommand(
@@ -189,6 +180,9 @@ type ProxmoxGuest = {
enableDocker: boolean;
};
/** The resolved host ctx.ssh.connect hands back, plus the sudo secret proxmox needs for pvesh. */
type PluginSshHostWithSudo = PluginHostRecord & { sudoPassword?: string };
type ProxmoxSource = {
source: "proxmox";
sourceHostId: number;
@@ -285,44 +279,30 @@ async function discoverProxmoxGuestsForHost(
parsedHostId: number,
onProgress?: (done: number, total: number) => void,
): Promise<{
host: SSHHost;
host: PluginSshHostWithSudo;
guests: ProxmoxGuest[];
credentialId: number | null;
defaultCredentialId: number | null;
jumpHosts: unknown[] | null;
config: ReturnType<typeof parseProxmoxConfig>;
}> {
if (!DataCrypto.canUserAccessData(userId)) {
const error = new Error("Session expired — please log in again");
(error as Error & { code?: string }).code = "SESSION_EXPIRED";
throw error;
}
const resolvedHost = await resolveHostById(parsedHostId, userId);
if (!resolvedHost) {
const error = new Error("Host not found");
(error as Error & { status?: number }).status = 404;
throw error;
}
const host = resolvedHost as SSHHost;
const proxmoxCfgRaw = parseJsonObject(host.proxmoxConfig);
const ctx = pluginCtx();
const proxmoxCfgRaw = await ctx.settings.getHost(
parsedHostId,
"proxmoxConfig",
);
const config = parseProxmoxConfig(proxmoxCfgRaw);
const hostCredentialId = host.credentialId ?? null;
const { client } = await connectSsh(host as unknown as PluginSshHost, {
const { client, host } = await connectSsh(parsedHostId, {
purpose: "proxmox",
timeoutMs: 35000,
overrides: { tryKeyboard: false, readyTimeout: 30000 },
});
const hostWithSudo = host as PluginSshHostWithSudo;
const hostCredentialId = (hostWithSudo.credentialId as number | null) ?? null;
try {
proxmoxLogger.info("Proxmox discovery SSH connection established", {
operation: "proxmox_discover",
hostId: parsedHostId,
userId,
});
ctx.log.info("Proxmox discovery SSH connection established");
const pveshCheck = await execCommand(
client,
@@ -337,7 +317,7 @@ async function discoverProxmoxGuestsForHost(
const resourcesJson = await execPveshCommand(
client,
"pvesh get /cluster/resources --output-format json 2>/dev/null",
host.sudoPassword,
hostWithSudo.sudoPassword,
);
let resources: Array<Record<string, unknown>>;
@@ -366,11 +346,9 @@ async function discoverProxmoxGuestsForHost(
if (r.template) continue;
const node = r.node as string;
if (!isSafeNodeName(node)) {
proxmoxLogger.warn("Skipping guest with unsafe node name", {
operation: "proxmox_discover",
node,
vmid: r.vmid,
});
ctx.log.warn(
`Skipping guest with unsafe node name: ${node} (vmid ${r.vmid})`,
);
continue;
}
guestBases.push({
@@ -389,7 +367,7 @@ async function discoverProxmoxGuestsForHost(
const cfgJson = await execPveshCommand(
client,
`pvesh get /nodes/${g.node}/lxc/${g.vmid}/config --output-format json 2>/dev/null`,
host.sudoPassword,
hostWithSudo.sudoPassword,
25000,
);
configIp = parseLxcIp(JSON.parse(cfgJson), config.preferredPrefixes);
@@ -404,7 +382,7 @@ async function discoverProxmoxGuestsForHost(
const ifRaw = await execPveshCommand(
client,
`pvesh get /nodes/${g.node}/lxc/${g.vmid}/interfaces --output-format json 2>/dev/null`,
host.sudoPassword,
hostWithSudo.sudoPassword,
12000,
);
const data = JSON.parse(ifRaw);
@@ -437,7 +415,7 @@ async function discoverProxmoxGuestsForHost(
const ifJson = await execPveshCommand(
client,
`pvesh get /nodes/${g.node}/qemu/${g.vmid}/agent/network-get-interfaces --output-format json 2>/dev/null`,
host.sudoPassword,
hostWithSudo.sudoPassword,
12000,
);
const data = JSON.parse(ifJson);
@@ -508,21 +486,16 @@ async function discoverProxmoxGuestsForHost(
enableDocker: matchesAny(g.name, config.dockerPatterns),
}));
proxmoxLogger.info("Proxmox discovery completed", {
operation: "proxmox_discover",
hostId: parsedHostId,
userId,
guestCount: guests.length,
});
ctx.log.info(
`Proxmox discovery completed for host ${parsedHostId}: ${guests.length} guest(s)`,
);
return {
host,
host: hostWithSudo,
guests,
credentialId: hostCredentialId,
defaultCredentialId: config.defaultCredentialId,
jumpHosts: parseProxmoxJumpHosts(
(host as unknown as { jumpHosts?: unknown }).jumpHosts,
),
jumpHosts: parseProxmoxJumpHosts(hostWithSudo.jumpHosts),
config,
};
} finally {
@@ -538,6 +511,7 @@ async function syncProxmoxHost(
userId: string,
sourceHostId: number,
): Promise<ProxmoxSyncResult> {
const ctx = pluginCtx();
const lockKey = `${userId}:${sourceHostId}`;
if (runningSyncs.has(lockKey)) {
return {
@@ -570,10 +544,10 @@ async function syncProxmoxHost(
);
const now = new Date().toISOString();
const existingHosts =
(await createCurrentHostRepository().listDecryptedByUserId(
userId,
)) as unknown as Record<string, unknown>[];
const existingHosts = (await ctx.hosts.listOwned()) as unknown as Record<
string,
unknown
>[];
const existingBySource = new Map<string, Record<string, unknown>>();
for (const host of existingHosts) {
const source = getProxmoxSource(host);
@@ -647,10 +621,9 @@ async function syncProxmoxHost(
update.credentialId = importAuth.credentialId;
update.overrideCredentialUsername = false;
}
await createCurrentHostRepository().updateEncryptedForUser(
userId,
await ctx.hosts.update(
existing.id as number,
update,
update as PluginHostUpdateInput,
);
result.updated++;
continue;
@@ -665,9 +638,8 @@ async function syncProxmoxHost(
enableRdp: connectionType === "rdp",
});
await createCurrentHostRepository().createEncryptedForUser(userId, {
await ctx.hosts.create({
...update,
userId,
createdAt: now,
pin: false,
authType: connectionType === "rdp" ? "password" : importAuth.authType,
@@ -709,7 +681,7 @@ async function syncProxmoxHost(
showTunnelInSidebar: 0,
showDockerInSidebar: 0,
showServerStatsInSidebar: 0,
});
} as unknown as PluginHostCreateInput);
result.created++;
}
@@ -720,8 +692,7 @@ async function syncProxmoxHost(
const source = getProxmoxSource(existing);
if (!source) continue;
const missingSince = source.missingSince || now;
await createCurrentHostRepository().updateEncryptedForUser(
userId,
await ctx.hosts.update(
existing.id as number,
{
tags: mergeTags(existing.tags, ["proxmox-missing"]),
@@ -733,13 +704,13 @@ async function syncProxmoxHost(
},
}),
updatedAt: now,
},
} as PluginHostUpdateInput,
);
result.markedMissing++;
}
}
await writeSyncStatus(userId, sourceHostId, {
await writeSyncStatus(sourceHostId, {
lastSyncAt: startedAt,
lastSyncStatus: "success",
lastSyncError: null,
@@ -750,7 +721,7 @@ async function syncProxmoxHost(
} catch (error) {
const message = getErrorMessage(error);
result.errors.push(message);
await writeSyncStatus(userId, sourceHostId, {
await writeSyncStatus(sourceHostId, {
lastSyncAt: startedAt,
lastSyncStatus: "error",
lastSyncError: message,
@@ -763,22 +734,46 @@ async function syncProxmoxHost(
}
async function writeSyncStatus(
userId: string,
hostId: number,
patch: Record<string, unknown>,
): Promise<void> {
const hostRepository = createCurrentHostRepository();
const row = await hostRepository.findByIdForUser(userId, hostId);
if (!row) return;
const config = parseJsonObject(row.proxmoxConfig);
await hostRepository.updateForUser(userId, hostId, {
proxmoxConfig: JSON.stringify({ ...config, ...patch }),
});
const ctx = pluginCtx();
const config = parseJsonObject(
await ctx.settings.getHost(hostId, "proxmoxConfig"),
);
await ctx.settings.setHost(hostId, "proxmoxConfig", { ...config, ...patch });
}
router.post("/sync", authenticateJWT, requireDataAccess, async (req, res) => {
/**
* @openapi
* /proxmox/sync:
* post:
* summary: Sync Proxmox guests for a host
* description: Re-runs discovery for a Proxmox node and creates, updates or marks missing the imported guest hosts.
* tags: [Proxmox]
* security:
* - bearerAuth: []
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* required: [hostId]
* properties:
* hostId:
* type: number
* responses:
* 200:
* description: Sync result.
* 400:
* description: Missing or invalid hostId.
* 500:
* description: Sync failed.
*/
router.post("/sync", async (req, res) => {
const { hostId } = req.body as { hostId?: unknown };
const userId = (req as unknown as AuthenticatedRequest).userId;
const userId = pluginCtx().currentActor()!;
const parsedHostId = Number(hostId);
if (!hostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
@@ -790,30 +785,48 @@ router.post("/sync", authenticateJWT, requireDataAccess, async (req, res) => {
return res.json(result);
} catch (err: unknown) {
const message = getErrorMessage(err);
const status =
(err as Error & { code?: string; status?: number }).code ===
"SESSION_EXPIRED"
? 401
: (err as Error & { status?: number }).status || 500;
proxmoxLogger.error("Proxmox sync failed", err, {
operation: "proxmox_sync",
hostId: parsedHostId,
userId,
});
const status = (err as Error & { status?: number }).status || 500;
pluginCtx().log.error(
`Proxmox sync failed for host ${parsedHostId}`,
err as Error,
);
return res.status(status).json({ error: `Sync failed: ${message}` });
}
});
async function runDueProxmoxAutoSyncs(): Promise<void> {
const ctx = pluginCtx();
try {
const rows = await createCurrentHostRepository().listProxmoxEnabled();
const {
createCurrentPluginSettingsRepository,
createCurrentHostRepository,
} =
await import("../../../../src/backend/database/repositories/factory.js");
const enabledRows = await createCurrentPluginSettingsRepository().listByKey(
"proxmox",
"host",
"enableProxmox",
);
const hostRepository = createCurrentHostRepository();
const now = Date.now();
for (const row of rows) {
const configRaw = parseJsonObject(row.proxmoxConfig);
for (const row of enabledRows) {
if (row.value !== "true") continue;
const hostId = Number(row.scopeId);
if (!Number.isInteger(hostId)) continue;
const host = await hostRepository.findById(hostId);
if (!host) continue;
const configRow = await createCurrentPluginSettingsRepository().get(
"proxmox",
"host",
String(hostId),
"proxmoxConfig",
);
const configRaw = configRow?.value ? JSON.parse(configRow.value) : {};
const config = parseProxmoxConfig(configRaw);
if (!config.autoSyncEnabled) continue;
if (!DataCrypto.canUserAccessData(row.userId)) continue;
const lastSyncAt =
typeof configRaw.lastSyncAt === "string"
@@ -822,18 +835,17 @@ async function runDueProxmoxAutoSyncs(): Promise<void> {
const intervalMs = config.syncIntervalMinutes * 60 * 1000;
if (lastSyncAt && now - lastSyncAt < intervalMs) continue;
syncProxmoxHost(row.userId, row.id).catch((error) => {
proxmoxLogger.error("Scheduled Proxmox sync failed", error, {
operation: "proxmox_auto_sync",
hostId: row.id,
userId: row.userId,
ctx
.asUser(host.userId, () => syncProxmoxHost(host.userId, hostId))
.catch((error) => {
ctx.log.error(
`Scheduled Proxmox sync failed for host ${hostId}`,
error as Error,
);
});
});
}
} catch (error) {
proxmoxLogger.error("Failed to scan Proxmox auto sync jobs", error, {
operation: "proxmox_auto_sync_scan",
});
ctx.log.error("Failed to scan Proxmox auto sync jobs", error as Error);
}
}
@@ -915,127 +927,109 @@ let proxmoxAutoSyncStartupTimer: NodeJS.Timeout | undefined;
* 500:
* description: Discovery failed.
*/
router.get(
"/discover/stream",
authenticateJWT,
requireDataAccess,
async (req, res) => {
const userId = (req as unknown as AuthenticatedRequest).userId;
const parsedHostId = Number((req.query as { hostId?: unknown }).hostId);
if (!parsedHostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
return res.status(400).json({ error: "Missing or invalid hostId" });
router.get("/discover/stream", async (req, res) => {
const userId = pluginCtx().currentActor()!;
const parsedHostId = Number((req.query as { hostId?: unknown }).hostId);
if (!parsedHostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
return res.status(400).json({ error: "Missing or invalid hostId" });
}
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-store, no-transform",
Connection: "keep-alive",
"X-Accel-Buffering": "no",
});
res.flushHeaders?.();
let closed = false;
const send = (event: string, data: unknown) => {
if (closed) return;
try {
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
} catch {
closed = true;
}
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-store, no-transform",
Connection: "keep-alive",
"X-Accel-Buffering": "no",
});
res.flushHeaders?.();
let closed = false;
const send = (event: string, data: unknown) => {
if (closed) return;
try {
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
} catch {
closed = true;
}
};
const heartbeat = setInterval(() => {
if (closed) return;
try {
res.write(": keepalive\n\n");
} catch {
closed = true;
clearInterval(heartbeat);
}
}, 15000);
req.on("close", () => {
};
const heartbeat = setInterval(() => {
if (closed) return;
try {
res.write(": keepalive\n\n");
} catch {
closed = true;
clearInterval(heartbeat);
});
}
}, 15000);
req.on("close", () => {
closed = true;
clearInterval(heartbeat);
});
try {
const discovery = await discoverProxmoxGuestsForHost(
userId,
parsedHostId,
(done, total) => send("progress", { done, total }),
);
send("result", {
guests: discovery.guests,
credentialId: discovery.credentialId,
defaultCredentialId: discovery.defaultCredentialId,
jumpHosts: discovery.jumpHosts,
});
} catch (err: unknown) {
const message = getErrorMessage(err);
proxmoxLogger.error("Proxmox discovery (stream) failed", err, {
operation: "proxmox_discover",
hostId: parsedHostId,
userId,
});
send("fail", { message });
} finally {
clearInterval(heartbeat);
if (!closed) {
try {
res.end();
} catch {
// ignore end errors
}
try {
const discovery = await discoverProxmoxGuestsForHost(
userId,
parsedHostId,
(done, total) => send("progress", { done, total }),
);
send("result", {
guests: discovery.guests,
credentialId: discovery.credentialId,
defaultCredentialId: discovery.defaultCredentialId,
jumpHosts: discovery.jumpHosts,
});
} catch (err: unknown) {
const message = getErrorMessage(err);
pluginCtx().log.error(
`Proxmox discovery (stream) failed for host ${parsedHostId}`,
err as Error,
);
send("fail", { message });
} finally {
clearInterval(heartbeat);
if (!closed) {
try {
res.end();
} catch {
// ignore end errors
}
}
},
);
}
});
router.post(
"/discover",
authenticateJWT,
requireDataAccess,
async (req, res) => {
const { hostId } = req.body as { hostId?: unknown };
const userId = (req as unknown as AuthenticatedRequest).userId;
router.post("/discover", async (req, res) => {
const { hostId } = req.body as { hostId?: unknown };
const userId = pluginCtx().currentActor()!;
const parsedHostId = Number(hostId);
if (!hostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
return res.status(400).json({ error: "Missing or invalid hostId" });
}
const parsedHostId = Number(hostId);
if (!hostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
return res.status(400).json({ error: "Missing or invalid hostId" });
}
try {
const discovery = await discoverProxmoxGuestsForHost(
userId,
parsedHostId,
);
return res.json({
guests: discovery.guests,
credentialId: discovery.credentialId,
defaultCredentialId: discovery.defaultCredentialId,
jumpHosts: discovery.jumpHosts,
});
} catch (err: unknown) {
const message = getErrorMessage(err);
proxmoxLogger.error("Proxmox discovery failed", err, {
operation: "proxmox_discover",
hostId: parsedHostId,
userId,
});
try {
const discovery = await discoverProxmoxGuestsForHost(userId, parsedHostId);
return res.json({
guests: discovery.guests,
credentialId: discovery.credentialId,
defaultCredentialId: discovery.defaultCredentialId,
jumpHosts: discovery.jumpHosts,
});
} catch (err: unknown) {
const message = getErrorMessage(err);
pluginCtx().log.error(
`Proxmox discovery failed for host ${parsedHostId}`,
err as Error,
);
const status =
(err as Error & { code?: string; status?: number }).code ===
"SESSION_EXPIRED"
? 401
: (err as Error & { status?: number }).status ||
(message.includes("Authentication failed") ||
message.includes("connect ECONNREFUSED") ||
message.includes("connect ETIMEDOUT")
? 422
: 500);
return res.status(status).json({ error: `Discovery failed: ${message}` });
}
},
);
const status =
(err as Error & { status?: number }).status ||
(message.includes("Authentication failed") ||
message.includes("connect ECONNREFUSED") ||
message.includes("connect ETIMEDOUT")
? 422
: 500);
return res.status(status).json({ error: `Discovery failed: ${message}` });
}
});
/** Called from activate(). Mounts this router at /proxmox via the shared
* dispatcher, and starts the background auto-sync scan (a 60s interval plus
@@ -0,0 +1,366 @@
/**
* Proxmox node stats: polling, viewer sessions and their routes.
*
* Moved here from host-metrics, which only ever hosted this because the
* plugin split predates B5. host-metrics must not import proxmox code, so
* this plugin owns its own polling manager, SSH connections and routes.
*/
import type { Router } from "express";
import express from "express";
import type { PluginContext } from "@termix/plugin-sdk/backend";
import { pluginCtx } from "./plugin-ctx.js";
import {
ProxmoxPollingManager,
parseProxmoxStatsConfig,
type ProxmoxStatsPollableHost,
} from "./proxmox-stats-polling.js";
import type { ProxmoxNodeHistoryRepository } from "./proxmox-node-history-repository.js";
const EMPTY_SNAPSHOT = {
node: {
cpu: { percent: null, cores: null, load: null },
memory: { percent: null, usedGiB: null, totalGiB: null },
disk: { percent: null, usedGiB: null, totalGiB: null },
uptime: { seconds: null, formatted: null },
system: { hostname: null, kernel: null, pveVersion: null },
},
network: { interfaces: [] },
guests: { guests: [], counts: { running: 0, stopped: 0, total: 0 } },
storage: { pools: [] },
cluster: { clustered: false },
lastChecked: new Date(0).toISOString(),
};
type PollableHost = ProxmoxStatsPollableHost & { enableProxmoxStats: boolean };
let pollingManager: ProxmoxPollingManager<PollableHost> | undefined;
let router: Router | undefined;
async function fetchHostForPolling(
ctx: PluginContext,
hostId: number,
userId: string,
): Promise<PollableHost | undefined> {
const enabled = await ctx.asUser(userId, () =>
ctx.settings.getHost<boolean>(hostId, "enableProxmoxStats"),
);
if (!enabled) return undefined;
const host = await ctx.asUser(userId, () => ctx.hosts.get(hostId));
if (!host) return undefined;
const proxmoxStatsConfig = await ctx.asUser(userId, () =>
ctx.settings.getHost(hostId, "proxmoxStatsConfig"),
);
return {
id: host.id,
userId: host.userId,
proxmoxStatsConfig: proxmoxStatsConfig as string | undefined,
enableProxmoxStats: true,
};
}
/** Called from activate(). Builds the polling manager and mounts its routes. */
export function startProxmoxStatsService(
ctx: PluginContext,
mountOn: Router,
historyRepository: ProxmoxNodeHistoryRepository,
): void {
pollingManager = new ProxmoxPollingManager<PollableHost>({
fetchHostById: (hostId, userId) => fetchHostForPolling(ctx, hostId, userId),
withSshConnection: (host, fn) =>
ctx.ssh.withConnection(
host.id,
{
pool: "proxmox-stats",
purpose: "proxmox",
overrides: { readyTimeout: 60000 },
},
fn,
),
historyRepository,
});
router = express.Router();
registerRoutes(router, pollingManager, historyRepository);
mountOn.use("/stats", router);
}
export function stopProxmoxStatsService(): void {
pollingManager?.destroy();
pollingManager = undefined;
router = undefined;
}
function registerRoutes(
app: Router,
manager: ProxmoxPollingManager<PollableHost>,
historyRepository: ProxmoxNodeHistoryRepository,
): void {
/**
* @openapi
* /plugin-api/proxmox/stats/{id}:
* get:
* summary: Get cached Proxmox node stats for a host
* description: Returns the most recently polled Proxmox stats snapshot for a host, or an empty skeleton if none has been collected yet.
* tags: [Proxmox]
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* responses:
* 200:
* description: Proxmox stats snapshot.
* 404:
* description: Stats not available yet.
*/
app.get("/:id", async (req, res) => {
const id = Number(req.params.id);
const cached = manager.getStats(id);
if (!cached) {
const errorState = manager.getError(id);
return res.status(404).json({
error: errorState?.error || "Stats not available",
...EMPTY_SNAPSHOT,
lastChecked: new Date().toISOString(),
});
}
res.json({
...cached.data,
lastChecked: new Date(cached.timestamp).toISOString(),
});
});
/**
* @openapi
* /plugin-api/proxmox/stats/start/{id}:
* post:
* summary: Start Proxmox stats collection
* description: Registers a viewer and starts (or reuses) polling for a host's Proxmox node stats.
* tags: [Proxmox]
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* responses:
* 200:
* description: Polling started, snapshot returned if already available.
* 403:
* description: Proxmox stats is not enabled for this host.
* 404:
* description: Host not found.
*/
app.post("/start/:id", async (req, res) => {
const id = Number(req.params.id);
const userId = pluginCtx().currentActor()!;
try {
const host = await fetchHostForPolling(pluginCtx(), id, userId);
if (!host) {
return res
.status(403)
.json({ error: "Proxmox stats is not enabled for this host" });
}
const viewerSessionId = `proxmox-viewer-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
manager.registerViewer(id, viewerSessionId, userId);
await manager.ensurePolling(host, userId);
const cached = manager.getStats(id);
if (cached) {
return res.json({
success: true,
viewerSessionId,
...cached.data,
lastChecked: new Date(cached.timestamp).toISOString(),
});
}
const errorState = manager.getError(id);
if (errorState) {
return res.json({
success: true,
viewerSessionId,
status: "error",
error: errorState.error,
});
}
return res.json({ success: true, viewerSessionId, status: "collecting" });
} catch (error) {
pluginCtx().log.error(
`Failed to start proxmox stats collection for host ${id}`,
error as Error,
);
res
.status(500)
.json({ error: "Failed to start proxmox stats collection" });
}
});
/**
* @openapi
* /plugin-api/proxmox/stats/stop/{id}:
* post:
* summary: Stop Proxmox stats collection
* description: Unregisters a viewer session for a host's Proxmox node stats polling.
* tags: [Proxmox]
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* requestBody:
* required: false
* content:
* application/json:
* schema:
* type: object
* properties:
* viewerSessionId:
* type: string
* responses:
* 200:
* description: Polling stopped successfully.
*/
app.post("/stop/:id", (req, res) => {
const id = Number(req.params.id);
const { viewerSessionId } = req.body as { viewerSessionId?: string };
if (viewerSessionId && typeof viewerSessionId === "string") {
manager.unregisterViewer(id, viewerSessionId);
}
res.json({ success: true });
});
/**
* @openapi
* /plugin-api/proxmox/stats/heartbeat:
* post:
* summary: Update Proxmox stats viewer heartbeat
* description: Keeps a Proxmox stats viewer session alive.
* tags: [Proxmox]
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* properties:
* viewerSessionId:
* type: string
* responses:
* 200:
* description: Heartbeat updated.
* 400:
* description: Invalid viewerSessionId.
* 404:
* description: Viewer session not found.
*/
app.post("/heartbeat", (req, res) => {
const { viewerSessionId } = req.body as { viewerSessionId?: string };
if (!viewerSessionId || typeof viewerSessionId !== "string") {
return res.status(400).json({ error: "Invalid viewerSessionId" });
}
const ok = manager.updateHeartbeat(viewerSessionId);
if (!ok) return res.status(404).json({ error: "Viewer session not found" });
res.json({ success: true });
});
/**
* @openapi
* /plugin-api/proxmox/stats/history/{hostId}:
* get:
* summary: Get historical Proxmox node stats for a host
* tags: [Proxmox]
* parameters:
* - in: path
* name: hostId
* required: true
* schema:
* type: integer
* - in: query
* name: range
* schema:
* type: string
* enum: [1h, 6h, 24h, 7d, 30d]
* - in: query
* name: from
* schema:
* type: string
* format: date-time
* - in: query
* name: to
* schema:
* type: string
* format: date-time
* responses:
* 200:
* description: Array of node history rows.
* 400:
* description: Invalid range or date.
*/
app.get("/history/:hostId", async (req, res) => {
const hostId = Number(req.params.hostId);
const { range, from, to } = req.query as Record<string, string | undefined>;
const RANGE_OFFSETS: Record<string, number> = {
"1h": 1 * 60 * 60 * 1000,
"6h": 6 * 60 * 60 * 1000,
"24h": 24 * 60 * 60 * 1000,
"7d": 7 * 24 * 60 * 60 * 1000,
"30d": 30 * 24 * 60 * 60 * 1000,
};
let fromTs: string;
let toTs = new Date().toISOString();
if (range) {
const offsetMs = RANGE_OFFSETS[range];
if (!offsetMs) {
return res
.status(400)
.json({ error: "Invalid range. Use 1h, 6h, 24h, 7d, or 30d" });
}
fromTs = new Date(Date.now() - offsetMs).toISOString();
} else if (from && to) {
const fromDate = new Date(from);
const toDate = new Date(to);
if (isNaN(fromDate.getTime()) || isNaN(toDate.getTime())) {
return res.status(400).json({ error: "Invalid from/to date format" });
}
fromTs = fromDate.toISOString();
toTs = toDate.toISOString();
} else {
fromTs = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
}
try {
const toSqlite = (iso: string) =>
iso.replace("T", " ").replace(/\.\d{3}Z$/, "");
const rows = (
await historyRepository.listRange(
hostId,
toSqlite(fromTs),
toSqlite(toTs),
)
).map((row) => ({
ts: row.ts,
cpu_percent: row.cpuPercent,
mem_percent: row.memPercent,
disk_percent: row.diskPercent,
net_rx_bytes: row.netRxBytes,
net_tx_bytes: row.netTxBytes,
}));
res.json({ rows, fromTs, toTs });
} catch (error) {
pluginCtx().log.error(
`Failed to fetch proxmox stats history for host ${hostId}`,
error as Error,
);
res.status(500).json({ error: "Failed to fetch proxmox stats history" });
}
});
}
export { parseProxmoxStatsConfig };
+64
View File
@@ -0,0 +1,64 @@
import {
adoptLegacyTable,
defineTable,
id,
integer,
real,
refHost,
refUser,
text,
timestamp,
} from "@termix/plugin-sdk/db";
/**
* Per-host historical node stats samples (CPU, memory, disk, network),
* polled on an interval while a viewer has the Proxmox stats tab open.
*
* Adopted from core's proxmox_node_history, so the column and index names are
* the legacy ones: the rename carries the existing rows across.
*/
export const proxmoxNodeHistory = adoptLegacyTable(
"proxmox_node_history",
defineTable("node_history", {
id: id(),
hostId: refHost(),
ts: timestamp().notNull().defaultNow(),
cpuPercent: real(),
memPercent: real(),
diskPercent: real(),
netRxBytes: integer(),
netTxBytes: integer(),
}),
);
/**
* One saved dashboard layout per user per host for the Proxmox stats tab.
*
* Adopted from core's proxmox_stats_preferences.
*/
export const proxmoxStatsPreferences = adoptLegacyTable(
"proxmox_stats_preferences",
defineTable(
"stats_preferences",
{
id: id(),
userId: refUser(),
hostId: refHost(),
// JSON-encoded layout. No secrets, so plain text.
layout: text().notNull(),
createdAt: timestamp().notNull().defaultNow(),
updatedAt: timestamp().notNull().defaultNow(),
},
{
indexes: [
{
name: "idx_proxmox_stats_prefs_user_host",
columns: ["userId", "hostId"],
unique: true,
},
],
},
),
);
export const tables = [proxmoxNodeHistory, proxmoxStatsPreferences];