mirror of
https://github.com/Termix-SSH/Termix.git
synced 2026-10-30 15:15:24 +00:00
feat: move proxmox node stats and collectors into the proxmox plugin
This commit is contained in:
1 parent
218c5088e7
commit
37c015a760
37 files changed
+3775
-227
No files matched your search
@@ -1,14 +1,36 @@
|
||||
import type { Router } from "express";
|
||||
import type { PluginContext } from "@termix/plugin-sdk/backend";
|
||||
import { setPluginSsh } from "./ssh.js";
|
||||
import { setPluginCtx } from "./plugin-ctx.js";
|
||||
import { startProxmoxService, stopProxmoxService } from "./routes.js";
|
||||
import {
|
||||
startProxmoxStatsService,
|
||||
stopProxmoxStatsService,
|
||||
} from "./stats-service.js";
|
||||
import { proxmoxNodeHistory } from "./tables.js";
|
||||
import { createProxmoxNodeHistoryRepository } from "./proxmox-node-history-repository.js";
|
||||
|
||||
export async function activate(ctx: PluginContext) {
|
||||
setPluginSsh(ctx.ssh);
|
||||
setPluginCtx(ctx);
|
||||
ctx.disposables.add(() => setPluginSsh(null));
|
||||
startProxmoxService(ctx.http.router());
|
||||
ctx.disposables.add(() => setPluginCtx(null));
|
||||
|
||||
const historyTable = await ctx.db.define(proxmoxNodeHistory);
|
||||
const historyRepository = createProxmoxNodeHistoryRepository(
|
||||
ctx.db,
|
||||
historyTable,
|
||||
);
|
||||
|
||||
const router = ctx.http.router<Router>();
|
||||
startProxmoxService(router);
|
||||
ctx.disposables.add(stopProxmoxService);
|
||||
startProxmoxStatsService(ctx, router, historyRepository);
|
||||
ctx.disposables.add(stopProxmoxStatsService);
|
||||
ctx.log.info("Proxmox routes mounted at /plugin-api/proxmox");
|
||||
}
|
||||
|
||||
export async function deactivate() {
|
||||
stopProxmoxService();
|
||||
stopProxmoxStatsService();
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { PluginContext } from "@termix/plugin-sdk/backend";
|
||||
|
||||
let current: PluginContext | null = null;
|
||||
|
||||
/** Set in activate, cleared on deactivate. */
|
||||
export function setPluginCtx(ctx: PluginContext | null): void {
|
||||
current = ctx;
|
||||
}
|
||||
|
||||
export function pluginCtx(): PluginContext {
|
||||
if (!current) throw new Error("The plugin is not active");
|
||||
return current;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { and, gte, lte, eq, asc, lt } from "drizzle-orm";
|
||||
import type { PluginDatabase } from "@termix/plugin-sdk/backend";
|
||||
|
||||
export interface ProxmoxNodeHistoryRow {
|
||||
id: number;
|
||||
hostId: number;
|
||||
ts: string;
|
||||
cpuPercent: number | null;
|
||||
memPercent: number | null;
|
||||
diskPercent: number | null;
|
||||
netRxBytes: number | null;
|
||||
netTxBytes: number | null;
|
||||
}
|
||||
|
||||
export interface ProxmoxNodeHistoryCreateInput {
|
||||
hostId: number;
|
||||
cpuPercent: number | null;
|
||||
memPercent: number | null;
|
||||
diskPercent: number | null;
|
||||
netRxBytes: number | null;
|
||||
netTxBytes: number | null;
|
||||
}
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
type Table = any;
|
||||
type Drizzle = any;
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
|
||||
export type ProxmoxNodeHistoryRepository = ReturnType<
|
||||
typeof createProxmoxNodeHistoryRepository
|
||||
>;
|
||||
|
||||
export function createProxmoxNodeHistoryRepository(
|
||||
db: PluginDatabase,
|
||||
table: Table,
|
||||
) {
|
||||
const client = () => db.client<Drizzle>();
|
||||
|
||||
return {
|
||||
async create(input: ProxmoxNodeHistoryCreateInput): Promise<void> {
|
||||
const drizzle = await client();
|
||||
await drizzle.insert(table).values(input);
|
||||
await db.persist();
|
||||
},
|
||||
|
||||
async listRange(
|
||||
hostId: number,
|
||||
fromIso: string,
|
||||
toIso: string,
|
||||
): Promise<ProxmoxNodeHistoryRow[]> {
|
||||
const drizzle = await client();
|
||||
return drizzle
|
||||
.select()
|
||||
.from(table)
|
||||
.where(
|
||||
and(
|
||||
eq(table.hostId, hostId),
|
||||
gte(table.ts, fromIso),
|
||||
lte(table.ts, toIso),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(table.ts));
|
||||
},
|
||||
|
||||
async pruneOlderThan(hostId: number, retentionDays: number): Promise<void> {
|
||||
// ts columns store CURRENT_TIMESTAMP as "YYYY-MM-DD HH:MM:SS" UTC, which
|
||||
// sorts lexicographically, so the cutoff is a plain string comparison.
|
||||
const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000)
|
||||
.toISOString()
|
||||
.slice(0, 19)
|
||||
.replace("T", " ");
|
||||
const drizzle = await client();
|
||||
await drizzle
|
||||
.delete(table)
|
||||
.where(and(eq(table.hostId, hostId), lt(table.ts, cutoff)));
|
||||
await db.persist();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// Proxmox node names are restricted to [a-zA-Z0-9-] by PVE itself,
|
||||
// but we validate defensively before using in a shell command.
|
||||
const SAFE_NODE_RE = /^[a-zA-Z0-9._-]{1,64}$/;
|
||||
|
||||
export function isSafeNodeName(name: string): boolean {
|
||||
return SAFE_NODE_RE.test(name);
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
import {
|
||||
collectProxmoxStats,
|
||||
type ProxmoxStatsSnapshot,
|
||||
} from "./proxmox/collect-proxmox-stats.js";
|
||||
import type { ProxmoxNodeHistoryRepository } from "./proxmox-node-history-repository.js";
|
||||
import { pluginCtx } from "./plugin-ctx.js";
|
||||
import type { Client } from "ssh2";
|
||||
|
||||
/**
|
||||
* Proxmox stats polling gets its own concurrency limiter so a burst of polls
|
||||
* on many hosts cannot starve one host's collection.
|
||||
*/
|
||||
class ConcurrentLimiter {
|
||||
private running = 0;
|
||||
private queue: Array<() => void> = [];
|
||||
constructor(private readonly max: number) {}
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (this.running >= this.max) {
|
||||
await new Promise<void>((resolve) => this.queue.push(resolve));
|
||||
}
|
||||
this.running++;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.running--;
|
||||
const next = this.queue.shift();
|
||||
if (next) next();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const proxmoxStatsPollLimiter = new ConcurrentLimiter(5);
|
||||
|
||||
export interface ProxmoxStatsPollableHost {
|
||||
id: number;
|
||||
userId: string;
|
||||
proxmoxStatsConfig?: string | ProxmoxStatsPollConfig | null;
|
||||
}
|
||||
|
||||
export interface ProxmoxStatsPollConfig {
|
||||
nodeName?: string | null;
|
||||
pollInterval?: number;
|
||||
enabledCards?: string[];
|
||||
}
|
||||
|
||||
const DEFAULT_POLL_INTERVAL_SECONDS = 60;
|
||||
|
||||
export function parseProxmoxStatsConfig(
|
||||
raw: string | ProxmoxStatsPollConfig | null | undefined,
|
||||
): ProxmoxStatsPollConfig {
|
||||
if (!raw) {
|
||||
return { nodeName: null, pollInterval: DEFAULT_POLL_INTERVAL_SECONDS };
|
||||
}
|
||||
if (typeof raw === "object") {
|
||||
return {
|
||||
nodeName: raw.nodeName ?? null,
|
||||
pollInterval: raw.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS,
|
||||
enabledCards: raw.enabledCards,
|
||||
};
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as ProxmoxStatsPollConfig;
|
||||
return {
|
||||
nodeName: parsed.nodeName ?? null,
|
||||
pollInterval: parsed.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS,
|
||||
enabledCards: parsed.enabledCards,
|
||||
};
|
||||
} catch {
|
||||
return { nodeName: null, pollInterval: DEFAULT_POLL_INTERVAL_SECONDS };
|
||||
}
|
||||
}
|
||||
|
||||
interface HostPollingEntry<THost extends ProxmoxStatsPollableHost> {
|
||||
host: THost;
|
||||
timer?: NodeJS.Timeout;
|
||||
viewerUserId?: string;
|
||||
}
|
||||
|
||||
interface ViewerDetail {
|
||||
sessionId: string;
|
||||
userId: string;
|
||||
hostId: number;
|
||||
lastHeartbeat: number;
|
||||
}
|
||||
|
||||
interface CachedSnapshot {
|
||||
data: ProxmoxStatsSnapshot;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
interface ErrorSnapshot {
|
||||
error: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export class ProxmoxPollingManager<
|
||||
THost extends ProxmoxStatsPollableHost = ProxmoxStatsPollableHost,
|
||||
> {
|
||||
private pollingConfigs = new Map<number, HostPollingEntry<THost>>();
|
||||
private snapshotStore = new Map<number, CachedSnapshot>();
|
||||
private errorStore = new Map<number, ErrorSnapshot>();
|
||||
private activeViewers = new Map<number, Set<string>>();
|
||||
private viewerDetails = new Map<string, ViewerDetail>();
|
||||
private inFlight = new Set<number>();
|
||||
private viewerCleanupInterval: NodeJS.Timeout;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
fetchHostById: (
|
||||
hostId: number,
|
||||
userId: string,
|
||||
) => Promise<THost | undefined>;
|
||||
withSshConnection: <T>(
|
||||
host: THost,
|
||||
fn: (client: Client) => Promise<T>,
|
||||
) => Promise<T>;
|
||||
historyRepository: ProxmoxNodeHistoryRepository;
|
||||
historyEnabled?: () => boolean;
|
||||
},
|
||||
) {
|
||||
this.viewerCleanupInterval = setInterval(() => {
|
||||
this.cleanupInactiveViewers();
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
private intervalWithJitter(intervalMs: number, hostId: number): number {
|
||||
const spread = Math.min(intervalMs * 0.2, 15_000);
|
||||
const jitter = (hostId * 1103515245) % Math.max(1, Math.floor(spread));
|
||||
return intervalMs + jitter;
|
||||
}
|
||||
|
||||
// Host Metrics' own retention setting is not on the core settings
|
||||
// allowlist, so this plugin uses a fixed default rather than reaching for
|
||||
// it. See FINISH-LIST.md.
|
||||
private readonly retentionDays = 7;
|
||||
|
||||
private async pollHostStats(
|
||||
host: THost,
|
||||
viewerUserId?: string,
|
||||
): Promise<void> {
|
||||
if (this.inFlight.has(host.id)) return;
|
||||
this.inFlight.add(host.id);
|
||||
|
||||
try {
|
||||
await proxmoxStatsPollLimiter.run(async () => {
|
||||
const userId = viewerUserId || host.userId;
|
||||
const refreshed =
|
||||
(await this.deps.fetchHostById(host.id, userId)) ?? host;
|
||||
const config = parseProxmoxStatsConfig(refreshed.proxmoxStatsConfig);
|
||||
|
||||
try {
|
||||
const snapshot = await this.deps.withSshConnection(
|
||||
refreshed,
|
||||
(client) => collectProxmoxStats(client, config.nodeName),
|
||||
);
|
||||
|
||||
this.snapshotStore.set(host.id, {
|
||||
data: snapshot,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
this.errorStore.delete(host.id);
|
||||
|
||||
if (this.deps.historyEnabled?.() ?? true) {
|
||||
await this.insertHistory(host.id, snapshot);
|
||||
}
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
this.errorStore.set(host.id, {
|
||||
error: message,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
pluginCtx().log.warn(
|
||||
`Proxmox stats poll failed for host ${host.id}: ${message}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
this.inFlight.delete(host.id);
|
||||
}
|
||||
}
|
||||
|
||||
private async insertHistory(
|
||||
hostId: number,
|
||||
snapshot: ProxmoxStatsSnapshot,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const iface = snapshot.network?.interfaces?.[0];
|
||||
const rxRaw = iface?.rxBytes ? parseInt(iface.rxBytes, 10) : null;
|
||||
const txRaw = iface?.txBytes ? parseInt(iface.txBytes, 10) : null;
|
||||
|
||||
await this.deps.historyRepository.create({
|
||||
hostId,
|
||||
cpuPercent: snapshot.node?.cpu?.percent ?? null,
|
||||
memPercent: snapshot.node?.memory?.percent ?? null,
|
||||
diskPercent: snapshot.node?.disk?.percent ?? null,
|
||||
netRxBytes: rxRaw !== null && !isNaN(rxRaw) ? rxRaw : null,
|
||||
netTxBytes: txRaw !== null && !isNaN(txRaw) ? txRaw : null,
|
||||
});
|
||||
|
||||
await this.deps.historyRepository.pruneOlderThan(
|
||||
hostId,
|
||||
this.retentionDays,
|
||||
);
|
||||
} catch (err) {
|
||||
pluginCtx().log.warn(
|
||||
`Failed to write proxmox node history for host ${hostId}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private startPollingForHost(host: THost, viewerUserId?: string): void {
|
||||
const existing = this.pollingConfigs.get(host.id);
|
||||
if (existing?.timer) {
|
||||
clearInterval(existing.timer);
|
||||
}
|
||||
|
||||
const config = parseProxmoxStatsConfig(host.proxmoxStatsConfig);
|
||||
const intervalMs = this.intervalWithJitter(
|
||||
(config.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS) * 1000,
|
||||
host.id,
|
||||
);
|
||||
|
||||
void this.pollHostStats(host, viewerUserId);
|
||||
|
||||
const timer = setInterval(() => {
|
||||
const latest = this.pollingConfigs.get(host.id);
|
||||
if (latest) {
|
||||
void this.pollHostStats(latest.host, latest.viewerUserId);
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
this.pollingConfigs.set(host.id, { host, timer, viewerUserId });
|
||||
}
|
||||
|
||||
private stopPollingForHost(hostId: number): void {
|
||||
const config = this.pollingConfigs.get(hostId);
|
||||
if (config?.timer) {
|
||||
clearInterval(config.timer);
|
||||
}
|
||||
this.pollingConfigs.delete(hostId);
|
||||
}
|
||||
|
||||
getStats(
|
||||
hostId: number,
|
||||
): { data: ProxmoxStatsSnapshot; timestamp: number } | undefined {
|
||||
return this.snapshotStore.get(hostId);
|
||||
}
|
||||
|
||||
getError(hostId: number): ErrorSnapshot | undefined {
|
||||
return this.errorStore.get(hostId);
|
||||
}
|
||||
|
||||
async ensurePolling(host: THost, viewerUserId?: string): Promise<void> {
|
||||
if (!this.pollingConfigs.has(host.id)) {
|
||||
this.startPollingForHost(host, viewerUserId);
|
||||
}
|
||||
if (!this.snapshotStore.has(host.id) && !this.inFlight.has(host.id)) {
|
||||
await this.pollHostStats(host, viewerUserId);
|
||||
}
|
||||
}
|
||||
|
||||
registerViewer = (
|
||||
hostId: number,
|
||||
sessionId: string,
|
||||
userId: string,
|
||||
): void => {
|
||||
if (!this.activeViewers.has(hostId)) {
|
||||
this.activeViewers.set(hostId, new Set());
|
||||
}
|
||||
this.activeViewers.get(hostId)!.add(sessionId);
|
||||
|
||||
this.viewerDetails.set(sessionId, {
|
||||
sessionId,
|
||||
userId,
|
||||
hostId,
|
||||
lastHeartbeat: Date.now(),
|
||||
});
|
||||
|
||||
if (this.activeViewers.get(hostId)!.size === 1) {
|
||||
Promise.resolve()
|
||||
.then(async () => {
|
||||
const host = await this.deps.fetchHostById(hostId, userId);
|
||||
if (host) {
|
||||
this.startPollingForHost(host, userId);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
pluginCtx().log.warn(
|
||||
`Proxmox stats startPollingForHost rejected (non-fatal) for host ${hostId}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
unregisterViewer = (hostId: number, sessionId: string): void => {
|
||||
const viewers = this.activeViewers.get(hostId);
|
||||
if (viewers) {
|
||||
viewers.delete(sessionId);
|
||||
if (viewers.size === 0) {
|
||||
this.activeViewers.delete(hostId);
|
||||
this.stopPollingForHost(hostId);
|
||||
}
|
||||
}
|
||||
this.viewerDetails.delete(sessionId);
|
||||
};
|
||||
|
||||
updateHeartbeat(sessionId: string): boolean {
|
||||
const viewer = this.viewerDetails.get(sessionId);
|
||||
if (viewer) {
|
||||
viewer.lastHeartbeat = Date.now();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private cleanupInactiveViewers(): void {
|
||||
const now = Date.now();
|
||||
const maxInactivity = 120000;
|
||||
|
||||
for (const [sessionId, viewer] of this.viewerDetails.entries()) {
|
||||
if (now - viewer.lastHeartbeat > maxInactivity) {
|
||||
this.unregisterViewer(viewer.hostId, sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
clearInterval(this.viewerCleanupInterval);
|
||||
for (const hostId of this.pollingConfigs.keys()) {
|
||||
this.stopPollingForHost(hostId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
|
||||
export interface ProxmoxClusterNodeEntry {
|
||||
name: string;
|
||||
online: boolean;
|
||||
local: boolean;
|
||||
ip: string | null;
|
||||
}
|
||||
|
||||
export type ProxmoxClusterHealthResult =
|
||||
| { clustered: false }
|
||||
| {
|
||||
clustered: true;
|
||||
quorate: boolean;
|
||||
clusterName: string | null;
|
||||
nodes: ProxmoxClusterNodeEntry[];
|
||||
};
|
||||
|
||||
const EMPTY_RESULT: ProxmoxClusterHealthResult = { clustered: false };
|
||||
|
||||
export async function collectProxmoxClusterHealth(
|
||||
client: Client,
|
||||
): Promise<ProxmoxClusterHealthResult> {
|
||||
try {
|
||||
const { stdout, code } = await execCommand(
|
||||
client,
|
||||
"pvesh get /cluster/status --output-format json",
|
||||
15000,
|
||||
);
|
||||
if (code !== 0) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const data = JSON.parse(stdout);
|
||||
if (!Array.isArray(data)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const entries = data as Array<Record<string, unknown>>;
|
||||
const clusterEntry = entries.find((e) => e.type === "cluster");
|
||||
if (!clusterEntry) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const nodes: ProxmoxClusterNodeEntry[] = entries
|
||||
.filter((e) => e.type === "node")
|
||||
.map((e) => ({
|
||||
name: typeof e.name === "string" ? e.name : "",
|
||||
online: e.online === 1 || e.online === true,
|
||||
local: e.local === 1 || e.local === true,
|
||||
ip: typeof e.ip === "string" && e.ip ? e.ip : null,
|
||||
}));
|
||||
|
||||
return {
|
||||
clustered: true,
|
||||
quorate: clusterEntry.quorate === 1 || clusterEntry.quorate === true,
|
||||
clusterName:
|
||||
typeof clusterEntry.name === "string" && clusterEntry.name
|
||||
? clusterEntry.name
|
||||
: null,
|
||||
nodes,
|
||||
};
|
||||
} catch {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
import { isSafeNodeName } from "../proxmox-shared.js";
|
||||
import {
|
||||
collectProxmoxNodeStatus,
|
||||
type ProxmoxNodeStatusResult,
|
||||
} from "./node-status-collector.js";
|
||||
import {
|
||||
collectProxmoxNodeNetwork,
|
||||
type ProxmoxNodeNetworkResult,
|
||||
} from "./node-network-collector.js";
|
||||
import {
|
||||
collectProxmoxGuestsSummary,
|
||||
type ProxmoxGuestsSummaryResult,
|
||||
} from "./guests-collector.js";
|
||||
import {
|
||||
collectProxmoxStorage,
|
||||
type ProxmoxStorageResult,
|
||||
} from "./storage-collector.js";
|
||||
import {
|
||||
collectProxmoxClusterHealth,
|
||||
type ProxmoxClusterHealthResult,
|
||||
} from "./cluster-health-collector.js";
|
||||
|
||||
export interface ProxmoxStatsSnapshot {
|
||||
node: ProxmoxNodeStatusResult;
|
||||
network: ProxmoxNodeNetworkResult;
|
||||
guests: ProxmoxGuestsSummaryResult;
|
||||
storage: ProxmoxStorageResult;
|
||||
cluster: ProxmoxClusterHealthResult;
|
||||
lastChecked: string;
|
||||
}
|
||||
|
||||
async function resolveNodeName(
|
||||
client: Client,
|
||||
configuredNodeName: string | null | undefined,
|
||||
): Promise<string> {
|
||||
if (configuredNodeName) {
|
||||
// An explicitly configured name is validated and used as-is - an unsafe
|
||||
// value is rejected outright rather than silently falling back to
|
||||
// auto-detection, which would mask a misconfigured (or malicious) override.
|
||||
if (!isSafeNodeName(configuredNodeName)) {
|
||||
throw new Error("Unable to determine a valid Proxmox node name");
|
||||
}
|
||||
return configuredNodeName;
|
||||
}
|
||||
|
||||
const { stdout } = await execCommand(client, "hostname", 10000);
|
||||
return stdout.trim();
|
||||
}
|
||||
|
||||
export async function collectProxmoxStats(
|
||||
client: Client,
|
||||
configuredNodeName: string | null | undefined,
|
||||
): Promise<ProxmoxStatsSnapshot> {
|
||||
const pveshCheck = await execCommand(
|
||||
client,
|
||||
"command -v pvesh >/dev/null 2>&1 && echo ok || echo missing",
|
||||
10000,
|
||||
);
|
||||
if (pveshCheck.stdout.trim() !== "ok") {
|
||||
throw new Error("pvesh not found — is this a Proxmox node?");
|
||||
}
|
||||
|
||||
const nodeName = await resolveNodeName(client, configuredNodeName);
|
||||
if (!isSafeNodeName(nodeName)) {
|
||||
throw new Error("Unable to determine a valid Proxmox node name");
|
||||
}
|
||||
|
||||
const [node, network, guests, storage, cluster] = await Promise.all([
|
||||
collectProxmoxNodeStatus(client, nodeName),
|
||||
collectProxmoxNodeNetwork(client, nodeName),
|
||||
collectProxmoxGuestsSummary(client, nodeName),
|
||||
collectProxmoxStorage(client, nodeName),
|
||||
collectProxmoxClusterHealth(client),
|
||||
]);
|
||||
|
||||
return {
|
||||
node,
|
||||
network,
|
||||
guests,
|
||||
storage,
|
||||
cluster,
|
||||
lastChecked: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { Client, ClientChannel } from "ssh2";
|
||||
|
||||
/** Runs a command over an already-open ssh2 client and collects its output. */
|
||||
export function execCommand(
|
||||
client: Client,
|
||||
command: string,
|
||||
timeoutMs = 30000,
|
||||
): Promise<{
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
code: number | null;
|
||||
}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let stream: ClientChannel | null = null;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(new Error(`Command timeout after ${timeoutMs}ms: ${command}`));
|
||||
}
|
||||
}, timeoutMs);
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
if (stream) {
|
||||
try {
|
||||
stream.removeAllListeners();
|
||||
if (stream.stderr) {
|
||||
stream.stderr.removeAllListeners();
|
||||
}
|
||||
stream.destroy();
|
||||
} catch {
|
||||
// expected - cleanup errors ignored
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
client.exec(command, { pty: false }, (err, _stream) => {
|
||||
if (err) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(err);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
stream = _stream;
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let exitCode: number | null = null;
|
||||
|
||||
stream
|
||||
.on("close", (code: number | undefined) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
exitCode = typeof code === "number" ? code : null;
|
||||
cleanup();
|
||||
resolve({ stdout, stderr, code: exitCode });
|
||||
}
|
||||
})
|
||||
.on("data", (data: Buffer) => {
|
||||
stdout += data.toString("utf8");
|
||||
})
|
||||
.on("error", (streamErr: Error) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(streamErr);
|
||||
}
|
||||
});
|
||||
|
||||
if (stream.stderr) {
|
||||
stream.stderr
|
||||
.on("data", (data: Buffer) => {
|
||||
stderr += data.toString("utf8");
|
||||
})
|
||||
.on("error", (stderrErr: Error) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(stderrErr);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function toFixedNum(
|
||||
n: number | null | undefined,
|
||||
digits = 2,
|
||||
): number | null {
|
||||
if (typeof n !== "number" || !Number.isFinite(n)) return null;
|
||||
return Number(n.toFixed(digits));
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "./common-utils.js";
|
||||
import { isSafeNodeName } from "../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxGuestSummaryEntry {
|
||||
vmid: number;
|
||||
name: string;
|
||||
type: "qemu" | "lxc";
|
||||
status: string;
|
||||
cpuPercent: number | null;
|
||||
memPercent: number | null;
|
||||
memUsedGiB: number | null;
|
||||
memTotalGiB: number | null;
|
||||
diskPercent: number | null;
|
||||
diskUsedGiB: number | null;
|
||||
diskTotalGiB: number | null;
|
||||
uptimeSeconds: number | null;
|
||||
}
|
||||
|
||||
export interface ProxmoxGuestsSummaryResult {
|
||||
guests: ProxmoxGuestSummaryEntry[];
|
||||
counts: { running: number; stopped: number; total: number };
|
||||
}
|
||||
|
||||
const EMPTY_RESULT: ProxmoxGuestsSummaryResult = {
|
||||
guests: [],
|
||||
counts: { running: 0, stopped: 0, total: 0 },
|
||||
};
|
||||
|
||||
function bytesToGiB(bytes: number): number {
|
||||
return bytes / (1024 * 1024 * 1024);
|
||||
}
|
||||
|
||||
export async function collectProxmoxGuestsSummary(
|
||||
client: Client,
|
||||
nodeName: string,
|
||||
): Promise<ProxmoxGuestsSummaryResult> {
|
||||
if (!isSafeNodeName(nodeName)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout, code } = await execCommand(
|
||||
client,
|
||||
"pvesh get /cluster/resources --output-format json",
|
||||
25000,
|
||||
);
|
||||
if (code !== 0) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const resources = JSON.parse(stdout);
|
||||
if (!Array.isArray(resources)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const guests: ProxmoxGuestSummaryEntry[] = [];
|
||||
for (const r of resources as Array<Record<string, unknown>>) {
|
||||
const type = r.type;
|
||||
if (type !== "qemu" && type !== "lxc") continue;
|
||||
if (r.node !== nodeName) continue;
|
||||
if (r.template === true || r.template === 1) continue;
|
||||
|
||||
const cpuFraction = typeof r.cpu === "number" ? r.cpu : null;
|
||||
const mem = typeof r.mem === "number" ? r.mem : null;
|
||||
const maxmem = typeof r.maxmem === "number" ? r.maxmem : null;
|
||||
const memPercent =
|
||||
mem !== null && maxmem !== null && maxmem > 0
|
||||
? Math.max(0, Math.min(100, (mem / maxmem) * 100))
|
||||
: null;
|
||||
|
||||
const disk = typeof r.disk === "number" ? r.disk : null;
|
||||
const maxdisk = typeof r.maxdisk === "number" ? r.maxdisk : null;
|
||||
// QEMU guests without a running agent report maxdisk: 0 - a false 0%
|
||||
// is worse than an honest "unknown", so treat that as no data at all.
|
||||
const hasDiskData = maxdisk !== null && maxdisk > 0;
|
||||
const diskPercent =
|
||||
hasDiskData && disk !== null
|
||||
? Math.max(0, Math.min(100, (disk / maxdisk) * 100))
|
||||
: null;
|
||||
|
||||
guests.push({
|
||||
vmid: typeof r.vmid === "number" ? r.vmid : Number(r.vmid),
|
||||
name:
|
||||
typeof r.name === "string" && r.name ? r.name : String(r.vmid ?? ""),
|
||||
type,
|
||||
status: typeof r.status === "string" ? r.status : "unknown",
|
||||
cpuPercent: toFixedNum(
|
||||
cpuFraction !== null ? cpuFraction * 100 : null,
|
||||
0,
|
||||
),
|
||||
memPercent: toFixedNum(memPercent, 0),
|
||||
memUsedGiB: mem !== null ? toFixedNum(bytesToGiB(mem), 2) : null,
|
||||
memTotalGiB: maxmem !== null ? toFixedNum(bytesToGiB(maxmem), 2) : null,
|
||||
diskPercent: toFixedNum(diskPercent, 0),
|
||||
diskUsedGiB:
|
||||
hasDiskData && disk !== null ? toFixedNum(bytesToGiB(disk), 2) : null,
|
||||
diskTotalGiB: hasDiskData ? toFixedNum(bytesToGiB(maxdisk), 2) : null,
|
||||
uptimeSeconds: typeof r.uptime === "number" ? r.uptime : null,
|
||||
});
|
||||
}
|
||||
|
||||
const running = guests.filter((g) => g.status === "running").length;
|
||||
const stopped = guests.filter((g) => g.status !== "running").length;
|
||||
|
||||
return {
|
||||
guests,
|
||||
counts: { running, stopped, total: guests.length },
|
||||
};
|
||||
} catch {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand } from "./common-utils.js";
|
||||
import { isSafeNodeName } from "../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxNodeNetworkInterface {
|
||||
name: string;
|
||||
ip: string | null;
|
||||
state: string | null;
|
||||
rxBytes: string | null;
|
||||
txBytes: string | null;
|
||||
}
|
||||
|
||||
export interface ProxmoxNodeNetworkResult {
|
||||
interfaces: ProxmoxNodeNetworkInterface[];
|
||||
}
|
||||
|
||||
const EMPTY_RESULT: ProxmoxNodeNetworkResult = { interfaces: [] };
|
||||
|
||||
async function collectFromProcNetDev(
|
||||
client: Client,
|
||||
): Promise<ProxmoxNodeNetworkResult> {
|
||||
const interfaces: ProxmoxNodeNetworkInterface[] = [];
|
||||
|
||||
try {
|
||||
const [addrOut, stateOut, procNetOut] = await Promise.all([
|
||||
execCommand(
|
||||
client,
|
||||
"ip -o addr show | awk '{print $2,$4}' | grep -v '^lo'",
|
||||
),
|
||||
execCommand(
|
||||
client,
|
||||
"ip -o link show | awk '{gsub(/:/, \"\", $2); print $2,$9}'",
|
||||
),
|
||||
execCommand(client, "cat /proc/net/dev"),
|
||||
]);
|
||||
|
||||
const ifMap = new Map<
|
||||
string,
|
||||
{ ip: string | null; state: string | null }
|
||||
>();
|
||||
for (const line of addrOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const name = parts[0];
|
||||
const ip = parts[1].split("/")[0];
|
||||
if (!ifMap.has(name)) ifMap.set(name, { ip, state: null });
|
||||
}
|
||||
}
|
||||
for (const line of stateOut.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const existing = ifMap.get(parts[0]);
|
||||
if (existing) existing.state = parts[1];
|
||||
}
|
||||
}
|
||||
|
||||
const rxTxMap = new Map<string, { rx: string; tx: string }>();
|
||||
for (const line of procNetOut.stdout.split("\n").slice(2)) {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
if (parts.length >= 10) {
|
||||
const ifName = parts[0].replace(":", "");
|
||||
rxTxMap.set(ifName, { rx: parts[1], tx: parts[9] });
|
||||
}
|
||||
}
|
||||
|
||||
for (const [name, data] of ifMap.entries()) {
|
||||
const rxTx = rxTxMap.get(name);
|
||||
interfaces.push({
|
||||
name,
|
||||
ip: data.ip,
|
||||
state: data.state,
|
||||
rxBytes: rxTx?.rx ?? null,
|
||||
txBytes: rxTx?.tx ?? null,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
return { interfaces };
|
||||
}
|
||||
|
||||
export async function collectProxmoxNodeNetwork(
|
||||
client: Client,
|
||||
nodeName: string,
|
||||
): Promise<ProxmoxNodeNetworkResult> {
|
||||
if (!isSafeNodeName(nodeName)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout, code } = await execCommand(
|
||||
client,
|
||||
`pvesh get /nodes/${nodeName}/netstat --output-format json`,
|
||||
15000,
|
||||
);
|
||||
if (code !== 0) {
|
||||
return collectFromProcNetDev(client);
|
||||
}
|
||||
|
||||
const data = JSON.parse(stdout);
|
||||
if (!Array.isArray(data)) {
|
||||
return collectFromProcNetDev(client);
|
||||
}
|
||||
|
||||
const interfaces: ProxmoxNodeNetworkInterface[] = data
|
||||
.filter(
|
||||
(entry): entry is Record<string, unknown> =>
|
||||
!!entry && typeof entry === "object",
|
||||
)
|
||||
.map((entry) => ({
|
||||
name:
|
||||
typeof entry.dev === "string" ? entry.dev : String(entry.dev ?? ""),
|
||||
ip: null,
|
||||
state: null,
|
||||
rxBytes:
|
||||
typeof entry.in === "number"
|
||||
? String(entry.in)
|
||||
: typeof entry.received === "number"
|
||||
? String(entry.received)
|
||||
: null,
|
||||
txBytes:
|
||||
typeof entry.out === "number"
|
||||
? String(entry.out)
|
||||
: typeof entry.transmitted === "number"
|
||||
? String(entry.transmitted)
|
||||
: null,
|
||||
}))
|
||||
.filter((iface) => iface.name && iface.name !== "lo");
|
||||
|
||||
if (interfaces.length === 0) {
|
||||
return collectFromProcNetDev(client);
|
||||
}
|
||||
|
||||
return { interfaces };
|
||||
} catch {
|
||||
return collectFromProcNetDev(client);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "./common-utils.js";
|
||||
import { isSafeNodeName } from "../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxNodeStatusResult {
|
||||
cpu: {
|
||||
percent: number | null;
|
||||
cores: number | null;
|
||||
load: [number, number, number] | null;
|
||||
};
|
||||
memory: {
|
||||
percent: number | null;
|
||||
usedGiB: number | null;
|
||||
totalGiB: number | null;
|
||||
};
|
||||
disk: {
|
||||
percent: number | null;
|
||||
usedGiB: number | null;
|
||||
totalGiB: number | null;
|
||||
};
|
||||
uptime: {
|
||||
seconds: number | null;
|
||||
formatted: string | null;
|
||||
};
|
||||
system: {
|
||||
hostname: string | null;
|
||||
kernel: string | null;
|
||||
pveVersion: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
const EMPTY_RESULT: ProxmoxNodeStatusResult = {
|
||||
cpu: { percent: null, cores: null, load: null },
|
||||
memory: { percent: null, usedGiB: null, totalGiB: null },
|
||||
disk: { percent: null, usedGiB: null, totalGiB: null },
|
||||
uptime: { seconds: null, formatted: null },
|
||||
system: { hostname: null, kernel: null, pveVersion: null },
|
||||
};
|
||||
|
||||
function bytesToGiB(bytes: number): number {
|
||||
return bytes / (1024 * 1024 * 1024);
|
||||
}
|
||||
|
||||
function formatUptime(seconds: number): string {
|
||||
const days = Math.floor(seconds / 86400);
|
||||
const hours = Math.floor((seconds % 86400) / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
return `${days}d ${hours}h ${minutes}m`;
|
||||
}
|
||||
|
||||
export async function collectProxmoxNodeStatus(
|
||||
client: Client,
|
||||
nodeName: string,
|
||||
): Promise<ProxmoxNodeStatusResult> {
|
||||
if (!isSafeNodeName(nodeName)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout, code } = await execCommand(
|
||||
client,
|
||||
`pvesh get /nodes/${nodeName}/status --output-format json`,
|
||||
25000,
|
||||
);
|
||||
if (code !== 0) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const data = JSON.parse(stdout) as Record<string, unknown>;
|
||||
|
||||
const cpuFraction = typeof data.cpu === "number" ? data.cpu : null;
|
||||
const cpuinfo = (data.cpuinfo as Record<string, unknown>) || {};
|
||||
const cores = typeof cpuinfo.cores === "number" ? cpuinfo.cores : null;
|
||||
const loadavgRaw = data.loadavg;
|
||||
let load: [number, number, number] | null = null;
|
||||
if (Array.isArray(loadavgRaw) && loadavgRaw.length >= 3) {
|
||||
const parsed = loadavgRaw
|
||||
.slice(0, 3)
|
||||
.map((v) => Number(v))
|
||||
.map((v) => (Number.isFinite(v) ? v : 0));
|
||||
load = parsed as [number, number, number];
|
||||
}
|
||||
|
||||
const memory = (data.memory as Record<string, unknown>) || {};
|
||||
const memUsed = typeof memory.used === "number" ? memory.used : null;
|
||||
const memTotal = typeof memory.total === "number" ? memory.total : null;
|
||||
const memPercent =
|
||||
memUsed !== null && memTotal !== null && memTotal > 0
|
||||
? Math.max(0, Math.min(100, (memUsed / memTotal) * 100))
|
||||
: null;
|
||||
|
||||
const rootfs = (data.rootfs as Record<string, unknown>) || {};
|
||||
const diskUsed = typeof rootfs.used === "number" ? rootfs.used : null;
|
||||
const diskTotal = typeof rootfs.total === "number" ? rootfs.total : null;
|
||||
const diskPercent =
|
||||
diskUsed !== null && diskTotal !== null && diskTotal > 0
|
||||
? Math.max(0, Math.min(100, (diskUsed / diskTotal) * 100))
|
||||
: null;
|
||||
|
||||
const uptimeSeconds = typeof data.uptime === "number" ? data.uptime : null;
|
||||
|
||||
return {
|
||||
cpu: {
|
||||
percent: toFixedNum(cpuFraction !== null ? cpuFraction * 100 : null, 0),
|
||||
cores,
|
||||
load,
|
||||
},
|
||||
memory: {
|
||||
percent: toFixedNum(memPercent, 0),
|
||||
usedGiB: memUsed !== null ? toFixedNum(bytesToGiB(memUsed), 2) : null,
|
||||
totalGiB:
|
||||
memTotal !== null ? toFixedNum(bytesToGiB(memTotal), 2) : null,
|
||||
},
|
||||
disk: {
|
||||
percent: toFixedNum(diskPercent, 0),
|
||||
usedGiB: diskUsed !== null ? toFixedNum(bytesToGiB(diskUsed), 2) : null,
|
||||
totalGiB:
|
||||
diskTotal !== null ? toFixedNum(bytesToGiB(diskTotal), 2) : null,
|
||||
},
|
||||
uptime: {
|
||||
seconds: uptimeSeconds,
|
||||
formatted: uptimeSeconds !== null ? formatUptime(uptimeSeconds) : null,
|
||||
},
|
||||
system: {
|
||||
hostname:
|
||||
typeof data.hostname === "string" && data.hostname
|
||||
? data.hostname
|
||||
: null,
|
||||
kernel:
|
||||
typeof data.kversion === "string" && data.kversion
|
||||
? data.kversion
|
||||
: null,
|
||||
pveVersion:
|
||||
typeof data.pveversion === "string" && data.pveversion
|
||||
? data.pveversion
|
||||
: null,
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { Client } from "ssh2";
|
||||
import { execCommand, toFixedNum } from "./common-utils.js";
|
||||
import { isSafeNodeName } from "../proxmox-shared.js";
|
||||
|
||||
export interface ProxmoxStoragePoolEntry {
|
||||
name: string;
|
||||
type: string;
|
||||
active: boolean;
|
||||
enabled: boolean;
|
||||
usedGiB: number | null;
|
||||
totalGiB: number | null;
|
||||
availGiB: number | null;
|
||||
percent: number | null;
|
||||
}
|
||||
|
||||
export interface ProxmoxStorageResult {
|
||||
pools: ProxmoxStoragePoolEntry[];
|
||||
}
|
||||
|
||||
const EMPTY_RESULT: ProxmoxStorageResult = { pools: [] };
|
||||
|
||||
function bytesToGiB(bytes: number): number {
|
||||
return bytes / (1024 * 1024 * 1024);
|
||||
}
|
||||
|
||||
export async function collectProxmoxStorage(
|
||||
client: Client,
|
||||
nodeName: string,
|
||||
): Promise<ProxmoxStorageResult> {
|
||||
if (!isSafeNodeName(nodeName)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout, code } = await execCommand(
|
||||
client,
|
||||
`pvesh get /nodes/${nodeName}/storage --output-format json`,
|
||||
25000,
|
||||
);
|
||||
if (code !== 0) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const data = JSON.parse(stdout);
|
||||
if (!Array.isArray(data)) {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
|
||||
const pools: ProxmoxStoragePoolEntry[] = (
|
||||
data as Array<Record<string, unknown>>
|
||||
).map((entry) => {
|
||||
const used = typeof entry.used === "number" ? entry.used : null;
|
||||
const total = typeof entry.total === "number" ? entry.total : null;
|
||||
const avail = typeof entry.avail === "number" ? entry.avail : null;
|
||||
const percent =
|
||||
used !== null && total !== null && total > 0
|
||||
? Math.max(0, Math.min(100, (used / total) * 100))
|
||||
: null;
|
||||
|
||||
return {
|
||||
name: typeof entry.storage === "string" ? entry.storage : "",
|
||||
type: typeof entry.type === "string" ? entry.type : "unknown",
|
||||
active: entry.active === 1 || entry.active === true,
|
||||
enabled: entry.enabled === 1 || entry.enabled === true,
|
||||
usedGiB: used !== null ? toFixedNum(bytesToGiB(used), 2) : null,
|
||||
totalGiB: total !== null ? toFixedNum(bytesToGiB(total), 2) : null,
|
||||
availGiB: avail !== null ? toFixedNum(bytesToGiB(avail), 2) : null,
|
||||
percent: toFixedNum(percent, 0),
|
||||
};
|
||||
});
|
||||
|
||||
return { pools };
|
||||
} catch {
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
}
|
||||
@@ -1,36 +1,27 @@
|
||||
import { getErrorMessage } from "../../../../src/backend/utils/error-message.js";
|
||||
import express, { type Router } from "express";
|
||||
import type { Client as SSHClient } from "ssh2";
|
||||
import { logger } from "../../../../src/backend/utils/logger.js";
|
||||
import { DataCrypto } from "../../../../src/backend/utils/data-crypto.js";
|
||||
import { createCurrentHostRepository } from "../../../../src/backend/database/repositories/factory.js";
|
||||
import { AuthManager } from "../../../../src/backend/utils/auth-manager.js";
|
||||
import {
|
||||
type AuthenticatedRequest,
|
||||
type SSHHost,
|
||||
} from "../../../../src/types/index.js";
|
||||
import { resolveHostById } from "../../../../src/backend/hosts/host-resolver.js";
|
||||
import type { PluginSshHost } from "@termix/plugin-sdk/backend";
|
||||
import { execElevated } from "../../../../src/backend/hosts/metrics-shared/exec-elevated.js";
|
||||
import type {
|
||||
PluginHostCreateInput,
|
||||
PluginHostRecord,
|
||||
PluginHostUpdateInput,
|
||||
} from "@termix/plugin-sdk/backend";
|
||||
import { connectSsh } from "./ssh.js";
|
||||
import { pluginCtx } from "./plugin-ctx.js";
|
||||
import { resolveProxmoxImportAuth } from "./proxmox-import-auth.js";
|
||||
import {
|
||||
parseProxmoxJumpHosts,
|
||||
serializeProxmoxJumpHosts,
|
||||
} from "./proxmox-jump-hosts.js";
|
||||
import { isSafeNodeName } from "../../../../src/backend/hosts/proxmox-shared.js";
|
||||
import { execElevated } from "../../../../src/backend/hosts/metrics-shared/exec-elevated.js";
|
||||
import { isSafeNodeName } from "./proxmox-shared.js";
|
||||
|
||||
const router = express.Router();
|
||||
const proxmoxLogger = logger;
|
||||
const runningSyncs = new Set<string>();
|
||||
|
||||
const MIN_SYNC_INTERVAL_MINUTES = 5;
|
||||
const DEFAULT_SYNC_INTERVAL_MINUTES = 15;
|
||||
|
||||
const authManager = AuthManager.getInstance();
|
||||
const authenticateJWT = authManager.createAuthMiddleware();
|
||||
const requireDataAccess = authManager.createDataAccessMiddleware();
|
||||
|
||||
// Helpers
|
||||
|
||||
function execCommand(
|
||||
@@ -189,6 +180,9 @@ type ProxmoxGuest = {
|
||||
enableDocker: boolean;
|
||||
};
|
||||
|
||||
/** The resolved host ctx.ssh.connect hands back, plus the sudo secret proxmox needs for pvesh. */
|
||||
type PluginSshHostWithSudo = PluginHostRecord & { sudoPassword?: string };
|
||||
|
||||
type ProxmoxSource = {
|
||||
source: "proxmox";
|
||||
sourceHostId: number;
|
||||
@@ -285,44 +279,30 @@ async function discoverProxmoxGuestsForHost(
|
||||
parsedHostId: number,
|
||||
onProgress?: (done: number, total: number) => void,
|
||||
): Promise<{
|
||||
host: SSHHost;
|
||||
host: PluginSshHostWithSudo;
|
||||
guests: ProxmoxGuest[];
|
||||
credentialId: number | null;
|
||||
defaultCredentialId: number | null;
|
||||
jumpHosts: unknown[] | null;
|
||||
config: ReturnType<typeof parseProxmoxConfig>;
|
||||
}> {
|
||||
if (!DataCrypto.canUserAccessData(userId)) {
|
||||
const error = new Error("Session expired — please log in again");
|
||||
(error as Error & { code?: string }).code = "SESSION_EXPIRED";
|
||||
throw error;
|
||||
}
|
||||
|
||||
const resolvedHost = await resolveHostById(parsedHostId, userId);
|
||||
if (!resolvedHost) {
|
||||
const error = new Error("Host not found");
|
||||
(error as Error & { status?: number }).status = 404;
|
||||
throw error;
|
||||
}
|
||||
|
||||
const host = resolvedHost as SSHHost;
|
||||
const proxmoxCfgRaw = parseJsonObject(host.proxmoxConfig);
|
||||
const ctx = pluginCtx();
|
||||
const proxmoxCfgRaw = await ctx.settings.getHost(
|
||||
parsedHostId,
|
||||
"proxmoxConfig",
|
||||
);
|
||||
const config = parseProxmoxConfig(proxmoxCfgRaw);
|
||||
|
||||
const hostCredentialId = host.credentialId ?? null;
|
||||
|
||||
const { client } = await connectSsh(host as unknown as PluginSshHost, {
|
||||
const { client, host } = await connectSsh(parsedHostId, {
|
||||
purpose: "proxmox",
|
||||
timeoutMs: 35000,
|
||||
overrides: { tryKeyboard: false, readyTimeout: 30000 },
|
||||
});
|
||||
const hostWithSudo = host as PluginSshHostWithSudo;
|
||||
const hostCredentialId = (hostWithSudo.credentialId as number | null) ?? null;
|
||||
|
||||
try {
|
||||
proxmoxLogger.info("Proxmox discovery SSH connection established", {
|
||||
operation: "proxmox_discover",
|
||||
hostId: parsedHostId,
|
||||
userId,
|
||||
});
|
||||
ctx.log.info("Proxmox discovery SSH connection established");
|
||||
|
||||
const pveshCheck = await execCommand(
|
||||
client,
|
||||
@@ -337,7 +317,7 @@ async function discoverProxmoxGuestsForHost(
|
||||
const resourcesJson = await execPveshCommand(
|
||||
client,
|
||||
"pvesh get /cluster/resources --output-format json 2>/dev/null",
|
||||
host.sudoPassword,
|
||||
hostWithSudo.sudoPassword,
|
||||
);
|
||||
|
||||
let resources: Array<Record<string, unknown>>;
|
||||
@@ -366,11 +346,9 @@ async function discoverProxmoxGuestsForHost(
|
||||
if (r.template) continue;
|
||||
const node = r.node as string;
|
||||
if (!isSafeNodeName(node)) {
|
||||
proxmoxLogger.warn("Skipping guest with unsafe node name", {
|
||||
operation: "proxmox_discover",
|
||||
node,
|
||||
vmid: r.vmid,
|
||||
});
|
||||
ctx.log.warn(
|
||||
`Skipping guest with unsafe node name: ${node} (vmid ${r.vmid})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
guestBases.push({
|
||||
@@ -389,7 +367,7 @@ async function discoverProxmoxGuestsForHost(
|
||||
const cfgJson = await execPveshCommand(
|
||||
client,
|
||||
`pvesh get /nodes/${g.node}/lxc/${g.vmid}/config --output-format json 2>/dev/null`,
|
||||
host.sudoPassword,
|
||||
hostWithSudo.sudoPassword,
|
||||
25000,
|
||||
);
|
||||
configIp = parseLxcIp(JSON.parse(cfgJson), config.preferredPrefixes);
|
||||
@@ -404,7 +382,7 @@ async function discoverProxmoxGuestsForHost(
|
||||
const ifRaw = await execPveshCommand(
|
||||
client,
|
||||
`pvesh get /nodes/${g.node}/lxc/${g.vmid}/interfaces --output-format json 2>/dev/null`,
|
||||
host.sudoPassword,
|
||||
hostWithSudo.sudoPassword,
|
||||
12000,
|
||||
);
|
||||
const data = JSON.parse(ifRaw);
|
||||
@@ -437,7 +415,7 @@ async function discoverProxmoxGuestsForHost(
|
||||
const ifJson = await execPveshCommand(
|
||||
client,
|
||||
`pvesh get /nodes/${g.node}/qemu/${g.vmid}/agent/network-get-interfaces --output-format json 2>/dev/null`,
|
||||
host.sudoPassword,
|
||||
hostWithSudo.sudoPassword,
|
||||
12000,
|
||||
);
|
||||
const data = JSON.parse(ifJson);
|
||||
@@ -508,21 +486,16 @@ async function discoverProxmoxGuestsForHost(
|
||||
enableDocker: matchesAny(g.name, config.dockerPatterns),
|
||||
}));
|
||||
|
||||
proxmoxLogger.info("Proxmox discovery completed", {
|
||||
operation: "proxmox_discover",
|
||||
hostId: parsedHostId,
|
||||
userId,
|
||||
guestCount: guests.length,
|
||||
});
|
||||
ctx.log.info(
|
||||
`Proxmox discovery completed for host ${parsedHostId}: ${guests.length} guest(s)`,
|
||||
);
|
||||
|
||||
return {
|
||||
host,
|
||||
host: hostWithSudo,
|
||||
guests,
|
||||
credentialId: hostCredentialId,
|
||||
defaultCredentialId: config.defaultCredentialId,
|
||||
jumpHosts: parseProxmoxJumpHosts(
|
||||
(host as unknown as { jumpHosts?: unknown }).jumpHosts,
|
||||
),
|
||||
jumpHosts: parseProxmoxJumpHosts(hostWithSudo.jumpHosts),
|
||||
config,
|
||||
};
|
||||
} finally {
|
||||
@@ -538,6 +511,7 @@ async function syncProxmoxHost(
|
||||
userId: string,
|
||||
sourceHostId: number,
|
||||
): Promise<ProxmoxSyncResult> {
|
||||
const ctx = pluginCtx();
|
||||
const lockKey = `${userId}:${sourceHostId}`;
|
||||
if (runningSyncs.has(lockKey)) {
|
||||
return {
|
||||
@@ -570,10 +544,10 @@ async function syncProxmoxHost(
|
||||
);
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const existingHosts =
|
||||
(await createCurrentHostRepository().listDecryptedByUserId(
|
||||
userId,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
const existingHosts = (await ctx.hosts.listOwned()) as unknown as Record<
|
||||
string,
|
||||
unknown
|
||||
>[];
|
||||
const existingBySource = new Map<string, Record<string, unknown>>();
|
||||
for (const host of existingHosts) {
|
||||
const source = getProxmoxSource(host);
|
||||
@@ -647,10 +621,9 @@ async function syncProxmoxHost(
|
||||
update.credentialId = importAuth.credentialId;
|
||||
update.overrideCredentialUsername = false;
|
||||
}
|
||||
await createCurrentHostRepository().updateEncryptedForUser(
|
||||
userId,
|
||||
await ctx.hosts.update(
|
||||
existing.id as number,
|
||||
update,
|
||||
update as PluginHostUpdateInput,
|
||||
);
|
||||
result.updated++;
|
||||
continue;
|
||||
@@ -665,9 +638,8 @@ async function syncProxmoxHost(
|
||||
enableRdp: connectionType === "rdp",
|
||||
});
|
||||
|
||||
await createCurrentHostRepository().createEncryptedForUser(userId, {
|
||||
await ctx.hosts.create({
|
||||
...update,
|
||||
userId,
|
||||
createdAt: now,
|
||||
pin: false,
|
||||
authType: connectionType === "rdp" ? "password" : importAuth.authType,
|
||||
@@ -709,7 +681,7 @@ async function syncProxmoxHost(
|
||||
showTunnelInSidebar: 0,
|
||||
showDockerInSidebar: 0,
|
||||
showServerStatsInSidebar: 0,
|
||||
});
|
||||
} as unknown as PluginHostCreateInput);
|
||||
result.created++;
|
||||
}
|
||||
|
||||
@@ -720,8 +692,7 @@ async function syncProxmoxHost(
|
||||
const source = getProxmoxSource(existing);
|
||||
if (!source) continue;
|
||||
const missingSince = source.missingSince || now;
|
||||
await createCurrentHostRepository().updateEncryptedForUser(
|
||||
userId,
|
||||
await ctx.hosts.update(
|
||||
existing.id as number,
|
||||
{
|
||||
tags: mergeTags(existing.tags, ["proxmox-missing"]),
|
||||
@@ -733,13 +704,13 @@ async function syncProxmoxHost(
|
||||
},
|
||||
}),
|
||||
updatedAt: now,
|
||||
},
|
||||
} as PluginHostUpdateInput,
|
||||
);
|
||||
result.markedMissing++;
|
||||
}
|
||||
}
|
||||
|
||||
await writeSyncStatus(userId, sourceHostId, {
|
||||
await writeSyncStatus(sourceHostId, {
|
||||
lastSyncAt: startedAt,
|
||||
lastSyncStatus: "success",
|
||||
lastSyncError: null,
|
||||
@@ -750,7 +721,7 @@ async function syncProxmoxHost(
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error);
|
||||
result.errors.push(message);
|
||||
await writeSyncStatus(userId, sourceHostId, {
|
||||
await writeSyncStatus(sourceHostId, {
|
||||
lastSyncAt: startedAt,
|
||||
lastSyncStatus: "error",
|
||||
lastSyncError: message,
|
||||
@@ -763,22 +734,46 @@ async function syncProxmoxHost(
|
||||
}
|
||||
|
||||
async function writeSyncStatus(
|
||||
userId: string,
|
||||
hostId: number,
|
||||
patch: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
const hostRepository = createCurrentHostRepository();
|
||||
const row = await hostRepository.findByIdForUser(userId, hostId);
|
||||
if (!row) return;
|
||||
const config = parseJsonObject(row.proxmoxConfig);
|
||||
await hostRepository.updateForUser(userId, hostId, {
|
||||
proxmoxConfig: JSON.stringify({ ...config, ...patch }),
|
||||
});
|
||||
const ctx = pluginCtx();
|
||||
const config = parseJsonObject(
|
||||
await ctx.settings.getHost(hostId, "proxmoxConfig"),
|
||||
);
|
||||
await ctx.settings.setHost(hostId, "proxmoxConfig", { ...config, ...patch });
|
||||
}
|
||||
|
||||
router.post("/sync", authenticateJWT, requireDataAccess, async (req, res) => {
|
||||
/**
|
||||
* @openapi
|
||||
* /proxmox/sync:
|
||||
* post:
|
||||
* summary: Sync Proxmox guests for a host
|
||||
* description: Re-runs discovery for a Proxmox node and creates, updates or marks missing the imported guest hosts.
|
||||
* tags: [Proxmox]
|
||||
* security:
|
||||
* - bearerAuth: []
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* required: [hostId]
|
||||
* properties:
|
||||
* hostId:
|
||||
* type: number
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Sync result.
|
||||
* 400:
|
||||
* description: Missing or invalid hostId.
|
||||
* 500:
|
||||
* description: Sync failed.
|
||||
*/
|
||||
router.post("/sync", async (req, res) => {
|
||||
const { hostId } = req.body as { hostId?: unknown };
|
||||
const userId = (req as unknown as AuthenticatedRequest).userId;
|
||||
const userId = pluginCtx().currentActor()!;
|
||||
|
||||
const parsedHostId = Number(hostId);
|
||||
if (!hostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
|
||||
@@ -790,30 +785,48 @@ router.post("/sync", authenticateJWT, requireDataAccess, async (req, res) => {
|
||||
return res.json(result);
|
||||
} catch (err: unknown) {
|
||||
const message = getErrorMessage(err);
|
||||
const status =
|
||||
(err as Error & { code?: string; status?: number }).code ===
|
||||
"SESSION_EXPIRED"
|
||||
? 401
|
||||
: (err as Error & { status?: number }).status || 500;
|
||||
proxmoxLogger.error("Proxmox sync failed", err, {
|
||||
operation: "proxmox_sync",
|
||||
hostId: parsedHostId,
|
||||
userId,
|
||||
});
|
||||
const status = (err as Error & { status?: number }).status || 500;
|
||||
pluginCtx().log.error(
|
||||
`Proxmox sync failed for host ${parsedHostId}`,
|
||||
err as Error,
|
||||
);
|
||||
return res.status(status).json({ error: `Sync failed: ${message}` });
|
||||
}
|
||||
});
|
||||
|
||||
async function runDueProxmoxAutoSyncs(): Promise<void> {
|
||||
const ctx = pluginCtx();
|
||||
try {
|
||||
const rows = await createCurrentHostRepository().listProxmoxEnabled();
|
||||
const {
|
||||
createCurrentPluginSettingsRepository,
|
||||
createCurrentHostRepository,
|
||||
} =
|
||||
await import("../../../../src/backend/database/repositories/factory.js");
|
||||
const enabledRows = await createCurrentPluginSettingsRepository().listByKey(
|
||||
"proxmox",
|
||||
"host",
|
||||
"enableProxmox",
|
||||
);
|
||||
const hostRepository = createCurrentHostRepository();
|
||||
|
||||
const now = Date.now();
|
||||
for (const row of rows) {
|
||||
const configRaw = parseJsonObject(row.proxmoxConfig);
|
||||
for (const row of enabledRows) {
|
||||
if (row.value !== "true") continue;
|
||||
const hostId = Number(row.scopeId);
|
||||
if (!Number.isInteger(hostId)) continue;
|
||||
|
||||
const host = await hostRepository.findById(hostId);
|
||||
if (!host) continue;
|
||||
|
||||
const configRow = await createCurrentPluginSettingsRepository().get(
|
||||
"proxmox",
|
||||
"host",
|
||||
String(hostId),
|
||||
"proxmoxConfig",
|
||||
);
|
||||
const configRaw = configRow?.value ? JSON.parse(configRow.value) : {};
|
||||
const config = parseProxmoxConfig(configRaw);
|
||||
if (!config.autoSyncEnabled) continue;
|
||||
if (!DataCrypto.canUserAccessData(row.userId)) continue;
|
||||
|
||||
const lastSyncAt =
|
||||
typeof configRaw.lastSyncAt === "string"
|
||||
@@ -822,18 +835,17 @@ async function runDueProxmoxAutoSyncs(): Promise<void> {
|
||||
const intervalMs = config.syncIntervalMinutes * 60 * 1000;
|
||||
if (lastSyncAt && now - lastSyncAt < intervalMs) continue;
|
||||
|
||||
syncProxmoxHost(row.userId, row.id).catch((error) => {
|
||||
proxmoxLogger.error("Scheduled Proxmox sync failed", error, {
|
||||
operation: "proxmox_auto_sync",
|
||||
hostId: row.id,
|
||||
userId: row.userId,
|
||||
ctx
|
||||
.asUser(host.userId, () => syncProxmoxHost(host.userId, hostId))
|
||||
.catch((error) => {
|
||||
ctx.log.error(
|
||||
`Scheduled Proxmox sync failed for host ${hostId}`,
|
||||
error as Error,
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
proxmoxLogger.error("Failed to scan Proxmox auto sync jobs", error, {
|
||||
operation: "proxmox_auto_sync_scan",
|
||||
});
|
||||
ctx.log.error("Failed to scan Proxmox auto sync jobs", error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -915,127 +927,109 @@ let proxmoxAutoSyncStartupTimer: NodeJS.Timeout | undefined;
|
||||
* 500:
|
||||
* description: Discovery failed.
|
||||
*/
|
||||
router.get(
|
||||
"/discover/stream",
|
||||
authenticateJWT,
|
||||
requireDataAccess,
|
||||
async (req, res) => {
|
||||
const userId = (req as unknown as AuthenticatedRequest).userId;
|
||||
const parsedHostId = Number((req.query as { hostId?: unknown }).hostId);
|
||||
if (!parsedHostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
|
||||
return res.status(400).json({ error: "Missing or invalid hostId" });
|
||||
router.get("/discover/stream", async (req, res) => {
|
||||
const userId = pluginCtx().currentActor()!;
|
||||
const parsedHostId = Number((req.query as { hostId?: unknown }).hostId);
|
||||
if (!parsedHostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
|
||||
return res.status(400).json({ error: "Missing or invalid hostId" });
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-store, no-transform",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
});
|
||||
res.flushHeaders?.();
|
||||
|
||||
let closed = false;
|
||||
const send = (event: string, data: unknown) => {
|
||||
if (closed) return;
|
||||
try {
|
||||
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
||||
} catch {
|
||||
closed = true;
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-store, no-transform",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
});
|
||||
res.flushHeaders?.();
|
||||
|
||||
let closed = false;
|
||||
const send = (event: string, data: unknown) => {
|
||||
if (closed) return;
|
||||
try {
|
||||
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
||||
} catch {
|
||||
closed = true;
|
||||
}
|
||||
};
|
||||
const heartbeat = setInterval(() => {
|
||||
if (closed) return;
|
||||
try {
|
||||
res.write(": keepalive\n\n");
|
||||
} catch {
|
||||
closed = true;
|
||||
clearInterval(heartbeat);
|
||||
}
|
||||
}, 15000);
|
||||
req.on("close", () => {
|
||||
};
|
||||
const heartbeat = setInterval(() => {
|
||||
if (closed) return;
|
||||
try {
|
||||
res.write(": keepalive\n\n");
|
||||
} catch {
|
||||
closed = true;
|
||||
clearInterval(heartbeat);
|
||||
});
|
||||
}
|
||||
}, 15000);
|
||||
req.on("close", () => {
|
||||
closed = true;
|
||||
clearInterval(heartbeat);
|
||||
});
|
||||
|
||||
try {
|
||||
const discovery = await discoverProxmoxGuestsForHost(
|
||||
userId,
|
||||
parsedHostId,
|
||||
(done, total) => send("progress", { done, total }),
|
||||
);
|
||||
send("result", {
|
||||
guests: discovery.guests,
|
||||
credentialId: discovery.credentialId,
|
||||
defaultCredentialId: discovery.defaultCredentialId,
|
||||
jumpHosts: discovery.jumpHosts,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = getErrorMessage(err);
|
||||
proxmoxLogger.error("Proxmox discovery (stream) failed", err, {
|
||||
operation: "proxmox_discover",
|
||||
hostId: parsedHostId,
|
||||
userId,
|
||||
});
|
||||
send("fail", { message });
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
if (!closed) {
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
// ignore end errors
|
||||
}
|
||||
try {
|
||||
const discovery = await discoverProxmoxGuestsForHost(
|
||||
userId,
|
||||
parsedHostId,
|
||||
(done, total) => send("progress", { done, total }),
|
||||
);
|
||||
send("result", {
|
||||
guests: discovery.guests,
|
||||
credentialId: discovery.credentialId,
|
||||
defaultCredentialId: discovery.defaultCredentialId,
|
||||
jumpHosts: discovery.jumpHosts,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = getErrorMessage(err);
|
||||
pluginCtx().log.error(
|
||||
`Proxmox discovery (stream) failed for host ${parsedHostId}`,
|
||||
err as Error,
|
||||
);
|
||||
send("fail", { message });
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
if (!closed) {
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
// ignore end errors
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
router.post(
|
||||
"/discover",
|
||||
authenticateJWT,
|
||||
requireDataAccess,
|
||||
async (req, res) => {
|
||||
const { hostId } = req.body as { hostId?: unknown };
|
||||
const userId = (req as unknown as AuthenticatedRequest).userId;
|
||||
router.post("/discover", async (req, res) => {
|
||||
const { hostId } = req.body as { hostId?: unknown };
|
||||
const userId = pluginCtx().currentActor()!;
|
||||
|
||||
const parsedHostId = Number(hostId);
|
||||
if (!hostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
|
||||
return res.status(400).json({ error: "Missing or invalid hostId" });
|
||||
}
|
||||
const parsedHostId = Number(hostId);
|
||||
if (!hostId || !Number.isInteger(parsedHostId) || parsedHostId <= 0) {
|
||||
return res.status(400).json({ error: "Missing or invalid hostId" });
|
||||
}
|
||||
|
||||
try {
|
||||
const discovery = await discoverProxmoxGuestsForHost(
|
||||
userId,
|
||||
parsedHostId,
|
||||
);
|
||||
return res.json({
|
||||
guests: discovery.guests,
|
||||
credentialId: discovery.credentialId,
|
||||
defaultCredentialId: discovery.defaultCredentialId,
|
||||
jumpHosts: discovery.jumpHosts,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = getErrorMessage(err);
|
||||
proxmoxLogger.error("Proxmox discovery failed", err, {
|
||||
operation: "proxmox_discover",
|
||||
hostId: parsedHostId,
|
||||
userId,
|
||||
});
|
||||
try {
|
||||
const discovery = await discoverProxmoxGuestsForHost(userId, parsedHostId);
|
||||
return res.json({
|
||||
guests: discovery.guests,
|
||||
credentialId: discovery.credentialId,
|
||||
defaultCredentialId: discovery.defaultCredentialId,
|
||||
jumpHosts: discovery.jumpHosts,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = getErrorMessage(err);
|
||||
pluginCtx().log.error(
|
||||
`Proxmox discovery failed for host ${parsedHostId}`,
|
||||
err as Error,
|
||||
);
|
||||
|
||||
const status =
|
||||
(err as Error & { code?: string; status?: number }).code ===
|
||||
"SESSION_EXPIRED"
|
||||
? 401
|
||||
: (err as Error & { status?: number }).status ||
|
||||
(message.includes("Authentication failed") ||
|
||||
message.includes("connect ECONNREFUSED") ||
|
||||
message.includes("connect ETIMEDOUT")
|
||||
? 422
|
||||
: 500);
|
||||
return res.status(status).json({ error: `Discovery failed: ${message}` });
|
||||
}
|
||||
},
|
||||
);
|
||||
const status =
|
||||
(err as Error & { status?: number }).status ||
|
||||
(message.includes("Authentication failed") ||
|
||||
message.includes("connect ECONNREFUSED") ||
|
||||
message.includes("connect ETIMEDOUT")
|
||||
? 422
|
||||
: 500);
|
||||
return res.status(status).json({ error: `Discovery failed: ${message}` });
|
||||
}
|
||||
});
|
||||
|
||||
/** Called from activate(). Mounts this router at /proxmox via the shared
|
||||
* dispatcher, and starts the background auto-sync scan (a 60s interval plus
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
/**
|
||||
* Proxmox node stats: polling, viewer sessions and their routes.
|
||||
*
|
||||
* Moved here from host-metrics, which only ever hosted this because the
|
||||
* plugin split predates B5. host-metrics must not import proxmox code, so
|
||||
* this plugin owns its own polling manager, SSH connections and routes.
|
||||
*/
|
||||
|
||||
import type { Router } from "express";
|
||||
import express from "express";
|
||||
import type { PluginContext } from "@termix/plugin-sdk/backend";
|
||||
import { pluginCtx } from "./plugin-ctx.js";
|
||||
import {
|
||||
ProxmoxPollingManager,
|
||||
parseProxmoxStatsConfig,
|
||||
type ProxmoxStatsPollableHost,
|
||||
} from "./proxmox-stats-polling.js";
|
||||
import type { ProxmoxNodeHistoryRepository } from "./proxmox-node-history-repository.js";
|
||||
|
||||
const EMPTY_SNAPSHOT = {
|
||||
node: {
|
||||
cpu: { percent: null, cores: null, load: null },
|
||||
memory: { percent: null, usedGiB: null, totalGiB: null },
|
||||
disk: { percent: null, usedGiB: null, totalGiB: null },
|
||||
uptime: { seconds: null, formatted: null },
|
||||
system: { hostname: null, kernel: null, pveVersion: null },
|
||||
},
|
||||
network: { interfaces: [] },
|
||||
guests: { guests: [], counts: { running: 0, stopped: 0, total: 0 } },
|
||||
storage: { pools: [] },
|
||||
cluster: { clustered: false },
|
||||
lastChecked: new Date(0).toISOString(),
|
||||
};
|
||||
|
||||
type PollableHost = ProxmoxStatsPollableHost & { enableProxmoxStats: boolean };
|
||||
|
||||
let pollingManager: ProxmoxPollingManager<PollableHost> | undefined;
|
||||
let router: Router | undefined;
|
||||
|
||||
async function fetchHostForPolling(
|
||||
ctx: PluginContext,
|
||||
hostId: number,
|
||||
userId: string,
|
||||
): Promise<PollableHost | undefined> {
|
||||
const enabled = await ctx.asUser(userId, () =>
|
||||
ctx.settings.getHost<boolean>(hostId, "enableProxmoxStats"),
|
||||
);
|
||||
if (!enabled) return undefined;
|
||||
const host = await ctx.asUser(userId, () => ctx.hosts.get(hostId));
|
||||
if (!host) return undefined;
|
||||
const proxmoxStatsConfig = await ctx.asUser(userId, () =>
|
||||
ctx.settings.getHost(hostId, "proxmoxStatsConfig"),
|
||||
);
|
||||
return {
|
||||
id: host.id,
|
||||
userId: host.userId,
|
||||
proxmoxStatsConfig: proxmoxStatsConfig as string | undefined,
|
||||
enableProxmoxStats: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Called from activate(). Builds the polling manager and mounts its routes. */
|
||||
export function startProxmoxStatsService(
|
||||
ctx: PluginContext,
|
||||
mountOn: Router,
|
||||
historyRepository: ProxmoxNodeHistoryRepository,
|
||||
): void {
|
||||
pollingManager = new ProxmoxPollingManager<PollableHost>({
|
||||
fetchHostById: (hostId, userId) => fetchHostForPolling(ctx, hostId, userId),
|
||||
withSshConnection: (host, fn) =>
|
||||
ctx.ssh.withConnection(
|
||||
host.id,
|
||||
{
|
||||
pool: "proxmox-stats",
|
||||
purpose: "proxmox",
|
||||
overrides: { readyTimeout: 60000 },
|
||||
},
|
||||
fn,
|
||||
),
|
||||
historyRepository,
|
||||
});
|
||||
|
||||
router = express.Router();
|
||||
registerRoutes(router, pollingManager, historyRepository);
|
||||
mountOn.use("/stats", router);
|
||||
}
|
||||
|
||||
export function stopProxmoxStatsService(): void {
|
||||
pollingManager?.destroy();
|
||||
pollingManager = undefined;
|
||||
router = undefined;
|
||||
}
|
||||
|
||||
function registerRoutes(
|
||||
app: Router,
|
||||
manager: ProxmoxPollingManager<PollableHost>,
|
||||
historyRepository: ProxmoxNodeHistoryRepository,
|
||||
): void {
|
||||
/**
|
||||
* @openapi
|
||||
* /plugin-api/proxmox/stats/{id}:
|
||||
* get:
|
||||
* summary: Get cached Proxmox node stats for a host
|
||||
* description: Returns the most recently polled Proxmox stats snapshot for a host, or an empty skeleton if none has been collected yet.
|
||||
* tags: [Proxmox]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Proxmox stats snapshot.
|
||||
* 404:
|
||||
* description: Stats not available yet.
|
||||
*/
|
||||
app.get("/:id", async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
const cached = manager.getStats(id);
|
||||
if (!cached) {
|
||||
const errorState = manager.getError(id);
|
||||
return res.status(404).json({
|
||||
error: errorState?.error || "Stats not available",
|
||||
...EMPTY_SNAPSHOT,
|
||||
lastChecked: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
res.json({
|
||||
...cached.data,
|
||||
lastChecked: new Date(cached.timestamp).toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /plugin-api/proxmox/stats/start/{id}:
|
||||
* post:
|
||||
* summary: Start Proxmox stats collection
|
||||
* description: Registers a viewer and starts (or reuses) polling for a host's Proxmox node stats.
|
||||
* tags: [Proxmox]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Polling started, snapshot returned if already available.
|
||||
* 403:
|
||||
* description: Proxmox stats is not enabled for this host.
|
||||
* 404:
|
||||
* description: Host not found.
|
||||
*/
|
||||
app.post("/start/:id", async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
const userId = pluginCtx().currentActor()!;
|
||||
|
||||
try {
|
||||
const host = await fetchHostForPolling(pluginCtx(), id, userId);
|
||||
if (!host) {
|
||||
return res
|
||||
.status(403)
|
||||
.json({ error: "Proxmox stats is not enabled for this host" });
|
||||
}
|
||||
|
||||
const viewerSessionId = `proxmox-viewer-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
|
||||
manager.registerViewer(id, viewerSessionId, userId);
|
||||
await manager.ensurePolling(host, userId);
|
||||
|
||||
const cached = manager.getStats(id);
|
||||
if (cached) {
|
||||
return res.json({
|
||||
success: true,
|
||||
viewerSessionId,
|
||||
...cached.data,
|
||||
lastChecked: new Date(cached.timestamp).toISOString(),
|
||||
});
|
||||
}
|
||||
const errorState = manager.getError(id);
|
||||
if (errorState) {
|
||||
return res.json({
|
||||
success: true,
|
||||
viewerSessionId,
|
||||
status: "error",
|
||||
error: errorState.error,
|
||||
});
|
||||
}
|
||||
return res.json({ success: true, viewerSessionId, status: "collecting" });
|
||||
} catch (error) {
|
||||
pluginCtx().log.error(
|
||||
`Failed to start proxmox stats collection for host ${id}`,
|
||||
error as Error,
|
||||
);
|
||||
res
|
||||
.status(500)
|
||||
.json({ error: "Failed to start proxmox stats collection" });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /plugin-api/proxmox/stats/stop/{id}:
|
||||
* post:
|
||||
* summary: Stop Proxmox stats collection
|
||||
* description: Unregisters a viewer session for a host's Proxmox node stats polling.
|
||||
* tags: [Proxmox]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: id
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* requestBody:
|
||||
* required: false
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* viewerSessionId:
|
||||
* type: string
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Polling stopped successfully.
|
||||
*/
|
||||
app.post("/stop/:id", (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
const { viewerSessionId } = req.body as { viewerSessionId?: string };
|
||||
if (viewerSessionId && typeof viewerSessionId === "string") {
|
||||
manager.unregisterViewer(id, viewerSessionId);
|
||||
}
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /plugin-api/proxmox/stats/heartbeat:
|
||||
* post:
|
||||
* summary: Update Proxmox stats viewer heartbeat
|
||||
* description: Keeps a Proxmox stats viewer session alive.
|
||||
* tags: [Proxmox]
|
||||
* requestBody:
|
||||
* required: true
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* viewerSessionId:
|
||||
* type: string
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Heartbeat updated.
|
||||
* 400:
|
||||
* description: Invalid viewerSessionId.
|
||||
* 404:
|
||||
* description: Viewer session not found.
|
||||
*/
|
||||
app.post("/heartbeat", (req, res) => {
|
||||
const { viewerSessionId } = req.body as { viewerSessionId?: string };
|
||||
if (!viewerSessionId || typeof viewerSessionId !== "string") {
|
||||
return res.status(400).json({ error: "Invalid viewerSessionId" });
|
||||
}
|
||||
const ok = manager.updateHeartbeat(viewerSessionId);
|
||||
if (!ok) return res.status(404).json({ error: "Viewer session not found" });
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /plugin-api/proxmox/stats/history/{hostId}:
|
||||
* get:
|
||||
* summary: Get historical Proxmox node stats for a host
|
||||
* tags: [Proxmox]
|
||||
* parameters:
|
||||
* - in: path
|
||||
* name: hostId
|
||||
* required: true
|
||||
* schema:
|
||||
* type: integer
|
||||
* - in: query
|
||||
* name: range
|
||||
* schema:
|
||||
* type: string
|
||||
* enum: [1h, 6h, 24h, 7d, 30d]
|
||||
* - in: query
|
||||
* name: from
|
||||
* schema:
|
||||
* type: string
|
||||
* format: date-time
|
||||
* - in: query
|
||||
* name: to
|
||||
* schema:
|
||||
* type: string
|
||||
* format: date-time
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Array of node history rows.
|
||||
* 400:
|
||||
* description: Invalid range or date.
|
||||
*/
|
||||
app.get("/history/:hostId", async (req, res) => {
|
||||
const hostId = Number(req.params.hostId);
|
||||
const { range, from, to } = req.query as Record<string, string | undefined>;
|
||||
|
||||
const RANGE_OFFSETS: Record<string, number> = {
|
||||
"1h": 1 * 60 * 60 * 1000,
|
||||
"6h": 6 * 60 * 60 * 1000,
|
||||
"24h": 24 * 60 * 60 * 1000,
|
||||
"7d": 7 * 24 * 60 * 60 * 1000,
|
||||
"30d": 30 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
|
||||
let fromTs: string;
|
||||
let toTs = new Date().toISOString();
|
||||
if (range) {
|
||||
const offsetMs = RANGE_OFFSETS[range];
|
||||
if (!offsetMs) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: "Invalid range. Use 1h, 6h, 24h, 7d, or 30d" });
|
||||
}
|
||||
fromTs = new Date(Date.now() - offsetMs).toISOString();
|
||||
} else if (from && to) {
|
||||
const fromDate = new Date(from);
|
||||
const toDate = new Date(to);
|
||||
if (isNaN(fromDate.getTime()) || isNaN(toDate.getTime())) {
|
||||
return res.status(400).json({ error: "Invalid from/to date format" });
|
||||
}
|
||||
fromTs = fromDate.toISOString();
|
||||
toTs = toDate.toISOString();
|
||||
} else {
|
||||
fromTs = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
try {
|
||||
const toSqlite = (iso: string) =>
|
||||
iso.replace("T", " ").replace(/\.\d{3}Z$/, "");
|
||||
const rows = (
|
||||
await historyRepository.listRange(
|
||||
hostId,
|
||||
toSqlite(fromTs),
|
||||
toSqlite(toTs),
|
||||
)
|
||||
).map((row) => ({
|
||||
ts: row.ts,
|
||||
cpu_percent: row.cpuPercent,
|
||||
mem_percent: row.memPercent,
|
||||
disk_percent: row.diskPercent,
|
||||
net_rx_bytes: row.netRxBytes,
|
||||
net_tx_bytes: row.netTxBytes,
|
||||
}));
|
||||
res.json({ rows, fromTs, toTs });
|
||||
} catch (error) {
|
||||
pluginCtx().log.error(
|
||||
`Failed to fetch proxmox stats history for host ${hostId}`,
|
||||
error as Error,
|
||||
);
|
||||
res.status(500).json({ error: "Failed to fetch proxmox stats history" });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export { parseProxmoxStatsConfig };
|
||||
@@ -0,0 +1,64 @@
|
||||
import {
|
||||
adoptLegacyTable,
|
||||
defineTable,
|
||||
id,
|
||||
integer,
|
||||
real,
|
||||
refHost,
|
||||
refUser,
|
||||
text,
|
||||
timestamp,
|
||||
} from "@termix/plugin-sdk/db";
|
||||
|
||||
/**
|
||||
* Per-host historical node stats samples (CPU, memory, disk, network),
|
||||
* polled on an interval while a viewer has the Proxmox stats tab open.
|
||||
*
|
||||
* Adopted from core's proxmox_node_history, so the column and index names are
|
||||
* the legacy ones: the rename carries the existing rows across.
|
||||
*/
|
||||
export const proxmoxNodeHistory = adoptLegacyTable(
|
||||
"proxmox_node_history",
|
||||
defineTable("node_history", {
|
||||
id: id(),
|
||||
hostId: refHost(),
|
||||
ts: timestamp().notNull().defaultNow(),
|
||||
cpuPercent: real(),
|
||||
memPercent: real(),
|
||||
diskPercent: real(),
|
||||
netRxBytes: integer(),
|
||||
netTxBytes: integer(),
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* One saved dashboard layout per user per host for the Proxmox stats tab.
|
||||
*
|
||||
* Adopted from core's proxmox_stats_preferences.
|
||||
*/
|
||||
export const proxmoxStatsPreferences = adoptLegacyTable(
|
||||
"proxmox_stats_preferences",
|
||||
defineTable(
|
||||
"stats_preferences",
|
||||
{
|
||||
id: id(),
|
||||
userId: refUser(),
|
||||
hostId: refHost(),
|
||||
// JSON-encoded layout. No secrets, so plain text.
|
||||
layout: text().notNull(),
|
||||
createdAt: timestamp().notNull().defaultNow(),
|
||||
updatedAt: timestamp().notNull().defaultNow(),
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
name: "idx_proxmox_stats_prefs_user_host",
|
||||
columns: ["userId", "hostId"],
|
||||
unique: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
export const tables = [proxmoxNodeHistory, proxmoxStatsPreferences];
|
||||
Reference in new issue
Block a user