mirror of
https://github.com/Termix-SSH/Termix.git
synced 2026-10-29 14:45:24 +00:00
feat: make each plugin an npm workspace built by a termix-plugin CLI
This commit is contained in:
LukeGus committed
1 parent
4758016f41
commit
5796186a1c
448 files changed
+3087
-2003
No files matched your search
@@ -0,0 +1,448 @@
|
||||
import type {
|
||||
AutomationDefinition,
|
||||
RunStatus,
|
||||
Step,
|
||||
} from "../../../../src/types/automations.js";
|
||||
import {
|
||||
DEFAULT_MAX_RUN_SECONDS,
|
||||
MAX_AUTOMATION_DEPTH,
|
||||
MAX_STEP_OUTPUT_BYTES,
|
||||
} from "../../../../src/types/automations.js";
|
||||
import { createCurrentAutomationRepository } from "../../../../src/backend/database/repositories/factory.js";
|
||||
import { statsLogger } from "../../../../src/backend/utils/logger.js";
|
||||
import { resolveHostById } from "../../../../src/backend/hosts/host-resolver.js";
|
||||
import { notifyAutomationInternalEvent } from "../../../../src/backend/hosts/automation-events.js";
|
||||
import { executeStep } from "./actions/index.js";
|
||||
import type { StepExecutionContext, StepResult } from "./actions/types.js";
|
||||
import { compare } from "./conditions.js";
|
||||
import { renderTemplate, type TemplateContext } from "./template.js";
|
||||
|
||||
export interface RunRequest {
|
||||
automationId: number;
|
||||
triggerType: string;
|
||||
triggerContext?: Record<string, unknown>;
|
||||
triggerHostId?: number;
|
||||
/** Overrides the automation's own dry-run flag, for "test run". */
|
||||
dryRun?: boolean;
|
||||
parentRunId?: number;
|
||||
ancestry?: number[];
|
||||
depth?: number;
|
||||
}
|
||||
|
||||
export interface RunOutcome {
|
||||
runId: number | null;
|
||||
status: RunStatus;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes automations.
|
||||
*
|
||||
* Both HTTP handlers and the metrics hooks live in this same process, so this
|
||||
* is a plain singleton rather than anything cross-process. State that has to
|
||||
* survive a restart (cooldowns, dwell windows) lives in the database; the only
|
||||
* thing held in memory is the set of runs currently in flight, which is
|
||||
* meaningless after a restart anyway.
|
||||
*/
|
||||
export class AutomationEngine {
|
||||
private static instance: AutomationEngine;
|
||||
|
||||
private readonly running = new Set<number>();
|
||||
private readonly queued = new Map<number, number>();
|
||||
|
||||
static getInstance(): AutomationEngine {
|
||||
if (!AutomationEngine.instance) {
|
||||
AutomationEngine.instance = new AutomationEngine();
|
||||
}
|
||||
return AutomationEngine.instance;
|
||||
}
|
||||
|
||||
isRunning(automationId: number): boolean {
|
||||
return this.running.has(automationId);
|
||||
}
|
||||
|
||||
async run(request: RunRequest): Promise<RunOutcome> {
|
||||
const repository = createCurrentAutomationRepository();
|
||||
const automation = await repository.findById(request.automationId);
|
||||
|
||||
if (!automation) {
|
||||
return { runId: null, status: "failed", error: "Automation not found" };
|
||||
}
|
||||
|
||||
const depth = request.depth ?? 0;
|
||||
const ancestry = request.ancestry ?? [];
|
||||
|
||||
// Refuse recursion before anything is recorded, so a cycle cannot spin.
|
||||
if (depth > MAX_AUTOMATION_DEPTH) {
|
||||
return {
|
||||
runId: null,
|
||||
status: "failed",
|
||||
error: `Maximum automation depth of ${MAX_AUTOMATION_DEPTH} exceeded`,
|
||||
};
|
||||
}
|
||||
if (ancestry.includes(automation.id)) {
|
||||
return {
|
||||
runId: null,
|
||||
status: "failed",
|
||||
error: `Automation ${automation.id} is already running in this chain`,
|
||||
};
|
||||
}
|
||||
|
||||
let definition: AutomationDefinition;
|
||||
try {
|
||||
definition = JSON.parse(automation.definition) as AutomationDefinition;
|
||||
} catch {
|
||||
return {
|
||||
runId: null,
|
||||
status: "failed",
|
||||
error: "Automation definition is not valid JSON",
|
||||
};
|
||||
}
|
||||
|
||||
// A second trigger while a run is in flight is recorded as skipped rather
|
||||
// than dropped silently, so the history explains what happened.
|
||||
//
|
||||
// The slot has to be claimed in the same tick as the check. It used to be
|
||||
// claimed several awaits later, so two triggers arriving together both
|
||||
// passed this test and both ran.
|
||||
let claimed = false;
|
||||
if (this.running.has(automation.id)) {
|
||||
const policy = automation.concurrencyPolicy;
|
||||
if (policy === "skip") {
|
||||
const run = await repository.createRun({
|
||||
automationId: automation.id,
|
||||
userId: automation.userId,
|
||||
triggerType: request.triggerType,
|
||||
triggerContext: JSON.stringify(request.triggerContext ?? {}),
|
||||
status: "skipped",
|
||||
});
|
||||
await repository.finishRun(run.id, {
|
||||
status: "skipped",
|
||||
error: "A previous run was still in progress",
|
||||
durationMs: 0,
|
||||
});
|
||||
return { runId: run.id, status: "skipped" };
|
||||
}
|
||||
if (policy === "queue") {
|
||||
const depthNow = this.queued.get(automation.id) ?? 0;
|
||||
if (depthNow >= 5) {
|
||||
return { runId: null, status: "skipped", error: "Queue is full" };
|
||||
}
|
||||
this.queued.set(automation.id, depthNow + 1);
|
||||
try {
|
||||
await this.waitUntilFree(automation.id);
|
||||
} finally {
|
||||
this.queued.set(
|
||||
automation.id,
|
||||
(this.queued.get(automation.id) ?? 1) - 1,
|
||||
);
|
||||
}
|
||||
// waitUntilFree gives up on its own deadline, so the slot may still be
|
||||
// taken. Only claim it when it is genuinely free.
|
||||
if (!this.running.has(automation.id)) {
|
||||
this.running.add(automation.id);
|
||||
claimed = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.running.add(automation.id);
|
||||
claimed = true;
|
||||
}
|
||||
|
||||
const dryRun = request.dryRun ?? automation.dryRun;
|
||||
const maxRunSeconds = automation.maxRunSeconds || DEFAULT_MAX_RUN_SECONDS;
|
||||
const startedAt = Date.now();
|
||||
|
||||
let run: { id: number };
|
||||
try {
|
||||
run = await repository.createRun({
|
||||
automationId: automation.id,
|
||||
userId: automation.userId,
|
||||
triggerType: request.triggerType,
|
||||
triggerContext: JSON.stringify(request.triggerContext ?? {}),
|
||||
status: "running",
|
||||
dryRun,
|
||||
parentRunId: request.parentRunId ?? null,
|
||||
});
|
||||
} catch (err) {
|
||||
// The slot is already claimed at this point, so it has to be given back
|
||||
// here; the finally below is only reached once a run row exists.
|
||||
if (claimed) this.running.delete(automation.id);
|
||||
return {
|
||||
runId: null,
|
||||
status: "failed",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
if (!claimed) this.running.add(automation.id);
|
||||
|
||||
const trigger = { ...(request.triggerContext ?? {}) };
|
||||
trigger.type ??= request.triggerType;
|
||||
let host: TemplateContext["host"];
|
||||
if (request.triggerHostId) {
|
||||
try {
|
||||
const resolved = await resolveHostById(
|
||||
request.triggerHostId,
|
||||
automation.userId,
|
||||
);
|
||||
if (resolved) {
|
||||
host = {
|
||||
id: request.triggerHostId,
|
||||
name: resolved.name || resolved.ip,
|
||||
ip: resolved.ip,
|
||||
username: resolved.username,
|
||||
port: resolved.port,
|
||||
};
|
||||
trigger.hostName ??= host.name;
|
||||
}
|
||||
} catch {
|
||||
// A notification should still run with its numeric host id.
|
||||
}
|
||||
}
|
||||
|
||||
const template: TemplateContext = {
|
||||
host,
|
||||
trigger,
|
||||
steps: {},
|
||||
vars: {},
|
||||
run: {
|
||||
id: run.id,
|
||||
automationId: automation.id,
|
||||
startedAt: new Date(startedAt).toISOString(),
|
||||
},
|
||||
};
|
||||
|
||||
const context: StepExecutionContext = {
|
||||
userId: automation.userId,
|
||||
automationId: automation.id,
|
||||
runId: run.id,
|
||||
dryRun,
|
||||
template,
|
||||
triggerHostId: request.triggerHostId,
|
||||
ancestry: [...ancestry, automation.id],
|
||||
depth,
|
||||
deadlineAt: startedAt + maxRunSeconds * 1000,
|
||||
};
|
||||
|
||||
let status: RunStatus = "success";
|
||||
let error: string | undefined;
|
||||
|
||||
try {
|
||||
const result = await this.runSteps(definition.steps ?? [], context, {
|
||||
index: 0,
|
||||
});
|
||||
if (result.halted?.status === "failed") {
|
||||
status = "failed";
|
||||
error = "Stopped by a stop step";
|
||||
} else if (result.failed) {
|
||||
status = "failed";
|
||||
error = result.error;
|
||||
}
|
||||
if (Date.now() >= context.deadlineAt) {
|
||||
status = "timeout";
|
||||
error = `Run exceeded ${maxRunSeconds}s`;
|
||||
}
|
||||
} catch (err) {
|
||||
status = "failed";
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
} finally {
|
||||
this.running.delete(automation.id);
|
||||
}
|
||||
|
||||
await repository.finishRun(run.id, {
|
||||
status,
|
||||
error: error ?? null,
|
||||
durationMs: Date.now() - startedAt,
|
||||
});
|
||||
|
||||
if (status === "failed") {
|
||||
statsLogger.warn(`Automation "${automation.name}" failed`, {
|
||||
operation: "automation_run_failed",
|
||||
automationId: automation.id,
|
||||
runId: run.id,
|
||||
error,
|
||||
});
|
||||
|
||||
// An automation_failed handler that itself fails must not re-announce
|
||||
// its own failure, so the event is not emitted for runs that this event
|
||||
// already started.
|
||||
if (request.triggerType !== "internal_event") {
|
||||
// Emitted onto the bus rather than imported from hosts/: the bus is
|
||||
// fire-and-forget, so this no longer needs the lazy-import dance that
|
||||
// was working around the repositories cycle.
|
||||
notifyAutomationInternalEvent(
|
||||
"automation_failed",
|
||||
automation.userId,
|
||||
undefined,
|
||||
{
|
||||
automationId: automation.id,
|
||||
automationName: automation.name,
|
||||
runId: run.id,
|
||||
error: error ?? null,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { runId: run.id, status, error };
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a list of steps in order, descending into if/else. Returns as soon as
|
||||
* a stop step halts the run or a failing step's policy says to stop.
|
||||
*/
|
||||
private async runSteps(
|
||||
steps: Step[],
|
||||
context: StepExecutionContext,
|
||||
cursor: { index: number },
|
||||
): Promise<{
|
||||
failed: boolean;
|
||||
error?: string;
|
||||
halted?: { status: "success" | "failed" };
|
||||
}> {
|
||||
const repository = createCurrentAutomationRepository();
|
||||
|
||||
for (const step of steps) {
|
||||
if (step.enabled === false) continue;
|
||||
|
||||
if (Date.now() >= context.deadlineAt) {
|
||||
return { failed: true, error: "Run deadline exceeded" };
|
||||
}
|
||||
|
||||
const stepIndex = cursor.index++;
|
||||
|
||||
if (step.type === "if") {
|
||||
const left = renderTemplate(step.condition.left, context.template);
|
||||
const right = renderTemplate(
|
||||
step.condition.right ?? "",
|
||||
context.template,
|
||||
);
|
||||
const matched = compare(left, step.condition.operator, right);
|
||||
|
||||
const rowId = await repository.createRunStep({
|
||||
runId: context.runId,
|
||||
stepIndex,
|
||||
stepId: step.id,
|
||||
stepType: "if",
|
||||
status: "running",
|
||||
});
|
||||
await repository.finishRunStep(rowId, {
|
||||
status: "success",
|
||||
output: `Condition ${matched ? "matched" : "did not match"}: ${left} ${step.condition.operator} ${right}`,
|
||||
});
|
||||
|
||||
const branch = matched ? step.then : (step.else ?? []);
|
||||
const result = await this.runSteps(branch, context, cursor);
|
||||
if (result.halted) return result;
|
||||
if (result.failed) return result;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (step.type === "run_automation") {
|
||||
const rowId = await repository.createRunStep({
|
||||
runId: context.runId,
|
||||
stepIndex,
|
||||
stepId: step.id,
|
||||
stepType: step.type,
|
||||
status: "running",
|
||||
});
|
||||
|
||||
const nested = await this.run({
|
||||
automationId: step.automationId,
|
||||
triggerType: "run_automation",
|
||||
triggerContext: { parentAutomationId: context.automationId },
|
||||
triggerHostId: context.triggerHostId,
|
||||
dryRun: context.dryRun,
|
||||
parentRunId: context.runId,
|
||||
ancestry: context.ancestry,
|
||||
depth: context.depth + 1,
|
||||
});
|
||||
|
||||
const nestedOk = nested.status === "success";
|
||||
await repository.finishRunStep(rowId, {
|
||||
status: nestedOk ? "success" : "failed",
|
||||
output: `Nested run ${nested.runId ?? "not started"}: ${nested.status}`,
|
||||
error: nested.error ?? null,
|
||||
});
|
||||
|
||||
if (!nestedOk && (step.onError ?? "stop") === "stop") {
|
||||
return { failed: true, error: nested.error ?? "Nested run failed" };
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const rowId = await repository.createRunStep({
|
||||
runId: context.runId,
|
||||
stepIndex,
|
||||
stepId: step.id,
|
||||
stepType: step.type,
|
||||
status: "running",
|
||||
});
|
||||
|
||||
let result: StepResult;
|
||||
try {
|
||||
result = await executeStep(step, context);
|
||||
} catch (err) {
|
||||
result = {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const { text, truncated } = truncate(result.output);
|
||||
await repository.finishRunStep(rowId, {
|
||||
status: result.success ? "success" : "failed",
|
||||
output: text,
|
||||
error: result.error ?? null,
|
||||
truncated,
|
||||
});
|
||||
|
||||
// Later steps read earlier output through {{steps.<id>.stdout}}.
|
||||
context.template.steps = {
|
||||
...context.template.steps,
|
||||
[step.id]: {
|
||||
stdout: result.output ?? "",
|
||||
code: result.success ? 0 : 1,
|
||||
},
|
||||
};
|
||||
if (result.vars) {
|
||||
context.template.vars = { ...context.template.vars, ...result.vars };
|
||||
}
|
||||
|
||||
if (result.halt) return { failed: false, halted: result.halt };
|
||||
|
||||
if (!result.success) {
|
||||
const policy = step.onError ?? "stop";
|
||||
if (policy === "stop") {
|
||||
return { failed: true, error: result.error };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { failed: false };
|
||||
}
|
||||
|
||||
private async waitUntilFree(automationId: number): Promise<void> {
|
||||
const started = Date.now();
|
||||
while (this.running.has(automationId)) {
|
||||
if (Date.now() - started > 60_000) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Keeps a single step's output from bloating the database. */
|
||||
function truncate(output: string | undefined): {
|
||||
text: string | null;
|
||||
truncated: boolean;
|
||||
} {
|
||||
if (!output) return { text: null, truncated: false };
|
||||
if (Buffer.byteLength(output, "utf8") <= MAX_STEP_OUTPUT_BYTES) {
|
||||
return { text: output, truncated: false };
|
||||
}
|
||||
return {
|
||||
text: output.slice(0, MAX_STEP_OUTPUT_BYTES) + "\n... (truncated)",
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user