feat(automations): add per-host maintenance mode and recurring schedules (#1521)

plugins/automations/tests/backend/maintenance-scheduler.test.ts

@@ -0,0 +1,47 @@
+import { expect, it, vi } from "vitest";
+import { createScheduler } from "../../src/backend/scheduler.js";
+
+it("discards sustained breaches during maintenance but still runs scheduled work", async () => {
+  const run = vi.fn(async () => ({ runId: 1, status: "success" as const }));
+  const clearBreach = vi.fn(async () => {});
+  const repository = {
+    listDueSchedules: async () => [{ automationId: 2, intervalSeconds: 60 }],
+    markScheduleTicked: vi.fn(),
+    listOpenBreaches: async () => [
+      {
+        automationId: 1,
+        stateKey: "7:cpu.percent",
+        breachStartedAt: "2000-01-01T00:00:00Z",
+      },
+    ],
+    findById: async () => ({
+      id: 1,
+      userId: "alice",
+      enabled: true,
+      definition: JSON.stringify({
+        trigger: { kind: "metric_threshold", forSeconds: 60 },
+      }),
+    }),
+    clearBreach,
+    failStaleRunningRuns: vi.fn(),
+    pruneRunsOlderThan: async () => 0,
+  };
+  const maintaining = vi.fn(async () => true);
+  const scheduler = createScheduler({
+    repository: repository as never,
+    engine: { run },
+    log: { debug() {}, info() {}, warn() {}, error() {} },
+    reconcile: [],
+    isMaintaining: maintaining,
+  });
+  await scheduler.tick();
+  expect(maintaining).toHaveBeenCalledWith("alice", 7);
+  expect(clearBreach).toHaveBeenCalledWith(1, "7:cpu.percent");
+  expect(run).toHaveBeenCalledTimes(1);
+  expect(run).toHaveBeenCalledWith(
+    expect.objectContaining({
+      automationId: 2,
+      triggerType: "schedule",
+    }),
+  );
+});

plugins/automations/tests/backend/maintenance-time.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it } from "vitest";
+import {
+  advanceMaintenance,
+  beginMaintenance,
+  nextOccurrence,
+  parsePlan,
+  plansOverlap,
+} from "../../src/backend/maintenance-time.js";
+import {
+  emptyMaintenance,
+  type MaintenancePlan,
+} from "../../src/maintenance.js";
+
+const at = (value: string) => Date.parse(value);
+const plan = (overrides: Partial<MaintenancePlan> = {}): MaintenancePlan => ({
+  id: "one",
+  reason: "Upgrade",
+  start: "2026-01-31T21:00:00.000Z",
+  nextStart: "2026-01-31T21:00:00.000Z",
+  durationMinutes: 60,
+  recurrence: "once",
+  graceMinutes: 20,
+  notifyOverdue: true,
+  ...overrides,
+});
+
+describe("maintenance calendar", () => {
+  it("keeps weekly UTC time across DST and skips missing monthly days", () => {
+    expect(
+      new Date(
+        nextOccurrence(
+          plan({ recurrence: "weekly" }),
+          at("2026-03-28T21:00Z"),
+        )!,
+      ).toISOString(),
+    ).toBe("2026-04-04T21:00:00.000Z");
+    expect(
+      new Date(
+        nextOccurrence(
+          plan({ recurrence: "monthly" }),
+          at("2026-01-31T21:00Z"),
+        )!,
+      ).toISOString(),
+    ).toBe("2026-03-31T21:00:00.000Z");
+    expect(nextOccurrence(plan(), at("2026-01-31T21:00Z"))).toBeNull();
+  });
+  it("rejects actual overlaps and accepts touching windows", () => {
+    expect(plansOverlap(plan(), plan({ start: "2026-01-31T21:59:00Z" }))).toBe(
+      true,
+    );
+    expect(plansOverlap(plan(), plan({ start: "2026-01-31T22:00:00Z" }))).toBe(
+      false,
+    );
+    expect(
+      plansOverlap(
+        plan({ recurrence: "monthly" }),
+        plan({ start: "2030-03-31T21:30:00Z" }),
+      ),
+    ).toBe(true);
+    expect(
+      plansOverlap(
+        plan({ recurrence: "weekly" }),
+        plan({ recurrence: "monthly", start: "2026-03-31T21:30:00Z" }),
+      ),
+    ).toBe(true);
+    expect(
+      plansOverlap(
+        plan({ recurrence: "weekly" }),
+        plan({ recurrence: "monthly", start: "2026-03-31T23:00:00Z" }),
+      ),
+    ).toBe(false);
+  });
+  it("does not end at ETM and merges a later nonoverlapping schedule", () => {
+    const state = emptyMaintenance();
+    state.plans = [
+      plan(),
+      plan({
+        id: "two",
+        reason: "Firmware",
+        start: "2026-01-31T22:30:00Z",
+        nextStart: "2026-01-31T22:30:00Z",
+      }),
+    ];
+    advanceMaintenance(state, at("2026-01-31T22:15Z"));
+    expect(state.active?.estimatedEnd).toBe("2026-01-31T22:00:00.000Z");
+    advanceMaintenance(state, at("2026-01-31T22:45Z"));
+    expect(state.active?.startedAt).toBe("2026-01-31T21:00:00.000Z");
+    expect(state.active?.estimatedEnd).toBe("2026-01-31T23:30:00.000Z");
+    expect(state.active?.reasons).toEqual(["Upgrade", "Firmware"]);
+    advanceMaintenance(state, at("2026-02-02T00:00Z"));
+    expect(state.active).not.toBeNull();
+  });
+  it("catches up after restart and never re-enters a consumed one-time plan", () => {
+    const state = emptyMaintenance();
+    state.plans = [plan({ recurrence: "monthly" })];
+    advanceMaintenance(state, at("2026-05-01T00:00Z"));
+    expect(state.active?.startedAt).toBe("2026-03-31T21:00:00.000Z");
+    expect(state.plans[0].nextStart).toBe("2026-05-31T21:00:00.000Z");
+    state.active = null;
+    advanceMaintenance(state, at("2026-05-01T00:00Z"));
+    expect(state.active).toBeNull();
+  });
+  it("preserves a delivered reminder unless a later estimate extends maintenance", () => {
+    const state = emptyMaintenance();
+    beginMaintenance(state, plan(), at("2026-01-31T21:00Z"));
+    state.active!.notified = true;
+    beginMaintenance(
+      state,
+      plan({ durationMinutes: 10 }),
+      at("2026-01-31T21:05Z"),
+    );
+    expect(state.active!.notified).toBe(true);
+    beginMaintenance(state, plan(), at("2026-01-31T22:30Z"));
+    expect(state.active!.notified).toBe(false);
+  });
+  it.each([
+    { reason: " " },
+    { durationMinutes: 0 },
+    { durationMinutes: 10081 },
+    { graceMinutes: -1 },
+    { recurrence: "daily" },
+    { notifyOverdue: "true" },
+    { start: "2020-01-01T00:00Z" },
+    { start: "2026-01-31T21:00:01Z" },
+  ])("rejects invalid input %j", (override) => {
+    expect(() =>
+      parsePlan(plan(override as never), at("2026-01-01T00:00Z")),
+    ).toThrow();
+  });
+});

plugins/automations/tests/backend/maintenance.test.ts

@@ -0,0 +1,209 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { startServer as createServer, host, type TestServer } from "./helpers";
+import { createMaintenanceRepository } from "../../src/backend/maintenance-repository.js";
+import { createMaintenanceService } from "../../src/backend/maintenance-service.js";
+
+function startServer(options: Parameters<typeof createServer>[0] = {}) {
+  return createServer({
+    ...options,
+    before: (db) => {
+      db.exec("INSERT INTO ssh_data (id) VALUES (1), (2)");
+      options.before?.(db);
+    },
+  });
+}
+
+let server: TestServer | undefined;
+afterEach(async () => {
+  vi.restoreAllMocks();
+  await server?.close();
+  server = undefined;
+});
+const input = {
+  action: "start",
+  reason: "OS upgrade",
+  durationMinutes: 60,
+  graceMinutes: 20,
+  notifyOverdue: true,
+};
+
+describe("host maintenance", () => {
+  it("scopes reads and edits to the host owner and enforces edit permission", async () => {
+    server = await startServer({ hosts: [host(1), host(2, "bob")] });
+    expect(
+      (await server.request("POST", "/maintenance/1", { body: input })).status,
+    ).toBe(200);
+    expect(
+      (await server.request("GET", "/maintenance", { user: "bob" })).body,
+    ).toEqual([]);
+    expect(
+      (
+        await server.request("POST", "/maintenance/1", {
+          user: "bob",
+          body: { action: "end" },
+        })
+      ).status,
+    ).toBe(404);
+    expect(
+      (await server.request("GET", "/maintenance/1", { user: "bob" })).status,
+    ).toBe(404);
+    expect(
+      (await server.request("POST", "/maintenance/1x", { body: input })).status,
+    ).toBe(400);
+    expect(
+      (await server.request("GET", "/maintenance/1", { user: null })).status,
+    ).toBe(401);
+    await server.close();
+    server = await startServer({ permissions: ["automations.view"] });
+    expect(
+      (await server.request("POST", "/maintenance/1", { body: input })).status,
+    ).toBe(403);
+  });
+  it("survives service restart, remains active beyond ETM, and reminds only once with fresh offline evidence", async () => {
+    server = await startServer();
+    let now = Date.parse("2026-01-01T12:00Z");
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    await server.request("POST", "/maintenance/1", { body: input });
+    const repository = await createMaintenanceRepository(server.db.database);
+    let service = createMaintenanceService(server.mock.ctx, repository);
+    now = Date.parse("2026-01-01T13:21Z");
+    server.mock.hostStatuses.set(1, {
+      status: "offline",
+      lastChecked: "2026-01-01T12:00Z",
+    });
+    await service.tick();
+    expect(server.mock.notifications).toHaveLength(0);
+    server.mock.hostStatuses.set(1, {
+      status: "reachable",
+      lastChecked: "2026-01-01T13:21Z",
+    });
+    await service.tick();
+    expect(server.mock.notifications).toHaveLength(0);
+    server.mock.hostStatuses.set(1, {
+      status: "offline",
+      lastChecked: "2026-01-01T13:21Z",
+    });
+    await service.tick();
+    expect(server.mock.notifications).toHaveLength(1);
+    expect(server.mock.notifications[0].actor).toBe("alice");
+    service = createMaintenanceService(server.mock.ctx, repository);
+    await service.tick();
+    expect(server.mock.notifications).toHaveLength(1);
+    expect((await service.read("alice", 1)).active).not.toBeNull();
+    await service.edit("alice", 1, "end", {});
+    expect(await service.isMaintaining("alice", 1)).toBe(false);
+  });
+  it("serializes concurrent starts and conflicting schedules without losing state", async () => {
+    server = await startServer();
+    const service = createMaintenanceService(
+      server.mock.ctx,
+      await createMaintenanceRepository(server.db.database),
+    );
+    const starts = await Promise.allSettled([
+      service.edit("alice", 1, "start", input),
+      service.edit("alice", 1, "start", input),
+    ]);
+    expect(starts.map((r) => r.status).sort()).toEqual([
+      "fulfilled",
+      "rejected",
+    ]);
+    const start = new Date(
+      Math.ceil((Date.now() + 86400000) / 60000) * 60000,
+    ).toISOString();
+    const schedules = await Promise.allSettled([
+      service.edit("alice", 1, "schedule", {
+        ...input,
+        start,
+        recurrence: "weekly",
+      }),
+      service.edit("alice", 1, "schedule", {
+        ...input,
+        start,
+        recurrence: "monthly",
+      }),
+    ]);
+    expect(schedules.map((r) => r.status).sort()).toEqual([
+      "fulfilled",
+      "rejected",
+    ]);
+    expect((await service.read("alice", 1)).plans).toHaveLength(1);
+  });
+  it("starts a due schedule before accepting status events and handles deletion", async () => {
+    server = await startServer();
+    let now = Date.parse("2026-01-01T12:00Z");
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    const service = createMaintenanceService(
+      server.mock.ctx,
+      await createMaintenanceRepository(server.db.database),
+    );
+    await service.edit("alice", 1, "schedule", {
+      ...input,
+      start: "2026-01-01T13:00:00Z",
+      recurrence: "once",
+    });
+    expect(await service.isMaintaining("alice", 1)).toBe(false);
+    now = Date.parse("2026-01-01T13:00Z");
+    expect(await service.isMaintaining("alice", 1)).toBe(true);
+    const state = await service.read("alice", 1);
+    await service.edit("alice", 1, "remove", { id: state.plans[0].id });
+    expect(await service.isMaintaining("alice", 1)).toBe(true);
+    await service.wipeUser("alice");
+    expect(await service.list("alice")).toEqual([]);
+  });
+  it("clears old dwell windows on start and end and cascades host deletion", async () => {
+    server = await startServer();
+    const created = await server.request("POST", "/", {
+      body: {
+        name: "CPU",
+        definition: {
+          version: 1,
+          trigger: {
+            kind: "metric_threshold",
+            hostSelector: { kind: "host", hostId: 1 },
+            metric: { path: "cpu.percent" },
+            operator: ">",
+            value: 90,
+            forSeconds: 60,
+            cooldownMinutes: 0,
+          },
+          steps: [],
+        },
+      },
+    });
+    expect(created.status).toBe(201);
+    const { createAutomationRepository } =
+      await import("../../src/backend/repository.js");
+    const repository = await createAutomationRepository(
+      server.db.database,
+      async () => [],
+    );
+    const seed = () =>
+      repository.upsertTriggerState({
+        automationId: created.body.id,
+        stateKey: "1",
+        breachStartedAt: "2000-01-01T00:00:00Z",
+      });
+    await seed();
+    expect(
+      (await server.request("POST", "/maintenance/1", { body: input })).status,
+    ).toBe(200);
+    expect(
+      (await repository.getTriggerState(created.body.id, "1"))?.breachStartedAt,
+    ).toBeNull();
+    await seed();
+    expect(
+      (
+        await server.request("POST", "/maintenance/1", {
+          body: { action: "end" },
+        })
+      ).status,
+    ).toBe(200);
+    expect(
+      (await repository.getTriggerState(created.body.id, "1"))?.breachStartedAt,
+    ).toBeNull();
+    server.db.sqlite.prepare("DELETE FROM ssh_data WHERE id = 1").run();
+    expect(
+      await (await createMaintenanceRepository(server.db.database)).list(),
+    ).toEqual([]);
+  });
+});

plugins/automations/tests/backend/triggers.test.ts

@@ -438,3 +438,132 @@ describe("watchedHosts", () => {
     expect((await triggers.watchedHosts("metric_threshold")).size).toBe(0);
   });
 });
+
+describe("maintenance suppression", () => {
+  it("updates the status baseline without firing or replaying maintenance edges", async () => {
+    let maintaining = false;
+    const guarded = createTriggers(
+      repository as never,
+      { run },
+      log,
+      async () => maintaining,
+    );
+    addAutomation({
+      kind: "host_status",
+      hostSelector: { kind: "all" },
+      to: "offline",
+      cooldownMinutes: 0,
+    });
+    await guarded.onStatus({ hostId: 7, ownerUserId: "user-1", online: true });
+    maintaining = true;
+    await guarded.onStatus({ hostId: 7, ownerUserId: "user-1", online: false });
+    maintaining = false;
+    await guarded.onStatus({ hostId: 7, ownerUserId: "user-1", online: false });
+    expect(run).not.toHaveBeenCalled();
+    await guarded.onStatus({ hostId: 7, ownerUserId: "user-1", online: true });
+    await guarded.onStatus({ hostId: 7, ownerUserId: "user-1", online: false });
+    expect(run).toHaveBeenCalledTimes(1);
+  });
+  it("maintains health baselines and suppresses container events", async () => {
+    const guarded = createTriggers(
+      repository as never,
+      { run },
+      log,
+      async () => true,
+    );
+    addAutomation({
+      kind: "health_check",
+      hostSelector: { kind: "all" },
+      to: "failing",
+      cooldownMinutes: 0,
+    });
+    addAutomation({
+      kind: "docker_event",
+      hostSelector: { kind: "all" },
+      event: "exited",
+      cooldownMinutes: 0,
+    });
+    await guarded.onHealthCheck({
+      hostId: 7,
+      userId: "user-1",
+      checkId: "http",
+      ok: true,
+    });
+    await guarded.onHealthCheck({
+      hostId: 7,
+      userId: "user-1",
+      checkId: "http",
+      ok: false,
+    });
+    await guarded.onDockerEvent({
+      hostId: 7,
+      ownerUserId: "user-1",
+      container: "app",
+      event: "exited",
+    });
+    expect(run).not.toHaveBeenCalled();
+    expect(triggerState.get("1:7:http")?.lastObservedState).toBe("failing");
+  });
+  it("clears a metric dwell window so maintenance time cannot count toward a later alert", async () => {
+    let maintaining = true;
+    const guarded = createTriggers(
+      repository as never,
+      { run },
+      log,
+      async () => maintaining,
+    );
+    addAutomation({
+      kind: "metric_threshold",
+      hostSelector: { kind: "all" },
+      metric: { path: "cpu.percent" },
+      operator: ">",
+      value: 90,
+      forSeconds: 60,
+      cooldownMinutes: 0,
+    });
+    triggerState.set("1:7", {
+      breachStartedAt: "2000-01-01T00:00:00Z",
+    });
+    const event = {
+      hostId: 7,
+      ownerUserId: "user-1",
+      metrics: { cpu: { percent: 99 } },
+    };
+    await guarded.onMetrics(event);
+    expect(triggerState.get("1:7")?.breachStartedAt).toBeNull();
+    maintaining = false;
+    await guarded.onMetrics(event);
+    expect(run).not.toHaveBeenCalled();
+    expect(triggerState.get("1:7")?.breachStartedAt).not.toBeNull();
+  });
+  it("suppresses tunnel disconnects but preserves security login events", async () => {
+    const guarded = createTriggers(
+      repository as never,
+      { run },
+      log,
+      async () => true,
+    );
+    addAutomation({
+      kind: "internal_event",
+      event: "tunnel_disconnected",
+      cooldownMinutes: 0,
+    });
+    addAutomation({
+      kind: "internal_event",
+      event: "user_login",
+      cooldownMinutes: 0,
+    });
+    await guarded.onInternalEvent({
+      event: "tunnel_disconnected",
+      userId: "user-1",
+      hostId: 7,
+    });
+    expect(run).not.toHaveBeenCalled();
+    await guarded.onInternalEvent({
+      event: "user_login",
+      userId: "user-1",
+      hostId: 7,
+    });
+    expect(run).toHaveBeenCalledTimes(1);
+  });
+});

plugins/automations/tests/frontend/maintenance.test.tsx

@@ -0,0 +1,141 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { fireEvent, waitFor, within } from "@testing-library/react";
+import {
+  renderWithApp,
+  type RenderedPluginApp,
+} from "@termix/plugin-sdk/testing";
+import type { PluginManifest } from "@termix/plugin-sdk/manifest";
+import type { PluginApiClient } from "@termix/plugin-sdk/frontend";
+import * as plugin from "../../src/frontend/index";
+import manifest from "../../manifest.json";
+import locales from "../../locales/en.json";
+import { emptyMaintenance, type HostMaintenance } from "../../src/maintenance";
+import { createMaintenanceStore } from "../../src/frontend/maintenance-store";
+
+let rendered: RenderedPluginApp | undefined;
+afterEach(async () => {
+  await rendered?.deactivate();
+  rendered = undefined;
+  vi.restoreAllMocks();
+});
+const active: HostMaintenance = {
+  plans: [],
+  active: {
+    startedAt: "2026-01-01T12:00Z",
+    estimatedEnd: "2026-01-01T13:00Z",
+    reasons: ["OS upgrade"],
+    graceMinutes: 20,
+    notifyOverdue: true,
+    notified: false,
+  },
+};
+const host = { id: 7, name: "Database", ip: "10.0.0.7" };
+function apiFor(state: HostMaintenance) {
+  return {
+    get: vi.fn(async () => ({ data: [{ hostId: 7, state }] })),
+    post: vi.fn(async () => ({ data: emptyMaintenance() })),
+  };
+}
+async function render(
+  api: ReturnType<typeof apiFor>,
+  permissions = ["automations.view", "automations.edit"],
+) {
+  rendered = await renderWithApp(plugin, {
+    manifest: manifest as unknown as PluginManifest,
+    locales,
+    api: api as unknown as PluginApiClient,
+    permissions,
+  });
+  return within(
+    rendered.renderTab("host_maintenance", { host, isVisible: true }),
+  );
+}
+
+describe("maintenance UI", () => {
+  it("shows the reason and estimate and confirms before ending the correct host", async () => {
+    const api = apiFor(active);
+    const view = await render(api);
+    await waitFor(() => expect(view.getByText("OS upgrade")).toBeTruthy());
+    fireEvent.click(view.getByRole("button", { name: "End maintenance" }));
+    expect(api.post).not.toHaveBeenCalled();
+    fireEvent.click(view.getByRole("button", { name: "Confirm end" }));
+    await waitFor(() =>
+      expect(api.post).toHaveBeenCalledWith("/maintenance/7", {
+        action: "end",
+      }),
+    );
+  });
+  it("offers a read-only view without edit permission", async () => {
+    const view = await render(apiFor(active), ["automations.view"]);
+    await waitFor(() => expect(view.getByText("OS upgrade")).toBeTruthy());
+    expect(view.queryByRole("button", { name: "End maintenance" })).toBeNull();
+    expect(view.queryByLabelText("Reason")).toBeNull();
+  });
+  it("submits a recurring UTC schedule with its grace period", async () => {
+    const api = apiFor(emptyMaintenance());
+    const view = await render(api);
+    await waitFor(() =>
+      expect(
+        view
+          .getByRole("button", { name: "Start now" })
+          .hasAttribute("disabled"),
+      ).toBe(false),
+    );
+    fireEvent.change(view.getByLabelText("Action"), {
+      target: { value: "schedule" },
+    });
+    fireEvent.change(view.getByLabelText("Reason"), {
+      target: { value: "Upgrade" },
+    });
+    fireEvent.change(view.getByLabelText("First start (UTC)"), {
+      target: { value: "2027-01-01T12:00" },
+    });
+    fireEvent.change(view.getByLabelText("Repeat"), {
+      target: { value: "monthly" },
+    });
+    fireEvent.click(view.getByRole("button", { name: "Add schedule" }));
+    await waitFor(() =>
+      expect(api.post).toHaveBeenCalledWith(
+        "/maintenance/7",
+        expect.objectContaining({
+          action: "schedule",
+          start: "2027-01-01T12:00:00.000Z",
+          recurrence: "monthly",
+          graceMinutes: 20,
+        }),
+      ),
+    );
+  });
+  it("disables edits when status cannot be loaded", async () => {
+    const api = apiFor(active);
+    api.get.mockRejectedValue(new Error("offline"));
+    const view = await render(api);
+    await waitFor(() =>
+      expect(view.getByRole("alert").textContent).toContain(
+        "Could not refresh",
+      ),
+    );
+    expect(
+      view.getByRole("button", { name: "Start now" }).hasAttribute("disabled"),
+    ).toBe(true);
+  });
+  it("does not let a stale refresh overwrite an acknowledged edit", async () => {
+    let resolve!: (value: unknown) => void;
+    const api = {
+      get: vi.fn(
+        () =>
+          new Promise((done) => {
+            resolve = done;
+          }),
+      ),
+      post: vi.fn(async () => ({ data: active })),
+    };
+    const store = createMaintenanceStore(api as unknown as PluginApiClient);
+    const request = store.refresh();
+    await store.edit(7, { action: "start" });
+    resolve({ data: [] });
+    await request;
+    expect(store.snapshot()[7]).toEqual(active);
+    store.dispose();
+  });
+});