feat: dev:all runner for web, electron and docker with plugin hot reload, termix-plugin dev

This commit is contained in:
LukeGus committed 2026-10-06 15:55:00 -05:00
1 parent 49fc241e06
commit ec80a8ef8d
24 files changed
+1848 -44

No files matched your search

+16 -12
View File
@@ -7,9 +7,9 @@
* pinned sha256 and unpacked.
*
* For development, `--local <dir>` (or TERMIX_LOCAL_PLUGINS) points at a
* folder of plugin repos like ../Termix-Plugins. A bundled plugin there whose
* sources changed since its last build is rebuilt first, then any built one
* (has dist/) is copied in place of its pin, so local plugin changes show up.
* folder of plugin repos like ../Termix-Plugins. A plugin there whose sources
* changed since its last build is rebuilt first, then any built one (has
* dist/) is copied in place of its pin, or added when it has no pin.
*/
const { execSync } = require("node:child_process");
@@ -33,14 +33,10 @@ async function main() {
const localArg =
flag !== -1 ? process.argv[flag + 1] : process.env.TERMIX_LOCAL_PLUGINS;
const localDir = localArg ? path.resolve(root, localArg) : null;
rebuildLocalPlugins(
localDir,
new Set(plugins.map((p) => p.id)),
(repo, id) => {
console.log(`building ${id}`);
execSync("npm run build", { cwd: repo, stdio: "inherit" });
},
);
rebuildLocalPlugins(localDir, null, (repo, id) => {
console.log(`building ${id}`);
execSync("npm run build", { cwd: repo, stdio: "inherit" });
});
const local = findLocalPluginBuilds(localDir);
fs.rmSync(destination, { recursive: true, force: true });
@@ -58,8 +54,16 @@ async function main() {
console.log(`unpacked ${plugin.id}`);
}
// A local plugin with no pin yet still gets staged, so new plugins work too.
const pinned = new Set(plugins.map((p) => p.id));
const extra = [...local.keys()].filter((id) => !pinned.has(id));
for (const id of extra) {
copyLocalBuild(local.get(id), path.join(destination, id));
console.log(`copied local build of ${id} (not pinned)`);
}
console.log(
`Bundled ${plugins.length} plugin(s): ${plugins.map((p) => p.id).join(", ")}`,
`Bundled ${plugins.length + extra.length} plugin(s): ${[...pinned, ...extra].join(", ")}`,
);
}
+9
View File
@@ -280,4 +280,13 @@ describe("rebuilding local plugins", () => {
expect(run).toEqual(["a"]);
expect(rebuildLocalPlugins(null, new Set(["a"]), () => {})).toEqual([]);
});
it("builds every stale repo when no ids are given", () => {
const dir = tempDir();
repo(dir, "Plugin-A", "a");
repo(dir, "Plugin-B", "b");
const run: string[] = [];
rebuildLocalPlugins(dir, null, (_repo: string, id: string) => run.push(id));
expect(run.sort()).toEqual(["a", "b"]);
});
});
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import {
SERVICES,
appRunArgs,
serviceEnv,
serviceRunArgs,
} from "./lib/dev-docker.mjs";
import { parseDevArgs } from "./lib/dev-runner.mjs";
describe("serviceRunArgs", () => {
it("runs a service on the dev network with its port, env and volume", () => {
const args = serviceRunArgs(SERVICES.postgres);
expect(args.slice(0, 6)).toEqual([
"run",
"-d",
"--name",
"termix-dev-postgres",
"--network",
"termix-dev",
]);
expect(args).toContain("55432:5432");
expect(args).toContain("POSTGRES_PASSWORD=termix");
expect(args).toContain("termix-dev-postgres:/var/lib/postgresql/data");
expect(args.at(-1)).toBe("postgres:16");
});
it("adds no volume for guacd", () => {
expect(serviceRunArgs(SERVICES.guacd)).not.toContain("-v");
});
});
describe("serviceEnv", () => {
it("is empty for plain sqlite", () => {
expect(serviceEnv(parseDevArgs([]), false)).toEqual({});
});
it("points a local backend at the published ports", () => {
expect(
serviceEnv(parseDevArgs(["--db", "postgres", "--guacd"]), false),
).toEqual({
DATABASE_DIALECT: "postgres",
DATABASE_URL: "postgres://termix:termix@127.0.0.1:55432/termix",
GUACD_HOST: "127.0.0.1",
GUACD_PORT: "4822",
});
});
it("points a container at the other containers by name", () => {
expect(
serviceEnv(parseDevArgs(["--db", "mysql", "--guacd"]), true),
).toEqual({
DATABASE_DIALECT: "mysql",
DATABASE_URL: "mysql://root:termix@termix-dev-mysql:3306/termix",
GUACD_HOST: "termix-dev-guacd",
GUACD_PORT: "4822",
});
});
});
describe("appRunArgs", () => {
it("mounts the data and the staged plugins", () => {
const args = appRunArgs({
port: 9000,
env: { GUACD_HOST: "termix-dev-guacd" },
dataMount: "termix-dev-data",
pluginsDir: "/repo/dist/plugins",
});
expect(args).toContain("9000:8080");
expect(args).toContain("GUACD_HOST=termix-dev-guacd");
expect(args).toContain("termix-dev-data:/app/data");
expect(args).toContain("/repo/dist/plugins:/app/dist/plugins");
expect(args.at(-1)).toBe("termix:dev");
});
});
+181
View File
@@ -0,0 +1,181 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
createDebouncer,
createSerialQueue,
dataDirFor,
findBuildablePlugins,
isPluginSourceChange,
needsDocker,
parseDevArgs,
parseTscStatus,
} from "./lib/dev-runner.mjs";
const cleanups: Array<() => void> = [];
afterEach(() => {
vi.useRealTimers();
while (cleanups.length) cleanups.pop()!();
});
describe("parseTscStatus", () => {
it("reads the error count from a watch summary", () => {
expect(
parseTscStatus(
"12:00:00 PM - Found 0 errors. Watching for file changes.",
),
).toBe(0);
expect(parseTscStatus("Found 1 error. Watching for file changes.")).toBe(1);
expect(parseTscStatus("src/a.ts(1,1): error TS2304: x")).toBeNull();
});
});
describe("isPluginSourceChange", () => {
it("accepts sources and skips build output", () => {
expect(
isPluginSourceChange(["src", "backend", "index.ts"].join("\\")),
).toBe(true);
expect(isPluginSourceChange("locales/en.json")).toBe(true);
expect(isPluginSourceChange("manifest.json")).toBe(true);
expect(isPluginSourceChange("dist/backend.js")).toBe(false);
expect(isPluginSourceChange("node_modules/x/index.js")).toBe(false);
expect(isPluginSourceChange("notes.txt")).toBe(false);
expect(isPluginSourceChange(null)).toBe(false);
});
});
describe("findBuildablePlugins", () => {
it("lists repos with a manifest and installed dependencies", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "termix-dev-"));
cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true }));
const make = (name: string, id: string, deps = true) => {
const repo = path.join(dir, name);
fs.mkdirSync(repo);
fs.writeFileSync(
path.join(repo, "manifest.json"),
JSON.stringify({ id }),
);
if (deps) fs.mkdirSync(path.join(repo, "node_modules"));
};
make("Plugin-B", "b");
make("Plugin-A", "a");
make("Plugin-C", "c", false);
fs.mkdirSync(path.join(dir, "notes"));
expect(findBuildablePlugins(dir).map((p) => p.id)).toEqual(["a", "b"]);
expect(findBuildablePlugins(null)).toEqual([]);
});
});
describe("createSerialQueue", () => {
it("runs one task at a time and survives a failure", async () => {
const queue = createSerialQueue();
const order: string[] = [];
const first = queue(async () => {
await new Promise((r) => setTimeout(r, 10));
order.push("first");
throw new Error("boom");
});
const second = queue(async () => order.push("second"));
await expect(first).rejects.toThrow("boom");
await second;
expect(order).toEqual(["first", "second"]);
});
});
describe("createDebouncer", () => {
it("fires once per quiet key and again for a change mid-run", async () => {
vi.useFakeTimers();
let release!: () => void;
const calls: string[] = [];
const trigger = createDebouncer(100, async (key: string) => {
calls.push(key);
if (calls.length === 1) await new Promise<void>((r) => (release = r));
});
trigger("a");
trigger("a");
await vi.advanceTimersByTimeAsync(100);
expect(calls).toEqual(["a"]);
trigger("a");
await vi.advanceTimersByTimeAsync(100);
expect(calls).toEqual(["a"]);
release();
await vi.advanceTimersByTimeAsync(100);
expect(calls).toEqual(["a", "a"]);
trigger.cancel();
});
});
describe("parseDevArgs", () => {
it("defaults to the web target with sqlite and local plugins", () => {
expect(parseDevArgs([])).toMatchObject({
target: "web",
db: "sqlite",
local: true,
guacd: false,
fresh: false,
port: 8081,
});
});
it("reads a target and options in any order", () => {
expect(
parseDevArgs(["--db", "Postgres", "electron", "--guacd", "--fresh"]),
).toMatchObject({
target: "electron",
db: "postgres",
guacd: true,
fresh: true,
});
expect(
parseDevArgs(["docker", "--port", "9000", "--no-cache", "--no-local"]),
).toMatchObject({
target: "docker",
port: 9000,
noCache: true,
local: false,
});
expect(parseDevArgs(["--plugins", "../x"]).pluginsDir).toBe("../x");
});
it("refuses unknown or incomplete options", () => {
expect(() => parseDevArgs(["--bogus"])).toThrow(/Unknown option/);
expect(() => parseDevArgs(["--db", "oracle"])).toThrow(/--db/);
expect(() => parseDevArgs(["--db"])).toThrow(/needs a value/);
expect(() => parseDevArgs(["--port", "abc"])).toThrow(/port/);
expect(() => parseDevArgs(["--fresh", "--data", "x"])).toThrow(/together/);
});
});
describe("dataDirFor", () => {
it("keeps each engine and fresh runs in their own folder", () => {
expect(dataDirFor(parseDevArgs([]))).toBeNull();
expect(dataDirFor(parseDevArgs(["--db", "mysql"]))).toBe("db/data-mysql");
expect(dataDirFor(parseDevArgs(["--fresh"]))).toBe("db/data-fresh");
expect(dataDirFor(parseDevArgs(["--fresh", "--db", "postgres"]))).toBe(
"db/data-fresh-postgres",
);
expect(dataDirFor(parseDevArgs(["--data", "x"]))).toBe("x");
expect(dataDirFor(parseDevArgs(["electron"]))).toBe("db/data-desktop");
expect(dataDirFor(parseDevArgs(["electron", "--db", "mysql"]))).toBe(
"db/data-desktop-mysql",
);
expect(dataDirFor(parseDevArgs(["electron", "--fresh"]))).toBe(
"db/data-fresh-desktop",
);
});
});
describe("needsDocker", () => {
it("is only true when something runs in a container", () => {
expect(needsDocker(parseDevArgs([]))).toBe(false);
expect(needsDocker(parseDevArgs(["electron"]))).toBe(false);
expect(needsDocker(parseDevArgs(["docker"]))).toBe(true);
expect(needsDocker(parseDevArgs(["--db", "postgres"]))).toBe(true);
expect(needsDocker(parseDevArgs(["--guacd"]))).toBe(true);
});
});
+716
View File
@@ -0,0 +1,716 @@
/**
* dev:all, one command for manual testing. Run with --help for the targets
* and options.
*
* Every target builds the SDK and stages plugins (local builds from
* ../Termix-Plugins when it exists), then:
*
* - web: compiles the backend in watch mode, starts it, then starts Vite. A
* backend or SDK change restarts the backend. A plugin change rebuilds that
* plugin and reloads it inside the running backend, and the browser swaps
* it in.
* - electron: web, plus the desktop app, restarted when electron/ changes.
* - docker: builds the production image and runs it with dist/plugins
* mounted. A plugin change rebuilds it and restarts the container. Core
* changes wait for "r", since an image build is slow.
*
* --db postgres|mysql and --guacd run those in Docker next to it.
*/
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import { spawn, spawnSync, fork } from "node:child_process";
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";
import {
HELP,
createDebouncer,
createSerialQueue,
dataDirFor,
findBuildablePlugins,
isPluginSourceChange,
needsDocker,
parseDevArgs,
parseTscStatus,
} from "./lib/dev-runner.mjs";
import {
APP_CONTAINER,
APP_IMAGE,
NETWORK,
SERVICES,
appRunArgs,
serviceEnv,
serviceRunArgs,
} from "./lib/dev-docker.mjs";
const require = createRequire(import.meta.url);
const { copyLocalBuild } = require("./lib/bundled-plugins.cjs");
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const BACKEND_ENTRY = path.join(
root,
"dist",
"backend",
"backend",
"starter.js",
);
const STAGED_PLUGINS = path.join(root, "dist", "plugins");
const SDK_SRC = path.join(root, "packages", "plugin-sdk", "src");
const ELECTRON_DIR = path.join(root, "electron");
const RELOAD_TIMEOUT_MS = 60_000;
const SHUTDOWN_TIMEOUT_MS = 10_000;
const DOCKER_READY_TIMEOUT_MS = 180_000;
const log = (message) => console.log(`\x1b[36m[dev]\x1b[0m ${message}`);
const warn = (message) => console.log(`\x1b[33m[dev]\x1b[0m ${message}`);
let options;
const queue = createSerialQueue();
const startedServices = [];
const watchers = [];
let shuttingDown = false;
let plugins = [];
let vite = null;
let backend = null;
let electron = null;
let tsc = null;
let dockerLogs = null;
let backendEnv = {};
function run(command, args, cwd = root) {
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
cwd,
stdio: ["ignore", "inherit", "inherit"],
shell: process.platform === "win32",
});
child.on("error", reject);
child.on("exit", (code) =>
code === 0
? resolve()
: reject(new Error(`${command} ${args.join(" ")} exited with ${code}`)),
);
});
}
function docker(args, { quiet = false } = {}) {
const result = spawnSync("docker", args, {
cwd: root,
encoding: "utf8",
stdio: quiet ? "pipe" : ["ignore", "inherit", "inherit"],
});
return { ok: result.status === 0, out: (result.stdout ?? "").trim() };
}
function watch(target, recursive, onChange) {
watchers.push(fs.watch(target, { recursive }, onChange));
}
function appUrl() {
if (options.target === "docker") return `http://localhost:${options.port}`;
return vite?.resolvedUrls?.local?.[0] ?? "http://localhost:5173";
}
function openBrowser(url) {
const [command, args] =
process.platform === "win32"
? ["cmd", ["/c", "start", "", url]]
: process.platform === "darwin"
? ["open", [url]]
: ["xdg-open", [url]];
spawn(command, args, { stdio: "ignore", detached: true }).unref();
}
// Docker side containers
function assertDocker() {
if (!docker(["info"], { quiet: true }).ok) {
throw new Error(
"Docker is not running. Start it, or drop --db, --guacd and the docker target.",
);
}
if (!docker(["network", "inspect", NETWORK], { quiet: true }).ok) {
docker(["network", "create", NETWORK], { quiet: true });
}
}
function containerState(name) {
const result = docker(["inspect", "-f", "{{.State.Running}}", name], {
quiet: true,
});
if (!result.ok) return "missing";
return result.out === "true" ? "running" : "stopped";
}
async function waitForService(service) {
if (!service.ready) return;
const deadline = Date.now() + 120_000;
while (Date.now() < deadline) {
if (
docker(["exec", service.container, ...service.ready], { quiet: true }).ok
)
return;
await new Promise((resolve) => setTimeout(resolve, 1000));
}
throw new Error(`${service.container} did not become ready`);
}
async function startService(name) {
const service = SERVICES[name];
if (options.fresh && service.volume) {
docker(["rm", "-f", service.container], { quiet: true });
docker(["volume", "rm", service.volume[0]], { quiet: true });
}
const state = containerState(service.container);
if (state === "running") {
log(`${service.container} is already running`);
} else {
log(`starting ${service.container}`);
const result =
state === "stopped"
? docker(["start", service.container], { quiet: true })
: docker(serviceRunArgs(service), { quiet: true });
if (!result.ok) throw new Error(`could not start ${service.container}`);
startedServices.push(service.container);
}
await waitForService(service);
}
async function startServices() {
if (options.db !== "sqlite") await startService(options.db);
if (options.guacd) await startService("guacd");
}
// Local backend, Vite and Electron
function notifyBrowser(data) {
vite?.ws.send({ type: "custom", event: "termix:plugins-reloaded", data });
}
function startBackend() {
const child = fork(BACKEND_ENTRY, [], {
cwd: root,
env: { ...process.env, ...backendEnv, TERMIX_DEV_RELOAD: "true" },
stdio: ["ignore", "inherit", "inherit", "ipc"],
});
const pending = new Map();
const state = { child, pending, ready: false, stopping: false };
state.readyPromise = new Promise((resolve, reject) => {
child.on("message", (msg) => {
if (msg?.type === "backend-ready") {
state.ready = true;
resolve();
return;
}
if (msg?.type === "backend-request") {
relayToElectron(child, msg);
return;
}
if (
msg?.type === "plugin-reloaded" ||
msg?.type === "plugin-reload-failed"
) {
pending.get(msg.id)?.(msg);
pending.delete(msg.id);
}
});
child.on("exit", (code) => {
for (const done of pending.values()) {
done({ type: "plugin-reload-failed", error: "backend exited" });
}
pending.clear();
if (backend === state) backend = null;
if (!state.ready) reject(new Error(`backend exited with ${code}`));
else if (!shuttingDown && !state.stopping) {
warn(
`backend exited with ${code}, press r or save a file to restart it`,
);
}
});
});
backend = state;
return state;
}
function stopChild(child, message) {
return new Promise((resolve) => {
if (!child || child.exitCode !== null || child.signalCode !== null) {
return resolve();
}
const timer = setTimeout(() => child.kill("SIGKILL"), SHUTDOWN_TIMEOUT_MS);
child.once("exit", () => {
clearTimeout(timer);
resolve();
});
try {
if (message && child.connected) child.send(message);
else child.kill();
} catch {
child.kill();
}
});
}
function stopBackend() {
const state = backend;
if (!state) return Promise.resolve();
backend = null;
state.stopping = true;
return stopChild(state.child, state.ready ? { type: "shutdown" } : null);
}
async function restartBackend(reason) {
log(reason);
await stopBackend();
if (shuttingDown) return;
fs.copyFileSync(
path.join(root, "src", "backend", "package.json"),
path.join(root, "dist", "backend", "package.json"),
);
const state = startBackend();
try {
await state.readyPromise;
} catch (error) {
warn(`${error.message}, press r or save a file to restart it`);
return;
}
log("backend ready");
if (!vite) {
await startVite();
if (options.target === "electron") startElectron();
printKeys();
} else {
notifyBrowser({});
}
}
async function startVite() {
const { createServer } = await import("vite");
vite = await createServer({
root,
configFile: path.join(root, "vite.config.ts"),
});
await vite.listen();
vite.printUrls();
}
// The packaged app forks the backend from Electron's main process, and they
// talk over that IPC channel (opening sign-in windows, the linked server's
// proxy settings). Here both are children of the runner, so it passes those
// messages along. The last proxy settings are kept for an app (re)started
// after the backend sent them.
let lastProxyConfig = null;
const REPLAY_PREFIX = "dev-replay-";
function relayToElectron(backendChild, msg) {
if (msg.channel === "sync-proxy-config") lastProxyConfig = msg;
if (electron?.connected) {
electron.send(msg);
return;
}
const ok = msg.channel === "sync-proxy-config";
try {
backendChild.send({
type: "backend-response",
id: msg.id,
ok,
...(ok
? { result: { success: true } }
: { error: "The desktop app is not running" }),
});
} catch {
// backend gone
}
}
function relayToBackend(msg) {
if (msg?.type !== "backend-response") return;
if (String(msg.id).startsWith(REPLAY_PREFIX)) return;
if (backend?.child.connected) backend.child.send(msg);
}
function startElectron() {
const env = {
...process.env,
NODE_ENV: "development",
TERMIX_DEV_RUNNER: "true",
};
// Set by some editors' terminals; Electron would start as plain Node.
delete env.ELECTRON_RUN_AS_NODE;
const child = spawn(require("electron"), ["."], {
cwd: root,
env,
stdio: ["ignore", "inherit", "inherit", "ipc"],
});
electron = child;
child.on("message", relayToBackend);
if (lastProxyConfig) {
child.send({ ...lastProxyConfig, id: `${REPLAY_PREFIX}${Date.now()}` });
}
log("desktop app started");
child.on("exit", () => {
if (electron !== child) return;
electron = null;
if (!shuttingDown) {
log("desktop app closed");
void shutdown();
}
});
}
async function restartElectron() {
const child = electron;
electron = null;
log("electron changed, restarting the desktop app");
await stopChild(child);
if (!shuttingDown) startElectron();
}
function reloadInBackend(id) {
const state = backend;
if (!state?.ready || !state.child.connected) return Promise.resolve(null);
return new Promise((resolve) => {
const timer = setTimeout(() => {
state.pending.delete(id);
resolve({ type: "plugin-reload-failed", error: "timed out" });
}, RELOAD_TIMEOUT_MS);
state.pending.set(id, (msg) => {
clearTimeout(timer);
resolve(msg);
});
state.child.send({ type: "plugin-reload", id });
});
}
function watchBackend() {
tsc = spawn(
process.execPath,
[
require.resolve("typescript/bin/tsc"),
"-p",
"tsconfig.node.json",
"--watch",
"--preserveWatchOutput",
"--pretty",
"false",
],
{ cwd: root, stdio: ["ignore", "pipe", "inherit"] },
);
let first = true;
let buffer = "";
tsc.stdout.on("data", (chunk) => {
buffer += chunk.toString();
const lines = buffer.split(/\r?\n/);
buffer = lines.pop();
for (const line of lines) {
const errors = parseTscStatus(line);
if (errors === null) {
if (line.includes("error TS")) console.log(line);
continue;
}
if (errors > 0) {
warn(`backend has ${errors} type error(s), fix them to restart it`);
continue;
}
const reason = first
? "starting the backend"
: "backend changed, restarting it";
first = false;
void queue(() => restartBackend(reason));
}
});
}
function watchSdk() {
const changed = createDebouncer(300, () =>
queue(async () => {
try {
await run("npm", ["run", "build:sdk"]);
} catch (error) {
warn(`SDK failed to build: ${error.message}`);
return;
}
await restartBackend("SDK changed, restarting the backend");
}),
);
watch(SDK_SRC, true, () => changed("sdk"));
}
function watchElectron() {
const changed = createDebouncer(300, () => restartElectron());
watch(ELECTRON_DIR, false, (_event, filename) => {
if (/\.(c?js)$/.test(String(filename)) && electron) changed("electron");
});
}
// Docker target
function dockerDataMount() {
if (options.data) return path.resolve(root, options.data);
const volume = options.fresh ? "termix-dev-data-fresh" : "termix-dev-data";
if (options.fresh) docker(["volume", "rm", "-f", volume], { quiet: true });
return volume;
}
async function buildImage() {
log(`building ${APP_IMAGE}, this takes a while`);
await run("docker", [
"build",
"-f",
"docker/Dockerfile",
"-t",
APP_IMAGE,
...(options.noCache ? ["--no-cache"] : []),
".",
]);
}
function followDockerLogs(since) {
dockerLogs?.kill();
dockerLogs = spawn(
"docker",
["logs", "-f", ...(since ? ["--since", since] : []), APP_CONTAINER],
{ cwd: root, stdio: ["ignore", "inherit", "inherit"] },
);
}
async function waitForApp() {
const deadline = Date.now() + DOCKER_READY_TIMEOUT_MS;
while (Date.now() < deadline && !shuttingDown) {
try {
const response = await fetch(appUrl());
if (response.ok) return true;
} catch {
// not up yet
}
await new Promise((resolve) => setTimeout(resolve, 1000));
}
return false;
}
let dockerMount = null;
async function runAppContainer() {
docker(["rm", "-f", APP_CONTAINER], { quiet: true });
dockerMount ??= dockerDataMount();
const result = docker(
appRunArgs({
port: options.port,
env: serviceEnv(options, true),
dataMount: dockerMount,
pluginsDir: STAGED_PLUGINS,
}),
{ quiet: true },
);
if (!result.ok) throw new Error(`could not start ${APP_CONTAINER}`);
followDockerLogs();
if (await waitForApp()) log(`ready on ${appUrl()}`);
else warn(`${APP_CONTAINER} is not answering on ${appUrl()} yet`);
}
async function restartAppContainer(reason) {
log(reason);
const since = new Date().toISOString();
docker(["restart", APP_CONTAINER], { quiet: true });
followDockerLogs(since);
if (await waitForApp()) log(`ready on ${appUrl()}`);
}
async function rebuildAndRunImage() {
try {
await buildImage();
} catch (error) {
warn(error.message);
return;
}
await runAppContainer();
}
function watchCoreForDocker() {
const changed = createDebouncer(500, () =>
warn("core changed, press r to rebuild the image"),
);
for (const dir of [path.join(root, "src"), SDK_SRC]) {
watch(dir, true, () => changed("core"));
}
}
// Plugins
async function rebuildPlugin({ id, repo }) {
log(`building ${id}`);
try {
await run("npm", ["run", "build"], repo);
} catch (error) {
warn(`${id} failed to build: ${error.message}`);
return;
}
const destination = path.join(STAGED_PLUGINS, id);
fs.rmSync(destination, { recursive: true, force: true });
copyLocalBuild(repo, destination);
if (options.target === "docker") {
await restartAppContainer(`${id} rebuilt, restarting ${APP_CONTAINER}`);
return;
}
const result = await reloadInBackend(id);
if (!result) {
log(`${id} staged, it loads when the backend starts`);
return;
}
if (result.type === "plugin-reload-failed") {
warn(`${id} did not reload: ${result.error}`);
return;
}
log(`${id} reloaded (${result.state})`);
notifyBrowser({ id });
}
function watchPlugins(dir) {
plugins = findBuildablePlugins(dir);
const byId = new Map(plugins.map((plugin) => [plugin.id, plugin]));
const changed = createDebouncer(300, (id) =>
queue(() => rebuildPlugin(byId.get(id))),
);
for (const plugin of plugins) {
watch(plugin.repo, true, (_event, filename) => {
if (isPluginSourceChange(filename)) changed(plugin.id);
});
}
log(`watching ${plugins.length} plugin(s) in ${dir}`);
}
// Keys and shutdown
function printKeys() {
if (!process.stdin.isTTY) return;
log("keys: r restart, p rebuild plugins, o open browser, q quit");
}
function listenForKeys() {
if (!process.stdin.isTTY) return;
process.stdin.setRawMode(true);
process.stdin.setEncoding("utf8");
process.stdin.resume();
process.stdin.on("data", (key) => {
if (key === "\u0003" || key === "q") {
void shutdown();
} else if (key === "r") {
void queue(() =>
options.target === "docker"
? rebuildAndRunImage()
: restartBackend("restarting the backend"),
);
} else if (key === "p") {
if (plugins.length === 0) warn("no local plugins to rebuild");
for (const plugin of plugins) void queue(() => rebuildPlugin(plugin));
} else if (key === "o") {
openBrowser(appUrl());
}
});
}
async function shutdown(code = 0) {
if (shuttingDown) return;
shuttingDown = true;
log("shutting down");
if (process.stdin.isTTY) process.stdin.setRawMode(false);
for (const watcher of watchers) watcher.close();
tsc?.kill();
dockerLogs?.kill();
await Promise.allSettled([stopBackend(), stopChild(electron), vite?.close()]);
if (options.target === "docker") {
docker(["rm", "-f", APP_CONTAINER], { quiet: true });
}
for (const container of startedServices) {
docker(["stop", container], { quiet: true });
}
process.exit(code);
}
async function main() {
try {
options = parseDevArgs(process.argv.slice(2));
} catch (error) {
console.error(error.message);
process.exit(1);
}
if (options.help) {
console.log(HELP);
return;
}
process.on("SIGINT", () => shutdown());
process.on("SIGTERM", () => shutdown());
const pluginsDir = options.local
? path.resolve(
root,
options.pluginsDir ??
process.env.TERMIX_LOCAL_PLUGINS ??
"../Termix-Plugins",
)
: null;
const localPlugins =
pluginsDir && fs.existsSync(pluginsDir) ? pluginsDir : null;
if (pluginsDir && !localPlugins)
warn(`${pluginsDir} not found, using pinned plugins`);
if (needsDocker(options)) {
assertDocker();
await startServices();
}
log("building the SDK");
await run("npm", ["run", "build:sdk"]);
log(
localPlugins
? `staging plugins with local builds from ${localPlugins}`
: "staging pinned plugins",
);
await run(process.execPath, [
"scripts/build-plugins.cjs",
...(localPlugins ? ["--local", localPlugins] : []),
]);
listenForKeys();
if (localPlugins) watchPlugins(localPlugins);
if (options.target === "docker") {
await buildImage();
await runAppContainer();
watchCoreForDocker();
printKeys();
return;
}
const dataDir = dataDirFor(options);
if (dataDir) {
const full = path.resolve(root, dataDir);
if (options.fresh) fs.rmSync(full, { recursive: true, force: true });
fs.mkdirSync(full, { recursive: true });
log(`data folder ${full}`);
}
backendEnv = {
...serviceEnv(options, false),
// The desktop app runs against its own embedded backend, which is what
// turns on the local profile, sync and the desktop-only routes.
...(options.target === "electron" ? { ELECTRON_EMBEDDED: "true" } : {}),
// A folder the runner picked starts empty on purpose, so the guard
// against a mistyped DATA_DIR must not stop it.
...(dataDir
? { DATA_DIR: path.resolve(root, dataDir), ALLOW_EMPTY_DATA_DIR: "true" }
: {}),
};
log("compiling the backend");
watchBackend();
watchSdk();
if (options.target === "electron") watchElectron();
}
main().catch(async (error) => {
console.error(error?.message ?? error);
await shutdown(1);
});
+5 -2
View File
@@ -231,7 +231,10 @@ function isLocalBuildStale(repo) {
);
}
/** Builds each listed plugin repo whose build is missing or out of date. */
/**
* Builds each plugin repo whose build is missing or out of date. ids limits
* it to those plugins; null builds every one.
*/
function rebuildLocalPlugins(dir, ids, run) {
const rebuilt = [];
if (!dir || !fs.existsSync(dir)) return rebuilt;
@@ -246,7 +249,7 @@ function rebuildLocalPlugins(dir, ids, run) {
} catch {
continue;
}
if (!ids.has(id) || !isLocalBuildStale(repo)) continue;
if ((ids && !ids.has(id)) || !isLocalBuildStale(repo)) continue;
run(repo, id);
rebuilt.push(id);
}
+109
View File
@@ -0,0 +1,109 @@
export const NETWORK = "termix-dev";
export const APP_CONTAINER = "termix-dev";
export const APP_IMAGE = "termix:dev";
/** Side containers dev:all can run. hostPort is what the local backend uses. */
export const SERVICES = {
postgres: {
container: "termix-dev-postgres",
image: "postgres:16",
hostPort: 55432,
port: 5432,
env: {
POSTGRES_USER: "termix",
POSTGRES_PASSWORD: "termix",
POSTGRES_DB: "termix",
},
volume: ["termix-dev-postgres", "/var/lib/postgresql/data"],
// Over TCP, so the server that only runs during first-time setup does
// not count as ready.
ready: ["pg_isready", "-h", "127.0.0.1", "-U", "termix", "-d", "termix"],
},
mysql: {
container: "termix-dev-mysql",
image: "mysql:8",
hostPort: 33306,
port: 3306,
env: { MYSQL_ROOT_PASSWORD: "termix", MYSQL_DATABASE: "termix" },
volume: ["termix-dev-mysql", "/var/lib/mysql"],
ready: ["mysqladmin", "ping", "-h", "127.0.0.1", "-uroot", "-ptermix"],
},
guacd: {
container: "termix-dev-guacd",
image: "guacamole/guacd:1.6.0",
hostPort: 4822,
port: 4822,
env: {},
volume: null,
ready: null,
},
};
function envArgs(env) {
return Object.entries(env).flatMap(([key, value]) => [
"-e",
`${key}=${value}`,
]);
}
export function serviceRunArgs(service) {
return [
"run",
"-d",
"--name",
service.container,
"--network",
NETWORK,
"-p",
`${service.hostPort}:${service.port}`,
...envArgs(service.env),
...(service.volume
? ["-v", `${service.volume[0]}:${service.volume[1]}`]
: []),
service.image,
];
}
/**
* Environment that points a backend at the side containers. inDocker means
* the backend runs in the dev network and reaches them by name.
*/
export function serviceEnv(options, inDocker) {
const env = {};
const address = (service) =>
inDocker
? `${service.container}:${service.port}`
: `127.0.0.1:${service.hostPort}`;
if (options.db === "postgres") {
env.DATABASE_DIALECT = "postgres";
env.DATABASE_URL = `postgres://termix:termix@${address(SERVICES.postgres)}/termix`;
} else if (options.db === "mysql") {
env.DATABASE_DIALECT = "mysql";
env.DATABASE_URL = `mysql://root:termix@${address(SERVICES.mysql)}/termix`;
}
if (options.guacd) {
const guacd = SERVICES.guacd;
env.GUACD_HOST = inDocker ? guacd.container : "127.0.0.1";
env.GUACD_PORT = String(inDocker ? guacd.port : guacd.hostPort);
}
return env;
}
export function appRunArgs({ port, env, dataMount, pluginsDir }) {
return [
"run",
"-d",
"--name",
APP_CONTAINER,
"--network",
NETWORK,
"-p",
`${port}:8080`,
...envArgs({ PORT: "8080", NODE_ENV: "development", ...env }),
"-v",
`${dataMount}:/app/data`,
"-v",
`${pluginsDir}:/app/dist/plugins`,
APP_IMAGE,
];
}
+185
View File
@@ -0,0 +1,185 @@
import fs from "node:fs";
import path from "node:path";
const ID = /^[a-z0-9][a-z0-9-]*$/;
const SOURCE_DIRS = new Set(["src", "locales", "migrations"]);
const SOURCE_FILES = new Set(["manifest.json", "package.json"]);
/** The error count from tsc --watch's "Found N errors" line, else null. */
export function parseTscStatus(line) {
const match = /Found (\d+) errors?\b.*Watching for file changes/.exec(line);
return match ? Number(match[1]) : null;
}
/** True when a changed path (relative to a plugin repo) feeds its build. */
export function isPluginSourceChange(filename) {
if (!filename) return false;
const parts = String(filename).replaceAll("\\", "/").split("/");
if (parts.length === 1) return SOURCE_FILES.has(parts[0]);
return SOURCE_DIRS.has(parts[0]);
}
/** Plugin repos under dir that can be built (manifest.json and node_modules). */
export function findBuildablePlugins(dir) {
const plugins = [];
if (!dir || !fs.existsSync(dir)) return plugins;
for (const name of fs.readdirSync(dir)) {
const repo = path.join(dir, name);
if (!fs.existsSync(path.join(repo, "node_modules"))) continue;
try {
const { id } = JSON.parse(
fs.readFileSync(path.join(repo, "manifest.json"), "utf8"),
);
if (typeof id === "string" && ID.test(id)) plugins.push({ id, repo });
} catch {
// not a plugin repo
}
}
return plugins.sort((a, b) => a.id.localeCompare(b.id));
}
/** Runs tasks one after another; a failed task does not stop the next. */
export function createSerialQueue() {
let tail = Promise.resolve();
return (task) => {
const run = tail.then(task, task);
tail = run.catch(() => {});
return run;
};
}
/**
* Calls fn(key) once a key has gone quiet for ms. A key that changes again
* while fn is still running for it runs once more afterwards.
*/
export function createDebouncer(ms, fn) {
const timers = new Map();
const running = new Set();
const again = new Set();
const fire = async (key) => {
timers.delete(key);
if (running.has(key)) {
again.add(key);
return;
}
running.add(key);
try {
await fn(key);
} finally {
running.delete(key);
if (again.delete(key)) trigger(key);
}
};
const trigger = (key) => {
clearTimeout(timers.get(key));
timers.set(
key,
setTimeout(() => void fire(key), ms),
);
};
trigger.cancel = () => {
for (const timer of timers.values()) clearTimeout(timer);
timers.clear();
};
return trigger;
}
export const TARGETS = ["web", "electron", "docker"];
export const DATABASES = ["sqlite", "postgres", "mysql"];
export const HELP = `Usage: npm run dev:all -- [target] [options]
Targets
web Backend and Vite on http://localhost:5173 (default)
electron Backend, Vite and the desktop app
docker Build the production image and run it with your local plugins
Options
--plugins <dir> Plugin repos to build and watch (default ../Termix-Plugins)
--no-local Use only the plugins pinned in docker/bundled-plugins.json
--db <engine> sqlite (default), postgres or mysql. Postgres and MySQL run in Docker
--guacd Run guacd in Docker for remote desktop
--fresh Start from empty data (first run, onboarding)
--data <dir> Data folder for the backend (default db/data, db/data-desktop for
electron, plus -<engine> for postgres or mysql)
--port <n> docker target: port on this machine (default 8081)
--no-cache docker target: build the image without the layer cache
-h, --help Show this
Keys while running
r restart the backend (docker: rebuild the image)
p rebuild every local plugin
o open the app in a browser
q quit`;
/** Reads dev:all arguments. Throws on anything it does not know. */
export function parseDevArgs(argv) {
const options = {
target: "web",
pluginsDir: null,
local: true,
db: "sqlite",
guacd: false,
fresh: false,
data: null,
port: 8081,
noCache: false,
help: false,
};
const value = (i, name) => {
const next = argv[i + 1];
if (!next || next.startsWith("--"))
throw new Error(`${name} needs a value`);
return next;
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (TARGETS.includes(arg)) options.target = arg;
else if (arg === "-h" || arg === "--help") options.help = true;
else if (arg === "--no-local") options.local = false;
else if (arg === "--guacd") options.guacd = true;
else if (arg === "--fresh") options.fresh = true;
else if (arg === "--no-cache") options.noCache = true;
else if (arg === "--plugins") options.pluginsDir = value(i++, arg);
else if (arg === "--data") options.data = value(i++, arg);
else if (arg === "--db") {
options.db = value(i++, arg).toLowerCase();
if (!DATABASES.includes(options.db)) {
throw new Error(`--db must be one of ${DATABASES.join(", ")}`);
}
} else if (arg === "--port") {
options.port = Number(value(i++, arg));
if (!Number.isInteger(options.port) || options.port < 1) {
throw new Error("--port must be a port number");
}
} else throw new Error(`Unknown option ${arg}. Run with --help.`);
}
if (options.fresh && options.data) {
throw new Error("--fresh and --data cannot be used together");
}
return options;
}
/**
* The data folder the local backend uses, or null for its own default. The
* desktop, each database engine and fresh runs get their own, so a desktop's
* local profile or another engine's keys never land in the everyday data.
*/
export function dataDirFor(options) {
if (options.data) return options.data;
const suffix = [
options.target === "electron" ? "desktop" : null,
options.db === "sqlite" ? null : options.db,
]
.filter(Boolean)
.map((part) => `-${part}`)
.join("");
if (options.fresh) return `db/data-fresh${suffix}`;
return suffix ? `db/data${suffix}` : null;
}
export function needsDocker(options) {
return (
options.target === "docker" || options.db !== "sqlite" || options.guacd
);
}
+98
View File
@@ -0,0 +1,98 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
isSourceChange,
pushPlugin,
resolveDevOptions,
} from "../packages/plugin-sdk/cli/commands/dev.mjs";
const cleanups: Array<() => void> = [];
afterEach(() => {
while (cleanups.length) cleanups.pop()!();
});
function json(status: number, body: unknown) {
return new Response(JSON.stringify(body), { status });
}
function tempFile(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "termix-plugin-dev-"));
cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true }));
const file = path.join(dir, "demo-1.0.0.tmxplug");
fs.writeFileSync(file, "archive");
return file;
}
describe("termix-plugin dev options", () => {
it("reads flags first, then the environment", () => {
expect(
resolveDevOptions(["--server", "http://a:8080/", "--key", "tmx_1"], {}),
).toEqual({ server: "http://a:8080", key: "tmx_1", once: false });
expect(
resolveDevOptions(["--once"], {
TERMIX_SERVER_URL: "http://b",
TERMIX_API_KEY: "tmx_2",
}),
).toEqual({ server: "http://b", key: "tmx_2", once: true });
expect(resolveDevOptions([], { TERMIX_API_KEY: "tmx_3" }).server).toBe(
"http://localhost:30001",
);
});
it("needs an API key", () => {
expect(() => resolveDevOptions([], {})).toThrow(/API key/);
});
it("watches sources only", () => {
expect(isSourceChange("src/frontend/index.tsx")).toBe(true);
expect(isSourceChange("README.md")).toBe(true);
expect(isSourceChange("dist/frontend.js")).toBe(false);
});
});
describe("pushPlugin", () => {
it("uploads the file then installs it with the capabilities it asks for", async () => {
const fetchImpl = vi
.fn()
.mockResolvedValueOnce(
json(200, { token: "t".repeat(32), capabilities: ["kv:own"] }),
)
.mockResolvedValueOnce(
json(200, { id: "demo", version: "1.0.0", state: "active" }),
);
const { result } = await pushPlugin(
tempFile(),
{ server: "http://s", key: "tmx_k" },
fetchImpl,
);
expect(result.state).toBe("active");
const [uploadUrl, upload] = fetchImpl.mock.calls[0];
expect(uploadUrl).toBe("http://s/plugins/upload");
expect(upload.headers.Authorization).toBe("Bearer tmx_k");
expect(upload.headers["Content-Type"]).toBe("application/octet-stream");
const [installUrl, install] = fetchImpl.mock.calls[1];
expect(installUrl).toBe(
`http://s/plugins/upload/${"t".repeat(32)}/install`,
);
expect(JSON.parse(install.body)).toEqual({ capabilities: ["kv:own"] });
});
it("explains a server with developer mode off", async () => {
const fetchImpl = vi.fn().mockResolvedValue(
json(403, {
error: "Installing from a file needs developer mode",
code: "DEVELOPER_MODE_OFF",
}),
);
await expect(
pushPlugin(tempFile(), { server: "http://s", key: "tmx_k" }, fetchImpl),
).rejects.toThrow(
/403 Installing from a file needs developer mode\n.*developer mode/,
);
});
});