feat: dev:all runner for web, electron and docker with plugin hot reload, termix-plugin dev

scripts/dev.mjs

@@ -0,0 +1,716 @@
+/**
+ * dev:all, one command for manual testing. Run with --help for the targets
+ * and options.
+ *
+ * Every target builds the SDK and stages plugins (local builds from
+ * ../Termix-Plugins when it exists), then:
+ *
+ * - web: compiles the backend in watch mode, starts it, then starts Vite. A
+ *   backend or SDK change restarts the backend. A plugin change rebuilds that
+ *   plugin and reloads it inside the running backend, and the browser swaps
+ *   it in.
+ * - electron: web, plus the desktop app, restarted when electron/ changes.
+ * - docker: builds the production image and runs it with dist/plugins
+ *   mounted. A plugin change rebuilds it and restarts the container. Core
+ *   changes wait for "r", since an image build is slow.
+ *
+ * --db postgres|mysql and --guacd run those in Docker next to it.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { spawn, spawnSync, fork } from "node:child_process";
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+import {
+  HELP,
+  createDebouncer,
+  createSerialQueue,
+  dataDirFor,
+  findBuildablePlugins,
+  isPluginSourceChange,
+  needsDocker,
+  parseDevArgs,
+  parseTscStatus,
+} from "./lib/dev-runner.mjs";
+import {
+  APP_CONTAINER,
+  APP_IMAGE,
+  NETWORK,
+  SERVICES,
+  appRunArgs,
+  serviceEnv,
+  serviceRunArgs,
+} from "./lib/dev-docker.mjs";
+
+const require = createRequire(import.meta.url);
+const { copyLocalBuild } = require("./lib/bundled-plugins.cjs");
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const BACKEND_ENTRY = path.join(
+  root,
+  "dist",
+  "backend",
+  "backend",
+  "starter.js",
+);
+const STAGED_PLUGINS = path.join(root, "dist", "plugins");
+const SDK_SRC = path.join(root, "packages", "plugin-sdk", "src");
+const ELECTRON_DIR = path.join(root, "electron");
+const RELOAD_TIMEOUT_MS = 60_000;
+const SHUTDOWN_TIMEOUT_MS = 10_000;
+const DOCKER_READY_TIMEOUT_MS = 180_000;
+
+const log = (message) => console.log(`\x1b[36m[dev]\x1b[0m ${message}`);
+const warn = (message) => console.log(`\x1b[33m[dev]\x1b[0m ${message}`);
+
+let options;
+const queue = createSerialQueue();
+const startedServices = [];
+const watchers = [];
+let shuttingDown = false;
+let plugins = [];
+let vite = null;
+let backend = null;
+let electron = null;
+let tsc = null;
+let dockerLogs = null;
+let backendEnv = {};
+
+function run(command, args, cwd = root) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd,
+      stdio: ["ignore", "inherit", "inherit"],
+      shell: process.platform === "win32",
+    });
+    child.on("error", reject);
+    child.on("exit", (code) =>
+      code === 0
+        ? resolve()
+        : reject(new Error(`${command} ${args.join(" ")} exited with ${code}`)),
+    );
+  });
+}
+
+function docker(args, { quiet = false } = {}) {
+  const result = spawnSync("docker", args, {
+    cwd: root,
+    encoding: "utf8",
+    stdio: quiet ? "pipe" : ["ignore", "inherit", "inherit"],
+  });
+  return { ok: result.status === 0, out: (result.stdout ?? "").trim() };
+}
+
+function watch(target, recursive, onChange) {
+  watchers.push(fs.watch(target, { recursive }, onChange));
+}
+
+function appUrl() {
+  if (options.target === "docker") return `http://localhost:${options.port}`;
+  return vite?.resolvedUrls?.local?.[0] ?? "http://localhost:5173";
+}
+
+function openBrowser(url) {
+  const [command, args] =
+    process.platform === "win32"
+      ? ["cmd", ["/c", "start", "", url]]
+      : process.platform === "darwin"
+        ? ["open", [url]]
+        : ["xdg-open", [url]];
+  spawn(command, args, { stdio: "ignore", detached: true }).unref();
+}
+
+// Docker side containers
+
+function assertDocker() {
+  if (!docker(["info"], { quiet: true }).ok) {
+    throw new Error(
+      "Docker is not running. Start it, or drop --db, --guacd and the docker target.",
+    );
+  }
+  if (!docker(["network", "inspect", NETWORK], { quiet: true }).ok) {
+    docker(["network", "create", NETWORK], { quiet: true });
+  }
+}
+
+function containerState(name) {
+  const result = docker(["inspect", "-f", "{{.State.Running}}", name], {
+    quiet: true,
+  });
+  if (!result.ok) return "missing";
+  return result.out === "true" ? "running" : "stopped";
+}
+
+async function waitForService(service) {
+  if (!service.ready) return;
+  const deadline = Date.now() + 120_000;
+  while (Date.now() < deadline) {
+    if (
+      docker(["exec", service.container, ...service.ready], { quiet: true }).ok
+    )
+      return;
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+  throw new Error(`${service.container} did not become ready`);
+}
+
+async function startService(name) {
+  const service = SERVICES[name];
+  if (options.fresh && service.volume) {
+    docker(["rm", "-f", service.container], { quiet: true });
+    docker(["volume", "rm", service.volume[0]], { quiet: true });
+  }
+  const state = containerState(service.container);
+  if (state === "running") {
+    log(`${service.container} is already running`);
+  } else {
+    log(`starting ${service.container}`);
+    const result =
+      state === "stopped"
+        ? docker(["start", service.container], { quiet: true })
+        : docker(serviceRunArgs(service), { quiet: true });
+    if (!result.ok) throw new Error(`could not start ${service.container}`);
+    startedServices.push(service.container);
+  }
+  await waitForService(service);
+}
+
+async function startServices() {
+  if (options.db !== "sqlite") await startService(options.db);
+  if (options.guacd) await startService("guacd");
+}
+
+// Local backend, Vite and Electron
+
+function notifyBrowser(data) {
+  vite?.ws.send({ type: "custom", event: "termix:plugins-reloaded", data });
+}
+
+function startBackend() {
+  const child = fork(BACKEND_ENTRY, [], {
+    cwd: root,
+    env: { ...process.env, ...backendEnv, TERMIX_DEV_RELOAD: "true" },
+    stdio: ["ignore", "inherit", "inherit", "ipc"],
+  });
+  const pending = new Map();
+  const state = { child, pending, ready: false, stopping: false };
+  state.readyPromise = new Promise((resolve, reject) => {
+    child.on("message", (msg) => {
+      if (msg?.type === "backend-ready") {
+        state.ready = true;
+        resolve();
+        return;
+      }
+      if (msg?.type === "backend-request") {
+        relayToElectron(child, msg);
+        return;
+      }
+      if (
+        msg?.type === "plugin-reloaded" ||
+        msg?.type === "plugin-reload-failed"
+      ) {
+        pending.get(msg.id)?.(msg);
+        pending.delete(msg.id);
+      }
+    });
+    child.on("exit", (code) => {
+      for (const done of pending.values()) {
+        done({ type: "plugin-reload-failed", error: "backend exited" });
+      }
+      pending.clear();
+      if (backend === state) backend = null;
+      if (!state.ready) reject(new Error(`backend exited with ${code}`));
+      else if (!shuttingDown && !state.stopping) {
+        warn(
+          `backend exited with ${code}, press r or save a file to restart it`,
+        );
+      }
+    });
+  });
+  backend = state;
+  return state;
+}
+
+function stopChild(child, message) {
+  return new Promise((resolve) => {
+    if (!child || child.exitCode !== null || child.signalCode !== null) {
+      return resolve();
+    }
+    const timer = setTimeout(() => child.kill("SIGKILL"), SHUTDOWN_TIMEOUT_MS);
+    child.once("exit", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+    try {
+      if (message && child.connected) child.send(message);
+      else child.kill();
+    } catch {
+      child.kill();
+    }
+  });
+}
+
+function stopBackend() {
+  const state = backend;
+  if (!state) return Promise.resolve();
+  backend = null;
+  state.stopping = true;
+  return stopChild(state.child, state.ready ? { type: "shutdown" } : null);
+}
+
+async function restartBackend(reason) {
+  log(reason);
+  await stopBackend();
+  if (shuttingDown) return;
+  fs.copyFileSync(
+    path.join(root, "src", "backend", "package.json"),
+    path.join(root, "dist", "backend", "package.json"),
+  );
+  const state = startBackend();
+  try {
+    await state.readyPromise;
+  } catch (error) {
+    warn(`${error.message}, press r or save a file to restart it`);
+    return;
+  }
+  log("backend ready");
+  if (!vite) {
+    await startVite();
+    if (options.target === "electron") startElectron();
+    printKeys();
+  } else {
+    notifyBrowser({});
+  }
+}
+
+async function startVite() {
+  const { createServer } = await import("vite");
+  vite = await createServer({
+    root,
+    configFile: path.join(root, "vite.config.ts"),
+  });
+  await vite.listen();
+  vite.printUrls();
+}
+
+// The packaged app forks the backend from Electron's main process, and they
+// talk over that IPC channel (opening sign-in windows, the linked server's
+// proxy settings). Here both are children of the runner, so it passes those
+// messages along. The last proxy settings are kept for an app (re)started
+// after the backend sent them.
+let lastProxyConfig = null;
+const REPLAY_PREFIX = "dev-replay-";
+
+function relayToElectron(backendChild, msg) {
+  if (msg.channel === "sync-proxy-config") lastProxyConfig = msg;
+  if (electron?.connected) {
+    electron.send(msg);
+    return;
+  }
+  const ok = msg.channel === "sync-proxy-config";
+  try {
+    backendChild.send({
+      type: "backend-response",
+      id: msg.id,
+      ok,
+      ...(ok
+        ? { result: { success: true } }
+        : { error: "The desktop app is not running" }),
+    });
+  } catch {
+    // backend gone
+  }
+}
+
+function relayToBackend(msg) {
+  if (msg?.type !== "backend-response") return;
+  if (String(msg.id).startsWith(REPLAY_PREFIX)) return;
+  if (backend?.child.connected) backend.child.send(msg);
+}
+
+function startElectron() {
+  const env = {
+    ...process.env,
+    NODE_ENV: "development",
+    TERMIX_DEV_RUNNER: "true",
+  };
+  // Set by some editors' terminals; Electron would start as plain Node.
+  delete env.ELECTRON_RUN_AS_NODE;
+  const child = spawn(require("electron"), ["."], {
+    cwd: root,
+    env,
+    stdio: ["ignore", "inherit", "inherit", "ipc"],
+  });
+  electron = child;
+  child.on("message", relayToBackend);
+  if (lastProxyConfig) {
+    child.send({ ...lastProxyConfig, id: `${REPLAY_PREFIX}${Date.now()}` });
+  }
+  log("desktop app started");
+  child.on("exit", () => {
+    if (electron !== child) return;
+    electron = null;
+    if (!shuttingDown) {
+      log("desktop app closed");
+      void shutdown();
+    }
+  });
+}
+
+async function restartElectron() {
+  const child = electron;
+  electron = null;
+  log("electron changed, restarting the desktop app");
+  await stopChild(child);
+  if (!shuttingDown) startElectron();
+}
+
+function reloadInBackend(id) {
+  const state = backend;
+  if (!state?.ready || !state.child.connected) return Promise.resolve(null);
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      state.pending.delete(id);
+      resolve({ type: "plugin-reload-failed", error: "timed out" });
+    }, RELOAD_TIMEOUT_MS);
+    state.pending.set(id, (msg) => {
+      clearTimeout(timer);
+      resolve(msg);
+    });
+    state.child.send({ type: "plugin-reload", id });
+  });
+}
+
+function watchBackend() {
+  tsc = spawn(
+    process.execPath,
+    [
+      require.resolve("typescript/bin/tsc"),
+      "-p",
+      "tsconfig.node.json",
+      "--watch",
+      "--preserveWatchOutput",
+      "--pretty",
+      "false",
+    ],
+    { cwd: root, stdio: ["ignore", "pipe", "inherit"] },
+  );
+  let first = true;
+  let buffer = "";
+  tsc.stdout.on("data", (chunk) => {
+    buffer += chunk.toString();
+    const lines = buffer.split(/\r?\n/);
+    buffer = lines.pop();
+    for (const line of lines) {
+      const errors = parseTscStatus(line);
+      if (errors === null) {
+        if (line.includes("error TS")) console.log(line);
+        continue;
+      }
+      if (errors > 0) {
+        warn(`backend has ${errors} type error(s), fix them to restart it`);
+        continue;
+      }
+      const reason = first
+        ? "starting the backend"
+        : "backend changed, restarting it";
+      first = false;
+      void queue(() => restartBackend(reason));
+    }
+  });
+}
+
+function watchSdk() {
+  const changed = createDebouncer(300, () =>
+    queue(async () => {
+      try {
+        await run("npm", ["run", "build:sdk"]);
+      } catch (error) {
+        warn(`SDK failed to build: ${error.message}`);
+        return;
+      }
+      await restartBackend("SDK changed, restarting the backend");
+    }),
+  );
+  watch(SDK_SRC, true, () => changed("sdk"));
+}
+
+function watchElectron() {
+  const changed = createDebouncer(300, () => restartElectron());
+  watch(ELECTRON_DIR, false, (_event, filename) => {
+    if (/\.(c?js)$/.test(String(filename)) && electron) changed("electron");
+  });
+}
+
+// Docker target
+
+function dockerDataMount() {
+  if (options.data) return path.resolve(root, options.data);
+  const volume = options.fresh ? "termix-dev-data-fresh" : "termix-dev-data";
+  if (options.fresh) docker(["volume", "rm", "-f", volume], { quiet: true });
+  return volume;
+}
+
+async function buildImage() {
+  log(`building ${APP_IMAGE}, this takes a while`);
+  await run("docker", [
+    "build",
+    "-f",
+    "docker/Dockerfile",
+    "-t",
+    APP_IMAGE,
+    ...(options.noCache ? ["--no-cache"] : []),
+    ".",
+  ]);
+}
+
+function followDockerLogs(since) {
+  dockerLogs?.kill();
+  dockerLogs = spawn(
+    "docker",
+    ["logs", "-f", ...(since ? ["--since", since] : []), APP_CONTAINER],
+    { cwd: root, stdio: ["ignore", "inherit", "inherit"] },
+  );
+}
+
+async function waitForApp() {
+  const deadline = Date.now() + DOCKER_READY_TIMEOUT_MS;
+  while (Date.now() < deadline && !shuttingDown) {
+    try {
+      const response = await fetch(appUrl());
+      if (response.ok) return true;
+    } catch {
+      // not up yet
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+  return false;
+}
+
+let dockerMount = null;
+
+async function runAppContainer() {
+  docker(["rm", "-f", APP_CONTAINER], { quiet: true });
+  dockerMount ??= dockerDataMount();
+  const result = docker(
+    appRunArgs({
+      port: options.port,
+      env: serviceEnv(options, true),
+      dataMount: dockerMount,
+      pluginsDir: STAGED_PLUGINS,
+    }),
+    { quiet: true },
+  );
+  if (!result.ok) throw new Error(`could not start ${APP_CONTAINER}`);
+  followDockerLogs();
+  if (await waitForApp()) log(`ready on ${appUrl()}`);
+  else warn(`${APP_CONTAINER} is not answering on ${appUrl()} yet`);
+}
+
+async function restartAppContainer(reason) {
+  log(reason);
+  const since = new Date().toISOString();
+  docker(["restart", APP_CONTAINER], { quiet: true });
+  followDockerLogs(since);
+  if (await waitForApp()) log(`ready on ${appUrl()}`);
+}
+
+async function rebuildAndRunImage() {
+  try {
+    await buildImage();
+  } catch (error) {
+    warn(error.message);
+    return;
+  }
+  await runAppContainer();
+}
+
+function watchCoreForDocker() {
+  const changed = createDebouncer(500, () =>
+    warn("core changed, press r to rebuild the image"),
+  );
+  for (const dir of [path.join(root, "src"), SDK_SRC]) {
+    watch(dir, true, () => changed("core"));
+  }
+}
+
+// Plugins
+
+async function rebuildPlugin({ id, repo }) {
+  log(`building ${id}`);
+  try {
+    await run("npm", ["run", "build"], repo);
+  } catch (error) {
+    warn(`${id} failed to build: ${error.message}`);
+    return;
+  }
+  const destination = path.join(STAGED_PLUGINS, id);
+  fs.rmSync(destination, { recursive: true, force: true });
+  copyLocalBuild(repo, destination);
+
+  if (options.target === "docker") {
+    await restartAppContainer(`${id} rebuilt, restarting ${APP_CONTAINER}`);
+    return;
+  }
+  const result = await reloadInBackend(id);
+  if (!result) {
+    log(`${id} staged, it loads when the backend starts`);
+    return;
+  }
+  if (result.type === "plugin-reload-failed") {
+    warn(`${id} did not reload: ${result.error}`);
+    return;
+  }
+  log(`${id} reloaded (${result.state})`);
+  notifyBrowser({ id });
+}
+
+function watchPlugins(dir) {
+  plugins = findBuildablePlugins(dir);
+  const byId = new Map(plugins.map((plugin) => [plugin.id, plugin]));
+  const changed = createDebouncer(300, (id) =>
+    queue(() => rebuildPlugin(byId.get(id))),
+  );
+  for (const plugin of plugins) {
+    watch(plugin.repo, true, (_event, filename) => {
+      if (isPluginSourceChange(filename)) changed(plugin.id);
+    });
+  }
+  log(`watching ${plugins.length} plugin(s) in ${dir}`);
+}
+
+// Keys and shutdown
+
+function printKeys() {
+  if (!process.stdin.isTTY) return;
+  log("keys: r restart, p rebuild plugins, o open browser, q quit");
+}
+
+function listenForKeys() {
+  if (!process.stdin.isTTY) return;
+  process.stdin.setRawMode(true);
+  process.stdin.setEncoding("utf8");
+  process.stdin.resume();
+  process.stdin.on("data", (key) => {
+    if (key === "\u0003" || key === "q") {
+      void shutdown();
+    } else if (key === "r") {
+      void queue(() =>
+        options.target === "docker"
+          ? rebuildAndRunImage()
+          : restartBackend("restarting the backend"),
+      );
+    } else if (key === "p") {
+      if (plugins.length === 0) warn("no local plugins to rebuild");
+      for (const plugin of plugins) void queue(() => rebuildPlugin(plugin));
+    } else if (key === "o") {
+      openBrowser(appUrl());
+    }
+  });
+}
+
+async function shutdown(code = 0) {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  log("shutting down");
+  if (process.stdin.isTTY) process.stdin.setRawMode(false);
+  for (const watcher of watchers) watcher.close();
+  tsc?.kill();
+  dockerLogs?.kill();
+  await Promise.allSettled([stopBackend(), stopChild(electron), vite?.close()]);
+  if (options.target === "docker") {
+    docker(["rm", "-f", APP_CONTAINER], { quiet: true });
+  }
+  for (const container of startedServices) {
+    docker(["stop", container], { quiet: true });
+  }
+  process.exit(code);
+}
+
+async function main() {
+  try {
+    options = parseDevArgs(process.argv.slice(2));
+  } catch (error) {
+    console.error(error.message);
+    process.exit(1);
+  }
+  if (options.help) {
+    console.log(HELP);
+    return;
+  }
+
+  process.on("SIGINT", () => shutdown());
+  process.on("SIGTERM", () => shutdown());
+
+  const pluginsDir = options.local
+    ? path.resolve(
+        root,
+        options.pluginsDir ??
+          process.env.TERMIX_LOCAL_PLUGINS ??
+          "../Termix-Plugins",
+      )
+    : null;
+  const localPlugins =
+    pluginsDir && fs.existsSync(pluginsDir) ? pluginsDir : null;
+  if (pluginsDir && !localPlugins)
+    warn(`${pluginsDir} not found, using pinned plugins`);
+
+  if (needsDocker(options)) {
+    assertDocker();
+    await startServices();
+  }
+
+  log("building the SDK");
+  await run("npm", ["run", "build:sdk"]);
+  log(
+    localPlugins
+      ? `staging plugins with local builds from ${localPlugins}`
+      : "staging pinned plugins",
+  );
+  await run(process.execPath, [
+    "scripts/build-plugins.cjs",
+    ...(localPlugins ? ["--local", localPlugins] : []),
+  ]);
+
+  listenForKeys();
+  if (localPlugins) watchPlugins(localPlugins);
+
+  if (options.target === "docker") {
+    await buildImage();
+    await runAppContainer();
+    watchCoreForDocker();
+    printKeys();
+    return;
+  }
+
+  const dataDir = dataDirFor(options);
+  if (dataDir) {
+    const full = path.resolve(root, dataDir);
+    if (options.fresh) fs.rmSync(full, { recursive: true, force: true });
+    fs.mkdirSync(full, { recursive: true });
+    log(`data folder ${full}`);
+  }
+  backendEnv = {
+    ...serviceEnv(options, false),
+    // The desktop app runs against its own embedded backend, which is what
+    // turns on the local profile, sync and the desktop-only routes.
+    ...(options.target === "electron" ? { ELECTRON_EMBEDDED: "true" } : {}),
+    // A folder the runner picked starts empty on purpose, so the guard
+    // against a mistyped DATA_DIR must not stop it.
+    ...(dataDir
+      ? { DATA_DIR: path.resolve(root, dataDir), ALLOW_EMPTY_DATA_DIR: "true" }
+      : {}),
+  };
+
+  log("compiling the backend");
+  watchBackend();
+  watchSdk();
+  if (options.target === "electron") watchElectron();
+}
+
+main().catch(async (error) => {
+  console.error(error?.message ?? error);
+  await shutdown(1);
+});