From ec80a8ef8d1e0f9c6f7654ef36c2807a7758a3a8 Mon Sep 17 00:00:00 2001 From: LukeGus Date: Tue, 6 Oct 2026 15:55:00 -0500 Subject: [PATCH] feat: dev:all runner for web, electron and docker with plugin hot reload, termix-plugin dev --- electron/main.cjs | 28 +- package.json | 1 + packages/plugin-sdk/README.md | 6 + packages/plugin-sdk/cli/commands/dev.mjs | 158 ++++ packages/plugin-sdk/cli/index.mjs | 5 + scripts/build-plugins.cjs | 28 +- scripts/bundled-plugins.test.ts | 9 + scripts/dev-docker.test.ts | 74 ++ scripts/dev-runner.test.ts | 181 +++++ scripts/dev.mjs | 716 +++++++++++++++++++ scripts/lib/bundled-plugins.cjs | 7 +- scripts/lib/dev-docker.mjs | 109 +++ scripts/lib/dev-runner.mjs | 185 +++++ scripts/plugin-dev.test.ts | 98 +++ src/backend/plugins/dev-reload.ts | 66 ++ src/backend/plugins/manage.ts | 38 + src/backend/starter.ts | 10 +- src/backend/tests/plugins/dev-reload.test.ts | 51 ++ src/backend/tests/plugins/manage.test.ts | 40 ++ src/ui/plugin-host/loader.ts | 5 + src/ui/shell/MobileBar.tsx | 19 +- src/ui/sidebar/AppRail.tsx | 34 +- src/ui/tests/shell/MobileBar.test.tsx | 12 + src/ui/tests/sidebar/AppRail.test.tsx | 12 +- 24 files changed, 1848 insertions(+), 44 deletions(-) create mode 100644 packages/plugin-sdk/cli/commands/dev.mjs create mode 100644 scripts/dev-docker.test.ts create mode 100644 scripts/dev-runner.test.ts create mode 100644 scripts/dev.mjs create mode 100644 scripts/lib/dev-docker.mjs create mode 100644 scripts/lib/dev-runner.mjs create mode 100644 scripts/plugin-dev.test.ts create mode 100644 src/backend/plugins/dev-reload.ts create mode 100644 src/backend/tests/plugins/dev-reload.test.ts diff --git a/electron/main.cjs b/electron/main.cjs index 01e53fb0d..d0214b38c 100644 --- a/electron/main.cjs +++ b/electron/main.cjs @@ -644,17 +644,30 @@ const BACKEND_REQUEST_HANDLERS = { }, }; +function isDevRunner() { + return ( + process.env.TERMIX_DEV_RUNNER === "true" && + typeof process.send === "function" + ); +} + async function handleBackendRequest(msg) { if (!msg || msg.type !== "backend-request") return; const { id, channel, payload } = msg; const handler = BACKEND_REQUEST_HANDLERS[channel]; + // Under dev:all the runner owns the backend and relays these messages. + const target = + backendProcess && !backendProcess.killed + ? backendProcess + : isDevRunner() + ? process + : null; const reply = (response) => { - if (backendProcess && !backendProcess.killed) { - try { - backendProcess.send({ type: "backend-response", id, ...response }); - } catch (error) { - logToFile("Failed to reply to backend request:", error.message); - } + if (!target) return; + try { + target.send({ type: "backend-response", id, ...response }); + } catch (error) { + logToFile("Failed to reply to backend request:", error.message); } }; if (!handler) { @@ -3764,6 +3777,9 @@ app.whenReady().then(async () => { logToFile( "Skipping embedded backend (isDev=true) - expecting separate dev:backend process", ); + if (isDevRunner()) { + process.on("message", (msg) => void handleBackendRequest(msg)); + } } createTray(); diff --git a/package.json b/package.json index 410b2fe7d..ecdbeffb2 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "build": "npm run build:sdk && vite build && tsc -p tsconfig.node.json && node -e \"require('fs').copyFileSync('src/backend/package.json','dist/backend/package.json')\" && npm run build:plugins", "build:backend": "npm run build:sdk && tsc -p tsconfig.node.json && node -e \"require('fs').copyFileSync('src/backend/package.json','dist/backend/package.json')\" && npm run build:plugins", "dev:backend": "npm run build:sdk && tsc -p tsconfig.node.json && node -e \"require('fs').copyFileSync('src/backend/package.json','dist/backend/package.json')\" && npm run build:plugins && node ./dist/backend/backend/starter.js", + "dev:all": "node scripts/dev.mjs", "dev:backend:local": "npm run build:sdk && tsc -p tsconfig.node.json && node -e \"require('fs').copyFileSync('src/backend/package.json','dist/backend/package.json')\" && npm run build:plugins:local && node ./dist/backend/backend/starter.js", "dev:docker": "docker stop termix-dev 2>nul & docker rm termix-dev 2>nul & docker build -f docker/Dockerfile -t termix:dev --no-cache . && docker run -d --name termix-dev -p 3000:3000 -p 8080:8080 -p 30001:30001 -v \"%cd%\\db\\data:/app/data\" termix:dev", "dev:docker:restart": "docker stop termix-dev 2>nul & docker rm termix-dev 2>nul & docker run -d --name termix-dev -p 8080:8080 -p 30001:30001 -v \"%cd%\\db\\data:/app/data\" termix:dev", diff --git a/packages/plugin-sdk/README.md b/packages/plugin-sdk/README.md index 0702434e3..60e65b05b 100644 --- a/packages/plugin-sdk/README.md +++ b/packages/plugin-sdk/README.md @@ -35,8 +35,14 @@ npx termix-plugin pack # write -.tmxplug npx termix-plugin sign # sign it, needs TERMIX_PLUGIN_SIGNING_KEY npx termix-plugin verify --key # check a signature npx termix-plugin keygen --out # make a new signing key pair +npx termix-plugin dev --server --key # install on a server and reinstall on every change ``` +`dev` needs developer mode on (Admin settings, Plugins) and an API key from an +admin account. It installs the plugin from a file, marked unverified, and +reinstalls it each time you save. `TERMIX_SERVER_URL` and `TERMIX_API_KEY` work +in place of the flags. Reload the page to pick up frontend changes. +
## Support diff --git a/packages/plugin-sdk/cli/commands/dev.mjs b/packages/plugin-sdk/cli/commands/dev.mjs new file mode 100644 index 000000000..fd08ca43f --- /dev/null +++ b/packages/plugin-sdk/cli/commands/dev.mjs @@ -0,0 +1,158 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { build } from "./build.mjs"; +import { pack } from "./pack.mjs"; +import { readManifest } from "../lib/plugin-dir.mjs"; + +const SOURCE_DIRS = new Set(["src", "locales", "migrations"]); +const SOURCE_FILES = new Set(["manifest.json", "package.json", "README.md"]); + +function readOption(args, name) { + const index = args.indexOf(name); + if (index === -1) return null; + const value = args[index + 1]; + if (!value) throw new Error(`${name} needs a value`); + return value; +} + +/** True when a changed path (relative to the plugin) feeds its build. */ +export function isSourceChange(filename) { + if (!filename) return false; + const parts = String(filename).replaceAll("\\", "/").split("/"); + if (parts.length === 1) return SOURCE_FILES.has(parts[0]); + return SOURCE_DIRS.has(parts[0]); +} + +export function resolveDevOptions(args, env = process.env) { + const server = ( + readOption(args, "--server") ?? + env.TERMIX_SERVER_URL ?? + "http://localhost:30001" + ).replace(/\/+$/, ""); + const key = readOption(args, "--key") ?? env.TERMIX_API_KEY ?? null; + if (!key) { + throw new Error( + "An admin API key is needed. Pass --key tmx_... or set TERMIX_API_KEY.", + ); + } + return { server, key, once: args.includes("--once") }; +} + +const HINTS = { + DEVELOPER_MODE_OFF: + "Turn on developer mode in Admin settings, under Plugins, then save again.", + SIGNED_ONLY: + "The server sets TERMIX_REQUIRE_SIGNED_PLUGINS, so it never installs unsigned files.", + BUNDLED_ID: "That id ships with Termix. Use a different id while developing.", + ALREADY_INSTALLED: + "Uninstall the copy installed from the registry first, then save again.", +}; + +async function request(fetchImpl, url, init) { + const response = await fetchImpl(url, init); + const text = await response.text(); + let body = null; + try { + body = text ? JSON.parse(text) : null; + } catch { + body = { error: text }; + } + if (!response.ok) { + const code = body?.code; + const hint = code && HINTS[code] ? `\n ${HINTS[code]}` : ""; + throw new Error( + `${response.status} ${body?.error ?? response.statusText}${hint}`, + ); + } + return body; +} + +/** Uploads a packed .tmxplug and installs it, agreeing to what it asks for. */ +export async function pushPlugin( + file, + { server, key }, + fetchImpl = globalThis.fetch, +) { + const auth = { Authorization: `Bearer ${key}` }; + const preview = await request(fetchImpl, `${server}/plugins/upload`, { + method: "POST", + headers: { ...auth, "Content-Type": "application/octet-stream" }, + body: fs.readFileSync(file), + }); + const result = await request( + fetchImpl, + `${server}/plugins/upload/${preview.token}/install`, + { + method: "POST", + headers: { ...auth, "Content-Type": "application/json" }, + body: JSON.stringify({ capabilities: preview.capabilities }), + }, + ); + return { preview, result }; +} + +/** + * Builds, packs and installs the plugin on a running Termix server, then + * does it again on every source change. The server needs developer mode on. + */ +export async function dev({ cwd, args = [] }) { + const options = resolveDevOptions(args); + const outDir = fs.mkdtempSync(path.join(os.tmpdir(), "termix-plugin-dev-")); + let lastCapabilities = null; + + const cycle = async () => { + const started = Date.now(); + await build({ cwd }); + const file = await pack({ cwd, args: ["--out", outDir] }); + const { preview, result } = await pushPlugin(file, options); + fs.rmSync(file, { force: true }); + const capabilities = preview.capabilities.join(", ") || "none"; + if (capabilities !== lastCapabilities) { + console.log(` capabilities ${capabilities}`); + lastCapabilities = capabilities; + } + console.log( + `installed ${result.id}@${result.version} on ${options.server} (${result.state}, ${Date.now() - started} ms)`, + ); + }; + + try { + await cycle(); + } catch (error) { + if (options.once) throw error; + console.error(error?.message ?? error); + } + if (options.once) return; + + const { id } = readManifest(cwd); + console.log(`watching ${id} for changes, Ctrl+C to stop`); + let timer = null; + let running = false; + let again = false; + const fire = async () => { + if (running) { + again = true; + return; + } + running = true; + try { + await cycle(); + } catch (error) { + console.error(error?.message ?? error); + } finally { + running = false; + if (again) { + again = false; + void fire(); + } + } + }; + fs.watch(cwd, { recursive: true }, (_event, filename) => { + if (!isSourceChange(filename)) return; + clearTimeout(timer); + timer = setTimeout(() => void fire(), 300); + }); + await new Promise(() => {}); +} diff --git a/packages/plugin-sdk/cli/index.mjs b/packages/plugin-sdk/cli/index.mjs index eef9063f8..61cf4be76 100755 --- a/packages/plugin-sdk/cli/index.mjs +++ b/packages/plugin-sdk/cli/index.mjs @@ -13,6 +13,7 @@ import { test } from "./commands/test.mjs"; import { pack } from "./commands/pack.mjs"; import { sign, verify, keygen } from "./commands/sign.mjs"; import { migrations } from "./commands/migrations.mjs"; +import { dev } from "./commands/dev.mjs"; const COMMANDS = { build, @@ -28,6 +29,7 @@ const COMMANDS = { verify, keygen, migrations, + dev, }; const [command, ...args] = process.argv.slice(2); @@ -46,6 +48,9 @@ if (!command || command === "--help" || command === "-h") { " verify Check a .tmxplug's .sig --key [,...]", " keygen Write a new signing key pair [--out dir]", " migrations Generate migrations from the plugin's table definitions", + " dev Build, pack and install on a running server, again on every change", + " [--server url] [--key tmx_...] [--once]", + " (or TERMIX_SERVER_URL and TERMIX_API_KEY; needs developer mode)", ].join("\n"), ); process.exit(command ? 0 : 1); diff --git a/scripts/build-plugins.cjs b/scripts/build-plugins.cjs index 53d85afb1..6fadfbae2 100644 --- a/scripts/build-plugins.cjs +++ b/scripts/build-plugins.cjs @@ -7,9 +7,9 @@ * pinned sha256 and unpacked. * * For development, `--local ` (or TERMIX_LOCAL_PLUGINS) points at a - * folder of plugin repos like ../Termix-Plugins. A bundled plugin there whose - * sources changed since its last build is rebuilt first, then any built one - * (has dist/) is copied in place of its pin, so local plugin changes show up. + * folder of plugin repos like ../Termix-Plugins. A plugin there whose sources + * changed since its last build is rebuilt first, then any built one (has + * dist/) is copied in place of its pin, or added when it has no pin. */ const { execSync } = require("node:child_process"); @@ -33,14 +33,10 @@ async function main() { const localArg = flag !== -1 ? process.argv[flag + 1] : process.env.TERMIX_LOCAL_PLUGINS; const localDir = localArg ? path.resolve(root, localArg) : null; - rebuildLocalPlugins( - localDir, - new Set(plugins.map((p) => p.id)), - (repo, id) => { - console.log(`building ${id}`); - execSync("npm run build", { cwd: repo, stdio: "inherit" }); - }, - ); + rebuildLocalPlugins(localDir, null, (repo, id) => { + console.log(`building ${id}`); + execSync("npm run build", { cwd: repo, stdio: "inherit" }); + }); const local = findLocalPluginBuilds(localDir); fs.rmSync(destination, { recursive: true, force: true }); @@ -58,8 +54,16 @@ async function main() { console.log(`unpacked ${plugin.id}`); } + // A local plugin with no pin yet still gets staged, so new plugins work too. + const pinned = new Set(plugins.map((p) => p.id)); + const extra = [...local.keys()].filter((id) => !pinned.has(id)); + for (const id of extra) { + copyLocalBuild(local.get(id), path.join(destination, id)); + console.log(`copied local build of ${id} (not pinned)`); + } + console.log( - `Bundled ${plugins.length} plugin(s): ${plugins.map((p) => p.id).join(", ")}`, + `Bundled ${plugins.length + extra.length} plugin(s): ${[...pinned, ...extra].join(", ")}`, ); } diff --git a/scripts/bundled-plugins.test.ts b/scripts/bundled-plugins.test.ts index 744f2a7a1..add9e9dca 100644 --- a/scripts/bundled-plugins.test.ts +++ b/scripts/bundled-plugins.test.ts @@ -280,4 +280,13 @@ describe("rebuilding local plugins", () => { expect(run).toEqual(["a"]); expect(rebuildLocalPlugins(null, new Set(["a"]), () => {})).toEqual([]); }); + + it("builds every stale repo when no ids are given", () => { + const dir = tempDir(); + repo(dir, "Plugin-A", "a"); + repo(dir, "Plugin-B", "b"); + const run: string[] = []; + rebuildLocalPlugins(dir, null, (_repo: string, id: string) => run.push(id)); + expect(run.sort()).toEqual(["a", "b"]); + }); }); diff --git a/scripts/dev-docker.test.ts b/scripts/dev-docker.test.ts new file mode 100644 index 000000000..325bbce30 --- /dev/null +++ b/scripts/dev-docker.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { + SERVICES, + appRunArgs, + serviceEnv, + serviceRunArgs, +} from "./lib/dev-docker.mjs"; +import { parseDevArgs } from "./lib/dev-runner.mjs"; + +describe("serviceRunArgs", () => { + it("runs a service on the dev network with its port, env and volume", () => { + const args = serviceRunArgs(SERVICES.postgres); + expect(args.slice(0, 6)).toEqual([ + "run", + "-d", + "--name", + "termix-dev-postgres", + "--network", + "termix-dev", + ]); + expect(args).toContain("55432:5432"); + expect(args).toContain("POSTGRES_PASSWORD=termix"); + expect(args).toContain("termix-dev-postgres:/var/lib/postgresql/data"); + expect(args.at(-1)).toBe("postgres:16"); + }); + + it("adds no volume for guacd", () => { + expect(serviceRunArgs(SERVICES.guacd)).not.toContain("-v"); + }); +}); + +describe("serviceEnv", () => { + it("is empty for plain sqlite", () => { + expect(serviceEnv(parseDevArgs([]), false)).toEqual({}); + }); + + it("points a local backend at the published ports", () => { + expect( + serviceEnv(parseDevArgs(["--db", "postgres", "--guacd"]), false), + ).toEqual({ + DATABASE_DIALECT: "postgres", + DATABASE_URL: "postgres://termix:termix@127.0.0.1:55432/termix", + GUACD_HOST: "127.0.0.1", + GUACD_PORT: "4822", + }); + }); + + it("points a container at the other containers by name", () => { + expect( + serviceEnv(parseDevArgs(["--db", "mysql", "--guacd"]), true), + ).toEqual({ + DATABASE_DIALECT: "mysql", + DATABASE_URL: "mysql://root:termix@termix-dev-mysql:3306/termix", + GUACD_HOST: "termix-dev-guacd", + GUACD_PORT: "4822", + }); + }); +}); + +describe("appRunArgs", () => { + it("mounts the data and the staged plugins", () => { + const args = appRunArgs({ + port: 9000, + env: { GUACD_HOST: "termix-dev-guacd" }, + dataMount: "termix-dev-data", + pluginsDir: "/repo/dist/plugins", + }); + expect(args).toContain("9000:8080"); + expect(args).toContain("GUACD_HOST=termix-dev-guacd"); + expect(args).toContain("termix-dev-data:/app/data"); + expect(args).toContain("/repo/dist/plugins:/app/dist/plugins"); + expect(args.at(-1)).toBe("termix:dev"); + }); +}); diff --git a/scripts/dev-runner.test.ts b/scripts/dev-runner.test.ts new file mode 100644 index 000000000..432e3ee65 --- /dev/null +++ b/scripts/dev-runner.test.ts @@ -0,0 +1,181 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + createDebouncer, + createSerialQueue, + dataDirFor, + findBuildablePlugins, + isPluginSourceChange, + needsDocker, + parseDevArgs, + parseTscStatus, +} from "./lib/dev-runner.mjs"; + +const cleanups: Array<() => void> = []; + +afterEach(() => { + vi.useRealTimers(); + while (cleanups.length) cleanups.pop()!(); +}); + +describe("parseTscStatus", () => { + it("reads the error count from a watch summary", () => { + expect( + parseTscStatus( + "12:00:00 PM - Found 0 errors. Watching for file changes.", + ), + ).toBe(0); + expect(parseTscStatus("Found 1 error. Watching for file changes.")).toBe(1); + expect(parseTscStatus("src/a.ts(1,1): error TS2304: x")).toBeNull(); + }); +}); + +describe("isPluginSourceChange", () => { + it("accepts sources and skips build output", () => { + expect( + isPluginSourceChange(["src", "backend", "index.ts"].join("\\")), + ).toBe(true); + expect(isPluginSourceChange("locales/en.json")).toBe(true); + expect(isPluginSourceChange("manifest.json")).toBe(true); + expect(isPluginSourceChange("dist/backend.js")).toBe(false); + expect(isPluginSourceChange("node_modules/x/index.js")).toBe(false); + expect(isPluginSourceChange("notes.txt")).toBe(false); + expect(isPluginSourceChange(null)).toBe(false); + }); +}); + +describe("findBuildablePlugins", () => { + it("lists repos with a manifest and installed dependencies", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "termix-dev-")); + cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true })); + const make = (name: string, id: string, deps = true) => { + const repo = path.join(dir, name); + fs.mkdirSync(repo); + fs.writeFileSync( + path.join(repo, "manifest.json"), + JSON.stringify({ id }), + ); + if (deps) fs.mkdirSync(path.join(repo, "node_modules")); + }; + make("Plugin-B", "b"); + make("Plugin-A", "a"); + make("Plugin-C", "c", false); + fs.mkdirSync(path.join(dir, "notes")); + + expect(findBuildablePlugins(dir).map((p) => p.id)).toEqual(["a", "b"]); + expect(findBuildablePlugins(null)).toEqual([]); + }); +}); + +describe("createSerialQueue", () => { + it("runs one task at a time and survives a failure", async () => { + const queue = createSerialQueue(); + const order: string[] = []; + const first = queue(async () => { + await new Promise((r) => setTimeout(r, 10)); + order.push("first"); + throw new Error("boom"); + }); + const second = queue(async () => order.push("second")); + await expect(first).rejects.toThrow("boom"); + await second; + expect(order).toEqual(["first", "second"]); + }); +}); + +describe("createDebouncer", () => { + it("fires once per quiet key and again for a change mid-run", async () => { + vi.useFakeTimers(); + let release!: () => void; + const calls: string[] = []; + const trigger = createDebouncer(100, async (key: string) => { + calls.push(key); + if (calls.length === 1) await new Promise((r) => (release = r)); + }); + + trigger("a"); + trigger("a"); + await vi.advanceTimersByTimeAsync(100); + expect(calls).toEqual(["a"]); + + trigger("a"); + await vi.advanceTimersByTimeAsync(100); + expect(calls).toEqual(["a"]); + release(); + await vi.advanceTimersByTimeAsync(100); + expect(calls).toEqual(["a", "a"]); + trigger.cancel(); + }); +}); + +describe("parseDevArgs", () => { + it("defaults to the web target with sqlite and local plugins", () => { + expect(parseDevArgs([])).toMatchObject({ + target: "web", + db: "sqlite", + local: true, + guacd: false, + fresh: false, + port: 8081, + }); + }); + + it("reads a target and options in any order", () => { + expect( + parseDevArgs(["--db", "Postgres", "electron", "--guacd", "--fresh"]), + ).toMatchObject({ + target: "electron", + db: "postgres", + guacd: true, + fresh: true, + }); + expect( + parseDevArgs(["docker", "--port", "9000", "--no-cache", "--no-local"]), + ).toMatchObject({ + target: "docker", + port: 9000, + noCache: true, + local: false, + }); + expect(parseDevArgs(["--plugins", "../x"]).pluginsDir).toBe("../x"); + }); + + it("refuses unknown or incomplete options", () => { + expect(() => parseDevArgs(["--bogus"])).toThrow(/Unknown option/); + expect(() => parseDevArgs(["--db", "oracle"])).toThrow(/--db/); + expect(() => parseDevArgs(["--db"])).toThrow(/needs a value/); + expect(() => parseDevArgs(["--port", "abc"])).toThrow(/port/); + expect(() => parseDevArgs(["--fresh", "--data", "x"])).toThrow(/together/); + }); +}); + +describe("dataDirFor", () => { + it("keeps each engine and fresh runs in their own folder", () => { + expect(dataDirFor(parseDevArgs([]))).toBeNull(); + expect(dataDirFor(parseDevArgs(["--db", "mysql"]))).toBe("db/data-mysql"); + expect(dataDirFor(parseDevArgs(["--fresh"]))).toBe("db/data-fresh"); + expect(dataDirFor(parseDevArgs(["--fresh", "--db", "postgres"]))).toBe( + "db/data-fresh-postgres", + ); + expect(dataDirFor(parseDevArgs(["--data", "x"]))).toBe("x"); + expect(dataDirFor(parseDevArgs(["electron"]))).toBe("db/data-desktop"); + expect(dataDirFor(parseDevArgs(["electron", "--db", "mysql"]))).toBe( + "db/data-desktop-mysql", + ); + expect(dataDirFor(parseDevArgs(["electron", "--fresh"]))).toBe( + "db/data-fresh-desktop", + ); + }); +}); + +describe("needsDocker", () => { + it("is only true when something runs in a container", () => { + expect(needsDocker(parseDevArgs([]))).toBe(false); + expect(needsDocker(parseDevArgs(["electron"]))).toBe(false); + expect(needsDocker(parseDevArgs(["docker"]))).toBe(true); + expect(needsDocker(parseDevArgs(["--db", "postgres"]))).toBe(true); + expect(needsDocker(parseDevArgs(["--guacd"]))).toBe(true); + }); +}); diff --git a/scripts/dev.mjs b/scripts/dev.mjs new file mode 100644 index 000000000..0d0716833 --- /dev/null +++ b/scripts/dev.mjs @@ -0,0 +1,716 @@ +/** + * dev:all, one command for manual testing. Run with --help for the targets + * and options. + * + * Every target builds the SDK and stages plugins (local builds from + * ../Termix-Plugins when it exists), then: + * + * - web: compiles the backend in watch mode, starts it, then starts Vite. A + * backend or SDK change restarts the backend. A plugin change rebuilds that + * plugin and reloads it inside the running backend, and the browser swaps + * it in. + * - electron: web, plus the desktop app, restarted when electron/ changes. + * - docker: builds the production image and runs it with dist/plugins + * mounted. A plugin change rebuilds it and restarts the container. Core + * changes wait for "r", since an image build is slow. + * + * --db postgres|mysql and --guacd run those in Docker next to it. + */ + +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { spawn, spawnSync, fork } from "node:child_process"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { + HELP, + createDebouncer, + createSerialQueue, + dataDirFor, + findBuildablePlugins, + isPluginSourceChange, + needsDocker, + parseDevArgs, + parseTscStatus, +} from "./lib/dev-runner.mjs"; +import { + APP_CONTAINER, + APP_IMAGE, + NETWORK, + SERVICES, + appRunArgs, + serviceEnv, + serviceRunArgs, +} from "./lib/dev-docker.mjs"; + +const require = createRequire(import.meta.url); +const { copyLocalBuild } = require("./lib/bundled-plugins.cjs"); + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const BACKEND_ENTRY = path.join( + root, + "dist", + "backend", + "backend", + "starter.js", +); +const STAGED_PLUGINS = path.join(root, "dist", "plugins"); +const SDK_SRC = path.join(root, "packages", "plugin-sdk", "src"); +const ELECTRON_DIR = path.join(root, "electron"); +const RELOAD_TIMEOUT_MS = 60_000; +const SHUTDOWN_TIMEOUT_MS = 10_000; +const DOCKER_READY_TIMEOUT_MS = 180_000; + +const log = (message) => console.log(`\x1b[36m[dev]\x1b[0m ${message}`); +const warn = (message) => console.log(`\x1b[33m[dev]\x1b[0m ${message}`); + +let options; +const queue = createSerialQueue(); +const startedServices = []; +const watchers = []; +let shuttingDown = false; +let plugins = []; +let vite = null; +let backend = null; +let electron = null; +let tsc = null; +let dockerLogs = null; +let backendEnv = {}; + +function run(command, args, cwd = root) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + stdio: ["ignore", "inherit", "inherit"], + shell: process.platform === "win32", + }); + child.on("error", reject); + child.on("exit", (code) => + code === 0 + ? resolve() + : reject(new Error(`${command} ${args.join(" ")} exited with ${code}`)), + ); + }); +} + +function docker(args, { quiet = false } = {}) { + const result = spawnSync("docker", args, { + cwd: root, + encoding: "utf8", + stdio: quiet ? "pipe" : ["ignore", "inherit", "inherit"], + }); + return { ok: result.status === 0, out: (result.stdout ?? "").trim() }; +} + +function watch(target, recursive, onChange) { + watchers.push(fs.watch(target, { recursive }, onChange)); +} + +function appUrl() { + if (options.target === "docker") return `http://localhost:${options.port}`; + return vite?.resolvedUrls?.local?.[0] ?? "http://localhost:5173"; +} + +function openBrowser(url) { + const [command, args] = + process.platform === "win32" + ? ["cmd", ["/c", "start", "", url]] + : process.platform === "darwin" + ? ["open", [url]] + : ["xdg-open", [url]]; + spawn(command, args, { stdio: "ignore", detached: true }).unref(); +} + +// Docker side containers + +function assertDocker() { + if (!docker(["info"], { quiet: true }).ok) { + throw new Error( + "Docker is not running. Start it, or drop --db, --guacd and the docker target.", + ); + } + if (!docker(["network", "inspect", NETWORK], { quiet: true }).ok) { + docker(["network", "create", NETWORK], { quiet: true }); + } +} + +function containerState(name) { + const result = docker(["inspect", "-f", "{{.State.Running}}", name], { + quiet: true, + }); + if (!result.ok) return "missing"; + return result.out === "true" ? "running" : "stopped"; +} + +async function waitForService(service) { + if (!service.ready) return; + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + if ( + docker(["exec", service.container, ...service.ready], { quiet: true }).ok + ) + return; + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + throw new Error(`${service.container} did not become ready`); +} + +async function startService(name) { + const service = SERVICES[name]; + if (options.fresh && service.volume) { + docker(["rm", "-f", service.container], { quiet: true }); + docker(["volume", "rm", service.volume[0]], { quiet: true }); + } + const state = containerState(service.container); + if (state === "running") { + log(`${service.container} is already running`); + } else { + log(`starting ${service.container}`); + const result = + state === "stopped" + ? docker(["start", service.container], { quiet: true }) + : docker(serviceRunArgs(service), { quiet: true }); + if (!result.ok) throw new Error(`could not start ${service.container}`); + startedServices.push(service.container); + } + await waitForService(service); +} + +async function startServices() { + if (options.db !== "sqlite") await startService(options.db); + if (options.guacd) await startService("guacd"); +} + +// Local backend, Vite and Electron + +function notifyBrowser(data) { + vite?.ws.send({ type: "custom", event: "termix:plugins-reloaded", data }); +} + +function startBackend() { + const child = fork(BACKEND_ENTRY, [], { + cwd: root, + env: { ...process.env, ...backendEnv, TERMIX_DEV_RELOAD: "true" }, + stdio: ["ignore", "inherit", "inherit", "ipc"], + }); + const pending = new Map(); + const state = { child, pending, ready: false, stopping: false }; + state.readyPromise = new Promise((resolve, reject) => { + child.on("message", (msg) => { + if (msg?.type === "backend-ready") { + state.ready = true; + resolve(); + return; + } + if (msg?.type === "backend-request") { + relayToElectron(child, msg); + return; + } + if ( + msg?.type === "plugin-reloaded" || + msg?.type === "plugin-reload-failed" + ) { + pending.get(msg.id)?.(msg); + pending.delete(msg.id); + } + }); + child.on("exit", (code) => { + for (const done of pending.values()) { + done({ type: "plugin-reload-failed", error: "backend exited" }); + } + pending.clear(); + if (backend === state) backend = null; + if (!state.ready) reject(new Error(`backend exited with ${code}`)); + else if (!shuttingDown && !state.stopping) { + warn( + `backend exited with ${code}, press r or save a file to restart it`, + ); + } + }); + }); + backend = state; + return state; +} + +function stopChild(child, message) { + return new Promise((resolve) => { + if (!child || child.exitCode !== null || child.signalCode !== null) { + return resolve(); + } + const timer = setTimeout(() => child.kill("SIGKILL"), SHUTDOWN_TIMEOUT_MS); + child.once("exit", () => { + clearTimeout(timer); + resolve(); + }); + try { + if (message && child.connected) child.send(message); + else child.kill(); + } catch { + child.kill(); + } + }); +} + +function stopBackend() { + const state = backend; + if (!state) return Promise.resolve(); + backend = null; + state.stopping = true; + return stopChild(state.child, state.ready ? { type: "shutdown" } : null); +} + +async function restartBackend(reason) { + log(reason); + await stopBackend(); + if (shuttingDown) return; + fs.copyFileSync( + path.join(root, "src", "backend", "package.json"), + path.join(root, "dist", "backend", "package.json"), + ); + const state = startBackend(); + try { + await state.readyPromise; + } catch (error) { + warn(`${error.message}, press r or save a file to restart it`); + return; + } + log("backend ready"); + if (!vite) { + await startVite(); + if (options.target === "electron") startElectron(); + printKeys(); + } else { + notifyBrowser({}); + } +} + +async function startVite() { + const { createServer } = await import("vite"); + vite = await createServer({ + root, + configFile: path.join(root, "vite.config.ts"), + }); + await vite.listen(); + vite.printUrls(); +} + +// The packaged app forks the backend from Electron's main process, and they +// talk over that IPC channel (opening sign-in windows, the linked server's +// proxy settings). Here both are children of the runner, so it passes those +// messages along. The last proxy settings are kept for an app (re)started +// after the backend sent them. +let lastProxyConfig = null; +const REPLAY_PREFIX = "dev-replay-"; + +function relayToElectron(backendChild, msg) { + if (msg.channel === "sync-proxy-config") lastProxyConfig = msg; + if (electron?.connected) { + electron.send(msg); + return; + } + const ok = msg.channel === "sync-proxy-config"; + try { + backendChild.send({ + type: "backend-response", + id: msg.id, + ok, + ...(ok + ? { result: { success: true } } + : { error: "The desktop app is not running" }), + }); + } catch { + // backend gone + } +} + +function relayToBackend(msg) { + if (msg?.type !== "backend-response") return; + if (String(msg.id).startsWith(REPLAY_PREFIX)) return; + if (backend?.child.connected) backend.child.send(msg); +} + +function startElectron() { + const env = { + ...process.env, + NODE_ENV: "development", + TERMIX_DEV_RUNNER: "true", + }; + // Set by some editors' terminals; Electron would start as plain Node. + delete env.ELECTRON_RUN_AS_NODE; + const child = spawn(require("electron"), ["."], { + cwd: root, + env, + stdio: ["ignore", "inherit", "inherit", "ipc"], + }); + electron = child; + child.on("message", relayToBackend); + if (lastProxyConfig) { + child.send({ ...lastProxyConfig, id: `${REPLAY_PREFIX}${Date.now()}` }); + } + log("desktop app started"); + child.on("exit", () => { + if (electron !== child) return; + electron = null; + if (!shuttingDown) { + log("desktop app closed"); + void shutdown(); + } + }); +} + +async function restartElectron() { + const child = electron; + electron = null; + log("electron changed, restarting the desktop app"); + await stopChild(child); + if (!shuttingDown) startElectron(); +} + +function reloadInBackend(id) { + const state = backend; + if (!state?.ready || !state.child.connected) return Promise.resolve(null); + return new Promise((resolve) => { + const timer = setTimeout(() => { + state.pending.delete(id); + resolve({ type: "plugin-reload-failed", error: "timed out" }); + }, RELOAD_TIMEOUT_MS); + state.pending.set(id, (msg) => { + clearTimeout(timer); + resolve(msg); + }); + state.child.send({ type: "plugin-reload", id }); + }); +} + +function watchBackend() { + tsc = spawn( + process.execPath, + [ + require.resolve("typescript/bin/tsc"), + "-p", + "tsconfig.node.json", + "--watch", + "--preserveWatchOutput", + "--pretty", + "false", + ], + { cwd: root, stdio: ["ignore", "pipe", "inherit"] }, + ); + let first = true; + let buffer = ""; + tsc.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop(); + for (const line of lines) { + const errors = parseTscStatus(line); + if (errors === null) { + if (line.includes("error TS")) console.log(line); + continue; + } + if (errors > 0) { + warn(`backend has ${errors} type error(s), fix them to restart it`); + continue; + } + const reason = first + ? "starting the backend" + : "backend changed, restarting it"; + first = false; + void queue(() => restartBackend(reason)); + } + }); +} + +function watchSdk() { + const changed = createDebouncer(300, () => + queue(async () => { + try { + await run("npm", ["run", "build:sdk"]); + } catch (error) { + warn(`SDK failed to build: ${error.message}`); + return; + } + await restartBackend("SDK changed, restarting the backend"); + }), + ); + watch(SDK_SRC, true, () => changed("sdk")); +} + +function watchElectron() { + const changed = createDebouncer(300, () => restartElectron()); + watch(ELECTRON_DIR, false, (_event, filename) => { + if (/\.(c?js)$/.test(String(filename)) && electron) changed("electron"); + }); +} + +// Docker target + +function dockerDataMount() { + if (options.data) return path.resolve(root, options.data); + const volume = options.fresh ? "termix-dev-data-fresh" : "termix-dev-data"; + if (options.fresh) docker(["volume", "rm", "-f", volume], { quiet: true }); + return volume; +} + +async function buildImage() { + log(`building ${APP_IMAGE}, this takes a while`); + await run("docker", [ + "build", + "-f", + "docker/Dockerfile", + "-t", + APP_IMAGE, + ...(options.noCache ? ["--no-cache"] : []), + ".", + ]); +} + +function followDockerLogs(since) { + dockerLogs?.kill(); + dockerLogs = spawn( + "docker", + ["logs", "-f", ...(since ? ["--since", since] : []), APP_CONTAINER], + { cwd: root, stdio: ["ignore", "inherit", "inherit"] }, + ); +} + +async function waitForApp() { + const deadline = Date.now() + DOCKER_READY_TIMEOUT_MS; + while (Date.now() < deadline && !shuttingDown) { + try { + const response = await fetch(appUrl()); + if (response.ok) return true; + } catch { + // not up yet + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + return false; +} + +let dockerMount = null; + +async function runAppContainer() { + docker(["rm", "-f", APP_CONTAINER], { quiet: true }); + dockerMount ??= dockerDataMount(); + const result = docker( + appRunArgs({ + port: options.port, + env: serviceEnv(options, true), + dataMount: dockerMount, + pluginsDir: STAGED_PLUGINS, + }), + { quiet: true }, + ); + if (!result.ok) throw new Error(`could not start ${APP_CONTAINER}`); + followDockerLogs(); + if (await waitForApp()) log(`ready on ${appUrl()}`); + else warn(`${APP_CONTAINER} is not answering on ${appUrl()} yet`); +} + +async function restartAppContainer(reason) { + log(reason); + const since = new Date().toISOString(); + docker(["restart", APP_CONTAINER], { quiet: true }); + followDockerLogs(since); + if (await waitForApp()) log(`ready on ${appUrl()}`); +} + +async function rebuildAndRunImage() { + try { + await buildImage(); + } catch (error) { + warn(error.message); + return; + } + await runAppContainer(); +} + +function watchCoreForDocker() { + const changed = createDebouncer(500, () => + warn("core changed, press r to rebuild the image"), + ); + for (const dir of [path.join(root, "src"), SDK_SRC]) { + watch(dir, true, () => changed("core")); + } +} + +// Plugins + +async function rebuildPlugin({ id, repo }) { + log(`building ${id}`); + try { + await run("npm", ["run", "build"], repo); + } catch (error) { + warn(`${id} failed to build: ${error.message}`); + return; + } + const destination = path.join(STAGED_PLUGINS, id); + fs.rmSync(destination, { recursive: true, force: true }); + copyLocalBuild(repo, destination); + + if (options.target === "docker") { + await restartAppContainer(`${id} rebuilt, restarting ${APP_CONTAINER}`); + return; + } + const result = await reloadInBackend(id); + if (!result) { + log(`${id} staged, it loads when the backend starts`); + return; + } + if (result.type === "plugin-reload-failed") { + warn(`${id} did not reload: ${result.error}`); + return; + } + log(`${id} reloaded (${result.state})`); + notifyBrowser({ id }); +} + +function watchPlugins(dir) { + plugins = findBuildablePlugins(dir); + const byId = new Map(plugins.map((plugin) => [plugin.id, plugin])); + const changed = createDebouncer(300, (id) => + queue(() => rebuildPlugin(byId.get(id))), + ); + for (const plugin of plugins) { + watch(plugin.repo, true, (_event, filename) => { + if (isPluginSourceChange(filename)) changed(plugin.id); + }); + } + log(`watching ${plugins.length} plugin(s) in ${dir}`); +} + +// Keys and shutdown + +function printKeys() { + if (!process.stdin.isTTY) return; + log("keys: r restart, p rebuild plugins, o open browser, q quit"); +} + +function listenForKeys() { + if (!process.stdin.isTTY) return; + process.stdin.setRawMode(true); + process.stdin.setEncoding("utf8"); + process.stdin.resume(); + process.stdin.on("data", (key) => { + if (key === "\u0003" || key === "q") { + void shutdown(); + } else if (key === "r") { + void queue(() => + options.target === "docker" + ? rebuildAndRunImage() + : restartBackend("restarting the backend"), + ); + } else if (key === "p") { + if (plugins.length === 0) warn("no local plugins to rebuild"); + for (const plugin of plugins) void queue(() => rebuildPlugin(plugin)); + } else if (key === "o") { + openBrowser(appUrl()); + } + }); +} + +async function shutdown(code = 0) { + if (shuttingDown) return; + shuttingDown = true; + log("shutting down"); + if (process.stdin.isTTY) process.stdin.setRawMode(false); + for (const watcher of watchers) watcher.close(); + tsc?.kill(); + dockerLogs?.kill(); + await Promise.allSettled([stopBackend(), stopChild(electron), vite?.close()]); + if (options.target === "docker") { + docker(["rm", "-f", APP_CONTAINER], { quiet: true }); + } + for (const container of startedServices) { + docker(["stop", container], { quiet: true }); + } + process.exit(code); +} + +async function main() { + try { + options = parseDevArgs(process.argv.slice(2)); + } catch (error) { + console.error(error.message); + process.exit(1); + } + if (options.help) { + console.log(HELP); + return; + } + + process.on("SIGINT", () => shutdown()); + process.on("SIGTERM", () => shutdown()); + + const pluginsDir = options.local + ? path.resolve( + root, + options.pluginsDir ?? + process.env.TERMIX_LOCAL_PLUGINS ?? + "../Termix-Plugins", + ) + : null; + const localPlugins = + pluginsDir && fs.existsSync(pluginsDir) ? pluginsDir : null; + if (pluginsDir && !localPlugins) + warn(`${pluginsDir} not found, using pinned plugins`); + + if (needsDocker(options)) { + assertDocker(); + await startServices(); + } + + log("building the SDK"); + await run("npm", ["run", "build:sdk"]); + log( + localPlugins + ? `staging plugins with local builds from ${localPlugins}` + : "staging pinned plugins", + ); + await run(process.execPath, [ + "scripts/build-plugins.cjs", + ...(localPlugins ? ["--local", localPlugins] : []), + ]); + + listenForKeys(); + if (localPlugins) watchPlugins(localPlugins); + + if (options.target === "docker") { + await buildImage(); + await runAppContainer(); + watchCoreForDocker(); + printKeys(); + return; + } + + const dataDir = dataDirFor(options); + if (dataDir) { + const full = path.resolve(root, dataDir); + if (options.fresh) fs.rmSync(full, { recursive: true, force: true }); + fs.mkdirSync(full, { recursive: true }); + log(`data folder ${full}`); + } + backendEnv = { + ...serviceEnv(options, false), + // The desktop app runs against its own embedded backend, which is what + // turns on the local profile, sync and the desktop-only routes. + ...(options.target === "electron" ? { ELECTRON_EMBEDDED: "true" } : {}), + // A folder the runner picked starts empty on purpose, so the guard + // against a mistyped DATA_DIR must not stop it. + ...(dataDir + ? { DATA_DIR: path.resolve(root, dataDir), ALLOW_EMPTY_DATA_DIR: "true" } + : {}), + }; + + log("compiling the backend"); + watchBackend(); + watchSdk(); + if (options.target === "electron") watchElectron(); +} + +main().catch(async (error) => { + console.error(error?.message ?? error); + await shutdown(1); +}); diff --git a/scripts/lib/bundled-plugins.cjs b/scripts/lib/bundled-plugins.cjs index 3056bfe14..d2ded1d6b 100644 --- a/scripts/lib/bundled-plugins.cjs +++ b/scripts/lib/bundled-plugins.cjs @@ -231,7 +231,10 @@ function isLocalBuildStale(repo) { ); } -/** Builds each listed plugin repo whose build is missing or out of date. */ +/** + * Builds each plugin repo whose build is missing or out of date. ids limits + * it to those plugins; null builds every one. + */ function rebuildLocalPlugins(dir, ids, run) { const rebuilt = []; if (!dir || !fs.existsSync(dir)) return rebuilt; @@ -246,7 +249,7 @@ function rebuildLocalPlugins(dir, ids, run) { } catch { continue; } - if (!ids.has(id) || !isLocalBuildStale(repo)) continue; + if ((ids && !ids.has(id)) || !isLocalBuildStale(repo)) continue; run(repo, id); rebuilt.push(id); } diff --git a/scripts/lib/dev-docker.mjs b/scripts/lib/dev-docker.mjs new file mode 100644 index 000000000..55644c883 --- /dev/null +++ b/scripts/lib/dev-docker.mjs @@ -0,0 +1,109 @@ +export const NETWORK = "termix-dev"; +export const APP_CONTAINER = "termix-dev"; +export const APP_IMAGE = "termix:dev"; + +/** Side containers dev:all can run. hostPort is what the local backend uses. */ +export const SERVICES = { + postgres: { + container: "termix-dev-postgres", + image: "postgres:16", + hostPort: 55432, + port: 5432, + env: { + POSTGRES_USER: "termix", + POSTGRES_PASSWORD: "termix", + POSTGRES_DB: "termix", + }, + volume: ["termix-dev-postgres", "/var/lib/postgresql/data"], + // Over TCP, so the server that only runs during first-time setup does + // not count as ready. + ready: ["pg_isready", "-h", "127.0.0.1", "-U", "termix", "-d", "termix"], + }, + mysql: { + container: "termix-dev-mysql", + image: "mysql:8", + hostPort: 33306, + port: 3306, + env: { MYSQL_ROOT_PASSWORD: "termix", MYSQL_DATABASE: "termix" }, + volume: ["termix-dev-mysql", "/var/lib/mysql"], + ready: ["mysqladmin", "ping", "-h", "127.0.0.1", "-uroot", "-ptermix"], + }, + guacd: { + container: "termix-dev-guacd", + image: "guacamole/guacd:1.6.0", + hostPort: 4822, + port: 4822, + env: {}, + volume: null, + ready: null, + }, +}; + +function envArgs(env) { + return Object.entries(env).flatMap(([key, value]) => [ + "-e", + `${key}=${value}`, + ]); +} + +export function serviceRunArgs(service) { + return [ + "run", + "-d", + "--name", + service.container, + "--network", + NETWORK, + "-p", + `${service.hostPort}:${service.port}`, + ...envArgs(service.env), + ...(service.volume + ? ["-v", `${service.volume[0]}:${service.volume[1]}`] + : []), + service.image, + ]; +} + +/** + * Environment that points a backend at the side containers. inDocker means + * the backend runs in the dev network and reaches them by name. + */ +export function serviceEnv(options, inDocker) { + const env = {}; + const address = (service) => + inDocker + ? `${service.container}:${service.port}` + : `127.0.0.1:${service.hostPort}`; + if (options.db === "postgres") { + env.DATABASE_DIALECT = "postgres"; + env.DATABASE_URL = `postgres://termix:termix@${address(SERVICES.postgres)}/termix`; + } else if (options.db === "mysql") { + env.DATABASE_DIALECT = "mysql"; + env.DATABASE_URL = `mysql://root:termix@${address(SERVICES.mysql)}/termix`; + } + if (options.guacd) { + const guacd = SERVICES.guacd; + env.GUACD_HOST = inDocker ? guacd.container : "127.0.0.1"; + env.GUACD_PORT = String(inDocker ? guacd.port : guacd.hostPort); + } + return env; +} + +export function appRunArgs({ port, env, dataMount, pluginsDir }) { + return [ + "run", + "-d", + "--name", + APP_CONTAINER, + "--network", + NETWORK, + "-p", + `${port}:8080`, + ...envArgs({ PORT: "8080", NODE_ENV: "development", ...env }), + "-v", + `${dataMount}:/app/data`, + "-v", + `${pluginsDir}:/app/dist/plugins`, + APP_IMAGE, + ]; +} diff --git a/scripts/lib/dev-runner.mjs b/scripts/lib/dev-runner.mjs new file mode 100644 index 000000000..0bfed7c57 --- /dev/null +++ b/scripts/lib/dev-runner.mjs @@ -0,0 +1,185 @@ +import fs from "node:fs"; +import path from "node:path"; + +const ID = /^[a-z0-9][a-z0-9-]*$/; +const SOURCE_DIRS = new Set(["src", "locales", "migrations"]); +const SOURCE_FILES = new Set(["manifest.json", "package.json"]); + +/** The error count from tsc --watch's "Found N errors" line, else null. */ +export function parseTscStatus(line) { + const match = /Found (\d+) errors?\b.*Watching for file changes/.exec(line); + return match ? Number(match[1]) : null; +} + +/** True when a changed path (relative to a plugin repo) feeds its build. */ +export function isPluginSourceChange(filename) { + if (!filename) return false; + const parts = String(filename).replaceAll("\\", "/").split("/"); + if (parts.length === 1) return SOURCE_FILES.has(parts[0]); + return SOURCE_DIRS.has(parts[0]); +} + +/** Plugin repos under dir that can be built (manifest.json and node_modules). */ +export function findBuildablePlugins(dir) { + const plugins = []; + if (!dir || !fs.existsSync(dir)) return plugins; + for (const name of fs.readdirSync(dir)) { + const repo = path.join(dir, name); + if (!fs.existsSync(path.join(repo, "node_modules"))) continue; + try { + const { id } = JSON.parse( + fs.readFileSync(path.join(repo, "manifest.json"), "utf8"), + ); + if (typeof id === "string" && ID.test(id)) plugins.push({ id, repo }); + } catch { + // not a plugin repo + } + } + return plugins.sort((a, b) => a.id.localeCompare(b.id)); +} + +/** Runs tasks one after another; a failed task does not stop the next. */ +export function createSerialQueue() { + let tail = Promise.resolve(); + return (task) => { + const run = tail.then(task, task); + tail = run.catch(() => {}); + return run; + }; +} + +/** + * Calls fn(key) once a key has gone quiet for ms. A key that changes again + * while fn is still running for it runs once more afterwards. + */ +export function createDebouncer(ms, fn) { + const timers = new Map(); + const running = new Set(); + const again = new Set(); + const fire = async (key) => { + timers.delete(key); + if (running.has(key)) { + again.add(key); + return; + } + running.add(key); + try { + await fn(key); + } finally { + running.delete(key); + if (again.delete(key)) trigger(key); + } + }; + const trigger = (key) => { + clearTimeout(timers.get(key)); + timers.set( + key, + setTimeout(() => void fire(key), ms), + ); + }; + trigger.cancel = () => { + for (const timer of timers.values()) clearTimeout(timer); + timers.clear(); + }; + return trigger; +} + +export const TARGETS = ["web", "electron", "docker"]; +export const DATABASES = ["sqlite", "postgres", "mysql"]; + +export const HELP = `Usage: npm run dev:all -- [target] [options] + +Targets + web Backend and Vite on http://localhost:5173 (default) + electron Backend, Vite and the desktop app + docker Build the production image and run it with your local plugins + +Options + --plugins Plugin repos to build and watch (default ../Termix-Plugins) + --no-local Use only the plugins pinned in docker/bundled-plugins.json + --db sqlite (default), postgres or mysql. Postgres and MySQL run in Docker + --guacd Run guacd in Docker for remote desktop + --fresh Start from empty data (first run, onboarding) + --data Data folder for the backend (default db/data, db/data-desktop for + electron, plus - for postgres or mysql) + --port docker target: port on this machine (default 8081) + --no-cache docker target: build the image without the layer cache + -h, --help Show this + +Keys while running + r restart the backend (docker: rebuild the image) + p rebuild every local plugin + o open the app in a browser + q quit`; + +/** Reads dev:all arguments. Throws on anything it does not know. */ +export function parseDevArgs(argv) { + const options = { + target: "web", + pluginsDir: null, + local: true, + db: "sqlite", + guacd: false, + fresh: false, + data: null, + port: 8081, + noCache: false, + help: false, + }; + const value = (i, name) => { + const next = argv[i + 1]; + if (!next || next.startsWith("--")) + throw new Error(`${name} needs a value`); + return next; + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (TARGETS.includes(arg)) options.target = arg; + else if (arg === "-h" || arg === "--help") options.help = true; + else if (arg === "--no-local") options.local = false; + else if (arg === "--guacd") options.guacd = true; + else if (arg === "--fresh") options.fresh = true; + else if (arg === "--no-cache") options.noCache = true; + else if (arg === "--plugins") options.pluginsDir = value(i++, arg); + else if (arg === "--data") options.data = value(i++, arg); + else if (arg === "--db") { + options.db = value(i++, arg).toLowerCase(); + if (!DATABASES.includes(options.db)) { + throw new Error(`--db must be one of ${DATABASES.join(", ")}`); + } + } else if (arg === "--port") { + options.port = Number(value(i++, arg)); + if (!Number.isInteger(options.port) || options.port < 1) { + throw new Error("--port must be a port number"); + } + } else throw new Error(`Unknown option ${arg}. Run with --help.`); + } + if (options.fresh && options.data) { + throw new Error("--fresh and --data cannot be used together"); + } + return options; +} + +/** + * The data folder the local backend uses, or null for its own default. The + * desktop, each database engine and fresh runs get their own, so a desktop's + * local profile or another engine's keys never land in the everyday data. + */ +export function dataDirFor(options) { + if (options.data) return options.data; + const suffix = [ + options.target === "electron" ? "desktop" : null, + options.db === "sqlite" ? null : options.db, + ] + .filter(Boolean) + .map((part) => `-${part}`) + .join(""); + if (options.fresh) return `db/data-fresh${suffix}`; + return suffix ? `db/data${suffix}` : null; +} + +export function needsDocker(options) { + return ( + options.target === "docker" || options.db !== "sqlite" || options.guacd + ); +} diff --git a/scripts/plugin-dev.test.ts b/scripts/plugin-dev.test.ts new file mode 100644 index 000000000..03fceb3c2 --- /dev/null +++ b/scripts/plugin-dev.test.ts @@ -0,0 +1,98 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + isSourceChange, + pushPlugin, + resolveDevOptions, +} from "../packages/plugin-sdk/cli/commands/dev.mjs"; + +const cleanups: Array<() => void> = []; + +afterEach(() => { + while (cleanups.length) cleanups.pop()!(); +}); + +function json(status: number, body: unknown) { + return new Response(JSON.stringify(body), { status }); +} + +function tempFile(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "termix-plugin-dev-")); + cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true })); + const file = path.join(dir, "demo-1.0.0.tmxplug"); + fs.writeFileSync(file, "archive"); + return file; +} + +describe("termix-plugin dev options", () => { + it("reads flags first, then the environment", () => { + expect( + resolveDevOptions(["--server", "http://a:8080/", "--key", "tmx_1"], {}), + ).toEqual({ server: "http://a:8080", key: "tmx_1", once: false }); + expect( + resolveDevOptions(["--once"], { + TERMIX_SERVER_URL: "http://b", + TERMIX_API_KEY: "tmx_2", + }), + ).toEqual({ server: "http://b", key: "tmx_2", once: true }); + expect(resolveDevOptions([], { TERMIX_API_KEY: "tmx_3" }).server).toBe( + "http://localhost:30001", + ); + }); + + it("needs an API key", () => { + expect(() => resolveDevOptions([], {})).toThrow(/API key/); + }); + + it("watches sources only", () => { + expect(isSourceChange("src/frontend/index.tsx")).toBe(true); + expect(isSourceChange("README.md")).toBe(true); + expect(isSourceChange("dist/frontend.js")).toBe(false); + }); +}); + +describe("pushPlugin", () => { + it("uploads the file then installs it with the capabilities it asks for", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + json(200, { token: "t".repeat(32), capabilities: ["kv:own"] }), + ) + .mockResolvedValueOnce( + json(200, { id: "demo", version: "1.0.0", state: "active" }), + ); + const { result } = await pushPlugin( + tempFile(), + { server: "http://s", key: "tmx_k" }, + fetchImpl, + ); + expect(result.state).toBe("active"); + + const [uploadUrl, upload] = fetchImpl.mock.calls[0]; + expect(uploadUrl).toBe("http://s/plugins/upload"); + expect(upload.headers.Authorization).toBe("Bearer tmx_k"); + expect(upload.headers["Content-Type"]).toBe("application/octet-stream"); + + const [installUrl, install] = fetchImpl.mock.calls[1]; + expect(installUrl).toBe( + `http://s/plugins/upload/${"t".repeat(32)}/install`, + ); + expect(JSON.parse(install.body)).toEqual({ capabilities: ["kv:own"] }); + }); + + it("explains a server with developer mode off", async () => { + const fetchImpl = vi.fn().mockResolvedValue( + json(403, { + error: "Installing from a file needs developer mode", + code: "DEVELOPER_MODE_OFF", + }), + ); + await expect( + pushPlugin(tempFile(), { server: "http://s", key: "tmx_k" }, fetchImpl), + ).rejects.toThrow( + /403 Installing from a file needs developer mode\n.*developer mode/, + ); + }); +}); diff --git a/src/backend/plugins/dev-reload.ts b/src/backend/plugins/dev-reload.ts new file mode 100644 index 000000000..6f15b68d3 --- /dev/null +++ b/src/backend/plugins/dev-reload.ts @@ -0,0 +1,66 @@ +/** + * IPC between the backend and scripts/dev.mjs, which forks it with + * TERMIX_DEV_RELOAD=true. The runner asks for a plugin to be loaded again + * after it rebuilt and copied it into the bundled folder. + */ + +import { pluginLogger } from "../utils/logger.js"; + +interface ReloadRequest { + type: "plugin-reload"; + id: string; +} + +type Send = (message: unknown) => void; +type Reload = (id: string) => Promise<{ version: string; state: string }>; + +function isReloadRequest(msg: unknown): msg is ReloadRequest { + return ( + !!msg && + typeof msg === "object" && + (msg as { type?: unknown }).type === "plugin-reload" && + typeof (msg as { id?: unknown }).id === "string" + ); +} + +export function isDevReloadEnabled(): boolean { + return ( + process.env.TERMIX_DEV_RELOAD === "true" && + typeof process.send === "function" + ); +} + +export async function handleDevMessage( + msg: unknown, + send: Send, + reload: Reload, +): Promise { + if (!isReloadRequest(msg)) return; + try { + const result = await reload(msg.id); + pluginLogger.info(`Reloaded ${msg.id}@${result.version}`, { + operation: "plugin_dev_reload", + }); + send({ type: "plugin-reloaded", id: msg.id, state: result.state }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + pluginLogger.error(`Could not reload ${msg.id}`, error, { + operation: "plugin_dev_reload", + }); + send({ type: "plugin-reload-failed", id: msg.id, error: message }); + } +} + +export function listenForDevReloads(): void { + if (!isDevReloadEnabled()) return; + process.on("message", (msg: unknown) => { + void handleDevMessage( + msg, + (reply) => process.send?.(reply), + async (id) => { + const { reloadBundledPlugin } = await import("./manage.js"); + return reloadBundledPlugin(id); + }, + ); + }); +} diff --git a/src/backend/plugins/manage.ts b/src/backend/plugins/manage.ts index 23ff51ae0..b47b7fdd2 100644 --- a/src/backend/plugins/manage.ts +++ b/src/backend/plugins/manage.ts @@ -1290,6 +1290,44 @@ export function installUpload( }); } +/** + * Loads the copy in the bundled folder again after it was rebuilt, for the + * dev runner. Running dependents stop first and start again after. + */ +export function reloadBundledPlugin( + pluginId: string, +): Promise<{ id: string; version: string; state: string }> { + return serialized(async () => { + const { loader } = getPluginRuntime(); + const previous = loader.get(pluginId); + if (previous && previous.source !== "bundled") { + throw new PluginManageError( + `${pluginId} is not loaded from the bundled folder`, + 409, + "NOT_BUNDLED", + ); + } + const dependents = previous + ? dependentChain(pluginId, await enabledIds()).filter( + (id) => loader.get(id)?.state === "active", + ) + : []; + for (const id of dependents) await deactivatePlugin(id); + await unloadPlugin(pluginId); + + const plugin = await loader.loadBundled(pluginId); + await registerLoadedPlugins([plugin]); + + let state = "stopped"; + if ((await enabledIds()).has(pluginId)) { + const { missing } = dependencyChain(pluginId); + state = missing.length > 0 ? "blocked" : await startAndRecord(pluginId); + } + for (const id of [...dependents].reverse()) await startAndRecord(id); + return { id: pluginId, version: plugin.manifest.version, state }; + }); +} + /** Test seam. */ export function resetUploads(): void { uploads.clear(); diff --git a/src/backend/starter.ts b/src/backend/starter.ts index b04f0fabe..32d612cc5 100644 --- a/src/backend/starter.ts +++ b/src/backend/starter.ts @@ -283,7 +283,10 @@ async function provisionLocalDesktopUserIfNeeded(): Promise { }); // Log output can be filtered by the configured level or split across chunks. - if (process.env.ELECTRON_EMBEDDED === "true") { + if ( + process.env.ELECTRON_EMBEDDED === "true" || + process.env.TERMIX_DEV_RELOAD === "true" + ) { process.send?.({ type: "backend-ready" }); } @@ -328,6 +331,11 @@ async function provisionLocalDesktopUserIfNeeded(): Promise { } }); + if (process.env.TERMIX_DEV_RELOAD === "true") { + const { listenForDevReloads } = await import("./plugins/dev-reload.js"); + listenForDevReloads(); + } + // A single bad request must not take the server down. Exit only on errors // that leave the process genuinely unusable; log and keep serving // otherwise, since these are almost always scoped to one connection. diff --git a/src/backend/tests/plugins/dev-reload.test.ts b/src/backend/tests/plugins/dev-reload.test.ts new file mode 100644 index 000000000..44555b9e5 --- /dev/null +++ b/src/backend/tests/plugins/dev-reload.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("../../utils/logger.js", () => ({ + pluginLogger: { info: vi.fn(), error: vi.fn() }, +})); + +const { handleDevMessage } = await import("../../plugins/dev-reload.js"); + +describe("dev reload messages", () => { + it("reloads the plugin and answers with its state", async () => { + const send = vi.fn(); + const reload = vi.fn(async () => ({ version: "1.0.0", state: "active" })); + await handleDevMessage( + { type: "plugin-reload", id: "snippets" }, + send, + reload, + ); + expect(reload).toHaveBeenCalledWith("snippets"); + expect(send).toHaveBeenCalledWith({ + type: "plugin-reloaded", + id: "snippets", + state: "active", + }); + }); + + it("answers with the error when the reload fails", async () => { + const send = vi.fn(); + await handleDevMessage( + { type: "plugin-reload", id: "snippets" }, + send, + async () => { + throw new Error("bad manifest"); + }, + ); + expect(send).toHaveBeenCalledWith({ + type: "plugin-reload-failed", + id: "snippets", + error: "bad manifest", + }); + }); + + it("ignores other messages", async () => { + const send = vi.fn(); + const reload = vi.fn(); + await handleDevMessage({ type: "shutdown" }, send, reload); + await handleDevMessage({ type: "plugin-reload" }, send, reload); + await handleDevMessage(null, send, reload); + expect(reload).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); +}); diff --git a/src/backend/tests/plugins/manage.test.ts b/src/backend/tests/plugins/manage.test.ts index fb461cfde..65a7c960b 100644 --- a/src/backend/tests/plugins/manage.test.ts +++ b/src/backend/tests/plugins/manage.test.ts @@ -445,6 +445,46 @@ describe("dependencies", () => { }); }); +describe("reloading a bundled plugin for the dev runner", () => { + it("loads the rebuilt copy and restarts what depends on it", async () => { + createFixturePlugin({ id: "base", root: bundled }); + createFixturePlugin({ + id: "addon", + root: bundled, + manifestOverrides: { dependencies: { base: "^1.0.0" } }, + }); + const loader = await boot(); + createFixturePlugin({ + id: "base", + root: bundled, + manifestOverrides: { version: "1.1.0" }, + }); + + const result = await manage.reloadBundledPlugin("base"); + expect(result).toEqual({ id: "base", version: "1.1.0", state: "active" }); + expect(loader.get("base")?.manifest.version).toBe("1.1.0"); + expect(loader.get("addon")?.state).toBe("active"); + expect(db.rows.get("base")?.version).toBe("1.1.0"); + }); + + it("loads and starts a plugin added since boot", async () => { + const loader = await boot(); + createFixturePlugin({ id: "fresh", root: bundled }); + const result = await manage.reloadBundledPlugin("fresh"); + expect(result.state).toBe("active"); + expect(loader.get("fresh")?.state).toBe("active"); + }); + + it("keeps a disabled plugin stopped", async () => { + createFixturePlugin({ id: "base", root: bundled }); + const loader = await boot(); + await manage.setPluginState("base", false); + const result = await manage.reloadBundledPlugin("base"); + expect(result.state).toBe("stopped"); + expect(loader.get("base")?.state).toBe("loaded"); + }); +}); + describe("consent", () => { it("refuses an install without the consented list, before any download", async () => { registry.index.plugins = [registryEntry("notes", [{ version: "1.0.0" }])]; diff --git a/src/ui/plugin-host/loader.ts b/src/ui/plugin-host/loader.ts index 2c3947f22..7753ceb87 100644 --- a/src/ui/plugin-host/loader.ts +++ b/src/ui/plugin-host/loader.ts @@ -398,6 +398,11 @@ function onPluginsChanged() { void syncPlugins(); } +// Sent by scripts/dev.mjs after it rebuilt a plugin or restarted the backend. +import.meta.hot?.on("termix:plugins-reloaded", () => { + if (started) void syncPlugins(); +}); + function onFocus() { const now = Date.now(); if (now - lastFocusSync < FOCUS_SYNC_INTERVAL_MS) return; diff --git a/src/ui/shell/MobileBar.tsx b/src/ui/shell/MobileBar.tsx index 60438e001..7ee5372ad 100644 --- a/src/ui/shell/MobileBar.tsx +++ b/src/ui/shell/MobileBar.tsx @@ -19,6 +19,7 @@ import { import { useHiddenRailTabs } from "@/sidebar/hidden-rail-tabs"; import { RailBadge } from "@/sidebar/RailBadge"; import { cn } from "@/lib/utils"; +import { isElectron } from "@/lib/electron"; const PRIMARY_COUNT = 4; @@ -183,14 +184,16 @@ export function MobileBar({ onOpenSettings(); }} /> - { - close(); - onLogout(); - }} - /> + {!isElectron() && ( + { + close(); + onLogout(); + }} + /> + )} diff --git a/src/ui/sidebar/AppRail.tsx b/src/ui/sidebar/AppRail.tsx index b1e6d2cdf..6dd13a911 100644 --- a/src/ui/sidebar/AppRail.tsx +++ b/src/ui/sidebar/AppRail.tsx @@ -25,6 +25,7 @@ import { useUiPreferencesContext } from "@/contexts/UiPreferencesContext"; import { toggleHiddenRailTab, useHiddenRailTabs } from "./hidden-rail-tabs"; import { RailBadge } from "./RailBadge"; import { rem } from "@/lib/rem"; +import { isElectron } from "@/lib/electron"; /** Core rail views; plugins add their own ids at runtime. */ export type CoreRailView = @@ -437,20 +438,25 @@ export function AppRail({ )} {footerItems.map(renderItem)} - {footerItems.length > 0 && } - + {footerItems.length > 0 && !isElectron() && ( + + )} + {/* The desktop signs in to its own local profile by itself. */} + {!isElectron() && ( + + )}
diff --git a/src/ui/tests/shell/MobileBar.test.tsx b/src/ui/tests/shell/MobileBar.test.tsx index 590109320..cf1a646f5 100644 --- a/src/ui/tests/shell/MobileBar.test.tsx +++ b/src/ui/tests/shell/MobileBar.test.tsx @@ -24,6 +24,7 @@ afterEach(() => { cleanup(); resetRegisteredRailItems(); localStorage.clear(); + delete (window as { IS_ELECTRON?: boolean }).IS_ELECTRON; }); function renderBar(onRailClick = vi.fn()) { @@ -77,6 +78,17 @@ describe("MobileBar", () => { expect(screen.getByText("nav.group.objects")).toBeTruthy(); }); + it("leaves logout out of More on the desktop", () => { + renderBar(); + fireEvent.click(screen.getByLabelText("common.more")); + expect(screen.getByText("common.logout")).toBeTruthy(); + cleanup(); + (window as { IS_ELECTRON?: boolean }).IS_ELECTRON = true; + renderBar(); + fireEvent.click(screen.getByLabelText("common.more")); + expect(screen.queryByText("common.logout")).toBeNull(); + }); + it("leaves out hidden items", () => { localStorage.setItem("hiddenRailTabs", JSON.stringify(["hosts"])); renderBar(); diff --git a/src/ui/tests/sidebar/AppRail.test.tsx b/src/ui/tests/sidebar/AppRail.test.tsx index abdee43c5..fc22a105e 100644 --- a/src/ui/tests/sidebar/AppRail.test.tsx +++ b/src/ui/tests/sidebar/AppRail.test.tsx @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { fireEvent, render, screen } from "@testing-library/react"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { Bell, Boxes } from "lucide-react"; import { AppRail } from "../../sidebar/AppRail"; @@ -25,6 +25,7 @@ vi.mock("@/hooks/use-permissions", () => ({ afterEach(() => { resetRegisteredRailItems(); localStorage.clear(); + delete (window as { IS_ELECTRON?: boolean }).IS_ELECTRON; }); function renderRail(onRailClick = vi.fn()) { @@ -44,6 +45,15 @@ function renderRail(onRailClick = vi.fn()) { } describe("AppRail", () => { + it("shows logout in the browser but not on the desktop", () => { + renderRail(); + expect(screen.getByLabelText("common.logout")).toBeTruthy(); + cleanup(); + (window as { IS_ELECTRON?: boolean }).IS_ELECTRON = true; + renderRail(); + expect(screen.queryByLabelText("common.logout")).toBeNull(); + }); + it("puts a footer item above the profile with its badge", async () => { registerRailItem({ id: "inbox",