Ethica/Termix
1
0 Fork 0
mirror of https://github.com/Termix-SSH/Termix.git synced 2026-10-29 14:45:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

release-2.9.0 (#1532)

Browse Source 
* fix(macos): skip single-instance lock in Mac App Store builds (#1454)

fix(macos): skip single-instance lock in Mac App Store builds

* Fix/backend optional sharp (#1455)

Fix/backend optional sharp

* feat: issue #1218 (#1218)

https://github.com/Termix-SSH/Support/issues/1218

* feat: Add option to use saved global custom theme under Connection Defaults (#1209)

https://github.com/Termix-SSH/Support/issues/1209

* feat: Add Fleet Sharing functionality to the UI (#1228)

https://github.com/Termix-SSH/Support/issues/1228

* feat: Add a list view to the Docker management page (#1237)

https://github.com/Termix-SSH/Support/issues/1237

* feat: issue #1264 (#1264)

https://github.com/Termix-SSH/Support/issues/1264

* feat: 2FA With FortiToken (#1288)

https://github.com/Termix-SSH/Support/issues/1288

* feat: Allow to disable showing paths to folders (#1274)

https://github.com/Termix-SSH/Support/issues/1274

* feat: Sync Network Graph between Desktop and Remote Server (#1245)

https://github.com/Termix-SSH/Support/issues/1245

* feat: Version Number on offline servers (#1291)

https://github.com/Termix-SSH/Support/issues/1291

* feat: Support OrbStack Docker socket path on macOS / Fix "Docker is not installed" on mac... (#1302)

https://github.com/Termix-SSH/Support/issues/1302

* feat: Sidebar host click should focus existing tab instead of opening a new connection (#1289)

https://github.com/Termix-SSH/Support/issues/1289

* fix: Search in side bar shows overlapping hosts when grouped by tags (#1303)

https://github.com/Termix-SSH/Support/issues/1303

* fix: I can't send a file on its own. (#1304)

https://github.com/Termix-SSH/Support/issues/1304

* fix: sudo password not autofilling (#1248)

https://github.com/Termix-SSH/Support/issues/1248

* fix: node-MainThread (#1300)

https://github.com/Termix-SSH/Support/issues/1300

* fix: Tunnel authentication with shared credentials (#1295)

https://github.com/Termix-SSH/Support/issues/1295

* fix: host key updates not syncing, metrics 404 race on first connect

Host key writes never bumped updatedAt so sync never picked them up.
Also retry the first metrics fetch briefly instead of failing right away.

* chore: increment ver

* feat: add category and icon fields to plugin manifest schema

* ci(deps): bump the github-actions group with 2 updates (#1458)

Bumps the github-actions group with 2 updates: [crowdin/github-action](https://github.com/crowdin/github-action) and [actions/github-script](https://github.com/actions/github-script).


Updates `crowdin/github-action` from 2 to 3
- [Release notes](https://github.com/crowdin/github-action/releases)
- [Commits](https://github.com/crowdin/github-action/compare/v2...v3)

Updates `actions/github-script` from 7 to 9
- [Release notes](https://github.com/actions/github-script/releases)
- [Commits](https://github.com/actions/github-script/compare/v7...v9)

---
updated-dependencies:
- dependency-name: crowdin/github-action
  dependency-version: '3'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: github-actions
- dependency-name: actions/github-script
  dependency-version: '9'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps-dev): bump the dev-patch-updates group with 13 updates (#1459)

Bumps the dev-patch-updates group with 13 updates:

| Package | From | To |
| --- | --- | --- |
| [@codemirror/commands](https://github.com/codemirror/commands) | `6.11.0` | `6.11.1` |
| [@codemirror/search](https://github.com/codemirror/search) | `6.7.1` | `6.7.2` |
| [@codemirror/view](https://github.com/codemirror/view) | `6.43.9` | `6.43.12` |
| [@testing-library/dom](https://github.com/testing-library/dom-testing-library) | `10.4.1` | `10.4.2` |
| [@testing-library/react](https://github.com/testing-library/react-testing-library) | `16.3.2` | `16.3.3` |
| [@testing-library/user-event](https://github.com/testing-library/user-event) | `14.6.5` | `14.6.7` |
| [@types/ssh2](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/ssh2) | `1.15.5` | `1.15.6` |
| [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/tree/HEAD/packages/plugin-react) | `6.1.0` | `6.1.1` |
| [cytoscape](https://github.com/cytoscape/cytoscape.js) | `3.34.1` | `3.34.3` |
| [eslint-plugin-react-refresh](https://github.com/ArnaudBarre/eslint-plugin-react-refresh) | `0.5.4` | `0.5.7` |
| [i18next](https://github.com/i18next/i18next) | `26.4.0` | `26.4.2` |
| [prettier](https://github.com/prettier/prettier) | `3.9.6` | `3.9.8` |
| [react-i18next](https://github.com/i18next/react-i18next) | `17.0.12` | `17.0.14` |


Updates `@codemirror/commands` from 6.11.0 to 6.11.1
- [Changelog](https://github.com/codemirror/commands/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codemirror/commands/commits)

Updates `@codemirror/search` from 6.7.1 to 6.7.2
- [Changelog](https://github.com/codemirror/search/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codemirror/search/commits)

Updates `@codemirror/view` from 6.43.9 to 6.43.12
- [Changelog](https://github.com/codemirror/view/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codemirror/view/commits)

Updates `@testing-library/dom` from 10.4.1 to 10.4.2
- [Release notes](https://github.com/testing-library/dom-testing-library/releases)
- [Changelog](https://github.com/testing-library/dom-testing-library/blob/main/CHANGELOG.md)
- [Commits](https://github.com/testing-library/dom-testing-library/compare/v10.4.1...v10.4.2)

Updates `@testing-library/react` from 16.3.2 to 16.3.3
- [Release notes](https://github.com/testing-library/react-testing-library/releases)
- [Changelog](https://github.com/testing-library/react-testing-library/blob/main/CHANGELOG.md)
- [Commits](https://github.com/testing-library/react-testing-library/compare/v16.3.2...v16.3.3)

Updates `@testing-library/user-event` from 14.6.5 to 14.6.7
- [Release notes](https://github.com/testing-library/user-event/releases)
- [Changelog](https://github.com/testing-library/user-event/blob/main/CHANGELOG.md)
- [Commits](https://github.com/testing-library/user-event/compare/v14.6.5...v14.6.7)

Updates `@types/ssh2` from 1.15.5 to 1.15.6
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/ssh2)

Updates `@vitejs/plugin-react` from 6.1.0 to 6.1.1
- [Release notes](https://github.com/vitejs/vite-plugin-react/releases)
- [Changelog](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/CHANGELOG.md)
- [Commits](https://github.com/vitejs/vite-plugin-react/commits/plugin-react@6.1.1/packages/plugin-react)

Updates `cytoscape` from 3.34.1 to 3.34.3
- [Release notes](https://github.com/cytoscape/cytoscape.js/releases)
- [Commits](https://github.com/cytoscape/cytoscape.js/compare/v3.34.1...v3.34.3)

Updates `eslint-plugin-react-refresh` from 0.5.4 to 0.5.7
- [Release notes](https://github.com/ArnaudBarre/eslint-plugin-react-refresh/releases)
- [Changelog](https://github.com/ArnaudBarre/eslint-plugin-react-refresh/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ArnaudBarre/eslint-plugin-react-refresh/compare/v0.5.4...v0.5.7)

Updates `i18next` from 26.4.0 to 26.4.2
- [Release notes](https://github.com/i18next/i18next/releases)
- [Changelog](https://github.com/i18next/i18next/blob/master/CHANGELOG.md)
- [Commits](https://github.com/i18next/i18next/compare/v26.4.0...v26.4.2)

Updates `prettier` from 3.9.6 to 3.9.8
- [Release notes](https://github.com/prettier/prettier/releases)
- [Changelog](https://github.com/prettier/prettier/blob/main/CHANGELOG.md)
- [Commits](https://github.com/prettier/prettier/compare/3.9.6...3.9.8)

Updates `react-i18next` from 17.0.12 to 17.0.14
- [Changelog](https://github.com/i18next/react-i18next/blob/master/CHANGELOG.md)
- [Commits](https://github.com/i18next/react-i18next/compare/v17.0.12...v17.0.14)

---
updated-dependencies:
- dependency-name: "@codemirror/commands"
  dependency-version: 6.11.1
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@codemirror/search"
  dependency-version: 6.7.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@codemirror/view"
  dependency-version: 6.43.12
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@testing-library/dom"
  dependency-version: 10.4.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@testing-library/react"
  dependency-version: 16.3.3
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@testing-library/user-event"
  dependency-version: 14.6.7
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@types/ssh2"
  dependency-version: 1.15.6
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: "@vitejs/plugin-react"
  dependency-version: 6.1.1
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: cytoscape
  dependency-version: 3.34.3
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: eslint-plugin-react-refresh
  dependency-version: 0.5.7
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: i18next
  dependency-version: 26.4.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: prettier
  dependency-version: 3.9.8
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
- dependency-name: react-i18next
  dependency-version: 17.0.14
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump the prod-patch-updates group with 6 updates (#1461)

Bumps the prod-patch-updates group with 6 updates:

| Package | From | To |
| --- | --- | --- |
| [@tanstack/react-virtual](https://github.com/TanStack/virtual/tree/HEAD/packages/react-virtual) | `3.14.10` | `3.14.13` |
| [compression](https://github.com/expressjs/compression) | `1.8.1` | `1.8.2` |
| [jose](https://github.com/panva/jose) | `6.2.9` | `6.2.12` |
| [jszip](https://github.com/Stuk/jszip) | `3.10.1` | `3.10.2` |
| [socks](https://github.com/JoshGlazebrook/socks) | `2.8.9` | `2.8.10` |
| [undici](https://github.com/nodejs/undici) | `8.10.0` | `8.10.2` |


Updates `@tanstack/react-virtual` from 3.14.10 to 3.14.13
- [Release notes](https://github.com/TanStack/virtual/releases)
- [Changelog](https://github.com/TanStack/virtual/blob/main/packages/react-virtual/CHANGELOG.md)
- [Commits](https://github.com/TanStack/virtual/commits/@tanstack/react-virtual@3.14.13/packages/react-virtual)

Updates `compression` from 1.8.1 to 1.8.2
- [Release notes](https://github.com/expressjs/compression/releases)
- [Changelog](https://github.com/expressjs/compression/blob/master/HISTORY.md)
- [Commits](https://github.com/expressjs/compression/compare/v1.8.1...v1.8.2)

Updates `jose` from 6.2.9 to 6.2.12
- [Release notes](https://github.com/panva/jose/releases)
- [Changelog](https://github.com/panva/jose/blob/main/CHANGELOG.md)
- [Commits](https://github.com/panva/jose/compare/v6.2.9...v6.2.12)

Updates `jszip` from 3.10.1 to 3.10.2
- [Changelog](https://github.com/Stuk/jszip/blob/main/CHANGES.md)
- [Commits](https://github.com/Stuk/jszip/compare/v3.10.1...v3.10.2)

Updates `socks` from 2.8.9 to 2.8.10
- [Release notes](https://github.com/JoshGlazebrook/socks/releases)
- [Commits](https://github.com/JoshGlazebrook/socks/compare/2.8.9...2.8.10)

Updates `undici` from 8.10.0 to 8.10.2
- [Release notes](https://github.com/nodejs/undici/releases)
- [Commits](https://github.com/nodejs/undici/compare/v8.10.0...v8.10.2)

---
updated-dependencies:
- dependency-name: "@tanstack/react-virtual"
  dependency-version: 3.14.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: compression
  dependency-version: 1.8.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: jose
  dependency-version: 6.2.12
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: jszip
  dependency-version: 3.10.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: socks
  dependency-version: 2.8.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: undici
  dependency-version: 8.10.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* feat: add plugin worker protocol, bootstrap and plugin_storage table

Worker-side ctx proxy plus the message envelope it talks over, and the
per-plugin key/value table behind ctx.storage with migrations for all
three dialects. Rows cascade with the plugin so uninstalling leaves
nothing behind.

* feat: add allowlisted host view for plugins

An allowlist rather than a list of fields to strip. A blocklist rots:
the moment a new secret-bearing column lands on SSHHost it starts
leaking silently. With an allowlist a new field stays invisible to
plugins until someone adds it here on purpose.

* feat: add plugin loader with manifest validation and crash restart

Validates an unpacked plugin, spawns a worker_threads worker and manages
activate/deactivate/crash-restart with backoff, disabling after three
tries.

The crash counter clears only after a plugin stays up past a stability
window, not on activation. Clearing it on activation let a plugin that
activates cleanly and then dies restart forever, because every attempt
looked like the first.

loader.ts documents what the worker boundary does and does not buy you.
Node's permission model is process-wide, not per-worker, so it cannot be
enabled for plugin workers alone and is not claimed here.

* feat: route internal events through a shared plugin event bus

Formalises three ad-hoc publish paths that already existed:
automation-events.ts, metrics/automation-bridge.ts and the
hostSessionStatus singleton, now the host.session.status topic.

Both invariants the old code relied on are preserved. Publishing stays
fire-and-forget, so a failing subscriber cannot disturb a metrics poll or
a host delete. The automations engine subscribes to the bus at scheduler
start instead of being imported ad hoc, which keeps the edge
one-directional: automations may import repositories, hosts modules must
not import automations.

* feat: add plugin ctx broker with permission gate and audit

Every ctx call a plugin makes is checked against its granted
capabilities, performed by the main thread, then audited. A capability
must be both declared in the manifest and granted in the database, so
widening a plugin's reach always needs a new manifest the user can see,
never just a database row.

Audit attribution is set by the broker, not the plugin. The plugin
supplies only the details, so it cannot forge the actor.

fix: stop plugin workers seeing plaintext credentials in error messages

A failing ctx.ssh.connect handed the worker the raw error from deep in
the SSH stack, which embeds the connect config. The worker received, in
full:

  Authentication failed for root using password <SECRET>

where <SECRET> was the host's real plaintext password. Any plugin with
ssh.exec could read it by catching the error, defeating the whole point
of the handle-based ssh API.

Errors crossing the boundary are now sanitised. Only PluginFacingError
and PluginPermissionError, both authored here, reach a plugin verbatim;
anything else becomes a generic message and the real one goes to the
log. This uses a marker class rather than matching on message text,
because matching would start leaking again the moment an upstream error
happened to contain a familiar phrase.

Covered by credential-isolation.test.ts, which plants a sentinel in
every secret-bearing field and asserts it appears in no message the
worker ever receives.

* feat: dispatch plugin HTTP routes and wire the runtime into startup

ctx.http.route now registers a real router on the /plugin-api/:pluginId
dispatcher, replacing its stub 404. A plugin never touches the Express
req/res: it gets a plain summary and returns a plain object, so sockets
and session cookies stay on this side of the boundary. Authorization and
cookie headers are not forwarded.

Plugins load last in the start-up sequence, after the server is fully
wired, and are terminated before the database on shutdown so none can
outlive it. A plugin that fails to load cannot stop the backend.

fix: correct the nginx plugin-api location regex

The pattern was ^/plugin-api/(/.*)?$, which needs a double slash and so
never matched /plugin-api/<id>/<route>. Every other block in the file
uses ^/prefix(/.*)?$. Left alone, plugin routes would work in dev and
404 only behind Docker.

* fix: update stale SFTP transfer test for single file destination paths

The test still asserted transferToHost was called with the destination
directory "/srv". Commit 7727fa09 (#1304) deliberately changed a single
file transfer to send the full destination path including the filename,
so the correct expectation is "/srv/remote-1.txt". The code was right
and the test was left behind.

* feat: ship the ssh terminal as a first-party in-process plugin

Adds a hardcoded process:transport-owner tier so the terminal can own its
WebSocket server and ssh2 clients, which cannot cross the worker boundary.

* feat: ship docker as a first-party in-process plugin

Moves docker backend and frontend into plugins/docker/, alongside
ssh-terminal. Adds a host editor tab seam, a dashboard card registry,
and a build step to compile plugin backends into dist.

* feat: ship host metrics as a first-party in-process plugin

* fix: harden plugin auth and add per-request identity for ctx calls

Require auth on /plugin-api, gate plugin enable/grant/revoke routes
behind admin.plugins.manage, and thread the calling user through
worker ctx.hosts/ctx.ssh calls instead of always using the install
owner. Also adds the plugin permissions grant/revoke UI.

* feat: ship the AI assistant as a first-party in-process plugin

Moves src/backend/ai and its routes into plugins/ai, mounted through a
small /ai dispatcher instead of its own port. Moves the ai-enabled and
ai-private-endpoints settings routes out of user-settings-routes.ts
into the plugin too, so it has no leftover core dependents.

* fix: remove compiled plugin JS left behind in plugins/ after build

copy-bundled-plugins.cjs compiled and rewrote plugin backend TS in
place, copied it to dist/plugins, but never deleted the .js it had
just emitted next to the .ts sources. Those files shadowed the real
source for vitest/tsx and had import paths rewritten for the dist
layout, so tests failed with "Cannot find module" once a local build
had run. Affects docker, host-metrics and ai, not just this one.

* feat: add a permission-gated backend service registry for plugins

* feat: add a permission-gated UI action registry and action slots for plugins

Moves the terminal toolbar's hardwired AI button into a slot the ssh-terminal
plugin offers and the ai plugin fills, gated on ai.services.use.

* feat: add cross-plugin secret references gated by the provider's RBAC permission

* feat: move proxmox discovery/import/sync into a first-party plugin

* feat: move remote desktop (RDP/VNC/Telnet) into a first-party plugin

Relocates guacamole backend/frontend source and tests into
plugins/remote-desktop, following the ssh-terminal/docker pattern.

* feat: wire remote-desktop plugin into core and rename i18n keys

Repoints core consumers (dispatcher, collab, session-sharing, tab
utils) at the plugin, adds isTabTypeAvailable gating for rdp/vnc/
telnet, widens hostCapability to accept an array, and renames the
guacamole i18n namespace to remoteDesktop.

* feat: move fleets into a first-party plugin

* feat: move automations into a first-party plugin

* feat: remove legacy alert rules, superseded by automations

Alert rules already auto-migrate into automations on boot and the old
engine stands down after. Deletes the rule/firing tables, routes, UI,
and AI tools; keeps notification channels since automations still use
them. Also fixes automations' notify step, which still called the
deleted repository.

* feat: move network topology into a first-party plugin

* feat: move tailscale device discovery and settings into a plugin

Adds plugins/tailscale/ with a devices sidebar tab, admin settings
section, and re-auth dialog. Host Metrics tailscale manager card and
SSH re-auth banner parsing stay in core.

* feat: move workspaces into a first-party plugin

* feat: move web endpoint into a first-party plugin

* feat: collapse plugin runtime to one tier behind a plugin SDK

* feat: make each plugin an npm workspace built by a termix-plugin CLI

* feat: give plugins their own tables, migrations and sync entities

* feat: serve plugin http and websockets from one mount instead of ports

* feat: namespace plugin permissions and keep them valid when a plugin is off

* feat: give plugins their own settings and one settings screen

* feat: load plugin frontends at runtime and remove plugin ids from the shell

* fix: mock the ai status fetch in its activate test

* feat: add auth extension points and one ssh connect pipeline

* feat: rebuild workspaces as the reference plugin

* fix: gate second factors after SSO/LDAP behind an admin setting

* feat: convert network-topology to a full plugin (B1)

* feat: move snippets backend and tables into a plugin (B2)

* feat: point core frontend snippets pickers at the plugin (B2)

* feat: add the snippets plugin (B2)

* feat: wire ai, automations, fleets and ssh-terminal to snippets (B2)

* feat: finish the tailscale plugin, including SSH auth (B3)

* feat: finish the fleets plugin, add ctx.hosts and host-commands SDK (B4)

* chore: update proxmox and host-metrics locale files for B5

* chore: regenerate drizzle migration journals for B5

* feat: add ctx.hosts create/update to the SDK, host-metrics drops proxmox

* feat: move proxmox node stats and collectors into the proxmox plugin

* feat: drop proxmox host columns, use plugin settings instead

* feat: finish the fleets plugin, add ctx.hosts and host-commands SDK (B4)

* feat: convert file manager into a plugin

* feat: convert ssh tunnels into a plugin

* feat: finish the web-endpoint plugin, add ctx.desktop (B8)

* feat: move the terminal fully into the ssh-terminal plugin (B9)

* feat: convert tmux monitoring into a plugin (B10)

* feat: convert serial console into the serial plugin

* feat: move session sharing and collab rooms into a plugin (B12)

* feat: turn session recording into a plugin (B13)

* feat: finish the remote-desktop plugin (B14)

* feat: finish the host-metrics plugin, move host status into core (B16)

* feat: finish the docker plugin (B15)

* feat: finish the automations plugin (B17)

* feat: finish the ai plugin (B18)

* feat: finish the homepage plugin, move service links and canvas out of core (B19)

* feat: convert wake-on-lan into a plugin (B20)

* feat: convert secret-sources into a plugin (B20)

* feat: move totp and passkeys into plugins (C1)

* feat: move sso and ldap logins into plugins (C2)

* feat: move opkssh and warpgate into plugins (C3)

* feat: move step-ca into a plugin (C4)

* feat: move vault ssh signing into a plugin (C5)

* feat: move termix identity into a plugin (C6)

* feat: move acme certificate automation into a plugin (C7)

* fix: finish 2.9.0 plugin moves and get every check green

* chore: lock the plugin boundary and remove 2.9.0 leftovers

* fix: harden the plugin runtime after the 2.9.0 security review

* feat: add plugin packing, signing, pinned trust and tmxplug bundling

* test: prove 2.8 to 2.9.0 upgrade is lossless and fix what it caught

* fix: give bundled plugin backends a real require for CJS deps

* feat: improve UI loading time and jitter

* feat: show credential sharing inline in the sidebar like host sharing

* feat: tailscale add host and copy ip, fix outbound fetch hang and plugin ESM warning

* feat: unify alerts into one plugin, channels, rules and inbox

* feat: rebuild desktop sync as linked server accounts, fix packaged plugin bundles and UI scaling

* fix: set blacksmith builder cache-key to stop broken gha cache fallback

* fix: link plugin-sdk workspace before npm ci in docker deps stage

* fix: link every plugin workspace before npm ci in docker deps stage

* fix(host-metrics): exclude image and firmware filesystems (#1469)

* fix(host-metrics): exclude image and firmware filesystems

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(hosts): normalize imported jump host references (#1470)

* fix(hosts): normalize imported jump host references

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(ci): build plugin SDK and repair MySQL settings migration (#1471)

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(file-manager): keep inline rename consistent (#1476)

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(file-manager): keep inline rename consistent

* fix(dashboard): reuse translated host status labels (#1475)

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(dashboard): reuse translated host status labels

* docs(api): clarify local and remote version fields (#1474)

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* docs(api): clarify local and remote version fields

* fix(file-manager): preserve selected backgrounds on hover (#1473)

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(file-manager): preserve selection feedback on hover

* test(ui): await dialog focus cleanup before teardown

* fix(file-manager): restore Ctrl/Cmd+F file search (#1472)

* fix(ci): build plugin SDK before dependent checks

* fix(mysql): keep plugin settings scope indexes within limits

* test(mysql): refresh reviewed plugin upgrade SQL

* fix(file-manager): focus file search with Ctrl or Cmd F

* fix: keep rounded-full circular in production builds

* fix(file-manager): cancel uploads and stop remote writes (#1477)

* fix(file-manager): cancel uploads and stop remote writes

* test(css): exercise the configured PostCSS plugin chain

* fix(file-manager): apply compact density to directory tree (#1478)

* fix(file-manager): apply compact density to directory tree

* test(css): exercise the configured PostCSS plugin chain

* fix(file-manager): preserve font sizes in compact mode (#1480)

* fix(file-manager): preserve font sizes in compact mode

* test(css): exercise the configured PostCSS plugin chain

* fix(file-manager): prioritize file names in narrow lists (#1483)

* fix(file-manager): prioritize file names in narrow lists

* test(css): exercise the configured PostCSS plugin chain

* test(identity): preserve full Ed25519 public keys in fixtures

* test(remote-desktop): always mutate ciphertext in tamper test

* fix(remote-desktop): probe the saved host guacd endpoint (#1481)

* fix(remote-desktop): probe the saved host guacd endpoint

* test(css): exercise the configured PostCSS plugin chain

* test(identity): preserve full Ed25519 public keys in fixtures

* fix(homepage): check allowlisted private service reachability (#1479)

* fix(homepage): check allowlisted private service reachability

* test(css): exercise the configured PostCSS plugin chain

* fix(remote-desktop): stop unsolicited clipboard read prompts (#1482)

* fix(remote-desktop): read clipboard only on explicit paste

* test(css): exercise the configured PostCSS plugin chain

* test(identity): preserve full Ed25519 public keys in fixtures

* test(remote-desktop): always mutate ciphertext in tamper test

* chore: simplify plugin readmes, changelogs and manifest descriptions

* fix: keep plugins loaded after signing in from the login screen

* fix(file-manager): accept OS file drops over existing rows (#1484)

* fix(file-manager): accept OS file drops over existing rows

* test(file-manager): target the file label in drop tests

* test(css): exercise the configured PostCSS plugin chain

* test(identity): preserve full Ed25519 public keys in fixtures

* test(remote-desktop): always mutate ciphertext in tamper test

* fix(file-manager): pipeline SFTP download reads (#1485)

* feat: add plugin sdk build to beta release

* fix: make link-store secret check immune to random base64 matches

* fix(ldap): preserve the directory-provided bind DN (#1486)

* fix: compute beta changes across dev branches and move the beta tag

* chore: untrack plugin sdk ARCHITECTURE.md

* chore: remove references to plugin sdk ARCHITECTURE.md

* feat: merge host editor terminal tabs into one owned by ssh-terminal

* feat: add descriptions to plugin host enable toggles

* fix(file-manager): show download progress in file viewers (#1487)

* fix(electron): signal backend readiness over IPC (#1488)

* fix(split-screen): allow layout creation without randomUUID (#1490)

* fix(split-screen): allow layout creation without randomUUID

* test(credentials): avoid flaky Ed25519 key generation

* fix(sidebar): remeasure visible rows after density changes (#1489)

* fix(sidebar): remeasure visible rows after density changes

* test(credentials): avoid flaky Ed25519 key generation

* feat(split-screen): redesign split screen with free-form panes and pane picker

* feat(sidebar): add host click behavior setting, default to always opening a new tab

* feat(snippets): redesign panel to match the host list

* fix(session-recording): stop session logs panel from refetching in a loop

* feat(snippets): bring back sharing, target hosts and drag reorder

* feat(ssh-terminal): add toolbar position, start state, display and fade host settings

* feat(ssh-terminal): add host setting to turn off password prompt auto-fill

* fix(ai): stop streams getting cut off and keep tool steps across messages

* feat: improve quick connect system using plugin api

* fix: proxmox guest sync, startup snippet sync, drop dead cloudflare ssh transport

* feat: move terminal settings, look and ssh tools into ssh-terminal plugin

* feat: move snippets, keybinding actions and macros out of core into plugins

* feat: store rdp, vnc and telnet logins in a generic per-protocol table

* feat: type the plugin host record, move plugin strings out of core and add lint checks

* feat: replace core homepage widget registry with generic extension points

* feat: plugins only see their own host settings in the browser

* feat: harden plugin api for separate repos, signed updates and generic external login

* fix: keep plugin css below core utilities and version plugin assets by their css

* fix: load and save plugin host settings in the editor and tighten settings row spacing

* feat: redesign user host defaults system

* fix(remote-desktop): release held keys on focus loss (#1498)

* fix(session-sharing): replace local presentation after takeover (#1500)

* fix(file-manager): refresh disk usage with directory listing (#1503)

* fix(ssh-terminal): preserve input order during local echo reconciliation (#1505)

* feat(remote-desktop): expose display zoom controls for RDP (#1514)

* fix(file-manager): restore text selection and copying in Markdown previews (#1518)

* fix(sidebar): keep compact host addresses visible (#1524)

* fix(hosts): preserve plugin settings when reopening the editor (#1499)

* fix(ssh-terminal): close owner sockets when presentations unmount (#1501)

* fix(session-sharing): synchronize shared SSH terminal dimensions (#1502)

* fix(session-sharing): show participant presence in member terminals (#1504)

* fix(ai): sync provider configuration and keys across linked devices (#1506)

* feat(ssh-terminal): add a duplicate tab action (#1510)

* feat(file-manager): allow disabling move-to-trash confirmation (#1511)

* feat(file-manager): default to the remote login directory (#1512)

* feat(homepage): synchronize canvas layouts across devices (#1515)

* fix(shortcuts): prevent accidental double-Shift palette activation (#1519)

* feat(tmux-monitor): add a unified multi-host view (#1520)

* fix(electron): allow explicit quit with active connections (#1522)

* feat: reconnect all disconnected terminal tabs (#1523)

* fix(hosts): preserve recording and sharing settings in the dev editor (#1509)

* fix(database): preserve RDP domain during startup backfill (#1497)

* feat(sidebar): add keyboard type-ahead search for hosts (#1516)

* feat(automations): add per-host maintenance mode and recurring schedules (#1521)

* feat(tmux): add a per-host mouse toggle and preserve remote configuration (#1513)

* fix(session-sharing): use the linked server for desktop meetings and links (#1507)

* fix(tunnels): support TOTP for client tunnels and jump hosts (#1508)

* fix: type errors and lint failures from merged plugin prs

* feat(installer): make the Windows desktop shortcut optional (#1517)

* fix: ship the plugin sdk and serialport in desktop builds

* fix: prefer ipv4 when resolving ssh host names (#1365)

* fix: readable terminal selection with the light theme (#1319)

* fix: build desktop session share links on the linked server (#1350)

* fix: show meeting guests to members and hide the roster from guests (#1353)

* fix: reliable file manager saves with sudo prompt and no base64 guessing (#1335)

* fix: show disk usage for the mount being browsed (#1347)

* fix: load the linux tray icon reliably and log tray failures (#1332)

* fix: cover the gnome close path in the desktop quit test (#1332)

* fix: prompt for jump host TOTP in terminal, file manager and metrics (#1312)

* feat: optional auto reconnect for dropped ssh sessions (#1321)

* feat: tighter compact host rows with inline tags (#1325)

* feat: bring back the alert feed homepage widget

* feat: pick target terminals for snippets again

* feat: share button on connection tabs again

* feat: recent servers list when linking the desktop app

* feat: list the fixed app shortcuts in keyboard shortcuts

* fix: translate hardcoded toasts and drop em dashes from ui text

* fix: translate leftover hardcoded ui text

* fix: toolbar files button opens at the shell directory again

* fix: show the remote desktop note under connection origin again

* chore: openapi docs for host metrics manager routes

* chore: openapi docs for undocumented routes and fix two broken doc blocks

* fix: declare terminal.listSessions use for snippet targets

* revert: drop the alert feed widget, the notification center replaces it

* feat: restore the alert feed homepage widget

* fix: quick connect fills the host address from plugin auth editors like tailscale

* fix: docker container logs use the base background color

* chore: fix dependabot alerts and apply safe npm upgrades

* fix: proxmox stats tab reads its switch from plugin host settings

* fix: count plugin ssh logins toward host status and stop showing reachable as a warning

* fix: hide the built-in source pill in the host editor

* fix: host status shows online when the port answers and only warns on a failed login

* fix: drop the top border on the split layout menu

* fix: host defaults tabs sit where the host editor puts them and skip plugins with nothing to default

* fix: remote desktop auth method labels and notes for vnc and telnet

* fix: add description to the web endpoint enable switch

* fix: crowdin sync and pretranslate for plugin locales, run plugin tests on release

* feat: move telemetry out of core into a usage statistics plugin

* fix: recent activity card no longer cuts off host names early

* fix: remove duplicate top border on split screen menu

* fix: redesign session log player controls, fix tooltip overlap and drop dot separators

* fix: file manager and sftp text showing raw i18n keys

* fix: onboarding rerunning for existing users after upgrading

* fix: fleet detail buttons overflowing the sidebar

* fix: session log player scrolling off screen horizontally

* fix: network graph lag when adding hosts and on every render

* fix: maintenance index name, shared host read only view and badge polling

* fix: meeting tabs, workspace tab merging, rail panels as tabs and general ui lag

* chore: update release notes

* fix: remote desktop tabs reconnecting on every tab switch

* fix: build plugin sdk before linting in release workflow

* chore: lint, format, and bump version to 2.9.0

* chore: skip crowdin pre-translate in release for now

* chore: sync Crowdin translations for 2.9.0

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Koi <38512047+datlt4@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com>
This commit is contained in:
Luke GustafsonKoidependabot[bot]ZacharyZcR
 authored and GitHub committed 2026-10-01 16:20:16 -05:00
1 parent 49fa488093
commit 3643e7af97
4405 files changed
+1341026 -289722
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/backend/sync/client/electron-proxy.ts
+36
View File
@@ -0,0 +1,36 @@
import {
isElectronIpcAvailable,
requestFromElectronMain,
} from "../../utils/electron-ipc-bridge.js";
import type { SyncLink } from "./link-store.js";
/**
* Chromium's own requests to the linked server (the sign-in page, remote
* terminals, file transfers) cannot set headers themselves, so Electron's
* main process adds the proxy headers and answers basic auth for that one
* origin. Sent again whenever the link changes; null clears it.
*/
export async function applyElectronProxyConfig(
link:
| (Pick<
SyncLink,
"serverUrl" | "customHeaders" | "basicAuth" | "allowInvalidCertificate"
> & { sessionToken?: string | null; status?: string })
| null,
): Promise<void> {
if (!isElectronIpcAvailable()) return;
await requestFromElectronMain(
"sync-proxy-config",
link
? {
origin: new URL(link.serverUrl).origin,
serverUrl: link.serverUrl,
token:
link.status === "signed_out" ? null : (link.sessionToken ?? null),
headers: link.customHeaders ?? [],
basicAuth: link.basicAuth ?? null,
allowInvalidCertificate: !!link.allowInvalidCertificate,
}
: null,
);
}
src/backend/sync/client/engine.ts
+758
View File
@@ -0,0 +1,758 @@
/**
* The linked desktop's side of sync.
*
* A pass pushes local edits, then pulls what changed on the server after the
* saved cursor, then backfills any entity type this device has never pulled
* (first link, a feature switched on, a category turned back on). Local
* edits are found the same way the server finds changes: by comparing each
* row's hash with the one recorded at the last sync.
*/
import os from "os";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import {
getEntity,
listEntities,
onEntitiesChanged,
type RegisteredSyncEntity,
} from "../../plugins/sync-registry.js";
import { syncLogger } from "../../utils/logger.js";
import { sendCoreAlert } from "../../notify/core-notify.js";
import { registerCoreSyncEntities } from "../entities.js";
import {
createResolvers,
deleteStoredRow,
loadWireRow,
loadWireRows,
writeWireRow,
} from "../store.js";
import {
listRecords,
putRecord,
recordKey,
saveConflict,
setRecordError,
type SyncRecord,
} from "../records.js";
import { orderSelfReferences } from "../wire.js";
import { onLocalWrite } from "../server/change-watcher.js";
import { getLink, updateLink, type SyncLink } from "./link-store.js";
import { RemoteError, remoteFetch, remoteJson } from "./http.js";
import { mirrorPlugins } from "./plugins.js";
import { applyElectronProxyConfig } from "./electron-proxy.js";
export interface RemoteChange {
seq: number;
entityType: string;
syncId: string;
revision: number;
deleted: boolean;
row: SyncRow | null;
}
interface ChangesPage {
changes: RemoteChange[];
nextCursor: number;
hasMore: boolean;
entityTypes: Array<{
type: string;
owner: string;
order: number;
readOnly: boolean;
}>;
}
interface PushOp {
entityType: string;
syncId: string;
baseRevision: number;
op: "upsert" | "delete";
row?: SyncRow;
}
type PushResult =
| { syncId: string; entityType: string; status: "ok"; revision: number }
| {
syncId: string;
entityType: string;
status: "conflict";
revision: number;
deleted: boolean;
row: SyncRow | null;
}
| { syncId: string; entityType: string; status: "rejected"; reason: string };
const MAX_BATCH_OPS = 200;
const MAX_BATCH_BYTES = 1_000_000;
const MAX_PAGES = 500;
const SAFETY_INTERVAL_MS = 5 * 60_000;
const REFRESH_CHECK_MS = 24 * 60 * 60_000;
const DELETED_HASH = "deleted";
export interface EngineState {
running: boolean;
/** Local changes found in the last pass that the server has not taken. */
pending: number;
/** Entity types the server runs, from the last pull. */
serverTypes: Array<{ type: string; owner: string; readOnly: boolean }>;
lastPassAt: string | null;
}
/** An entity this device could sync: registered here, and on the server. */
function syncable(
entity: RegisteredSyncEntity,
link: SyncLink,
serverTypes: Set<string> | null,
): boolean {
if (link.disabledTypes.includes(entity.type)) return false;
if (serverTypes && !serverTypes.has(entity.type)) return false;
return true;
}
function sortChanges(changes: RemoteChange[]): RemoteChange[] {
const byType = new Map<string, RemoteChange[]>();
for (const change of changes) {
const list = byType.get(change.entityType) ?? [];
list.push(change);
byType.set(change.entityType, list);
}
const types = [...byType.keys()].sort(
(a, b) => (getEntity(a)?.order ?? 1e6) - (getEntity(b)?.order ?? 1e6),
);
const out: RemoteChange[] = [];
for (const type of types) {
const list = byType.get(type)!;
const entity = getEntity(type);
// Only the latest record of each row matters.
const latest = new Map<string, RemoteChange>();
for (const change of list) latest.set(change.syncId, change);
const live = [...latest.values()].filter((change) => !change.deleted);
const gone = [...latest.values()].filter((change) => change.deleted);
if (entity) {
const bySyncId = new Map(live.map((change) => [change.syncId, change]));
const ordered = orderSelfReferences(
entity,
live.map((change) => ({
...(change.row ?? {}),
syncId: change.syncId,
})),
);
out.push(...ordered.map((row) => bySyncId.get(row.syncId as string)!));
} else {
out.push(...live);
}
out.push(...gone);
}
return out;
}
export class SyncEngine {
private running = false;
private again = false;
private timer: NodeJS.Timeout | null = null;
private safety: NodeJS.Timeout | null = null;
private backoffMs = 0;
private events: AbortController | null = null;
private eventsFor: string | null = null;
private lastRefreshCheck = 0;
private serverTypes: Set<string> | null = null;
private stopFns: Array<() => void> = [];
private alerted = false;
readonly state: EngineState = {
running: false,
pending: 0,
serverTypes: [],
lastPassAt: null,
};
start(): void {
if (this.safety) return;
registerCoreSyncEntities();
this.safety = setInterval(() => this.request(0), SAFETY_INTERVAL_MS);
this.safety.unref?.();
this.stopFns.push(onLocalWrite(() => this.request(2000)));
// A plugin that came up may bring entity types to backfill.
this.stopFns.push(onEntitiesChanged(() => this.request(3000)));
this.request(3000);
}
stop(): void {
if (this.safety) clearInterval(this.safety);
if (this.timer) clearTimeout(this.timer);
this.safety = this.timer = null;
this.closeEvents();
for (const stop of this.stopFns.splice(0)) stop();
}
/** Runs a pass after `delayMs`, folding requests that arrive meanwhile. */
request(delayMs = 0): void {
if (this.timer) {
if (delayMs > 0) return;
clearTimeout(this.timer);
}
this.timer = setTimeout(() => {
this.timer = null;
void this.run();
}, delayMs);
this.timer.unref?.();
}
/** Runs a pass now and waits for it. */
async syncNow(): Promise<void> {
if (this.running) {
this.again = true;
while (this.running) await new Promise((r) => setTimeout(r, 100));
return;
}
await this.run();
}
/** Forgets what the server runs, after a relink or unlink. */
reset(): void {
this.serverTypes = null;
this.state.serverTypes = [];
this.state.pending = 0;
this.alerted = false;
this.backoffMs = 0;
this.closeEvents();
}
private async run(): Promise<void> {
if (this.running) {
this.again = true;
return;
}
this.running = this.state.running = true;
try {
await this.pass();
} finally {
this.running = this.state.running = false;
this.state.lastPassAt = new Date().toISOString();
if (this.again) {
this.again = false;
this.request(0);
}
}
}
private async pass(): Promise<void> {
const link = await getLink();
if (!link?.sessionToken) {
this.closeEvents();
return;
}
await updateLink({ status: "syncing" });
try {
await this.refreshSession(link);
const fresh = (await getLink())!;
await mirrorPlugins(fresh);
// A first pass pulls before it pushes, so rows both sides already
// have are matched up instead of sent back as edits.
const firstPass = fresh.knownTypes.length === 0;
if (!firstPass) {
await this.push(fresh);
await this.pull(fresh);
}
await this.backfill((await getLink())!);
if (firstPass) await this.push((await getLink())!);
this.backoffMs = 0;
this.alerted = false;
await updateLink({
status: "idle",
lastError: null,
lastSyncAt: new Date().toISOString(),
});
const done = (await getLink())!;
await applyElectronProxyConfig(done).catch(() => {});
this.openEvents(done);
} catch (error) {
await this.fail(error);
}
}
private async fail(error: unknown): Promise<void> {
const remote = error instanceof RemoteError ? error : null;
const message = error instanceof Error ? error.message : String(error);
syncLogger.warn("Sync pass failed", {
operation: "sync_pass",
kind: remote?.kind ?? "internal",
error: message,
});
if (remote?.kind === "signed_out") {
const link = await updateLink({
status: "signed_out",
lastError: message,
});
await applyElectronProxyConfig(link).catch(() => {});
this.closeEvents();
if (!this.alerted) {
this.alerted = true;
await sendCoreAlert({
title: "Sync is paused",
body: "This device was signed out of the server it syncs with. Sign in again from Sync to keep going.",
severity: "warning",
category: "termix.sync.signed_out",
dedupeKey: "sync:signed_out",
audience: "admins",
link: { tab: "sync" },
});
}
return;
}
const offline = remote?.kind === "offline";
await updateLink({
status: offline ? "offline" : "error",
lastError: message,
});
this.backoffMs = Math.min(
this.backoffMs ? this.backoffMs * 2 : 15_000,
10 * 60_000,
);
this.request(this.backoffMs);
}
private async refreshSession(link: SyncLink): Promise<void> {
if (Date.now() - this.lastRefreshCheck < REFRESH_CHECK_MS) return;
const result = await remoteJson<{ token: string | null }>(
link,
"/sync/v2/session/refresh",
{ method: "POST", body: {} },
);
this.lastRefreshCheck = Date.now();
if (result.token) await updateLink({ sessionToken: result.token });
}
/** Local rows the server does not have in this form yet. */
async outbox(link: SyncLink): Promise<{
ops: PushOp[];
hashes: Map<string, string>;
}> {
const records = await listRecords(link.userId);
const resolvers = createResolvers(link.userId);
const ops: PushOp[] = [];
const hashes = new Map<string, string>();
for (const entity of listEntities()) {
if (entity.readOnly) continue;
if (!link.knownTypes.includes(entity.type)) continue;
if (!syncable(entity, link, this.serverTypes)) continue;
const { rows, failed } = await loadWireRows(
entity,
link.userId,
resolvers,
);
for (const row of rows.values()) {
const key = recordKey(entity.type, row.syncId);
const record = records.get(key);
if (record && !record.deleted && record.hash === row.hash) continue;
if (record?.errorHash === row.hash) continue;
ops.push({
entityType: entity.type,
syncId: row.syncId,
baseRevision: record?.revision ?? 0,
op: "upsert",
row: row.wire,
});
hashes.set(key, row.hash);
}
for (const record of records.values()) {
if (record.entityType !== entity.type || record.deleted) continue;
if (record.revision === 0) continue;
if (rows.has(record.syncId) || failed.has(record.syncId)) continue;
if (entity.covers && !entity.covers(record.syncId)) continue;
if (record.errorHash === DELETED_HASH) continue;
ops.push({
entityType: entity.type,
syncId: record.syncId,
baseRevision: record.revision,
op: "delete",
});
}
}
return { ops, hashes };
}
private async push(link: SyncLink): Promise<void> {
const { ops, hashes } = await this.outbox(link);
this.state.pending = ops.length;
if (ops.length === 0) return;
for (const batch of batches(ops)) {
const { results } = await remoteJson<{ results: PushResult[] }>(
link,
"/sync/v2/push",
{ method: "POST", body: { ops: batch }, timeoutMs: 120_000 },
);
const byKey = new Map(
batch.map((op) => [recordKey(op.entityType, op.syncId), op]),
);
for (const result of results) {
const key = recordKey(result.entityType, result.syncId);
const op = byKey.get(key);
if (!op) continue;
await this.settle(link, op, result, hashes.get(key) ?? null);
}
this.state.pending = Math.max(0, this.state.pending - batch.length);
}
}
private async settle(
link: SyncLink,
op: PushOp,
result: PushResult,
hash: string | null,
): Promise<void> {
const userId = link.userId;
const entity = getEntity(op.entityType);
if (!entity) return;
if (result.status === "ok") {
await putRecord(userId, op.entityType, op.syncId, {
revision: result.revision,
hash: op.op === "delete" ? null : hash,
deleted: op.op === "delete",
});
return;
}
if (result.status === "rejected") {
await setRecordError(
userId,
op.entityType,
op.syncId,
result.reason,
hash ?? DELETED_HASH,
);
return;
}
// Conflict: the server moved on since this device last saw the row. Its
// version wins here, and a local edit it replaced is kept for the user.
if (op.op === "upsert" && op.row) {
await saveConflict(
userId,
op.entityType,
op.syncId,
op.row,
result.revision,
);
}
const resolvers = createResolvers(userId);
if (result.deleted || !result.row) {
await deleteStoredRow(entity, userId, op.syncId);
await putRecord(userId, op.entityType, op.syncId, {
revision: result.revision,
hash: null,
deleted: true,
});
return;
}
await writeWireRow(
entity,
userId,
{ ...result.row, syncId: op.syncId },
resolvers,
);
const written = await loadWireRow(entity, userId, op.syncId, resolvers);
await putRecord(userId, op.entityType, op.syncId, {
revision: result.revision,
hash: written?.hash ?? null,
deleted: false,
});
}
private async fetchChanges(
link: SyncLink,
cursor: number,
types?: string[],
): Promise<{ changes: RemoteChange[]; cursor: number }> {
const all: RemoteChange[] = [];
let next = cursor;
for (let page = 0; page < MAX_PAGES; page++) {
const query = new URLSearchParams({
cursor: String(next),
limit: "1000",
});
if (types?.length) query.set("types", types.join(","));
const result = await remoteJson<ChangesPage>(
link,
`/sync/v2/changes?${query.toString()}`,
{ timeoutMs: 120_000 },
);
all.push(...result.changes);
next = result.nextCursor;
this.serverTypes = new Set(result.entityTypes.map((entry) => entry.type));
this.state.serverTypes = result.entityTypes.map((entry) => ({
type: entry.type,
owner: entry.owner,
readOnly: entry.readOnly,
}));
if (!result.hasMore) break;
}
return { changes: all, cursor: next };
}
private async pull(link: SyncLink): Promise<void> {
const { changes, cursor } = await this.fetchChanges(link, link.cursor);
const known = new Set(link.knownTypes);
await this.apply(
link,
changes.filter(
(change) =>
known.has(change.entityType) &&
!link.disabledTypes.includes(change.entityType),
),
false,
);
await updateLink({ cursor });
}
/** Pulls every row of types this device has not pulled before. */
private async backfill(link: SyncLink): Promise<void> {
if (!this.serverTypes) {
await this.fetchChanges(link, Number.MAX_SAFE_INTEGER);
}
const fresh = listEntities()
.map((entity) => entity.type)
.filter(
(type) =>
!link.knownTypes.includes(type) &&
!link.disabledTypes.includes(type) &&
this.serverTypes?.has(type),
);
if (fresh.length === 0) return;
const { changes, cursor } = await this.fetchChanges(link, 0, fresh);
await this.apply(link, changes, true);
const current = (await getLink())!;
await updateLink({
knownTypes: [...new Set([...current.knownTypes, ...fresh])],
// On a first pass the backfill is the whole pull, so it sets the cursor.
...(link.knownTypes.length === 0 ? { cursor } : {}),
});
}
/**
* Writes pulled changes. `adopt` is for a first pull of a type: a local row
* with no record yet takes the server's version without a conflict.
*/
private async apply(
link: SyncLink,
changes: RemoteChange[],
adopt: boolean,
): Promise<void> {
if (changes.length === 0) return;
const userId = link.userId;
const records = await listRecords(userId);
let missed = false;
const resolvers = createResolvers(userId, () => {
missed = true;
});
const retry: RemoteChange[] = [];
for (const change of sortChanges(changes)) {
const entity = getEntity(change.entityType);
if (!entity) continue;
const key = recordKey(change.entityType, change.syncId);
const record = records.get(key);
if (record && record.revision >= change.revision) continue;
try {
if (!entity.readOnly) {
await this.keepLocalEdit(
entity,
userId,
change,
record,
adopt,
resolvers,
);
}
if (change.deleted) {
await deleteStoredRow(entity, userId, change.syncId);
await putRecord(userId, change.entityType, change.syncId, {
revision: change.revision,
hash: null,
deleted: true,
});
continue;
}
missed = false;
await writeWireRow(
entity,
userId,
{ ...(change.row ?? {}), syncId: change.syncId },
resolvers,
);
if (missed) retry.push(change);
await this.recordApplied(entity, userId, change, resolvers);
} catch (error) {
syncLogger.warn("Could not apply a synced change", {
operation: "sync_apply",
entityType: change.entityType,
syncId: change.syncId,
error: error instanceof Error ? error.message : String(error),
});
}
}
// Rows that pointed at something written later in the same pull.
for (const change of retry) {
const entity = getEntity(change.entityType)!;
try {
await writeWireRow(
entity,
userId,
{ ...(change.row ?? {}), syncId: change.syncId },
resolvers,
);
await this.recordApplied(entity, userId, change, resolvers);
} catch {
// The first write stands.
}
}
}
private async recordApplied(
entity: RegisteredSyncEntity,
userId: string,
change: RemoteChange,
resolvers: ReturnType<typeof createResolvers>,
): Promise<void> {
const written = entity.readOnly
? null
: await loadWireRow(entity, userId, change.syncId, resolvers);
await putRecord(userId, change.entityType, change.syncId, {
revision: change.revision,
hash: written?.hash ?? null,
deleted: false,
});
}
/** Saves a local edit the incoming change is about to replace. */
private async keepLocalEdit(
entity: RegisteredSyncEntity,
userId: string,
change: RemoteChange,
record: SyncRecord | undefined,
adopt: boolean,
resolvers: ReturnType<typeof createResolvers>,
): Promise<void> {
if (!record && adopt) return;
const local = await loadWireRow(
entity,
userId,
change.syncId,
resolvers,
).catch(() => null);
if (!local) return;
if (record && !record.deleted && record.hash === local.hash) return;
await saveConflict(
userId,
change.entityType,
change.syncId,
local.wire,
change.revision,
);
}
private openEvents(link: SyncLink): void {
const key = `${link.serverUrl}|${link.sessionToken}`;
if (this.events && this.eventsFor === key) return;
this.closeEvents();
const controller = new AbortController();
this.events = controller;
this.eventsFor = key;
void this.listen(link, controller);
}
private closeEvents(): void {
this.events?.abort();
this.events = null;
this.eventsFor = null;
}
private async listen(
link: SyncLink,
controller: AbortController,
): Promise<void> {
try {
const response = await remoteFetch(link, "/sync/v2/events", {
signal: controller.signal,
});
if (!response.ok || !response.body)
throw new Error(`HTTP ${response.status}`);
const decoder = new TextDecoder();
let buffer = "";
for await (const chunk of response.body) {
buffer += decoder.decode(chunk as Uint8Array, { stream: true });
let index;
while ((index = buffer.indexOf("\n\n")) >= 0) {
const block = buffer.slice(0, index);
buffer = buffer.slice(index + 2);
if (/^event: change$/m.test(block)) this.request(500);
}
}
} catch {
// Dropped or refused: the safety interval still syncs, and the next
// successful pass reopens the stream.
}
if (this.events === controller) {
this.events = null;
this.eventsFor = null;
if (!controller.signal.aborted) {
setTimeout(() => this.request(0), 30_000).unref?.();
}
}
}
/** Where the link's proxy settings have to reach outside the backend. */
async applyProxyConfig(): Promise<void> {
await applyElectronProxyConfig(await getLink());
}
}
function* batches(ops: PushOp[]): Generator<PushOp[]> {
let batch: PushOp[] = [];
let bytes = 0;
for (const op of ops) {
const size = JSON.stringify(op).length;
if (
batch.length > 0 &&
(batch.length >= MAX_BATCH_OPS || bytes + size > MAX_BATCH_BYTES)
) {
yield batch;
batch = [];
bytes = 0;
}
batch.push(op);
bytes += size;
}
if (batch.length > 0) yield batch;
}
let engine: SyncEngine | null = null;
export function getSyncEngine(): SyncEngine {
if (!engine) engine = new SyncEngine();
return engine;
}
/** Starts syncing on a desktop. A server install never links. */
export function startDesktopSync(): void {
if (process.env.ELECTRON_EMBEDDED !== "true") return;
const instance = getSyncEngine();
instance.start();
void instance.applyProxyConfig().catch(() => {});
}
export function defaultDeviceName(): string {
return os.hostname() || "Termix Desktop";
}
src/backend/sync/client/http.ts
+260
View File
@@ -0,0 +1,260 @@
/**
* Requests from a desktop to the server it is linked to.
*
* The Termix session travels as the jwt cookie rather than an Authorization
* header, so a proxy in front that uses basic auth can have that header to
* itself. Proxy headers (a Cloudflare Access service token, a gateway key)
* are added to every request.
*/
import { Agent, ProxyAgent, fetch as undiciFetch } from "undici";
import type { Dispatcher } from "undici";
import type { BasicAuth, ProxyHeader } from "./link-store.js";
export interface RemoteTarget {
serverUrl: string;
sessionToken?: string | null;
customHeaders?: ProxyHeader[];
basicAuth?: BasicAuth | null;
allowInvalidCertificate?: boolean;
}
export type RemoteErrorKind =
/** Could not reach the server at all. */
| "offline"
/** TLS failed and the certificate is not allowed. */
| "tls"
/** A proxy answered instead of Termix (its login page, a 404 page). */
| "proxy"
/** The proxy wants basic auth. */
| "basic_auth"
/** Termix refused the session: revoked, expired or signed out. */
| "signed_out"
/** Termix answered with an error. */
| "server";
export class RemoteError extends Error {
constructor(
readonly kind: RemoteErrorKind,
message: string,
readonly status?: number,
) {
super(message);
}
}
const TLS_CODES = new Set([
"DEPTH_ZERO_SELF_SIGNED_CERT",
"SELF_SIGNED_CERT_IN_CHAIN",
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
"CERT_HAS_EXPIRED",
"ERR_TLS_CERT_ALTNAME_INVALID",
"CERT_UNTRUSTED",
]);
function envProxy(url: string): string | null {
const proxy =
process.env.https_proxy ||
process.env.HTTPS_PROXY ||
process.env.http_proxy ||
process.env.HTTP_PROXY;
if (!proxy) return null;
const hostname = new URL(url).hostname.toLowerCase();
const noProxy = (process.env.no_proxy || process.env.NO_PROXY || "")
.split(",")
.map((entry) =>
entry
.trim()
.toLowerCase()
.replace(/^\*?\./, ""),
)
.filter(Boolean);
if (
noProxy.some(
(entry) => hostname === entry || hostname.endsWith(`.${entry}`),
)
)
return null;
return proxy;
}
const dispatchers = new Map<string, Dispatcher>();
function dispatcherFor(target: RemoteTarget): Dispatcher {
const insecure = !!target.allowInvalidCertificate;
const proxy = envProxy(target.serverUrl);
const key = `${insecure}|${proxy ?? ""}`;
let dispatcher = dispatchers.get(key);
if (!dispatcher) {
const tls = insecure ? { rejectUnauthorized: false } : {};
dispatcher = proxy
? new ProxyAgent({ uri: proxy, requestTls: tls })
: new Agent({ connect: tls });
dispatchers.set(key, dispatcher);
}
return dispatcher;
}
/** "https://x.example/termix/" + "/sync/v2/info" keeps the base path. */
export function remoteUrl(serverUrl: string, pathname: string): string {
return `${serverUrl.replace(/\/+$/, "")}${pathname}`;
}
export function normalizeServerUrl(input: string): string | null {
const trimmed = input.trim();
if (!trimmed) return null;
if (
/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) &&
!/^https?:\/\//i.test(trimmed)
) {
return null;
}
const withScheme = /^https?:\/\//i.test(trimmed)
? trimmed
: `https://${trimmed}`;
try {
const url = new URL(withScheme);
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
url.hash = "";
url.search = "";
return url.toString().replace(/\/+$/, "");
} catch {
return null;
}
}
export function requestHeaders(target: RemoteTarget): Record<string, string> {
const headers: Record<string, string> = {
"X-Electron-App": "true",
"User-Agent": "Termix-Desktop-Sync",
};
for (const header of target.customHeaders ?? []) {
if (header.name && header.value) headers[header.name] = header.value;
}
if (target.basicAuth?.username) {
headers.Authorization = `Basic ${Buffer.from(
`${target.basicAuth.username}:${target.basicAuth.password ?? ""}`,
).toString("base64")}`;
}
if (target.sessionToken) headers.Cookie = `jwt=${target.sessionToken}`;
return headers;
}
function describeNetworkError(error: unknown): RemoteError {
const cause = (error as { cause?: { code?: string; message?: string } })
?.cause;
const code = cause?.code ?? (error as { code?: string })?.code ?? "";
if (TLS_CODES.has(code) || /certificate/i.test(cause?.message ?? "")) {
return new RemoteError("tls", cause?.message || code);
}
if ((error as Error)?.name === "AbortError") {
return new RemoteError("offline", "The server did not answer in time");
}
return new RemoteError(
"offline",
cause?.message || (error as Error)?.message || "Could not reach the server",
);
}
export interface RemoteRequest {
method?: string;
body?: unknown;
timeoutMs?: number;
signal?: AbortSignal;
}
/** Sends a request and returns the raw response, classifying failures. */
export async function remoteFetch(
target: RemoteTarget,
pathname: string,
request: RemoteRequest = {},
) {
const url = remoteUrl(target.serverUrl, pathname);
const headers = requestHeaders(target);
if (request.body !== undefined) headers["Content-Type"] = "application/json";
const controller = new AbortController();
const timeout = request.timeoutMs
? setTimeout(() => controller.abort(), request.timeoutMs)
: null;
request.signal?.addEventListener("abort", () => controller.abort(), {
once: true,
});
let response;
try {
response = await undiciFetch(url, {
method: request.method ?? "GET",
headers,
body:
request.body !== undefined ? JSON.stringify(request.body) : undefined,
dispatcher: dispatcherFor(target),
signal: controller.signal,
});
} catch (error) {
throw describeNetworkError(error);
} finally {
if (timeout) clearTimeout(timeout);
}
return response;
}
/** Sends a request and parses a JSON answer from Termix. */
export async function remoteJson<T>(
target: RemoteTarget,
pathname: string,
request: RemoteRequest = {},
): Promise<T> {
const response = await remoteFetch(target, pathname, {
timeoutMs: 60_000,
...request,
});
const type = response.headers.get("content-type") ?? "";
const text = await response.text();
if (!type.includes("application/json")) {
if (
response.status === 401 &&
/basic/i.test(response.headers.get("www-authenticate") ?? "")
) {
throw new RemoteError(
"basic_auth",
"The server asks for a username and password before Termix",
401,
);
}
throw new RemoteError(
"proxy",
`Something other than Termix answered (${response.status})`,
response.status,
);
}
let data: unknown;
try {
data = text ? JSON.parse(text) : null;
} catch {
throw new RemoteError(
"proxy",
"The server sent invalid JSON",
response.status,
);
}
if (response.status === 401) {
throw new RemoteError(
"signed_out",
(data as { error?: string })?.error || "Signed out",
401,
);
}
if (!response.ok) {
throw new RemoteError(
"server",
(data as { error?: string })?.error || `Server error ${response.status}`,
response.status,
);
}
return data as T;
}
src/backend/sync/client/link-store.ts
+233
View File
@@ -0,0 +1,233 @@
/**
* The desktop's link to a server, one row in sync_link. Secrets (the session
* token, proxy headers, basic auth) are stored encrypted with the system key.
*/
import { eq } from "drizzle-orm";
import { syncLink } from "../../database/db/schema.js";
import { createCurrentRepositoryContext } from "../../database/repositories/factory.js";
import { DatabaseSaveTrigger } from "../../utils/database-save-trigger.js";
import {
decryptSystemSecret,
encryptSystemSecret,
} from "../../utils/system-secret-crypto.js";
export type LinkStatus =
"idle" | "syncing" | "offline" | "error" | "signed_out";
export interface LinkAccount {
remoteUserId?: string;
username?: string;
isAdmin?: boolean;
roles?: string[];
permissions?: string[];
}
export interface ProxyHeader {
name: string;
value: string;
}
export interface BasicAuth {
username: string;
password: string;
}
export interface SyncLink {
id: number;
userId: string;
serverUrl: string;
serverName: string | null;
serverVersion: string | null;
sessionToken: string | null;
customHeaders: ProxyHeader[];
basicAuth: BasicAuth | null;
allowInvalidCertificate: boolean;
remoteUserId: string | null;
remoteUsername: string | null;
account: LinkAccount | null;
/** Entity types switched off on this device. */
disabledTypes: string[];
/** Entity types this device has fully pulled at least once. */
knownTypes: string[];
cursor: number;
status: LinkStatus;
lastError: string | null;
linkedAt: string;
lastSyncAt: string | null;
}
type Row = typeof syncLink.$inferSelect;
function db() {
return createCurrentRepositoryContext().drizzle;
}
function parseJson<T>(value: string | null, fallback: T): T {
if (!value) return fallback;
try {
return JSON.parse(value) as T;
} catch {
return fallback;
}
}
async function decryptJson<T>(value: string | null, fallback: T): Promise<T> {
if (!value) return fallback;
try {
return JSON.parse(await decryptSystemSecret(value)) as T;
} catch {
return fallback;
}
}
async function fromRow(row: Row): Promise<SyncLink> {
let sessionToken: string | null = null;
if (row.sessionToken) {
try {
sessionToken = await decryptSystemSecret(row.sessionToken);
} catch {
sessionToken = null;
}
}
const scope = parseJson<{ disabled?: string[] }>(row.scope, {});
return {
id: row.id,
userId: row.userId,
serverUrl: row.serverUrl,
serverName: row.serverName,
serverVersion: row.serverVersion,
sessionToken,
customHeaders: await decryptJson<ProxyHeader[]>(row.customHeaders, []),
basicAuth: await decryptJson<BasicAuth | null>(row.basicAuth, null),
allowInvalidCertificate: !!row.allowInvalidCertificate,
remoteUserId: row.remoteUserId,
remoteUsername: row.remoteUsername,
account: parseJson<LinkAccount | null>(row.account, null),
disabledTypes: Array.isArray(scope.disabled) ? scope.disabled : [],
knownTypes: parseJson<string[]>(row.knownTypes, []),
cursor: Number(row.cursor) || 0,
status: (row.status as LinkStatus) || "idle",
lastError: row.lastError,
linkedAt: row.linkedAt,
lastSyncAt: row.lastSyncAt,
};
}
export async function getLink(): Promise<SyncLink | null> {
const [row] = await db().select().from(syncLink).limit(1);
return row ? fromRow(row) : null;
}
export interface NewLink {
userId: string;
serverUrl: string;
serverName?: string | null;
serverVersion?: string | null;
sessionToken: string;
customHeaders?: ProxyHeader[];
basicAuth?: BasicAuth | null;
allowInvalidCertificate?: boolean;
remoteUserId?: string | null;
remoteUsername?: string | null;
}
export async function createLink(link: NewLink): Promise<SyncLink> {
await db().delete(syncLink);
await db()
.insert(syncLink)
.values({
userId: link.userId,
serverUrl: link.serverUrl,
serverName: link.serverName ?? null,
serverVersion: link.serverVersion ?? null,
sessionToken: await encryptSystemSecret(link.sessionToken),
customHeaders: link.customHeaders?.length
? await encryptSystemSecret(JSON.stringify(link.customHeaders))
: null,
basicAuth: link.basicAuth
? await encryptSystemSecret(JSON.stringify(link.basicAuth))
: null,
allowInvalidCertificate: !!link.allowInvalidCertificate,
remoteUserId: link.remoteUserId ?? null,
remoteUsername: link.remoteUsername ?? null,
cursor: 0,
status: "idle",
knownTypes: "[]",
linkedAt: new Date().toISOString(),
});
await DatabaseSaveTrigger.forceSave("sync_link");
return (await getLink())!;
}
export interface LinkPatch {
serverName?: string | null;
serverVersion?: string | null;
sessionToken?: string | null;
customHeaders?: ProxyHeader[];
basicAuth?: BasicAuth | null;
allowInvalidCertificate?: boolean;
remoteUserId?: string | null;
remoteUsername?: string | null;
account?: LinkAccount | null;
disabledTypes?: string[];
knownTypes?: string[];
cursor?: number;
status?: LinkStatus;
lastError?: string | null;
lastSyncAt?: string | null;
}
export async function updateLink(patch: LinkPatch): Promise<SyncLink | null> {
const current = await getLink();
if (!current) return null;
const set: Partial<Row> = {};
if (patch.serverName !== undefined) set.serverName = patch.serverName;
if (patch.serverVersion !== undefined)
set.serverVersion = patch.serverVersion;
if (patch.sessionToken !== undefined) {
set.sessionToken = patch.sessionToken
? await encryptSystemSecret(patch.sessionToken)
: null;
}
if (patch.customHeaders !== undefined) {
set.customHeaders = patch.customHeaders.length
? await encryptSystemSecret(JSON.stringify(patch.customHeaders))
: null;
}
if (patch.basicAuth !== undefined) {
set.basicAuth = patch.basicAuth
? await encryptSystemSecret(JSON.stringify(patch.basicAuth))
: null;
}
if (patch.allowInvalidCertificate !== undefined) {
set.allowInvalidCertificate = patch.allowInvalidCertificate;
}
if (patch.remoteUserId !== undefined) set.remoteUserId = patch.remoteUserId;
if (patch.remoteUsername !== undefined) {
set.remoteUsername = patch.remoteUsername;
}
if (patch.account !== undefined) {
set.account = patch.account ? JSON.stringify(patch.account) : null;
}
if (patch.disabledTypes !== undefined) {
set.scope = JSON.stringify({ disabled: patch.disabledTypes });
}
if (patch.knownTypes !== undefined) {
set.knownTypes = JSON.stringify(patch.knownTypes);
}
if (patch.cursor !== undefined) set.cursor = patch.cursor;
if (patch.status !== undefined) set.status = patch.status;
if (patch.lastError !== undefined) set.lastError = patch.lastError;
if (patch.lastSyncAt !== undefined) set.lastSyncAt = patch.lastSyncAt;
if (Object.keys(set).length > 0) {
await db().update(syncLink).set(set).where(eq(syncLink.id, current.id));
DatabaseSaveTrigger.triggerSave("sync_link");
}
return getLink();
}
export async function deleteLink(): Promise<void> {
await db().delete(syncLink);
await DatabaseSaveTrigger.forceSave("sync_link");
}
src/backend/sync/client/plugins.ts
+245
View File
@@ -0,0 +1,245 @@
/**
* A linked desktop runs the features its server runs.
*
* For every plugin in "mirror" mode, the server decides: a plugin installed
* there is installed here, on or off to match, with the capabilities an
* admin granted it there. Plugins marked "local" are left to the desktop and
* "server" ones never run on a linked desktop.
*/
import fs from "fs";
import path from "path";
import semver from "semver";
import type { PluginDesktopMode } from "@termix/plugin-sdk/manifest";
import { syncLogger } from "../../utils/logger.js";
import { sendCoreAlert } from "../../notify/core-notify.js";
import {
getPluginRuntime,
installPluginArtifact,
setPluginEnabled,
unloadPlugin,
} from "../../plugins/index.js";
import { getPluginsDir } from "../../plugins/paths.js";
import {
createCurrentPluginPermissionGrantRepository,
createCurrentPluginRepository,
} from "../../database/repositories/factory.js";
import { remoteFetch, remoteJson } from "./http.js";
import type { SyncLink } from "./link-store.js";
export interface RemotePlugin {
id: string;
name: string;
version: string;
source: "bundled" | "user";
/** The server can send a signed .tmxplug for it. */
artifact?: boolean;
enabled: boolean;
active: boolean;
desktop: PluginDesktopMode;
syncEntities: string[];
granted: string[];
}
let lastRemote: RemotePlugin[] = [];
const alertedVersions = new Set<string>();
/** What the server ran at the last sync, for the sync panel. */
export function lastRemotePlugins(): RemotePlugin[] {
return lastRemote;
}
async function isEnabledLocally(pluginId: string): Promise<boolean> {
const record = await createCurrentPluginRepository().findById(pluginId);
return record?.state === "enabled";
}
const PLUGIN_ID_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
async function download(link: SyncLink, plugin: RemotePlugin): Promise<string> {
const response = await remoteFetch(
link,
`/sync/v2/plugins/${encodeURIComponent(plugin.id)}/package`,
{ timeoutMs: 120_000 },
);
if (!response.ok) {
throw new Error(`Could not download ${plugin.id} (${response.status})`);
}
const buffer = Buffer.from(await response.arrayBuffer());
const dir = getPluginsDir();
await fs.promises.mkdir(dir, { recursive: true });
const file = path.join(dir, `${plugin.id}.tmxplug`);
await fs.promises.writeFile(file, buffer);
const signature = response.headers.get("x-termix-signature");
const sigFile = `${file}.sig`;
if (signature) {
await fs.promises.writeFile(
sigFile,
Buffer.from(signature, "base64").toString("utf8"),
);
} else {
await fs.promises.rm(sigFile, { force: true });
}
return file;
}
async function matchGrants(plugin: RemotePlugin): Promise<void> {
const repository = createCurrentPluginPermissionGrantRepository();
const local = getPluginRuntime().loader.get(plugin.id);
if (!local || local.source !== "user") return;
// Grants are only copied for code a trusted key signed. Anything else the
// desktop's own admin has to review, as with any plugin dropped on disk.
if (!local.signedBy) return;
const declared = new Set(local.manifest.capabilities);
const wanted = new Set(plugin.granted.filter((cap) => declared.has(cap)));
const existing = await repository.listByPlugin(plugin.id);
for (const grant of existing) {
if (!wanted.has(grant.capability)) {
await repository.revoke(plugin.id, grant.capability);
}
}
const have = new Set(existing.map((grant) => grant.capability));
for (const capability of wanted) {
if (have.has(capability)) continue;
await repository.grant({
pluginId: plugin.id,
capability,
grantedBy: null,
source: "admin",
});
}
}
async function alertOlderDesktop(plugin: RemotePlugin): Promise<void> {
const key = `${plugin.id}@${plugin.version}`;
if (alertedVersions.has(key)) return;
alertedVersions.add(key);
await sendCoreAlert({
title: "Update the desktop app",
body: `The server runs a newer ${plugin.name} (${plugin.version}) than this app has. Update the app to get it.`,
severity: "info",
category: "termix.sync.desktop_outdated",
dedupeKey: `sync:outdated:${key}`,
audience: "admins",
});
}
async function alertUnsigned(plugin: RemotePlugin): Promise<void> {
const key = `unsigned:${plugin.id}@${plugin.version}`;
if (alertedVersions.has(key)) return;
alertedVersions.add(key);
await sendCoreAlert({
title: "Review a feature from your server",
body: `${plugin.name} (${plugin.version}) was copied from your server but is not signed, so it stays off here until you turn it on.`,
severity: "warning",
category: "termix.sync.unsigned_plugin",
dedupeKey: `sync:${key}`,
audience: "admins",
});
}
async function mirrorOne(link: SyncLink, plugin: RemotePlugin): Promise<void> {
// The id becomes a file name below, so it must be a real plugin id.
if (typeof plugin.id !== "string" || !PLUGIN_ID_PATTERN.test(plugin.id)) {
throw new Error("The server sent a plugin with an invalid id");
}
const { loader } = getPluginRuntime();
let local = loader.get(plugin.id);
const mode = local?.manifest.desktop ?? plugin.desktop ?? "mirror";
if (mode === "server") {
if (local && (await isEnabledLocally(plugin.id))) {
await setPluginEnabled(plugin.id, false);
}
return;
}
if (mode !== "mirror") return;
if (plugin.source === "bundled") {
const newer = !local || semver.gt(plugin.version, local.manifest.version);
if (newer && plugin.artifact && local?.state !== "activating") {
// A signed update of a bundled plugin. The loader checks the signature
// and that the version really is newer.
const wasEnabled = await isEnabledLocally(plugin.id);
if (local) await unloadPlugin(plugin.id);
try {
local = await installPluginArtifact(await download(link, plugin));
} catch (error) {
await alertOlderDesktop(plugin);
throw error;
}
if (wasEnabled) await setPluginEnabled(plugin.id, true);
} else if (newer) {
await alertOlderDesktop(plugin);
}
if (!local) return;
} else if (plugin.enabled || local) {
const outdated =
!local ||
local.source !== "user" ||
local.manifest.version !== plugin.version;
if (outdated && (!local || local.source === "user")) {
const wasEnabled = local ? await isEnabledLocally(plugin.id) : false;
if (local) await unloadPlugin(plugin.id);
const file = await download(link, plugin);
local = await installPluginArtifact(file);
if (wasEnabled && !plugin.enabled) {
await setPluginEnabled(plugin.id, false);
}
}
await matchGrants(plugin);
// Unsigned code is only ever switched off from here, never on.
if (!local.signedBy) {
const enabledHere = await isEnabledLocally(plugin.id);
if (!plugin.enabled && enabledHere) {
await setPluginEnabled(plugin.id, false);
} else if (plugin.enabled && !enabledHere) {
await alertUnsigned(plugin);
}
return;
}
}
if (!local) return;
if (plugin.enabled !== (await isEnabledLocally(plugin.id))) {
await setPluginEnabled(plugin.id, plugin.enabled);
} else if (plugin.enabled && local.state !== "active") {
await setPluginEnabled(plugin.id, true);
}
}
export async function mirrorPlugins(link: SyncLink): Promise<void> {
const { plugins } = await remoteJson<{ plugins: RemotePlugin[] }>(
link,
"/sync/v2/plugins",
);
lastRemote = plugins;
const remoteIds = new Set(plugins.map((plugin) => plugin.id));
for (const plugin of plugins) {
try {
await mirrorOne(link, plugin);
} catch (error) {
syncLogger.warn("Could not match a feature to the server", {
operation: "sync_plugin_mirror",
pluginId: plugin.id,
error: error instanceof Error ? error.message : String(error),
});
}
}
// A plugin the server no longer has is switched off here. Its data stays.
for (const local of getPluginRuntime().loader.list()) {
if (remoteIds.has(local.id) || local.source !== "user") continue;
if ((local.manifest.desktop ?? "mirror") !== "mirror") continue;
if (await isEnabledLocally(local.id)) {
await setPluginEnabled(local.id, false).catch(() => {});
}
}
}
/** Whether a plugin's on/off state belongs to the linked server. */
export function isServerManaged(pluginId: string): boolean {
const local = getPluginRuntime().loader.get(pluginId);
return (local?.manifest.desktop ?? "mirror") !== "local";
}
src/backend/sync/client/routes.ts
+761
View File
@@ -0,0 +1,761 @@
/**
* The desktop's own sync routes, under /sync/link. They only exist in the
* embedded desktop backend; on a server they answer 404.
*/
import express, {
type NextFunction,
type Request,
type Response,
} from "express";
import { and, eq, isNotNull } from "drizzle-orm";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { syncLogger } from "../../utils/logger.js";
import { hosts, sshCredentials } from "../../database/db/schema.js";
import { createCurrentRepositoryContext } from "../../database/repositories/factory.js";
import { getPluginRuntime } from "../../plugins/index.js";
import { getEntity, listEntities } from "../../plugins/sync-registry.js";
import { registerCoreSyncEntities } from "../entities.js";
import {
createResolvers,
deleteStoredRow,
listStoredRows,
regenerateSyncIds,
writeWireRow,
} from "../store.js";
import {
clearRecordErrors,
deleteConflict,
deleteRecordsForUser,
getConflict,
listConflicts,
listRecordErrors,
} from "../records.js";
import { SYNC_PROTOCOL_VERSION } from "../protocol.js";
import {
createLink,
deleteLink,
getLink,
updateLink,
type BasicAuth,
type ProxyHeader,
type SyncLink,
} from "./link-store.js";
import {
normalizeServerUrl,
RemoteError,
remoteJson,
type RemoteTarget,
} from "./http.js";
import { defaultDeviceName, getSyncEngine } from "./engine.js";
import { applyElectronProxyConfig } from "./electron-proxy.js";
import { lastRemotePlugins } from "./plugins.js";
const router = express.Router();
const authManager = AuthManager.getInstance();
const authenticateJWT = authManager.createAuthMiddleware();
function desktopOnly(_req: Request, res: Response, next: NextFunction) {
if (process.env.ELECTRON_EMBEDDED !== "true") {
res.status(404).json({ error: "Not found" });
return;
}
next();
}
const guard = [desktopOnly, authenticateJWT];
function userOf(req: Request): string {
return (req as AuthenticatedRequest).userId;
}
function readHeaders(value: unknown): ProxyHeader[] {
if (!Array.isArray(value)) return [];
return value
.filter(
(header): header is ProxyHeader =>
!!header &&
typeof header.name === "string" &&
typeof header.value === "string" &&
/^[A-Za-z0-9-]{1,100}$/.test(header.name.trim()),
)
.map((header) => ({ name: header.name.trim(), value: header.value }))
.slice(0, 20);
}
function readBasicAuth(value: unknown): BasicAuth | null {
if (!value || typeof value !== "object") return null;
const { username, password } = value as Record<string, unknown>;
if (typeof username !== "string" || !username) return null;
return {
username,
password: typeof password === "string" ? password : "",
};
}
function readTarget(body: Record<string, unknown>): RemoteTarget | null {
const serverUrl =
typeof body.serverUrl === "string"
? normalizeServerUrl(body.serverUrl)
: null;
if (!serverUrl) return null;
return {
serverUrl,
customHeaders: readHeaders(body.customHeaders),
basicAuth: readBasicAuth(body.basicAuth),
allowInvalidCertificate: body.allowInvalidCertificate === true,
};
}
/** What the sync panel can show about each entity type. */
function describeEntities(link: SyncLink | null) {
registerCoreSyncEntities();
const names = new Map<string, string>();
for (const plugin of getPluginRuntime().loader.list()) {
names.set(plugin.id, plugin.manifest.name);
}
for (const plugin of lastRemotePlugins()) {
if (!names.has(plugin.id)) names.set(plugin.id, plugin.name);
}
const serverTypes = getSyncEngine().state.serverTypes;
const seen = new Set<string>();
const entries = [
...listEntities().map((entity) => ({
type: entity.type,
owner: entity.owner,
readOnly: !!entity.readOnly,
})),
...serverTypes,
];
return entries
.filter((entry) => {
if (seen.has(entry.type) || entry.type === "accountProfile") return false;
seen.add(entry.type);
return true;
})
.map((entry) => ({
type: entry.type,
owner: entry.owner,
ownerName:
entry.owner === "core" ? null : (names.get(entry.owner) ?? entry.owner),
readOnly: entry.readOnly,
onServer: serverTypes.some((server) => server.type === entry.type),
onDevice: !!getEntity(entry.type),
enabled: !link?.disabledTypes.includes(entry.type),
}));
}
async function statusBody() {
const link = await getLink();
const engine = getSyncEngine();
if (!link) {
return { linked: false, entities: describeEntities(null) };
}
return {
linked: true,
serverUrl: link.serverUrl,
serverName: link.serverName,
serverVersion: link.serverVersion,
account: link.account ?? {
username: link.remoteUsername ?? undefined,
},
status: engine.state.running ? "syncing" : link.status,
lastError: link.lastError,
linkedAt: link.linkedAt,
lastSyncAt: link.lastSyncAt,
pending: engine.state.pending,
conflicts: (await listConflicts(link.userId)).length,
errors: (await listRecordErrors(link.userId)).length,
firstSyncDone: link.knownTypes.length > 0,
allowInvalidCertificate: link.allowInvalidCertificate,
customHeaders: link.customHeaders.map((header) => ({
name: header.name,
value: "",
set: true,
})),
basicAuth: link.basicAuth
? { username: link.basicAuth.username, password: "", set: true }
: null,
entities: describeEntities(link),
};
}
/** Removes read-only copies of what other users shared with the account. */
async function removeSharedCopies(userId: string): Promise<void> {
const db = createCurrentRepositoryContext().drizzle;
const sharedHosts = await db
.select({ id: hosts.id })
.from(hosts)
.where(and(eq(hosts.userId, userId), isNotNull(hosts.sharedSource)));
const { deleteOwnedHost } = await import("../../hosts/delete-host.js");
for (const host of sharedHosts) await deleteOwnedHost(userId, host.id);
const sharedCredentials = await db
.select({ id: sshCredentials.id })
.from(sshCredentials)
.where(
and(
eq(sshCredentials.userId, userId),
isNotNull(sshCredentials.sharedSource),
),
);
const { deleteOwnedCredential } =
await import("../../hosts/delete-credential.js");
for (const credential of sharedCredentials) {
await deleteOwnedCredential(userId, credential.id);
}
}
/** Deletes every row that syncs, keeping local-only ones. */
async function wipeSyncedData(userId: string): Promise<void> {
const entities = [...listEntities()].reverse();
for (const entity of entities) {
if (entity.readOnly) continue;
try {
if (entity.load) {
for (const row of await entity.load(userId)) {
await entity.erase?.(userId, String(row.syncId));
}
continue;
}
if (!entity.table || entity.singleton) continue;
for (const row of await listStoredRows(entity, userId)) {
if (entity.shouldSync && entity.shouldSync(row) === false) continue;
if (typeof row.syncId !== "string") continue;
await deleteStoredRow(entity, userId, row.syncId);
}
} catch (error) {
syncLogger.warn("Could not clear synced data", {
operation: "sync_wipe",
entityType: entity.type,
error: error instanceof Error ? error.message : String(error),
});
}
}
}
function problemFor(error: unknown) {
if (!(error instanceof RemoteError)) {
return { problem: "unreachable", message: String(error) };
}
switch (error.kind) {
case "tls":
return { problem: "tls_untrusted", message: error.message };
case "basic_auth":
return { problem: "basic_auth_required", message: error.message };
case "proxy":
return error.status === 404
? { problem: "not_termix", message: error.message }
: { problem: "proxy_login", message: error.message };
case "signed_out":
case "server":
// A Termix older than 2.9 has no /sync/v2 and answers with an error.
return { problem: "version_mismatch", message: error.message };
default:
return { problem: "unreachable", message: error.message };
}
}
/**
* @openapi
* /sync/link/status:
* get:
* summary: This desktop's link to a server
* description: Desktop only. Whether the app is linked, to which server and account, how syncing is going, and which kinds of data sync.
* tags:
* - Sync
* responses:
* 200:
* description: The link status.
*/
router.get("/link/status", ...guard, async (_req: Request, res: Response) => {
try {
res.json(await statusBody());
} catch (error) {
syncLogger.error("Failed to read sync status", error, {
operation: "sync_status",
});
res.status(500).json({ error: "Failed to read sync status" });
}
});
/**
* @openapi
* /sync/link/probe:
* post:
* summary: Check a server address before linking
* description: Desktop only. Tries the server with the given proxy settings and says what is in the way, if anything. On success the proxy settings are also applied to the desktop's browser session so the sign-in page loads.
* tags:
* - Sync
* responses:
* 200:
* description: The server's name and version, or the problem found.
*/
router.post("/link/probe", ...guard, async (req: Request, res: Response) => {
const target = readTarget(req.body ?? {});
if (!target) return res.json({ ok: false, problem: "invalid_url" });
try {
const info = await remoteJson<{
termix?: boolean;
name?: string;
version?: string;
protocol?: number;
}>(target, "/sync/v2/info", { timeoutMs: 15_000 });
if (!info?.termix) return res.json({ ok: false, problem: "not_termix" });
if (info.protocol !== SYNC_PROTOCOL_VERSION) {
return res.json({
ok: false,
problem: "version_mismatch",
version: info.version,
});
}
await applyElectronProxyConfig({
serverUrl: target.serverUrl,
customHeaders: target.customHeaders ?? [],
basicAuth: target.basicAuth ?? null,
allowInvalidCertificate: !!target.allowInvalidCertificate,
}).catch(() => {});
res.json({
ok: true,
serverUrl: target.serverUrl,
name: info.name ?? "Termix",
version: info.version ?? null,
});
} catch (error) {
res.json({ ok: false, ...problemFor(error) });
}
});
/**
* @openapi
* /sync/link/preview:
* get:
* summary: How much this desktop has that would sync
* description: Desktop only. Row counts per kind of data, shown before choosing whether to merge this desktop's data into the account or replace it.
* tags:
* - Sync
* responses:
* 200:
* description: Counts per entity type.
*/
router.get("/link/preview", ...guard, async (req: Request, res: Response) => {
const userId = userOf(req);
registerCoreSyncEntities();
const counts: Record<string, number> = {};
for (const entity of listEntities()) {
if (entity.readOnly) continue;
try {
if (entity.load) {
counts[entity.type] = (await entity.load(userId)).length;
continue;
}
if (!entity.table) continue;
counts[entity.type] = (await listStoredRows(entity, userId)).filter(
(row) => !entity.shouldSync || entity.shouldSync(row) !== false,
).length;
} catch {
counts[entity.type] = 0;
}
}
res.json({ counts });
});
/**
* @openapi
* /sync/link/complete:
* post:
* summary: Link this desktop to a server account
* description: Desktop only. Takes the session from signing in to the server, trades it for a long-lived desktop session, and starts syncing. "merge" uploads this desktop's data into the account; "replace" deletes it here first and takes the account's.
* tags:
* - Sync
* responses:
* 200:
* description: Linked; the new status.
* 400:
* description: Missing server or session.
* 502:
* description: The server refused the session.
*/
router.post("/link/complete", ...guard, async (req: Request, res: Response) => {
const userId = userOf(req);
const body = req.body ?? {};
const target = readTarget(body);
const token = typeof body.token === "string" ? body.token : "";
const mode = body.mode === "replace" ? "replace" : "merge";
if (!target || !token) {
return res.status(400).json({ error: "Missing server or session" });
}
try {
const deviceName =
typeof body.deviceName === "string" && body.deviceName.trim()
? body.deviceName.trim()
: defaultDeviceName();
const linked = await remoteJson<{
token: string;
user: { id: string; username: string };
server?: { version?: string | null };
}>({ ...target, sessionToken: token }, "/sync/v2/link", {
method: "POST",
body: { deviceName },
});
const previous = await getLink();
if (previous) {
await removeSharedCopies(previous.userId);
await deleteRecordsForUser(previous.userId);
}
if (mode === "replace") {
await wipeSyncedData(userId);
}
await deleteRecordsForUser(userId);
const link = await createLink({
userId,
serverUrl: target.serverUrl,
serverName: typeof body.serverName === "string" ? body.serverName : null,
serverVersion: linked.server?.version ?? null,
sessionToken: linked.token,
customHeaders: target.customHeaders,
basicAuth: target.basicAuth,
allowInvalidCertificate: target.allowInvalidCertificate,
remoteUserId: linked.user.id,
remoteUsername: linked.user.username,
});
const engine = getSyncEngine();
engine.reset();
await applyElectronProxyConfig(link).catch(() => {});
engine.request(0);
res.json(await statusBody());
} catch (error) {
syncLogger.warn("Linking to a server failed", {
operation: "sync_link_complete",
error: error instanceof Error ? error.message : String(error),
});
res.status(502).json({ ...problemFor(error) });
}
});
/**
* @openapi
* /sync/link/relogin:
* post:
* summary: Sign this desktop back in after its session ended
* description: Desktop only. Takes a new sign-in session for the same account and resumes syncing where it stopped.
* tags:
* - Sync
* responses:
* 200:
* description: Signed in again; the new status.
* 409:
* description: The sign-in was for a different account.
*/
router.post("/link/relogin", ...guard, async (req: Request, res: Response) => {
const token = typeof req.body?.token === "string" ? req.body.token : "";
const link = await getLink();
if (!link || !token) return res.status(400).json({ error: "Not linked" });
try {
const linked = await remoteJson<{
token: string;
user: { id: string; username: string };
}>({ ...link, sessionToken: token }, "/sync/v2/link", {
method: "POST",
body: { deviceName: defaultDeviceName() },
});
if (link.remoteUserId && linked.user.id !== link.remoteUserId) {
return res.status(409).json({ error: "different_account" });
}
await updateLink({
sessionToken: linked.token,
status: "idle",
lastError: null,
});
getSyncEngine().request(0);
res.json(await statusBody());
} catch (error) {
res.status(502).json({ ...problemFor(error) });
}
});
/**
* @openapi
* /sync/link/unlink:
* post:
* summary: Unlink this desktop from its server
* description: Desktop only. Ends the desktop's session on the server when it can be reached. With keepData the synced data stays on this device as its own; otherwise it is deleted. Copies of what others shared with the account are always removed.
* tags:
* - Sync
* responses:
* 200:
* description: Unlinked.
*/
router.post("/link/unlink", ...guard, async (req: Request, res: Response) => {
const keepData = req.body?.keepData !== false;
const link = await getLink();
if (!link) return res.json(await statusBody());
const userId = link.userId;
try {
await remoteJson(link, "/sync/v2/unlink", {
method: "POST",
body: {},
timeoutMs: 10_000,
}).catch(() => {});
const engine = getSyncEngine();
engine.reset();
await removeSharedCopies(userId);
if (keepData) {
// New syncIds so this data reads as new if it is ever linked again.
for (const entity of listEntities()) {
if (!entity.readOnly) await regenerateSyncIds(entity, userId);
}
} else {
await wipeSyncedData(userId);
}
await deleteRecordsForUser(userId);
await deleteLink();
await applyElectronProxyConfig(null).catch(() => {});
res.json(await statusBody());
} catch (error) {
syncLogger.error("Unlinking failed", error, { operation: "sync_unlink" });
res.status(500).json({ error: "Failed to unlink" });
}
});
/**
* @openapi
* /sync/link/sync:
* post:
* summary: Sync now
* description: Desktop only. Runs a sync pass and waits for it.
* tags:
* - Sync
* responses:
* 200:
* description: The status after the pass.
*/
router.post("/link/sync", ...guard, async (_req: Request, res: Response) => {
await getSyncEngine().syncNow();
res.json(await statusBody());
});
/**
* @openapi
* /sync/link/settings:
* put:
* summary: Change what syncs and how the server is reached
* description: Desktop only. disabledTypes lists the kinds of data that stay on this device. Proxy headers and basic auth left blank keep their saved values.
* tags:
* - Sync
* responses:
* 200:
* description: The new status.
* 400:
* description: Not linked.
*/
router.put("/link/settings", ...guard, async (req: Request, res: Response) => {
const link = await getLink();
if (!link) return res.status(400).json({ error: "Not linked" });
const body = req.body ?? {};
const patch: Parameters<typeof updateLink>[0] = {};
if (Array.isArray(body.disabledTypes)) {
const disabled = body.disabledTypes.filter(
(type: unknown): type is string =>
typeof type === "string" && type !== "accountProfile",
);
const changed = new Set([
...disabled.filter((type: string) => !link.disabledTypes.includes(type)),
...link.disabledTypes.filter((type) => !disabled.includes(type)),
]);
patch.disabledTypes = disabled;
// A type switched back on is pulled again in full.
patch.knownTypes = link.knownTypes.filter((type) => !changed.has(type));
}
if (Array.isArray(body.customHeaders)) {
const saved = new Map(link.customHeaders.map((h) => [h.name, h.value]));
patch.customHeaders = (body.customHeaders as unknown[])
.filter(
(header): header is { name: string; value?: string } =>
!!header && typeof (header as { name?: unknown }).name === "string",
)
.map((header) => ({
name: header.name,
value: header.value || saved.get(header.name) || "",
}))
.filter((header) => header.value);
patch.customHeaders = readHeaders(patch.customHeaders);
}
if (body.basicAuth === null) {
patch.basicAuth = null;
} else if (body.basicAuth && typeof body.basicAuth === "object") {
const next = readBasicAuth(body.basicAuth);
if (next && !next.password && link.basicAuth) {
next.password = link.basicAuth.password;
}
patch.basicAuth = next;
}
if (typeof body.allowInvalidCertificate === "boolean") {
patch.allowInvalidCertificate = body.allowInvalidCertificate;
}
const updated = await updateLink(patch);
await applyElectronProxyConfig(updated).catch(() => {});
getSyncEngine().request(0);
res.json(await statusBody());
});
/**
* @openapi
* /sync/link/conflicts:
* get:
* summary: Local edits that lost to a newer edit on the server
* description: Desktop only. The server's version was kept; each entry holds this device's version so it can be restored.
* tags:
* - Sync
* responses:
* 200:
* description: The conflicts, oldest first.
*/
router.get("/link/conflicts", ...guard, async (req: Request, res: Response) => {
const conflicts = await listConflicts(userOf(req));
res.json({
conflicts: conflicts.map((conflict) => {
let row: SyncRow = {};
try {
row = JSON.parse(conflict.localRow);
} catch {
row = {};
}
return {
id: conflict.id,
entityType: conflict.entityType,
syncId: conflict.syncId,
name: row.name ?? row.title ?? row.label ?? row.ip ?? conflict.syncId,
createdAt: conflict.createdAt,
};
}),
});
});
/**
* @openapi
* /sync/link/conflicts/{id}:
* post:
* summary: Settle a conflict
* description: Desktop only. keep "mine" puts this device's version back, and the next sync sends it to the server. keep "server" drops it.
* tags:
* - Sync
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* responses:
* 200:
* description: Settled.
* 404:
* description: No such conflict.
*/
router.post(
"/link/conflicts/:id",
...guard,
async (req: Request, res: Response) => {
const userId = userOf(req);
const conflict = await getConflict(userId, Number(req.params.id));
if (!conflict) return res.status(404).json({ error: "Not found" });
try {
if (req.body?.keep === "mine") {
const entity = getEntity(conflict.entityType);
if (entity && !entity.readOnly) {
const row = JSON.parse(conflict.localRow) as SyncRow;
await writeWireRow(
entity,
userId,
{ ...row, syncId: conflict.syncId },
createResolvers(userId),
);
}
}
await deleteConflict(userId, conflict.id);
getSyncEngine().request(0);
res.json({ success: true });
} catch (error) {
syncLogger.warn("Could not settle a sync conflict", {
operation: "sync_conflict",
error: error instanceof Error ? error.message : String(error),
});
res.status(500).json({ error: "Failed to settle conflict" });
}
},
);
/**
* @openapi
* /sync/link/errors:
* get:
* summary: Changes the server refused
* description: Desktop only. Each entry names the record and why the server refused it. A refused change is not sent again until it changes or is retried.
* tags:
* - Sync
* responses:
* 200:
* description: The refused changes.
*/
router.get("/link/errors", ...guard, async (req: Request, res: Response) => {
const errors = await listRecordErrors(userOf(req));
res.json({
errors: errors.map((record) => ({
entityType: record.entityType,
syncId: record.syncId,
reason: record.error,
})),
});
});
/**
* @openapi
* /sync/link/errors/retry:
* post:
* summary: Send refused changes again
* description: Desktop only. Clears every refusal and syncs.
* tags:
* - Sync
* responses:
* 200:
* description: The status after the pass.
*/
router.post(
"/link/errors/retry",
...guard,
async (req: Request, res: Response) => {
await clearRecordErrors(userOf(req));
await getSyncEngine().syncNow();
res.json(await statusBody());
},
);
/**
* @openapi
* /sync/link/session:
* get:
* summary: The linked server and session, for connecting to it directly
* description: Desktop only. Used when a host is set to connect through the server rather than from this device.
* tags:
* - Sync
* responses:
* 200:
* description: The server address and session token, or nulls when not linked.
*/
router.get("/link/session", ...guard, async (_req: Request, res: Response) => {
const link = await getLink();
if (!link?.sessionToken || link.status === "signed_out") {
return res.json({ serverUrl: null, token: null });
}
res.json({ serverUrl: link.serverUrl, token: link.sessionToken });
});
export default router;
src/backend/sync/entities.ts
+670
View File
@@ -0,0 +1,670 @@
/**
* Core's own sync entities, registered the same way a plugin registers one.
*
* Order values leave room between them (10, 30, 50...) so a plugin can slot
* an entity in without renumbering; anything referenced syncs first.
*/
import { and, eq } from "drizzle-orm";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
import {
hosts,
sshCredentials,
sshFolders,
userPreferences,
} from "../database/db/schema.js";
import {
CORE_OWNER,
listEntities,
registerEntity,
} from "../plugins/sync-registry.js";
import { FieldCrypto } from "../utils/field-crypto.js";
import { syncLogger } from "../utils/logger.js";
import { getPluginRuntime } from "../plugins/index.js";
import { singletonSyncId } from "./wire.js";
import { parseDefaultOverrides } from "../../types/host-defaults.js";
import {
exportHostPluginSettings,
importHostPluginSettings,
} from "./host-plugin-settings.js";
import {
exportProtocolLogins,
importProtocolLogins,
} from "./host-protocol-auth.js";
import {
coversHostDefault,
eraseHostDefault,
loadAdminDefaults,
loadHostDefaults,
writeAdminDefaults,
writeHostDefault,
} from "./host-defaults.js";
const CREDENTIAL_REFERENCE = {
field: "credentialId",
syncField: "credentialSyncId",
entityType: "sshCredentials",
} as const;
const HOST_REFERENCES = [
CREDENTIAL_REFERENCE,
{
field: "parentHostId",
syncField: "parentHostSyncId",
entityType: "hosts",
},
{ field: "jumpHosts[].hostId", entityType: "hosts" },
] as const;
/** Columns that only mean something on the device that holds them. */
const HOST_LOCAL_FIELDS = [
"connectionOrigin",
"localOnly",
"sharedSource",
"hostKeyLastVerified",
];
/**
* Columns kept until 3.0.0 that nothing reads any more: their values moved
* into plugin host settings, which sync with the host's pluginSettings. They
* stay off the wire so a stale local id never reaches the other side.
*/
const HOST_RETIRED_FIELDS = ["quickActions"];
let registered = false;
/** Registers core's entities. Idempotent. */
export function registerCoreSyncEntities(): void {
if (
registered &&
listEntities().some((entity) => entity.owner === CORE_OWNER)
)
return;
registered = true;
registerEntity(
CORE_OWNER,
{
type: "sshCredentials",
table: sshCredentials,
order: 10,
encryptedFields: FieldCrypto.fieldsFor("ssh_credentials"),
readOnlyFields: ["usageCount", "lastUsed", "sharedSource"],
shouldSync: (row) => !row.sharedSource,
permissions: {
create: "credentials.create",
update: "credentials.edit",
delete: "credentials.delete",
},
afterWrite: async ({ id, userId, created }) => {
if (id === null || created) return;
const { SharedHostSecretsManager } =
await import("../utils/shared-host-secrets-manager.js");
await SharedHostSecretsManager.getInstance().resyncHostsForCredential(
id,
userId,
);
const { SharedCredentialSecretsManager } =
await import("../utils/shared-credential-secrets-manager.js");
await SharedCredentialSecretsManager.getInstance().resyncCredential(
id,
userId,
);
},
remove: async (row, userId) => {
const { deleteOwnedCredential } =
await import("../hosts/delete-credential.js");
await deleteOwnedCredential(userId, row.id as number);
},
},
{
// A credential shared with the user can back their own host.
canReference: async (userId, _ownerId, id) => {
const { findUsableCredential } =
await import("../hosts/usable-credential.js");
return (await findUsableCredential(id, userId)) !== null;
},
},
);
// vaultProfiles (order 20) belongs to the vault plugin.
registerEntity(CORE_OWNER, {
type: "sshFolders",
table: sshFolders,
order: 30,
references: [CREDENTIAL_REFERENCE],
readOnlyFields: ["localOnly"],
shouldSync: (row) => !row.localOnly,
permissions: {
create: "hosts.create",
update: "hosts.edit",
delete: "hosts.delete",
},
});
registerEntity(
CORE_OWNER,
{
type: "hosts",
table: hosts,
order: 50,
encryptedFields: FieldCrypto.fieldsFor("ssh_data"),
readOnlyFields: [...HOST_LOCAL_FIELDS, ...HOST_RETIRED_FIELDS],
references: HOST_REFERENCES,
shouldSync: (row) => !row.localOnly && !row.sharedSource,
permissions: {
create: "hosts.create",
update: "hosts.edit",
delete: "hosts.delete",
},
serialize: async (row, resolveSyncId) => ({
...row,
pluginSettings:
typeof row.id === "number"
? await exportHostPluginSettings(row.id)
: {},
protocolAuth:
typeof row.id === "number"
? await exportProtocolLogins(
row.id,
String(row.userId),
resolveSyncId,
)
: {},
}),
afterWrite: async ({ id, userId, wire, resolveId }) => {
if (id === null) return;
// The server owns host defaults: it keeps what the host set itself
// and resolves the rest. A desktop takes the server's values as sent.
const { getLink } = await import("./client/link-store.js");
const onServer = !(await getLink().catch(() => null));
await importHostPluginSettings(
id,
wire.pluginSettings,
onServer ? parseDefaultOverrides(wire.defaultOverrides) : null,
);
await importProtocolLogins(id, userId, wire.protocolAuth, resolveId);
await dropInvalidParent(userId, id);
if (onServer) {
const { applyDefaultsAfterHostWrite } =
await import("../hosts/defaults/index.js");
await applyDefaultsAfterHostWrite(id);
}
const { SharedHostSecretsManager } =
await import("../utils/shared-host-secrets-manager.js");
await SharedHostSecretsManager.getInstance().resyncHost(id);
},
remove: async (row, userId) => {
const { deleteOwnedHost } = await import("../hosts/delete-host.js");
await deleteOwnedHost(userId, row.id as number);
},
},
{
canReference: async (userId, _ownerId, id) => {
const { PermissionManager } =
await import("../utils/permission-manager.js");
const access = await PermissionManager.getInstance().canAccessHost(
userId,
id,
"view",
);
return access.hasAccess;
},
},
);
// After hosts: a folder's jump chain names hosts by syncId.
registerEntity(
CORE_OWNER,
{ type: "hostDefaults", table: null, order: 60 },
{
load: loadHostDefaults,
write: writeHostDefault,
erase: eraseHostDefault,
covers: (syncId) => coversHostDefault(syncId, activeManifest),
},
);
registerEntity(
CORE_OWNER,
{ type: "hostDefaultsAdmin", table: null, order: 7, singleton: true },
{
readOnly: true,
load: loadAdminDefaults,
write: writeAdminDefaults,
erase: async () => {},
},
);
// dashboardServiceLinks (70) and homepageItems (80) belong to homepage.
registerEntity(CORE_OWNER, {
type: "userPreferences",
table: userPreferences,
order: 90,
singleton: true,
readOnlyFields: ["storageMode"],
// A keybinding parameter a plugin declares with a syncEntity holds a
// local row id of that entity; the wire carries its syncId.
serialize: (row, resolveSyncId) =>
mapKeybindingReferences(row, async (entityType, value) => {
const id = Number(value);
return Number.isInteger(id) ? resolveSyncId(entityType, id) : null;
}),
deserialize: (row, resolveId) =>
mapKeybindingReferences(row, async (entityType, value) => {
const id = await resolveId(entityType, value);
return id === null ? null : String(id);
}),
});
registerEntity(
CORE_OWNER,
{ type: "pluginUserSettings", table: null, order: 95 },
{
load: loadPluginUserSettings,
write: writePluginUserSetting,
erase: erasePluginUserSetting,
covers: (syncId) => !!activeManifest(syncId.split(":")[0]),
},
);
registerEntity(
CORE_OWNER,
{ type: "accountProfile", table: null, order: 5, singleton: true },
{
readOnly: true,
load: loadAccountProfile,
write: writeAccountProfile,
erase: async () => {},
},
);
registerEntity(
CORE_OWNER,
{ type: "sharedCredentials", table: null, order: 15 },
{
readOnly: true,
load: loadSharedCredentials,
write: (userId, wire) => writeSharedCopy("sshCredentials", userId, wire),
erase: (userId, syncId) =>
eraseSharedCopy("sshCredentials", userId, syncId),
},
);
registerEntity(
CORE_OWNER,
{ type: "sharedHosts", table: null, order: 55 },
{
readOnly: true,
load: loadSharedHosts,
write: (userId, wire) => writeSharedCopy("hosts", userId, wire),
erase: (userId, syncId) => eraseSharedCopy("hosts", userId, syncId),
},
);
}
/**
* Rewrites every keybinding parameter whose declaration names a sync entity,
* keeping the column's storage form. A value that does not map becomes null.
*/
export async function mapKeybindingReferences(
row: SyncRow,
map: (entityType: string, value: string) => Promise<string | null>,
): Promise<SyncRow> {
const stored = row.customKeybindings;
if (typeof stored !== "string" || !stored) return row;
let bindings: unknown;
try {
bindings = JSON.parse(stored);
} catch {
return row;
}
if (!Array.isArray(bindings)) return row;
const { findKeybindingAction } =
await import("../database/routes/keybinding-validation.js");
let changed = false;
for (const binding of bindings) {
const action = (binding as { action?: Record<string, unknown> } | null)
?.action;
if (!action || typeof action.type !== "string") continue;
const params = findKeybindingAction(action.type)?.params ?? {};
for (const [name, param] of Object.entries(params)) {
const value = action[name];
if (!param.syncEntity || typeof value !== "string" || !value) continue;
action[name] = await map(param.syncEntity, value);
changed = true;
}
}
return changed
? { ...row, customKeybindings: JSON.stringify(bindings) }
: row;
}
/** A synced parent link that would make a cycle here is dropped. */
async function dropInvalidParent(userId: string, hostId: number) {
const { createCurrentRepositoryContext } =
await import("../database/repositories/factory.js");
const db = createCurrentRepositoryContext().drizzle;
const [row] = await db
.select({ parentHostId: hosts.parentHostId })
.from(hosts)
.where(and(eq(hosts.id, hostId), eq(hosts.userId, userId)))
.limit(1);
if (typeof row?.parentHostId !== "number") return;
const { validateParentHostId } =
await import("../database/routes/host-parent-validation.js");
if (await validateParentHostId(userId, hostId, row.parentHostId)) {
await db
.update(hosts)
.set({ parentHostId: null })
.where(and(eq(hosts.id, hostId), eq(hosts.userId, userId)));
}
}
function activeManifests(): PluginManifest[] {
try {
return getPluginRuntime()
.loader.list()
.filter((plugin) => plugin.state === "active" && plugin.manifest)
.map((plugin) => plugin.manifest as PluginManifest);
} catch {
return [];
}
}
function activeManifest(pluginId: string): PluginManifest | undefined {
return activeManifests().find((manifest) => manifest.id === pluginId);
}
async function loadPluginUserSettings(userId: string): Promise<SyncRow[]> {
const { declaredFields, getSetting } = await import("../plugins/settings.js");
const { createCurrentPluginSettingsRepository } =
await import("../database/repositories/factory.js");
const repository = createCurrentPluginSettingsRepository();
const rows: SyncRow[] = [];
for (const manifest of activeManifests()) {
const fields = declaredFields(manifest, "user");
if (fields.length === 0) continue;
const stored = new Set(
(await repository.getAll(manifest.id, "user", userId))
.filter((row) => row.value !== null)
.map((row) => row.key),
);
for (const field of fields) {
if (!stored.has(field.key)) continue;
rows.push({
syncId: `${manifest.id}:${field.key}`,
pluginId: manifest.id,
key: field.key,
value: await getSetting(manifest, "user", userId, field.key),
});
}
}
return rows;
}
async function writePluginUserSetting(userId: string, wire: SyncRow) {
const manifest = activeManifest(String(wire.pluginId));
if (!manifest) return;
const { setSetting } = await import("../plugins/settings.js");
const error = await setSetting(
manifest,
"user",
userId,
String(wire.key),
wire.value,
);
if (error) throw new Error(error);
}
async function erasePluginUserSetting(userId: string, syncId: string) {
const [pluginId, ...rest] = syncId.split(":");
const { createCurrentPluginSettingsRepository } =
await import("../database/repositories/factory.js");
await createCurrentPluginSettingsRepository().delete(
pluginId,
"user",
userId,
rest.join(":"),
);
}
async function loadAccountProfile(userId: string): Promise<SyncRow[]> {
const { createCurrentUserRepository, createCurrentRoleRepository } =
await import("../database/repositories/factory.js");
const user = await createCurrentUserRepository().findById(userId);
if (!user) return [];
const { PermissionManager } = await import("../utils/permission-manager.js");
const roles = await createCurrentRoleRepository().listUserRoles(userId);
return [
{
syncId: singletonSyncId("accountProfile"),
remoteUserId: userId,
username: user.username,
isAdmin: !!user.isAdmin,
roles: roles.map((role) => role.roleName).sort(),
permissions: (
await PermissionManager.getInstance().getUserPermissions(userId)
).sort(),
},
];
}
async function writeAccountProfile(userId: string, wire: SyncRow) {
const { updateLink } = await import("./client/link-store.js");
await updateLink({
remoteUserId: (wire.remoteUserId as string) ?? null,
remoteUsername: (wire.username as string) ?? null,
account: {
remoteUserId: wire.remoteUserId as string,
username: wire.username as string,
isAdmin: !!wire.isAdmin,
roles: (wire.roles as string[]) ?? [],
permissions: (wire.permissions as string[]) ?? [],
},
});
if (typeof wire.username === "string" && wire.username) {
const { createCurrentUserRepository } =
await import("../database/repositories/factory.js");
const users = createCurrentUserRepository();
const current = await users.findById(userId);
if (current && current.username !== wire.username) {
await users.update(userId, { username: wire.username });
}
}
}
/** Host columns a shared copy never carries. */
const SHARED_HOST_DROP = [
"id",
"userId",
"credentialId",
"parentHostId",
"pluginSettings",
"protocolAuth",
"quickActions",
"createdAt",
"updatedAt",
"isShared",
"permissionLevel",
...HOST_LOCAL_FIELDS,
];
async function loadSharedHosts(userId: string): Promise<SyncRow[]> {
const {
createCurrentRbacAccessRepository,
createCurrentRoleRepository,
createCurrentHostResolutionRepository,
createCurrentUserRepository,
} = await import("../database/repositories/factory.js");
const roleIds = await createCurrentRoleRepository().listUserRoleIds(userId);
const entries =
await createCurrentRbacAccessRepository().listVisibleHostAccessEntries(
userId,
roleIds,
);
const hostRepository = createCurrentHostResolutionRepository();
const owned = await hostRepository.listOwnedHostIds(userId);
const levels = new Map<number, string>();
for (const entry of entries) {
if (owned.has(entry.hostId)) continue;
if (!levels.has(entry.hostId)) {
levels.set(entry.hostId, entry.permissionLevel);
}
}
if (levels.size === 0) return [];
const { resolveHostById } = await import("../hosts/host-resolver.js");
const users = createCurrentUserRepository();
const ownerNames = new Map<string, string>();
const rows: SyncRow[] = [];
for (const [hostId, permissionLevel] of levels) {
try {
const host = (await resolveHostById(
hostId,
userId,
)) as unknown as SyncRow | null;
if (!host || typeof host.syncId !== "string") continue;
const ownerId = String(host.userId);
if (!ownerNames.has(ownerId)) {
ownerNames.set(
ownerId,
(await users.findById(ownerId))?.username ?? "",
);
}
const wire: SyncRow = {};
for (const [field, value] of Object.entries(host)) {
if (SHARED_HOST_DROP.includes(field)) continue;
wire[field] =
value !== null && typeof value === "object"
? JSON.stringify(value)
: value;
}
wire.jumpHosts = await jumpHostsToSyncIds(host.jumpHosts);
wire.syncId = host.syncId;
wire.shared = {
hostId,
owner: ownerNames.get(ownerId) ?? "",
permissionLevel,
};
rows.push(wire);
} catch (error) {
syncLogger.warn("Could not include a shared host in sync", {
operation: "sync_shared_host",
hostId,
error: error instanceof Error ? error.message : String(error),
});
}
}
return rows;
}
async function jumpHostsToSyncIds(value: unknown): Promise<string | null> {
const list = Array.isArray(value) ? value : [];
if (list.length === 0) return null;
const { createCurrentHostRepository } =
await import("../database/repositories/factory.js");
const repository = createCurrentHostRepository();
const mapped = [];
for (const item of list) {
const hostId = (item as { hostId?: unknown })?.hostId;
if (typeof hostId !== "number") continue;
const target = await repository.findById(hostId);
if (target?.syncId) mapped.push({ ...item, hostId: target.syncId });
}
return JSON.stringify(mapped);
}
async function loadSharedCredentials(userId: string): Promise<SyncRow[]> {
const {
createCurrentCredentialAccessRepository,
createCurrentCredentialRepository,
createCurrentRoleRepository,
createCurrentUserRepository,
} = await import("../database/repositories/factory.js");
const roleIds = await createCurrentRoleRepository().listUserRoleIds(userId);
const grants =
await createCurrentCredentialAccessRepository().listSharedWithUser(
userId,
roleIds,
);
if (grants.length === 0) return [];
const { findUsableCredential } =
await import("../hosts/usable-credential.js");
const credentials = createCurrentCredentialRepository();
const users = createCurrentUserRepository();
const rows: SyncRow[] = [];
for (const grant of grants) {
try {
const row = (await credentials.findById(
grant.credentialId,
)) as SyncRow | null;
const usable = (await findUsableCredential(
grant.credentialId,
userId,
)) as unknown as SyncRow | null;
if (!row || !usable || typeof row.syncId !== "string") continue;
rows.push({
syncId: row.syncId,
name: row.name,
description: row.description ?? null,
tags: row.tags ?? null,
authType: usable.authType,
username: usable.username ?? null,
password: usable.password ?? null,
privateKey: usable.privateKey ?? null,
keyPassword: usable.keyPassword ?? null,
keyType: usable.keyType ?? null,
publicKey: usable.publicKey ?? null,
certPublicKey: usable.certPublicKey ?? null,
shared: {
credentialId: grant.credentialId,
owner: (await users.findById(grant.ownerId))?.username ?? "",
permissionLevel: grant.permissionLevel,
},
});
} catch (error) {
syncLogger.warn("Could not include a shared credential in sync", {
operation: "sync_shared_credential",
credentialId: grant.credentialId,
error: error instanceof Error ? error.message : String(error),
});
}
}
return rows;
}
/** Stores a read-only copy of something shared with the linked account. */
async function writeSharedCopy(
entityType: "hosts" | "sshCredentials",
userId: string,
wire: SyncRow,
): Promise<void> {
const { getEntity } = await import("../plugins/sync-registry.js");
const { writeWireRow, createResolvers } = await import("./store.js");
const entity = getEntity(entityType);
if (!entity) return;
const { shared, ...rest } = wire;
await writeWireRow(entity, userId, rest, createResolvers(userId), {
sharedSource: JSON.stringify(shared ?? {}),
});
}
async function eraseSharedCopy(
entityType: "hosts" | "sshCredentials",
userId: string,
syncId: string,
): Promise<void> {
const { getEntity } = await import("../plugins/sync-registry.js");
const { deleteStoredRow, findStoredRow } = await import("./store.js");
const entity = getEntity(entityType);
if (!entity) return;
const row = await findStoredRow(entity, userId, syncId);
if (!row?.sharedSource) return;
await deleteStoredRow(entity, userId, syncId);
}
src/backend/sync/host-defaults.ts
+335
View File
@@ -0,0 +1,335 @@
/**
* Host defaults over sync.
*
* A user's own and folder defaults travel as one row per key. A value that
* names a row by local id (the credential in an auth default, the hosts in a
* jump chain, a plugin's own ids) travels by syncId and is translated back on
* the other side. The server's admin defaults reach a linked desktop as one
* read-only snapshot, so the desktop resolves a host the way its server does.
*/
import { and, eq } from "drizzle-orm";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import type { PluginManifest } from "@termix/plugin-sdk/manifest";
import { hosts, sshCredentials, sshFolders } from "../database/db/schema.js";
import {
createCurrentHostDefaultsRepository,
createCurrentRepositoryContext,
createCurrentSettingsRepository,
} from "../database/repositories/factory.js";
import { consume } from "../plugins/registry.js";
import {
CORE_NAMESPACE,
joinDefaultKey,
splitDefaultKey,
} from "../../types/host-defaults.js";
import {
decodeDefaultValue,
encodeDefaultValue,
REMOTE_ADMIN_DEFAULTS_KEY,
} from "../hosts/defaults/levels.js";
import type { PluginHostSettingsSync } from "./host-plugin-settings.js";
import { singletonSyncId } from "./wire.js";
type IdMapper = (id: number) => Promise<string | null>;
type SyncIdMapper = (syncId: string) => Promise<number | null>;
function db() {
return createCurrentRepositoryContext().drizzle;
}
async function credentialSyncId(id: number): Promise<string | null> {
const [row] = await db()
.select({ syncId: sshCredentials.syncId })
.from(sshCredentials)
.where(eq(sshCredentials.id, id))
.limit(1);
return row?.syncId ?? null;
}
async function hostSyncId(id: number): Promise<string | null> {
const [row] = await db()
.select({ syncId: hosts.syncId })
.from(hosts)
.where(eq(hosts.id, id))
.limit(1);
return row?.syncId ?? null;
}
function localCredential(userId: string): SyncIdMapper {
return async (syncId) => {
const [row] = await db()
.select({ id: sshCredentials.id })
.from(sshCredentials)
.where(
and(
eq(sshCredentials.syncId, syncId),
eq(sshCredentials.userId, userId),
),
)
.limit(1);
return row?.id ?? null;
};
}
function localHost(userId: string): SyncIdMapper {
return async (syncId) => {
const [row] = await db()
.select({ id: hosts.id })
.from(hosts)
.where(and(eq(hosts.syncId, syncId), eq(hosts.userId, userId)))
.limit(1);
return row?.id ?? null;
};
}
function hostSettingsHook(
pluginId: string,
): PluginHostSettingsSync | undefined {
return consume<PluginHostSettingsSync>(`${pluginId}.hostSettingsSync`);
}
/** A value on its way out: local ids become syncIds. */
export async function exportDefaultValue(
namespace: string,
key: string,
value: unknown,
maps: { credential: IdMapper; host: IdMapper } = {
credential: credentialSyncId,
host: hostSyncId,
},
): Promise<unknown> {
if (namespace !== CORE_NAMESPACE) {
const hook = hostSettingsHook(namespace);
return hook?.exportValue ? hook.exportValue(key, value) : value;
}
if (key === "auth" && value && typeof value === "object") {
const auth = value as Record<string, unknown>;
if (typeof auth.credentialId !== "number") return value;
const { credentialId, ...rest } = auth;
return { ...rest, credentialSyncId: await maps.credential(credentialId) };
}
if (key === "jumpHosts" && Array.isArray(value)) {
const mapped = [];
for (const entry of value) {
const hostId = Number((entry as { hostId?: unknown })?.hostId);
const syncId = Number.isInteger(hostId) ? await maps.host(hostId) : null;
if (syncId) mapped.push({ hostSyncId: syncId });
}
return mapped;
}
return value;
}
/** A value on its way in: syncIds become this side's ids. */
export async function importDefaultValue(
namespace: string,
key: string,
value: unknown,
maps: { credential: SyncIdMapper; host: SyncIdMapper },
): Promise<unknown> {
if (namespace !== CORE_NAMESPACE) {
const hook = hostSettingsHook(namespace);
return hook?.importValue ? hook.importValue(key, value) : value;
}
if (key === "auth" && value && typeof value === "object") {
const auth = value as Record<string, unknown>;
if (typeof auth.credentialSyncId !== "string") return value;
const { credentialSyncId, ...rest } = auth;
return { ...rest, credentialId: await maps.credential(credentialSyncId) };
}
if (key === "jumpHosts" && Array.isArray(value)) {
const mapped = [];
for (const entry of value) {
const syncId = (entry as { hostSyncId?: unknown })?.hostSyncId;
const hostId =
typeof syncId === "string" ? await maps.host(syncId) : null;
if (hostId !== null) mapped.push({ hostId });
}
return mapped;
}
return value;
}
export async function loadHostDefaults(userId: string): Promise<SyncRow[]> {
const repository = createCurrentHostDefaultsRepository();
const rows = await repository.listOwnedBy(userId);
const folderSyncIds = new Map<number, string>();
const folders = await db()
.select({
id: sshFolders.id,
syncId: sshFolders.syncId,
localOnly: sshFolders.localOnly,
})
.from(sshFolders)
.where(eq(sshFolders.userId, userId));
for (const folder of folders) {
if (folder.syncId && !folder.localOnly) {
folderSyncIds.set(folder.id, folder.syncId);
}
}
const result: SyncRow[] = [];
for (const row of rows) {
const fullKey = joinDefaultKey(row.namespace, row.key);
let syncId: string;
let folderSyncId: string | null = null;
if (row.level === "user") {
syncId = `u:${fullKey}`;
} else if (row.level === "folder" && row.folderId !== null) {
const folder = folderSyncIds.get(row.folderId);
if (!folder) continue;
folderSyncId = folder;
syncId = `f:${folder}:${fullKey}`;
} else {
continue;
}
result.push({
syncId,
level: row.level,
folderSyncId,
namespace: row.namespace,
key: row.key,
value: await exportDefaultValue(
row.namespace,
row.key,
decodeDefaultValue(row.value),
),
});
}
return result;
}
/** Splits "u:<ns.key>" or "f:<folderSyncId>:<ns.key>". */
export function parseHostDefaultsSyncId(syncId: string): {
level: "user" | "folder";
folderSyncId: string | null;
fullKey: string;
} | null {
if (syncId.startsWith("u:")) {
return { level: "user", folderSyncId: null, fullKey: syncId.slice(2) };
}
if (syncId.startsWith("f:")) {
const rest = syncId.slice(2);
const colon = rest.indexOf(":");
if (colon <= 0) return null;
return {
level: "folder",
folderSyncId: rest.slice(0, colon),
fullKey: rest.slice(colon + 1),
};
}
return null;
}
async function folderIdFor(
userId: string,
folderSyncId: string,
): Promise<number | null> {
const [row] = await db()
.select({ id: sshFolders.id })
.from(sshFolders)
.where(
and(eq(sshFolders.syncId, folderSyncId), eq(sshFolders.userId, userId)),
)
.limit(1);
return row?.id ?? null;
}
async function scopeFor(
userId: string,
parsed: NonNullable<ReturnType<typeof parseHostDefaultsSyncId>>,
) {
if (parsed.level === "user") return { level: "user" as const, userId };
const folderId = parsed.folderSyncId
? await folderIdFor(userId, parsed.folderSyncId)
: null;
if (folderId === null) return null;
return { level: "folder" as const, userId, folderId };
}
export async function writeHostDefault(
userId: string,
wire: SyncRow,
): Promise<void> {
const parsed = parseHostDefaultsSyncId(String(wire.syncId));
if (!parsed) return;
const scope = await scopeFor(userId, parsed);
if (!scope) return;
const [namespace, key] = splitDefaultKey(parsed.fullKey);
const value = await importDefaultValue(namespace, key, wire.value, {
credential: localCredential(userId),
host: localHost(userId),
});
await createCurrentHostDefaultsRepository().apply(
scope,
[{ namespace, key, value: encodeDefaultValue(value) }],
[],
null,
);
await afterSyncedLevelChange(userId);
}
export async function eraseHostDefault(
userId: string,
syncId: string,
): Promise<void> {
const parsed = parseHostDefaultsSyncId(syncId);
if (!parsed) return;
const scope = await scopeFor(userId, parsed);
if (!scope) return;
const [namespace, key] = splitDefaultKey(parsed.fullKey);
await createCurrentHostDefaultsRepository().apply(
scope,
[],
[{ namespace, key }],
null,
);
await afterSyncedLevelChange(userId);
}
/**
* A level a desktop pushed changes what the server's hosts resolve to. On a
* desktop the hosts arrive already resolved, so nothing runs there.
*/
async function afterSyncedLevelChange(userId: string): Promise<void> {
const { getLink } = await import("./client/link-store.js");
if (await getLink().catch(() => null)) return;
const { recompute } = await import("../hosts/defaults/recompute.js");
void recompute({ userIds: [userId] }).catch(() => {});
}
export function coversHostDefault(
syncId: string,
activeManifest: (pluginId: string) => PluginManifest | undefined,
): boolean {
const parsed = parseHostDefaultsSyncId(syncId);
if (!parsed) return false;
const [namespace] = splitDefaultKey(parsed.fullKey);
return namespace === CORE_NAMESPACE || !!activeManifest(namespace);
}
export async function loadAdminDefaults(): Promise<SyncRow[]> {
const rows = await createCurrentHostDefaultsRepository().listScope({
level: "admin",
});
const values: Record<string, unknown> = {};
for (const row of rows) {
values[joinDefaultKey(row.namespace, row.key)] = decodeDefaultValue(
row.value,
);
}
return [{ syncId: singletonSyncId("hostDefaultsAdmin"), values }];
}
export async function writeAdminDefaults(
_userId: string,
wire: SyncRow,
): Promise<void> {
const values =
wire.values && typeof wire.values === "object" ? wire.values : {};
await createCurrentSettingsRepository().set(
REMOTE_ADMIN_DEFAULTS_KEY,
JSON.stringify(values),
);
}
src/backend/sync/host-plugin-settings.ts
+93
View File
@@ -0,0 +1,93 @@
/**
* Host plugin settings over sync. A host row travels with a
* pluginSettings map of its non-secret values; a plugin whose value refers to
* one of its own rows by local id translates it with a registered
* ctx.registry.provide("<id>.hostSettingsSync", { exportValue, importValue }).
*/
import { consume } from "../plugins/registry.js";
import {
declaredFields,
getAllSettings,
setSetting,
} from "../plugins/settings.js";
import { databaseLogger } from "../utils/logger.js";
import type { DefaultOverrides } from "../../types/host-defaults.js";
import {
hostSettingsPlugins,
type HostPluginSettings,
} from "../database/routes/host-plugin-settings.js";
export interface PluginHostSettingsSync {
exportValue?: (key: string, value: unknown) => unknown | Promise<unknown>;
importValue?: (key: string, value: unknown) => unknown | Promise<unknown>;
}
/** What a host carries to the other side of a sync pair. */
export async function exportHostPluginSettings(
hostId: number,
): Promise<HostPluginSettings> {
const result: HostPluginSettings = {};
for (const manifest of hostSettingsPlugins()) {
const fields = declaredFields(manifest, "host").filter(
(field) => field.type !== "secret",
);
if (fields.length === 0) continue;
const values = await getAllSettings(manifest, "host", hostId, {
redactSecrets: true,
});
const hook = consume<PluginHostSettingsSync>(
`${manifest.id}.hostSettingsSync`,
);
const own: Record<string, unknown> = {};
for (const field of fields) {
const value = values[field.key];
own[field.key] = hook?.exportValue
? await hook.exportValue(field.key, value)
: value;
}
result[manifest.id] = own;
}
return result;
}
/**
* Writes what a synced host carried, translated back to local ids. With
* `only`, a classified namespace takes just the keys the host sets itself;
* the rest follow this side's defaults and are written by the materialize
* pass that follows.
*/
export async function importHostPluginSettings(
hostId: number,
carried: unknown,
only?: DefaultOverrides | null,
): Promise<void> {
if (!carried || typeof carried !== "object") return;
const byPlugin = carried as HostPluginSettings;
for (const manifest of hostSettingsPlugins()) {
const own = byPlugin[manifest.id];
if (!own || typeof own !== "object") continue;
const hook = consume<PluginHostSettingsSync>(
`${manifest.id}.hostSettingsSync`,
);
const ownKeys = only?.[manifest.id];
for (const field of declaredFields(manifest, "host")) {
if (field.type === "secret" || !(field.key in own)) continue;
if (ownKeys && !ownKeys.includes(field.key)) continue;
try {
const value = hook?.importValue
? await hook.importValue(field.key, own[field.key])
: own[field.key];
await setSetting(manifest, "host", hostId, field.key, value);
} catch (error) {
databaseLogger.warn("Could not apply a synced host plugin setting", {
operation: "sync_host_plugin_settings",
pluginId: manifest.id,
hostId,
key: field.key,
error: error instanceof Error ? error.message : String(error),
});
}
}
}
}
src/backend/sync/host-protocol-auth.ts
+84
View File
@@ -0,0 +1,84 @@
/**
* A host's plugin protocol logins over sync. They travel on the host row as
* `protocolAuth`, keyed by protocol id, with the owner's secrets decrypted
* for the wire like the host's own and a credential named by its syncId.
*/
import type { SyncRow } from "@termix/plugin-sdk/backend";
import type { HostProtocolLogin } from "../database/repositories/host-protocol-auth-repository.js";
import {
fromPortableLogin,
listProtocolLogins,
replaceProtocolLogins,
toPortableLogins,
} from "../hosts/protocol-auth/protocol-auth.js";
import { syncLogger } from "../utils/logger.js";
type ResolveSyncId = (entityType: string, id: number) => Promise<string | null>;
type ResolveId = (entityType: string, syncId: string) => Promise<number | null>;
const CREDENTIALS = "sshCredentials";
export async function exportProtocolLogins(
hostId: number,
ownerId: string,
resolveSyncId: ResolveSyncId,
): Promise<SyncRow> {
const out: SyncRow = {};
const portable = toPortableLogins(await listProtocolLogins(hostId, ownerId));
for (const [protocol, login] of Object.entries(portable)) {
const { credentialId, ...rest } = login;
out[protocol] = {
...rest,
credentialSyncId: credentialId
? await resolveSyncId(CREDENTIALS, credentialId)
: null,
};
}
return out;
}
/**
* Writes what a synced host carried. A row from a peer that sends no
* protocolAuth leaves the stored logins alone.
*/
export async function importProtocolLogins(
hostId: number,
userId: string,
carried: unknown,
resolveId: ResolveId | undefined,
): Promise<void> {
if (!carried || typeof carried !== "object" || Array.isArray(carried)) {
return;
}
try {
const current = new Map(
(await listProtocolLogins(hostId, userId)).map((login) => [
login.protocol,
login,
]),
);
const logins: HostProtocolLogin[] = [];
for (const [protocol, value] of Object.entries(carried)) {
const syncId = (value as { credentialSyncId?: unknown })
?.credentialSyncId;
let credentialId: number | null = null;
if (typeof syncId === "string" && syncId) {
// A credential not here yet keeps whatever the login pointed at.
credentialId =
(resolveId ? await resolveId(CREDENTIALS, syncId) : null) ??
current.get(protocol)?.credentialId ??
null;
}
const login = fromPortableLogin(protocol, value, credentialId);
if (login) logins.push(login);
}
await replaceProtocolLogins(userId, hostId, logins);
} catch (error) {
syncLogger.warn("Could not apply a synced host's protocol logins", {
operation: "sync_host_protocol_auth",
hostId,
error: error instanceof Error ? error.message : String(error),
});
}
}
src/backend/sync/protocol.ts
+2
View File
@@ -0,0 +1,2 @@
/** Bumped when a desktop and server can no longer understand each other. */
export const SYNC_PROTOCOL_VERSION = 1;
src/backend/sync/records.ts
+248
View File
@@ -0,0 +1,248 @@
/**
* The per-record sync bookkeeping in sync_records. See the table's comment in
* schema.ts for what the columns mean on a server and on a desktop.
*/
import { and, asc, eq, gt, inArray, sql } from "drizzle-orm";
import { syncConflicts, syncRecords } from "../database/db/schema.js";
import { createCurrentRepositoryContext } from "../database/repositories/factory.js";
import { DatabaseSaveTrigger } from "../utils/database-save-trigger.js";
export type SyncRecord = typeof syncRecords.$inferSelect;
export type SyncConflict = typeof syncConflicts.$inferSelect;
export function recordKey(entityType: string, syncId: string): string {
return `${entityType}\u0000${syncId}`;
}
function db() {
return createCurrentRepositoryContext().drizzle;
}
export async function listRecords(
userId: string,
entityType?: string,
): Promise<Map<string, SyncRecord>> {
const rows = await db()
.select()
.from(syncRecords)
.where(
entityType
? and(
eq(syncRecords.userId, userId),
eq(syncRecords.entityType, entityType),
)
: eq(syncRecords.userId, userId),
);
return new Map(
rows.map((row) => [recordKey(row.entityType, row.syncId), row]),
);
}
export async function getRecord(
userId: string,
entityType: string,
syncId: string,
): Promise<SyncRecord | null> {
const [row] = await db()
.select()
.from(syncRecords)
.where(
and(
eq(syncRecords.userId, userId),
eq(syncRecords.entityType, entityType),
eq(syncRecords.syncId, syncId),
),
)
.limit(1);
return row ?? null;
}
export interface RecordValues {
revision: number;
seq?: number;
hash: string | null;
deleted: boolean;
error?: string | null;
errorHash?: string | null;
}
export async function putRecord(
userId: string,
entityType: string,
syncId: string,
values: RecordValues,
): Promise<void> {
const set = {
revision: values.revision,
hash: values.hash,
deleted: values.deleted,
error: values.error ?? null,
errorHash: values.errorHash ?? null,
updatedAt: new Date().toISOString(),
...(values.seq !== undefined ? { seq: values.seq } : {}),
};
const existing = await getRecord(userId, entityType, syncId);
if (existing) {
await db()
.update(syncRecords)
.set(set)
.where(eq(syncRecords.id, existing.id));
} else {
await db()
.insert(syncRecords)
.values({ userId, entityType, syncId, seq: 0, ...set });
}
DatabaseSaveTrigger.triggerSave("sync_record");
}
export async function setRecordError(
userId: string,
entityType: string,
syncId: string,
error: string,
errorHash: string,
): Promise<void> {
const existing = await getRecord(userId, entityType, syncId);
if (existing) {
await db()
.update(syncRecords)
.set({ error, errorHash })
.where(eq(syncRecords.id, existing.id));
} else {
await db().insert(syncRecords).values({
userId,
entityType,
syncId,
revision: 0,
seq: 0,
hash: null,
deleted: false,
error,
errorHash,
});
}
DatabaseSaveTrigger.triggerSave("sync_record");
}
export async function clearRecordErrors(userId: string): Promise<void> {
await db()
.update(syncRecords)
.set({ error: null, errorHash: null })
.where(eq(syncRecords.userId, userId));
}
export async function listRecordErrors(userId: string): Promise<SyncRecord[]> {
const rows = await db()
.select()
.from(syncRecords)
.where(eq(syncRecords.userId, userId));
return rows.filter((row) => !!row.error);
}
export async function listChanges(
userId: string,
cursor: number,
limit: number,
types?: string[],
): Promise<SyncRecord[]> {
const conditions = [
eq(syncRecords.userId, userId),
gt(syncRecords.seq, cursor),
];
if (types && types.length > 0) {
conditions.push(inArray(syncRecords.entityType, types));
}
return db()
.select()
.from(syncRecords)
.where(and(...conditions))
.orderBy(asc(syncRecords.seq))
.limit(limit);
}
export async function deleteRecordsForUser(userId: string): Promise<void> {
await db().delete(syncRecords).where(eq(syncRecords.userId, userId));
await db().delete(syncConflicts).where(eq(syncConflicts.userId, userId));
DatabaseSaveTrigger.triggerSave("sync_record");
}
let nextSeq: number | null = null;
/**
* The next position in the change feed. One counter for every user, so a
* seq is never reused; a single process owns it, seeded from the table.
*/
export async function allocateSeq(): Promise<number> {
if (nextSeq === null) {
const [row] = await db()
.select({ max: sql<number>`max(${syncRecords.seq})` })
.from(syncRecords);
const max = Number(row?.max ?? 0);
nextSeq = (Number.isFinite(max) ? max : 0) + 1;
}
return nextSeq++;
}
/** Test seam. */
export function resetSeqAllocator(): void {
nextSeq = null;
}
export async function saveConflict(
userId: string,
entityType: string,
syncId: string,
localRow: Record<string, unknown>,
serverRevision: number,
): Promise<void> {
await db()
.delete(syncConflicts)
.where(
and(
eq(syncConflicts.userId, userId),
eq(syncConflicts.entityType, entityType),
eq(syncConflicts.syncId, syncId),
),
);
await db()
.insert(syncConflicts)
.values({
userId,
entityType,
syncId,
localRow: JSON.stringify(localRow),
serverRevision,
});
DatabaseSaveTrigger.triggerSave("sync_conflict");
}
export async function listConflicts(userId: string): Promise<SyncConflict[]> {
return db()
.select()
.from(syncConflicts)
.where(eq(syncConflicts.userId, userId))
.orderBy(asc(syncConflicts.createdAt));
}
export async function getConflict(
userId: string,
id: number,
): Promise<SyncConflict | null> {
const [row] = await db()
.select()
.from(syncConflicts)
.where(and(eq(syncConflicts.userId, userId), eq(syncConflicts.id, id)))
.limit(1);
return row ?? null;
}
export async function deleteConflict(
userId: string,
id: number,
): Promise<void> {
await db()
.delete(syncConflicts)
.where(and(eq(syncConflicts.userId, userId), eq(syncConflicts.id, id)));
DatabaseSaveTrigger.triggerSave("sync_conflict");
}
src/backend/sync/server/change-watcher.ts
+30
View File
@@ -0,0 +1,30 @@
import type { NextFunction, Request, Response } from "express";
import { markChanged } from "./feed.js";
const READS = new Set(["GET", "HEAD", "OPTIONS"]);
const writeListeners = new Set<() => void>();
/** Called after every successful write request, for a linked desktop's engine. */
export function onLocalWrite(listener: () => void): () => void {
writeListeners.add(listener);
return () => writeListeners.delete(listener);
}
/**
* Any successful write may change what a linked desktop should have, so it
* nudges the change feed. Sync's own pushes reconcile themselves.
*/
export function syncChangeWatcher(
req: Request,
res: Response,
next: NextFunction,
): void {
if (!READS.has(req.method) && !req.path.startsWith("/sync/")) {
res.on("finish", () => {
if (res.statusCode >= 400) return;
markChanged();
for (const listener of writeListeners) listener();
});
}
next();
}
src/backend/sync/server/feed.ts
+220
View File
@@ -0,0 +1,220 @@
/**
* The server's change feed.
*
* Nothing records a change as it happens. Instead the reconciler compares
* every synced row a user owns with sync_records and bumps the record of
* anything new, changed or gone. That catches every write path, including
* plugin tables, bulk routes and background jobs, without each of them having
* to remember to report it.
*/
import type { Response } from "express";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import { listEntities } from "../../plugins/sync-registry.js";
import { syncLogger } from "../../utils/logger.js";
import { registerCoreSyncEntities } from "../entities.js";
import { createResolvers, loadWireRows } from "../store.js";
import { allocateSeq, listRecords, putRecord, recordKey } from "../records.js";
const locks = new Map<string, Promise<unknown>>();
/** Runs one user's sync work at a time: reconcile and push must not interleave. */
export async function withUserLock<T>(
userId: string,
fn: () => Promise<T>,
): Promise<T> {
const previous = locks.get(userId) ?? Promise.resolve();
const run = previous.catch(() => {}).then(fn);
locks.set(userId, run);
try {
return await run;
} finally {
if (locks.get(userId) === run) locks.delete(userId);
}
}
interface Snapshot {
at: number;
generation: number;
wire: Map<string, SyncRow>;
}
const snapshots = new Map<string, Snapshot>();
const SNAPSHOT_TTL_MS = 60_000;
let generation = 0;
export interface ReconcileResult {
changed: boolean;
/** Every live row on the wire, by recordKey. */
wire: Map<string, SyncRow>;
}
/**
* Brings the user's records up to date. Call inside withUserLock. Skipped
* when nothing was written since the last run and it is recent, unless
* forced.
*/
export async function reconcileUser(
userId: string,
options: { force?: boolean } = {},
): Promise<ReconcileResult> {
const snapshot = snapshots.get(userId);
if (
!options.force &&
snapshot &&
snapshot.generation === generation &&
Date.now() - snapshot.at < SNAPSHOT_TTL_MS
) {
return { changed: false, wire: snapshot.wire };
}
registerCoreSyncEntities();
const startedAt = generation;
const records = await listRecords(userId);
const resolvers = createResolvers(userId);
const wire = new Map<string, SyncRow>();
let changed = false;
for (const entity of listEntities()) {
let loaded;
try {
loaded = await loadWireRows(entity, userId, resolvers);
} catch (error) {
// An entity that cannot load right now is left alone rather than read
// as every row deleted.
syncLogger.warn("Could not read a sync entity", {
operation: "sync_reconcile",
entityType: entity.type,
userId,
error: error instanceof Error ? error.message : String(error),
});
continue;
}
for (const row of loaded.rows.values()) {
const key = recordKey(entity.type, row.syncId);
wire.set(key, row.wire);
const record = records.get(key);
if (record && !record.deleted && record.hash === row.hash) continue;
await putRecord(userId, entity.type, row.syncId, {
revision: (record?.revision ?? 0) + 1,
seq: await allocateSeq(),
hash: row.hash,
deleted: false,
});
changed = true;
}
for (const record of records.values()) {
if (record.entityType !== entity.type || record.deleted) continue;
if (loaded.rows.has(record.syncId) || loaded.failed.has(record.syncId)) {
continue;
}
if (entity.covers && !entity.covers(record.syncId)) continue;
await putRecord(userId, entity.type, record.syncId, {
revision: record.revision + 1,
seq: await allocateSeq(),
hash: null,
deleted: true,
});
changed = true;
}
}
snapshots.set(userId, { at: Date.now(), generation: startedAt, wire });
if (changed) notifyListeners(userId);
return { changed, wire };
}
/** Forgets a user's snapshot so the next reconcile reads everything. */
export function invalidateSnapshot(userId: string): void {
snapshots.delete(userId);
}
const listeners = new Map<string, Set<Response>>();
let debounce: NodeJS.Timeout | null = null;
let heartbeat: NodeJS.Timeout | null = null;
let sweep: NodeJS.Timeout | null = null;
/**
* Something was written. Desktops listening for changes get their
* account reconciled shortly after, since a write by one user (an owner
* editing a shared host) can change what another one sees.
*/
export function markChanged(): void {
generation++;
if (listeners.size === 0) return;
if (debounce) clearTimeout(debounce);
debounce = setTimeout(() => {
debounce = null;
void reconcileListeners();
}, 1500);
debounce.unref?.();
}
async function reconcileListeners(): Promise<void> {
for (const userId of [...listeners.keys()]) {
try {
await withUserLock(userId, () => reconcileUser(userId));
} catch (error) {
syncLogger.warn("Background sync reconcile failed", {
operation: "sync_reconcile_background",
userId,
error: error instanceof Error ? error.message : String(error),
});
}
}
}
function notifyListeners(userId: string): void {
const set = listeners.get(userId);
if (!set) return;
for (const res of set) {
res.write(`event: change\ndata: {}\n\n`);
}
}
export function addListener(userId: string, res: Response): () => void {
let set = listeners.get(userId);
if (!set) {
set = new Set();
listeners.set(userId, set);
}
set.add(res);
ensureTimers();
return () => {
set!.delete(res);
if (set!.size === 0) listeners.delete(userId);
};
}
function ensureTimers(): void {
if (!heartbeat) {
heartbeat = setInterval(() => {
for (const set of listeners.values()) {
for (const res of set) res.write(`: ping\n\n`);
}
}, 25_000);
heartbeat.unref?.();
}
if (!sweep) {
// Background jobs write without a request; this catches them.
sweep = setInterval(() => {
generation++;
void reconcileListeners();
}, 60_000);
sweep.unref?.();
}
}
/** Test seam. */
export function resetFeed(): void {
snapshots.clear();
listeners.clear();
locks.clear();
generation = 0;
for (const timer of [debounce, heartbeat, sweep]) {
if (timer) clearTimeout(timer);
}
debounce = heartbeat = sweep = null;
}
src/backend/sync/server/push.ts
+250
View File
@@ -0,0 +1,250 @@
/**
* Applying a desktop's pushed changes on the server.
*
* Each op names the revision its sender last saw. If the server has moved on
* since, the op is a conflict and the server's version is sent back instead
* of being overwritten. Every op gets its own result, so one bad row never
* holds up the rest of the batch.
*/
import type { SyncRow } from "@termix/plugin-sdk/backend";
import { getEntity } from "../../plugins/sync-registry.js";
import { PermissionManager } from "../../utils/permission-manager.js";
import { syncLogger } from "../../utils/logger.js";
import {
createResolvers,
deleteStoredRow,
loadWireRow,
writeWireRow,
} from "../store.js";
import { allocateSeq, getRecord, putRecord, recordKey } from "../records.js";
import { orderSelfReferences } from "../wire.js";
import { reconcileUser } from "./feed.js";
export interface PushOp {
entityType: string;
syncId: string;
baseRevision: number;
op: "upsert" | "delete";
row?: SyncRow;
}
export type PushResult =
| { syncId: string; entityType: string; status: "ok"; revision: number }
| {
syncId: string;
entityType: string;
status: "conflict";
revision: number;
deleted: boolean;
row: SyncRow | null;
}
| {
syncId: string;
entityType: string;
status: "rejected";
reason: string;
};
export const MAX_PUSH_OPS = 500;
function isPushOp(value: unknown): value is PushOp {
if (!value || typeof value !== "object") return false;
const op = value as Record<string, unknown>;
return (
typeof op.entityType === "string" &&
typeof op.syncId === "string" &&
op.syncId.length > 0 &&
op.syncId.length <= 191 &&
typeof op.baseRevision === "number" &&
(op.op === "upsert" || op.op === "delete") &&
(op.op === "delete" || (!!op.row && typeof op.row === "object"))
);
}
export function parsePushOps(body: unknown): PushOp[] | null {
const ops = (body as { ops?: unknown })?.ops;
if (!Array.isArray(ops) || ops.length > MAX_PUSH_OPS) return null;
return ops.every(isPushOp) ? ops : null;
}
/** Upserts in dependency order, then deletes in reverse, children first. */
export function orderOps(ops: PushOp[]): PushOp[] {
const order = (op: PushOp) => getEntity(op.entityType)?.order ?? 1_000_000;
const upserts = ops
.filter((op) => op.op === "upsert")
.sort((a, b) => order(a) - order(b));
const deletes = ops
.filter((op) => op.op === "delete")
.sort((a, b) => order(b) - order(a));
const orderedUpserts: PushOp[] = [];
const byType = new Map<string, PushOp[]>();
for (const op of upserts) {
const list = byType.get(op.entityType) ?? [];
list.push(op);
byType.set(op.entityType, list);
}
for (const [type, list] of byType) {
const entity = getEntity(type);
if (!entity) {
orderedUpserts.push(...list);
continue;
}
const bySyncId = new Map(list.map((op) => [op.syncId, op]));
const rows = orderSelfReferences(
entity,
list.map((op) => ({ ...(op.row as SyncRow), syncId: op.syncId })),
);
for (const row of rows) {
orderedUpserts.push(bySyncId.get(row.syncId as string)!);
}
}
return [...orderedUpserts, ...deletes];
}
/** Call inside withUserLock. */
export async function applyPush(
userId: string,
ops: PushOp[],
): Promise<PushResult[]> {
const { wire } = await reconcileUser(userId, { force: true });
const permissions = PermissionManager.getInstance();
let missed = false;
const resolvers = createResolvers(userId, () => {
missed = true;
});
const results = new Map<string, PushResult>();
// Rows written before something they point at, for a second pass.
const retry: PushOp[] = [];
for (const op of orderOps(ops)) {
const key = recordKey(op.entityType, op.syncId);
const base = { syncId: op.syncId, entityType: op.entityType };
const entity = getEntity(op.entityType);
if (!entity) {
results.set(key, { ...base, status: "rejected", reason: "unknown_type" });
continue;
}
if (entity.readOnly) {
results.set(key, { ...base, status: "rejected", reason: "read_only" });
continue;
}
const record = await getRecord(userId, op.entityType, op.syncId);
const currentRevision = record?.revision ?? 0;
if (op.baseRevision !== currentRevision) {
results.set(key, {
...base,
status: "conflict",
revision: currentRevision,
deleted: !!record?.deleted,
row: record?.deleted ? null : (wire.get(key) ?? null),
});
continue;
}
const exists = !!record && !record.deleted;
const permission =
op.op === "delete"
? entity.permissions?.delete
: exists
? entity.permissions?.update
: entity.permissions?.create;
if (permission && !(await permissions.hasPermission(userId, permission))) {
results.set(key, { ...base, status: "rejected", reason: "permission" });
continue;
}
try {
if (op.op === "delete") {
await deleteStoredRow(entity, userId, op.syncId);
await putRecord(userId, op.entityType, op.syncId, {
revision: currentRevision + 1,
seq: await allocateSeq(),
hash: null,
deleted: true,
});
wire.delete(key);
results.set(key, {
...base,
status: "ok",
revision: currentRevision + 1,
});
continue;
}
const row = { ...(op.row as SyncRow), syncId: op.syncId };
if (entity.shouldSync && entity.shouldSync(row) === false) {
results.set(key, { ...base, status: "rejected", reason: "not_synced" });
continue;
}
missed = false;
await writeWireRow(entity, userId, row, resolvers);
if (missed) retry.push(op);
const written = await loadWireRow(entity, userId, op.syncId, resolvers);
await putRecord(userId, op.entityType, op.syncId, {
revision: currentRevision + 1,
seq: await allocateSeq(),
hash: written?.hash ?? null,
deleted: false,
});
if (written) wire.set(key, written.wire);
results.set(key, {
...base,
status: "ok",
revision: currentRevision + 1,
});
} catch (error) {
const reason = error instanceof Error ? error.message : String(error);
syncLogger.warn("Rejected a pushed sync change", {
operation: "sync_push",
userId,
entityType: op.entityType,
syncId: op.syncId,
error: reason,
});
results.set(key, {
...base,
status: "rejected",
reason: `invalid: ${reason}`.slice(0, 500),
});
}
}
for (const op of retry) {
const entity = getEntity(op.entityType)!;
try {
await writeWireRow(
entity,
userId,
{ ...(op.row as SyncRow), syncId: op.syncId },
resolvers,
);
const written = await loadWireRow(entity, userId, op.syncId, resolvers);
const record = await getRecord(userId, op.entityType, op.syncId);
if (record && written) {
await putRecord(userId, op.entityType, op.syncId, {
revision: record.revision,
seq: record.seq,
hash: written.hash,
deleted: false,
});
}
} catch {
// The first write stands.
}
}
// Refresh the snapshot so the next pull serves what was just written.
await reconcileUser(userId, { force: true });
return ops.map(
(op) =>
results.get(recordKey(op.entityType, op.syncId)) ?? {
syncId: op.syncId,
entityType: op.entityType,
status: "rejected",
reason: "duplicate",
},
);
}
src/backend/sync/server/routes.ts
+583
View File
@@ -0,0 +1,583 @@
import express, { type Request, type Response } from "express";
import fs from "fs";
import path from "path";
import os from "os";
import * as tar from "tar";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { syncLogger } from "../../utils/logger.js";
import { getLocalVersion } from "../../utils/app-version.js";
import { getEntity, listEntities } from "../../plugins/sync-registry.js";
import { getPluginRuntime } from "../../plugins/index.js";
import {
createCurrentSessionRepository,
createCurrentSettingsRepository,
createCurrentUserRepository,
createCurrentPluginRepository,
createCurrentPluginPermissionGrantRepository,
} from "../../database/repositories/factory.js";
import { resolveBrandingSettings } from "../../database/routes/branding-settings.js";
import { registerCoreSyncEntities } from "../entities.js";
import { listChanges, recordKey } from "../records.js";
import { addListener, reconcileUser, withUserLock } from "./feed.js";
import { applyPush, parsePushOps } from "./push.js";
import { SYNC_PROTOCOL_VERSION } from "../protocol.js";
const SESSION_DAYS = 365;
const REFRESH_WITHIN_MS = 60 * 24 * 60 * 60 * 1000;
const MAX_PAGE = 1000;
const router = express.Router();
const authManager = AuthManager.getInstance();
const authenticateJWT = authManager.createAuthMiddleware();
registerCoreSyncEntities();
function userOf(req: Request): string {
return (req as AuthenticatedRequest).userId;
}
/**
* @openapi
* /sync/v2/info:
* get:
* summary: Describe this server to a desktop that wants to link to it
* description: Public. Lets the desktop app confirm it reached Termix and not a proxy's login page, and that the sync protocol matches.
* tags:
* - Sync
* responses:
* 200:
* description: Server name, version and sync protocol version.
*/
router.get("/v2/info", async (_req: Request, res: Response) => {
let name = "Termix";
try {
name = (await resolveBrandingSettings(createCurrentSettingsRepository()))
.appName;
} catch {
// The default name is fine.
}
res.setHeader("X-Termix-Sync", String(SYNC_PROTOCOL_VERSION));
res.json({
termix: true,
name,
version: getLocalVersion(),
protocol: SYNC_PROTOCOL_VERSION,
});
});
/**
* @openapi
* /sync/v2/link:
* post:
* summary: Link a desktop to this account
* description: Trades the short login session the desktop just signed in with for a long-lived desktop session, which shows in the account's sessions list and is revoked like any other. The login session is ended.
* tags:
* - Sync
* requestBody:
* content:
* application/json:
* schema:
* type: object
* properties:
* deviceName:
* type: string
* responses:
* 200:
* description: The desktop session token and the account it belongs to.
* 401:
* description: Not signed in.
*/
router.post(
"/v2/link",
authenticateJWT,
async (req: Request, res: Response) => {
const userId = userOf(req);
try {
const user = await createCurrentUserRepository().findById(userId);
if (!user) return res.status(401).json({ error: "User not found" });
const rawName =
typeof req.body?.deviceName === "string" ? req.body.deviceName : "";
const deviceName = rawName.trim().slice(0, 100) || "Termix Desktop";
const token = await authManager.generateJWTToken(userId, {
expiresIn: `${SESSION_DAYS}d`,
deviceType: "desktop",
deviceInfo: `${deviceName} (sync)`,
});
const loginSession = (req as AuthenticatedRequest).sessionId;
if (loginSession) await authManager.revokeSession(loginSession);
res.json({
token,
user: { id: user.id, username: user.username },
server: { version: getLocalVersion() },
});
} catch (error) {
syncLogger.error("Failed to link a desktop", error, {
operation: "sync_link",
userId,
});
res.status(500).json({ error: "Failed to link" });
}
},
);
/**
* @openapi
* /sync/v2/session/refresh:
* post:
* summary: Keep a linked desktop's session from expiring
* description: Returns a fresh desktop session when the current one ends within 60 days, and ends the old one. Otherwise returns null and the desktop keeps its token.
* tags:
* - Sync
* responses:
* 200:
* description: The new token, or null.
* 401:
* description: Session revoked or expired.
*/
router.post(
"/v2/session/refresh",
authenticateJWT,
async (req: Request, res: Response) => {
const userId = userOf(req);
const sessionId = (req as AuthenticatedRequest).sessionId;
try {
if (!sessionId) return res.json({ token: null });
const session =
await createCurrentSessionRepository().findById(sessionId);
if (!session) return res.status(401).json({ error: "Session not found" });
const remaining = new Date(session.expiresAt).getTime() - Date.now();
if (remaining > REFRESH_WITHIN_MS) return res.json({ token: null });
const token = await authManager.generateJWTToken(userId, {
expiresIn: `${SESSION_DAYS}d`,
deviceType: "desktop",
deviceInfo: session.deviceInfo,
});
await authManager.revokeSession(sessionId);
res.json({ token });
} catch (error) {
syncLogger.error("Failed to refresh a desktop session", error, {
operation: "sync_session_refresh",
userId,
});
res.status(500).json({ error: "Failed to refresh session" });
}
},
);
/**
* @openapi
* /sync/v2/unlink:
* post:
* summary: End a linked desktop's session
* description: Called by a desktop when it unlinks, so its session stops working here right away.
* tags:
* - Sync
* responses:
* 200:
* description: The session was ended.
*/
router.post(
"/v2/unlink",
authenticateJWT,
async (req: Request, res: Response) => {
const sessionId = (req as AuthenticatedRequest).sessionId;
try {
if (sessionId) await authManager.revokeSession(sessionId);
res.json({ success: true });
} catch (error) {
syncLogger.error("Failed to end a desktop session", error, {
operation: "sync_unlink",
});
res.status(500).json({ error: "Failed to unlink" });
}
},
);
/**
* @openapi
* /sync/v2/changes:
* get:
* summary: Pull what changed for this account after a cursor
* description: Returns records in change order. Each has the entity type, syncId, revision, whether it is deleted, and the row when it is not. Records of entity types this server does not currently run are skipped but still move the cursor.
* tags:
* - Sync
* parameters:
* - in: query
* name: cursor
* schema:
* type: integer
* - in: query
* name: limit
* schema:
* type: integer
* - in: query
* name: types
* description: Comma-separated entity types, to backfill only those.
* schema:
* type: string
* responses:
* 200:
* description: A page of changes, the next cursor and whether more remain.
* 500:
* description: Failed to read changes.
*/
router.get(
"/v2/changes",
authenticateJWT,
async (req: Request, res: Response) => {
const userId = userOf(req);
const cursor = Math.max(0, Number(req.query.cursor) || 0);
const limit = Math.min(
MAX_PAGE,
Math.max(1, Number(req.query.limit) || MAX_PAGE),
);
const types =
typeof req.query.types === "string" && req.query.types
? req.query.types.split(",").filter(Boolean)
: undefined;
try {
const page = await withUserLock(userId, async () => {
const { wire } = await reconcileUser(userId);
const records = await listChanges(userId, cursor, limit + 1, types);
return { wire, records };
});
const hasMore = page.records.length > limit;
const records = page.records.slice(0, limit);
const changes = [];
for (const record of records) {
if (!getEntity(record.entityType)) continue;
const row = record.deleted
? null
: (page.wire.get(recordKey(record.entityType, record.syncId)) ??
null);
// A live record with no row failed to serialize; it comes again once it
// changes, rather than arriving as a delete.
if (!record.deleted && !row) continue;
changes.push({
seq: record.seq,
entityType: record.entityType,
syncId: record.syncId,
revision: record.revision,
deleted: !!record.deleted,
row,
});
}
res.json({
changes,
nextCursor: records.length ? records[records.length - 1].seq : cursor,
hasMore,
entityTypes: listEntities().map((entity) => ({
type: entity.type,
owner: entity.owner,
order: entity.order,
readOnly: !!entity.readOnly,
})),
});
} catch (error) {
syncLogger.error("Failed to read sync changes", error, {
operation: "sync_changes",
userId,
});
res.status(500).json({ error: "Failed to read changes" });
}
},
);
/**
* @openapi
* /sync/v2/push:
* post:
* summary: Push a desktop's changes
* description: Each op carries the revision the desktop last saw. Ops are applied in dependency order and each gets its own result, ok with the new revision, conflict with the server's version, or rejected with a reason.
* tags:
* - Sync
* requestBody:
* content:
* application/json:
* schema:
* type: object
* properties:
* ops:
* type: array
* items:
* type: object
* responses:
* 200:
* description: One result per op, in the order sent.
* 400:
* description: Malformed ops, or more than 500.
* 500:
* description: Failed to apply changes.
*/
router.post(
"/v2/push",
authenticateJWT,
async (req: Request, res: Response) => {
const userId = userOf(req);
const ops = parsePushOps(req.body);
if (!ops) return res.status(400).json({ error: "Invalid ops" });
try {
const results = await withUserLock(userId, () => applyPush(userId, ops));
res.json({ results });
} catch (error) {
syncLogger.error("Failed to apply pushed changes", error, {
operation: "sync_push",
userId,
});
res.status(500).json({ error: "Failed to apply changes" });
}
},
);
/**
* @openapi
* /sync/v2/events:
* get:
* summary: Stream a notice whenever this account's data changes
* description: Server-sent events. A "change" event means there is something new to pull. Comments are sent every 25 seconds to keep proxies from closing the stream.
* tags:
* - Sync
* responses:
* 200:
* description: An event stream.
*/
router.get("/v2/events", authenticateJWT, (req: Request, res: Response) => {
const userId = userOf(req);
res.setHeader("Content-Type", "text/event-stream");
res.setHeader("Cache-Control", "no-cache, no-transform");
res.setHeader("Connection", "keep-alive");
res.setHeader("X-Accel-Buffering", "no");
res.flushHeaders?.();
res.write(`event: ready\ndata: {}\n\n`);
const remove = addListener(userId, res);
req.on("close", remove);
});
/**
* @openapi
* /sync/v2/hosts/{syncId}:
* get:
* summary: This server's id for a host, by its sync id
* description: For a linked desktop that asks the server to act on a host (a tunnel through it), which needs the server's own id. Only for hosts the caller can see.
* tags:
* - Sync
* parameters:
* - in: path
* name: syncId
* required: true
* schema:
* type: string
* responses:
* 200:
* description: The host's id and name.
* 404:
* description: No such host the caller can see.
*/
router.get(
"/v2/hosts/:syncId",
authenticateJWT,
async (req: Request, res: Response) => {
const userId = userOf(req);
try {
const { createCurrentHostResolutionRepository } =
await import("../../database/repositories/factory.js");
const hostId =
await createCurrentHostResolutionRepository().findHostIdBySyncId(
String(req.params.syncId),
);
if (hostId === null) return res.status(404).json({ error: "Not found" });
const { PermissionManager } =
await import("../../utils/permission-manager.js");
const access = await PermissionManager.getInstance().canAccessHost(
userId,
hostId,
"view",
);
if (!access.hasAccess)
return res.status(404).json({ error: "Not found" });
const { createCurrentHostRepository } =
await import("../../database/repositories/factory.js");
const row = await createCurrentHostRepository().findById(hostId);
res.json({ id: hostId, name: row?.name ?? row?.ip ?? null });
} catch (error) {
syncLogger.error("Failed to look up a host by sync id", error, {
operation: "sync_host_lookup",
});
res.status(500).json({ error: "Failed to look up host" });
}
},
);
/**
* @openapi
* /sync/v2/host-status:
* get:
* summary: Status of the hosts this account can see, keyed by sync id
* description: For a linked desktop whose hosts connect through this server, so it can show their status against its own copies.
* tags:
* - Sync
* responses:
* 200:
* description: Status entries by host sync id.
*/
router.get(
"/v2/host-status",
authenticateJWT,
async (req: Request, res: Response) => {
const userId = userOf(req);
try {
const { hostStatusService } =
await import("../../hosts/status/host-status-service.js");
const { PermissionManager } =
await import("../../utils/permission-manager.js");
const { createCurrentHostRepository } =
await import("../../database/repositories/factory.js");
const statuses = await hostStatusService.statusesFor(userId, null);
const entries = [...statuses.entries()];
const allowed =
await PermissionManager.getInstance().filterAccessibleHostIds(
userId,
entries.map(([id]) => id),
);
const hostRepository = createCurrentHostRepository();
const result: Record<string, unknown> = {};
for (const [id, entry] of entries) {
if (!allowed.has(id)) continue;
const row = await hostRepository.findById(id);
if (row?.syncId) result[row.syncId] = entry;
}
res.json(result);
} catch (error) {
syncLogger.error("Failed to read host status for sync", error, {
operation: "sync_host_status",
});
res.status(500).json({ error: "Failed to read host status" });
}
},
);
async function listPluginsForDesktop() {
return getPluginRuntime().loader.list();
}
/**
* @openapi
* /sync/v2/plugins:
* get:
* summary: The features this server runs, for a linked desktop to mirror
* description: Every installed plugin with its version, where it came from, whether it is on, how a desktop should treat it and which sync entities it owns.
* tags:
* - Sync
* responses:
* 200:
* description: The plugin list.
*/
router.get(
"/v2/plugins",
authenticateJWT,
async (_req: Request, res: Response) => {
try {
const records = new Map(
(await createCurrentPluginRepository().listAll()).map((record) => [
record.id,
record,
]),
);
const grants = createCurrentPluginPermissionGrantRepository();
const plugins = [];
for (const plugin of await listPluginsForDesktop()) {
plugins.push({
id: plugin.id,
name: plugin.manifest.name,
version: plugin.manifest.version,
source: plugin.source,
// A bundled plugin updated from a signed .tmxplug can be mirrored.
artifact: !!plugin.artifact,
enabled: records.get(plugin.id)?.state === "enabled",
active: plugin.state === "active",
desktop: plugin.manifest.desktop ?? "mirror",
syncEntities: plugin.manifest.contributes?.syncEntities ?? [],
granted: (await grants.listByPlugin(plugin.id)).map(
(grant) => grant.capability,
),
});
}
res.json({ plugins });
} catch (error) {
syncLogger.error("Failed to list plugins for sync", error, {
operation: "sync_plugins",
});
res.status(500).json({ error: "Failed to list plugins" });
}
},
);
/**
* @openapi
* /sync/v2/plugins/{id}/package:
* get:
* summary: Download an installed plugin so a linked desktop can install it
* description: Only for plugins installed on this server, or bundled ones replaced by a signed update. Sends the original .tmxplug when there is one, with its signature in the X-Termix-Signature header, otherwise a fresh archive of the plugin folder.
* tags:
* - Sync
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: string
* responses:
* 200:
* description: The plugin archive.
* 404:
* description: No installed plugin with that id.
*/
router.get(
"/v2/plugins/:id/package",
authenticateJWT,
async (req: Request, res: Response) => {
const pluginId = String(req.params.id);
try {
const plugin = (await listPluginsForDesktop()).find(
(candidate) => candidate.id === pluginId,
);
if (!plugin || (plugin.source !== "user" && !plugin.artifact)) {
return res.status(404).json({ error: "Plugin not found" });
}
const artifact = plugin.artifact ?? "";
res.setHeader("Content-Type", "application/gzip");
if (fs.existsSync(artifact)) {
const sig = `${artifact}.sig`;
if (fs.existsSync(sig)) {
res.setHeader(
"X-Termix-Signature",
Buffer.from(await fs.promises.readFile(sig, "utf8")).toString(
"base64",
),
);
}
return fs.createReadStream(artifact).pipe(res);
}
const temp = path.join(
os.tmpdir(),
`termix-plugin-${pluginId}-${Date.now()}.tmxplug`,
);
await tar.c({ gzip: true, cwd: plugin.dir, file: temp }, ["."]);
const stream = fs.createReadStream(temp);
stream.on("close", () => void fs.promises.rm(temp, { force: true }));
stream.pipe(res);
} catch (error) {
syncLogger.error("Failed to package a plugin for sync", error, {
operation: "sync_plugin_package",
pluginId,
});
if (!res.headersSent) {
res.status(500).json({ error: "Failed to package plugin" });
}
}
},
);
export default router;
src/backend/sync/shared-copy-guard.ts
+39
View File
@@ -0,0 +1,39 @@
import type { NextFunction, Request, Response } from "express";
import { eq } from "drizzle-orm";
import { hosts, sshCredentials } from "../database/db/schema.js";
import { createCurrentRepositoryContext } from "../database/repositories/factory.js";
/**
* On a linked desktop, hosts and credentials someone shared with the account
* are read-only copies kept in step by sync. Editing or deleting one here
* would be undone by the next sync, so it is refused with a clear reason.
*/
export function rejectSharedCopyWrites(
kind: "host" | "credential",
pattern: RegExp,
) {
const table = kind === "host" ? hosts : sshCredentials;
return async (req: Request, res: Response, next: NextFunction) => {
if (process.env.ELECTRON_EMBEDDED !== "true") return next();
if (req.method !== "PUT" && req.method !== "DELETE") return next();
const match = req.path.match(pattern);
if (!match) return next();
try {
const [row] = await createCurrentRepositoryContext()
.drizzle.select({ sharedSource: table.sharedSource })
.from(table)
.where(eq(table.id, Number(match[1])))
.limit(1);
if (row?.sharedSource) {
res.status(403).json({
error: "This was shared with you and is managed on the server",
code: "SHARED_COPY",
});
return;
}
} catch {
// Let the route answer for itself.
}
next();
};
}
src/backend/sync/store.ts
+383
View File
@@ -0,0 +1,383 @@
/**
* Reading and writing a registered entity's rows, whatever table they are in.
*
* Every registration names a drizzle table with an id, a syncId (unless it is
* a singleton) and an owner column, and that is all this relies on. Writes
* read the row back by its key rather than using RETURNING, which MySQL lacks.
*/
import crypto from "crypto";
import { and, eq, getTableColumns, type SQL } from "drizzle-orm";
import type { SyncRow } from "@termix/plugin-sdk/backend";
import { createCurrentRepositoryContext } from "../database/repositories/factory.js";
import { DataCrypto } from "../utils/data-crypto.js";
import { SystemCrypto } from "../utils/system-crypto.js";
import { DatabaseSaveTrigger } from "../utils/database-save-trigger.js";
import {
getEntity,
type RegisteredSyncEntity,
} from "../plugins/sync-registry.js";
import {
decryptFields,
deserializeReferences,
encryptFields,
hashWire,
serializeReferences,
singletonSyncId,
toWire,
toWritable,
type ResolveId,
type ResolveSyncId,
} from "./wire.js";
/* eslint-disable @typescript-eslint/no-explicit-any */
type AnyTable = any;
function tableOf(entity: RegisteredSyncEntity): AnyTable {
if (!entity.table) {
throw new Error(`Sync entity "${entity.type}" has no table`);
}
return entity.table as AnyTable;
}
function db() {
return createCurrentRepositoryContext().drizzle as any;
}
function ownerColumn(entity: RegisteredSyncEntity) {
const column = tableOf(entity)[entity.userColumn];
if (!column) {
throw new Error(
`Sync entity "${entity.type}" has no ${entity.userColumn} column`,
);
}
return column;
}
export function locate(
entity: RegisteredSyncEntity,
userId: string,
syncId: string,
): SQL {
const table = tableOf(entity);
if (entity.singleton) return eq(ownerColumn(entity), userId);
return and(eq(table.syncId, syncId), eq(ownerColumn(entity), userId))!;
}
export function rowSyncId(entity: RegisteredSyncEntity, row: SyncRow): string {
return entity.singleton
? singletonSyncId(entity.type)
: (row.syncId as string);
}
let hashKey: Buffer | null = null;
export async function getHashKey(): Promise<Buffer> {
if (!hashKey) {
const secret = await SystemCrypto.getInstance().getEncryptionKey();
hashKey = Buffer.from(
crypto.hkdfSync(
"sha256",
secret,
Buffer.alloc(0),
"termix-sync-hash",
32,
),
);
}
return hashKey;
}
/** Test seam. */
export function resetHashKey(): void {
hashKey = null;
}
export async function listStoredRows(
entity: RegisteredSyncEntity,
userId: string,
): Promise<SyncRow[]> {
const rows = (await db()
.select()
.from(tableOf(entity))
.where(eq(ownerColumn(entity), userId))) as SyncRow[];
return rows;
}
export async function findStoredRow(
entity: RegisteredSyncEntity,
userId: string,
syncId: string,
): Promise<SyncRow | null> {
const [row] = (await db()
.select()
.from(tableOf(entity))
.where(locate(entity, userId, syncId))
.limit(1)) as SyncRow[];
return row ?? null;
}
/** Gives an old row with no syncId one, so it can take part. */
async function ensureSyncId(
entity: RegisteredSyncEntity,
row: SyncRow,
): Promise<string> {
if (entity.singleton) return singletonSyncId(entity.type);
if (typeof row.syncId === "string" && row.syncId) return row.syncId;
const syncId = crypto.randomUUID();
const table = tableOf(entity);
await db().update(table).set({ syncId }).where(eq(table.id, row.id));
row.syncId = syncId;
return syncId;
}
export interface Resolvers {
resolveSyncId: ResolveSyncId;
resolveId: ResolveId;
}
/**
* Id and syncId lookups for references, scoped to rows the user owns. A core
* entity can widen that (a credential shared with the user is a valid
* reference target on the server).
*/
export function createResolvers(
userId: string,
onMissing?: (entityType: string, syncId: string) => void,
): Resolvers {
const allowed = async (
entityType: string,
ownerId: unknown,
id: number,
): Promise<boolean> => {
if (ownerId === userId) return true;
const canReference = getEntity(entityType)?.canReference;
return canReference ? canReference(userId, String(ownerId), id) : false;
};
return {
resolveSyncId: async (entityType, id) => {
const target = getEntity(entityType);
if (!target?.table || target.singleton) return null;
const table = tableOf(target);
const [row] = await db()
.select({ syncId: table.syncId, owner: ownerColumn(target) })
.from(table)
.where(eq(table.id, id))
.limit(1);
if (!row || !(await allowed(entityType, row.owner, id))) return null;
return row.syncId ?? null;
},
resolveId: async (entityType, syncId) => {
const target = getEntity(entityType);
if (!target?.table || target.singleton) {
onMissing?.(entityType, syncId);
return null;
}
const table = tableOf(target);
const rows = await db()
.select({ id: table.id, owner: ownerColumn(target) })
.from(table)
.where(eq(table.syncId, syncId));
const own = rows.find((row: any) => row.owner === userId);
if (own) return own.id;
for (const row of rows) {
if (await allowed(entityType, row.owner, row.id)) return row.id;
}
onMissing?.(entityType, syncId);
return null;
},
};
}
export interface WireRow {
syncId: string;
wire: SyncRow;
hash: string;
}
export interface LoadResult {
rows: Map<string, WireRow>;
/** Rows that exist but could not be serialized, so must not read as deleted. */
failed: Set<string>;
}
/** The user's rows of one entity, on the wire and hashed. */
export async function loadWireRows(
entity: RegisteredSyncEntity,
userId: string,
resolvers: Resolvers,
options: { filter?: (row: SyncRow) => boolean } = {},
): Promise<LoadResult> {
const key = await getHashKey();
const rows = new Map<string, WireRow>();
const failed = new Set<string>();
if (entity.load) {
for (const wire of await entity.load(userId)) {
const syncId = String(wire.syncId);
rows.set(syncId, { syncId, wire, hash: hashWire(entity, wire, key) });
}
return { rows, failed };
}
const dataKey = DataCrypto.getUserDataKey(userId);
for (const stored of await listStoredRows(entity, userId)) {
if (entity.shouldSync && entity.shouldSync(stored) === false) continue;
if (options.filter && !options.filter(stored)) continue;
const syncId = await ensureSyncId(entity, stored);
try {
const plain = decryptFields(entity, stored, dataKey);
const serialized = await serializeReferences(
entity,
plain,
resolvers.resolveSyncId,
);
const wire = toWire(entity, serialized, syncId);
rows.set(syncId, { syncId, wire, hash: hashWire(entity, wire, key) });
} catch {
failed.add(syncId);
}
}
return { rows, failed };
}
/** One row on the wire, or null when it is gone or excluded. */
export async function loadWireRow(
entity: RegisteredSyncEntity,
userId: string,
syncId: string,
resolvers: Resolvers,
): Promise<WireRow | null> {
if (entity.load) {
const all = await loadWireRows(entity, userId, resolvers);
return all.rows.get(syncId) ?? null;
}
const stored = await findStoredRow(entity, userId, syncId);
if (!stored) return null;
if (entity.shouldSync && entity.shouldSync(stored) === false) return null;
const plain = decryptFields(
entity,
stored,
DataCrypto.getUserDataKey(userId),
);
const serialized = await serializeReferences(
entity,
plain,
resolvers.resolveSyncId,
);
const wire = toWire(entity, serialized, syncId);
return { syncId, wire, hash: hashWire(entity, wire, await getHashKey()) };
}
function columnsOf(entity: RegisteredSyncEntity): Set<string> {
return new Set(Object.keys(getTableColumns(tableOf(entity))));
}
export interface WriteResult {
id: number | null;
created: boolean;
}
/**
* Writes an inbound wire row for the user. `extra` is set on top of the
* inbound fields, for columns this side owns (a shared copy's source).
*/
export async function writeWireRow(
entity: RegisteredSyncEntity,
userId: string,
wire: SyncRow,
resolvers: Resolvers,
extra: SyncRow = {},
): Promise<WriteResult> {
const syncId = String(wire.syncId);
if (entity.write) {
await entity.write(userId, wire);
DatabaseSaveTrigger.triggerSave("sync_write");
return { id: null, created: false };
}
const table = tableOf(entity);
const current = await findStoredRow(entity, userId, syncId);
const resolved = await deserializeReferences(
entity,
wire,
resolvers.resolveId,
current,
);
const columns = columnsOf(entity);
const writable = Object.fromEntries(
Object.entries({ ...toWritable(entity, resolved), ...extra }).filter(
([field]) => columns.has(field),
),
);
const encrypted = encryptFields(
entity,
writable,
DataCrypto.getUserDataKey(userId),
current ? String(current.id) : `sync-${syncId}`,
);
if (columns.has("updatedAt")) encrypted.updatedAt = new Date().toISOString();
const where = locate(entity, userId, syncId);
if (current) {
await db().update(table).set(encrypted).where(where);
} else {
const insert: SyncRow = { ...encrypted, [entity.userColumn]: userId };
if (!entity.singleton) insert.syncId = syncId;
await db().insert(table).values(insert);
}
const written = await findStoredRow(entity, userId, syncId);
const id = typeof written?.id === "number" ? written.id : null;
if (entity.afterWrite) {
await entity.afterWrite({
id,
userId,
wire,
created: !current,
resolveId: resolvers.resolveId,
});
}
DatabaseSaveTrigger.triggerSave("sync_write");
return { id, created: !current };
}
/** Deletes the user's row, through the entity's own remove when it has one. */
export async function deleteStoredRow(
entity: RegisteredSyncEntity,
userId: string,
syncId: string,
): Promise<boolean> {
if (entity.erase) {
await entity.erase(userId, syncId);
DatabaseSaveTrigger.triggerSave("sync_delete");
return true;
}
const current = await findStoredRow(entity, userId, syncId);
if (!current) return false;
if (entity.remove) {
await entity.remove(current, userId);
} else {
await db()
.delete(tableOf(entity))
.where(locate(entity, userId, syncId));
}
DatabaseSaveTrigger.triggerSave("sync_delete");
return true;
}
/** Sets fresh syncIds on every row the user owns, so they read as new. */
export async function regenerateSyncIds(
entity: RegisteredSyncEntity,
userId: string,
): Promise<void> {
if (entity.singleton || entity.load || !entity.table) return;
const table = tableOf(entity);
for (const row of await listStoredRows(entity, userId)) {
await db()
.update(table)
.set({ syncId: crypto.randomUUID() })
.where(eq(table.id, row.id));
}
}
src/backend/sync/wire.ts
+347
View File
@@ -0,0 +1,347 @@
/**
* Turning a stored row into its wire form and back.
*
* On the wire a row has plaintext secrets, a syncId, and syncIds in place of
* every local numeric id it references. Local ids mean nothing on the other
* side of a link, and secrets are encrypted with a key only this side has.
*/
import crypto from "crypto";
import type { SyncEntityReference, SyncRow } from "@termix/plugin-sdk/backend";
import { FieldCrypto } from "../utils/field-crypto.js";
import type { RegisteredSyncEntity } from "../plugins/sync-registry.js";
export type ResolveSyncId = (
entityType: string,
id: number,
) => Promise<string | null>;
export type ResolveId = (
entityType: string,
syncId: string,
) => Promise<number | null>;
/** Keys that describe where a row lives rather than what it says. */
const LOCATION_KEYS = ["id", "syncId", "createdAt", "updatedAt"];
export function singletonSyncId(entityType: string): string {
return `${entityType}:singleton`;
}
interface JsonPath {
column: string;
steps: Array<{ key: string; each: boolean }>;
}
/** "jumpHosts[].hostId" -> column jumpHosts, then each item, then hostId. */
export function parseReferencePath(field: string): JsonPath | null {
if (!field.includes(".") && !field.includes("[]")) return null;
const parts = field.split(".");
const head = parts.shift()!;
const headEach = head.endsWith("[]");
const column = headEach ? head.slice(0, -2) : head;
const steps: JsonPath["steps"] = [];
if (headEach) steps.push({ key: "", each: true });
for (const part of parts) {
const each = part.endsWith("[]");
steps.push({ key: each ? part.slice(0, -2) : part, each });
}
return { column, steps };
}
async function rewritePath(
value: unknown,
steps: JsonPath["steps"],
map: (leaf: unknown) => Promise<unknown>,
): Promise<unknown> {
if (steps.length === 0) return map(value);
const [step, ...rest] = steps;
if (step.key === "" && step.each) {
if (!Array.isArray(value)) return value;
return Promise.all(value.map((item) => rewritePath(item, rest, map)));
}
if (!value || typeof value !== "object" || Array.isArray(value)) {
return value;
}
const object = value as Record<string, unknown>;
if (!(step.key in object)) return value;
const child = object[step.key];
if (step.each) {
if (!Array.isArray(child)) return value;
return {
...object,
[step.key]: await Promise.all(
child.map((item) => rewritePath(item, rest, map)),
),
};
}
return { ...object, [step.key]: await rewritePath(child, rest, map) };
}
/** Rewrites every leaf a JSON path points at, keeping the column's storage form. */
async function rewriteJsonColumn(
row: SyncRow,
path: JsonPath,
map: (leaf: unknown) => Promise<unknown>,
): Promise<void> {
const stored = row[path.column];
if (stored == null || stored === "") return;
let parsed: unknown = stored;
const asString = typeof stored === "string";
if (asString) {
try {
parsed = JSON.parse(stored as string);
} catch {
return;
}
}
const rewritten = await rewritePath(parsed, path.steps, map);
row[path.column] = asString ? JSON.stringify(rewritten) : rewritten;
}
function wireField(reference: SyncEntityReference): string {
return reference.syncField ?? `${reference.field}SyncId`;
}
/** Local ids to syncIds. */
export async function serializeReferences(
entity: RegisteredSyncEntity,
row: SyncRow,
resolveSyncId: ResolveSyncId,
): Promise<SyncRow> {
const out = { ...row };
for (const reference of entity.references ?? []) {
const path = parseReferencePath(reference.field);
if (path) {
await rewriteJsonColumn(out, path, async (leaf) => {
const id = typeof leaf === "string" ? Number(leaf) : leaf;
if (typeof id !== "number" || !Number.isInteger(id)) return leaf;
return (await resolveSyncId(reference.entityType, id)) ?? null;
});
continue;
}
const id = out[reference.field];
out[wireField(reference)] =
typeof id === "number"
? await resolveSyncId(reference.entityType, id)
: null;
delete out[reference.field];
}
return entity.serialize ? entity.serialize(out, resolveSyncId) : out;
}
/**
* SyncIds back to local ids.
*
* A syncId with no local row keeps whatever the column already holds here
* (`current`), rather than failing the whole row. That happens when the
* target's category is not synced on this device, or it has not arrived yet.
*/
export async function deserializeReferences(
entity: RegisteredSyncEntity,
row: SyncRow,
resolveId: ResolveId,
current: SyncRow | null,
): Promise<SyncRow> {
const out = { ...row };
for (const reference of entity.references ?? []) {
const path = parseReferencePath(reference.field);
if (path) {
await rewriteJsonColumn(out, path, async (leaf) => {
if (typeof leaf !== "string" || !leaf) return leaf;
const id = await resolveId(reference.entityType, leaf);
if (id === null) return null;
return reference.idType === "string" ? String(id) : id;
});
continue;
}
const field = wireField(reference);
const syncId = out[field];
delete out[field];
delete out[reference.field];
if (syncId == null || syncId === "") {
out[reference.field] = null;
continue;
}
if (typeof syncId !== "string") continue;
const id = await resolveId(reference.entityType, syncId);
if (id !== null) {
out[reference.field] = id;
} else if (current && reference.field in current) {
out[reference.field] = current[reference.field];
} else {
out[reference.field] = null;
}
}
return entity.deserialize ? entity.deserialize(out, resolveId) : out;
}
/**
* Decrypts the entity's secret fields. Throws when one cannot be decrypted:
* sending an empty value instead would wipe the secret on the other side.
*/
export function decryptFields(
entity: RegisteredSyncEntity,
row: SyncRow,
key: Buffer | null,
): SyncRow {
if (entity.encryptedFields.length === 0) return row;
const out = { ...row };
for (const field of entity.encryptedFields) {
const value = out[field];
if (typeof value !== "string" || !FieldCrypto.isEncrypted(value)) continue;
if (!key) throw new Error("No data key to decrypt synced fields");
out[field] = FieldCrypto.decryptField(value, key, String(row.id), field);
}
return out;
}
export function encryptFields(
entity: RegisteredSyncEntity,
row: SyncRow,
key: Buffer | null,
recordId: string,
): SyncRow {
if (entity.encryptedFields.length === 0) return row;
const out = { ...row };
for (const field of entity.encryptedFields) {
const value = out[field];
if (typeof value !== "string" || !value) continue;
if (FieldCrypto.isEncrypted(value)) continue;
if (!key) throw new Error("No data key to encrypt synced fields");
out[field] = FieldCrypto.encryptField(value, key, recordId, field);
}
return out;
}
/** What goes on the wire: no local ids, no owner, no fields that stay here. */
export function toWire(
entity: RegisteredSyncEntity,
row: SyncRow,
syncId: string,
): SyncRow {
const out = { ...row };
delete out.id;
delete out[entity.userColumn];
if (entity.userColumn !== "userId") delete out.userId;
for (const field of entity.readOnlyFields) delete out[field];
out.syncId = syncId;
return out;
}
/** What may be written from an inbound row. */
export function toWritable(
entity: RegisteredSyncEntity,
wire: SyncRow,
): SyncRow {
const out = { ...wire };
delete out.id;
delete out.syncId;
delete out.userId;
delete out[entity.userColumn];
delete out.createdAt;
delete out.updatedAt;
for (const field of entity.readOnlyFields) delete out[field];
for (const field of entity.hashIgnore ?? []) delete out[field];
return out;
}
function stable(value: unknown): unknown {
if (Array.isArray(value)) return value.map(stable);
if (value && typeof value === "object") {
const object = value as Record<string, unknown>;
return Object.keys(object)
.sort()
.reduce<Record<string, unknown>>((acc, key) => {
if (object[key] !== undefined) acc[key] = stable(object[key]);
return acc;
}, {});
}
return value;
}
/**
* Booleans are stored as 0/1 on SQLite and true/false elsewhere, and JSON
* columns come back as text or parsed depending on the driver. Leveled here
* so the same content always hashes the same.
*/
function normalize(value: unknown): unknown {
if (value === true) return 1;
if (value === false) return 0;
if (value === undefined) return null;
if (typeof value === "string" && /^[[{]/.test(value)) {
try {
return stable(JSON.parse(value));
} catch {
return value;
}
}
return stable(value);
}
/**
* Content hash of a wire row, keyed so a stored hash cannot be used to guess
* a weak secret it covers.
*/
export function hashWire(
entity: RegisteredSyncEntity,
wire: SyncRow,
key: Buffer,
): string {
const skip = new Set([
...LOCATION_KEYS,
...entity.readOnlyFields,
...(entity.hashIgnore ?? []),
]);
const content: Record<string, unknown> = {};
for (const [field, value] of Object.entries(wire)) {
if (skip.has(field)) continue;
const normalized = normalize(value);
if (normalized === null || normalized === "") continue;
content[field] = normalized;
}
return crypto
.createHmac("sha256", key)
.update(JSON.stringify(stable(content)))
.digest("hex");
}
/** Parents before children when an entity references itself. */
export function orderSelfReferences(
entity: RegisteredSyncEntity,
rows: SyncRow[],
): SyncRow[] {
const self = (entity.references ?? []).find(
(reference) =>
reference.entityType === entity.type &&
!parseReferencePath(reference.field),
);
if (!self) return rows;
const field = wireField(self);
const bySyncId = new Map(
rows
.filter((row) => typeof row.syncId === "string")
.map((row) => [row.syncId as string, row]),
);
const ordered: SyncRow[] = [];
const done = new Set<SyncRow>();
const visiting = new Set<SyncRow>();
const visit = (row: SyncRow) => {
if (done.has(row) || visiting.has(row)) return;
visiting.add(row);
const parent = row[field];
if (typeof parent === "string") {
const parentRow = bySyncId.get(parent);
if (parentRow) visit(parentRow);
}
visiting.delete(row);
done.add(row);
ordered.push(row);
};
rows.forEach(visit);
return ordered;
}