Ethica/Termix
1
0 Fork 0
mirror of https://github.com/Termix-SSH/Termix.git synced 2026-10-29 22:55:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

release-2.7.0 (#1264)

Browse Source 
* feat: redesign host/credential sidebars with synced preferences and manual drag-to-reorder

* chore: run format

* chore(deps-dev): bump @types/pg in the dev-patch-updates group (#1162)

Bumps the dev-patch-updates group with 1 update: [@types/pg](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/pg).


Updates `@types/pg` from 8.20.0 to 8.20.3
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/pg)

---
updated-dependencies:
- dependency-name: "@types/pg"
  dependency-version: 8.20.3
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: dev-patch-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps-dev): bump the dev-minor-updates group with 4 updates (#1163)

Bumps the dev-minor-updates group with 4 updates: [react-hook-form](https://github.com/react-hook-form/react-hook-form), [react-icons](https://github.com/react-icons/react-icons), [typescript-eslint](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/typescript-eslint) and [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite).


Updates `react-hook-form` from 7.79.0 to 7.84.0
- [Release notes](https://github.com/react-hook-form/react-hook-form/releases)
- [Changelog](https://github.com/react-hook-form/react-hook-form/blob/master/CHANGELOG.md)
- [Commits](https://github.com/react-hook-form/react-hook-form/compare/v7.79.0...v7.84.0)

Updates `react-icons` from 5.6.0 to 5.7.0
- [Release notes](https://github.com/react-icons/react-icons/releases)
- [Commits](https://github.com/react-icons/react-icons/compare/v5.6.0...v5.7.0)

Updates `typescript-eslint` from 8.61.1 to 8.66.0
- [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases)
- [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/main/packages/typescript-eslint/CHANGELOG.md)
- [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v8.66.0/packages/typescript-eslint)

Updates `vite` from 8.0.16 to 8.2.0
- [Release notes](https://github.com/vitejs/vite/releases)
- [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md)
- [Commits](https://github.com/vitejs/vite/commits/create-vite@8.2.0/packages/vite)

---
updated-dependencies:
- dependency-name: react-hook-form
  dependency-version: 7.84.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: dev-minor-updates
- dependency-name: react-icons
  dependency-version: 5.7.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: dev-minor-updates
- dependency-name: typescript-eslint
  dependency-version: 8.66.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: dev-minor-updates
- dependency-name: vite
  dependency-version: 8.2.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: dev-minor-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump the prod-patch-updates group with 3 updates (#1164)

Bumps the prod-patch-updates group with 3 updates: [jose](https://github.com/panva/jose), [js-yaml](https://github.com/nodeca/js-yaml) and [nanoid](https://github.com/ai/nanoid).


Updates `jose` from 6.2.7 to 6.2.8
- [Release notes](https://github.com/panva/jose/releases)
- [Changelog](https://github.com/panva/jose/blob/main/CHANGELOG.md)
- [Commits](https://github.com/panva/jose/compare/v6.2.7...v6.2.8)

Updates `js-yaml` from 5.2.2 to 5.2.3
- [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md)
- [Commits](https://github.com/nodeca/js-yaml/compare/5.2.2...5.2.3)

Updates `nanoid` from 6.0.0 to 6.0.1
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/6.0.0...6.0.1)

---
updated-dependencies:
- dependency-name: jose
  dependency-version: 6.2.8
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: js-yaml
  dependency-version: 5.2.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
- dependency-name: nanoid
  dependency-version: 6.0.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: prod-patch-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump undici in the prod-minor-updates group (#1165)

Bumps the prod-minor-updates group with 1 update: [undici](https://github.com/nodejs/undici).


Updates `undici` from 8.9.0 to 8.10.0
- [Release notes](https://github.com/nodejs/undici/releases)
- [Commits](https://github.com/nodejs/undici/compare/v8.9.0...v8.10.0)

---
updated-dependencies:
- dependency-name: undici
  dependency-version: 8.10.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: prod-minor-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps-dev): bump the major-updates group with 2 updates (#1166)

Bumps the major-updates group with 2 updates: [@types/better-sqlite3](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/better-sqlite3) and [jsdom](https://github.com/jsdom/jsdom).


Updates `@types/better-sqlite3` from 7.6.13 to 9.6.0
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/better-sqlite3)

Updates `jsdom` from 29.1.1 to 30.0.1
- [Release notes](https://github.com/jsdom/jsdom/releases)
- [Commits](https://github.com/jsdom/jsdom/compare/v29.1.1...v30.0.1)

---
updated-dependencies:
- dependency-name: "@types/better-sqlite3"
  dependency-version: 9.6.0
  dependency-type: direct:development
  update-type: version-update:semver-major
  dependency-group: major-updates
- dependency-name: jsdom
  dependency-version: 30.0.1
  dependency-type: direct:development
  update-type: version-update:semver-major
  dependency-group: major-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* fix: stop resyncAutoIncrement failing on tables without an id column (#1173)

The Postgres branch asked pg_get_serial_sequence(table, 'id') about every
table a fixture had inserted into. That function raises 42703 when the
column does not exist, rather than returning null, so any seed touching a
table keyed on something else took down the fixture.

host_sidebar_preferences is keyed on user_id and has no id at all, which
is why the Postgres job on dev-2.7.0 fails for every pull request.

Drive the lookup from information_schema so a missing id column yields no
row instead of an error. A text primary key still returns a null sequence
and is still skipped, as before.

* chore: install the git hooks that were already configured (#1174)

husky, lint-staged, commitlint and their config have been in the repo
since v1.8.0 (#429): .husky/pre-commit runs lint-staged, .husky/commit-msg
runs commitlint, the lint-staged globs are in package.json and the
commitlint rules in .commitlintrc.json.

None of it has ever run. husky only takes effect once it sets
core.hooksPath, and that happens in the prepare lifecycle script, which
the package did not define -- so every clone installed the tooling and
left the hooks unwired.

That is why formatting keeps failing in CI rather than locally: three of
the four open pull requests fail lint-and-build on prettier alone,
touching between one and five files each, and the check is the first place
anyone finds out.

prepare falls back to true so a checkout without a .git directory cannot
break installation. The Docker build passes --ignore-scripts, so it never
runs this at all.

Also pin the Prettier extension to the repo's own copy via
prettier.prettierPath, and let .vscode/settings.json out of .gitignore so
it applies to everyone. The extension bundles its own prettier otherwise,
which formats to a different version's rules than the one CI enforces.

* fix: derive the ssh_credentials rebuild from the live schema (#1172)

The startup rebuild that drops the old username NOT NULL constraint
restated the table's columns as a literal and then copied rows with
INSERT INTO temp SELECT <every live column>. The table has gained columns
since that literal was written — cert_public_key, pin, sort_order and
sync_id are all added by addColumnIfNotExists before the rebuild runs —
so the destination was narrower than the source. SQLite rejected the
INSERT on a column count mismatch, the error was swallowed as a warning,
and the constraint survived every restart.

Read the CREATE TABLE statement back from sqlite_master and rewrite just
the table name and the username constraint, so the replacement table
cannot fall behind the real one. Copy rows by explicit column name rather
than positionally, and replay the table's indexes, which DROP TABLE would
otherwise take with it along with the sync_id uniqueness.

* fix: make audit_logs.user_id nullable on fresh SQLite installs (#1171)

The audit trail is meant to outlive the account it belongs to: deleting a
user nulls user_id and keeps username for attribution. schema.ts, the
Drizzle migrations and AuditLogRepository.anonymizeByUserId were all
written against that, but the runtime bootstrap still created
user_id TEXT NOT NULL.

A second CREATE TABLE IF NOT EXISTS further down migrateSchema() had the
correct nullable column, but it can never run — the primary bootstrap has
already created the table, so IF NOT EXISTS is a no-op. Every fresh
install therefore got the old constraint, and user deletion failed with
"NOT NULL constraint failed: audit_logs.user_id" for any account that had
logged in at least once, via both the admin delete path and the OIDC
account-link cleanup.

Fix the primary bootstrap, and rebuild the table on existing databases
using the same pattern already used for ssh_credentials.username, since
SQLite cannot ALTER a column.

* fix: key the sync upsert on the row it just looked up (#1175)

A sync push locates the stored row twice -- once to decide insert vs
update, once to write it -- and the two lookups were spelled out
separately. Only the read knew about singleton entities; the write always
keyed on table.id.

userPreferences is the only singleton, and user_preferences is the one
synced table with no id column: its primary key is user_id. table.id was
therefore undefined, and drizzle emitted a comparison with nothing on its
left:

  ( = ? and "user_preferences"."user_id" = ?)

The insert branch was unaffected, so the first push of preferences
succeeded and every push after it -- the steady state -- failed with
SqliteError: near "=": syntax error. Preference sync never converged, and
both sides ship the same handler, so the desktop's embedded backend failed
identically.

Extract the lookup into locateSyncRow() and use it for the read, the
update and the tombstone delete, so the three cannot drift apart again.
The tombstone path already handled singletons correctly; it now shares the
one expression rather than keeping a third copy of it.

* fix: refuse an SSH connection whose host id resolves elsewhere (#1176)

A client identifies a host by the numeric row id of the database it is
displaying. With the desktop connection origin set to "Remote server",
that id is resolved against the sync server's ssh_data instead, and the
two autoincrement sequences need not line up -- they diverge as soon as
each side accumulates inserts and deletes in a different order.

resolveHostById() then returns whichever row owns that id here, and the
handler takes the address, the credentials, the jump hosts and the stored
host key from it. The session opens on a machine the user did not pick,
while the host list, host details and export all keep showing the right
one. Commands run on the wrong server, a host key mismatch is reported
for the wrong reason, and anything typed at the prompt goes to the wrong
place.

Compare the resolved address against the one the client sent, and refuse
when they disagree. Checking at the point the row is loaded covers every
use of it rather than each site separately. Addresses are compared with
brackets stripped and casing folded, so an IPv6 literal or a hostname
written differently is not treated as a different machine; when the
server has no address stored, the client's own details are used as
before.

This stops the wrong-machine session. It does not make delegated
connections work when the ids have drifted -- that needs the host to be
addressed by syncId across the boundary, which the connection protocol
does not currently carry.

* fix: refuse SFTP and Docker console on a mismatched host id too (#1177)

The wrong-machine guard added for SSH covered one of the paths that
resolve a client-supplied host id against this server's ssh_data. The
file manager and the Docker console take the same id from the same client
and dial whatever row owns it here.

The file manager then browses, edits and deletes files on that machine,
and the Docker console attaches to its daemon -- both while the UI shows
the host the user actually picked.

Reuse hostAddressMismatch at each point the row is loaded. The two file
manager sites sit inside "failed to resolve credentials, carry on"
handlers, so the refusal is a distinct error type those catches rethrow;
swallowing it would resume the connection this is meant to stop. The
Docker console reports over its socket, as it does for every other
refusal.

The user-facing wording now lives next to the check instead of being
written out at each site.

Still uncovered, and not fixable this way: file-manager's transfer
session, jump-host-chain and the proxmox routes resolve an id with no
client-supplied address to compare it against. Those need the host to be
addressed by syncId across the boundary.

* feat: address hosts by syncId when a connection is delegated (#1178)

A numeric host id belongs to the database that produced it. The desktop
app lists hosts from its embedded database and names them by row id, so
when a connection is delegated to a sync server that id is resolved
against a different table, whose autoincrement sequence has no reason to
agree. The row it lands on is a different machine, and it supplies the
address, the credentials, the jump hosts and the stored host key.

#1176 and #1177 made that refuse rather than connect. Refusing is right,
but it leaves "Remote server" unusable once the ids have drifted, which
is the state the reporter was in.

syncId already names a host identically on both sides -- remote sync
relies on it, ssh_data.sync_id is unique, and the API already returns it.
It just never reached the backend: hostToSSHHost() builds its result field
by field and dropped it.

Carry it through, and resolve with it when it is present:

  resolveHostBySyncId(syncId, userId)   // translate, then reuse
    -> findHostIdBySyncId(syncId)       // this database's own row id
    -> resolveHostById(hostId, userId)  // permissions, decryption, audit

The translation is deliberately not scoped to a user -- sync_id is unique
across the table and a shared host belongs to someone else -- so access
stays with the permission check in the id-based path, which the new tests
cover.

An unknown syncId resolves to nothing rather than falling back to the
numeric id: an unknown host is precisely where guessing picks the wrong
machine. Clients that send no syncId are unchanged, address comparison
included, so an older desktop keeps its safety net instead of breaking.

* fix(homepage): make the System Overview update indicator able to fire (#1168)

The widget's "Update available" row and orange version text were unreachable,
for two independent reasons that each alone would have been enough.

It called `getVersionInfo(false)`, and `checkRemote=false` makes /version return
early with `{localVersion, status: "update_check_disabled"}` -- no GitHub fetch,
no remote version, nothing to compare. It then read `info.updateAvailable`, a
field the route does not return in either mode; the success response carries
status, localVersion, version, remoteVersion, latest_release, cached and
cache_age. `Boolean(undefined)` is false, always. The read type-checked only
because `getVersionInfo()` is declared as `Record<string, unknown>`, so a
property name that does not exist is indistinguishable from one that does.

Let the endpoint do the comparison and read `status === "requires_update"`,
which is what the dashboard stats bar and the profile panel badge already do.

The row's label was `homepage.overviewUpdate`, whose English string is "Up to
date" -- as the label of an update-available row it read "Up to date / Update
available". Nobody has seen that, because the row has never rendered; fixing
the indicator without the label would have shipped it. Give it its own key.
That leaves `homepage.overviewUpdate` unused; it is left in place rather than
removed, since it would be the natural value for an always-visible row and that
is a product decision, not part of this fix.

* fix: capture real client IP for SSH login alerts behind reverse proxy (#1169)

* fix: capture real client IP for SSH login alerts behind reverse proxy

The WebSocket terminal handler used req.socket.remoteAddress for the
"user logged in" alert message, which is the immediate TCP peer (the
reverse proxy) rather than the actual client IP forwarded via
X-Forwarded-For. This made trust-proxy config on Traefik irrelevant
since Termix never read the header for this code path.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* test: cover getClientIp forwarded-header and socket fallback paths

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

* fix: keep already-shared hosts sharing their SSH authentication (#1179)

Sharing a host used to hand the owner's SSH authentication to the
recipient unconditionally. 2.6.1 put that behind ssh_data.share_ssh_auth,
added as NOT NULL DEFAULT 0.

Existing rows took the default, so every host shared before the upgrade
stopped supplying credentials the moment the column appeared. The snapshot
in collectProtocolSnapshots() is guarded by host.shareSshAuth, so nothing
was captured; resolveRecipientSharedHostAuthentication() then fell through
to "required" and the recipient got "No valid authentication method
provided" on a host that had worked the day before. Downgrading to 2.6.0
restored it, since that code has no such column to consult.

Backfill the flag for hosts that already appear in host_access. That is
where the previous behaviour was in effect and where the owner had already
agreed to share; hosts nobody has shared keep the new default and stay off
until their owner shares them.

Guarded by a settings key so it runs once. Without that, an owner who
turns sharing back off would have it turned on again by the next restart.

* fix: let a single credential disable 2FA again (#1180)

The disable dialog has one field, labelled "Enter TOTP code or password",
and its caller passes that value as disableTOTP(input) -- so it arrives as
`password` with `totp_code` undefined. That call has been unchanged since
v2.3.0.

2.5.1 changed the route to require both:

  if (!totp_code || (!userRecord.isOidc && !password)) -> 400

replacing `const credential = password || totp_code`. The first check has
rejected every attempt since, whatever the user typed, so nobody has been
able to turn 2FA off -- the client reports the generic "Failed to disable
2FA", which hides which check failed.

Take one credential again and try it as a TOTP code, a backup code, then
the account password. verifyTotpReauth still refuses the password itself,
so that comparison stays in the route; an OIDC user has no password hash
and reaches neither.

The backup-codes route has the same shape but no caller in the UI -- its
codes are returned when TOTP is enabled -- so it is left alone rather than
changed blind.

* fix: attach user-managed CA certificates over SFTP too (#1181)

opkssh-cert-auth.ts exports two helpers that end in the same
_applyCertToConnection: setupOPKSSHCertAuth, and setupCACertAuth for
user-managed CA-signed -cert.pub files. The file manager called the first
one twice and the second one never.

So a host whose key is paired with a CA-signed certificate authenticated
in a terminal and failed over SFTP, while OPKSSH certificates -- going
through the other helper -- worked in both. The file manager was not
missing certificate support in general; it was missing one of the two
paths into it.

The connection also never carried the certificate to begin with:
cert_public_key was not among the fields copied into resolvedCredentials,
so both places that build an SFTP connection now read it and attach it
where the private key is prepared -- the dedicated transfer session and
the main connect route.

An unusable certificate is logged and skipped rather than failing the
connection. The key alone may still be accepted, which is what happened
while this was not wired up at all, and turning that into a hard failure
would break setups that currently work.

Reported in #1160 with the call-site asymmetry already traced; the
reporter noted they could not confirm the link to their failure, having
moved off SSH CAs. The asymmetry is real either way and reproduces the
symptom exactly.

* fix: authenticate the desktop Docker console WebSocket (#1182)

The console WS opted out of the query token:

  buildOriginWsUrl({ ..., includeLocalJwt: false })

leaving it with no credential at all on the desktop. The browser
WebSocket API cannot set an Authorization header, and while Electron's
main process injects a remembered JWT cookie, it requires an exact origin
match -- the cookie belongs to the API origin (localhost:30001) while the
console connects to 127.0.0.1:30009, so nothing is attached.

The backend then closes the handshake with 1008 before it logs anything,
which is why the log has no docker-console entries while stats and logs
polling keep succeeding on the same host. The web build is unaffected: it
connects same-origin and its cookie is sent normally.

Drop the opt-out so the console carries the local JWT like the SSH
terminal does -- the same token, the same query parameter, and the
backend already reads it there.

Guacamole passes includeLocalJwt: false too, but rdp/vnc/telnet always
resolve to "remote", so that call never reaches the local branch.

* fix: use getClientIp in getRequestMeta for correct audit-log IPs (#1183)

* fix: capture real client IP for SSH login alerts behind reverse proxy

The WebSocket terminal handler used req.socket.remoteAddress for the
"user logged in" alert message, which is the immediate TCP peer (the
reverse proxy) rather than the actual client IP forwarded via
X-Forwarded-For. This made trust-proxy config on Traefik irrelevant
since Termix never read the header for this code path.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* test: cover getClientIp forwarded-header and socket fallback paths

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fix: use getClientIp in getRequestMeta for correct audit-log IPs

getRequestMeta had near-duplicate, strictly worse forwarded-header
logic: the array branch didn't split/trim, there was no socket-peer
fallback, and it returned "" instead of "unknown". Delegate to
getClientIp so the audit trail gets the same correctness as the
terminal login-alert path.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

* feat: add terminal image handoff (#1170)

* chore: sync Crowdin translations

* fix(homepage): make the System Overview update indicator able to fire (#1168)

The widget's "Update available" row and orange version text were unreachable,
for two independent reasons that each alone would have been enough.

It called `getVersionInfo(false)`, and `checkRemote=false` makes /version return
early with `{localVersion, status: "update_check_disabled"}` -- no GitHub fetch,
no remote version, nothing to compare. It then read `info.updateAvailable`, a
field the route does not return in either mode; the success response carries
status, localVersion, version, remoteVersion, latest_release, cached and
cache_age. `Boolean(undefined)` is false, always. The read type-checked only
because `getVersionInfo()` is declared as `Record<string, unknown>`, so a
property name that does not exist is indistinguishable from one that does.

Let the endpoint do the comparison and read `status === "requires_update"`,
which is what the dashboard stats bar and the profile panel badge already do.

The row's label was `homepage.overviewUpdate`, whose English string is "Up to
date" -- as the label of an update-available row it read "Up to date / Update
available". Nobody has seen that, because the row has never rendered; fixing
the indicator without the label would have shipped it. Give it its own key.
That leaves `homepage.overviewUpdate` unused; it is left in place rather than
removed, since it would be the natural value for an always-visible row and that
is a product decision, not part of this fix.

* fix: capture real client IP for SSH login alerts behind reverse proxy (#1169)

* fix: capture real client IP for SSH login alerts behind reverse proxy

The WebSocket terminal handler used req.socket.remoteAddress for the
"user logged in" alert message, which is the immediate TCP peer (the
reverse proxy) rather than the actual client IP forwarded via
X-Forwarded-For. This made trust-proxy config on Traefik irrelevant
since Termix never read the header for this code path.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* test: cover getClientIp forwarded-header and socket fallback paths

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

* feat: add terminal image handoff

Add authenticated browser upload and clipboard image handoff for terminal agents. Normalize images through Sharp, enforce storage and request limits, preserve host-visible paths, and provide a stable three-button terminal toolbar.

* docs: document terminal image handoff deployment

---------

Co-authored-by: LukeGus <bugattiguy527@gmail.com>
Co-authored-by: kacperpietrzyk <105545577+kacperpietrzyk@users.noreply.github.com>
Co-authored-by: Brennan Neoh <497569+brennanneoh@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

* fix(desktop): stop suppressing the update prompt, and make the version badge reachable (#1167)

* fix(desktop): stop suppressing the update prompt for users who need it

The startup update modal stored its dismissal under the local app version
rather than the remote version being offered, and the up-to-date branch
wrote that key with no user interaction at all. A user who launched while
current had their own version recorded; once the next release shipped,
`dismissedVersion === currentVersion` still held and the modal was skipped
on every launch. It reappeared only after the user had already updated --
the inverse of what it is for. Present since v2.3.0.

Key the dismissal on the offered remote version instead. The change is
backward compatible: an existing key holding 2.6.0 compares unequal against
a remote 2.6.1, so affected installs are prompted on their next launch. When
the check itself fails there is no remote version, so nothing is recorded and
no future prompt is suppressed.

That left the version badge as the only remaining signal, and it was an inert
span on both surfaces that render it -- the profile panel and the dashboard
stats bar -- even though the `getVersionInfo()` response it is built from
already carries `latest_release.html_url`. Extract the duplicated badge into
`components/version-badge.tsx` and make the update case a link to the release,
with an accessible name that says where it goes. The beta and stable cases
stay inert.

`getVersionInfo()` returned `Record<string, unknown>`, so the release URL was
unreachable without a cast; give it a `VersionInfo` type that keeps an index
signature, since `SystemOverviewWidget` reads `updateAvailable` off the same
response.

* test: cover the read that actually reaches the badge

The extracted VersionBadge is unit-tested, but the line that decides whether
it ever receives a URL -- pulling `latest_release.html_url` out of the version
response -- was duplicated at both call sites and asserted nowhere. A wrong
property there compiles (the response type keeps an index signature) and every
existing test still passes.

Give it a name, `releaseUrlFrom`, use it from both surfaces, and test it: the
happy path, a response with no release, a release with no URL, and a missing
response, since the caller's fetch can reject. Empty string is the contract the
badge reads as "nothing to link to", so it stays an inert span rather than
rendering a dead anchor.

* docs: state the index signature's real reason

The comment claimed the version endpoint carries fields beyond the typed ones,
citing `updateAvailable`. It does not -- `GET /version` returns status,
localVersion, version, remoteVersion, latest_release, cached and cache_age, and
nothing else. SystemOverviewWidget reads `updateAvailable` off it regardless,
which is why the permissive index signature has to stay, but that is a stale
read rather than an undocumented field. Say so accurately.

* Send alerts in Discord channels with Webhooks (#1158)

* feat(utils): add discord webhook sender

Add a utility to send alert embeds to Discord webhooks.

* fix(utils): validate DNS and use global fetch for outbound requests

Prevent private destination access and rely on global fetch after DNS validation.

* chore(logger): include extra context in logs

Show additional sanitized context entries for clearer diagnostics.

* feat(alerts): support discord channel type in routes and engine

Accept discord channels and route alerts to the Discord sender.

* feat(ui): add Discord option to notification channel dialog

Allow creating/editing Discord webhook channels with username/avatar.

* fix(ui/api): accept structured config payload for notification channels

Allow the client to pass structured config objects (or strings) when creating/updating channels.

* chore: sync Crowdin translations

* fix(homepage): make the System Overview update indicator able to fire (#1168)

The widget's "Update available" row and orange version text were unreachable,
for two independent reasons that each alone would have been enough.

It called `getVersionInfo(false)`, and `checkRemote=false` makes /version return
early with `{localVersion, status: "update_check_disabled"}` -- no GitHub fetch,
no remote version, nothing to compare. It then read `info.updateAvailable`, a
field the route does not return in either mode; the success response carries
status, localVersion, version, remoteVersion, latest_release, cached and
cache_age. `Boolean(undefined)` is false, always. The read type-checked only
because `getVersionInfo()` is declared as `Record<string, unknown>`, so a
property name that does not exist is indistinguishable from one that does.

Let the endpoint do the comparison and read `status === "requires_update"`,
which is what the dashboard stats bar and the profile panel badge already do.

The row's label was `homepage.overviewUpdate`, whose English string is "Up to
date" -- as the label of an update-available row it read "Up to date / Update
available". Nobody has seen that, because the row has never rendered; fixing
the indicator without the label would have shipped it. Give it its own key.
That leaves `homepage.overviewUpdate` unused; it is left in place rather than
removed, since it would be the natural value for an always-visible row and that
is a product decision, not part of this fix.

* fix: capture real client IP for SSH login alerts behind reverse proxy (#1169)

* fix: capture real client IP for SSH login alerts behind reverse proxy

The WebSocket terminal handler used req.socket.remoteAddress for the
"user logged in" alert message, which is the immediate TCP peer (the
reverse proxy) rather than the actual client IP forwarded via
X-Forwarded-For. This made trust-proxy config on Traefik irrelevant
since Termix never read the header for this code path.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* test: cover getClientIp forwarded-header and socket fallback paths

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

* chore: add url to SENSITIVE_FIELDS for discord url

* fix: enforce SSRF protection on outbound fetches

Use `undici.fetch` with the custom DNS lookup hook to ensure the validated
DNS resolution is the one used for the connection. Fix DNS lookup/address
validation bugs and add coverage for private, public and invalid addresses,
including the resolution issue affecting Discord endpoints.

* chore: prettier format

* fix: validate all DNS addresses and close dispatcher

* fix DNS lookup validation and callback handling
* update safe outbound fetch tests
* ensure created dispatcher is properly closed

* chore: remode url from SENSITIVE_FIELDS for other logs

---------

Co-authored-by: LukeGus <bugattiguy527@gmail.com>
Co-authored-by: kacperpietrzyk <105545577+kacperpietrzyk@users.noreply.github.com>
Co-authored-by: Brennan Neoh <497569+brennanneoh@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

* fix tmux UTF-8 path handling (#1157)

Co-authored-by: Carl <scarlettme@qq.com>

* chore: update package lock

* chore: update gitnore

* fix: [BUG] (#1049)

https://github.com/Termix-SSH/Support/issues/1049

* fix: test commitlint path fix (#1021)

* fix: SGR mouse-tracking escape codes printed as text (#1023)

* fix: quote $1 in commit-msg hook so it works from git worktrees

* fix: [BUG] could not connect to the database (#1057)

https://github.com/Termix-SSH/Support/issues/1057

* fix: [BUG] VNC connect macOS screen sharing failed (#1063)

https://github.com/Termix-SSH/Support/issues/1063

* fix: [BUG] Meta key (#1075)

https://github.com/Termix-SSH/Support/issues/1075

* fix: [BUG] Remote sync doesn't work with Termix behind nginx proxy (#1085)

https://github.com/Termix-SSH/Support/issues/1085

* fix: [BUG] webhook not working (#1080)

https://github.com/Termix-SSH/Support/issues/1080

* fix: [BUG] First server sync doesn't refresh UI (#1084)

https://github.com/Termix-SSH/Support/issues/1084

* fix: [BUG] How to enable SSL using custom certificate (#1083)

https://github.com/Termix-SSH/Support/issues/1083

* fix: [BUG] Sudo Password Auto-fill Persistance (#1098)

https://github.com/Termix-SSH/Support/issues/1098

* feat: [FEATURE] Expand Snippets Function (#1031)

https://github.com/Termix-SSH/Support/issues/1031

* feat: [FEATURE] (#1055)

https://github.com/Termix-SSH/Support/issues/1055

* feat: [FEATURE] Support for Headscale API Keys (hskey prefix) and Custom API Endpoints (#1013)

https://github.com/Termix-SSH/Support/issues/1013

* feat: [FEATURE] Allow paste on non https (#1026)

https://github.com/Termix-SSH/Support/issues/1026

* feat: be-azerty layout (#1073)

https://github.com/Termix-SSH/Support/issues/1073

* feat: Keyboard shortcuts to move between open tabs (#1069)

https://github.com/Termix-SSH/Support/issues/1069

* feat: Session Logs as a downloadable text file (#1058)

https://github.com/Termix-SSH/Support/issues/1058

* fix: persist and auto-fill saved SSH and sudo passwords

* fix: persist docker runtime selection and docker manager UI issues

* feat: Allow excluding specific mounts from disk usage metrics (#1046)

https://github.com/Termix-SSH/Support/issues/1046

* feat: Expand Snippets Function (#1031)

https://github.com/Termix-SSH/Support/issues/1031

* chore: restore the prettier baseline on dev-2.7.0 (#1185)

Five files on dev-2.7.0 do not match prettier, so `npx prettier --check .`
fails and takes lint-and-build with it — on every pull request, whatever
it changes.

Formatting only, produced by `npx prettier --write` on exactly the files
the check names. No logic touched: tsc passes for both configs, backend
148 files / 1106 tests and UI 71 files / 479 tests all pass.

* test: keep the tmux escaping test runnable on Windows (#1184)

The escaping check ran its command through /bin/sh. That binary does not
exist on Windows, and Windows is a supported platform for the desktop
app, so `npm test` fails there on a test about string quoting. CI is
ubuntu-only and would never see it.

Assert the escaped string directly, which covers the rule on every
platform, and keep the round trip through a real shell as a separate case
guarded by platform -- it is the stronger evidence where a shell exists.

* chore: drop the unreachable table probes from migrateSchema (#1186)

Eleven blocks in migrateSchema() guarded a CREATE TABLE IF NOT EXISTS
behind SELECT id FROM <table> LIMIT 1, for tables the primary bootstrap
had already created earlier in the same startup. The probe could not
throw, so the catch never ran.

Two of those unreachable copies had drifted from the definition actually
in use. sessions had lost ON DELETE CASCADE, and session_recordings still
carried user_id TEXT NOT NULL with ON DELETE CASCADE and no username --
the shape from before audit trails were made to outlive the account. They
would have taken effect had anything ever reordered startup.

Kept, because they are not the same thing:

  - blocks whose catch runs ALTER TABLE ADD COLUMN. CREATE TABLE IF NOT
    EXISTS is a no-op on a table that exists, so a database created before
    a column was added still needs the ALTER. Those probe a column, not a
    table.
  - blocks that are a table's only creation point.
  - the user_open_tabs block, which is a data migration; its SELECT is a
    precondition, not a probe.

Deletion only, no behaviour change.

* fix: repair the frontend type-check and clear the 299 errors behind it (#1189)

* fix: repair frontend type-check configuration and the errors it exposed

The root tsconfig.json is solution-style with "files": [], so the
`npx tsc --noEmit` that CI runs compiles nothing at all. Frontend types
have therefore never been checked, and 299 errors had accumulated behind
that no-op. This clears just over half of them; nothing here changes
runtime behaviour.

Configuration:
- "@/types" resolved through the "@/*" fallback to src/ui/types, which
  does not exist. Added an explicit mapping to src/types/index.ts.
- src/vite-env.d.ts sits outside the include list, so import.meta.env and
  the ?url import suffix were unknown. Added.
- src/ui/types/ held a single file, keybindings.ts, while every other
  shared type lives in src/types/. Six modules imported it as
  "@/types/keybindings" and silently resolved to nothing. Moved.

Type definitions that had fallen behind the code:
- guacamoleConfig and terminalConfig were Record<string, unknown> in
  ui-types while the editor read concrete fields off them. Both now use
  the real interfaces; GuacamoleConfig is extracted from its inline
  definition in guacamole-api.ts so the two cannot drift again.
- customThemeColors and TerminalTheme["colors"] described the same object
  with different optionality. Aligned.
- FileWindow declared its own SSHHost whose authType was "password" | "key",
  which no longer matches the eight the app supports.
- connectSSH and listSSHFiles returned Record<string, unknown>, so every
  field the callers destructured arrived as unknown.
- AxiosRequestConfig and AxiosResponse were used without being imported.

Also adds asHttpError() for the handful of catch blocks that reached into
an unknown binding, and narrows the Host | HostFolder comparator and the
RailItem union at the points where the discriminant was not carrying.

Note: dbHealthMonitor.reportDatabaseError was being called with a second
argument it does not accept, so the authenticated-or-not flag was already
being discarded at runtime. Dropped the argument to match the signature;
whether that flag was meant to gate the report is worth a separate look.

* fix: clear more of the frontend type-check baseline

Continues the previous commit; 140 errors down to 70. Three of these were
real defects rather than missing annotations.

Defects:
- DashboardTab counted active tunnels by comparing status to "CONNECTED",
  but CONNECTION_STATES.CONNECTED is "connected" and that is what the
  tunnel manager emits, so the count was always zero. Now compares against
  the constant.
- QuickActionsCard requires isAdmin and gates a block of admin-only actions
  on it, but neither call site passed it — those actions never rendered.
  Both call sites also passed isAdmin to HostStatusCard, which does not
  accept it; the prop had evidently been moved and the call sites missed.
- The host editor stores jump host ids as strings and sent them straight to
  an API typed for numbers. Backend host lookups compare against an integer
  column, which a string does not match on Postgres or MySQL. Converted.

Types brought in line with the data:
- Host and HostData were missing hasPassword, hasSudoPassword, sortOrder,
  instanceId, connectionOrigin, vaultProfileId, syncId, and the "vault"
  authType; TabContextTab was missing the "tunnel" tab, which TabContext
  already branched on.
- statsConfig and terminalConfig used inline shapes that had drifted from
  StatsConfig and TerminalConfig. Both now reference the real interfaces;
  excludedMounts, which the editor reads, was added to StatsConfig.
- downloadSSHFile, generateKeyPair and generatePublicKeyFromPrivate all
  returned Record<string, unknown> while callers read named fields.
- The Guacamole declarations were missing Keyboard.reset, Client.onfile,
  InputStream.sendAck, Status.Code and BlobReader, all already in use.
- NetworkTopologyNode/Edge could not be discriminated, though the graph
  code tells them apart by testing for source/target.

ProxyNode.type is now 4 | 5 | "http" | "socks4" | "socks5". The editor
writes the string spellings while proxy-helper.ts tests for "http" and
casts everything else to 4|5 before handing it to the socks client, so a
chained proxy reaches it as "socks5" rather than 5. Typed as what is
actually stored; reconciling the two spellings needs a migration decision
and is left alone here.

* fix: continue clearing the frontend type-check baseline

70 errors down to 44.

Dead configuration removed:
- Terminal set terminal.options.bellStyle on xterm, which dropped the
  option in v5. The host editor still exposes the setting and stores it;
  it has simply had no effect on the terminal since that upgrade. Making
  the bell work again means handling the onBell event and is left alone.
- CodeEditor passed scrollPastEnd to basicSetup, which has no such option.
- FileManager passed an id to openWindow, which assigns its own and
  discards what it is given — the component was already being rendered
  under a different id than the one the caller held.

Widgets that were registered but unreachable:
- DockerActivityWidget and SshQuickConnectWidget register under
  "docker_activity" and "ssh_quick_connect", neither of which was in
  WidgetTypeId, and both referenced config interfaces that did not exist.
  Added the ids and the two interfaces, inferred from their edit forms and
  defaultConfig.

More endpoints given their real return types: getRecentFiles,
getPinnedFiles, getFolderShortcuts (arrays, not records), downloadSSHFile,
copySSHItem, generateKeyPair, generatePublicKeyFromPrivate and getSnippets.

parseGuacamoleConfig() handles the host row carrying guacamoleConfig either
parsed or as raw JSON, which GuacamoleApp was reading fields off directly.
TerminalHostConfig was missing name, which it reads for the activity log.

* fix: continue clearing the frontend type-check baseline

44 errors down to 17.

Host and AuditLog are now type aliases rather than interfaces. An
interface has no implicit index signature, so neither could be assigned
to the `[key: string]: unknown` shapes that TerminalHostConfig,
HostMetricsTab's HostConfig and several helpers declare — eight errors
came from that alone.

More dead configuration:
- i18n passed checkWhitelist to the language detector, which no longer
  has that option; supportedLngs already covers it.
- SSHAuthDialog passed scrollPastEnd to basicSetup, same as CodeEditor.
- AudioPreview's onLoadedMetadata never fired: react-h5-audio-player
  spells the prop onLoadedMetaData.
- MarkdownRenderer destructured `inline` from code(), which react-markdown
  removed in v9, so the flag was always undefined and every inline span
  took the block branch when it happened to carry a language class. Now
  derived from whether a className is present at all.
- SnippetsPanel put a title prop on a lucide icon, which does not forward
  it; changed to aria-label so the hint is actually reachable.

updateHostConfig in TabContext replaced tab.hostConfig wholesale with the
six-field literal it receives, dropping everything else the tab held about
the host. It now merges onto the existing config.

Also: getReleasesRSS, getUserAlerts and getVersionInfo have real return
types (UpdateLog kept private copies of two of them, and VersionInfo was
missing `version`, which the endpoint sends and the panel renders);
wakeOnLan and vncCredentialId get the numeric ids they are typed for; and
the tmux formatter takes i18next's TFunction instead of a hand-written
signature it does not satisfy.

* fix: clear the last frontend type errors and make CI actually run the check

Baseline is now zero, so the check can be turned on.

`npx tsc --noEmit` — what CI ran and what `npm run type-check` was — compiles
nothing: the root tsconfig.json is solution-style with "files": [], and
plain tsc does not follow project references. Both are now `tsc -b`, which
builds tsconfig.app.json and tsconfig.node.json. Verified by planting a type
error and watching the command fail.

Last defects in this batch:
- patchOpenTab could not carry hostId, so quick-connect's "save this host
  and attach the tab to it" call was passing a field excluded from the
  type all the way down. The column exists and updateForUser spreads
  whatever it receives, so the write worked; only the types disagreed.
  Widened front to back.
- The file-comparison window opened without x, y, width or height — every
  other openWindow call passes them — and sent a `type` field WindowInstance
  does not have.
- HostEditor gated a block on authType === "warpgate", which is not one of
  the eight authType values. Unreachable, and it held only a label and a
  description. Removed.
- FileManager passed onLoadDirectory to a sidebar that neither declares nor
  reads it, and FileManagerApp passed embedded to a FileManager that has no
  such prop.
- TunnelApp's minimal Host was missing three required flags.

The remainder were assertions at boundaries that are genuinely loose: bulk
host import takes rows assembled from untyped input and validates them
server-side, and a vi.fn() whose body only throws infers never.

* feat: add drive file browser and drag-and-drop upload for RDP (#1187)

Drive redirection could already be enabled per host, but the redirected
drive lived inside guacd with no way to reach it from the browser: the
client never handled onfilesystem, so the mounted volume was writable
from Windows and invisible from Termix.

Add a file browser panel that lists the drive, downloads files, and
uploads them, plus drag-and-drop onto the display which opens the panel
and uploads into the directory currently shown. The disable-upload and
disable-download connection settings are honoured by the UI, not just
passed to guacd.

A rejected upload stops the BlobWriter without firing onerror or
oncomplete, so the error ack is watched explicitly; otherwise the
transfer would hang forever. Directory reads carry a deadline for the
same reason.

Also declares Guacamole.Object, Client.onfilesystem, BlobReader and
BlobWriter in the local type definitions, which previously omitted them.

* fix: keep the mouse working on touch-capable devices in RDP/VNC (#1190)

Reported as "mouse input broken, keyboard fine" after 2.5.1 (#1102).

2.5.1 bound Guacamole.Mouse unconditionally. 2.6.0 replaced that with a
three-way branch on touchMode, and the touch branches replace the mouse
binding instead of adding to it:

    if (touchMode === "touchscreen")      new Guacamole.Mouse.Touchscreen(el)
    else if (touchMode === "touchpad")    new Guacamole.Mouse.Touchpad(el)
    else                                  new Guacamole.Mouse(el)

The two do not overlap. Guacamole.Mouse listens for mousedown/mousemove/
mouseup; Touchscreen and Touchpad listen only for touchstart/touchmove/
touchend. So in a touch mode nothing is listening for the mouse at all.

touchMode defaults to "touchscreen" whenever navigator.maxTouchPoints > 0,
which is true of every laptop with a touchscreen — machines that are still
driven by a mouse. Those users lost the pointer entirely while the keyboard
kept working, because Guacamole.Keyboard is bound independently.

The physical pointer is now always bound and a touch emulator is layered on
top when one is selected. Extracted to bindPointerInput() so the binding is
testable; the test fails against the old branch.

Note the issue also carries a second, unrelated report where well-formed
mouse frames do reach guacd and the VNC leg ignores them. That one is not
this, and the guacd image is pinned to 1.6.0 in both 2.5.1 and 2.6.1, so it
is not an upgrade either.

* fix: deduplicate /api/folders requests to prevent intermittent folder disappearance (#1191)

* chore: sync Crowdin translations

* fix: deduplicate /api/folders requests to prevent intermittent folder disappearance

getSSHFolders() had no request deduplication while getSSHHosts() used a TTL
cache with in-flight dedupe. When loadHosts() fired multiple times during
rapid navigation between Credentials and Hosts panels, the folder response
could arrive after the hosts response, causing the sidebar tree to render
without folder metadata.

- Add foldersCache (10s TTL) in hosts-request-cache.ts
- Wrap getSSHFolders() API call in getCachedSSHFolders()
- Invalidate folders cache on renameFolder, updateFolderMetadata,
  deleteAllHostsInFolder, and renameCredentialFolder
- Include foldersCache in invalidateHostsAndStatusCaches()

Closes Termix-SSH/Support#1103

Signed-off-by: RawNuke <67506722+RawNuke@users.noreply.github.com>

---------

Signed-off-by: RawNuke <67506722+RawNuke@users.noreply.github.com>
Co-authored-by: LukeGus <bugattiguy527@gmail.com>

* chore(deps): bump undici from 8.9.0 to 8.10.0 in the prod-minor-updates group (#1195)

* chore: sync Crowdin translations

* chore(deps): bump undici in the prod-minor-updates group

Bumps the prod-minor-updates group with 1 update: [undici](https://github.com/nodejs/undici).


Updates `undici` from 8.9.0 to 8.10.0
- [Release notes](https://github.com/nodejs/undici/releases)
- [Commits](https://github.com/nodejs/undici/compare/v8.9.0...v8.10.0)

---
updated-dependencies:
- dependency-name: undici
  dependency-version: 8.10.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: prod-minor-updates
...

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: LukeGus <bugattiguy527@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* feat: proxmox metrics integration

* feat: add folder select to the host multi select feature

* feat: implement context aware terminal toolbar with quick links, host info, image pasting, etc

* feat: made toolbar open file manager at path

* fix: delete folder route not invalidating host list cache

* fix: match host list icons with tab bar iconfix

* fix: change sidebar reset button icon to seperate against fullscreen button

* feat: unify connection system and add connection logs to guacd hosts

* fix: make mobile terminal scrollback match xterm wheel behavior (#1198)

* fix: route mobile terminal scrolling through xterm viewport

* docs: document mobile terminal touch scrolling

* chore: add a note to not place files in docs

* chore: remove touch imput from docs

* feat: improve snippet system with variable snippets and collapse settings

* feat: new fleet system with snippet, packages, files, and inventory features

* fix: command pallete not loading new activity and made enter load first item

* feat: add subhost from parent host organization feature

* feat: add workspaces feature to save tab layout

* perf: greatly improved performance across metrics polling and host management for enterprise users

* feat: add a onboarding system with a new interface simplicity system

* feat: finalize the multi dialect database system

* fix: bind trusted MFA devices to client installs (#1202)

* fix: merge OIDC group claims across sources (#1203)

* fix: allow disabling SSH keepalives (#1204)

* fix: distinguish reachable and available hosts (#1206)

* fix: throttle session activity persistence (#1207)

* fix: preserve saved RDP connection settings (#1208)

* fix: authenticate tunnel status stream (#1209)

* fix: select quick-created credentials (#1210)

* fix: stagger initial metrics collection (#1211)

* fix: stagger initial metrics collection

* fix: admit reachable hosts to initial metrics

* fix: prevent long host names shifting dashboard metrics (#1205)

* feat: add global touch input settings (#1201)

Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com>

* fix: keep host list row sizing stable (#1213)

* fix(guacamole): correct Windows key mapping (#1216)

* fix: normalize OIDC discovery issuer URLs (#1218)

* fix: prompt for RDP domain credentials (#1212)

* fix: route status checks by connection origin (#1214)

* fix: restore desktop Tailscale configuration (#1215)

* fix(docker): restore Node 24 for ssh2 native crypto (#1217)

* feat: added new automations feature with events, channels, and steps

* feat: allowed some tabs in the app rail to be opened as its own tab or in a new right sidebar

* feat: expand onboarding process with more customization/features

* feat: initial implementation of the termix ai feature

* chore: run linter

* fix: issue #424 (#424)

https://github.com/Termix-SSH/Support/issues/424

* fix: Not working without internet connection. Missing OPKSSH binary in pre-built image. (#1133)

https://github.com/Termix-SSH/Support/issues/1133

* fix: SQLite forceSave on telemetry writes causes periodic SSH terminal stalls in 2.6.x (#1109)

https://github.com/Termix-SSH/Support/issues/1109

* feat: How to enable SSL using custom certificate (#1083)

https://github.com/Termix-SSH/Support/issues/1083

* fix: show profile API key after creation (#1221)

* feat: add trusted proxy authentication (#1222)

* fix: clarify SSH agent authentication (#1224)

* feat: add first-class split screen tabs (#1226)

* feat: add split tab data model

* feat: make split screens top-level tabs

* feat: persist and manage split layouts

* feat: launch native RDP on Windows desktop (#1223)

* feat: launch native RDP on Windows

* style: format native RDP launcher

* feat: enhance custom disk and network metrics (#1220)

* feat: enhance host disk and network metrics

* fix: align enhanced metrics types

* fix: preserve Proxmox guest identity on edit (#1219)

* fix: preserve Proxmox guest identity on edit

* fix: type Proxmox guest source metadata

* chore: dead-code cleanup and small refactors (#1225)

* chore: remove dead code and unused exports

* chore: remove unused api client functions

* chore: remove unused backend helpers

* refactor: extract getErrorMessage helper for repeated error extraction

* refactor: unify error message extraction across backend with getErrorMessage

* refactor: unify error message extraction in frontend with getErrorMessage

* refactor: merge duplicate imports from the same module

* refactor: use Array.includes in TabBar

* chore: drop biome, keep prettier as the single formatter

* style: apply prettier formatting to refactored files

* fix: close active tab with Ctrl+W on Windows

* fix: make tray Quit terminate the desktop app

* feat: verify host transfer integrity

* fix: reuse transfer sessions during verification

* feat: select the fastest host transfer route

* feat: tune host transfers adaptively

* feat: adapt background polling to activity (#1233)

* feat: adapt background polling to activity

* feat: extend adaptive polling coverage

* feat: make polling cost and network aware (#1234)

* feat: make repeat navigation feel instant (#1235)

* feat: make file operations feel immediate (#1236)

* feat: preload likely user actions (#1237)

* feat: preload likely file previews

* feat: preload likely host tools

* feat: preload likely file viewers

* fix: replace stale terminal input listeners

* feat: add links to docs for all new features

* chore: update readme

* fix: warn before discarding host changes (#1229)

* feat: learn local host action preferences (#1238)

* feat(terminal-toolbar): add bounded movable desktop toolbar (#1239)

* feat: add local adaptive decision engine (#1240)

* feat: adapt speculative resource usage (#1241)

* feat: persist adaptive transfer profiles (#1242)

* Fix .preferred_username when using LDAP login. (#1243)

* chore: sync Crowdin translations

* Fix .preferred_username when using LDAP login. Strips internal LDAP prefix from username.

---------

Co-authored-by: LukeGus <bugattiguy527@gmail.com>

* feat: learn direct transfer routes (#1244)

* feat: learn speculative preload usefulness (#1245)

* fix: - Adjusting the SSH Authentication from Vault to something else fails (#1152)

https://github.com/Termix-SSH/Support/issues/1152

* fix: terminal graphical display, special characters inserted, distorted - `midnight comma... (#1145)

https://github.com/Termix-SSH/Support/issues/1145

* feat: single click on host in list opens session - should be only on double click (#1146)

https://github.com/Termix-SSH/Support/issues/1146

* feat: Terminal: custom font/ font selection/ how-to for adding a font - `MesloLGS NF` (#1140)

https://github.com/Termix-SSH/Support/issues/1140

* fix: revert host single click to open session, make double click an option (#1146)

Single click opens a session again by default. The old double click
behavior can be turned on in Customize Sidebar.

* chore: drop prettier check from beta release workflow, run formatter

* chore: patch dependabot vulnerabilities via npm overrides

* fix: reset adaptive resource state between tests to stop cross-test leaks

* feat: replace terminal toolbar density popover with a native select

* fix: pin hardwareConcurrency in adaptive budget tests so CI cores don't change the tier

* fix: allow dylib files in mac universal arch rules so mas build packages sharp

* feat: add file manager trash (#1250)

* feat: add inheritable connection defaults (#1246)

* feat: add desktop local terminal (#1247)

* feat: add interactive terminal macros (#1248)

* feat: add adaptive SSH local echo (#1249)

* fix: sync desktop host changes immediately (#1252)

* fix: route desktop sharing through synced server (#1253)

* Fix terminal image uploads and add safe diagnostics (#1254)

* feat: add configurable terminal image storage backends

* feat: add admin image storage settings

* fix: preserve native clipboard PNG uploads

* fix: quote terminal image paths safely

* docs: record image storage security remediation plan

* fix: close remote image SFTP channels

* fix: restrict remote image SFTP permissions

* fix: bound remote image SFTP writes

* fix: add best effort remote image retention

* fix: cap normalized image output size

* fix: bound concurrent image processing

* fix: fail closed on local image inspection errors

* test: cover fail closed image storage and atomic settings

* fix: enforce remote image quota and upload admission

* fix: serialize remote quota and verify existing paths

* fix: use synchronous sqlite settings transaction

* fix: keep settings transactions portable across dialects

* fix: bound image processing admission queue

* fix: serialize remote image quota across processes

* fix: recover stale remote image locks safely

* fix: preserve remote storage errors during unlock

* fix: fail closed when stale lock removal fails

* fix: harden image upload resource and storage cleanup

* fix: bound SFTP operations and lock lifetime

* fix: bound SFTP acquisition and cleanup callbacks

* fix: close late SFTP channels and test cleanup stalls

* fix: preserve SFTP inspection client context

* feat: add image upload source metadata

* fix: expose image upload metadata in logs

* chore: exclude internal plan from pull request

* style: apply prettier formatting

---------

Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com>

* fix: batch of security hardening fixes (#1255)

* fix: bind desktop auto-session loopback check to the TCP peer address

* fix: escape HTML entities in Vault OIDC callback responses

* fix: route homepage ping and rss through the SSRF-safe outbound fetch

* fix: scope tunnel status endpoints to hosts the caller can access

* chore: update release notes

* chore: update release notes to write more about the ai integration

* fix: unbreak windows and macos electron builds after node-pty

Install Spectre-mitigated MSVC libs on the Windows runner and cover
node-pty's spawn-helper in the macOS universal arch rules.

* fix: rework connection defaults ui into a dialog and add missing i18n keys

* fix: rework macros panel with i18n, plain text matching, and list layout

* feat: add docs links for trash, connection defaults, and local echo

* fix: make image storage and trash tests pass on windows

* fix: stop docs links squeezing sidebar panel headers

* fix: put automations docs link back on the tabs row

* fix(desktop): keep Linux credential storage working on unrecognised desktops (#1261)

Chromium resolves safeStorage's backend from XDG_CURRENT_DESKTOP and falls
back to the basic_text store for any desktop it has no mapping for, which
covers every wlroots-style compositor (Hyprland, sway, niri, river).
isEncryptionAvailable() reports false for that store, so saveRemoteSyncJwt
refused every write and the OIDC sign-in it was storing appeared to succeed.
The sync engine then found no JWT and reported the session as expired, which
sent users looking at their OIDC provider for a fault that was never there.

Name the libsecret backend explicitly on those desktops. They run an ordinary
Secret Service, so that is enough to make encryption available again. KWallet
desktops keep their auto-detected backend, an explicit --password-store still
wins, and no stored secret can be orphaned by the switch because
isEncryptionAvailable() gated every write that would have created one.

Also stop discarding the {success: false} the main process returns when it
cannot store a credential: on a machine with no Secret Service at all, the
sign-in now says so instead of silently completing.

Co-authored-by: alexandre-vl <rafaelsenchais@gmail.com>

* chore: update release notes

* chore: update release notes

* fix(file-manager): widen trash dialog so names and paths are not cut off

* fix(sidebar): stop hover action tray overlapping the row below it

* fix(hosts): make real status colors toggle actually apply

* feat(local-terminal): add rail button and fix hardcoded tab label

* chore: update release notes

* fix(ai): hide assistant everywhere when admin disables it globally

* fix(automations): fix concurrency race, wire docker and internal event triggers

Claim the in-flight slot in the same tick it is checked, poll container
state for docker_event triggers, emit the internal events, apply the
schedule time zone, and expose the concurrency policy in the editor.

* fix(sidebar): rework host and credential drag-to-reorder

Adds a lock toggle in the sort menu and fixes reorder positioning,
cross-folder drops, and the duplicate drop indicator.

* chore(sidebar): drop unused sortKey prop from host and credential trees

* fix(sidebar): fix row height in click tray mode so status stripes stop overlapping

* fix(onboarding): remove add-first-host step that closed onboarding mid-flow

* fix(release): upload release notes so Mac App Store review submission stops failing

* chore: sync Crowdin translations for 2.7.0

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: RawNuke <67506722+RawNuke@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com>
Co-authored-by: kacperpietrzyk <105545577+kacperpietrzyk@users.noreply.github.com>
Co-authored-by: Brennan Neoh <497569+brennanneoh@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
Co-authored-by: T3rM1nAt0-R <niraj.sangani91@gmail.com>
Co-authored-by: Horziox <horziox.dev@gmail.com>
Co-authored-by: William Shi <184219650@qq.com>
Co-authored-by: Carl <scarlettme@qq.com>
Co-authored-by: Raw_Nuke <67506722+RawNuke@users.noreply.github.com>
Co-authored-by: njz-cvm <njz@cvm.com>
Co-authored-by: Alexandre VARGAS <alexandre.vargas.lopez@gmail.com>
Co-authored-by: alexandre-vl <rafaelsenchais@gmail.com>
This commit is contained in:
+4
Luke Gustafsondependabot[bot]ZacharyZcRkacperpietrzykBrennan NeohClaude Sonnet 5T3rM1nAt0-RHorzioxWilliam ShiCarlRaw_Nukenjz-cvmAlexandre VARGASalexandre-vl
 authored and GitHub committed 2026-08-19 14:12:06 -05:00
1 parent 5021ccf3e2
commit 7ae1648c25
837 files changed
+387941 -15831
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/backend/hosts/credential-username.ts
+22 -1
View File
@@ -51,9 +51,30 @@ export async function expandOidcUsername(
const { createCurrentUserRepository } =
await import("../database/repositories/factory.js");
const user = await createCurrentUserRepository().findById(userId);
const oidcIdentifier = user?.oidcIdentifier;
let oidcIdentifier = user?.oidcIdentifier;
if (!oidcIdentifier) return username;
const match = /^ldap:(\d+):(.+)$/.exec(oidcIdentifier);
if (match) {
// Make sure the SSO provider is actually LDAP, to prevent spoofing.
const { createCurrentSsoProviderRepository } =
await import("../database/repositories/factory.js");
const claimedProviderId = Number(match[1]);
const provider =
user?.ssoProviderId != null
? await createCurrentSsoProviderRepository().findById(
user.ssoProviderId,
)
: null;
if (
provider?.type === "ldap" &&
user?.ssoProviderId === claimedProviderId
) {
oidcIdentifier = match[2];
}
}
return username.replace(/\$oidc\.preferred_username/g, oidcIdentifier);
} catch {
return username;
src/backend/hosts/docker/console.ts
+58 -9
View File
@@ -1,3 +1,5 @@
import { getErrorMessage } from "../../utils/error-message.js";
import { StringDecoder } from "string_decoder";
import { Client as SSHClient } from "ssh2";
import { SSH_ALGORITHMS } from "../../utils/ssh-algorithms.js";
import { WebSocketServer, WebSocket } from "ws";
@@ -12,6 +14,11 @@ import {
type ContainerRuntime,
} from "./container-runtime.js";
import { resolveSshConnectConfigHost } from "../ssh-dns.js";
import {
hostAddressMismatch,
HOST_ADDRESS_MISMATCH_MESSAGE,
HOST_NOT_ON_THIS_SERVER_MESSAGE,
} from "../terminal/host-identity.js";
const sshLogger = systemLogger;
@@ -387,14 +394,49 @@ wss.on("connection", async (ws: WebSocket, req) => {
try {
// Resolve host with credentials server-side
const { resolveHostById } = await import("../host-resolver.js");
const resolvedHost = await resolveHostById(hostId, userId);
const { resolveHostById, resolveHostBySyncId } =
await import("../host-resolver.js");
// syncId names the host on both sides of a sync pair; the numeric
// id only names it in the database the client is displaying.
const hostSyncId = hostConfig?.syncId;
const resolvedHost = hostSyncId
? await resolveHostBySyncId(hostSyncId, userId)
: await resolveHostById(hostId, userId);
if (!resolvedHost) {
ws.send(
JSON.stringify({
type: "error",
message: "Host not found",
message: hostSyncId
? HOST_NOT_ON_THIS_SERVER_MESSAGE
: "Host not found",
}),
);
return;
}
// The connection below dials resolvedHost.ip outright, so if this
// server has a different machine under the id the client sent, the
// console opens on that machine's Docker daemon instead.
if (
!hostSyncId &&
hostAddressMismatch(hostConfig?.ip, resolvedHost.ip)
) {
sshLogger.error(
"Refusing Docker console: host id resolves to a different address here",
undefined,
{
operation: "docker_console_host_id_mismatch",
hostId,
userId,
clientIp: hostConfig?.ip,
resolvedIp: resolvedHost.ip,
},
);
ws.send(
JSON.stringify({
type: "error",
message: HOST_ADDRESS_MISMATCH_MESSAGE,
}),
);
return;
@@ -601,12 +643,19 @@ wss.on("connection", async (ws: WebSocket, req) => {
containerId,
});
// Buffers incomplete multi-byte UTF-8 sequences across chunk
// boundaries so box-drawing/special characters don't get
// corrupted when a character is split across TCP packets.
const decoder = new StringDecoder("utf-8");
stream.on("data", (data: Buffer) => {
if (ws.readyState === WebSocket.OPEN) {
const text = decoder.write(data);
if (!text) return;
ws.send(
JSON.stringify({
type: "output",
data: data.toString("utf8"),
data: text,
}),
);
}
@@ -669,10 +718,10 @@ wss.on("connection", async (ws: WebSocket, req) => {
ws.send(
JSON.stringify({
type: "error",
message:
error instanceof Error
? error.message
: "Failed to connect to container",
message: getErrorMessage(
error,
"Failed to connect to container",
),
}),
);
}
@@ -734,7 +783,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
ws.send(
JSON.stringify({
type: "error",
message: error instanceof Error ? error.message : "An error occurred",
message: getErrorMessage(error, "An error occurred"),
}),
);
}
src/backend/hosts/docker/container-routes.ts
+11 -11
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import type express from "express";
import { logger } from "../../utils/logger.js";
import {
@@ -151,8 +152,7 @@ export function registerDockerContainerRoutes(
});
res.status(500).json({
error:
error instanceof Error ? error.message : "Failed to list containers",
error: getErrorMessage(error, "Failed to list containers"),
});
}
});
@@ -232,7 +232,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
error: "Container not found",
@@ -329,7 +329,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -429,7 +429,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -529,7 +529,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -629,7 +629,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -729,7 +729,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -838,7 +838,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -982,7 +982,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
@@ -1104,7 +1104,7 @@ export function registerDockerContainerRoutes(
} catch (error) {
session.activeOperations--;
const errorMsg = error instanceof Error ? error.message : "";
const errorMsg = getErrorMessage(error, "");
if (errorMsg.includes("No such container")) {
return res.status(404).json({
success: false,
src/backend/hosts/docker/index.ts
+2
View File
@@ -1,6 +1,7 @@
import express from "express";
import cookieParser from "cookie-parser";
import { createCorsMiddleware } from "../../utils/cors-config.js";
import { createCompressionMiddleware } from "../../utils/compression-config.js";
import { logger } from "../../utils/logger.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { registerDockerContainerRoutes } from "./container-routes.js";
@@ -20,6 +21,7 @@ const sshLogger = logger;
const app = express();
app.use(createCompressionMiddleware());
app.use(createCorsMiddleware(["GET", "POST", "PUT", "DELETE", "OPTIONS"]));
app.use(cookieParser());
src/backend/hosts/docker/routes.ts
+19 -29
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import express from "express";
import axios from "axios";
import { Client as SSHClient } from "ssh2";
@@ -13,12 +14,15 @@ import { resolveHostById } from "../host-resolver.js";
import { createConnectionLog } from "../connection-log.js";
import { DataCrypto } from "../../utils/data-crypto.js";
import { AuthManager } from "../../utils/auth-manager.js";
import type { AuthenticatedRequest } from "../../../types/index.js";
import {
type AuthenticatedRequest,
type ProxyNode,
type SSHHost,
} from "../../../types/index.js";
import {
createSocks5Connection,
type SOCKS5Config,
} from "../../utils/socks5-helper.js";
import type { SSHHost, ProxyNode } from "../../../types/index.js";
import type {
LogEntry,
ConnectionStage,
@@ -337,16 +341,13 @@ export function registerDockerSshRoutes(app: express.Express): void {
operation: "docker_connect",
sessionId,
hostId,
error:
opksshError instanceof Error
? opksshError.message
: "Unknown error",
error: getErrorMessage(opksshError),
});
connectionLogs.push(
createConnectionLog(
"error",
"docker_auth",
`OPKSSH authentication failed: ${opksshError instanceof Error ? opksshError.message : "Unknown error"}`,
`OPKSSH authentication failed: ${getErrorMessage(opksshError)}`,
),
);
return res.status(500).json({
@@ -367,10 +368,7 @@ export function registerDockerSshRoutes(app: express.Express): void {
config.passphrase = resolvedCredentials.keyPassword;
}
} catch (error) {
const message =
error instanceof Error
? error.message
: "Invalid private key format";
const message = getErrorMessage(error, "Invalid private key format");
sshLogger.error("SSH key processing error", error, {
operation: "docker_connect",
sessionId,
@@ -988,17 +986,14 @@ export function registerDockerSshRoutes(app: express.Express): void {
createConnectionLog(
"error",
"jump",
`Jump host connection failed: ${jumpError instanceof Error ? jumpError.message : "Unknown error"}`,
`Jump host connection failed: ${getErrorMessage(jumpError)}`,
),
);
if (!responseSent) {
responseSent = true;
return res.status(500).json({
error:
"Jump host connection failed: " +
(jumpError instanceof Error
? jumpError.message
: "Unknown error"),
"Jump host connection failed: " + getErrorMessage(jumpError),
connectionLogs,
});
}
@@ -1028,17 +1023,13 @@ export function registerDockerSshRoutes(app: express.Express): void {
createConnectionLog(
"error",
"proxy",
`Proxy connection failed: ${proxyError instanceof Error ? proxyError.message : "Unknown error"}`,
`Proxy connection failed: ${getErrorMessage(proxyError)}`,
),
);
if (!responseSent) {
responseSent = true;
return res.status(500).json({
error:
"Proxy connection failed: " +
(proxyError instanceof Error
? proxyError.message
: "Unknown error"),
error: "Proxy connection failed: " + getErrorMessage(proxyError),
connectionLogs,
});
}
@@ -1059,12 +1050,12 @@ export function registerDockerSshRoutes(app: express.Express): void {
createConnectionLog(
"error",
"docker_connecting",
`Connection error: ${error instanceof Error ? error.message : "Unknown error"}`,
`Connection error: ${getErrorMessage(error)}`,
),
);
res.status(500).json({
success: false,
message: error instanceof Error ? error.message : "Unknown error",
message: getErrorMessage(error),
connectionLogs,
});
}
@@ -1264,7 +1255,7 @@ export function registerDockerSshRoutes(app: express.Express): void {
operation: "activity_log_error",
userId: session.userId,
hostId: session.hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
})();
@@ -1444,7 +1435,7 @@ export function registerDockerSshRoutes(app: express.Express): void {
operation: "activity_log_error",
userId: session.userId,
hostId: session.hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
})();
@@ -1653,8 +1644,7 @@ export function registerDockerSshRoutes(app: express.Express): void {
});
} catch (daemonError) {
session.activeOperations--;
const errorMsg =
daemonError instanceof Error ? daemonError.message : "";
const errorMsg = getErrorMessage(daemonError, "");
if (errorMsg.includes("Cannot connect to the Docker daemon")) {
return res.json({
@@ -1702,7 +1692,7 @@ export function registerDockerSshRoutes(app: express.Express): void {
res.status(500).json({
available: false,
error: error instanceof Error ? error.message : "Validation failed",
error: getErrorMessage(error, "Validation failed"),
});
}
});
src/backend/hosts/file-manager/ca-cert-auth.ts
+51
View File
@@ -0,0 +1,51 @@
import { getErrorMessage } from "../../utils/error-message.js";
import type { Client as SSHClient, ConnectConfig } from "ssh2";
import { fileLogger } from "../../utils/logger.js";
/**
* Attaches a user-managed CA-signed certificate to an SFTP connection, if the
* host carries one.
*
* The terminal has always done this. The file manager never did: it imports
* and calls `setupOPKSSHCertAuth`, but `setupCACertAuth` — the sibling helper
* for user-managed `-cert.pub` files — had no call site here at all. A host
* whose key is paired with a CA-signed certificate therefore authenticated in
* a terminal and failed in the file manager, while OPKSSH certificates worked
* in both. That asymmetry is the whole bug.
*
* `cert_public_key` is a column on `ssh_data` but is not part of the shared
* `Host` type, so callers pass whichever record they hold and it is read off
* structurally — the same way the terminal reads it.
*
* A certificate that cannot be applied is logged and skipped rather than
* failing the connection: the private key alone may still be accepted, which
* is exactly what happened before any of this was wired up.
*/
export async function applyCACertIfPresent(
config: Record<string, unknown>,
client: SSHClient,
privateKey: Buffer | string,
source: { certPublicKey?: string | null },
username: string,
passphrase?: string,
): Promise<void> {
const certPublicKey = source.certPublicKey;
if (!certPublicKey || !certPublicKey.trim()) return;
try {
const { setupCACertAuth } = await import("../opkssh-cert-auth.js");
await setupCACertAuth(
config as ConnectConfig,
client,
privateKey,
certPublicKey,
username,
passphrase,
);
} catch (certError) {
fileLogger.warn("CA certificate setup failed, continuing with key only", {
operation: "sftp_ca_cert_auth_failed",
error: getErrorMessage(certError),
});
}
}
src/backend/hosts/file-manager/direct-transfer-routing.ts
+72
View File
@@ -0,0 +1,72 @@
export interface DirectTransferEndpoint {
host: string;
port: number;
username: string;
}
export const DIRECT_TRANSFER_MIN_IMPROVEMENT = 0.2;
export const DIRECT_TRANSFER_MIN_BYTES = 32 * 1024 * 1024;
export function shouldBenchmarkDirectTransfer(totalBytes: number): boolean {
return totalBytes >= DIRECT_TRANSFER_MIN_BYTES;
}
export function quoteShell(value: string): string {
return `'${value.replace(/'/g, `'"'"'`)}'`;
}
function formatHost(host: string): string {
return host.includes(":") && !host.startsWith("[") ? `[${host}]` : host;
}
export function buildDirectSshCommand(
endpoint: DirectTransferEndpoint,
): string {
const target = `${endpoint.username}@${formatHost(endpoint.host)}`;
return [
"ssh",
"-o BatchMode=yes",
"-o ConnectTimeout=5",
"-o StrictHostKeyChecking=yes",
`-p ${endpoint.port}`,
quoteShell(target),
].join(" ");
}
export function buildDirectProbeCommand(
endpoint: DirectTransferEndpoint,
): string {
return `${buildDirectSshCommand(endpoint)} ${quoteShell("command -v rsync >/dev/null")}`;
}
export function buildDirectRsyncCommand(
endpoint: DirectTransferEndpoint,
sourcePaths: string[],
destPath: string,
destIsDirectory: boolean,
): string {
const sshTransport = buildDirectSshCommand(endpoint).replace(/ '[^']+'$/, "");
const targetHost = `${endpoint.username}@${formatHost(endpoint.host)}`;
const targetPath = destIsDirectory
? `${destPath.replace(/\/+$/, "") || "/"}/`
: destPath;
const sources = sourcePaths.map(quoteShell).join(" ");
const target = quoteShell(`${targetHost}:${targetPath}`);
return [
"rsync -a --partial --append-verify --protect-args --info=progress2",
`-e ${quoteShell(sshTransport)}`,
"--",
sources,
target,
].join(" ");
}
export function shouldUseDirectTransfer(
directMs: number,
relayMs: number,
minImprovement = DIRECT_TRANSFER_MIN_IMPROVEMENT,
): boolean {
if (directMs <= 0 || relayMs <= 0) return false;
return directMs <= relayMs * (1 - minImprovement);
}
src/backend/hosts/file-manager/index.ts
+139 -34
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import express from "express";
import {
logAudit,
@@ -5,6 +6,7 @@ import {
getRequestMeta,
} from "../../utils/audit-logger.js";
import { createCorsMiddleware } from "../../utils/cors-config.js";
import { createCompressionMiddleware } from "../../utils/compression-config.js";
import cookieParser from "cookie-parser";
import axios from "axios";
import { Client as SSHClient } from "ssh2";
@@ -12,7 +14,11 @@ import { SSH_ALGORITHMS } from "../../utils/ssh-algorithms.js";
import { createCurrentHostResolutionRepository } from "../../database/repositories/factory.js";
import { fileLogger } from "../../utils/logger.js";
import { AuthManager } from "../../utils/auth-manager.js";
import type { AuthenticatedRequest, ProxyNode } from "../../../types/index.js";
import {
type AuthenticatedRequest,
type ProxyNode,
type SSHHost,
} from "../../../types/index.js";
import {
createSocks5Connection,
type SOCKS5Config,
@@ -23,7 +29,6 @@ import type {
} from "../../../types/connection-log.js";
import { SSHHostKeyVerifier } from "../host-key-verifier.js";
import { resolveHostById } from "../host-resolver.js";
import type { SSHHost } from "../../../types/index.js";
import {
startHostTransfer,
getTransferStatus,
@@ -49,12 +54,68 @@ import {
import { registerFileListingRoutes } from "./list-routes.js";
import { registerFileOperationRoutes } from "./operation-routes.js";
import { resolveSshConnectConfigHost } from "../ssh-dns.js";
import { resolveSshKeepalive } from "../ssh-keepalive.js";
import {
hostAddressMismatch,
HostAddressMismatchError,
HostNotOnThisServerError,
} from "../terminal/host-identity.js";
import { registerFileDownloadRoutes } from "./download-routes.js";
import { registerFileActionRoutes } from "./action-routes.js";
import { applyAgentAuth } from "../terminal-auth-helpers.js";
import { applyCACertIfPresent } from "./ca-cert-auth.js";
/**
* The host id came from whichever database the client is displaying. If this
* server has a different machine under that id — desktop and sync server
* autoincrement sequences drift apart — then the address, the credentials and
* the jump hosts resolved from it all belong to that other machine, and the
* user would browse, edit and delete its files believing they are on the host
* they picked.
*/
function assertResolvedHost(
clientIp: unknown,
hostSyncId: string | null | undefined,
resolvedHost: { ip?: string } | null | undefined,
hostId: number,
userId: string,
): void {
// Named by sync identity: it either exists here or it does not. Falling back
// to the numeric id is what picks the wrong machine.
if (hostSyncId) {
if (resolvedHost) return;
fileLogger.error(
"Refusing SFTP connection: host is not known to this server",
undefined,
{
operation: "file_manager_host_sync_id_unknown",
hostId,
userId,
},
);
throw new HostNotOnThisServerError();
}
// Older clients send only the numeric id, which means a different host here.
if (!hostAddressMismatch(clientIp, resolvedHost?.ip)) return;
fileLogger.error(
"Refusing SFTP connection: host id resolves to a different address here",
undefined,
{
operation: "file_manager_host_id_mismatch",
hostId,
userId,
clientIp,
resolvedIp: resolvedHost?.ip,
},
);
throw new HostAddressMismatchError();
}
const app = express();
app.use(createCompressionMiddleware());
app.use(createCorsMiddleware(["GET", "POST", "PUT", "DELETE", "OPTIONS"]));
app.use(cookieParser());
app.use(express.json({ limit: "1gb" }));
@@ -211,6 +272,15 @@ async function buildDedicatedTransferConnectConfig(
.replace(/\r/g, "\n");
config.privateKey = Buffer.from(cleanKey, "utf8");
if (host.keyPassword) config.passphrase = host.keyPassword;
await applyCACertIfPresent(
config,
client,
config.privateKey as Buffer,
host as { certPublicKey?: string | null },
username,
host.keyPassword,
);
} else if (authType === "password") {
if (!host.password) {
throw new Error("Password required for transfer connection");
@@ -368,8 +438,12 @@ async function openDedicatedTransferSession(
throw new Error("Host not found for transfer connection");
}
if (sshSessions[dedicatedSessionId]?.isConnected) {
closeDedicatedTransferSession(dedicatedSessionId);
const existingSession = sshSessions[dedicatedSessionId];
if (
existingSession?.isConnected &&
verifySessionOwnership(existingSession, userId)
) {
return existingSession;
}
const client = new SSHClient();
@@ -639,6 +713,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
const {
sessionId,
hostId,
syncId: hostSyncId,
ip,
port,
username,
@@ -734,6 +809,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
keyPassword,
authType,
sudoPassword: undefined as string | undefined,
certPublicKey: undefined as string | undefined,
};
let hostKeepaliveInterval: number | undefined;
let hostKeepaliveCountMax: number | undefined;
@@ -751,8 +827,12 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
let resolvedSocks5ProxyChain = socks5ProxyChain;
if (hostId && userId && !password && !sshKey) {
try {
const { resolveHostById } = await import("../host-resolver.js");
const resolvedHost = await resolveHostById(hostId, userId);
const { resolveHostById, resolveHostBySyncId } =
await import("../host-resolver.js");
const resolvedHost = hostSyncId
? await resolveHostBySyncId(hostSyncId, userId)
: await resolveHostById(hostId, userId);
assertResolvedHost(ip, hostSyncId, resolvedHost, hostId, userId);
if (resolvedHost) {
resolvedIp = resolvedHost.ip;
resolvedPort = resolvedHost.port;
@@ -763,6 +843,8 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
keyPassword: resolvedHost.keyPassword,
authType: resolvedHost.authType,
sudoPassword: resolvedHost.sudoPassword as string | undefined,
certPublicKey: (resolvedHost as { certPublicKey?: string })
.certPublicKey,
};
resolvedTerminalConfig = resolvedHost.terminalConfig as unknown as
Record<string, unknown> | undefined;
@@ -800,17 +882,26 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
);
}
} catch (error) {
if (
error instanceof HostAddressMismatchError ||
error instanceof HostNotOnThisServerError
)
throw error;
fileLogger.warn(`Failed to resolve host credentials for ${hostId}`, {
operation: "ssh_credentials",
hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
} else if (credentialId && hostId && userId) {
// Legacy: credential resolution from credentialId
try {
const { resolveHostById } = await import("../host-resolver.js");
const resolvedHost = await resolveHostById(hostId, userId);
const { resolveHostById, resolveHostBySyncId } =
await import("../host-resolver.js");
const resolvedHost = hostSyncId
? await resolveHostBySyncId(hostSyncId, userId)
: await resolveHostById(hostId, userId);
assertResolvedHost(ip, hostSyncId, resolvedHost, hostId, userId);
if (resolvedHost) {
resolvedIp = resolvedHost.ip;
resolvedPort = resolvedHost.port;
@@ -821,6 +912,8 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
keyPassword: resolvedHost.keyPassword,
authType: resolvedHost.authType,
sudoPassword: resolvedHost.sudoPassword as string | undefined,
certPublicKey: (resolvedHost as { certPublicKey?: string })
.certPublicKey,
};
resolvedTerminalConfig = resolvedHost.terminalConfig as unknown as
Record<string, unknown> | undefined;
@@ -858,29 +951,33 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
);
}
} catch (error) {
if (
error instanceof HostAddressMismatchError ||
error instanceof HostNotOnThisServerError
)
throw error;
fileLogger.warn(`Failed to resolve credentials for host ${hostId}`, {
operation: "ssh_credentials",
hostId,
credentialId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
}
const preloadedHostData = await SSHHostKeyVerifier.preloadHostData(hostId);
const keepalive = resolveSshKeepalive(
hostKeepaliveInterval,
hostKeepaliveCountMax,
60000,
5,
);
const config: Record<string, unknown> = {
host: resolvedIp?.replace(/^\[|\]$/g, "") || resolvedIp,
port: resolvedPort,
username: resolvedUsername,
tryKeyboard: true,
keepaliveInterval:
typeof hostKeepaliveInterval === "number"
? Math.max(5000, hostKeepaliveInterval * 1000)
: 60000,
keepaliveCountMax:
typeof hostKeepaliveCountMax === "number"
? Math.max(1, hostKeepaliveCountMax)
: 5,
...keepalive,
readyTimeout: 60000,
tcpKeepAlive: true,
tcpKeepAliveInitialDelay: 30000,
@@ -954,11 +1051,23 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
if (resolvedCredentials.keyPassword)
config.passphrase = resolvedCredentials.keyPassword;
await applyCACertIfPresent(
config,
client,
config.privateKey as Buffer,
resolvedCredentials,
resolvedUsername,
resolvedCredentials.keyPassword,
);
connectionLogs.push(
createConnectionLog(
"info",
"sftp_auth",
"Using SSH key authentication",
resolvedCredentials.certPublicKey?.trim()
? "Using SSH key authentication with CA certificate"
: "Using SSH key authentication",
),
);
} catch (keyError) {
@@ -1038,14 +1147,13 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
operation: "file_connect",
sessionId,
hostId,
error:
opksshError instanceof Error ? opksshError.message : "Unknown error",
error: getErrorMessage(opksshError),
});
connectionLogs.push(
createConnectionLog(
"error",
"sftp_auth",
`OPKSSH authentication failed: ${opksshError instanceof Error ? opksshError.message : "Unknown error"}`,
`OPKSSH authentication failed: ${getErrorMessage(opksshError)}`,
),
);
return res.status(500).json({
@@ -1225,7 +1333,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
operation: "activity_log_error",
userId,
hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
})();
@@ -1720,7 +1828,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
createConnectionLog(
"error",
"jump",
`Jump host error: ${error instanceof Error ? error.message : "Unknown error"}`,
`Jump host error: ${getErrorMessage(error)}`,
),
);
return res.status(500).json({
@@ -1760,13 +1868,11 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
createConnectionLog(
"error",
"proxy",
`Proxy connection failed: ${proxyError instanceof Error ? proxyError.message : "Unknown error"}`,
`Proxy connection failed: ${getErrorMessage(proxyError)}`,
),
);
return res.status(500).json({
error:
"Proxy connection failed: " +
(proxyError instanceof Error ? proxyError.message : "Unknown error"),
error: "Proxy connection failed: " + getErrorMessage(proxyError),
connectionLogs,
});
}
@@ -1917,7 +2023,7 @@ app.post("/ssh/file_manager/ssh/connect-totp", async (req, res) => {
operation: "activity_log_error",
userId: session.userId,
hostId: session.hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
})();
@@ -2115,7 +2221,7 @@ app.post("/ssh/file_manager/ssh/connect-warpgate", async (req, res) => {
operation: "activity_log_error",
userId: session.userId,
hostId: session.hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
})();
@@ -2829,7 +2935,7 @@ app.post("/ssh/file_manager/ssh/transferMethodPreview", async (req, res) => {
sourcePaths,
});
res.status(500).json({
error: err instanceof Error ? err.message : "Failed to preview method",
error: getErrorMessage(err, "Failed to preview method"),
});
}
});
@@ -2880,7 +2986,7 @@ app.post("/ssh/file_manager/ssh/transferToHost", async (req, res) => {
const rawParallel = Number(parallelSegmentCountRaw);
const parallelSegmentCount = Number.isFinite(rawParallel)
? Math.max(1, Math.min(8, Math.floor(rawParallel)))
: 2;
: undefined;
const { transferId } = startHostTransfer(hostTransferDeps, {
sourceSessionId,
@@ -2968,8 +3074,7 @@ app.post(
);
res.json(result);
} catch (err) {
const message =
err instanceof Error ? err.message : "Failed to clean up transfer";
const message = getErrorMessage(err, "Failed to clean up transfer");
const status = message === "Transfer not found" ? 404 : 400;
res.status(status).json({ error: message });
}
src/backend/hosts/file-manager/list-routes.ts
+4 -6
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import type { Express } from "express";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { fileLogger } from "../../utils/logger.js";
@@ -194,8 +195,7 @@ export function registerFileListingRoutes(
tryFallbackMethod();
});
} catch (sftpErr: unknown) {
const errMsg =
sftpErr instanceof Error ? sftpErr.message : "Unknown error";
const errMsg = getErrorMessage(sftpErr);
fileLogger.warn(`SFTP connection error, trying fallback: ${errMsg}`);
tryFallbackMethod();
}
@@ -325,8 +325,7 @@ export function registerFileListingRoutes(
);
} catch (execErr: unknown) {
sshConn.activeOperations--;
const errMsg =
execErr instanceof Error ? execErr.message : "Unknown error";
const errMsg = getErrorMessage(execErr);
fileLogger.error(`Fallback listFiles exec failed: ${errMsg}`);
if (!res.headersSent) {
return res.status(500).json({ error: errMsg });
@@ -455,8 +454,7 @@ export function registerFileListingRoutes(
});
} catch (execErr: unknown) {
sshConn.activeOperations--;
const errMsg =
execErr instanceof Error ? execErr.message : "Unknown error";
const errMsg = getErrorMessage(execErr);
fileLogger.error(`Sudo listFiles exec failed: ${errMsg}`);
if (!res.headersSent) {
return res.status(500).json({ error: errMsg });
src/backend/hosts/file-manager/operation-routes.ts
+172 -2
View File
@@ -1,8 +1,25 @@
import type { Express } from "express";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { fileLogger } from "../../utils/logger.js";
import { execChannel, execWithSudo, type SSHSession } from "./session.js";
import {
execChannel,
execWithSudo,
getSessionSftp,
type SSHSession,
} from "./session.js";
import { buildDeleteCommand } from "./operation-commands.js";
import {
emptyTrash,
listTrash,
moveToTrash,
permanentlyDeleteTrashItem,
restoreTrashItem,
} from "./trash-service.js";
import {
createCurrentSettingsRepository,
getCurrentSettingValue,
} from "../../database/repositories/factory.js";
import { PermissionManager } from "../../utils/permission-manager.js";
type FileOperationRoutesDeps = {
sshSessions: Record<string, SSHSession>;
@@ -13,6 +30,128 @@ export function registerFileOperationRoutes(
app: Express,
{ sshSessions, verifySessionOwnership }: FileOperationRoutesDeps,
): void {
const permissionManager = PermissionManager.getInstance();
const getTrashRetentionDays = () => {
try {
const value = Number(
getCurrentSettingValue("file_manager_trash_retention_days"),
);
return Number.isInteger(value) && value >= 1 && value <= 3650 ? value : 7;
} catch {
return 7;
}
};
async function ownedSession(
req: AuthenticatedRequest,
res: import("express").Response,
) {
const sessionId = String(req.body?.sessionId ?? req.query?.sessionId ?? "");
const session = sshSessions[sessionId];
if (!sessionId || !session?.isConnected) {
res.status(400).json({ error: "SSH connection not established" });
return null;
}
if (!verifySessionOwnership(session, req.userId)) {
res.status(403).json({ error: "Session access denied" });
return null;
}
session.lastActive = Date.now();
return session;
}
app.get("/ssh/file_manager/ssh/trash", async (req, res) => {
const session = await ownedSession(
req as unknown as AuthenticatedRequest,
res,
);
if (!session) return;
try {
res.json({
items: await listTrash(
await getSessionSftp(session),
getTrashRetentionDays(),
),
retentionDays: getTrashRetentionDays(),
canManageRetention: await permissionManager.isAdmin(
(req as AuthenticatedRequest).userId,
),
});
} catch (error) {
fileLogger.error("Failed to list trash", error);
res.status(500).json({ error: (error as Error).message });
}
});
app.post("/ssh/file_manager/ssh/trash/:id/restore", async (req, res) => {
const session = await ownedSession(
req as unknown as AuthenticatedRequest,
res,
);
if (!session) return;
try {
res.json({
item: await restoreTrashItem(
await getSessionSftp(session),
req.params.id,
),
});
} catch (error) {
const message = (error as Error).message;
res
.status(message.includes("already exists") ? 409 : 500)
.json({ error: message });
}
});
app.delete("/ssh/file_manager/ssh/trash/:id", async (req, res) => {
const session = await ownedSession(
req as unknown as AuthenticatedRequest,
res,
);
if (!session) return;
try {
await permanentlyDeleteTrashItem(
await getSessionSftp(session),
req.params.id,
);
res.json({ success: true });
} catch (error) {
res.status(500).json({ error: (error as Error).message });
}
});
app.delete("/ssh/file_manager/ssh/trash", async (req, res) => {
const session = await ownedSession(req as AuthenticatedRequest, res);
if (!session) return;
try {
res.json({ deleted: await emptyTrash(await getSessionSftp(session)) });
} catch (error) {
res.status(500).json({ error: (error as Error).message });
}
});
app.put("/ssh/file_manager/ssh/trash-retention", async (req, res) => {
const userId = (req as AuthenticatedRequest).userId;
if (!(await permissionManager.isAdmin(userId))) {
return res.status(403).json({ error: "Admin access required" });
}
const retentionDays = Number(req.body?.retentionDays);
if (
!Number.isInteger(retentionDays) ||
retentionDays < 1 ||
retentionDays > 3650
) {
return res
.status(400)
.json({ error: "Retention must be between 1 and 3650 days" });
}
await createCurrentSettingsRepository().upsert(
"file_manager_trash_retention_days",
String(retentionDays),
);
return res.json({ retentionDays });
});
/**
* @openapi
* /ssh/file_manager/ssh/createFile:
@@ -342,7 +481,7 @@ export function registerFileOperationRoutes(
* description: Failed to delete item.
*/
app.delete("/ssh/file_manager/ssh/deleteItem", async (req, res) => {
const { sessionId, path: itemPath, isDirectory } = req.body;
const { sessionId, path: itemPath, isDirectory, permanent } = req.body;
const sshConn = sshSessions[sessionId];
const userId = (req as AuthenticatedRequest).userId;
@@ -371,6 +510,37 @@ export function registerFileOperationRoutes(
});
sshConn.lastActive = Date.now();
if (!permanent) {
try {
const sftp = await getSessionSftp(sshConn);
await listTrash(sftp, getTrashRetentionDays());
const item = await moveToTrash(sftp, itemPath);
fileLogger.success("Item moved to trash", {
operation: "file_trash_success",
sessionId,
userId,
path: itemPath,
trashId: item.id,
});
return res.json({
message: "Item moved to trash",
path: itemPath,
trashItem: item,
});
} catch (error) {
fileLogger.error("Failed to move item to trash", error, {
operation: "file_trash_failed",
sessionId,
userId,
path: itemPath,
});
return res.status(409).json({
error: (error as Error).message,
trashUnavailable: true,
});
}
}
const { command: deleteCommand, commandWithSuccess } = buildDeleteCommand(
itemPath,
Boolean(isDirectory),
src/backend/hosts/file-manager/transfer-engine.ts
+651 -52
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import { randomUUID } from "crypto";
import { networkInterfaces } from "os";
import { performance } from "node:perf_hooks";
@@ -18,11 +19,30 @@ import {
} from "../transfer-paths.js";
import {
buildTransferScanSummary,
estimateIncompressibleSample,
getArchiveTransferReasonKey,
resolveArchiveTransferMethod,
type TransferMethodPreference,
type TransferScanSummary,
} from "./transfer-routing.js";
import { verifySftpFileIntegrity } from "./transfer-integrity.js";
import {
buildDirectProbeCommand,
buildDirectRsyncCommand,
quoteShell,
shouldBenchmarkDirectTransfer,
shouldUseDirectTransfer,
type DirectTransferEndpoint,
} from "./direct-transfer-routing.js";
import {
getRecentDirectRouteDecision,
getTransferProfile,
initializeTransferProfiles,
recordDirectRouteBenchmark,
recordDirectRouteOutcome,
recordTransferProfile,
selectTransferTuning,
} from "./transfer-tuning.js";
export type {
TransferMethodPreference,
@@ -53,6 +73,7 @@ export interface SSHSessionLike {
userId?: string;
ip?: string;
port?: number;
username?: string;
transferPlatform?: TransferPlatform;
}
@@ -76,10 +97,15 @@ export interface HostTransferDeps {
}
export type TransferPhase =
"compressing" | "transferring" | "extracting" | "reconnecting";
| "compressing"
| "transferring"
| "benchmarking"
| "verifying"
| "extracting"
| "reconnecting";
export type TransferStatus =
"running" | "success" | "partial" | "error" | "cancelled";
export type TransferMethod = "stream" | "tar" | "item_sftp";
export type TransferMethod = "stream" | "tar" | "item_sftp" | "direct_rsync";
export type TransferHopId =
"source_read" | "dest_sftp_write" | "dest_local_write";
@@ -97,6 +123,9 @@ export interface TransferTimings {
compressMs?: number;
transferMs?: number;
extractMs?: number;
verifyMs?: number;
directBenchmarkMs?: number;
relayBenchmarkMs?: number;
sourceDeleteMs?: number;
totalMs?: number;
transferBytes?: number;
@@ -136,6 +165,7 @@ export interface TransferProgress {
partialDestRemaining?: boolean;
cleanupCompleted?: boolean;
retryable?: boolean;
integrityVerified?: boolean;
requestSnapshot?: {
sourceSessionId: string;
sourcePaths: string[];
@@ -155,12 +185,16 @@ export interface TransferRequest {
move?: boolean;
userId: string;
methodPreference?: TransferMethodPreference;
/** Parallel 256 MiB segment lanes for single-file SFTP copy (default 2). */
/** Explicit parallel lane override; omitted values are tuned automatically. */
parallelSegmentCount?: number;
}
const activeTransfers = new Map<string, TransferProgress>();
const cancelRequestedTransfers = new Set<string>();
const directRouteCache = new Map<
string,
{ useDirect: boolean; expiresAt: number; directMs?: number; relayMs?: number }
>();
/** In-flight pipelined SFTP reads; force-closed when the user cancels. */
interface ActiveXferControl {
@@ -314,6 +348,7 @@ interface TransferReconnectContext {
dedicatedSourceSessionId: string;
dedicatedDestSessionId: string;
transferId: string;
profileKey?: string;
}
type TransferReconnectMeta = Omit<
@@ -376,7 +411,8 @@ function buildStreamTransferTimings(
const wallStart = progress?.startedAt ?? Date.now();
const totalMs = elapsedMs(wallStart);
const prepare = prepareDestMs ?? progress?.timings?.prepareDestMs ?? 0;
const dataMs = Math.max(1, totalMs - prepare);
const verify = progress?.timings?.verifyMs ?? 0;
const dataMs = Math.max(1, totalMs - prepare - verify);
return {
...progress?.timings,
@@ -394,6 +430,7 @@ async function finalizeStreamTransferIfDestAtSize(
destPath: string,
expectedSize: number,
extra: Partial<TransferProgress> = {},
verify?: () => Promise<void>,
): Promise<boolean> {
try {
const destSize = await probeDestResumeOffset(
@@ -405,6 +442,8 @@ async function finalizeStreamTransferIfDestAtSize(
return false;
}
await verify?.();
fileLogger.info("Destination file complete — finalizing transfer", {
operation: "host_transfer_dest_complete",
transferId,
@@ -435,6 +474,7 @@ async function tryFinalizeStreamTransferIfDestComplete(
destPath: string,
expectedSize: number,
extra: Partial<TransferProgress> = {},
verify?: () => Promise<void>,
): Promise<boolean> {
try {
const destSftp = await deps.getSessionSftp(destSession);
@@ -444,6 +484,7 @@ async function tryFinalizeStreamTransferIfDestComplete(
destPath,
expectedSize,
extra,
verify,
);
} catch {
return false;
@@ -787,6 +828,11 @@ function execCommand(
deps: HostTransferDeps,
session: SSHSessionLike,
command: string,
options: {
shouldAbort?: () => boolean;
onOutput?: (chunk: Buffer) => void;
timeoutMs?: number;
} = {},
): Promise<{ code: number; stderr: string }> {
return new Promise((resolve, reject) => {
deps.execChannel(session, command, (err, stream) => {
@@ -794,17 +840,44 @@ function execCommand(
reject(err);
return;
}
let settled = false;
const startedAt = Date.now();
let stderr = "";
stream.on("data", () => {
/* consume stdout */
const abortTimer =
options.shouldAbort || options.timeoutMs
? setInterval(() => {
const timedOut =
options.timeoutMs !== undefined &&
Date.now() - startedAt >= options.timeoutMs;
if ((!options.shouldAbort?.() && !timedOut) || settled) return;
settled = true;
clearInterval(abortTimer);
stream.signal("KILL");
stream.close();
reject(
timedOut
? new Error(`Command timed out after ${options.timeoutMs}ms`)
: new TransferCancelledError(),
);
}, 250)
: undefined;
stream.on("data", (data: Buffer) => {
options.onOutput?.(data);
});
stream.stderr.on("data", (data: Buffer) => {
stderr += data.toString();
options.onOutput?.(data);
});
stream.on("close", (code: number) => {
if (settled) return;
settled = true;
if (abortTimer) clearInterval(abortTimer);
resolve({ code: code ?? 0, stderr });
});
stream.on("error", (streamErr: Error) => {
if (settled) return;
settled = true;
if (abortTimer) clearInterval(abortTimer);
reject(streamErr);
});
});
@@ -1088,6 +1161,53 @@ function elapsedMs(start: number): number {
return Date.now() - start;
}
async function verifyTransferredFile(
deps: HostTransferDeps,
transferId: string,
reconnectMeta: TransferReconnectMeta,
sourcePath: string,
destPath: string,
): Promise<void> {
updateTransfer(transferId, { phase: "verifying" });
const verifyStart = Date.now();
const [sourceSession, destSession] = await Promise.all([
deps.openDedicatedTransferSession(
reconnectMeta.browseSourceSessionId,
reconnectMeta.dedicatedSourceSessionId,
reconnectMeta.userId,
transferId,
{ allowBrowseDisconnected: true },
),
deps.openDedicatedTransferSession(
reconnectMeta.browseDestSessionId,
reconnectMeta.dedicatedDestSessionId,
reconnectMeta.userId,
transferId,
{ allowBrowseDisconnected: true },
),
]);
const [sourceSftp, destSftp] = await Promise.all([
deps.getSessionSftp(sourceSession),
deps.getSessionSftp(destSession),
]);
await verifySftpFileIntegrity(
sourceSftp,
destSftp,
sourcePath,
destPath,
createTransferShouldAbort(transferId),
() => new TransferCancelledError(),
);
const current = activeTransfers.get(transferId);
updateTransfer(transferId, {
integrityVerified: true,
timings: {
...current?.timings,
verifyMs: (current?.timings?.verifyMs ?? 0) + elapsedMs(verifyStart),
},
});
}
export function computeTransferMbPerSec(
bytes: number,
ms: number,
@@ -1284,7 +1404,52 @@ async function scanSourcePathsForRouting(
...work.map((w) => ({ sourcePath: w.sourcePath, size: w.size })),
);
}
return buildTransferScanSummary(scanItems);
const summary = buildTransferScanSummary(scanItems);
const candidates = [...scanItems]
.filter((item) => item.size > 0)
.sort((a, b) => b.size - a.size)
.slice(0, 3);
let sampledBytes = 0;
let sampledIncompressibleBytes = 0;
for (const item of candidates) {
throwIfCancelled(transferId);
try {
const sample = await readSftpSample(sftp, item.sourcePath, item.size);
sampledBytes += sample.length;
if (estimateIncompressibleSample(sample)) {
sampledIncompressibleBytes += sample.length;
}
} catch {
/* Extension-based routing remains the safe fallback. */
}
}
if (sampledBytes > 0) {
summary.sampledIncompressibleRatio =
sampledIncompressibleBytes / sampledBytes;
}
return summary;
}
async function readSftpSample(
sftp: SFTPWrapper,
path: string,
fileSize: number,
): Promise<Buffer> {
const sampleSize = Math.min(64 * 1024, fileSize);
const position = Math.max(0, Math.floor((fileSize - sampleSize) / 2));
const handle = await promisifySftpOpen(sftp, path, SFTP_OPEN_READ, 0o666);
try {
const buffer = Buffer.alloc(sampleSize);
const bytesRead = await new Promise<number>((resolve, reject) => {
sftp.read(handle, buffer, 0, sampleSize, position, (err, count) => {
if (err) reject(err);
else resolve(count);
});
});
return buffer.subarray(0, bytesRead);
} finally {
await promisifySftpClose(sftp, handle).catch(() => {});
}
}
function promisifySftpOpen(
@@ -1326,6 +1491,7 @@ interface PipelinedXferOptions {
fileSize?: number;
initialOffset?: number;
parallelSegmentCount?: number;
pipelineConcurrency?: number;
onProgress?: (bytes: number) => void;
shouldAbort?: () => boolean;
transferId?: string;
@@ -1440,7 +1606,7 @@ async function runFastSftpCopy(
sourceReadClock: ReturnType<typeof createHopWallClock>,
destWriteClock: ReturnType<typeof createHopWallClock>,
): Promise<void> {
let concurrency = SFTP_XFER_CONCURRENCY;
let concurrency = options.pipelineConcurrency ?? SFTP_XFER_CONCURRENCY;
let chunkSize = SFTP_XFER_CHUNK_SIZE;
let bufsize = chunkSize * concurrency;
while (bufsize > byteLength && concurrency > 1) {
@@ -1694,8 +1860,7 @@ async function runFastSftpCopySegmentedSequential(
throw err;
}
const message =
err instanceof Error ? err.message : "Segment transfer failed";
const message = getErrorMessage(err, "Segment transfer failed");
const recoverable =
options.reconnect &&
isRecoverableTransferError(err) &&
@@ -1987,8 +2152,7 @@ async function runFastSftpCopySegmentedParallel(
throw err;
}
const message =
err instanceof Error ? err.message : "Segment transfer failed";
const message = getErrorMessage(err, "Segment transfer failed");
const recoverable =
isRecoverableTransferError(err) &&
attempts < SFTP_PARALLEL_SEGMENT_MAX_ATTEMPTS;
@@ -2255,7 +2419,7 @@ async function pipelinedSftpFile(
const reset = await resetDedicatedTransferSessions(
options.reconnect,
attempts,
err instanceof Error ? err.message : "copy failed",
getErrorMessage(err, "copy failed"),
);
sftpSource = reset.sourceSftp;
sftpDest = reset.destSftp;
@@ -2330,7 +2494,7 @@ async function pipelinedSftpToLocalFile(
const transferStart = Date.now();
await promisifyFastGet(sourceSftp, sourcePath, localPath, {
concurrency: SFTP_XFER_CONCURRENCY,
concurrency: options.pipelineConcurrency ?? SFTP_XFER_CONCURRENCY,
chunkSize: SFTP_XFER_CHUNK_SIZE,
fileSize,
step: (_total, chunk) => {
@@ -2361,6 +2525,13 @@ async function transferFileData(
onResumeOffset?: (offset: number) => void,
parallelSegmentCount?: number,
): Promise<PipelinedXferStats> {
const tuning = selectTransferTuning(
fileSize,
reconnect?.profileKey
? getTransferProfile(reconnect.profileKey)
: undefined,
parallelSegmentCount,
);
const pipeOptions: PipelinedXferOptions = {
fileSize,
onProgress,
@@ -2368,25 +2539,392 @@ async function transferFileData(
transferId,
reconnect,
onResumeOffset,
parallelSegmentCount,
parallelSegmentCount: tuning.parallelSegmentCount,
pipelineConcurrency: tuning.pipelineConcurrency,
};
if (isLocalSshEndpoint(destSession.ip)) {
return pipelinedSftpToLocalFile(
sourceSftp,
sourcePath,
destPath,
pipeOptions,
);
if (transferId) {
updateTransfer(transferId, {
parallelSegmentCount: tuning.parallelSegmentCount,
});
}
return pipelinedSftpFile(
const startedAt = Date.now();
const shouldProfile = fileSize >= SFTP_XFER_SEGMENT_THRESHOLD;
fileLogger.info("Selected adaptive transfer tuning", {
operation: "host_transfer_tuning",
transferId,
fileSize,
parallelSegmentCount: tuning.parallelSegmentCount,
pipelineConcurrency: tuning.pipelineConcurrency,
profileSamples: reconnect?.profileKey
? getTransferProfile(reconnect.profileKey)?.samples
: undefined,
explicitParallelOverride: parallelSegmentCount !== undefined,
});
try {
const stats = isLocalSshEndpoint(destSession.ip)
? await pipelinedSftpToLocalFile(
sourceSftp,
sourcePath,
destPath,
pipeOptions,
)
: await pipelinedSftpFile(
sourceSftp,
destSftp,
sourcePath,
destPath,
pipeOptions,
);
if (shouldProfile && reconnect?.profileKey) {
recordTransferProfile(reconnect.profileKey, {
bytes: stats.bytes,
durationMs: Math.max(1, Date.now() - startedAt),
lanes: tuning.parallelSegmentCount,
pipelineConcurrency: tuning.pipelineConcurrency,
failed: false,
});
}
return stats;
} catch (err) {
if (shouldProfile && reconnect?.profileKey) {
recordTransferProfile(reconnect.profileKey, {
bytes: 0,
durationMs: Math.max(1, Date.now() - startedAt),
lanes: tuning.parallelSegmentCount,
pipelineConcurrency: tuning.pipelineConcurrency,
failed: true,
});
}
throw err;
}
}
const DIRECT_BENCHMARK_BYTES = 8 * 1024 * 1024;
const DIRECT_ROUTE_CACHE_MS = 10 * 60 * 1000;
function getDirectEndpoint(
sourceSession: SSHSessionLike,
destSession: SSHSessionLike,
): DirectTransferEndpoint | null {
if (
!sourceSession.ip ||
!destSession.ip ||
!destSession.username ||
!destSession.port ||
sourceSession.transferPlatform !== "unix" ||
destSession.transferPlatform !== "unix"
) {
return null;
}
return {
host: destSession.ip,
port: destSession.port,
username: destSession.username,
};
}
function directRouteKey(
sourceSession: SSHSessionLike,
endpoint: DirectTransferEndpoint,
): string {
return `${sourceSession.username ?? ""}@${sourceSession.ip}->${endpoint.username}@${endpoint.host}:${endpoint.port}`;
}
function transferProfileKey(
sourceSession: SSHSessionLike,
destSession: SSHSessionLike,
userId: string,
): string {
return `${userId}:${sourceSession.username ?? ""}@${sourceSession.ip ?? "local"}:${sourceSession.port ?? 22}->${destSession.username ?? ""}@${destSession.ip ?? "local"}:${destSession.port ?? 22}`;
}
async function benchmarkTransferRoutes(
deps: HostTransferDeps,
transferId: string,
sourceSession: SSHSessionLike,
destSession: SSHSessionLike,
endpoint: DirectTransferEndpoint,
profileKey?: string,
): Promise<{ useDirect: boolean; directMs?: number; relayMs?: number }> {
const key = directRouteKey(sourceSession, endpoint);
const cached = directRouteCache.get(key);
if (cached && cached.expiresAt > Date.now()) return cached;
const learned = profileKey
? getRecentDirectRouteDecision(profileKey, DIRECT_ROUTE_CACHE_MS)
: undefined;
if (learned) {
const result = {
...learned,
expiresAt: Date.now() + DIRECT_ROUTE_CACHE_MS,
};
directRouteCache.set(key, result);
return result;
}
updateTransfer(transferId, { phase: "benchmarking" });
const probe = await execCommand(
deps,
sourceSession,
`command -v rsync >/dev/null && ${buildDirectProbeCommand(endpoint)}`,
{ timeoutMs: 8000 },
).catch(() => ({ code: 1, stderr: "" }));
if (probe.code !== 0) {
const result = {
useDirect: false,
expiresAt: Date.now() + DIRECT_ROUTE_CACHE_MS,
};
directRouteCache.set(key, result);
return result;
}
const probeId = randomUUID();
const sourceProbe = `/tmp/termix-route-${probeId}.bin`;
const directProbe = `/tmp/termix-route-${probeId}.direct`;
const relayProbe = `/tmp/termix-route-${probeId}.relay`;
const cleanup = async () => {
await Promise.allSettled([
execCommand(deps, sourceSession, `rm -f ${quoteShell(sourceProbe)}`),
execCommand(
deps,
destSession,
`rm -f ${quoteShell(directProbe)} ${quoteShell(relayProbe)}`,
),
]);
};
try {
const created = await execCommand(
deps,
sourceSession,
`dd if=/dev/zero of=${quoteShell(sourceProbe)} bs=1048576 count=8 status=none`,
{ timeoutMs: 8000 },
);
if (created.code !== 0) throw new Error("Failed to create route probe");
const directStart = performance.now();
const direct = await execCommand(
deps,
sourceSession,
buildDirectRsyncCommand(endpoint, [sourceProbe], directProbe, false),
{ timeoutMs: 15000 },
);
const directMs = performance.now() - directStart;
if (direct.code !== 0) throw new Error("Direct route probe failed");
const [sourceSftp, destSftp] = await Promise.all([
deps.getSessionSftp(sourceSession),
deps.getSessionSftp(destSession),
]);
const relayStart = performance.now();
const relayDeadline = Date.now() + 15000;
await transferFileData(
sourceSftp,
destSftp,
destSession,
sourceProbe,
relayProbe,
DIRECT_BENCHMARK_BYTES,
undefined,
() => Date.now() >= relayDeadline,
);
const relayMs = performance.now() - relayStart;
const profile = profileKey
? recordDirectRouteBenchmark(profileKey, directMs, relayMs)
: undefined;
const useDirect =
shouldUseDirectTransfer(directMs, relayMs) &&
(!profile || profile.outcomeSamples < 2 || profile.failureRate < 0.25);
const result = {
useDirect,
directMs,
relayMs,
expiresAt: Date.now() + DIRECT_ROUTE_CACHE_MS,
};
directRouteCache.set(key, result);
updateTransfer(transferId, {
timings: {
...activeTransfers.get(transferId)?.timings,
directBenchmarkMs: directMs,
relayBenchmarkMs: relayMs,
},
});
return result;
} catch {
const result = {
useDirect: false,
expiresAt: Date.now() + DIRECT_ROUTE_CACHE_MS,
};
directRouteCache.set(key, result);
return result;
} finally {
await cleanup();
}
}
async function tryAdaptiveDirectTransfer(
deps: HostTransferDeps,
transferId: string,
sourceSession: SSHSessionLike,
destSession: SSHSessionLike,
sourcePaths: string[],
destPath: string,
move: boolean,
reconnectMeta: TransferReconnectMeta,
): Promise<TransferProgress | null> {
const endpoint = getDirectEndpoint(sourceSession, destSession);
if (!endpoint) return null;
const sourceSftp = await deps.getSessionSftp(sourceSession);
const firstStats = await promisifySftpStat(sourceSftp, sourcePaths[0]);
const destIsDirectory = sourcePaths.length > 1 || firstStats.isDirectory();
const summary = await scanSourcePathsForRouting(
sourceSftp,
destSftp,
sourcePath,
destPath,
pipeOptions,
sourcePaths,
transferId,
);
if (!shouldBenchmarkDirectTransfer(summary.totalBytes)) return null;
const route = await benchmarkTransferRoutes(
deps,
transferId,
sourceSession,
destSession,
endpoint,
reconnectMeta.profileKey,
);
if (!route.useDirect) return null;
if (destIsDirectory) {
await ensureDestDirectory(deps, destSession, destPath);
} else {
await ensureDestParentForFile(deps, destSession, destPath);
}
updateTransfer(transferId, {
method: "direct_rsync",
phase: "transferring",
bytesTransferred: 0,
totalBytes: summary.totalBytes,
});
const transferStart = performance.now();
let outputBuffer = "";
try {
const result = await execCommand(
deps,
sourceSession,
buildDirectRsyncCommand(endpoint, sourcePaths, destPath, destIsDirectory),
{
shouldAbort: createTransferShouldAbort(transferId),
onOutput: (chunk) => {
outputBuffer = `${outputBuffer}${chunk.toString()}`.slice(-2048);
const matches = [...outputBuffer.matchAll(/([\d,]+)\s+(\d+)%/g)];
const last = matches.at(-1);
if (!last) return;
const bytes = Number(last[1].replace(/,/g, ""));
if (Number.isFinite(bytes)) {
updateTransfer(transferId, {
bytesTransferred: Math.min(bytes, summary.totalBytes),
});
}
},
},
);
if (result.code !== 0) {
throw new Error(result.stderr || "Direct rsync transfer failed");
}
const workItems: FileWorkItem[] = [];
if (destIsDirectory) {
for (const sourcePath of sourcePaths) {
workItems.push(
...(await collectFileWorkItems(sourceSftp, sourcePath, destPath)),
);
}
} else {
workItems.push({
sourcePath: sourcePaths[0],
destPath,
mode: firstStats.mode,
size: firstStats.size,
});
}
for (const item of workItems) {
await verifyTransferredFile(
deps,
transferId,
reconnectMeta,
item.sourcePath,
item.destPath,
);
}
if (move) {
await deleteSourcePathsAfterSuccess(
deps,
transferId,
sourceSession,
sourcePaths,
);
}
const transferMs = performance.now() - transferStart;
if (reconnectMeta.profileKey) {
recordDirectRouteOutcome(
reconnectMeta.profileKey,
false,
Date.now(),
DIRECT_ROUTE_CACHE_MS,
);
}
return finalizeTransfer(transferId, {
status: "success",
phase: "verifying",
method: "direct_rsync",
bytesTransferred: summary.totalBytes,
totalBytes: summary.totalBytes,
sourcePaths,
destPath,
sourceDeleted: move,
moveRequested: move,
integrityVerified: true,
timings: {
...activeTransfers.get(transferId)?.timings,
transferMs,
transferBytes: summary.totalBytes,
endToEndMbPerSec: computeTransferMbPerSec(
summary.totalBytes,
transferMs,
),
},
});
} catch (error) {
if (error instanceof TransferCancelledError) throw error;
fileLogger.warn("Direct transfer failed; falling back to relay", {
operation: "host_transfer_direct_fallback",
transferId,
error: getErrorMessage(error, "Direct transfer failed"),
});
if (reconnectMeta.profileKey) {
recordDirectRouteOutcome(
reconnectMeta.profileKey,
true,
Date.now(),
DIRECT_ROUTE_CACHE_MS,
);
}
directRouteCache.set(directRouteKey(sourceSession, endpoint), {
useDirect: false,
expiresAt: Date.now() + DIRECT_ROUTE_CACHE_MS,
});
updateTransfer(transferId, {
method: undefined,
phase: "transferring",
bytesTransferred: 0,
integrityVerified: false,
});
return null;
}
}
async function transferSingleFile(
@@ -2398,6 +2936,7 @@ async function transferSingleFile(
destPath: string,
move: boolean,
reconnectMeta: TransferReconnectMeta,
requestedParallelSegments?: number,
): Promise<TransferProgress> {
const sourceSftp = await deps.getSessionSftp(sourceSession);
const destSftp = await deps.getSessionSftp(destSession);
@@ -2439,9 +2978,12 @@ async function transferSingleFile(
updateTransfer(transferId, { bytesTransferred: absoluteOffset });
};
const parallelLanes = clampParallelSegmentCount(
activeTransfers.get(transferId)?.parallelSegmentCount,
const tuning = selectTransferTuning(
stats.size,
reconnect.profileKey ? getTransferProfile(reconnect.profileKey) : undefined,
requestedParallelSegments,
);
const parallelLanes = tuning.parallelSegmentCount;
const useAbsoluteProgressOnly = parallelLanes > 1;
throwIfCancelled(transferId);
@@ -2463,7 +3005,15 @@ async function transferSingleFile(
transferId,
reconnect,
syncProgress,
parallelLanes,
requestedParallelSegments,
);
await verifyTransferredFile(
deps,
transferId,
reconnectMeta,
sourcePath,
destPath,
);
if (move) {
@@ -2605,8 +3155,16 @@ async function transferViaTar(
syncProgress,
);
mergeXferStats(xferStats, fileStats);
await verifyTransferredFile(
deps,
transferId,
reconnectMeta,
tempArchive,
tempArchive,
);
} catch (err) {
if (!(err instanceof TransferCancelledError)) {
await promisifySftpUnlink(destSftp, tempArchive).catch(() => {});
await cleanupDestItems(deps, destSession, destPath, basenames);
}
throw err;
@@ -2746,6 +3304,14 @@ async function transferViaItemSftp(
},
);
mergeXferStats(xferStats, itemXferStats);
await verifyTransferredFile(
deps,
transferId,
reconnectMeta,
item.sourcePath,
item.destPath,
);
updateTransfer(transferId, { phase: "transferring" });
if (destPlatform !== "windows") {
await promisifySftpChmod(destSftp, item.destPath, item.mode);
}
@@ -2760,6 +3326,7 @@ async function transferViaItemSftp(
continue;
}
if (!(err instanceof TransferCancelledError)) {
await deletePathSftp(destSftp, item.destPath).catch(() => {});
for (const created of [...createdFiles].reverse()) {
await deletePathSftp(destSftp, created).catch(() => {});
}
@@ -2794,6 +3361,7 @@ async function transferViaItemSftp(
destPath,
sourceDeleted: status === "success" && move,
moveRequested: move,
integrityVerified: status === "success",
timings: {
...activeTransfers.get(transferId)?.timings,
transferMs,
@@ -2807,6 +3375,7 @@ async function runTransfer(
transferId: string,
request: TransferRequest,
): Promise<void> {
await initializeTransferProfiles();
const {
sourceSessionId: browseSourceSessionId,
sourcePaths,
@@ -2815,13 +3384,10 @@ async function runTransfer(
move = false,
userId,
methodPreference = "auto",
parallelSegmentCount:
requestParallelSegments = DEFAULT_PARALLEL_SEGMENT_COUNT,
parallelSegmentCount: requestParallelSegments,
} = request;
const parallelSegmentCount = clampParallelSegmentCount(
requestParallelSegments,
);
const parallelSegmentCount = requestParallelSegments;
const dedicatedSourceSessionId = `xfer:${transferId}:src`;
const dedicatedDestSessionId = `xfer:${transferId}:dst`;
@@ -2848,11 +3414,18 @@ async function runTransfer(
transferId,
);
reconnectMeta.profileKey = transferProfileKey(
sourceSession,
destSession,
userId,
);
sourceSession.lastActive = Date.now();
destSession.lastActive = Date.now();
updateTransfer(transferId, {
parallelSegmentCount,
parallelSegmentCount:
parallelSegmentCount ?? DEFAULT_PARALLEL_SEGMENT_COUNT,
dedicatedSourceSessionId,
dedicatedDestSessionId,
});
@@ -2887,6 +3460,29 @@ async function runTransfer(
}
}
const directResult =
methodPreference === "auto"
? await tryAdaptiveDirectTransfer(
deps,
transferId,
sourceSession,
destSession,
sourcePaths,
destPath,
move,
reconnectMeta,
)
: null;
if (directResult) {
updateTransfer(transferId, {
timings: {
...directResult.timings,
totalMs: elapsedMs(runStart),
},
});
return;
}
let useArchive = sourcePaths.length > 1;
if (sourcePaths.length === 1) {
@@ -2904,6 +3500,7 @@ async function runTransfer(
destPath,
move,
reconnectMeta,
parallelSegmentCount,
);
} else {
const prepareStart = Date.now();
@@ -3007,6 +3604,7 @@ async function runTransfer(
totalItems: current?.totalItems,
moveRequested: move,
sourceDeleted: false,
integrityVerified: false,
timings: {
...current?.timings,
totalMs: elapsedMs(runStart),
@@ -3023,7 +3621,7 @@ async function runTransfer(
}
const current = activeTransfers.get(transferId);
const message = err instanceof Error ? err.message : "Transfer failed";
const message = getErrorMessage(err, "Transfer failed");
if (
sourcePaths.length === 1 &&
@@ -3050,6 +3648,14 @@ async function runTransfer(
moveRequested: move,
sourceDeleted: current.sourceDeleted,
},
() =>
verifyTransferredFile(
deps,
transferId,
reconnectMeta,
sourcePaths[0],
current.destPath!,
),
);
if (finalized) {
fileLogger.info("Host transfer succeeded after destination verify", {
@@ -3079,6 +3685,7 @@ async function runTransfer(
itemsCompleted: current?.itemsCompleted,
totalItems: current?.totalItems,
moveRequested: move,
integrityVerified: false,
timings: {
...current?.timings,
totalMs: elapsedMs(runStart),
@@ -3244,10 +3851,7 @@ export function retryHostTransfer(
await runTransfer(deps, transferId, {
...latest.requestSnapshot,
userId,
parallelSegmentCount:
latest.requestSnapshot?.parallelSegmentCount ??
latest.parallelSegmentCount ??
DEFAULT_PARALLEL_SEGMENT_COUNT,
parallelSegmentCount: latest.requestSnapshot.parallelSegmentCount,
});
})();
@@ -3301,14 +3905,11 @@ export async function previewArchiveTransferMethod(
);
const sourceSftp = await deps.getSessionSftp(sourceSession);
const scanItems: Array<{ sourcePath: string; size: number }> = [];
for (const sourcePath of sourcePaths) {
const work = await collectFileWorkItems(sourceSftp, sourcePath, "/");
scanItems.push(
...work.map((w) => ({ sourcePath: w.sourcePath, size: w.size })),
);
}
const scanSummary = buildTransferScanSummary(scanItems);
const scanSummary = await scanSourcePathsForRouting(
sourceSftp,
sourcePaths,
"preview",
);
const sourceHasTar =
sourcePlatform === "unix" && (await checkTarAvailable(deps, sourceSession));
@@ -3374,9 +3975,7 @@ export function startHostTransfer(
destPath: request.destPath,
move: request.move,
methodPreference: request.methodPreference,
parallelSegmentCount: clampParallelSegmentCount(
request.parallelSegmentCount,
),
parallelSegmentCount: request.parallelSegmentCount,
},
});
src/backend/hosts/file-manager/transfer-integrity.ts
+54
View File
@@ -0,0 +1,54 @@
import { createHash } from "node:crypto";
type SFTPWrapper = import("ssh2").SFTPWrapper;
export interface TransferIntegrityResult {
algorithm: "sha256";
digest: string;
}
export async function hashSftpFile(
sftp: SFTPWrapper,
path: string,
shouldAbort: () => boolean = () => false,
createAbortError: () => Error = () => new Error("Transfer cancelled"),
): Promise<string> {
const stream = sftp.createReadStream(path);
const hash = createHash("sha256");
try {
for await (const chunk of stream) {
if (shouldAbort()) {
const error = createAbortError();
stream.destroy(error);
throw error;
}
hash.update(chunk);
}
} finally {
stream.destroy();
}
if (shouldAbort()) throw createAbortError();
return hash.digest("hex");
}
export async function verifySftpFileIntegrity(
sourceSftp: SFTPWrapper,
destSftp: SFTPWrapper,
sourcePath: string,
destPath: string,
shouldAbort: () => boolean = () => false,
createAbortError: () => Error = () => new Error("Transfer cancelled"),
): Promise<TransferIntegrityResult> {
const [sourceDigest, destDigest] = await Promise.all([
hashSftpFile(sourceSftp, sourcePath, shouldAbort, createAbortError),
hashSftpFile(destSftp, destPath, shouldAbort, createAbortError),
]);
if (sourceDigest !== destDigest) {
throw new Error(`SHA-256 verification failed for ${sourcePath}`);
}
return { algorithm: "sha256", digest: sourceDigest };
}
src/backend/hosts/file-manager/transfer-routing.ts
+16 -4
View File
@@ -1,4 +1,5 @@
import type { TransferPlatform } from "../transfer-paths.js";
import { deflateRawSync } from "node:zlib";
export type TransferMethodPreference = "auto" | "tar" | "item_sftp";
@@ -8,6 +9,8 @@ export interface TransferScanSummary {
largestFileBytes: number;
/** Share of total bytes in likely incompressible file types (0–1). */
incompressibleRatio: number;
/** Share inferred from bounded content samples, when available. */
sampledIncompressibleRatio?: number;
}
const INCOMPRESSIBLE_EXT =
@@ -43,6 +46,15 @@ export function buildTransferScanSummary(
};
}
export function estimateIncompressibleSample(data: Buffer): boolean {
if (data.length === 0) return false;
return deflateRawSync(data, { level: 1 }).length / data.length >= 0.9;
}
function effectiveIncompressibleRatio(summary: TransferScanSummary): number {
return summary.sampledIncompressibleRatio ?? summary.incompressibleRatio;
}
/**
* Choose tar vs per-item SFTP for directory / multi-file transfers.
* Single-file stream transfers bypass this entirely.
@@ -72,8 +84,8 @@ export function resolveArchiveTransferMethod(
return "item_sftp";
}
const { fileCount, totalBytes, largestFileBytes, incompressibleRatio } =
summary;
const { fileCount, totalBytes, largestFileBytes } = summary;
const incompressibleRatio = effectiveIncompressibleRatio(summary);
if (fileCount === 0) {
return "item_sftp";
@@ -154,8 +166,8 @@ export function getArchiveTransferReasonKey(
return "tar_unavailable";
}
const { fileCount, totalBytes, largestFileBytes, incompressibleRatio } =
summary;
const { fileCount, totalBytes, largestFileBytes } = summary;
const incompressibleRatio = effectiveIncompressibleRatio(summary);
if (
fileCount > 1 &&
src/backend/hosts/file-manager/transfer-tuning.ts
+387
View File
@@ -0,0 +1,387 @@
import { createHash } from "node:crypto";
import { promises as fs } from "node:fs";
import path from "node:path";
export interface TransferPerformanceProfile {
throughputBps: number;
failureRate: number;
samples: number;
preferredLanes: number;
pipelineConcurrency: number;
updatedAt: number;
}
export interface TransferTuning {
parallelSegmentCount: number;
pipelineConcurrency: number;
}
export interface DirectRouteProfile {
directMs: number;
relayMs: number;
failureRate: number;
benchmarkSamples: number;
outcomeSamples: number;
benchmarkedAt: number;
cooldownUntil?: number;
updatedAt: number;
}
const MB = 1024 * 1024;
const PROFILE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
const MAX_PROFILES = 128;
const STORE_VERSION = 1;
const STORE_FILENAME = "adaptive-transfer-profiles.json";
const profiles = new Map<string, TransferPerformanceProfile>();
const directRoutes = new Map<string, DirectRouteProfile>();
let loadedPath: string | undefined;
let loadPromise: Promise<void> | undefined;
let persistTimer: ReturnType<typeof setTimeout> | undefined;
let persistPromise = Promise.resolve();
interface PersistedProfiles {
version: typeof STORE_VERSION;
profiles: Record<string, TransferPerformanceProfile>;
directRoutes: Record<string, DirectRouteProfile>;
}
function validDirectRoute(value: unknown): value is DirectRouteProfile {
if (!value || typeof value !== "object") return false;
const profile = value as Partial<DirectRouteProfile>;
return (
Number.isFinite(profile.directMs) &&
Number(profile.directMs) > 0 &&
Number.isFinite(profile.relayMs) &&
Number(profile.relayMs) > 0 &&
Number.isFinite(profile.failureRate) &&
Number(profile.failureRate) >= 0 &&
Number(profile.failureRate) <= 1 &&
Number.isFinite(profile.benchmarkSamples) &&
Number(profile.benchmarkSamples) > 0 &&
Number.isFinite(profile.outcomeSamples) &&
Number(profile.outcomeSamples) >= 0 &&
Number.isFinite(profile.benchmarkedAt) &&
(profile.cooldownUntil === undefined ||
Number.isFinite(profile.cooldownUntil)) &&
Number.isFinite(profile.updatedAt)
);
}
function storePath(): string {
const dataDir =
process.env.DATA_DIR || path.join(process.cwd(), "db", "data");
return path.join(dataDir, STORE_FILENAME);
}
function profileId(key: string): string {
return createHash("sha256").update(key).digest("hex");
}
function validProfile(value: unknown): value is TransferPerformanceProfile {
if (!value || typeof value !== "object") return false;
const profile = value as Partial<TransferPerformanceProfile>;
return (
Number.isFinite(profile.throughputBps) &&
Number(profile.throughputBps) >= 0 &&
Number.isFinite(profile.failureRate) &&
Number(profile.failureRate) >= 0 &&
Number(profile.failureRate) <= 1 &&
Number.isFinite(profile.samples) &&
Number(profile.samples) > 0 &&
Number.isFinite(profile.preferredLanes) &&
Number.isFinite(profile.pipelineConcurrency) &&
Number.isFinite(profile.updatedAt)
);
}
function trimProfiles(now = Date.now()): void {
for (const [key, profile] of profiles) {
if (now - profile.updatedAt > PROFILE_TTL_MS) profiles.delete(key);
}
const recent = [...profiles.entries()]
.sort(([, a], [, b]) => b.updatedAt - a.updatedAt)
.slice(0, MAX_PROFILES);
profiles.clear();
for (const [key, profile] of recent) profiles.set(key, profile);
for (const [key, profile] of directRoutes) {
if (now - profile.updatedAt > PROFILE_TTL_MS) directRoutes.delete(key);
}
const recentRoutes = [...directRoutes.entries()]
.sort(([, a], [, b]) => b.updatedAt - a.updatedAt)
.slice(0, MAX_PROFILES);
directRoutes.clear();
for (const [key, profile] of recentRoutes) directRoutes.set(key, profile);
}
async function persistProfiles(): Promise<void> {
trimProfiles();
const target = storePath();
const temporary = `${target}.${process.pid}.tmp`;
const payload: PersistedProfiles = {
version: STORE_VERSION,
profiles: Object.fromEntries(profiles),
directRoutes: Object.fromEntries(directRoutes),
};
try {
await fs.mkdir(path.dirname(target), { recursive: true });
await fs.writeFile(temporary, JSON.stringify(payload), {
encoding: "utf8",
mode: 0o600,
});
await fs.rename(temporary, target);
} catch {
await fs.rm(temporary, { force: true }).catch(() => {});
}
}
function queuePersist(): void {
if (persistTimer) clearTimeout(persistTimer);
persistTimer = setTimeout(() => {
persistTimer = undefined;
persistPromise = persistPromise.then(persistProfiles);
}, 250);
persistTimer.unref?.();
}
export async function initializeTransferProfiles(): Promise<void> {
const target = storePath();
if (loadedPath === target) return loadPromise;
loadedPath = target;
loadPromise = (async () => {
profiles.clear();
directRoutes.clear();
try {
const parsed = JSON.parse(
await fs.readFile(target, "utf8"),
) as Partial<PersistedProfiles>;
if (parsed.version !== STORE_VERSION || !parsed.profiles) return;
for (const [key, profile] of Object.entries(parsed.profiles)) {
if (/^[a-f0-9]{64}$/.test(key) && validProfile(profile)) {
profiles.set(key, profile);
}
}
for (const [key, profile] of Object.entries(parsed.directRoutes ?? {})) {
if (/^[a-f0-9]{64}$/.test(key) && validDirectRoute(profile)) {
directRoutes.set(key, profile);
}
}
trimProfiles();
} catch {
// Missing or malformed local learning data must not affect transfers.
}
})();
return loadPromise;
}
export async function flushTransferProfiles(): Promise<void> {
if (persistTimer) {
clearTimeout(persistTimer);
persistTimer = undefined;
}
persistPromise = persistPromise.then(persistProfiles);
await persistPromise;
}
function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, Math.floor(value)));
}
export function selectTransferTuning(
fileSize: number,
profile?: TransferPerformanceProfile,
requestedLanes?: number,
): TransferTuning {
if (fileSize < 32 * MB) {
return { parallelSegmentCount: 1, pipelineConcurrency: 8 };
}
let lanes = fileSize >= 1024 * MB ? 4 : 2;
let pipelineConcurrency = fileSize >= 256 * MB ? 32 : 16;
if (profile) {
lanes = profile.preferredLanes;
pipelineConcurrency = profile.pipelineConcurrency;
if (profile.failureRate >= 0.2) {
lanes = Math.min(lanes, 2);
pipelineConcurrency = Math.min(pipelineConcurrency, 16);
}
}
if (requestedLanes !== undefined) lanes = requestedLanes;
const segmentCapacity = Math.max(1, Math.ceil(fileSize / (256 * MB)));
return {
parallelSegmentCount: clamp(lanes, 1, Math.min(8, segmentCapacity)),
pipelineConcurrency: clamp(pipelineConcurrency, 8, 64),
};
}
export function updateTransferProfile(
previous: TransferPerformanceProfile | undefined,
observation: {
bytes: number;
durationMs: number;
lanes: number;
pipelineConcurrency: number;
failed: boolean;
now?: number;
},
): TransferPerformanceProfile {
const now = observation.now ?? Date.now();
const throughputBps =
observation.durationMs > 0
? (observation.bytes * 1000) / observation.durationMs
: 0;
const weight = previous ? 0.25 : 1;
const smoothedThroughput = previous
? previous.throughputBps * (1 - weight) + throughputBps * weight
: throughputBps;
const failure = observation.failed ? 1 : 0;
const failureRate = previous
? previous.failureRate * (1 - weight) + failure * weight
: failure;
let preferredLanes = observation.lanes;
let pipelineConcurrency = observation.pipelineConcurrency;
if (failureRate >= 0.2) {
preferredLanes = Math.max(1, Math.floor(preferredLanes / 2));
pipelineConcurrency = Math.max(8, Math.floor(pipelineConcurrency / 2));
} else if (
!observation.failed &&
previous &&
previous.samples >= 2 &&
throughputBps > previous.throughputBps * 1.25
) {
preferredLanes = Math.min(8, preferredLanes + 1);
pipelineConcurrency = Math.min(64, pipelineConcurrency + 8);
}
return {
throughputBps: smoothedThroughput,
failureRate,
samples: (previous?.samples ?? 0) + 1,
preferredLanes,
pipelineConcurrency,
updatedAt: now,
};
}
export function getTransferProfile(
key: string,
now = Date.now(),
): TransferPerformanceProfile | undefined {
const id = profileId(key);
const profile = profiles.get(id);
if (!profile) return undefined;
if (now - profile.updatedAt <= PROFILE_TTL_MS) return profile;
profiles.delete(id);
queuePersist();
return undefined;
}
export function recordTransferProfile(
key: string,
observation: Parameters<typeof updateTransferProfile>[1],
): TransferPerformanceProfile {
const profile = updateTransferProfile(getTransferProfile(key), observation);
profiles.set(profileId(key), profile);
trimProfiles(observation.now);
queuePersist();
return profile;
}
export function getDirectRouteProfile(
key: string,
now = Date.now(),
): DirectRouteProfile | undefined {
const id = profileId(key);
const profile = directRoutes.get(id);
if (!profile) return undefined;
if (now - profile.updatedAt <= PROFILE_TTL_MS) return profile;
directRoutes.delete(id);
queuePersist();
return undefined;
}
export function recordDirectRouteBenchmark(
key: string,
directMs: number,
relayMs: number,
now = Date.now(),
): DirectRouteProfile {
const previous = getDirectRouteProfile(key, now);
const weight = previous ? 0.25 : 1;
const profile: DirectRouteProfile = {
directMs: previous
? previous.directMs * (1 - weight) + directMs * weight
: directMs,
relayMs: previous
? previous.relayMs * (1 - weight) + relayMs * weight
: relayMs,
failureRate: previous?.failureRate ?? 0,
benchmarkSamples: (previous?.benchmarkSamples ?? 0) + 1,
outcomeSamples: previous?.outcomeSamples ?? 0,
benchmarkedAt: now,
cooldownUntil: previous?.cooldownUntil,
updatedAt: now,
};
directRoutes.set(profileId(key), profile);
trimProfiles(now);
queuePersist();
return profile;
}
export function recordDirectRouteOutcome(
key: string,
failed: boolean,
now = Date.now(),
cooldownMs = 10 * 60 * 1000,
): DirectRouteProfile | undefined {
const previous = getDirectRouteProfile(key, now);
if (!previous) return undefined;
const failure = failed ? 1 : 0;
const weight = previous.outcomeSamples > 0 ? 0.25 : 1;
const profile = {
...previous,
failureRate: previous.failureRate * (1 - weight) + failure * weight,
outcomeSamples: previous.outcomeSamples + 1,
cooldownUntil: failed ? now + cooldownMs : undefined,
updatedAt: now,
};
directRoutes.set(profileId(key), profile);
queuePersist();
return profile;
}
export function getRecentDirectRouteDecision(
key: string,
maxAgeMs: number,
now = Date.now(),
): { useDirect: boolean; directMs: number; relayMs: number } | undefined {
const profile = getDirectRouteProfile(key, now);
if (!profile) return undefined;
if ((profile.cooldownUntil ?? 0) > now) {
return {
useDirect: false,
directMs: profile.directMs,
relayMs: profile.relayMs,
};
}
if (now - profile.benchmarkedAt > maxAgeMs) return undefined;
return {
useDirect:
profile.failureRate < 0.2 && profile.directMs <= profile.relayMs * 0.8,
directMs: profile.directMs,
relayMs: profile.relayMs,
};
}
export function clearTransferProfiles(): void {
if (persistTimer) clearTimeout(persistTimer);
persistTimer = undefined;
loadedPath = undefined;
loadPromise = undefined;
profiles.clear();
directRoutes.clear();
}
src/backend/hosts/file-manager/trash-service.ts
+234
View File
@@ -0,0 +1,234 @@
import crypto from "node:crypto";
import path from "node:path";
import type { SFTPWrapper, Stats } from "ssh2";
export interface TrashItem {
id: string;
name: string;
originalPath: string;
isDirectory: boolean;
deletedAt: string;
size: number;
}
type StoredTrashItem = TrashItem & { trashPath: string };
const TRASH_DIR = ".termix-trash";
const ID_PATTERN = /^[0-9a-f-]{36}$/i;
function call<T>(
run: (done: (error: Error | undefined, value: T) => void) => void,
) {
return new Promise<T>((resolve, reject) => {
run((error, value) => (error ? reject(error) : resolve(value)));
});
}
function stat(sftp: SFTPWrapper, target: string) {
return call<Stats>((done) => sftp.stat(target, done));
}
function lstat(sftp: SFTPWrapper, target: string) {
return call<Stats>((done) => sftp.lstat(target, done));
}
function readdir(sftp: SFTPWrapper, target: string) {
return call<import("ssh2").FileEntry[]>((done) => sftp.readdir(target, done));
}
function readFile(sftp: SFTPWrapper, target: string) {
return call<Buffer>((done) => sftp.readFile(target, done));
}
function writeFile(sftp: SFTPWrapper, target: string, data: string) {
return call<void>((done) => sftp.writeFile(target, data, done));
}
function rename(sftp: SFTPWrapper, from: string, to: string) {
return call<void>((done) => sftp.rename(from, to, done));
}
function unlink(sftp: SFTPWrapper, target: string) {
return call<void>((done) => sftp.unlink(target, done));
}
function mkdir(sftp: SFTPWrapper, target: string) {
return call<void>((done) => sftp.mkdir(target, done));
}
function rmdir(sftp: SFTPWrapper, target: string) {
return call<void>((done) => sftp.rmdir(target, done));
}
async function exists(sftp: SFTPWrapper, target: string) {
try {
await stat(sftp, target);
return true;
} catch {
return false;
}
}
async function ensureDirectory(sftp: SFTPWrapper, target: string) {
const normalized = target.replace(/\\/g, "/");
const root = normalized.startsWith("/") ? "/" : "";
const parts = normalized.split("/").filter(Boolean);
let current = root;
for (const part of parts) {
current =
current === "/" ? `/${part}` : current ? `${current}/${part}` : part;
if (await exists(sftp, current)) continue;
await mkdir(sftp, current);
}
}
async function removeTree(sftp: SFTPWrapper, target: string): Promise<void> {
const targetStat = await lstat(sftp, target);
if (!targetStat.isDirectory()) {
await unlink(sftp, target);
return;
}
for (const entry of await readdir(sftp, target)) {
if (entry.filename === "." || entry.filename === "..") continue;
await removeTree(sftp, path.posix.join(target, entry.filename));
}
await rmdir(sftp, target);
}
async function trashPaths(sftp: SFTPWrapper) {
const home = await call<string>((done) => sftp.realpath(".", done));
const root = path.posix.join(home.replace(/\\/g, "/"), TRASH_DIR);
const files = path.posix.join(root, "files");
const info = path.posix.join(root, "info");
await ensureDirectory(sftp, files);
await ensureDirectory(sftp, info);
return { root, files, info };
}
export function isSafeTrashSource(itemPath: string, trashRoot: string) {
const normalized = path.posix.normalize(itemPath.replace(/\\/g, "/"));
const root = path.posix.normalize(trashRoot);
return (
normalized !== "." &&
normalized !== "/" &&
!/^[A-Za-z]:\/?$/.test(normalized) &&
normalized !== root &&
!normalized.startsWith(`${root}/`)
);
}
function publicItem(item: StoredTrashItem): TrashItem {
const { trashPath: _trashPath, ...result } = item;
return result;
}
async function readStoredItem(
sftp: SFTPWrapper,
dirs: { root: string; files: string; info: string },
id: string,
): Promise<StoredTrashItem> {
if (!ID_PATTERN.test(id)) throw new Error("Invalid trash item id");
const parsed = JSON.parse(
(await readFile(sftp, path.posix.join(dirs.info, `${id}.json`))).toString(
"utf8",
),
) as StoredTrashItem;
if (
parsed.id !== id ||
parsed.trashPath !== path.posix.join(dirs.files, id) ||
!isSafeTrashSource(parsed.originalPath, dirs.root)
) {
throw new Error("Invalid trash metadata");
}
return parsed;
}
export async function moveToTrash(
sftp: SFTPWrapper,
itemPath: string,
): Promise<TrashItem> {
const dirs = await trashPaths(sftp);
if (!isSafeTrashSource(itemPath, dirs.root)) {
throw new Error("This path cannot be moved to trash");
}
const itemStat = await lstat(sftp, itemPath);
const id = crypto.randomUUID();
const trashPath = path.posix.join(dirs.files, id);
const item: StoredTrashItem = {
id,
name: path.posix.basename(itemPath.replace(/\\/g, "/")),
originalPath: itemPath,
trashPath,
isDirectory: itemStat.isDirectory(),
deletedAt: new Date().toISOString(),
size: itemStat.size,
};
await rename(sftp, itemPath, trashPath);
try {
await writeFile(
sftp,
path.posix.join(dirs.info, `${id}.json`),
JSON.stringify(item),
);
} catch (error) {
await rename(sftp, trashPath, itemPath).catch(() => {});
throw error;
}
return publicItem(item);
}
export async function listTrash(
sftp: SFTPWrapper,
retentionDays: number,
): Promise<TrashItem[]> {
const dirs = await trashPaths(sftp);
const cutoff = Date.now() - retentionDays * 86_400_000;
const items: TrashItem[] = [];
for (const entry of await readdir(sftp, dirs.info)) {
if (!entry.filename.endsWith(".json")) continue;
const id = entry.filename.slice(0, -5);
try {
const item = await readStoredItem(sftp, dirs, id);
if (new Date(item.deletedAt).getTime() < cutoff) {
if (await exists(sftp, item.trashPath))
await removeTree(sftp, item.trashPath);
await unlink(sftp, path.posix.join(dirs.info, entry.filename));
continue;
}
if (await exists(sftp, item.trashPath)) items.push(publicItem(item));
else await unlink(sftp, path.posix.join(dirs.info, entry.filename));
} catch {
// Ignore corrupt metadata without exposing arbitrary paths to deletion.
}
}
return items.sort((a, b) => b.deletedAt.localeCompare(a.deletedAt));
}
export async function restoreTrashItem(sftp: SFTPWrapper, id: string) {
const dirs = await trashPaths(sftp);
const item = await readStoredItem(sftp, dirs, id);
if (await exists(sftp, item.originalPath)) {
throw new Error("A file already exists at the original path");
}
await rename(sftp, item.trashPath, item.originalPath);
await unlink(sftp, path.posix.join(dirs.info, `${id}.json`));
return publicItem(item);
}
export async function permanentlyDeleteTrashItem(
sftp: SFTPWrapper,
id: string,
) {
const dirs = await trashPaths(sftp);
const item = await readStoredItem(sftp, dirs, id);
if (await exists(sftp, item.trashPath))
await removeTree(sftp, item.trashPath);
await unlink(sftp, path.posix.join(dirs.info, `${id}.json`));
}
export async function emptyTrash(sftp: SFTPWrapper) {
const items = await listTrash(sftp, 365_000);
for (const item of items) await permanentlyDeleteTrashItem(sftp, item.id);
return items.length;
}
src/backend/hosts/guacamole/guacamole-server.ts
+13 -4
View File
@@ -1,12 +1,16 @@
import GuacamoleLite from "guacamole-lite";
import { guacLogger } from "../../utils/logger.js";
import { GuacamoleTokenService } from "./token-service.js";
import { getCurrentSettingValue } from "../../database/repositories/factory.js";
import {
GuacamoleTokenService,
type GuacamoleRecordingMetadata,
} from "./token-service.js";
import {
createCurrentSessionRecordingRepository,
getCurrentSettingValue,
} from "../../database/repositories/factory.js";
import { resolveGuacdOptions } from "../../utils/guacd-config.js";
import fs from "fs";
import path from "path";
import { createCurrentSessionRecordingRepository } from "../../database/repositories/factory.js";
import type { GuacamoleRecordingMetadata } from "./token-service.js";
const tokenService = GuacamoleTokenService.getInstance();
@@ -178,6 +182,11 @@ const clientOptions = {
cursor: "remote",
width: 1280,
height: 720,
// macOS Screen Sharing negotiates its VNC security type over several
// round trips (RFB type 30 -> 33/36/2/35) and can fail the first
// attempt; retrying lets guacd's VNC client re-establish instead of
// guacd giving up immediately (Support#1063).
autoretry: 2,
},
telnet: {
"terminal-type": "xterm-256color",
src/backend/hosts/guacamole/rdp-settings.ts
+36
View File
@@ -0,0 +1,36 @@
interface RdpSettingsInput {
port: number;
domain?: string;
security?: string;
ignoreCert: boolean;
guacConfig: Record<string, unknown>;
guacdOverrides: Record<string, unknown>;
}
export function buildRdpSettings({
port,
domain,
security,
ignoreCert,
guacConfig,
guacdOverrides,
}: RdpSettingsInput): Record<string, unknown> {
return {
...guacConfig,
port,
domain,
...(security === undefined ? {} : { security }),
"ignore-cert": ignoreCert,
...guacdOverrides,
};
}
export function resolveRdpDomain(
authType: string | null,
promptedDomain: unknown,
storedDomain: string,
): string {
return authType === "none" && typeof promptedDomain === "string"
? promptedDomain
: storedDomain;
}
src/backend/hosts/guacamole/routes.ts
+18 -11
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import express from "express";
import { GuacamoleTokenService } from "./token-service.js";
import { withRecordingSettings } from "./recording-settings.js";
@@ -21,6 +22,7 @@ import {
getRequestMeta,
} from "../../utils/audit-logger.js";
import { resolveJumpTunnelEndpoint } from "./jump-tunnel-endpoint.js";
import { buildRdpSettings, resolveRdpDomain } from "./rdp-settings.js";
const router = express.Router();
const tokenService = GuacamoleTokenService.getInstance();
@@ -284,7 +286,7 @@ router.post(
guacLogger.warn("Failed to parse guacamole config", {
operation: "guac_config_parse_error",
hostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
}
@@ -355,7 +357,7 @@ router.post(
operation: "guac_shared_secret_resolve",
hostId,
protocol: connectionType,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
} else {
@@ -385,7 +387,7 @@ router.post(
guacLogger.warn("Failed to resolve RDP credential", {
operation: "guac_rdp_credential_resolve",
hostId,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
}
@@ -404,7 +406,7 @@ router.post(
guacLogger.warn("Failed to resolve VNC credential", {
operation: "guac_vnc_credential_resolve",
hostId,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
}
@@ -426,7 +428,7 @@ router.post(
guacLogger.warn("Failed to resolve Telnet credential", {
operation: "guac_telnet_credential_resolve",
hostId,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
}
@@ -472,8 +474,13 @@ router.post(
username = "";
password = "";
}
const domain =
const storedDomain =
(host.rdpDomain as string) || (host.domain as string) || "";
const domain = resolveRdpDomain(
rdpAuthTypeForConnect,
req.body?.promptedDomain,
storedDomain,
);
// Establish SSH tunnel if jump hosts are configured
let jumpHosts: Array<{ hostId: number }> = [];
@@ -618,22 +625,22 @@ router.post(
hostname,
username,
password,
{
buildRdpSettings({
port,
domain,
security:
(host.rdpSecurity as string) ||
(host.security as string) ||
undefined,
"ignore-cert":
ignoreCert:
host.rdpIgnoreCert !== undefined
? !!host.rdpIgnoreCert
: host.ignoreCert !== undefined
? !!host.ignoreCert
: true,
...guacConfig,
...guacdOverrides,
},
guacConfig,
guacdOverrides,
}),
recordingMetadata,
termixMeta,
);
src/backend/hosts/host-resolver.ts
+31 -3
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../utils/error-message.js";
import {
createCurrentHostResolutionRepository,
createCurrentVaultProfileRepository,
@@ -17,6 +18,33 @@ import type { HostAction } from "../utils/permission-manager.js";
const sshLogger = logger;
/**
* Resolve a host the client named by its sync identity.
*
* `id` is an autoincrement belonging to whichever database produced the row.
* When the desktop app delegates a connection to a sync server, the two
* sequences have no reason to agree, and resolving the client's id here lands
* on whatever host happens to own that number — a different machine, with its
* own address, credentials and host key. `syncId` is the same string on both
* sides, so it names the host the user actually picked.
*
* Returns null when the sync id is unknown here, rather than falling back to
* the numeric id: an unknown host is exactly the case where guessing picks the
* wrong machine.
*/
export async function resolveHostBySyncId(
syncId: string,
userId: string,
): Promise<SSHHost | null> {
const hostId =
await createCurrentHostResolutionRepository().findHostIdBySyncId(syncId);
if (hostId === null) return null;
// Permissions, decryption, shared-host handling and auditing all belong to
// the id-based path; this only decides which row it is pointed at.
return resolveHostById(hostId, userId);
}
/**
* Resolve a host with its credentials server-side by hostId.
* This avoids passing credentials through the frontend.
@@ -154,7 +182,7 @@ export async function resolveHostById(
sshLogger.warn("Failed to resolve folder credential for host", {
operation: "host_resolver_folder_credential",
hostId,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
}
@@ -190,7 +218,7 @@ export async function resolveHostById(
sshLogger.warn("Failed to resolve credential for host", {
operation: "host_resolver_credential",
hostId,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
}
@@ -216,7 +244,7 @@ export async function resolveHostById(
sshLogger.warn("Failed to resolve vault profile for host", {
operation: "host_resolver_vault_profile",
hostId,
error: e instanceof Error ? e.message : "Unknown",
error: getErrorMessage(e, "Unknown"),
});
}
}
src/backend/hosts/metrics/alert-engine.ts
+58 -6
View File
@@ -5,6 +5,7 @@ import {
type AlertPayload,
type NotificationChannel,
} from "../../utils/notification-sender.js";
import { sendDiscord } from "../../utils/discord-sender.js";
type AlertTriggerType =
| "host_offline"
@@ -28,6 +29,24 @@ interface AlertRule {
cooldownMinutes: number;
}
/**
* Set once the alert rules have been copied into automations. From then on the
* automations engine owns evaluation, and this one stands down rather than
* sending a second notification for every rule.
*
* The class is left in place so an install that has not migrated yet, or one
* rolled back, still alerts exactly as before.
*/
let supersededByAutomations = false;
export function markAlertEngineSuperseded(): void {
supersededByAutomations = true;
}
export function isAlertEngineSuperseded(): boolean {
return supersededByAutomations;
}
export class AlertEngine {
private static instance: AlertEngine;
@@ -55,6 +74,7 @@ export class AlertEngine {
disk?: { percent: number | null } | null;
},
): Promise<void> {
if (supersededByAutomations) return;
const rules = (await this.loadRulesForHost(hostId)).filter((r) =>
["cpu_threshold", "memory_threshold", "disk_threshold"].includes(
r.triggerType,
@@ -103,6 +123,7 @@ export class AlertEngine {
}
async evaluateStatus(hostId: number, isOnline: boolean): Promise<void> {
if (supersededByAutomations) return;
const currentStatus = isOnline ? "online" : "offline";
const lastStatus = this.lastStatusMap.get(hostId);
@@ -138,6 +159,7 @@ export class AlertEngine {
ok: boolean,
detail?: string,
): Promise<void> {
if (supersededByAutomations) return;
const stateKey = `${hostId}:${checkId}`;
const lastOk = this.healthCheckStateMap.get(stateKey);
this.healthCheckStateMap.set(stateKey, ok);
@@ -173,6 +195,7 @@ export class AlertEngine {
sshUser: string,
fromIp: string,
): Promise<void> {
if (supersededByAutomations) return;
const rules = (await this.loadRulesForHostUser(hostId, userId)).filter(
(r) => r.triggerType === "user_login",
);
@@ -236,13 +259,42 @@ export class AlertEngine {
const channels = await this.loadChannelsForRule(rule.id);
for (const channel of channels) {
sendNotification(channel, payload).catch((err) => {
statsLogger.warn("Failed to send notification", {
operation: "notification_delivery_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
if (channel.type === "discord") {
// parse config and send via Discord sender directly
try {
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(channel.config) as Record<string, unknown>;
} catch {
statsLogger.warn("Failed to parse discord channel config", {
operation: "discord_config_parse_error",
channelId: channel.id,
});
continue;
}
sendDiscord(parsed as any, payload).catch((err) => {
statsLogger.warn("Failed to send discord notification", {
operation: "discord_notification_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
});
});
} catch (err) {
statsLogger.warn("Failed to send discord notification", {
operation: "discord_notification_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
});
}
} else {
sendNotification(channel, payload).catch((err) => {
statsLogger.warn("Failed to send notification", {
operation: "notification_delivery_error",
channelId: channel.id,
error: err instanceof Error ? err.message : String(err),
});
});
});
}
}
}
src/backend/hosts/metrics/automation-bridge.ts
+62
View File
@@ -0,0 +1,62 @@
/**
* One-line hand-off from the metrics poller to the automations engine.
*
* The poller is already a very large module, so the hooks it calls live here
* instead. Everything is fire-and-forget and imported lazily: a failure in the
* automations layer must never disturb metric collection, and a static import
* would create a cycle (automations reads repositories, which the metrics
* module also pulls in).
*/
import type { MetricsSnapshot } from "../../automations/conditions.js";
export function notifyAutomationMetrics(
hostId: number,
ownerUserId: string,
metrics: MetricsSnapshot,
): void {
if (!ownerUserId) return;
import("../../automations/triggers.js")
.then((triggers) => triggers.onMetrics({ hostId, ownerUserId, metrics }))
.catch(() => {});
}
export function notifyAutomationStatus(
hostId: number,
ownerUserId: string,
online: boolean,
): void {
if (!ownerUserId) return;
import("../../automations/triggers.js")
.then((triggers) => triggers.onStatus({ hostId, ownerUserId, online }))
.catch(() => {});
}
export function notifyAutomationHealthCheck(
hostId: number,
userId: string,
checkId: string,
ok: boolean,
detail?: string,
): void {
if (!userId) return;
import("../../automations/triggers.js")
.then((triggers) =>
triggers.onHealthCheck({ hostId, userId, checkId, ok, detail }),
)
.catch(() => {});
}
export function notifyAutomationInternalEvent(
event: string,
userId: string,
hostId?: number,
details?: Record<string, unknown>,
): void {
if (!userId) return;
import("../../automations/triggers.js")
.then((triggers) =>
triggers.onInternalEvent({ event, userId, hostId, details }),
)
.catch(() => {});
}
src/backend/hosts/metrics/helpers.ts
+12
View File
@@ -21,6 +21,18 @@ export function isTcpPingEnabled(statsConfig: TcpPingStatsConfig): boolean {
return statsConfig.statusCheckEnabled && !statsConfig.disableTcpPing;
}
export function parseStatusHostIds(value: unknown): Set<number> | null {
if (value === undefined) return null;
if (typeof value !== "string") return new Set();
return new Set(
value
.split(",")
.map(Number)
.filter((id) => Number.isSafeInteger(id) && id > 0),
);
}
export function tcpPingThroughJumpHost(
jumpClient: Pick<Client, "forwardOut" | "end">,
host: string,
src/backend/hosts/metrics/host-status.test.ts
+24
View File
@@ -0,0 +1,24 @@
import { describe, expect, it } from "vitest";
import {
statusAfterAuthentication,
statusAfterReachabilityCheck,
} from "./host-status.js";
describe("host availability status", () => {
it("does not call a TCP-reachable host online before authentication", () => {
expect(statusAfterReachabilityCheck(true)).toBe("reachable");
});
it("keeps a verified host online across later reachability checks", () => {
expect(statusAfterReachabilityCheck(true, "online")).toBe("online");
});
it("marks successful authentication online", () => {
expect(statusAfterAuthentication(true, "reachable")).toBe("online");
});
it("downgrades failed authentication without hiding reachability", () => {
expect(statusAfterAuthentication(false, "online")).toBe("reachable");
expect(statusAfterAuthentication(false, "offline")).toBe("offline");
});
});
src/backend/hosts/metrics/host-status.ts
+17
View File
@@ -0,0 +1,17 @@
export type HostStatus = "online" | "reachable" | "offline";
export function statusAfterReachabilityCheck(
reachable: boolean,
current?: HostStatus,
): HostStatus {
if (!reachable) return "offline";
return current === "online" ? "online" : "reachable";
}
export function statusAfterAuthentication(
authenticated: boolean,
current?: HostStatus,
): HostStatus {
if (authenticated) return "online";
return current === "offline" ? "offline" : "reachable";
}
src/backend/hosts/metrics/index.ts
+277 -48
View File
@@ -1,6 +1,8 @@
import { getErrorMessage } from "../../utils/error-message.js";
import express from "express";
import net from "net";
import { createCorsMiddleware } from "../../utils/cors-config.js";
import { createCompressionMiddleware } from "../../utils/compression-config.js";
import cookieParser from "cookie-parser";
import { Client, type ConnectConfig } from "ssh2";
import { SSH_ALGORITHMS } from "../../utils/ssh-algorithms.js";
@@ -47,7 +49,14 @@ import { registerHostMetricsSettingsRoutes } from "./settings-routes.js";
import { registerHostMetricsViewerRoutes } from "./viewer-routes.js";
import { registerHostMetricsPreferencesRoutes } from "./preferences-routes.js";
import { registerHostMetricsHistoryRoutes } from "./history-routes.js";
import { registerProxmoxStatsRoutes } from "./proxmox-stats-routes.js";
import { registerProxmoxStatsHistoryRoutes } from "./proxmox-stats-history-routes.js";
import { ProxmoxPollingManager } from "./proxmox-stats-polling.js";
import { AlertEngine } from "./alert-engine.js";
import {
notifyAutomationMetrics,
notifyAutomationStatus,
} from "./automation-bridge.js";
import { registerManagerRoutes } from "./managers/index.js";
import { resolveSshConnectConfigHost } from "../ssh-dns.js";
import { AccessDeniedError } from "./managers/route-helpers.js";
@@ -56,9 +65,15 @@ import { createJumpHostChain } from "../jump-host-chain.js";
import { resolveHostById } from "../host-resolver.js";
import {
isTcpPingEnabled,
parseStatusHostIds,
supportsMetrics,
tcpPingThroughJumpHost,
} from "./helpers.js";
import {
type HostStatus,
statusAfterAuthentication,
statusAfterReachabilityCheck,
} from "./host-status.js";
import { createConnectionLog } from "../connection-log.js";
import {
cleanupMetricsSession,
@@ -70,9 +85,12 @@ import {
} from "./sessions.js";
import {
authFailureTracker,
canStartInitialMetrics,
hostPollCache,
initialMetricsPollLimiter,
metricsCache,
metricsPollLimiter,
metricsConcurrencyFor,
pollingBackoff,
requestQueue,
statusPollLimiter,
@@ -81,8 +99,6 @@ import {
const authManager = AuthManager.getInstance();
const permissionManager = PermissionManager.getInstance();
type HostStatus = "online" | "offline";
interface SSHHostWithCredentials {
id: number;
name: string;
@@ -106,6 +122,8 @@ interface SSHHostWithCredentials {
tunnelConnections: unknown[];
jumpHosts?: Array<{ hostId: number }>;
statsConfig?: string | StatsConfig;
enableProxmoxStats?: boolean;
proxmoxStatsConfig?: unknown;
createdAt: string;
updatedAt: string;
userId: string;
@@ -137,6 +155,8 @@ interface StatsConfig {
metricsInterval: number;
useGlobalMetricsInterval?: boolean;
disableTcpPing?: boolean;
excludedMounts?: string[];
monitoredMounts?: Array<{ path: string; label?: string }>;
}
const DEFAULT_STATS_CONFIG: StatsConfig = {
@@ -157,6 +177,7 @@ const DEFAULT_STATS_CONFIG: StatsConfig = {
statusCheckInterval: 60,
metricsEnabled: true,
metricsInterval: 30,
excludedMounts: [],
};
interface HostPollingConfig {
@@ -183,6 +204,7 @@ class PollingManager {
/** Skip stacking another status/metrics poll while one is already running. */
private statusInFlight = new Set<number>();
private metricsInFlight = new Set<number>();
private initialMetricsRequested = new Set<number>();
constructor() {
this.viewerCleanupInterval = setInterval(() => {
@@ -190,6 +212,30 @@ class PollingManager {
}, 60000);
}
/**
* Keeps poll concurrency matched to how many hosts are actually being
* polled, so a sweep still finishes inside its interval as a fleet grows.
*/
private syncPollConcurrency(): void {
const metricsHosts = Array.from(this.pollingConfigs.values()).filter(
(config) => config.statsConfig.metricsEnabled,
).length;
const target = metricsConcurrencyFor(metricsHosts);
if (target === metricsPollLimiter.limit) return;
const previous = metricsPollLimiter.limit;
metricsPollLimiter.setLimit(target);
statsLogger.info(
`Metrics poll concurrency ${previous} -> ${target} for ${metricsHosts} host(s)`,
{
operation: "metrics_concurrency_resize",
hosts: metricsHosts,
concurrency: target,
},
);
}
/** Spread timers so N hosts do not fire on the same wall-clock second. */
private intervalWithJitter(intervalMs: number, hostId: number): number {
const spread = Math.min(intervalMs * 0.2, 15_000);
@@ -265,6 +311,54 @@ class PollingManager {
});
}
private scheduleInitialMetricsPoll(
host: SSHHostWithCredentials,
viewerUserId?: string,
): void {
if (
this.metricsStore.has(host.id) ||
this.initialMetricsRequested.has(host.id)
) {
return;
}
this.initialMetricsRequested.add(host.id);
void initialMetricsPollLimiter
.run(async () => {
if (
!canStartInitialMetrics(
this.statusStore.get(host.id)?.status,
this.activeViewers.has(host.id),
isTcpPingEnabled(
this.pollingConfigs.get(host.id)?.statsConfig ??
this.parseStatsConfig(host.statsConfig),
),
)
) {
return;
}
if (this.metricsInFlight.has(host.id)) return;
this.metricsInFlight.add(host.id);
try {
await metricsPollLimiter.run(() =>
this.pollHostMetrics(host, viewerUserId),
);
} finally {
this.metricsInFlight.delete(host.id);
}
})
.catch((err) => {
statsLogger.error("Initial metrics polling failed", err, {
operation: "initial_metrics_poll_unhandled",
hostId: host.id,
});
})
.finally(() => {
this.initialMetricsRequested.delete(host.id);
});
}
private getGlobalDefaults(): {
statusCheckInterval: number;
metricsInterval: number;
@@ -312,7 +406,7 @@ class PollingManager {
parsed = temp as StatsConfig;
} catch (error) {
statsLogger.warn(
`Failed to parse statsConfig: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to parse statsConfig: ${getErrorMessage(error)}`,
{
operation: "parse_stats_config_error",
statsConfigStr,
@@ -376,6 +470,7 @@ class PollingManager {
if (!isTcpPingEnabled(statsConfig) && !statsConfig.metricsEnabled) {
this.pollingConfigs.delete(host.id);
this.syncPollConcurrency();
this.statusStore.delete(host.id);
this.metricsStore.delete(host.id);
return;
@@ -386,6 +481,7 @@ class PollingManager {
statsConfig,
viewerUserId,
};
this.pollingConfigs.set(host.id, config);
if (isTcpPingEnabled(statsConfig)) {
const intervalMs = this.intervalWithJitter(
@@ -411,21 +507,18 @@ class PollingManager {
host.id,
);
// First sample still awaited (gated) so callers can rely on a warm cache.
if (!this.metricsInFlight.has(host.id)) {
this.metricsInFlight.add(host.id);
try {
await metricsPollLimiter.run(() =>
this.pollHostMetrics(host, viewerUserId),
);
} catch (err) {
statsLogger.error("Metrics polling failed", err, {
operation: "metrics_poll_unhandled",
hostId: host.id,
});
} finally {
this.metricsInFlight.delete(host.id);
}
// Viewer registration can arrive for an entire fleet at once. Only
// collect the first heavy SSH sample after the cheap status probe has
// confirmed the host is reachable; the status completion path below
// starts it when the result was not already cached.
if (
canStartInitialMetrics(
this.statusStore.get(host.id)?.status,
this.activeViewers.has(host.id),
isTcpPingEnabled(statsConfig),
)
) {
this.scheduleInitialMetricsPoll(host, viewerUserId);
}
config.metricsTimer = setInterval(() => {
@@ -445,7 +538,7 @@ class PollingManager {
this.metricsStore.delete(host.id);
}
this.pollingConfigs.set(host.id, config);
this.syncPollConcurrency();
}
private async pollHostStatus(
@@ -484,13 +577,23 @@ class PollingManager {
isOnline = await tcpPing(refreshedHost.ip, pingPort, 5000);
}
const statusEntry: StatusEntry = {
status: isOnline ? "online" : "offline",
status: statusAfterReachabilityCheck(
isOnline,
this.statusStore.get(refreshedHost.id)?.status,
),
lastChecked: new Date().toISOString(),
};
this.statusStore.set(refreshedHost.id, statusEntry);
if (isOnline && this.activeViewers.has(refreshedHost.id)) {
const config = this.pollingConfigs.get(refreshedHost.id);
if (config?.statsConfig.metricsEnabled) {
this.scheduleInitialMetricsPoll(config.host, config.viewerUserId);
}
}
AlertEngine.getInstance()
.evaluateStatus(refreshedHost.id, isOnline)
.catch(() => {});
notifyAutomationStatus(refreshedHost.id, refreshedHost.userId, isOnline);
} catch {
const statusEntry: StatusEntry = {
status: "offline",
@@ -500,6 +603,7 @@ class PollingManager {
AlertEngine.getInstance()
.evaluateStatus(refreshedHost.id, false)
.catch(() => {});
notifyAutomationStatus(refreshedHost.id, refreshedHost.userId, false);
}
}
@@ -535,8 +639,19 @@ class PollingManager {
return;
}
let authenticated = false;
try {
const metrics = await collectMetrics(refreshedHost);
const metrics = await collectMetrics(refreshedHost, () => {
authenticated = true;
this.statusStore.set(refreshedHost.id, {
status: statusAfterAuthentication(true),
lastChecked: new Date().toISOString(),
});
});
this.statusStore.set(refreshedHost.id, {
status: statusAfterAuthentication(true),
lastChecked: new Date().toISOString(),
});
this.metricsStore.set(refreshedHost.id, {
data: metrics,
timestamp: Date.now(),
@@ -545,9 +660,19 @@ class PollingManager {
AlertEngine.getInstance()
.evaluateMetrics(refreshedHost.id, metrics)
.catch(() => {});
notifyAutomationMetrics(refreshedHost.id, refreshedHost.userId, metrics);
pollingBackoff.reset(refreshedHost.id);
authFailureTracker.reset(refreshedHost.id);
} catch (error) {
if (!authenticated) {
this.statusStore.set(refreshedHost.id, {
status: statusAfterAuthentication(
false,
this.statusStore.get(refreshedHost.id)?.status,
),
lastChecked: new Date().toISOString(),
});
}
const isAuthError =
error instanceof Error &&
(error.message.includes("authentication") ||
@@ -643,6 +768,8 @@ class PollingManager {
}
this.pollingConfigs.delete(hostId);
this.syncPollConcurrency();
if (clearData) {
this.statusStore.delete(hostId);
this.metricsStore.delete(hostId);
@@ -685,6 +812,24 @@ class PollingManager {
}
}
async reconcileStatusPolling(
userId: string,
allowedHostIds: Set<number>,
): Promise<void> {
const hosts = await fetchAllHosts(userId);
for (const host of hosts) {
if (allowedHostIds.has(host.id)) {
if (!this.pollingConfigs.has(host.id)) {
await this.startPollingForHost(host, { statusOnly: true });
}
continue;
}
this.stopPollingForHost(host.id, true);
}
}
async refreshHostPolling(userId: string): Promise<void> {
const hosts = await fetchAllHosts(userId);
const currentHostIds = new Set(hosts.map((h) => h.id));
@@ -714,7 +859,11 @@ class PollingManager {
for (const [hostId, config] of this.pollingConfigs.entries()) {
const status = this.statusStore.get(hostId);
if (!status || status.status === "online") {
if (
!status ||
status.status === "online" ||
status.status === "reachable"
) {
hostsToRefresh.push({
host: config.host,
viewerUserId: config.viewerUserId,
@@ -838,6 +987,7 @@ function validateHostId(
}
const app = express();
app.use(createCompressionMiddleware());
app.use(createCorsMiddleware());
app.use(cookieParser());
app.use(express.json({ limit: "1mb" }));
@@ -897,7 +1047,7 @@ async function fetchAllHosts(
}
} catch (err) {
statsLogger.warn(
`Failed to resolve credentials for host ${host.id}: ${err instanceof Error ? err.message : "Unknown error"}`,
`Failed to resolve credentials for host ${host.id}: ${getErrorMessage(err)}`,
);
}
}
@@ -1089,7 +1239,7 @@ async function resolveHostCredentials(
}
} catch (error) {
statsLogger.warn(
`Failed to resolve credential ${host.credentialId} for host ${host.id}: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to resolve credential ${host.credentialId} for host ${host.id}: ${getErrorMessage(error)}`,
);
addLegacyCredentials(baseHost, host);
if (baseHost.authType === "credential") {
@@ -1107,7 +1257,7 @@ async function resolveHostCredentials(
return baseHost as unknown as SSHHostWithCredentials;
} catch (error) {
statsLogger.error(
`Failed to resolve host credentials for host ${host.id}: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to resolve host credentials for host ${host.id}: ${getErrorMessage(error)}`,
);
return undefined;
}
@@ -1213,7 +1363,7 @@ async function buildSshConfig(
}
} catch (keyError) {
statsLogger.error(
`SSH key format error for host ${host.ip}: ${keyError instanceof Error ? keyError.message : "Unknown error"}`,
`SSH key format error for host ${host.ip}: ${getErrorMessage(keyError)}`,
);
throw new Error(`Invalid SSH key format for host ${host.ip}`, {
cause: keyError,
@@ -1321,10 +1471,7 @@ function createSshFactory(host: SSHHostWithCredentials): () => Promise<Client> {
}
} catch (proxyError) {
throw new Error(
"Proxy connection failed: " +
(proxyError instanceof Error
? proxyError.message
: "Unknown error"),
"Proxy connection failed: " + getErrorMessage(proxyError),
{ cause: proxyError },
);
}
@@ -1450,7 +1597,17 @@ async function withSshConnection<T>(
return withConnection(key, factory, fn);
}
async function collectMetrics(host: SSHHostWithCredentials): Promise<{
const proxmoxPollingManager = new ProxmoxPollingManager<SSHHostWithCredentials>(
{
fetchHostById,
withSshConnection,
},
);
async function collectMetrics(
host: SSHHostWithCredentials,
onAuthenticated?: () => void,
): Promise<{
cpu: {
percent: number | null;
cores: number | null;
@@ -1476,6 +1633,8 @@ async function collectMetrics(host: SSHHostWithCredentials): Promise<{
state: string;
rxBytes: string | null;
txBytes: string | null;
rxRateBps: number | null;
txRateBps: number | null;
}>;
};
uptime: {
@@ -1510,6 +1669,7 @@ async function collectMetrics(host: SSHHostWithCredentials): Promise<{
const cached = metricsCache.get(host.id);
if (cached) {
onAuthenticated?.();
return cached as ReturnType<typeof collectMetrics> extends Promise<infer T>
? T
: never;
@@ -1520,10 +1680,22 @@ async function collectMetrics(host: SSHHostWithCredentials): Promise<{
const existingSession = metricsSessions[sessionKey];
try {
const excludedMounts = pollingManager.parseStatsConfig(
host.statsConfig,
).excludedMounts;
const monitoredMounts = pollingManager.parseStatsConfig(
host.statsConfig,
).monitoredMounts;
const collectFn = async (client: Client) => {
onAuthenticated?.();
const cpu = await collectCpuMetrics(client);
const memory = await collectMemoryMetrics(client);
const disk = await collectDiskMetrics(client);
const disk = await collectDiskMetrics(
client,
excludedMounts,
monitoredMounts,
);
const network = await collectNetworkMetrics(client);
const uptime = await collectUptimeMetrics(client);
const processes = await collectProcessesMetrics(client);
@@ -1749,15 +1921,28 @@ app.get("/status", async (req, res) => {
});
}
const statuses = pollingManager.getAllStatuses();
if (statuses.size === 0) {
const requestedHostIds = parseStatusHostIds(req.query.hostIds);
if (requestedHostIds !== null) {
await pollingManager.reconcileStatusPolling(userId, requestedHostIds);
} else if (pollingManager.getAllStatuses().size === 0) {
await pollingManager.initializePolling(userId);
}
// One batched permission resolution for the whole fleet; this endpoint is
// polled every few seconds, and a per-host check made it linear in host
// count against the database.
const entries = Array.from(pollingManager.getAllStatuses().entries());
const allowed = await permissionManager.filterAccessibleHostIds(
userId,
entries.map(([id]) => id),
);
const result: Record<number, StatusEntry> = {};
for (const [id, entry] of pollingManager.getAllStatuses().entries()) {
const access = await permissionManager.canAccessHost(userId, id, "connect");
if (access.hasAccess) {
for (const [id, entry] of entries) {
if (
allowed.has(id) &&
(requestedHostIds === null || requestedHostIds.has(id))
) {
result[id] = entry;
}
}
@@ -2420,7 +2605,7 @@ app.post("/metrics/start/:id", validateHostId, async (req, res) => {
createConnectionLog(
"error",
"proxy",
`Jump host connection failed: ${error instanceof Error ? error.message : "Unknown error"}`,
`Jump host connection failed: ${getErrorMessage(error)}`,
),
);
reject(error);
@@ -2456,7 +2641,7 @@ app.post("/metrics/start/:id", validateHostId, async (req, res) => {
createConnectionLog(
"error",
"proxy",
`SOCKS5 proxy connection failed: ${error instanceof Error ? error.message : "Unknown error"}`,
`SOCKS5 proxy connection failed: ${getErrorMessage(error)}`,
),
);
reject(error);
@@ -2489,14 +2674,11 @@ app.post("/metrics/start/:id", validateHostId, async (req, res) => {
createConnectionLog(
"error",
"stats_connecting",
`Failed to start metrics: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to start metrics: ${getErrorMessage(error)}`,
),
);
res.status(500).json({
error:
error instanceof Error
? error.message
: "Failed to start metrics collection",
error: getErrorMessage(error, "Failed to start metrics collection"),
connectionLogs,
});
}
@@ -2567,10 +2749,7 @@ app.post("/metrics/stop/:id", validateHostId, async (req, res) => {
error: error instanceof Error ? error.message : String(error),
});
res.status(500).json({
error:
error instanceof Error
? error.message
: "Failed to stop metrics collection",
error: getErrorMessage(error, "Failed to stop metrics collection"),
});
}
});
@@ -2747,6 +2926,20 @@ app.post("/metrics/connect-totp", async (req, res) => {
}
});
// Lets automations keep metrics flowing for hosts they watch, through the same
// viewer refcount the UI uses rather than around it.
import("../../automations/headless-viewer.js")
.then(({ setViewerRegistry }) => {
setViewerRegistry({
registerViewer: (hostId, sessionId, userId) =>
pollingManager.registerViewer(hostId, sessionId, userId),
unregisterViewer: (hostId, sessionId) =>
pollingManager.unregisterViewer(hostId, sessionId),
updateHeartbeat: (sessionId) => pollingManager.updateHeartbeat(sessionId),
});
})
.catch(() => {});
registerHostMetricsViewerRoutes(app, {
fetchHostById,
supportsMetrics: (host: SSHHostWithCredentials) => supportsMetrics(host),
@@ -2784,6 +2977,40 @@ registerHostMetricsHistoryRoutes(app, {
(await permissionManager.canAccessHost(userId, hostId, level)).hasAccess,
});
registerHostMetricsViewerRoutes<
SSHHostWithCredentials,
{ metricsEnabled: boolean }
>(app, {
fetchHostById,
// Proxmox Stats viewers are gated by enableProxmoxStats + host-type support,
// not the Host Metrics statsConfig - fold both checks in here since
// supportsMetrics receives the full host, unlike parseStatsConfig below.
supportsMetrics: (host: SSHHostWithCredentials) =>
supportsMetrics(host) && host.enableProxmoxStats === true,
parseStatsConfig: () => ({ metricsEnabled: true }),
updateHeartbeat: (viewerSessionId) =>
proxmoxPollingManager.updateHeartbeat(viewerSessionId),
registerViewer: (hostId, viewerSessionId, userId) =>
proxmoxPollingManager.registerViewer(hostId, viewerSessionId, userId),
unregisterViewer: (hostId, viewerSessionId) =>
proxmoxPollingManager.unregisterViewer(hostId, viewerSessionId),
pathPrefix: "proxmox-stats",
});
registerProxmoxStatsRoutes(app, {
validateHostId,
fetchHostById,
canAccessHost: async (userId, hostId, level) =>
(await permissionManager.canAccessHost(userId, hostId, level)).hasAccess,
pollingManager: proxmoxPollingManager,
});
registerProxmoxStatsHistoryRoutes(app, {
validateHostId,
canAccessHost: async (userId, hostId, level) =>
(await permissionManager.canAccessHost(userId, hostId, level)).hasAccess,
});
registerManagerRoutes(app, {
validateHostId,
runOnHost: async (hostId, userId, level, fn) => {
@@ -2814,12 +3041,14 @@ registerManagerRoutes(app, {
process.on("SIGINT", () => {
pollingManager.destroy();
proxmoxPollingManager.destroy();
connectionPool.destroy();
process.exit(0);
});
process.on("SIGTERM", () => {
pollingManager.destroy();
proxmoxPollingManager.destroy();
connectionPool.destroy();
process.exit(0);
});
src/backend/hosts/metrics/managers/health.ts
+15
View File
@@ -8,6 +8,7 @@ import { shellSingleQuote } from "./exec-elevated.js";
import { isValidPort } from "./validation.js";
import type { ManagerRoutesDeps } from "./types.js";
import { AlertEngine } from "../alert-engine.js";
import { notifyAutomationHealthCheck } from "../automation-bridge.js";
export interface HealthCheck {
id: string;
@@ -165,6 +166,13 @@ export function registerHealthRoutes(
r.detail ?? undefined,
)
.catch(() => {});
notifyAutomationHealthCheck(
host.id,
userId,
r.checkId,
r.ok,
r.detail ?? undefined,
);
}
}
@@ -244,6 +252,13 @@ export function registerHealthRoutes(
r.detail ?? undefined,
)
.catch(() => {});
notifyAutomationHealthCheck(
host.id,
userId,
r.checkId,
r.ok,
r.detail ?? undefined,
);
}
}
return { results };
src/backend/hosts/metrics/managers/logs.ts
+1 -2
View File
@@ -1,8 +1,7 @@
import type { Express } from "express";
import { execCommand } from "../widgets/common-utils.js";
import { execElevated } from "./exec-elevated.js";
import { execElevated, shellSingleQuote } from "./exec-elevated.js";
import { managerHandler, ManagerInputError } from "./route-helpers.js";
import { shellSingleQuote } from "./exec-elevated.js";
import { isAllowedPath, isValidSystemdUnit } from "./validation.js";
import type { ManagerRoutesDeps } from "./types.js";
src/backend/hosts/metrics/managers/route-helpers.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../../utils/error-message.js";
import type { Request, Response } from "express";
import type { Client } from "ssh2";
import type { AuthenticatedRequest } from "../../../../types/index.js";
@@ -54,7 +55,7 @@ export function managerHandler(
error: error instanceof Error ? error.message : String(error),
});
return res.status(500).json({
error: error instanceof Error ? error.message : "Operation failed",
error: getErrorMessage(error, "Operation failed"),
});
}
};
src/backend/hosts/metrics/managers/wireguard.ts
+1 -2
View File
@@ -1,7 +1,6 @@
import type { Express } from "express";
import { execElevated } from "./exec-elevated.js";
import { managerHandler } from "./route-helpers.js";
import { ManagerInputError } from "./route-helpers.js";
import { managerHandler, ManagerInputError } from "./route-helpers.js";
import type { ManagerRoutesDeps } from "./types.js";
import {
isValidWireGuardInterface,
src/backend/hosts/metrics/proxmox-stats-history-routes.ts
+135
View File
@@ -0,0 +1,135 @@
import type { Express, RequestHandler } from "express";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { createCurrentProxmoxNodeHistoryRepository } from "../../database/repositories/factory.js";
import { statsLogger } from "../../utils/logger.js";
import type { HostAction } from "../../utils/permission-manager.js";
type ProxmoxStatsHistoryRoutesDeps = {
validateHostId: RequestHandler;
canAccessHost: (
userId: string,
hostId: number,
level: HostAction,
) => Promise<boolean>;
};
const RANGE_OFFSETS: Record<string, number> = {
"1h": 1 * 60 * 60 * 1000,
"6h": 6 * 60 * 60 * 1000,
"24h": 24 * 60 * 60 * 1000,
"7d": 7 * 24 * 60 * 60 * 1000,
"30d": 30 * 24 * 60 * 60 * 1000,
};
export function registerProxmoxStatsHistoryRoutes(
app: Express,
{ validateHostId, canAccessHost }: ProxmoxStatsHistoryRoutesDeps,
): void {
/**
* @openapi
* /proxmox-stats/history/{hostId}:
* get:
* summary: Get historical Proxmox node stats for a host
* tags:
* - Proxmox Stats
* parameters:
* - in: path
* name: hostId
* required: true
* schema:
* type: integer
* - in: query
* name: range
* schema:
* type: string
* enum: [1h, 6h, 24h, 7d, 30d]
* - in: query
* name: from
* schema:
* type: string
* format: date-time
* - in: query
* name: to
* schema:
* type: string
* format: date-time
* responses:
* 200:
* description: Array of node history rows.
* 403:
* description: Access denied.
*/
app.get(
"/proxmox-stats/history/:hostId",
validateHostId,
async (req, res) => {
const hostId = Number(req.params.hostId);
const userId = (req as AuthenticatedRequest).userId;
try {
const hasAccess = await canAccessHost(userId, hostId, "connect");
if (!hasAccess) {
return res.status(403).json({ error: "Access denied" });
}
const { range, from, to } = req.query as Record<
string,
string | undefined
>;
let fromTs: string;
let toTs: string = new Date().toISOString();
if (range) {
const offsetMs = RANGE_OFFSETS[range];
if (!offsetMs) {
return res
.status(400)
.json({ error: "Invalid range. Use 1h, 6h, 24h, 7d, or 30d" });
}
fromTs = new Date(Date.now() - offsetMs).toISOString();
} else if (from && to) {
const fromDate = new Date(from);
const toDate = new Date(to);
if (isNaN(fromDate.getTime()) || isNaN(toDate.getTime())) {
return res
.status(400)
.json({ error: "Invalid from/to date format" });
}
fromTs = fromDate.toISOString();
toTs = toDate.toISOString();
} else {
fromTs = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
}
const toSqlite = (iso: string) =>
iso.replace("T", " ").replace(/\.\d{3}Z$/, "");
const rows = (
await createCurrentProxmoxNodeHistoryRepository().listRange(
hostId,
toSqlite(fromTs),
toSqlite(toTs),
)
).map((row) => ({
ts: row.ts,
cpu_percent: row.cpuPercent,
mem_percent: row.memPercent,
disk_percent: row.diskPercent,
net_rx_bytes: row.netRxBytes,
net_tx_bytes: row.netTxBytes,
}));
res.json({ rows, fromTs, toTs });
} catch (error) {
statsLogger.error("Failed to fetch proxmox stats history", {
operation: "proxmox_stats_history_fetch_error",
hostId,
error: error instanceof Error ? error.message : String(error),
});
res
.status(500)
.json({ error: "Failed to fetch proxmox stats history" });
}
},
);
}
src/backend/hosts/metrics/proxmox-stats-polling.ts
+332
View File
@@ -0,0 +1,332 @@
import { statsLogger } from "../../utils/logger.js";
import {
createCurrentProxmoxNodeHistoryRepository,
getCurrentSettingValue,
} from "../../database/repositories/factory.js";
import { ConcurrentLimiter } from "./state.js";
import {
collectProxmoxStats,
type ProxmoxStatsSnapshot,
} from "./proxmox/collect-proxmox-stats.js";
import type { Client } from "ssh2";
/**
* Proxmox Stats polling gets its own limiter, separate from Host Metrics'
* metricsPollLimiter in state.ts, so a burst of Proxmox polls can never starve
* regular Host Metrics collection (or vice versa).
*/
const proxmoxStatsPollLimiter = new ConcurrentLimiter(5);
export interface ProxmoxStatsPollableHost {
id: number;
userId: string;
proxmoxStatsConfig?: string | ProxmoxStatsPollConfig | null;
}
export interface ProxmoxStatsPollConfig {
nodeName?: string | null;
pollInterval?: number;
enabledCards?: string[];
}
const DEFAULT_POLL_INTERVAL_SECONDS = 60;
export function parseProxmoxStatsConfig(
raw: string | ProxmoxStatsPollConfig | null | undefined,
): ProxmoxStatsPollConfig {
if (!raw) {
return { nodeName: null, pollInterval: DEFAULT_POLL_INTERVAL_SECONDS };
}
if (typeof raw === "object") {
return {
nodeName: raw.nodeName ?? null,
pollInterval: raw.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS,
enabledCards: raw.enabledCards,
};
}
try {
const parsed = JSON.parse(raw) as ProxmoxStatsPollConfig;
return {
nodeName: parsed.nodeName ?? null,
pollInterval: parsed.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS,
enabledCards: parsed.enabledCards,
};
} catch {
return { nodeName: null, pollInterval: DEFAULT_POLL_INTERVAL_SECONDS };
}
}
interface HostPollingEntry<THost extends ProxmoxStatsPollableHost> {
host: THost;
timer?: NodeJS.Timeout;
viewerUserId?: string;
}
interface ViewerDetail {
sessionId: string;
userId: string;
hostId: number;
lastHeartbeat: number;
}
interface CachedSnapshot {
data: ProxmoxStatsSnapshot;
timestamp: number;
}
interface ErrorSnapshot {
error: string;
timestamp: number;
}
export class ProxmoxPollingManager<
THost extends ProxmoxStatsPollableHost = ProxmoxStatsPollableHost,
> {
private pollingConfigs = new Map<number, HostPollingEntry<THost>>();
private snapshotStore = new Map<number, CachedSnapshot>();
private errorStore = new Map<number, ErrorSnapshot>();
private activeViewers = new Map<number, Set<string>>();
private viewerDetails = new Map<string, ViewerDetail>();
private inFlight = new Set<number>();
private viewerCleanupInterval: NodeJS.Timeout;
constructor(
private readonly deps: {
fetchHostById: (
hostId: number,
userId: string,
) => Promise<THost | undefined>;
withSshConnection: <T>(
host: THost,
fn: (client: Client) => Promise<T>,
) => Promise<T>;
historyEnabled?: () => boolean;
},
) {
this.viewerCleanupInterval = setInterval(() => {
this.cleanupInactiveViewers();
}, 60000);
}
private intervalWithJitter(intervalMs: number, hostId: number): number {
const spread = Math.min(intervalMs * 0.2, 15_000);
const jitter = (hostId * 1103515245) % Math.max(1, Math.floor(spread));
return intervalMs + jitter;
}
private getRetentionDays(): number {
try {
const value = getCurrentSettingValue("metrics_history_retention_days");
const days = value ? parseInt(value, 10) : 7;
return isNaN(days) || days < 1 ? 7 : Math.min(days, 90);
} catch {
return 7;
}
}
private async pollHostStats(
host: THost,
viewerUserId?: string,
): Promise<void> {
if (this.inFlight.has(host.id)) return;
this.inFlight.add(host.id);
try {
await proxmoxStatsPollLimiter.run(async () => {
const userId = viewerUserId || host.userId;
const refreshed =
(await this.deps.fetchHostById(host.id, userId)) ?? host;
const config = parseProxmoxStatsConfig(refreshed.proxmoxStatsConfig);
try {
const snapshot = await this.deps.withSshConnection(
refreshed,
(client) => collectProxmoxStats(client, config.nodeName),
);
this.snapshotStore.set(host.id, {
data: snapshot,
timestamp: Date.now(),
});
this.errorStore.delete(host.id);
if (this.deps.historyEnabled?.() ?? true) {
await this.insertHistory(host.id, snapshot);
}
} catch (error) {
const message =
error instanceof Error ? error.message : String(error);
this.errorStore.set(host.id, {
error: message,
timestamp: Date.now(),
});
statsLogger.warn("Proxmox stats poll failed", {
operation: "proxmox_stats_poll_failed",
hostId: host.id,
error: message,
});
}
});
} finally {
this.inFlight.delete(host.id);
}
}
private async insertHistory(
hostId: number,
snapshot: ProxmoxStatsSnapshot,
): Promise<void> {
try {
const iface = snapshot.network?.interfaces?.[0];
const rxRaw = iface?.rxBytes ? parseInt(iface.rxBytes, 10) : null;
const txRaw = iface?.txBytes ? parseInt(iface.txBytes, 10) : null;
const repository = createCurrentProxmoxNodeHistoryRepository();
await repository.create({
hostId,
cpuPercent: snapshot.node?.cpu?.percent ?? null,
memPercent: snapshot.node?.memory?.percent ?? null,
diskPercent: snapshot.node?.disk?.percent ?? null,
netRxBytes: rxRaw !== null && !isNaN(rxRaw) ? rxRaw : null,
netTxBytes: txRaw !== null && !isNaN(txRaw) ? txRaw : null,
});
await repository.pruneOlderThan(hostId, this.getRetentionDays());
} catch (err) {
statsLogger.warn("Failed to write proxmox node history", {
operation: "insert_proxmox_node_history",
hostId,
error: err instanceof Error ? err.message : String(err),
});
}
}
private startPollingForHost(host: THost, viewerUserId?: string): void {
const existing = this.pollingConfigs.get(host.id);
if (existing?.timer) {
clearInterval(existing.timer);
}
const config = parseProxmoxStatsConfig(host.proxmoxStatsConfig);
const intervalMs = this.intervalWithJitter(
(config.pollInterval ?? DEFAULT_POLL_INTERVAL_SECONDS) * 1000,
host.id,
);
void this.pollHostStats(host, viewerUserId);
const timer = setInterval(() => {
const latest = this.pollingConfigs.get(host.id);
if (latest) {
void this.pollHostStats(latest.host, latest.viewerUserId);
}
}, intervalMs);
this.pollingConfigs.set(host.id, { host, timer, viewerUserId });
}
private stopPollingForHost(hostId: number): void {
const config = this.pollingConfigs.get(hostId);
if (config?.timer) {
clearInterval(config.timer);
}
this.pollingConfigs.delete(hostId);
}
getStats(
hostId: number,
): { data: ProxmoxStatsSnapshot; timestamp: number } | undefined {
return this.snapshotStore.get(hostId);
}
getError(hostId: number): ErrorSnapshot | undefined {
return this.errorStore.get(hostId);
}
async ensurePolling(host: THost, viewerUserId?: string): Promise<void> {
if (!this.pollingConfigs.has(host.id)) {
this.startPollingForHost(host, viewerUserId);
}
if (!this.snapshotStore.has(host.id) && !this.inFlight.has(host.id)) {
await this.pollHostStats(host, viewerUserId);
}
}
registerViewer = (
hostId: number,
sessionId: string,
userId: string,
): void => {
if (!this.activeViewers.has(hostId)) {
this.activeViewers.set(hostId, new Set());
}
this.activeViewers.get(hostId)!.add(sessionId);
this.viewerDetails.set(sessionId, {
sessionId,
userId,
hostId,
lastHeartbeat: Date.now(),
});
if (this.activeViewers.get(hostId)!.size === 1) {
Promise.resolve()
.then(async () => {
const host = await this.deps.fetchHostById(hostId, userId);
if (host) {
this.startPollingForHost(host, userId);
}
})
.catch((err) => {
statsLogger.warn(
"Proxmox stats startPollingForHost rejected (non-fatal)",
{
operation: "proxmox_stats_start_unhandled",
hostId,
userId,
error: err instanceof Error ? err.message : String(err),
},
);
});
}
};
unregisterViewer = (hostId: number, sessionId: string): void => {
const viewers = this.activeViewers.get(hostId);
if (viewers) {
viewers.delete(sessionId);
if (viewers.size === 0) {
this.activeViewers.delete(hostId);
this.stopPollingForHost(hostId);
}
}
this.viewerDetails.delete(sessionId);
};
updateHeartbeat(sessionId: string): boolean {
const viewer = this.viewerDetails.get(sessionId);
if (viewer) {
viewer.lastHeartbeat = Date.now();
return true;
}
return false;
}
private cleanupInactiveViewers(): void {
const now = Date.now();
const maxInactivity = 120000;
for (const [sessionId, viewer] of this.viewerDetails.entries()) {
if (now - viewer.lastHeartbeat > maxInactivity) {
this.unregisterViewer(viewer.hostId, sessionId);
}
}
}
destroy(): void {
clearInterval(this.viewerCleanupInterval);
for (const hostId of this.pollingConfigs.keys()) {
this.stopPollingForHost(hostId);
}
}
}
src/backend/hosts/metrics/proxmox-stats-routes.ts
+258
View File
@@ -0,0 +1,258 @@
import { getErrorMessage } from "../../utils/error-message.js";
import type { Express, RequestHandler } from "express";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { statsLogger } from "../../utils/logger.js";
import { DataCrypto } from "../../utils/data-crypto.js";
import type { HostAction } from "../../utils/permission-manager.js";
import {
type ProxmoxPollingManager,
type ProxmoxStatsPollableHost,
} from "./proxmox-stats-polling.js";
const EMPTY_SNAPSHOT = {
node: {
cpu: { percent: null, cores: null, load: null },
memory: { percent: null, usedGiB: null, totalGiB: null },
disk: { percent: null, usedGiB: null, totalGiB: null },
uptime: { seconds: null, formatted: null },
system: { hostname: null, kernel: null, pveVersion: null },
},
network: { interfaces: [] },
guests: { guests: [], counts: { running: 0, stopped: 0, total: 0 } },
storage: { pools: [] },
cluster: { clustered: false },
lastChecked: new Date(0).toISOString(),
};
type ProxmoxStatsHost = ProxmoxStatsPollableHost & {
enableProxmoxStats?: boolean;
};
type ProxmoxStatsRoutesDeps = {
validateHostId: RequestHandler;
fetchHostById: (
hostId: number,
userId: string,
) => Promise<ProxmoxStatsHost | undefined>;
canAccessHost: (
userId: string,
hostId: number,
level: HostAction,
) => Promise<boolean>;
pollingManager: ProxmoxPollingManager<ProxmoxStatsHost>;
};
export function registerProxmoxStatsRoutes(
app: Express,
{
validateHostId,
fetchHostById,
canAccessHost,
pollingManager,
}: ProxmoxStatsRoutesDeps,
): void {
/**
* @openapi
* /proxmox-stats/{id}:
* get:
* summary: Get cached Proxmox node stats for a host
* description: Returns the most recently polled Proxmox Stats snapshot for a host, or an empty skeleton if none has been collected yet.
* tags:
* - Proxmox Stats
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* responses:
* 200:
* description: Proxmox stats snapshot.
* 401:
* description: Session expired - please log in again.
* 404:
* description: Stats not available yet.
*/
app.get("/proxmox-stats/:id", validateHostId, async (req, res) => {
const id = Number(req.params.id);
const userId = (req as AuthenticatedRequest).userId;
if (DataCrypto.getUserDataKey(userId) === null) {
return res.status(401).json({
error: "Session expired - please log in again",
code: "SESSION_EXPIRED",
});
}
const cached = pollingManager.getStats(id);
if (!cached) {
const errorState = pollingManager.getError(id);
return res.status(404).json({
error: errorState?.error || "Stats not available",
...EMPTY_SNAPSHOT,
lastChecked: new Date().toISOString(),
});
}
res.json({
...cached.data,
lastChecked: new Date(cached.timestamp).toISOString(),
});
});
/**
* @openapi
* /proxmox-stats/start/{id}:
* post:
* summary: Start Proxmox stats collection
* description: Registers a viewer and starts (or reuses) polling for a host's Proxmox node stats.
* tags:
* - Proxmox Stats
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* responses:
* 200:
* description: Polling started, snapshot returned if already available.
* 401:
* description: Session expired - please log in again.
* 403:
* description: Proxmox Stats is not enabled for this host.
* 404:
* description: Host not found.
*/
app.post("/proxmox-stats/start/:id", validateHostId, async (req, res) => {
const id = Number(req.params.id);
const userId = (req as AuthenticatedRequest).userId;
if (DataCrypto.getUserDataKey(userId) === null) {
return res.status(401).json({
error: "Session expired - please log in again",
code: "SESSION_EXPIRED",
});
}
try {
if (!(await canAccessHost(userId, id, "connect"))) {
return res.status(403).json({ error: "No access to this host" });
}
const host = await fetchHostById(id, userId);
if (!host) {
return res.status(404).json({ error: "Host not found" });
}
if (!host.enableProxmoxStats) {
return res
.status(403)
.json({ error: "Proxmox Stats is not enabled for this host" });
}
const viewerSessionId = `proxmox-viewer-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
pollingManager.registerViewer(id, viewerSessionId, userId);
await pollingManager.ensurePolling(host, userId);
const cached = pollingManager.getStats(id);
if (cached) {
return res.json({
success: true,
viewerSessionId,
...cached.data,
lastChecked: new Date(cached.timestamp).toISOString(),
});
}
const errorState = pollingManager.getError(id);
if (errorState) {
return res.json({
success: true,
viewerSessionId,
status: "error",
error: errorState.error,
});
}
return res.json({
success: true,
viewerSessionId,
status: "collecting",
});
} catch (error) {
statsLogger.error("Failed to start proxmox stats collection", {
operation: "proxmox_stats_start_error",
hostId: id,
error: error instanceof Error ? error.message : String(error),
});
res.status(500).json({
error: getErrorMessage(
error,
"Failed to start proxmox stats collection",
),
});
}
});
/**
* @openapi
* /proxmox-stats/stop/{id}:
* post:
* summary: Stop Proxmox stats collection
* description: Unregisters a viewer session for a host's Proxmox node stats polling.
* tags:
* - Proxmox Stats
* parameters:
* - in: path
* name: id
* required: true
* schema:
* type: integer
* requestBody:
* required: false
* content:
* application/json:
* schema:
* type: object
* properties:
* viewerSessionId:
* type: string
* responses:
* 200:
* description: Polling stopped successfully.
* 401:
* description: Session expired - please log in again.
*/
app.post("/proxmox-stats/stop/:id", validateHostId, async (req, res) => {
const id = Number(req.params.id);
const userId = (req as AuthenticatedRequest).userId;
const { viewerSessionId } = req.body as { viewerSessionId?: string };
if (DataCrypto.getUserDataKey(userId) === null) {
return res.status(401).json({
error: "Session expired - please log in again",
code: "SESSION_EXPIRED",
});
}
try {
if (viewerSessionId && typeof viewerSessionId === "string") {
pollingManager.unregisterViewer(id, viewerSessionId);
}
res.json({ success: true });
} catch (error) {
statsLogger.error("Failed to stop proxmox stats collection", {
operation: "proxmox_stats_stop_error",
hostId: id,
error: error instanceof Error ? error.message : String(error),
});
res.status(500).json({
error: getErrorMessage(
error,
"Failed to stop proxmox stats collection",
),
});
}
});
}
src/backend/hosts/metrics/proxmox/cluster-health-collector.ts
+67
View File
@@ -0,0 +1,67 @@
import type { Client } from "ssh2";
import { execCommand } from "../widgets/common-utils.js";
export interface ProxmoxClusterNodeEntry {
name: string;
online: boolean;
local: boolean;
ip: string | null;
}
export type ProxmoxClusterHealthResult =
| { clustered: false }
| {
clustered: true;
quorate: boolean;
clusterName: string | null;
nodes: ProxmoxClusterNodeEntry[];
};
const EMPTY_RESULT: ProxmoxClusterHealthResult = { clustered: false };
export async function collectProxmoxClusterHealth(
client: Client,
): Promise<ProxmoxClusterHealthResult> {
try {
const { stdout, code } = await execCommand(
client,
"pvesh get /cluster/status --output-format json",
15000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const data = JSON.parse(stdout);
if (!Array.isArray(data)) {
return EMPTY_RESULT;
}
const entries = data as Array<Record<string, unknown>>;
const clusterEntry = entries.find((e) => e.type === "cluster");
if (!clusterEntry) {
return EMPTY_RESULT;
}
const nodes: ProxmoxClusterNodeEntry[] = entries
.filter((e) => e.type === "node")
.map((e) => ({
name: typeof e.name === "string" ? e.name : "",
online: e.online === 1 || e.online === true,
local: e.local === 1 || e.local === true,
ip: typeof e.ip === "string" && e.ip ? e.ip : null,
}));
return {
clustered: true,
quorate: clusterEntry.quorate === 1 || clusterEntry.quorate === true,
clusterName:
typeof clusterEntry.name === "string" && clusterEntry.name
? clusterEntry.name
: null,
nodes,
};
} catch {
return EMPTY_RESULT;
}
}
src/backend/hosts/metrics/proxmox/collect-proxmox-stats.ts
+86
View File
@@ -0,0 +1,86 @@
import type { Client } from "ssh2";
import { execCommand } from "../widgets/common-utils.js";
import { isSafeNodeName } from "../../proxmox-shared.js";
import {
collectProxmoxNodeStatus,
type ProxmoxNodeStatusResult,
} from "./node-status-collector.js";
import {
collectProxmoxNodeNetwork,
type ProxmoxNodeNetworkResult,
} from "./node-network-collector.js";
import {
collectProxmoxGuestsSummary,
type ProxmoxGuestsSummaryResult,
} from "./guests-collector.js";
import {
collectProxmoxStorage,
type ProxmoxStorageResult,
} from "./storage-collector.js";
import {
collectProxmoxClusterHealth,
type ProxmoxClusterHealthResult,
} from "./cluster-health-collector.js";
export interface ProxmoxStatsSnapshot {
node: ProxmoxNodeStatusResult;
network: ProxmoxNodeNetworkResult;
guests: ProxmoxGuestsSummaryResult;
storage: ProxmoxStorageResult;
cluster: ProxmoxClusterHealthResult;
lastChecked: string;
}
async function resolveNodeName(
client: Client,
configuredNodeName: string | null | undefined,
): Promise<string> {
if (configuredNodeName) {
// An explicitly configured name is validated and used as-is - an unsafe
// value is rejected outright rather than silently falling back to
// auto-detection, which would mask a misconfigured (or malicious) override.
if (!isSafeNodeName(configuredNodeName)) {
throw new Error("Unable to determine a valid Proxmox node name");
}
return configuredNodeName;
}
const { stdout } = await execCommand(client, "hostname", 10000);
return stdout.trim();
}
export async function collectProxmoxStats(
client: Client,
configuredNodeName: string | null | undefined,
): Promise<ProxmoxStatsSnapshot> {
const pveshCheck = await execCommand(
client,
"command -v pvesh >/dev/null 2>&1 && echo ok || echo missing",
10000,
);
if (pveshCheck.stdout.trim() !== "ok") {
throw new Error("pvesh not found — is this a Proxmox node?");
}
const nodeName = await resolveNodeName(client, configuredNodeName);
if (!isSafeNodeName(nodeName)) {
throw new Error("Unable to determine a valid Proxmox node name");
}
const [node, network, guests, storage, cluster] = await Promise.all([
collectProxmoxNodeStatus(client, nodeName),
collectProxmoxNodeNetwork(client, nodeName),
collectProxmoxGuestsSummary(client, nodeName),
collectProxmoxStorage(client, nodeName),
collectProxmoxClusterHealth(client),
]);
return {
node,
network,
guests,
storage,
cluster,
lastChecked: new Date().toISOString(),
};
}
src/backend/hosts/metrics/proxmox/guests-collector.ts
+113
View File
@@ -0,0 +1,113 @@
import type { Client } from "ssh2";
import { execCommand, toFixedNum } from "../widgets/common-utils.js";
import { isSafeNodeName } from "../../proxmox-shared.js";
export interface ProxmoxGuestSummaryEntry {
vmid: number;
name: string;
type: "qemu" | "lxc";
status: string;
cpuPercent: number | null;
memPercent: number | null;
memUsedGiB: number | null;
memTotalGiB: number | null;
diskPercent: number | null;
diskUsedGiB: number | null;
diskTotalGiB: number | null;
uptimeSeconds: number | null;
}
export interface ProxmoxGuestsSummaryResult {
guests: ProxmoxGuestSummaryEntry[];
counts: { running: number; stopped: number; total: number };
}
const EMPTY_RESULT: ProxmoxGuestsSummaryResult = {
guests: [],
counts: { running: 0, stopped: 0, total: 0 },
};
function bytesToGiB(bytes: number): number {
return bytes / (1024 * 1024 * 1024);
}
export async function collectProxmoxGuestsSummary(
client: Client,
nodeName: string,
): Promise<ProxmoxGuestsSummaryResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
"pvesh get /cluster/resources --output-format json",
25000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const resources = JSON.parse(stdout);
if (!Array.isArray(resources)) {
return EMPTY_RESULT;
}
const guests: ProxmoxGuestSummaryEntry[] = [];
for (const r of resources as Array<Record<string, unknown>>) {
const type = r.type;
if (type !== "qemu" && type !== "lxc") continue;
if (r.node !== nodeName) continue;
if (r.template === true || r.template === 1) continue;
const cpuFraction = typeof r.cpu === "number" ? r.cpu : null;
const mem = typeof r.mem === "number" ? r.mem : null;
const maxmem = typeof r.maxmem === "number" ? r.maxmem : null;
const memPercent =
mem !== null && maxmem !== null && maxmem > 0
? Math.max(0, Math.min(100, (mem / maxmem) * 100))
: null;
const disk = typeof r.disk === "number" ? r.disk : null;
const maxdisk = typeof r.maxdisk === "number" ? r.maxdisk : null;
// QEMU guests without a running agent report maxdisk: 0 - a false 0%
// is worse than an honest "unknown", so treat that as no data at all.
const hasDiskData = maxdisk !== null && maxdisk > 0;
const diskPercent =
hasDiskData && disk !== null
? Math.max(0, Math.min(100, (disk / maxdisk) * 100))
: null;
guests.push({
vmid: typeof r.vmid === "number" ? r.vmid : Number(r.vmid),
name:
typeof r.name === "string" && r.name ? r.name : String(r.vmid ?? ""),
type,
status: typeof r.status === "string" ? r.status : "unknown",
cpuPercent: toFixedNum(
cpuFraction !== null ? cpuFraction * 100 : null,
0,
),
memPercent: toFixedNum(memPercent, 0),
memUsedGiB: mem !== null ? toFixedNum(bytesToGiB(mem), 2) : null,
memTotalGiB: maxmem !== null ? toFixedNum(bytesToGiB(maxmem), 2) : null,
diskPercent: toFixedNum(diskPercent, 0),
diskUsedGiB:
hasDiskData && disk !== null ? toFixedNum(bytesToGiB(disk), 2) : null,
diskTotalGiB: hasDiskData ? toFixedNum(bytesToGiB(maxdisk), 2) : null,
uptimeSeconds: typeof r.uptime === "number" ? r.uptime : null,
});
}
const running = guests.filter((g) => g.status === "running").length;
const stopped = guests.filter((g) => g.status !== "running").length;
return {
guests,
counts: { running, stopped, total: guests.length },
};
} catch {
return EMPTY_RESULT;
}
}
src/backend/hosts/metrics/proxmox/node-network-collector.ts
+145
View File
@@ -0,0 +1,145 @@
import type { Client } from "ssh2";
import { execCommand } from "../widgets/common-utils.js";
import { isSafeNodeName } from "../../proxmox-shared.js";
export interface ProxmoxNodeNetworkInterface {
name: string;
ip: string | null;
state: string | null;
rxBytes: string | null;
txBytes: string | null;
}
export interface ProxmoxNodeNetworkResult {
interfaces: ProxmoxNodeNetworkInterface[];
}
const EMPTY_RESULT: ProxmoxNodeNetworkResult = { interfaces: [] };
async function collectFromProcNetDev(
client: Client,
): Promise<ProxmoxNodeNetworkResult> {
const interfaces: ProxmoxNodeNetworkInterface[] = [];
try {
const [addrOut, stateOut, procNetOut] = await Promise.all([
execCommand(
client,
"ip -o addr show | awk '{print $2,$4}' | grep -v '^lo'",
),
execCommand(
client,
"ip -o link show | awk '{gsub(/:/, \"\", $2); print $2,$9}'",
),
execCommand(client, "cat /proc/net/dev"),
]);
const ifMap = new Map<
string,
{ ip: string | null; state: string | null }
>();
for (const line of addrOut.stdout
.split("\n")
.map((l) => l.trim())
.filter(Boolean)) {
const parts = line.split(/\s+/);
if (parts.length >= 2) {
const name = parts[0];
const ip = parts[1].split("/")[0];
if (!ifMap.has(name)) ifMap.set(name, { ip, state: null });
}
}
for (const line of stateOut.stdout
.split("\n")
.map((l) => l.trim())
.filter(Boolean)) {
const parts = line.split(/\s+/);
if (parts.length >= 2) {
const existing = ifMap.get(parts[0]);
if (existing) existing.state = parts[1];
}
}
const rxTxMap = new Map<string, { rx: string; tx: string }>();
for (const line of procNetOut.stdout.split("\n").slice(2)) {
const parts = line.trim().split(/\s+/);
if (parts.length >= 10) {
const ifName = parts[0].replace(":", "");
rxTxMap.set(ifName, { rx: parts[1], tx: parts[9] });
}
}
for (const [name, data] of ifMap.entries()) {
const rxTx = rxTxMap.get(name);
interfaces.push({
name,
ip: data.ip,
state: data.state,
rxBytes: rxTx?.rx ?? null,
txBytes: rxTx?.tx ?? null,
});
}
} catch {
return EMPTY_RESULT;
}
return { interfaces };
}
export async function collectProxmoxNodeNetwork(
client: Client,
nodeName: string,
): Promise<ProxmoxNodeNetworkResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
`pvesh get /nodes/${nodeName}/netstat --output-format json`,
15000,
);
if (code !== 0) {
return collectFromProcNetDev(client);
}
const data = JSON.parse(stdout);
if (!Array.isArray(data)) {
return collectFromProcNetDev(client);
}
const interfaces: ProxmoxNodeNetworkInterface[] = data
.filter(
(entry): entry is Record<string, unknown> =>
!!entry && typeof entry === "object",
)
.map((entry) => ({
name:
typeof entry.dev === "string" ? entry.dev : String(entry.dev ?? ""),
ip: null,
state: null,
rxBytes:
typeof entry.in === "number"
? String(entry.in)
: typeof entry.received === "number"
? String(entry.received)
: null,
txBytes:
typeof entry.out === "number"
? String(entry.out)
: typeof entry.transmitted === "number"
? String(entry.transmitted)
: null,
}))
.filter((iface) => iface.name && iface.name !== "lo");
if (interfaces.length === 0) {
return collectFromProcNetDev(client);
}
return { interfaces };
} catch {
return collectFromProcNetDev(client);
}
}
src/backend/hosts/metrics/proxmox/node-status-collector.ts
+142
View File
@@ -0,0 +1,142 @@
import type { Client } from "ssh2";
import { execCommand, toFixedNum } from "../widgets/common-utils.js";
import { isSafeNodeName } from "../../proxmox-shared.js";
export interface ProxmoxNodeStatusResult {
cpu: {
percent: number | null;
cores: number | null;
load: [number, number, number] | null;
};
memory: {
percent: number | null;
usedGiB: number | null;
totalGiB: number | null;
};
disk: {
percent: number | null;
usedGiB: number | null;
totalGiB: number | null;
};
uptime: {
seconds: number | null;
formatted: string | null;
};
system: {
hostname: string | null;
kernel: string | null;
pveVersion: string | null;
};
}
const EMPTY_RESULT: ProxmoxNodeStatusResult = {
cpu: { percent: null, cores: null, load: null },
memory: { percent: null, usedGiB: null, totalGiB: null },
disk: { percent: null, usedGiB: null, totalGiB: null },
uptime: { seconds: null, formatted: null },
system: { hostname: null, kernel: null, pveVersion: null },
};
function bytesToGiB(bytes: number): number {
return bytes / (1024 * 1024 * 1024);
}
function formatUptime(seconds: number): string {
const days = Math.floor(seconds / 86400);
const hours = Math.floor((seconds % 86400) / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
return `${days}d ${hours}h ${minutes}m`;
}
export async function collectProxmoxNodeStatus(
client: Client,
nodeName: string,
): Promise<ProxmoxNodeStatusResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
`pvesh get /nodes/${nodeName}/status --output-format json`,
25000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const data = JSON.parse(stdout) as Record<string, unknown>;
const cpuFraction = typeof data.cpu === "number" ? data.cpu : null;
const cpuinfo = (data.cpuinfo as Record<string, unknown>) || {};
const cores = typeof cpuinfo.cores === "number" ? cpuinfo.cores : null;
const loadavgRaw = data.loadavg;
let load: [number, number, number] | null = null;
if (Array.isArray(loadavgRaw) && loadavgRaw.length >= 3) {
const parsed = loadavgRaw
.slice(0, 3)
.map((v) => Number(v))
.map((v) => (Number.isFinite(v) ? v : 0));
load = parsed as [number, number, number];
}
const memory = (data.memory as Record<string, unknown>) || {};
const memUsed = typeof memory.used === "number" ? memory.used : null;
const memTotal = typeof memory.total === "number" ? memory.total : null;
const memPercent =
memUsed !== null && memTotal !== null && memTotal > 0
? Math.max(0, Math.min(100, (memUsed / memTotal) * 100))
: null;
const rootfs = (data.rootfs as Record<string, unknown>) || {};
const diskUsed = typeof rootfs.used === "number" ? rootfs.used : null;
const diskTotal = typeof rootfs.total === "number" ? rootfs.total : null;
const diskPercent =
diskUsed !== null && diskTotal !== null && diskTotal > 0
? Math.max(0, Math.min(100, (diskUsed / diskTotal) * 100))
: null;
const uptimeSeconds = typeof data.uptime === "number" ? data.uptime : null;
return {
cpu: {
percent: toFixedNum(cpuFraction !== null ? cpuFraction * 100 : null, 0),
cores,
load,
},
memory: {
percent: toFixedNum(memPercent, 0),
usedGiB: memUsed !== null ? toFixedNum(bytesToGiB(memUsed), 2) : null,
totalGiB:
memTotal !== null ? toFixedNum(bytesToGiB(memTotal), 2) : null,
},
disk: {
percent: toFixedNum(diskPercent, 0),
usedGiB: diskUsed !== null ? toFixedNum(bytesToGiB(diskUsed), 2) : null,
totalGiB:
diskTotal !== null ? toFixedNum(bytesToGiB(diskTotal), 2) : null,
},
uptime: {
seconds: uptimeSeconds,
formatted: uptimeSeconds !== null ? formatUptime(uptimeSeconds) : null,
},
system: {
hostname:
typeof data.hostname === "string" && data.hostname
? data.hostname
: null,
kernel:
typeof data.kversion === "string" && data.kversion
? data.kversion
: null,
pveVersion:
typeof data.pveversion === "string" && data.pveversion
? data.pveversion
: null,
},
};
} catch {
return EMPTY_RESULT;
}
}
src/backend/hosts/metrics/proxmox/storage-collector.ts
+76
View File
@@ -0,0 +1,76 @@
import type { Client } from "ssh2";
import { execCommand, toFixedNum } from "../widgets/common-utils.js";
import { isSafeNodeName } from "../../proxmox-shared.js";
export interface ProxmoxStoragePoolEntry {
name: string;
type: string;
active: boolean;
enabled: boolean;
usedGiB: number | null;
totalGiB: number | null;
availGiB: number | null;
percent: number | null;
}
export interface ProxmoxStorageResult {
pools: ProxmoxStoragePoolEntry[];
}
const EMPTY_RESULT: ProxmoxStorageResult = { pools: [] };
function bytesToGiB(bytes: number): number {
return bytes / (1024 * 1024 * 1024);
}
export async function collectProxmoxStorage(
client: Client,
nodeName: string,
): Promise<ProxmoxStorageResult> {
if (!isSafeNodeName(nodeName)) {
return EMPTY_RESULT;
}
try {
const { stdout, code } = await execCommand(
client,
`pvesh get /nodes/${nodeName}/storage --output-format json`,
25000,
);
if (code !== 0) {
return EMPTY_RESULT;
}
const data = JSON.parse(stdout);
if (!Array.isArray(data)) {
return EMPTY_RESULT;
}
const pools: ProxmoxStoragePoolEntry[] = (
data as Array<Record<string, unknown>>
).map((entry) => {
const used = typeof entry.used === "number" ? entry.used : null;
const total = typeof entry.total === "number" ? entry.total : null;
const avail = typeof entry.avail === "number" ? entry.avail : null;
const percent =
used !== null && total !== null && total > 0
? Math.max(0, Math.min(100, (used / total) * 100))
: null;
return {
name: typeof entry.storage === "string" ? entry.storage : "",
type: typeof entry.type === "string" ? entry.type : "unknown",
active: entry.active === 1 || entry.active === true,
enabled: entry.enabled === 1 || entry.enabled === true,
usedGiB: used !== null ? toFixedNum(bytesToGiB(used), 2) : null,
totalGiB: total !== null ? toFixedNum(bytesToGiB(total), 2) : null,
availGiB: avail !== null ? toFixedNum(bytesToGiB(avail), 2) : null,
percent: toFixedNum(percent, 0),
};
});
return { pools };
} catch {
return EMPTY_RESULT;
}
}
src/backend/hosts/metrics/state.ts
+92 -4
View File
@@ -258,7 +258,7 @@ export class ConcurrentLimiter {
private active = 0;
private readonly waiters: Array<() => void> = [];
constructor(private readonly maxConcurrent: number) {
constructor(private maxConcurrent: number) {
if (maxConcurrent < 1) {
throw new Error("maxConcurrent must be >= 1");
}
@@ -272,11 +272,55 @@ export class ConcurrentLimiter {
return this.waiters.length;
}
get limit(): number {
return this.maxConcurrent;
}
/**
* Changes the ceiling at runtime.
*
* Raising it wakes the waiters the new headroom allows, so a queue that built
* up under the old limit drains at once instead of one job at a time as
* running work finishes. Lowering it never interrupts work already running;
* the new limit simply applies from the next release onward.
*/
setLimit(maxConcurrent: number): void {
if (maxConcurrent < 1) {
throw new Error("maxConcurrent must be >= 1");
}
this.maxConcurrent = maxConcurrent;
this.releaseWaiters();
}
/**
* Wakes as many queued callers as there is now room for.
*
* `woken` counts callers that have been resumed but have not yet reached the
* `active += 1` on the far side of their await. Without it the occupancy
* looks lower than it really is for a microtask, and the loop would release
* past the ceiling.
*/
private releaseWaiters(): void {
while (
this.active + this.woken < this.maxConcurrent &&
this.waiters.length > 0
) {
this.woken += 1;
this.waiters.shift()!();
}
}
private woken = 0;
async run<T>(fn: () => Promise<T>): Promise<T> {
if (this.active >= this.maxConcurrent) {
// Queue behind anything already woken, so a caller arriving mid-handoff
// cannot jump the queue and push occupancy past the ceiling.
if (this.active + this.woken >= this.maxConcurrent) {
await new Promise<void>((resolve) => {
this.waiters.push(resolve);
});
// Resumed by a slot handoff; that reservation is now consumed.
this.woken = Math.max(0, this.woken - 1);
}
this.active += 1;
@@ -284,12 +328,42 @@ export class ConcurrentLimiter {
return await fn();
} finally {
this.active -= 1;
const next = this.waiters.shift();
if (next) next();
this.releaseWaiters();
}
}
}
/**
* How many metrics polls may run at once for a given number of polled hosts.
*
* A metrics poll is an SSH exec, so this cannot simply be unbounded — but the
* old fixed ceiling of 5 meant a sweep of 500 hosts took ~40s against a 30s
* interval, so polling fell permanently behind and a host could wait over a
* minute for a "30 second" metric. Scaling with the fleet keeps a sweep inside
* its interval; the cap keeps file descriptors and CPU bounded.
*/
export const METRICS_CONCURRENCY_ENV = "METRICS_POLL_CONCURRENCY";
const MIN_METRICS_CONCURRENCY = 5;
const MAX_METRICS_CONCURRENCY = 50;
/** Aim to spend about a twentieth of the interval per sweep wave. */
const HOSTS_PER_WORKER = 20;
export function metricsConcurrencyFor(
hostCount: number,
env: NodeJS.ProcessEnv = process.env,
): number {
const override = Number(env[METRICS_CONCURRENCY_ENV]);
if (Number.isFinite(override) && override >= 1) {
return Math.min(Math.floor(override), MAX_METRICS_CONCURRENCY);
}
const scaled = Math.ceil(Math.max(0, hostCount) / HOSTS_PER_WORKER);
return Math.min(
Math.max(scaled, MIN_METRICS_CONCURRENCY),
MAX_METRICS_CONCURRENCY,
);
}
/** Short-lived host snapshots for polling — avoids decrypting host rows every tick. */
export class HostPollCache<THost extends { id: number } = { id: number }> {
private cache = new Map<
@@ -330,9 +404,23 @@ export class HostPollCache<THost extends { id: number } = { id: number }> {
export const statusPollLimiter = new ConcurrentLimiter(20);
/** SSH metrics execs are expensive; keep concurrency tight. */
export const metricsPollLimiter = new ConcurrentLimiter(5);
/** Viewer registration is bursty; admit only two first samples at a time. */
export const initialMetricsPollLimiter = new ConcurrentLimiter(2);
export function canStartInitialMetrics(
status: HostStatus | undefined,
hasViewers: boolean,
statusCheckEnabled = true,
): boolean {
return (
hasViewers &&
(!statusCheckEnabled || status === "reachable" || status === "online")
);
}
export const hostPollCache = new HostPollCache(30_000);
export const requestQueue = new RequestQueue();
export const metricsCache = new MetricsCache();
export const authFailureTracker = new AuthFailureTracker();
export const pollingBackoff = new PollingBackoff();
import type { HostStatus } from "./host-status.js";
src/backend/hosts/metrics/viewer-routes.ts
+8 -7
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import type { Express } from "express";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { statsLogger } from "../../utils/logger.js";
@@ -21,6 +22,8 @@ type HostMetricsViewerRoutesDeps<
userId: string,
) => void;
unregisterViewer: (hostId: number, viewerSessionId: string) => void;
/** Route path segment, e.g. "metrics" -> /metrics/heartbeat. Defaults to "metrics". */
pathPrefix?: string;
};
export function registerHostMetricsViewerRoutes<
@@ -35,6 +38,7 @@ export function registerHostMetricsViewerRoutes<
updateHeartbeat,
registerViewer,
unregisterViewer,
pathPrefix = "metrics",
}: HostMetricsViewerRoutesDeps<THost, TStatsConfig>,
): void {
/**
@@ -66,7 +70,7 @@ export function registerHostMetricsViewerRoutes<
* 500:
* description: Failed to update heartbeat.
*/
app.post("/metrics/heartbeat", async (req, res) => {
app.post(`/${pathPrefix}/heartbeat`, async (req, res) => {
const { viewerSessionId } = req.body;
const userId = (req as AuthenticatedRequest).userId;
@@ -125,7 +129,7 @@ export function registerHostMetricsViewerRoutes<
* 500:
* description: Failed to register viewer.
*/
app.post("/metrics/register-viewer", async (req, res) => {
app.post(`/${pathPrefix}/register-viewer`, async (req, res) => {
const { hostId } = req.body;
const userId = (req as AuthenticatedRequest).userId;
@@ -154,10 +158,7 @@ export function registerHostMetricsViewerRoutes<
operation: "register_viewer_lookup",
hostId,
userId,
error:
lookupErr instanceof Error
? lookupErr.message
: String(lookupErr),
error: getErrorMessage(lookupErr, String(lookupErr)),
},
);
}
@@ -255,7 +256,7 @@ export function registerHostMetricsViewerRoutes<
* 500:
* description: Failed to unregister viewer.
*/
app.post("/metrics/unregister-viewer", async (req, res) => {
app.post(`/${pathPrefix}/unregister-viewer`, async (req, res) => {
const { hostId, viewerSessionId } = req.body;
const userId = (req as AuthenticatedRequest).userId;
src/backend/hosts/metrics/widgets/disk-collector.ts
+124 -21
View File
@@ -5,12 +5,14 @@ const PSEUDO_FS_RE = /^(tmpfs|devtmpfs|overlay|udev|none|shm)$/;
export interface DfRow {
filesystem: string;
type: string;
mount: string;
parts: string[];
}
export interface DiskFilesystem {
filesystem: string;
type: string;
mount: string;
percent: number | null;
usedHuman: string | null;
@@ -19,8 +21,22 @@ export interface DiskFilesystem {
usedBytes: number | null;
totalBytes: number | null;
availableBytes: number | null;
label?: string;
}
export interface MonitoredMount {
path: string;
label?: string;
}
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
// Parses `df -T -P`-style output: Filesystem, Type, then the size columns,
// with Mounted-on last. The Type column (e.g. ext4, nfs4, cifs) is what lets
// callers filter network shares out by filesystem type rather than guessing
// from the source path.
export function parseDfLines(output: string): DfRow[] {
return output
.split("\n")
@@ -28,15 +44,18 @@ export function parseDfLines(output: string): DfRow[] {
.filter(Boolean)
.map((line) => {
const parts = line.split(/\s+/);
return { filesystem: parts[0] || "", mount: parts[5] || "", parts };
return {
filesystem: parts[0] || "",
type: parts[1] || "",
mount: parts[6] || "",
parts,
};
})
.filter(
(row) => row.parts.length >= 6 && !PSEUDO_FS_RE.test(row.filesystem),
);
.filter((row) => row.parts.length >= 7 && !PSEUDO_FS_RE.test(row.type));
}
// Finds the index of the most-utilized real filesystem in a `df -B1`-style
// row set (parts[1] = total bytes, parts[2] = used bytes), so a nearly-full
// Finds the index of the most-utilized real filesystem in a `df -T -B1`-style
// row set (parts[2] = total bytes, parts[3] = used bytes), so a nearly-full
// secondary mount (e.g. /data) isn't hidden behind a healthy root filesystem.
export function findWorstMountIndex(bytesRows: DfRow[]): {
index: number;
@@ -48,8 +67,8 @@ export function findWorstMountIndex(bytesRows: DfRow[]): {
let worstTotalBytes = 0;
bytesRows.forEach((row, index) => {
const totalBytes = Number(row.parts[1]);
const usedBytes = Number(row.parts[2]);
const totalBytes = Number(row.parts[2]);
const usedBytes = Number(row.parts[3]);
if (
!Number.isFinite(totalBytes) ||
!Number.isFinite(usedBytes) ||
@@ -74,9 +93,9 @@ export function findWorstMountIndex(bytesRows: DfRow[]): {
};
}
// Merges the `df -B1` and `df -h` row sets into one filesystem list. Byte rows
// drive the maths; human rows only supply the display strings, matched by mount
// point so a mismatched row count can't shift the columns.
// Merges the `df -T -B1` and `df -T -h` row sets into one filesystem list.
// Byte rows drive the maths; human rows only supply the display strings,
// matched by mount point so a mismatched row count can't shift the columns.
export function buildFilesystemList(
bytesRows: DfRow[],
humanRows: DfRow[],
@@ -85,9 +104,9 @@ export function buildFilesystemList(
return bytesRows
.map((row, index) => {
const totalBytes = Number(row.parts[1]);
const usedBytes = Number(row.parts[2]);
const availableBytes = Number(row.parts[3]);
const totalBytes = Number(row.parts[2]);
const usedBytes = Number(row.parts[3]);
const availableBytes = Number(row.parts[4]);
if (!Number.isFinite(totalBytes) || totalBytes <= 0) return null;
const humanRow = aligned
@@ -100,11 +119,12 @@ export function buildFilesystemList(
return {
filesystem: row.filesystem,
type: row.type,
mount: row.mount,
percent: toFixedNum(percent, 0),
usedHuman: humanRow?.parts[2] || null,
totalHuman: humanRow?.parts[1] || null,
availableHuman: humanRow?.parts[3] || null,
usedHuman: humanRow?.parts[3] || null,
totalHuman: humanRow?.parts[2] || null,
availableHuman: humanRow?.parts[4] || null,
usedBytes: Number.isFinite(usedBytes) ? usedBytes : null,
totalBytes,
availableBytes: Number.isFinite(availableBytes) ? availableBytes : null,
@@ -133,7 +153,60 @@ export function selectPrimaryFilesystem(
return best;
}
export async function collectDiskMetrics(client: Client): Promise<{
// Excluded mounts are user-configured per host: an exact mount-path match
// (e.g. "/mnt/nas") or a filesystem-type substring match (e.g. "nfs" matches
// nfs/nfs4, "cifs" matches cifs/smb3), so network shares can be dropped from
// the headline percent and the filesystem list without hiding local disks.
export function filterExcludedFilesystems(
filesystems: DiskFilesystem[],
excludedMounts?: string[] | null,
): DiskFilesystem[] {
if (!excludedMounts || excludedMounts.length === 0) return filesystems;
const normalized = excludedMounts
.map((entry) => entry.trim().toLowerCase())
.filter(Boolean);
if (normalized.length === 0) return filesystems;
return filesystems.filter((fs) => {
const mount = fs.mount.toLowerCase();
const type = fs.type.toLowerCase();
return !normalized.some(
(entry) => mount === entry || (type && type.includes(entry)),
);
});
}
export function mergeMonitoredFilesystems(
filesystems: DiskFilesystem[],
monitored: MonitoredMount[],
customFilesystems: Array<DiskFilesystem | null>,
): DiskFilesystem[] {
const result = [...filesystems];
monitored.forEach((entry, index) => {
const path = entry.path.trim();
const custom = customFilesystems[index];
if (!path || !custom) return;
const existing = result.find((fs) => fs.mount === path);
if (existing) {
existing.label = entry.label?.trim() || undefined;
return;
}
result.push({
...custom,
mount: path,
label: entry.label?.trim() || undefined,
});
});
return result;
}
export async function collectDiskMetrics(
client: Client,
excludedMounts?: string[] | null,
monitoredMounts?: MonitoredMount[] | null,
): Promise<{
percent: number | null;
usedHuman: string | null;
totalHuman: string | null;
@@ -143,13 +216,43 @@ export async function collectDiskMetrics(client: Client): Promise<{
}> {
try {
const [diskOutHuman, diskOutBytes] = await Promise.all([
execCommand(client, "df -h -P | tail -n +2"),
execCommand(client, "df -B1 -P | tail -n +2"),
execCommand(client, "df -hT -P | tail -n +2"),
execCommand(client, "df -TB1 -P | tail -n +2"),
]);
const humanRows = parseDfLines(diskOutHuman.stdout);
const bytesRows = parseDfLines(diskOutBytes.stdout);
const filesystems = buildFilesystemList(bytesRows, humanRows);
let detected = buildFilesystemList(bytesRows, humanRows);
const monitored = (monitoredMounts ?? []).filter((entry) =>
Boolean(entry.path.trim()),
);
if (monitored.length > 0) {
const customFilesystems = await Promise.all(
monitored.map(async (entry) => {
const path = shellQuote(entry.path.trim());
try {
const [customHuman, customBytes] = await Promise.all([
execCommand(client, `df -hT -P -- ${path} | tail -n +2`),
execCommand(client, `df -TB1 -P -- ${path} | tail -n +2`),
]);
return (
buildFilesystemList(
parseDfLines(customBytes.stdout),
parseDfLines(customHuman.stdout),
)[0] ?? null
);
} catch {
return null;
}
}),
);
detected = mergeMonitoredFilesystems(
detected,
monitored,
customFilesystems,
);
}
const filesystems = filterExcludedFilesystems(detected, excludedMounts);
const primary = selectPrimaryFilesystem(filesystems);
return {
src/backend/hosts/metrics/widgets/network-collector.ts
+62 -8
View File
@@ -1,6 +1,45 @@
import type { Client } from "ssh2";
import { execCommand } from "./common-utils.js";
export interface NetworkCounters {
rx: string;
tx: string;
}
export function parseNetworkCounters(
output: string,
): Map<string, NetworkCounters> {
const counters = new Map<string, NetworkCounters>();
for (const line of output.split("\n").slice(2)) {
const parts = line.trim().split(/\s+/);
if (parts.length >= 10) {
counters.set(parts[0].replace(":", ""), {
rx: parts[1],
tx: parts[9],
});
}
}
return counters;
}
export function counterRate(
before: string | undefined,
after: string | undefined,
elapsedSeconds: number,
): number | null {
const first = Number(before);
const second = Number(after);
if (
!Number.isFinite(first) ||
!Number.isFinite(second) ||
second < first ||
elapsedSeconds <= 0
) {
return null;
}
return Math.round((second - first) / elapsedSeconds);
}
export async function collectNetworkMetrics(client: Client): Promise<{
interfaces: Array<{
name: string;
@@ -8,6 +47,8 @@ export async function collectNetworkMetrics(client: Client): Promise<{
state: string;
rxBytes: string | null;
txBytes: string | null;
rxRateBps: number | null;
txRateBps: number | null;
}>;
}> {
const interfaces: Array<{
@@ -16,16 +57,18 @@ export async function collectNetworkMetrics(client: Client): Promise<{
state: string;
rxBytes: string | null;
txBytes: string | null;
rxRateBps: number | null;
txRateBps: number | null;
}> = [];
try {
const ifconfigOut = await execCommand(
client,
"ip -o addr show | awk '{print $2,$4}' | grep -v '^lo'",
"ip -o addr show 2>/dev/null | awk '{print $2,$4}' | grep -v '^lo' || true",
);
const netStatOut = await execCommand(
client,
"ip -o link show | awk '{gsub(/:/, \"\", $2); print $2,$9}'",
"ip -o link show 2>/dev/null | awk '{gsub(/:/, \"\", $2); print $2,$9}' || true",
);
const addrs = ifconfigOut.stdout
@@ -50,32 +93,41 @@ export async function collectNetworkMetrics(client: Client): Promise<{
const parts = line.split(/\s+/);
if (parts.length >= 2) {
const name = parts[0];
if (name === "lo") continue;
const state = parts[1];
const existing = ifMap.get(name);
if (existing) {
existing.state = state;
} else {
ifMap.set(name, { ip: "", state });
}
}
}
try {
const firstReadAt = Date.now();
const procNet = await execCommand(client, "cat /proc/net/dev");
const rxTxMap = new Map<string, { rx: string; tx: string }>();
for (const line of procNet.stdout.split("\n").slice(2)) {
const parts = line.trim().split(/\s+/);
if (parts.length >= 10) {
const ifName = parts[0].replace(":", "");
rxTxMap.set(ifName, { rx: parts[1], tx: parts[9] });
await new Promise((resolve) => setTimeout(resolve, 500));
const procNetAfter = await execCommand(client, "cat /proc/net/dev");
const elapsedSeconds = (Date.now() - firstReadAt) / 1000;
const rxTxMap = parseNetworkCounters(procNet.stdout);
const afterMap = parseNetworkCounters(procNetAfter.stdout);
if (ifMap.size === 0) {
for (const name of rxTxMap.keys()) {
if (name !== "lo") ifMap.set(name, { ip: "", state: "UNKNOWN" });
}
}
for (const [name, data] of ifMap.entries()) {
const rxTx = rxTxMap.get(name);
const after = afterMap.get(name);
interfaces.push({
name,
ip: data.ip,
state: data.state,
rxBytes: rxTx?.rx ?? null,
txBytes: rxTx?.tx ?? null,
rxRateBps: counterRate(rxTx?.rx, after?.rx, elapsedSeconds),
txRateBps: counterRate(rxTx?.tx, after?.tx, elapsedSeconds),
});
}
} catch {
@@ -86,6 +138,8 @@ export async function collectNetworkMetrics(client: Client): Promise<{
state: data.state,
rxBytes: null,
txBytes: null,
rxRateBps: null,
txRateBps: null,
});
}
}
src/backend/hosts/metrics/widgets/processes-collector.ts
+4 -1
View File
@@ -23,7 +23,10 @@ export async function collectProcessesMetrics(client: Client): Promise<{
}> = [];
try {
const psOut = await execCommand(client, "ps aux --sort=-%cpu | head -n 11");
const psOut = await execCommand(
client,
"(ps aux --sort=-%cpu 2>/dev/null || ps aux) | head -n 11",
);
const psLines = psOut.stdout
.split("\n")
.map((l) => l.trim())
src/backend/hosts/opkssh-auth.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../utils/error-message.js";
import { spawn, ChildProcess } from "child_process";
import { randomUUID } from "crypto";
import { WebSocket } from "ws";
@@ -519,7 +520,7 @@ export async function startOPKSSHAuth(
JSON.stringify({
type: "opkssh_error",
requestId,
error: `Failed to start OPKSSH authentication: ${error instanceof Error ? error.message : "Unknown error"}`,
error: `Failed to start OPKSSH authentication: ${getErrorMessage(error)}`,
}),
);
return "";
src/backend/hosts/opkssh-cert-auth.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../utils/error-message.js";
// SSH certificate authentication workarounds for ssh2.
// ssh2 doesn't support OpenSSH cert auth natively — this module grafts
// the certificate onto the parsed key, wraps ECDSA signing to convert
@@ -357,7 +358,7 @@ export async function setupCACertAuth(
const parsed = passphrase ? parseKey(keyBuf, passphrase) : parseKey(keyBuf);
if (parsed instanceof Error || !parsed) {
const errMsg = parsed instanceof Error ? parsed.message : "unknown error";
const errMsg = getErrorMessage(parsed, "unknown error");
throw new Error(`Failed to parse private key for CA cert auth: ${errMsg}`);
}
const privKey = (
src/backend/hosts/proxmox-shared.ts
+7
View File
@@ -0,0 +1,7 @@
// Proxmox node names are restricted to [a-zA-Z0-9-] by PVE itself,
// but we validate defensively before using in a shell command.
const SAFE_NODE_RE = /^[a-zA-Z0-9._-]{1,64}$/;
export function isSafeNodeName(name: string): boolean {
return SAFE_NODE_RE.test(name);
}
src/backend/hosts/serial.ts
+3 -3
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../utils/error-message.js";
import { WebSocketServer, WebSocket, type RawData } from "ws";
import { SerialPort } from "serialport";
import { AuthManager } from "../utils/auth-manager.js";
@@ -109,7 +110,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
} catch (err) {
send({
type: "error",
data: err instanceof Error ? err.message : "Failed to list ports",
data: getErrorMessage(err, "Failed to list ports"),
});
}
break;
@@ -172,8 +173,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
} catch (err) {
send({
type: "error",
data:
err instanceof Error ? err.message : "Failed to open serial port",
data: getErrorMessage(err, "Failed to open serial port"),
});
}
break;
src/backend/hosts/session-sharing/routes.ts
+1 -2
View File
@@ -1,6 +1,5 @@
import crypto from "crypto";
import express from "express";
import type { Request, Response } from "express";
import express, { type Request, type Response } from "express";
import type { AuthenticatedRequest } from "../../../types/index.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { PermissionManager } from "../../utils/permission-manager.js";
src/backend/hosts/ssh-client-factory.ts
+223
View File
@@ -0,0 +1,223 @@
import { getErrorMessage } from "../utils/error-message.js";
import { Client, type ConnectConfig } from "ssh2";
import type { SSHHost } from "../../types/index.js";
import { SSH_ALGORITHMS } from "../utils/ssh-algorithms.js";
import { preparePrivateKeyForSSH2 } from "../utils/ssh-key-utils.js";
import {
createSocks5Connection,
type SOCKS5Config,
} from "../utils/socks5-helper.js";
import { SSHHostKeyVerifier } from "./host-key-verifier.js";
import { createJumpHostChain } from "./jump-host-chain.js";
import { resolveSshConnectConfigHost } from "./ssh-dns.js";
/**
* Non-interactive SSH connection helpers shared by fleet execution.
*
* Mirrors buildSshConfig/createSshFactory/getPoolKey in hosts/metrics/index.ts,
* minus the keyboard-interactive TOTP prompt handling (metrics/terminal-only)
* and OPKSSH/Vault cert setup (not supported as fleet auth for v1 - those flows
* need an interactive browser step that has no place in a batch fleet run).
*/
export function getFleetPoolKey(host: SSHHost): string {
const socks5Key = host.useSocks5
? `:socks5:${host.socks5Host}:${host.socks5Port}`
: "";
return `fleet:${host.userId}:${host.ip}:${host.port}:${host.username}${socks5Key}`;
}
export async function buildFleetSshConfig(
host: SSHHost,
): Promise<ConnectConfig> {
const base: ConnectConfig = {
host: host.ip?.replace(/^\[|\]$/g, "") || host.ip,
port: host.port,
username: host.username,
keepaliveInterval: 30000,
keepaliveCountMax: 3,
readyTimeout: 30000,
tcpKeepAlive: true,
tcpKeepAliveInitialDelay: 30000,
hostVerifier: await SSHHostKeyVerifier.createHostVerifier(
host.id,
host.ip,
host.port,
null,
host.userId || "",
false,
),
env: {
TERM: "xterm-256color",
LANG: "en_US.UTF-8",
LC_ALL: "en_US.UTF-8",
},
algorithms: {
kex: [
"curve25519-sha256",
"curve25519-sha256@libssh.org",
"ecdh-sha2-nistp521",
"ecdh-sha2-nistp384",
"ecdh-sha2-nistp256",
"diffie-hellman-group-exchange-sha256",
"diffie-hellman-group14-sha256",
"diffie-hellman-group14-sha1",
"diffie-hellman-group-exchange-sha1",
"diffie-hellman-group1-sha1",
],
serverHostKey: [
"ssh-ed25519",
"ecdsa-sha2-nistp521",
"ecdsa-sha2-nistp384",
"ecdsa-sha2-nistp256",
"rsa-sha2-512",
"rsa-sha2-256",
"ssh-rsa",
"ssh-dss",
],
cipher: SSH_ALGORITHMS.cipher,
hmac: [
"hmac-sha2-512-etm@openssh.com",
"hmac-sha2-256-etm@openssh.com",
"hmac-sha2-512",
"hmac-sha2-256",
"hmac-sha1",
"hmac-md5",
],
compress: ["none", "zlib@openssh.com", "zlib"],
},
} as ConnectConfig;
const authType = host.authType;
if (authType === "password" || authType === "credential") {
if (!host.password) {
throw new Error(`No password available for host ${host.ip}`);
}
base.password = host.password;
} else if (authType === "key") {
if (!host.key) {
throw new Error(`No SSH key available for host ${host.ip}`);
}
(base as Record<string, unknown>).privateKey = preparePrivateKeyForSSH2(
host.key,
host.keyPassword,
);
if (host.keyPassword) {
(base as Record<string, unknown>).passphrase = host.keyPassword;
}
} else if (authType === "none" || authType === "tailscale") {
// no credentials needed
} else {
throw new Error(
`Unsupported authentication type '${authType}' for fleet execution on host ${host.ip}`,
);
}
return base;
}
export function createFleetSshFactory(host: SSHHost): () => Promise<Client> {
return async () => {
const config = await buildFleetSshConfig(host);
const client = new Client();
const proxyConfig: SOCKS5Config | null =
host.useSocks5 &&
(host.socks5Host ||
(host.socks5ProxyChain && host.socks5ProxyChain.length > 0))
? {
useSocks5: host.useSocks5,
socks5Host: host.socks5Host,
socks5Port: host.socks5Port,
socks5Username: host.socks5Username,
socks5Password: host.socks5Password,
socks5ProxyChain: host.socks5ProxyChain,
}
: null;
const hasJumpHosts =
host.jumpHosts && host.jumpHosts.length > 0 && host.userId;
let jumpClient: Client | null = null;
if (hasJumpHosts) {
jumpClient = await createJumpHostChain(host.jumpHosts!, host.userId!);
if (!jumpClient) {
throw new Error("Failed to establish jump host chain");
}
} else if (proxyConfig) {
try {
const proxySocket = await createSocks5Connection(
host.ip,
host.port,
proxyConfig,
);
if (proxySocket) {
config.sock = proxySocket;
}
} catch (proxyError) {
throw new Error(
"Proxy connection failed: " + getErrorMessage(proxyError),
{ cause: proxyError },
);
}
}
return new Promise<Client>((resolve, reject) => {
const timeout = setTimeout(() => {
client.end();
jumpClient?.end();
reject(new Error("SSH connection timeout"));
}, 30000);
client.on("ready", () => {
clearTimeout(timeout);
resolve(client);
});
client.on("close", () => {
jumpClient?.end();
});
client.on("error", (err) => {
clearTimeout(timeout);
jumpClient?.end();
reject(err);
});
if (jumpClient) {
jumpClient.forwardOut(
"127.0.0.1",
0,
host.ip,
host.port,
(err, stream) => {
if (err) {
clearTimeout(timeout);
jumpClient!.end();
reject(
new Error(
"Failed to forward through jump host: " + err.message,
),
);
return;
}
config.sock = stream;
client.connect(config);
},
);
} else if (config.sock) {
client.connect(config);
} else {
resolveSshConnectConfigHost(config)
.then(() => {
client.connect(config);
})
.catch((error) => {
clearTimeout(timeout);
reject(error);
});
}
});
};
}
src/backend/hosts/ssh-keepalive.test.ts
+25
View File
@@ -0,0 +1,25 @@
import { describe, expect, it } from "vitest";
import { resolveSshKeepalive } from "./ssh-keepalive.js";
describe("resolveSshKeepalive", () => {
it("preserves zero to disable SSH keepalives", () => {
expect(resolveSshKeepalive(0, 0, 30000, 5)).toEqual({
keepaliveInterval: 0,
keepaliveCountMax: 0,
});
});
it("uses defaults when keepalive settings are absent", () => {
expect(resolveSshKeepalive(undefined, undefined, 60000, 5)).toEqual({
keepaliveInterval: 60000,
keepaliveCountMax: 5,
});
});
it("enforces the existing minimums for positive settings", () => {
expect(resolveSshKeepalive(1, 0.5, 30000, 5)).toEqual({
keepaliveInterval: 5000,
keepaliveCountMax: 1,
});
});
});
src/backend/hosts/ssh-keepalive.ts
+23
View File
@@ -0,0 +1,23 @@
const MIN_KEEPALIVE_INTERVAL_MS = 5000;
export function resolveSshKeepalive(
intervalSeconds: number | undefined,
countMax: number | undefined,
defaultIntervalMs: number,
defaultCountMax: number,
) {
return {
keepaliveInterval:
typeof intervalSeconds !== "number"
? defaultIntervalMs
: intervalSeconds === 0
? 0
: Math.max(MIN_KEEPALIVE_INTERVAL_MS, intervalSeconds * 1000),
keepaliveCountMax:
typeof countMax !== "number"
? defaultCountMax
: countMax === 0
? 0
: Math.max(1, countMax),
};
}
src/backend/hosts/terminal/host-identity.ts
+77
View File
@@ -0,0 +1,77 @@
/**
* Comparable form of a host address: bracketed IPv6 literals and hostname
* casing are presentation, not identity.
*/
export function normalizeHostAddress(value: unknown): string {
if (typeof value !== "string") return "";
return value
.replace(/^\[|\]$/g, "")
.trim()
.toLowerCase();
}
/**
* Whether a host id resolved to a different machine than the client meant.
*
* A client addresses a host by the numeric row id of the database it is
* displaying. When the desktop app delegates a connection to a remote sync
* server, the id is resolved against *that* server's table instead, and
* autoincrement ids need not line up between the two — they diverge as soon as
* the sides accumulate inserts and deletes in a different order.
*
* The resolved row then supplies the address, the credentials, the jump hosts
* and the stored host key, so a mismatch opens an interactive shell on a
* machine the user did not pick.
*
* Returns false when the server has no address to compare: the caller falls
* back to what the client supplied, which is the behaviour of every setup that
* never stored the host server-side.
*/
export function hostAddressMismatch(
clientAddress: unknown,
resolvedAddress: unknown,
): boolean {
const resolved = normalizeHostAddress(resolvedAddress);
if (!resolved) return false;
return resolved !== normalizeHostAddress(clientAddress);
}
/**
* Shown to the user on every path that refuses a mismatch, so the wording of
* the one thing they can act on does not depend on which feature they used.
*/
export const HOST_ADDRESS_MISMATCH_MESSAGE =
"Host mismatch: this server resolved the selected host to a different machine, so the connection was refused. " +
"The host ids on this device and on the sync server have drifted apart. " +
'Set the connection origin to "This device" for this host, or re-run a full sync, then try again.';
/**
* Shown when the client named a host by sync identity that this server does
* not have. Distinct from a mismatch: nothing was resolved at all, so the
* remedy is to sync the host across rather than to pick a different origin.
*/
export const HOST_NOT_ON_THIS_SERVER_MESSAGE =
"This host does not exist on the sync server, so the connection was refused. " +
'Run a sync so the server knows about it, or set the connection origin to "This device" for this host.';
/**
* Thrown where a mismatch is reported by rejecting rather than by messaging
* the socket. Callers whose host-resolution is wrapped in a "failed to resolve
* credentials, carry on" catch must let this one through: continuing is the
* behaviour being prevented.
*/
export class HostAddressMismatchError extends Error {
constructor() {
super(HOST_ADDRESS_MISMATCH_MESSAGE);
this.name = "HostAddressMismatchError";
}
}
/** Counterpart of {@link HostAddressMismatchError} for an unknown sync id. */
export class HostNotOnThisServerError extends Error {
constructor() {
super(HOST_NOT_ON_THIS_SERVER_MESSAGE);
this.name = "HostNotOnThisServerError";
}
}
src/backend/hosts/terminal/index.ts
+107 -54
View File
@@ -1,3 +1,5 @@
import { getErrorMessage } from "../../utils/error-message.js";
import { StringDecoder } from "string_decoder";
import { WebSocketServer, WebSocket, type RawData } from "ws";
import ssh2Pkg, {
type Client as SSHClientType,
@@ -45,13 +47,22 @@ import {
import { isWindowsSftpPath, sftpPathToLocalPath } from "../transfer-paths.js";
import { preparePrivateKeyForSSH2 } from "../../utils/ssh-key-utils.js";
import { triggerLoginAlert } from "../../utils/alert-trigger.js";
import { getClientIp } from "../../utils/request-origin.js";
import { isRetriableDnsError, resolveHostForSshConnect } from "../ssh-dns.js";
import { resolveSshKeepalive } from "../ssh-keepalive.js";
import {
hostAddressMismatch,
HOST_ADDRESS_MISMATCH_MESSAGE,
HOST_NOT_ON_THIS_SERVER_MESSAGE,
} from "./host-identity.js";
interface ConnectToHostData {
cols: number;
rows: number;
hostConfig: {
id: number;
/** Names the host across a sync pair; `id` only names it locally. */
syncId?: string | null;
instanceId?: string;
ip: string;
port: number;
@@ -326,7 +337,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
error,
{
operation: "websocket_connection_auth_error",
ip: req.socket.remoteAddress,
ip: getClientIp(req),
},
);
ws.close(1008, "Authentication required");
@@ -502,8 +513,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
connectData.hostConfig.userId = userId;
}
handleConnectToHost(connectData).catch((error) => {
const errMsg =
error instanceof Error ? error.message : "Unknown error";
const errMsg = getErrorMessage(error);
if (
errMsg.includes("Cannot parse privateKey") &&
errMsg.includes("no passphrase")
@@ -956,8 +966,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
};
handleConnectToHost(reconnectData).catch((error) => {
const errMsg =
error instanceof Error ? error.message : "Unknown error";
const errMsg = getErrorMessage(error);
if (
errMsg.includes("Cannot parse privateKey") &&
errMsg.includes("no passphrase")
@@ -1097,7 +1106,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
type: "error",
message:
"Failed to connect after authentication: " +
(error instanceof Error ? error.message : "Unknown error"),
getErrorMessage(error),
}),
);
});
@@ -1143,10 +1152,10 @@ wss.on("connection", async (ws: WebSocket, req) => {
JSON.stringify({
type: "vault_error",
hostId: vaultData.hostId,
error:
error instanceof Error
? error.message
: "Failed to start Vault authentication",
error: getErrorMessage(
error,
"Failed to start Vault authentication",
),
}),
);
}
@@ -1204,7 +1213,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
type: "error",
message:
"Failed to connect after authentication: " +
(error instanceof Error ? error.message : "Unknown error"),
getErrorMessage(error),
}),
);
});
@@ -1322,6 +1331,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
const { hostConfig, initialPath, executeCommand, tmuxAttachSession } = data;
const {
id,
syncId: hostSyncId,
ip: rawIp,
port: clientPort,
username: clientUsername,
@@ -1471,11 +1481,65 @@ wss.on("connection", async (ws: WebSocket, req) => {
if (id && userId) {
try {
const { resolveHostById } = await import("../host-resolver.js");
resolvedHostData = (await resolveHostById(
id,
userId,
)) as unknown as typeof resolvedHostData;
const { resolveHostById, resolveHostBySyncId } =
await import("../host-resolver.js");
// Prefer the sync identity. A numeric id belongs to whichever database
// produced it, so on a sync server it names a different host than the
// desktop app meant; syncId is the same string on both sides.
resolvedHostData = (hostSyncId
? await resolveHostBySyncId(hostSyncId, userId)
: await resolveHostById(id, userId)) as unknown as
typeof resolvedHostData | null;
if (hostSyncId && !resolvedHostData) {
sshLogger.error(
"Refusing to connect: host is not known to this server",
undefined,
{
operation: "ssh_connect_host_sync_id_unknown",
hostId: id,
userId,
},
);
ws.send(
JSON.stringify({
type: "error",
message: HOST_NOT_ON_THIS_SERVER_MESSAGE,
}),
);
cleanupAuthState(connectionTimeout);
return;
}
// Older clients send only the numeric id, which cannot be trusted to
// mean the same host here. Everything below is taken from the row it
// lands on -- the address, the credentials, the jump hosts, the stored
// host key -- so compare the address before using any of it.
if (
!hostSyncId &&
hostAddressMismatch(clientIp, resolvedHostData?.ip)
) {
sshLogger.error(
"Refusing to connect: host id resolves to a different address here",
undefined,
{
operation: "ssh_connect_host_id_mismatch",
hostId: id,
userId,
clientIp,
resolvedIp: resolvedHostData?.ip,
},
);
ws.send(
JSON.stringify({
type: "error",
message: HOST_ADDRESS_MISMATCH_MESSAGE,
}),
);
cleanupAuthState(connectionTimeout);
return;
}
if (resolvedHostData) {
if (
@@ -1522,7 +1586,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
sshLogger.warn(`Failed to resolve server-side host data for ${id}`, {
operation: "ssh_host_data",
hostId: id,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
}
@@ -1563,7 +1627,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
sshLogger.warn(`Failed to resolve host credentials for ${id}`, {
operation: "ssh_credentials",
hostId: id,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
} else if (credentialId && id && userId) {
@@ -1588,7 +1652,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
operation: "ssh_credentials",
hostId: id,
credentialId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
}
@@ -1633,8 +1697,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
);
}
} catch (error) {
const message =
error instanceof Error ? error.message : "Unknown error";
const message = getErrorMessage(error);
sshLogger.error("SSH hostname resolution failed", error, {
operation: "terminal_dns_resolve",
hostId: id,
@@ -1988,10 +2051,17 @@ wss.on("connection", async (ws: WebSocket, req) => {
}
const boundSessionId = currentSessionId;
// A single TCP/SSH packet boundary can split a multi-byte UTF-8
// character (e.g. the box-drawing glyphs mc/htop use for borders).
// Buffer.toString("utf-8") on each chunk independently replaces the
// split bytes with U+FFFD, which shows up as corrupted/inserted
// characters. StringDecoder carries incomplete trailing bytes over
// to the next chunk so multi-byte characters decode correctly.
const decoder = new StringDecoder("utf-8");
stream.on("data", (data: Buffer) => {
try {
const utf8String = data.toString("utf-8");
const utf8String = decoder.write(data);
if (!utf8String) return;
@@ -2184,7 +2254,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
id,
hostConfig.userId,
username,
req.socket.remoteAddress ?? "unknown",
getClientIp(req),
).catch(() => {});
}
@@ -2220,8 +2290,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
operation: "activity_log_error",
userId: hostConfig.userId,
hostId: id,
error:
error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
})();
@@ -2698,6 +2767,12 @@ wss.on("connection", async (ws: WebSocket, req) => {
const hostKeepaliveInterval = hostConfig.terminalConfig?.keepaliveInterval;
const hostKeepaliveCountMax = hostConfig.terminalConfig?.keepaliveCountMax;
const keepalive = resolveSshKeepalive(
hostKeepaliveInterval,
hostKeepaliveCountMax,
30000,
5,
);
// Pre-fetch the stored host key before connect so the verifier callback
// runs synchronously during SSH key exchange, avoiding LoginGraceTime
@@ -2709,14 +2784,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
port,
username,
tryKeyboard: resolvedCredentials.authType !== "tailscale",
keepaliveInterval:
typeof hostKeepaliveInterval === "number"
? Math.max(5000, hostKeepaliveInterval * 1000)
: 30000,
keepaliveCountMax:
typeof hostKeepaliveCountMax === "number"
? Math.max(1, hostKeepaliveCountMax)
: 5,
...keepalive,
readyTimeout:
resolvedCredentials.authType === "tailscale"
? TAILSCALE_CHECK_TIMEOUT_MS
@@ -2828,18 +2896,12 @@ wss.on("connection", async (ws: WebSocket, req) => {
operation: "ca_cert_auth_setup_failed",
userId,
hostId: id,
error:
certError instanceof Error
? certError.message
: String(certError),
error: getErrorMessage(certError, String(certError)),
});
}
}
} catch (keyError) {
const message =
keyError instanceof Error
? keyError.message
: "Invalid private key format";
const message = getErrorMessage(keyError, "Invalid private key format");
sshLogger.error("SSH key format error: " + message);
ws.send(
JSON.stringify({
@@ -2898,10 +2960,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
JSON.stringify({
type: "error",
message:
"OPKSSH authentication failed: " +
(opksshError instanceof Error
? opksshError.message
: "Unknown error"),
"OPKSSH authentication failed: " + getErrorMessage(opksshError),
}),
);
return;
@@ -2953,9 +3012,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
type: "error",
message:
"Vault SSH signer authentication failed: " +
(vaultError instanceof Error
? vaultError.message
: "Unknown error"),
getErrorMessage(vaultError),
}),
);
return;
@@ -3104,7 +3161,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
type: "error",
message:
"Cloudflare tunnel connection failed: " +
(cfError instanceof Error ? cfError.message : "Unknown error"),
getErrorMessage(cfError),
}),
);
cleanupAuthState(connectionTimeout);
@@ -3214,11 +3271,7 @@ wss.on("connection", async (ws: WebSocket, req) => {
ws.send(
JSON.stringify({
type: "error",
message:
"Proxy connection failed: " +
(proxyError instanceof Error
? proxyError.message
: "Unknown error"),
message: "Proxy connection failed: " + getErrorMessage(proxyError),
}),
);
if (currentSessionId) {
src/backend/hosts/terminal/session-manager.ts
+35 -4
View File
@@ -14,6 +14,12 @@ const SESSION_LOGS_DIR = path.join(DATA_DIR, "session_logs");
const DEFAULT_TIMEOUT_MINUTES = 30;
const HEALTH_CHECK_INTERVAL_MS = 60_000;
const MAX_SESSIONS_PER_USER = 10;
// Coalesces recording writes: a chatty SSH stream can emit dozens of "data"
// events per second, and appending to disk on every single one saturates the
// libuv threadpool (default size 4), starving unrelated fs/DNS/crypto work
// and stalling the WS ping/pong health check enough to look like connection
// drops. Batch pending lines and flush on a short trailing edge instead.
const RECORDING_FLUSH_INTERVAL_MS = 300;
export interface SessionParticipant {
ws: WebSocket;
@@ -54,6 +60,8 @@ export interface TerminalSession {
recordingId: number | null;
recordingWriteChain: Promise<void>;
recordingPersistChain: Promise<void>;
pendingRecordingData: string;
recordingFlushTimer: NodeJS.Timeout | null;
tmuxSessionName: string | null;
sessionLoggingEnabled: boolean;
sessionStartedAt: number;
@@ -199,6 +207,8 @@ class TerminalSessionManager {
recordingId: null,
recordingWriteChain: Promise.resolve(),
recordingPersistChain: Promise.resolve(),
pendingRecordingData: "",
recordingFlushTimer: null,
tmuxSessionName: null,
sessionLoggingEnabled,
sessionStartedAt: now,
@@ -599,6 +609,11 @@ class TerminalSessionManager {
private maybePersistLog(session: TerminalSession, force = false): void {
if (!session.sessionLoggingEnabled) return;
if (session.recordingFlushTimer) {
clearTimeout(session.recordingFlushTimer);
session.recordingFlushTimer = null;
this.flushRecording(session);
}
if (session.recordingBytes === 0) return;
if (!force && session.recordingBytes === session.lastPersistedBytes) return;
session.lastPersistedBytes = session.recordingBytes;
@@ -710,21 +725,37 @@ class TerminalSessionManager {
return;
const elapsed = (Date.now() - session.sessionStartedAt) / 1000;
const line = `${JSON.stringify([elapsed, type, data])}\n`;
const firstEvent = session.recordingBytes === 0;
session.recordingBytes += Buffer.byteLength(line);
session.pendingRecordingData += line;
if (!session.recordingFlushTimer) {
session.recordingFlushTimer = setTimeout(() => {
session.recordingFlushTimer = null;
this.flushRecording(session);
}, RECORDING_FLUSH_INTERVAL_MS);
}
}
/** Coalesces buffered recording lines into a single disk write. */
private flushRecording(session: TerminalSession): void {
if (!session.recordingPath || !session.pendingRecordingData) return;
const chunk = session.pendingRecordingData;
session.pendingRecordingData = "";
const firstWrite = session.recordingBytes === Buffer.byteLength(chunk);
session.recordingWriteChain = session.recordingWriteChain.then(async () => {
if (firstEvent) {
if (firstWrite) {
await fs.promises.mkdir(path.dirname(session.recordingPath!), {
recursive: true,
});
await fs.promises.writeFile(
session.recordingPath!,
`${session.recordingHeader}${line}`,
`${session.recordingHeader}${chunk}`,
"utf8",
);
return;
}
await fs.promises.appendFile(session.recordingPath!, line, "utf8");
await fs.promises.appendFile(session.recordingPath!, chunk, "utf8");
});
}
src/backend/hosts/tmux/helper.ts
+2 -17
View File
@@ -22,12 +22,12 @@ const TMUX_PATH_DIRS = [
];
export function withTmuxPath(command: string): string {
const script = `PATH=${TMUX_PATH_DIRS.join(":")}:$PATH; export PATH; ${command}`;
const script = `PATH=${TMUX_PATH_DIRS.join(":")}:"$PATH"; export PATH; ${command}`;
return `/bin/sh -c ${shellEscape(script)}`;
}
export function tmuxCommand(args: string): string {
return withTmuxPath(`tmux ${args}`);
return withTmuxPath(`tmux -u ${args}`);
}
/**
@@ -184,21 +184,6 @@ export function attachOrCreateTmuxSession(
/**
* Query the name of the most recently created tmux session via exec channel.
*/
export async function queryNewestTmuxSession(
conn: Client,
): Promise<string | null> {
try {
const output = await execCommand(
conn,
tmuxCommand(
`list-sessions -F "#{session_created}:#{session_name}" 2>/dev/null | sort -rn | head -1 | cut -d: -f2-`,
),
);
return output || null;
} catch {
return null;
}
}
function shellEscape(s: string): string {
return "'" + s.replace(/'/g, "'\\''") + "'";
src/backend/hosts/tmux/index.ts
+5 -2
View File
@@ -1,7 +1,9 @@
import { getErrorMessage } from "../../utils/error-message.js";
import express from "express";
import cookieParser from "cookie-parser";
import { Client, type ConnectConfig } from "ssh2";
import { createCorsMiddleware } from "../../utils/cors-config.js";
import { createCompressionMiddleware } from "../../utils/compression-config.js";
import { AuthManager } from "../../utils/auth-manager.js";
import { DataCrypto } from "../../utils/data-crypto.js";
import {
@@ -326,6 +328,7 @@ async function collectPaneMetrics(
const app = express();
const authManager = AuthManager.getInstance();
app.use(createCompressionMiddleware());
app.use(createCorsMiddleware(["GET", "POST", "PUT", "DELETE", "OPTIONS"]));
app.use(cookieParser());
app.use(express.json({ limit: "1mb" }));
@@ -389,7 +392,7 @@ async function requireHost(
}
function toErrorMessage(err: unknown): string {
return err instanceof Error ? err.message : "Unknown error";
return getErrorMessage(err);
}
// Destructive tmux actions terminate processes on the remote host, so they
@@ -430,7 +433,7 @@ type TmuxErrorCode =
"TMUX_NOT_INSTALLED" | "TMUX_NO_SERVER" | "HOST_UNREACHABLE" | "TMUX_ERROR";
function classifyTmuxError(err: unknown): TmuxErrorCode {
const msg = err instanceof Error ? err.message : "";
const msg = getErrorMessage(err, "");
if (/command not found|exited with code 127/i.test(msg))
return "TMUX_NOT_INSTALLED";
if (/no server running|lost server/i.test(msg)) return "TMUX_NO_SERVER";
src/backend/hosts/tunnel/index.ts
+2
View File
@@ -2,6 +2,7 @@ import express from "express";
import { createServer } from "http";
import { createCorsMiddleware } from "../../utils/cors-config.js";
import { createCompressionMiddleware } from "../../utils/compression-config.js";
import cookieParser from "cookie-parser";
import { WebSocketServer } from "ws";
@@ -25,6 +26,7 @@ import { initializeAutoStartTunnels } from "./manager.js";
const authManager = AuthManager.getInstance();
const app = express();
app.use(createCompressionMiddleware());
app.use(createCorsMiddleware(["GET", "POST", "PUT", "DELETE", "OPTIONS"]));
app.use(cookieParser());
app.use(express.json());
src/backend/hosts/tunnel/manager.ts
+86 -33
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import { type Response } from "express";
import {
createServer as createTcpServer,
@@ -42,7 +43,9 @@ import {
} from "./utils.js";
import { resolveSshConnectConfigHost } from "../ssh-dns.js";
import { PermissionManager } from "../../utils/permission-manager.js";
import { handleSocks5Connect } from "./socks5-relay.js";
import { notifyAutomationInternalEvent } from "../metrics/automation-bridge.js";
export const activeTunnels = new Map<string, Client>();
export const retryCounters = new Map<string, number>();
@@ -64,7 +67,9 @@ export const lastTunnelErrorTypes = new Map<
export const tunnelConfigs = new Map<string, TunnelConfig>();
export const activeTunnelProcesses = new Map<string, ChildProcess>();
export const pendingTunnelOperations = new Map<string, Promise<void>>();
export const tunnelStatusClients = new Set<Response>();
// SSE clients mapped to the user they belong to, so snapshots can be
// filtered to tunnels that user can actually reach.
export const tunnelStatusClients = new Map<Response, string>();
export const INTERNAL_HOST_API_BASE_URL = "http://localhost:30001/host/db/host";
export const AUTOSTART_FETCH_RETRIES = 6;
@@ -80,7 +85,7 @@ export function describeAxiosError(error: unknown): string {
: error.message;
}
return error instanceof Error ? error.message : "Unknown error";
return getErrorMessage(error);
}
export async function fetchInternalHosts(
@@ -213,18 +218,63 @@ export function getAllTunnelStatus(): Record<string, TunnelStatus> {
return tunnelStatus;
}
// Tunnel names embed host labels and destination endpoints, and error text
// can leak internal hostnames -- so visibility is scoped to tunnels whose
// source host the requesting user can access (owner, share grant, admin),
// mirroring the ownership model of the connect/disconnect routes.
export async function getTunnelStatusForUser(
userId: string,
): Promise<Record<string, TunnelStatus>> {
const permissionManager = PermissionManager.getInstance();
const accessibleHostIds = await permissionManager.filterAccessibleHostIds(
userId,
[...tunnelConfigs.values()]
.map((config) => config.sourceHostId)
.filter((id) => Number.isInteger(id)),
);
const tunnelStatus: Record<string, TunnelStatus> = {};
connectionStatus.forEach((status, name) => {
const sourceHostId = tunnelConfigs.get(name)?.sourceHostId;
if (sourceHostId !== undefined && accessibleHostIds.has(sourceHostId)) {
tunnelStatus[name] = status;
}
});
return tunnelStatus;
}
export async function canAccessTunnel(
userId: string,
tunnelName: string,
): Promise<boolean> {
const sourceHostId = tunnelConfigs.get(tunnelName)?.sourceHostId;
if (sourceHostId === undefined) return false;
const permissionManager = PermissionManager.getInstance();
const access = await permissionManager.canAccessHost(
userId,
sourceHostId,
"connect",
);
return access.hasAccess;
}
export function sendTunnelStatusSnapshot(res: Response): void {
try {
res.write(
`event: statuses\ndata: ${JSON.stringify(getAllTunnelStatus())}\n\n`,
);
} catch {
const userId = tunnelStatusClients.get(res);
if (userId === undefined) {
tunnelStatusClients.delete(res);
return;
}
void getTunnelStatusForUser(userId)
.then((statuses) => {
res.write(`event: statuses\ndata: ${JSON.stringify(statuses)}\n\n`);
})
.catch(() => {
tunnelStatusClients.delete(res);
});
}
export function broadcastTunnelStatusSnapshot(): void {
for (const client of tunnelStatusClients) {
for (const client of tunnelStatusClients.keys()) {
sendTunnelStatusSnapshot(client);
}
}
@@ -405,6 +455,18 @@ export async function handleDisconnect(
return;
}
// Past the manual branch, so this is a drop the user did not ask for.
const ownerUserId =
tunnelConfig?.sourceUserId ?? tunnelConfig?.requestingUserId;
if (ownerUserId) {
notifyAutomationInternalEvent(
"tunnel_disconnected",
ownerUserId,
tunnelConfig?.sourceHostId,
{ tunnelName },
);
}
if (retryExhaustedTunnels.has(tunnelName)) {
broadcastTunnelStatus(tunnelName, {
connected: false,
@@ -492,7 +554,7 @@ export async function handleDisconnect(
activeTunnels.delete(tunnelName);
connectSSHTunnel(tunnelConfig, retryCount).catch((error) => {
tunnelLogger.error(
`Failed to connect tunnel ${tunnelConfig.name}: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to connect tunnel ${tunnelConfig.name}: ${getErrorMessage(error)}`,
);
});
}
@@ -966,7 +1028,7 @@ export async function connectSSHTunnel(
operation: "tunnel_connect",
tunnelName,
sourceHostId: tunnelConfig.sourceHostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
} else if (tunnelConfig.sourceCredentialId && effectiveUserId) {
@@ -993,7 +1055,7 @@ export async function connectSSHTunnel(
operation: "tunnel_connect",
tunnelName,
credentialId: tunnelConfig.sourceCredentialId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
}
@@ -1034,7 +1096,7 @@ export async function connectSSHTunnel(
}
} catch (error) {
tunnelLogger.warn(
`Failed to resolve endpoint credentials for tunnel ${tunnelName}: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to resolve endpoint credentials for tunnel ${tunnelName}: ${getErrorMessage(error)}`,
);
}
} else if (tunnelConfig.endpointCredentialId) {
@@ -1263,8 +1325,7 @@ export async function connectSSHTunnel(
});
setupPingInterval(tunnelName);
} catch (error) {
const message =
error instanceof Error ? error.message : "Failed to create tunnel";
const message = getErrorMessage(error, "Failed to create tunnel");
const errorType = classifyTunnelError(message);
tunnelLogger.error("Failed to create managed tunnel", error, {
operation: "managed_tunnel_create_failed",
@@ -1362,8 +1423,7 @@ export async function connectSSHTunnel(
resolvedSourceCredentials.keyPassword,
);
} catch (error) {
const message =
error instanceof Error ? error.message : "Invalid SSH key format";
const message = getErrorMessage(error, "Invalid SSH key format");
tunnelLogger.error(
`Invalid SSH key format for tunnel '${tunnelName}': ${message}`,
undefined,
@@ -1458,17 +1518,13 @@ export async function connectSSHTunnel(
hasProxyAuth: !!(
tunnelConfig.socks5Username && tunnelConfig.socks5Password
),
errorMessage:
socks5Error instanceof Error ? socks5Error.message : "Unknown error",
errorMessage: getErrorMessage(socks5Error),
});
broadcastTunnelStatus(tunnelName, {
connected: false,
status: CONNECTION_STATES.FAILED,
reason:
"SOCKS5 proxy connection failed: " +
(socks5Error instanceof Error
? socks5Error.message
: "Unknown error"),
"SOCKS5 proxy connection failed: " + getErrorMessage(socks5Error),
});
tunnelConnecting.delete(tunnelName);
return;
@@ -1487,10 +1543,10 @@ export async function connectSSHTunnel(
broadcastTunnelStatus(tunnelName, {
connected: false,
status: CONNECTION_STATES.FAILED,
reason:
error instanceof Error
? error.message
: "Failed to resolve tunnel source hostname",
reason: getErrorMessage(
error,
"Failed to resolve tunnel source hostname",
),
});
tunnelConnecting.delete(tunnelName);
return;
@@ -1546,7 +1602,7 @@ export async function killRemoteTunnelByMarker(
tunnelLogger.warn("Failed to resolve source credentials for cleanup", {
tunnelName,
sourceHostId: tunnelConfig.sourceHostId,
error: error instanceof Error ? error.message : "Unknown error",
error: getErrorMessage(error),
});
}
}
@@ -1673,10 +1729,7 @@ export async function killRemoteTunnelByMarker(
},
);
throw new Error(
"SOCKS5 proxy connection failed: " +
(socks5Error instanceof Error
? socks5Error.message
: "Unknown error"),
"SOCKS5 proxy connection failed: " + getErrorMessage(socks5Error),
{ cause: socks5Error },
);
}
@@ -1891,7 +1944,7 @@ export async function initializeAutoStartTunnels(): Promise<void> {
setTimeout(() => {
connectSSHTunnel(tunnelConfig, 0).catch((error) => {
tunnelLogger.error(
`Failed to connect tunnel ${tunnelConfig.name}: ${error instanceof Error ? error.message : "Unknown error"}`,
`Failed to connect tunnel ${tunnelConfig.name}: ${getErrorMessage(error)}`,
);
});
}, 1000);
@@ -1899,7 +1952,7 @@ export async function initializeAutoStartTunnels(): Promise<void> {
} catch (error) {
tunnelLogger.error(
"Failed to initialize auto-start tunnels:",
error instanceof Error ? error.message : "Unknown error",
getErrorMessage(error),
);
}
}
src/backend/hosts/tunnel/routes.ts
+17 -11
View File
@@ -1,3 +1,4 @@
import { getErrorMessage } from "../../utils/error-message.js";
import express, { type Response } from "express";
import axios from "axios";
@@ -35,7 +36,8 @@ import {
handleDisconnect,
sendTunnelStatusSnapshot,
isSingleHostTunnel,
getAllTunnelStatus,
getTunnelStatusForUser,
canAccessTunnel,
findHostByTunnelEndpoint,
connectSSHTunnel,
} from "./manager.js";
@@ -49,12 +51,12 @@ export function registerTunnelRoutes(app: express.Express): void {
app.get(
"/ssh/tunnel/status",
authenticateJWT,
(req: AuthenticatedRequest, res: Response) => {
async (req: AuthenticatedRequest, res: Response) => {
if (!req.userId) {
return res.status(401).json({ error: "Authentication required" });
}
res.json(getAllTunnelStatus());
res.json(await getTunnelStatusForUser(req.userId));
},
);
@@ -74,7 +76,7 @@ export function registerTunnelRoutes(app: express.Express): void {
});
res.flushHeaders?.();
tunnelStatusClients.add(res);
tunnelStatusClients.set(res, req.userId);
sendTunnelStatusSnapshot(res);
const heartbeat = setInterval(() => {
@@ -117,7 +119,7 @@ export function registerTunnelRoutes(app: express.Express): void {
app.get(
"/ssh/tunnel/status/:tunnelName",
authenticateJWT,
(req: AuthenticatedRequest, res: Response) => {
async (req: AuthenticatedRequest, res: Response) => {
if (!req.userId) {
return res.status(401).json({ error: "Authentication required" });
}
@@ -126,6 +128,13 @@ export function registerTunnelRoutes(app: express.Express): void {
const tunnelName = Array.isArray(tunnelNameParam)
? tunnelNameParam[0]
: tunnelNameParam;
// 404 rather than 403 for foreign tunnels: the name itself carries
// host metadata, so confirming its existence would leak it.
if (!(await canAccessTunnel(req.userId, tunnelName))) {
return res.status(404).json({ error: "Tunnel not found" });
}
const status = connectionStatus.get(tunnelName);
if (!status) {
@@ -336,10 +345,7 @@ export function registerTunnelRoutes(app: express.Express): void {
{
operation: "tunnel_endpoint_credential_resolve",
endpointHostId: endpointHost.id,
error:
credError instanceof Error
? credError.message
: "Unknown",
error: getErrorMessage(credError, "Unknown"),
},
);
}
@@ -356,7 +362,7 @@ export function registerTunnelRoutes(app: express.Express): void {
},
);
throw new Error(
`Failed to resolve endpoint host: ${resolveError instanceof Error ? resolveError.message : "Unknown error"}`,
`Failed to resolve endpoint host: ${getErrorMessage(resolveError)}`,
{ cause: resolveError },
);
}
@@ -399,7 +405,7 @@ export function registerTunnelRoutes(app: express.Express): void {
broadcastTunnelStatus(tunnelName, {
connected: false,
status: CONNECTION_STATES.FAILED,
reason: err instanceof Error ? err.message : "Unknown error",
reason: getErrorMessage(err),
});
tunnelConnecting.delete(tunnelName);
})