mirror of
https://github.com/Termix-SSH/Termix.git
synced 2026-10-30 23:25:24 +00:00
release-2.7.1 (#1296)
* Add Helm and GitOps deployment setup * fix: build better-sqlite3 from source in Docker (#1267) * fix: preserve runtime SSL settings (#1268) * fix: support forwarding from the memory SSH agent (#1269) * fix: support forwarding from the memory agent * style: format memory agent test * fix: prompt for encrypted SFTP key passphrases (#1270) * fix: prompt for SFTP key passphrases * style: format SSH key utility test * fix: include host context in automation notifications (#1271) * fix: include host context in automation notifications * style: format automation notification changes * fix: reserve sidebar height for host tags (#1272) * fix: keep host action rows stable at large font sizes (#1273) * fix: honor certificate setting during server probe (#1274) * fix: package standard Linux icon sizes (#1275) * fix: avoid duplicate Docker HTTPS listener (#1276) * Fix host status without metrics collection (#1277) * fix: allow eight-digit secure auth codes (#1263) Allow TOTP prompts to accept secure auth codes longer than six digits without blocking valid authentication attempts. Generated with Codebuff 🤖 Co-authored-by: Chetan <chetan.development@gmail.com> Co-authored-by: Codebuff <noreply@codebuff.com> * Harden Helm deployment defaults * Update Helm workflow action * Exclude Helm templates from Prettier * Fix browser RDP file drops (#1279) * Fix Proxmox guest credential usernames (#1280) * Add WSL local terminal option (#1281) * refactor: split the transfer engine into focused modules (#1282) * refactor: extract SFTP promisify helpers into sftp-promisify module * refactor: extract transfer timing and rate stats into transfer-stats module * refactor: extract transfer error classes and recovery checks into transfer-errors module * refactor: extract host/path utility helpers into transfer-host-utils module * refactor: extract SFTP directory tree helpers into transfer-sftp-dir module * refactor: extract segment copy job builder into transfer-segment-copy module * refactor: extract file scan and sample helpers into transfer-scan module * refactor: move throttled progress helper into transfer-stats module * style: format transfer modules * perf: optimize tmux monitor aggregation (#1283) * fix: reserve credential tag row height (#1284) * feat: edit AI provider model settings (#1285) * fix: clarify click-to-expand host setting (#1286) * fix: allow portable imports on remote databases (#1287) * fix: allow HTTPS to share the configured port (#1288) * fix: resolve synced jump hosts on the server (#1289) * fix: make terminal clipboard shortcuts layout independent (#1290) * fix: use compatible fetch dispatcher for Tailscale (#1291) * fix: add OIDC environment recovery override (#1292) * fix: coalesce rapid mobile terminal input (#1293) * fix: coalesce rapid mobile terminal input * fix: support clean xterm patch installs * fix: resolve synced remote desktop host IDs (#1295) * feat: make the SFTP file manager path bar editable (#1294) Co-authored-by: Maxime Bonillo <257463937+dropafterfree@users.noreply.github.com> Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com> * feat: add passkey sign in to the login screen * fix: remove rounded corners from the host list search bar * fix: stop image storage settings text wrapping to one word per line * fix: prevent malformed websocket messages from crashing the server * chore: increment version * fix: remove gaps between host rows in the sidebar list Keep sub-pixel row measurements and stop wiping the size cache on hover. * fix: Failed to connect through jump hosts (#1180) https://github.com/Termix-SSH/Support/issues/1180 * feat: Progress bar for file downloads in the file manager (#1158) https://github.com/Termix-SSH/Support/issues/1158 * feat: Allow setting Silent OIDC Login via ENV var (#1174) https://github.com/Termix-SSH/Support/issues/1174 * feat: `IdentityFile` to limit the number of attempts by agents (#1165) https://github.com/Termix-SSH/Support/issues/1165 * feat: Credentials clone (#1159) https://github.com/Termix-SSH/Support/issues/1159 * chore: update release notes * docs: move helm setup guide to the docs site * fix: type errors in FilteredAgent agent identity handling * fix: remove stale better-sqlite3 prebuilds so the source build is used * fix: actually build better-sqlite3 from source so arm64 docker images work * fix: credential edit pencil in host editor and add clone action to credential list * fix: clear editingHost so the credential pencil actually opens the editor * chore: run format and lint * fix: folder drag and drop upload failing in the file manager * chore: sync Crowdin translations for 2.7.1 --------- Co-authored-by: alex-ctms <alex-ctms@users.noreply.github.com> Co-authored-by: ZacharyZcR <zacharyzcr1984@gmail.com> Co-authored-by: Chetan Kumar <74929596+ckloop@users.noreply.github.com> Co-authored-by: Chetan <chetan.development@gmail.com> Co-authored-by: Codebuff <noreply@codebuff.com> Co-authored-by: ZacharyZcR <payasonorahc@protonmail.com> Co-authored-by: dropafterfree <maxime.bonillo@gmail.com> Co-authored-by: Maxime Bonillo <257463937+dropafterfree@users.noreply.github.com>
This commit is contained in:
165 files changed
+6778
-2090
No files matched your search
@@ -9,7 +9,7 @@ import { createCorsMiddleware } from "../../utils/cors-config.js";
|
||||
import { createCompressionMiddleware } from "../../utils/compression-config.js";
|
||||
import cookieParser from "cookie-parser";
|
||||
import axios from "axios";
|
||||
import { Client as SSHClient } from "ssh2";
|
||||
import ssh2Pkg, { Client as SSHClient } from "ssh2";
|
||||
import { SSH_ALGORITHMS } from "../../utils/ssh-algorithms.js";
|
||||
import { createCurrentHostResolutionRepository } from "../../database/repositories/factory.js";
|
||||
import { fileLogger } from "../../utils/logger.js";
|
||||
@@ -42,8 +42,11 @@ import {
|
||||
} from "./transfer-engine.js";
|
||||
import { registerFileContentRoutes } from "./content-routes.js";
|
||||
import { createConnectionLog } from "../connection-log.js";
|
||||
import { createJumpHostChain } from "../jump-host-chain.js";
|
||||
import { preparePrivateKeyForSSH2 } from "../../utils/ssh-key-utils.js";
|
||||
import { createJumpHostChain, JumpHostChainError } from "../jump-host-chain.js";
|
||||
import {
|
||||
isPrivateKeyPassphraseError,
|
||||
preparePrivateKeyForSSH2,
|
||||
} from "../../utils/ssh-key-utils.js";
|
||||
import {
|
||||
ChannelOpenSerializer,
|
||||
execChannel,
|
||||
@@ -840,7 +843,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
resolvedCredentials = {
|
||||
password: resolvedHost.password,
|
||||
sshKey: resolvedHost.key,
|
||||
keyPassword: resolvedHost.keyPassword,
|
||||
keyPassword: keyPassword || resolvedHost.keyPassword,
|
||||
authType: resolvedHost.authType,
|
||||
sudoPassword: resolvedHost.sudoPassword as string | undefined,
|
||||
certPublicKey: (resolvedHost as { certPublicKey?: string })
|
||||
@@ -909,7 +912,7 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
resolvedCredentials = {
|
||||
password: resolvedHost.password,
|
||||
sshKey: resolvedHost.key,
|
||||
keyPassword: resolvedHost.keyPassword,
|
||||
keyPassword: keyPassword || resolvedHost.keyPassword,
|
||||
authType: resolvedHost.authType,
|
||||
sudoPassword: resolvedHost.sudoPassword as string | undefined,
|
||||
certPublicKey: (resolvedHost as { certPublicKey?: string })
|
||||
@@ -1049,6 +1052,12 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
resolvedCredentials.keyPassword,
|
||||
);
|
||||
|
||||
const parsedKey = ssh2Pkg.utils.parseKey(
|
||||
config.privateKey as Buffer,
|
||||
resolvedCredentials.keyPassword,
|
||||
);
|
||||
if (parsedKey instanceof Error) throw parsedKey;
|
||||
|
||||
if (resolvedCredentials.keyPassword)
|
||||
config.passphrase = resolvedCredentials.keyPassword;
|
||||
|
||||
@@ -1071,6 +1080,10 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
),
|
||||
);
|
||||
} catch (keyError) {
|
||||
if (isPrivateKeyPassphraseError(keyError)) {
|
||||
return res.json({ status: "passphrase_required", connectionLogs });
|
||||
}
|
||||
|
||||
fileLogger.error("SSH key format error for file manager", {
|
||||
operation: "file_connect",
|
||||
sessionId,
|
||||
@@ -1832,7 +1845,10 @@ app.post("/ssh/file_manager/ssh/connect", async (req, res) => {
|
||||
),
|
||||
);
|
||||
return res.status(500).json({
|
||||
error: "Failed to connect through jump hosts",
|
||||
error:
|
||||
error instanceof JumpHostChainError
|
||||
? `Failed to connect through jump hosts: ${error.message}`
|
||||
: "Failed to connect through jump hosts",
|
||||
connectionLogs,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
type SFTPWrapper = import("ssh2").SFTPWrapper;
|
||||
|
||||
export const SFTP_OPEN_READ = 0x00000001;
|
||||
export const SFTP_OPEN_WRITE = 0x00000002 | 0x00000008 | 0x00000010;
|
||||
export const SFTP_OPEN_WRITE_RESUME = 0x00000001 | 0x00000002 | 0x00000008;
|
||||
|
||||
export function promisifySftpStat(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<import("ssh2").Stats> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.stat(path, (err, stats) => {
|
||||
if (err) reject(err);
|
||||
else resolve(stats);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpUnlink(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.unlink(path, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpRmdir(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.rmdir(path, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpMkdir(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
mode: number,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.mkdir(path, { mode }, (err) => {
|
||||
if (err && (err as NodeJS.ErrnoException).code !== "EEXIST") {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpChmod(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
mode: number,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.chmod(path, mode, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpReaddir(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<Array<{ filename: string; attrs: import("ssh2").Stats }>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.readdir(path, (err, list) => {
|
||||
if (err) reject(err);
|
||||
else resolve(list);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpOpen(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
flags: number,
|
||||
mode: number,
|
||||
): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.open(path, flags, mode, (err, handle) => {
|
||||
if (err) reject(err);
|
||||
else resolve(handle);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpClose(
|
||||
sftp: SFTPWrapper,
|
||||
handle: Buffer,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.close(handle, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function promisifySftpFstat(
|
||||
sftp: SFTPWrapper,
|
||||
handle: Buffer,
|
||||
): Promise<import("ssh2").Stats> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.fstat(handle, (err, stats) => {
|
||||
if (err) reject(err);
|
||||
else resolve(stats);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
import { getErrorMessage } from "../../utils/error-message.js";
|
||||
import { randomUUID } from "crypto";
|
||||
import { networkInterfaces } from "os";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import type { ClientChannel } from "ssh2";
|
||||
import { fileLogger } from "../../utils/logger.js";
|
||||
import {
|
||||
basename,
|
||||
buildPathFromSegments,
|
||||
dirname,
|
||||
getWorkingDir,
|
||||
inferPlatformFromPath,
|
||||
@@ -14,7 +12,6 @@ import {
|
||||
normalizeSftpPath,
|
||||
pathsOverlap,
|
||||
sftpPathToLocalPath,
|
||||
splitPathSegments,
|
||||
type TransferPlatform,
|
||||
} from "../transfer-paths.js";
|
||||
import {
|
||||
@@ -26,6 +23,60 @@ import {
|
||||
type TransferScanSummary,
|
||||
} from "./transfer-routing.js";
|
||||
import { verifySftpFileIntegrity } from "./transfer-integrity.js";
|
||||
import {
|
||||
promisifySftpChmod,
|
||||
promisifySftpClose,
|
||||
promisifySftpFstat,
|
||||
promisifySftpOpen,
|
||||
promisifySftpStat,
|
||||
promisifySftpUnlink,
|
||||
} from "./sftp-promisify.js";
|
||||
import {
|
||||
buildTransferHopTimings,
|
||||
computeTransferMbPerSec,
|
||||
createEmptyXferStats,
|
||||
createHopWallClock,
|
||||
createThrottledProgress,
|
||||
elapsedMs,
|
||||
hopSpanMs,
|
||||
mergeXferStats,
|
||||
noteHopEnd,
|
||||
noteHopStart,
|
||||
type PipelinedXferStats,
|
||||
type TransferTimings,
|
||||
} from "./transfer-stats.js";
|
||||
import {
|
||||
TransferCancelledError,
|
||||
TransferStalledError,
|
||||
isRecoverableTransferError,
|
||||
} from "./transfer-errors.js";
|
||||
import {
|
||||
escapeShell,
|
||||
isLocalSshEndpoint,
|
||||
isPermissionError,
|
||||
isRootOnlyPath,
|
||||
} from "./transfer-host-utils.js";
|
||||
import {
|
||||
SFTP_OPEN_READ,
|
||||
SFTP_OPEN_WRITE,
|
||||
SFTP_OPEN_WRITE_RESUME,
|
||||
} from "./sftp-promisify.js";
|
||||
import {
|
||||
collectFileWorkItems,
|
||||
readSftpSample,
|
||||
type FileWorkItem,
|
||||
} from "./transfer-scan.js";
|
||||
import {
|
||||
deletePathSftp,
|
||||
ensureDirectoryTreeSftp,
|
||||
} from "./transfer-sftp-dir.js";
|
||||
import {
|
||||
DEFAULT_PARALLEL_SEGMENT_COUNT,
|
||||
SFTP_XFER_SEGMENT_SIZE,
|
||||
buildSegmentCopyJobs,
|
||||
clampParallelSegmentCount,
|
||||
type SegmentCopyJob,
|
||||
} from "./transfer-segment-copy.js";
|
||||
import {
|
||||
buildDirectProbeCommand,
|
||||
buildDirectRsyncCommand,
|
||||
@@ -107,31 +158,11 @@ export type TransferStatus =
|
||||
"running" | "success" | "partial" | "error" | "cancelled";
|
||||
export type TransferMethod = "stream" | "tar" | "item_sftp" | "direct_rsync";
|
||||
|
||||
export type TransferHopId =
|
||||
"source_read" | "dest_sftp_write" | "dest_local_write";
|
||||
|
||||
export interface TransferHopMetrics {
|
||||
id: TransferHopId;
|
||||
bytes: number;
|
||||
/** Wall-clock span from first I/O on this hop to last I/O complete. */
|
||||
spanMs: number;
|
||||
mbPerSec: number;
|
||||
}
|
||||
|
||||
export interface TransferTimings {
|
||||
prepareDestMs?: number;
|
||||
compressMs?: number;
|
||||
transferMs?: number;
|
||||
extractMs?: number;
|
||||
verifyMs?: number;
|
||||
directBenchmarkMs?: number;
|
||||
relayBenchmarkMs?: number;
|
||||
sourceDeleteMs?: number;
|
||||
totalMs?: number;
|
||||
transferBytes?: number;
|
||||
endToEndMbPerSec?: number;
|
||||
hops?: TransferHopMetrics[];
|
||||
}
|
||||
export type {
|
||||
TransferHopId,
|
||||
TransferHopMetrics,
|
||||
TransferTimings,
|
||||
} from "./transfer-stats.js";
|
||||
|
||||
export interface TransferProgress {
|
||||
transferId: string;
|
||||
@@ -204,34 +235,6 @@ interface ActiveXferControl {
|
||||
const activeXferControls = new Map<string, ActiveXferControl>();
|
||||
const cancelWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
|
||||
class TransferCancelledError extends Error {
|
||||
constructor() {
|
||||
super("Transfer cancelled");
|
||||
this.name = "TransferCancelledError";
|
||||
}
|
||||
}
|
||||
|
||||
class TransferStalledError extends Error {
|
||||
readonly byteOffset?: number;
|
||||
readonly segmentIndex?: number;
|
||||
|
||||
constructor(byteOffset?: number, segmentIndex?: number) {
|
||||
const pos = byteOffset !== undefined ? ` at byte offset ${byteOffset}` : "";
|
||||
const seg = segmentIndex !== undefined ? ` (segment ${segmentIndex})` : "";
|
||||
super(`Transfer stalled — no data moved for 45 seconds${pos}${seg}`);
|
||||
this.name = "TransferStalledError";
|
||||
this.byteOffset = byteOffset;
|
||||
this.segmentIndex = segmentIndex;
|
||||
}
|
||||
}
|
||||
|
||||
class TransferConnectionLostError extends Error {
|
||||
constructor(message = "Transfer SSH connection lost") {
|
||||
super(message);
|
||||
this.name = "TransferConnectionLostError";
|
||||
}
|
||||
}
|
||||
|
||||
function throwIfCancelled(transferId: string): void {
|
||||
if (cancelRequestedTransfers.has(transferId)) {
|
||||
throw new TransferCancelledError();
|
||||
@@ -315,8 +318,6 @@ const SMALL_FILE_SYNC_THRESHOLD = 10 * 1024 * 1024;
|
||||
const SFTP_XFER_CHUNK_SIZE = 256 * 1024;
|
||||
/** Pipelined in-flight READ requests per leg (ssh2 fastGet/fastPut default is 64). */
|
||||
const SFTP_XFER_CONCURRENCY = 32;
|
||||
/** Reset pipelined scheduler every segment to avoid long-run deadlocks at GiB boundaries. */
|
||||
const SFTP_XFER_SEGMENT_SIZE = 256 * 1024 * 1024;
|
||||
/** Files above this size use segmented copy; smaller files use a single scheduler run. */
|
||||
const SFTP_XFER_SEGMENT_THRESHOLD = 32 * 1024 * 1024;
|
||||
/** Per-segment attempts before giving up (sequential and parallel). */
|
||||
@@ -328,17 +329,9 @@ const SFTP_SEQUENTIAL_COPY_MAX_ATTEMPTS = 2;
|
||||
const SFTP_PARALLEL_COPY_MAX_ATTEMPTS = 2;
|
||||
/** Short backoff before opening fresh dedicated SSH sessions. */
|
||||
const TRANSFER_SESSION_RESET_DELAYS_MS = [1000, 2000, 3000];
|
||||
const DEFAULT_PARALLEL_SEGMENT_COUNT = 2;
|
||||
const MAX_PARALLEL_SEGMENT_COUNT = 8;
|
||||
const TRANSFER_HANDLE_CLOSE_TIMEOUT_MS = 2500;
|
||||
const HUNG_TRANSFER_MS = 90_000;
|
||||
const HUNG_RECONNECTING_MS = 180_000;
|
||||
const TRANSFER_PROGRESS_INTERVAL_MS = 200;
|
||||
const SFTP_OPEN_READ = 0x00000001;
|
||||
/** WRITE | CREATE | TRUNCATE — new file or overwrite from start. */
|
||||
const SFTP_OPEN_WRITE = 0x00000002 | 0x00000008 | 0x00000010;
|
||||
/** READ | WRITE | CREATE — resume into an existing partial file without truncating. */
|
||||
const SFTP_OPEN_WRITE_RESUME = 0x00000001 | 0x00000002 | 0x00000008;
|
||||
|
||||
interface TransferReconnectContext {
|
||||
deps: HostTransferDeps;
|
||||
@@ -364,31 +357,6 @@ function buildTransferReconnectContext(
|
||||
return { deps, transferId, ...meta };
|
||||
}
|
||||
|
||||
function isRecoverableTransferConnectionError(err: unknown): boolean {
|
||||
if (!(err instanceof Error)) return false;
|
||||
const msg = err.message.toLowerCase();
|
||||
return (
|
||||
msg.includes("no response from server") ||
|
||||
msg.includes("connection lost") ||
|
||||
msg.includes("not connected") ||
|
||||
msg.includes("econnreset") ||
|
||||
msg.includes("econnrefused") ||
|
||||
msg.includes("etimedout") ||
|
||||
msg.includes("socket hang up") ||
|
||||
msg.includes("protocol error") ||
|
||||
msg.includes("connection closed") ||
|
||||
msg.includes("channel open failure")
|
||||
);
|
||||
}
|
||||
|
||||
function isRecoverableTransferError(err: unknown): boolean {
|
||||
return (
|
||||
err instanceof TransferStalledError ||
|
||||
err instanceof TransferConnectionLostError ||
|
||||
isRecoverableTransferConnectionError(err)
|
||||
);
|
||||
}
|
||||
|
||||
async function probeDestResumeOffset(
|
||||
destSftp: SFTPWrapper,
|
||||
destPath: string,
|
||||
@@ -560,69 +528,6 @@ async function resetDedicatedTransferSessions(
|
||||
return { sourceSession, destSession, sourceSftp, destSftp };
|
||||
}
|
||||
|
||||
let cachedLocalAddresses: Set<string> | null = null;
|
||||
|
||||
function normalizeHostAddress(host: string): string {
|
||||
const trimmed = host.trim().toLowerCase();
|
||||
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||
return trimmed.slice(1, -1);
|
||||
}
|
||||
return trimmed.split(":")[0] ?? trimmed;
|
||||
}
|
||||
|
||||
function getLocalAddresses(): Set<string> {
|
||||
if (cachedLocalAddresses) return cachedLocalAddresses;
|
||||
|
||||
const addresses = new Set(["127.0.0.1", "::1", "localhost"]);
|
||||
for (const ifaces of Object.values(networkInterfaces())) {
|
||||
if (!ifaces) continue;
|
||||
for (const iface of ifaces) {
|
||||
if (!iface.internal && iface.family === "IPv4") {
|
||||
addresses.add(iface.address.toLowerCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
cachedLocalAddresses = addresses;
|
||||
return addresses;
|
||||
}
|
||||
|
||||
export function isLocalSshEndpoint(ip?: string): boolean {
|
||||
if (!ip) return false;
|
||||
const bare = normalizeHostAddress(ip);
|
||||
if (bare === "localhost" || bare === "127.0.0.1" || bare === "::1") {
|
||||
return true;
|
||||
}
|
||||
return getLocalAddresses().has(bare);
|
||||
}
|
||||
|
||||
function createThrottledProgress(onProgress?: (bytes: number) => void) {
|
||||
let pending = 0;
|
||||
let lastFlush = 0;
|
||||
|
||||
const flush = () => {
|
||||
if (pending > 0) {
|
||||
onProgress?.(pending);
|
||||
pending = 0;
|
||||
lastFlush = Date.now();
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
add(bytes: number) {
|
||||
pending += bytes;
|
||||
const now = Date.now();
|
||||
if (now - lastFlush >= TRANSFER_PROGRESS_INTERVAL_MS) {
|
||||
flush();
|
||||
}
|
||||
},
|
||||
flush,
|
||||
};
|
||||
}
|
||||
|
||||
function escapeShell(s: string): string {
|
||||
return s.replace(/'/g, "'\"'\"'");
|
||||
}
|
||||
|
||||
async function detectTransferPlatform(
|
||||
deps: HostTransferDeps,
|
||||
session: SSHSessionLike,
|
||||
@@ -680,150 +585,6 @@ async function detectTransferPlatform(
|
||||
return "unix";
|
||||
}
|
||||
|
||||
function isRootOnlyPath(path: string): boolean {
|
||||
const normalized = normalizeSftpPath(path);
|
||||
return (
|
||||
normalized === "/" ||
|
||||
/^\/[A-Za-z]:$/.test(normalized) ||
|
||||
/^[A-Za-z]:$/.test(normalized)
|
||||
);
|
||||
}
|
||||
|
||||
function isPermissionError(err: Error): boolean {
|
||||
const msg = err.message.toLowerCase();
|
||||
return (
|
||||
msg.includes("permission denied") ||
|
||||
msg.includes("eacces") ||
|
||||
msg.includes("access denied")
|
||||
);
|
||||
}
|
||||
|
||||
function promisifySftpStat(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<import("ssh2").Stats> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.stat(path, (err, stats) => {
|
||||
if (err) reject(err);
|
||||
else resolve(stats);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function promisifySftpUnlink(sftp: SFTPWrapper, path: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.unlink(path, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function promisifySftpRmdir(sftp: SFTPWrapper, path: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.rmdir(path, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function ensureDirectoryTreeSftp(
|
||||
sftp: SFTPWrapper,
|
||||
dirPath: string,
|
||||
created: Set<string> = new Set(),
|
||||
): Promise<void> {
|
||||
const normalized = normalizeSftpPath(dirPath);
|
||||
if (!normalized || isRootOnlyPath(normalized)) return;
|
||||
|
||||
const { root, segments } = splitPathSegments(normalized);
|
||||
if (segments.length === 0) return;
|
||||
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
const current = buildPathFromSegments(root, segments, i + 1);
|
||||
if (created.has(current)) continue;
|
||||
|
||||
try {
|
||||
await promisifySftpMkdir(sftp, current, 0o755);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code !== "EEXIST") {
|
||||
try {
|
||||
const stats = await promisifySftpStat(sftp, current);
|
||||
if (!stats.isDirectory()) throw err;
|
||||
} catch {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
created.add(current);
|
||||
}
|
||||
}
|
||||
|
||||
async function deletePathSftp(sftp: SFTPWrapper, path: string): Promise<void> {
|
||||
let stats: import("ssh2").Stats;
|
||||
try {
|
||||
stats = await promisifySftpStat(sftp, path);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
const entries = await promisifySftpReaddir(sftp, path);
|
||||
for (const entry of entries) {
|
||||
if (entry.filename === "." || entry.filename === "..") continue;
|
||||
await deletePathSftp(sftp, joinPath(path, entry.filename));
|
||||
}
|
||||
await promisifySftpRmdir(sftp, path);
|
||||
return;
|
||||
}
|
||||
|
||||
if (stats.isFile()) {
|
||||
await promisifySftpUnlink(sftp, path);
|
||||
}
|
||||
}
|
||||
|
||||
function promisifySftpMkdir(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
mode: number,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.mkdir(path, { mode }, (err) => {
|
||||
if (err && (err as NodeJS.ErrnoException).code !== "EEXIST") {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function promisifySftpChmod(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
mode: number,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.chmod(path, mode, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function promisifySftpReaddir(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<Array<{ filename: string; attrs: import("ssh2").Stats }>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.readdir(path, (err, list) => {
|
||||
if (err) reject(err);
|
||||
else resolve(list);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function execCommand(
|
||||
deps: HostTransferDeps,
|
||||
session: SSHSessionLike,
|
||||
@@ -1157,10 +918,6 @@ function finalizeTransfer(
|
||||
return result;
|
||||
}
|
||||
|
||||
function elapsedMs(start: number): number {
|
||||
return Date.now() - start;
|
||||
}
|
||||
|
||||
async function verifyTransferredFile(
|
||||
deps: HostTransferDeps,
|
||||
transferId: string,
|
||||
@@ -1208,103 +965,6 @@ async function verifyTransferredFile(
|
||||
});
|
||||
}
|
||||
|
||||
export function computeTransferMbPerSec(
|
||||
bytes: number,
|
||||
ms: number,
|
||||
): number | undefined {
|
||||
if (ms <= 0 || bytes <= 0) return undefined;
|
||||
return ((bytes / ms) * 1000) / (1024 * 1024);
|
||||
}
|
||||
|
||||
interface HopWallClock {
|
||||
firstAt: number | null;
|
||||
lastAt: number | null;
|
||||
}
|
||||
|
||||
function createHopWallClock(): HopWallClock {
|
||||
return { firstAt: null, lastAt: null };
|
||||
}
|
||||
|
||||
function noteHopStart(
|
||||
clock: HopWallClock,
|
||||
t: number = performance.now(),
|
||||
): void {
|
||||
if (clock.firstAt === null) clock.firstAt = t;
|
||||
}
|
||||
|
||||
function noteHopEnd(clock: HopWallClock, t: number = performance.now()): void {
|
||||
clock.lastAt = t;
|
||||
}
|
||||
|
||||
function hopSpanMs(clock: HopWallClock): number {
|
||||
if (clock.firstAt === null || clock.lastAt === null) return 0;
|
||||
return Math.max(0, clock.lastAt - clock.firstAt);
|
||||
}
|
||||
|
||||
interface PipelinedXferStats {
|
||||
bytes: number;
|
||||
sourceReadSpanMs: number;
|
||||
destWriteSpanMs: number;
|
||||
destWriteKind: "sftp" | "local";
|
||||
}
|
||||
|
||||
function createEmptyXferStats(): PipelinedXferStats {
|
||||
return {
|
||||
bytes: 0,
|
||||
sourceReadSpanMs: 0,
|
||||
destWriteSpanMs: 0,
|
||||
destWriteKind: "sftp",
|
||||
};
|
||||
}
|
||||
|
||||
function mergeXferStats(
|
||||
target: PipelinedXferStats,
|
||||
source: PipelinedXferStats,
|
||||
): void {
|
||||
target.bytes += source.bytes;
|
||||
target.sourceReadSpanMs += source.sourceReadSpanMs;
|
||||
target.destWriteSpanMs += source.destWriteSpanMs;
|
||||
target.destWriteKind = source.destWriteKind;
|
||||
}
|
||||
|
||||
function buildTransferHopTimings(
|
||||
stats: PipelinedXferStats,
|
||||
transferMs: number,
|
||||
): Pick<TransferTimings, "transferBytes" | "endToEndMbPerSec" | "hops"> {
|
||||
const hops: TransferHopMetrics[] = [];
|
||||
|
||||
const sourceRate = computeTransferMbPerSec(
|
||||
stats.bytes,
|
||||
stats.sourceReadSpanMs,
|
||||
);
|
||||
if (sourceRate !== undefined) {
|
||||
hops.push({
|
||||
id: "source_read",
|
||||
bytes: stats.bytes,
|
||||
spanMs: stats.sourceReadSpanMs,
|
||||
mbPerSec: sourceRate,
|
||||
});
|
||||
}
|
||||
|
||||
const destHopId: TransferHopId =
|
||||
stats.destWriteKind === "local" ? "dest_local_write" : "dest_sftp_write";
|
||||
const destRate = computeTransferMbPerSec(stats.bytes, stats.destWriteSpanMs);
|
||||
if (destRate !== undefined) {
|
||||
hops.push({
|
||||
id: destHopId,
|
||||
bytes: stats.bytes,
|
||||
spanMs: stats.destWriteSpanMs,
|
||||
mbPerSec: destRate,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
transferBytes: stats.bytes,
|
||||
endToEndMbPerSec: computeTransferMbPerSec(stats.bytes, transferMs),
|
||||
hops,
|
||||
};
|
||||
}
|
||||
|
||||
async function deleteSourcePathsAfterSuccess(
|
||||
deps: HostTransferDeps,
|
||||
transferId: string,
|
||||
@@ -1334,63 +994,6 @@ async function ensureDestParentForFile(
|
||||
await ensureDestDirectory(deps, destSession, parent);
|
||||
}
|
||||
|
||||
interface FileWorkItem {
|
||||
sourcePath: string;
|
||||
destPath: string;
|
||||
mode: number;
|
||||
size: number;
|
||||
}
|
||||
|
||||
async function collectFileWorkItems(
|
||||
sftp: SFTPWrapper,
|
||||
sourcePath: string,
|
||||
destRoot: string,
|
||||
destBaseName?: string,
|
||||
): Promise<FileWorkItem[]> {
|
||||
const stats = await promisifySftpStat(sftp, sourcePath);
|
||||
const name = destBaseName ?? basename(sourcePath);
|
||||
const destPath = joinPath(destRoot, name);
|
||||
|
||||
if (stats.isFile()) {
|
||||
return [
|
||||
{
|
||||
sourcePath,
|
||||
destPath,
|
||||
mode: stats.mode & 0o7777,
|
||||
size: stats.size,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (!stats.isDirectory()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const items: FileWorkItem[] = [];
|
||||
const walk = async (srcDir: string, dstDir: string) => {
|
||||
const entries = await promisifySftpReaddir(sftp, srcDir);
|
||||
for (const entry of entries) {
|
||||
if (entry.filename === "." || entry.filename === "..") continue;
|
||||
const srcChild = joinPath(srcDir, entry.filename);
|
||||
const dstChild = joinPath(dstDir, entry.filename);
|
||||
|
||||
if (entry.attrs.isDirectory()) {
|
||||
await walk(srcChild, dstChild);
|
||||
} else if (entry.attrs.isFile()) {
|
||||
items.push({
|
||||
sourcePath: srcChild,
|
||||
destPath: dstChild,
|
||||
mode: entry.attrs.mode & 0o7777,
|
||||
size: entry.attrs.size,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await walk(sourcePath, destPath);
|
||||
return items;
|
||||
}
|
||||
|
||||
async function scanSourcePathsForRouting(
|
||||
sftp: SFTPWrapper,
|
||||
sourcePaths: string[],
|
||||
@@ -1430,63 +1033,6 @@ async function scanSourcePathsForRouting(
|
||||
return summary;
|
||||
}
|
||||
|
||||
async function readSftpSample(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
fileSize: number,
|
||||
): Promise<Buffer> {
|
||||
const sampleSize = Math.min(64 * 1024, fileSize);
|
||||
const position = Math.max(0, Math.floor((fileSize - sampleSize) / 2));
|
||||
const handle = await promisifySftpOpen(sftp, path, SFTP_OPEN_READ, 0o666);
|
||||
try {
|
||||
const buffer = Buffer.alloc(sampleSize);
|
||||
const bytesRead = await new Promise<number>((resolve, reject) => {
|
||||
sftp.read(handle, buffer, 0, sampleSize, position, (err, count) => {
|
||||
if (err) reject(err);
|
||||
else resolve(count);
|
||||
});
|
||||
});
|
||||
return buffer.subarray(0, bytesRead);
|
||||
} finally {
|
||||
await promisifySftpClose(sftp, handle).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function promisifySftpOpen(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
flags: number,
|
||||
mode: number,
|
||||
): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.open(path, flags, mode, (err, handle) => {
|
||||
if (err) reject(err);
|
||||
else resolve(handle);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function promisifySftpClose(sftp: SFTPWrapper, handle: Buffer): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.close(handle, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function promisifySftpFstat(
|
||||
sftp: SFTPWrapper,
|
||||
handle: Buffer,
|
||||
): Promise<import("ssh2").Stats> {
|
||||
return new Promise((resolve, reject) => {
|
||||
sftp.fstat(handle, (err, stats) => {
|
||||
if (err) reject(err);
|
||||
else resolve(stats);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
interface PipelinedXferOptions {
|
||||
fileSize?: number;
|
||||
initialOffset?: number;
|
||||
@@ -1500,43 +1046,6 @@ interface PipelinedXferOptions {
|
||||
onResumeOffset?: (offset: number) => void;
|
||||
}
|
||||
|
||||
interface SegmentCopyJob {
|
||||
offset: number;
|
||||
length: number;
|
||||
segmentIndex: number;
|
||||
}
|
||||
|
||||
function clampParallelSegmentCount(value?: number): number {
|
||||
const n = value ?? DEFAULT_PARALLEL_SEGMENT_COUNT;
|
||||
return Math.max(1, Math.min(MAX_PARALLEL_SEGMENT_COUNT, Math.floor(n)));
|
||||
}
|
||||
|
||||
function buildSegmentCopyJobs(
|
||||
fileSize: number,
|
||||
initialOffset: number,
|
||||
destResumeSize: number,
|
||||
): SegmentCopyJob[] {
|
||||
const jobs: SegmentCopyJob[] = [];
|
||||
for (
|
||||
let offset = initialOffset;
|
||||
offset < fileSize;
|
||||
offset += SFTP_XFER_SEGMENT_SIZE
|
||||
) {
|
||||
const length = Math.min(SFTP_XFER_SEGMENT_SIZE, fileSize - offset);
|
||||
const segmentIndex = Math.floor(offset / SFTP_XFER_SEGMENT_SIZE);
|
||||
if (destResumeSize >= offset + length) {
|
||||
continue;
|
||||
}
|
||||
const start = destResumeSize > offset ? destResumeSize : offset;
|
||||
jobs.push({
|
||||
offset: start,
|
||||
length: offset + length - start,
|
||||
segmentIndex,
|
||||
});
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
function closeAllTransferSessions(
|
||||
deps: HostTransferDeps,
|
||||
ctx: TransferReconnectContext,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
export class TransferCancelledError extends Error {
|
||||
constructor() {
|
||||
super("Transfer cancelled");
|
||||
this.name = "TransferCancelledError";
|
||||
}
|
||||
}
|
||||
|
||||
export class TransferStalledError extends Error {
|
||||
readonly byteOffset?: number;
|
||||
readonly segmentIndex?: number;
|
||||
|
||||
constructor(byteOffset?: number, segmentIndex?: number) {
|
||||
const pos = byteOffset !== undefined ? ` at byte offset ${byteOffset}` : "";
|
||||
const seg = segmentIndex !== undefined ? ` (segment ${segmentIndex})` : "";
|
||||
super(`Transfer stalled — no data moved for 45 seconds${pos}${seg}`);
|
||||
this.name = "TransferStalledError";
|
||||
this.byteOffset = byteOffset;
|
||||
this.segmentIndex = segmentIndex;
|
||||
}
|
||||
}
|
||||
|
||||
export class TransferConnectionLostError extends Error {
|
||||
constructor(message = "Transfer SSH connection lost") {
|
||||
super(message);
|
||||
this.name = "TransferConnectionLostError";
|
||||
}
|
||||
}
|
||||
|
||||
export function isRecoverableTransferConnectionError(err: unknown): boolean {
|
||||
if (!(err instanceof Error)) return false;
|
||||
const msg = err.message.toLowerCase();
|
||||
return (
|
||||
msg.includes("no response from server") ||
|
||||
msg.includes("connection lost") ||
|
||||
msg.includes("not connected") ||
|
||||
msg.includes("econnreset") ||
|
||||
msg.includes("econnrefused") ||
|
||||
msg.includes("etimedout") ||
|
||||
msg.includes("socket hang up") ||
|
||||
msg.includes("protocol error") ||
|
||||
msg.includes("connection closed") ||
|
||||
msg.includes("channel open failure")
|
||||
);
|
||||
}
|
||||
|
||||
export function isRecoverableTransferError(err: unknown): boolean {
|
||||
return (
|
||||
err instanceof TransferStalledError ||
|
||||
err instanceof TransferConnectionLostError ||
|
||||
isRecoverableTransferConnectionError(err)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { networkInterfaces } from "os";
|
||||
import { normalizeSftpPath } from "../transfer-paths.js";
|
||||
|
||||
let cachedLocalAddresses: Set<string> | null = null;
|
||||
|
||||
export function normalizeHostAddress(host: string): string {
|
||||
const trimmed = host.trim().toLowerCase();
|
||||
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||
return trimmed.slice(1, -1);
|
||||
}
|
||||
return trimmed.split(":")[0] ?? trimmed;
|
||||
}
|
||||
|
||||
function getLocalAddresses(): Set<string> {
|
||||
if (cachedLocalAddresses) return cachedLocalAddresses;
|
||||
|
||||
const addresses = new Set(["127.0.0.1", "::1", "localhost"]);
|
||||
for (const ifaces of Object.values(networkInterfaces())) {
|
||||
if (!ifaces) continue;
|
||||
for (const iface of ifaces) {
|
||||
if (!iface.internal && iface.family === "IPv4") {
|
||||
addresses.add(iface.address.toLowerCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
cachedLocalAddresses = addresses;
|
||||
return addresses;
|
||||
}
|
||||
|
||||
export function isLocalSshEndpoint(ip?: string): boolean {
|
||||
if (!ip) return false;
|
||||
const bare = normalizeHostAddress(ip);
|
||||
if (bare === "localhost" || bare === "127.0.0.1" || bare === "::1") {
|
||||
return true;
|
||||
}
|
||||
return getLocalAddresses().has(bare);
|
||||
}
|
||||
|
||||
export function escapeShell(s: string): string {
|
||||
return s.replace(/'/g, "'\"'\"'");
|
||||
}
|
||||
|
||||
export function isRootOnlyPath(path: string): boolean {
|
||||
const normalized = normalizeSftpPath(path);
|
||||
return (
|
||||
normalized === "/" ||
|
||||
/^\/[A-Za-z]:$/.test(normalized) ||
|
||||
/^[A-Za-z]:$/.test(normalized)
|
||||
);
|
||||
}
|
||||
|
||||
export function isPermissionError(err: Error): boolean {
|
||||
const msg = err.message.toLowerCase();
|
||||
return (
|
||||
msg.includes("permission denied") ||
|
||||
msg.includes("eacces") ||
|
||||
msg.includes("access denied")
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { basename, joinPath } from "../transfer-paths.js";
|
||||
import {
|
||||
SFTP_OPEN_READ,
|
||||
promisifySftpClose,
|
||||
promisifySftpOpen,
|
||||
promisifySftpReaddir,
|
||||
promisifySftpStat,
|
||||
} from "./sftp-promisify.js";
|
||||
|
||||
type SFTPWrapper = import("ssh2").SFTPWrapper;
|
||||
|
||||
export interface FileWorkItem {
|
||||
sourcePath: string;
|
||||
destPath: string;
|
||||
mode: number;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export async function collectFileWorkItems(
|
||||
sftp: SFTPWrapper,
|
||||
sourcePath: string,
|
||||
destRoot: string,
|
||||
destBaseName?: string,
|
||||
): Promise<FileWorkItem[]> {
|
||||
const stats = await promisifySftpStat(sftp, sourcePath);
|
||||
const name = destBaseName ?? basename(sourcePath);
|
||||
const destPath = joinPath(destRoot, name);
|
||||
|
||||
if (stats.isFile()) {
|
||||
return [
|
||||
{
|
||||
sourcePath,
|
||||
destPath,
|
||||
mode: stats.mode & 0o7777,
|
||||
size: stats.size,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (!stats.isDirectory()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const items: FileWorkItem[] = [];
|
||||
const walk = async (srcDir: string, dstDir: string) => {
|
||||
const entries = await promisifySftpReaddir(sftp, srcDir);
|
||||
for (const entry of entries) {
|
||||
if (entry.filename === "." || entry.filename === "..") continue;
|
||||
const srcChild = joinPath(srcDir, entry.filename);
|
||||
const dstChild = joinPath(dstDir, entry.filename);
|
||||
|
||||
if (entry.attrs.isDirectory()) {
|
||||
await walk(srcChild, dstChild);
|
||||
} else if (entry.attrs.isFile()) {
|
||||
items.push({
|
||||
sourcePath: srcChild,
|
||||
destPath: dstChild,
|
||||
mode: entry.attrs.mode & 0o7777,
|
||||
size: entry.attrs.size,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await walk(sourcePath, destPath);
|
||||
return items;
|
||||
}
|
||||
|
||||
export async function readSftpSample(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
fileSize: number,
|
||||
): Promise<Buffer> {
|
||||
const sampleSize = Math.min(64 * 1024, fileSize);
|
||||
const position = Math.max(0, Math.floor((fileSize - sampleSize) / 2));
|
||||
const handle = await promisifySftpOpen(sftp, path, SFTP_OPEN_READ, 0o666);
|
||||
try {
|
||||
const buffer = Buffer.alloc(sampleSize);
|
||||
const bytesRead = await new Promise<number>((resolve, reject) => {
|
||||
sftp.read(handle, buffer, 0, sampleSize, position, (err, count) => {
|
||||
if (err) reject(err);
|
||||
else resolve(count);
|
||||
});
|
||||
});
|
||||
return buffer.subarray(0, bytesRead);
|
||||
} finally {
|
||||
await promisifySftpClose(sftp, handle).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
export const SFTP_XFER_SEGMENT_SIZE = 256 * 1024 * 1024;
|
||||
export const DEFAULT_PARALLEL_SEGMENT_COUNT = 2;
|
||||
export const MAX_PARALLEL_SEGMENT_COUNT = 8;
|
||||
|
||||
export interface SegmentCopyJob {
|
||||
offset: number;
|
||||
length: number;
|
||||
segmentIndex: number;
|
||||
}
|
||||
|
||||
export function clampParallelSegmentCount(value?: number): number {
|
||||
const n = value ?? DEFAULT_PARALLEL_SEGMENT_COUNT;
|
||||
return Math.max(1, Math.min(MAX_PARALLEL_SEGMENT_COUNT, Math.floor(n)));
|
||||
}
|
||||
|
||||
export function buildSegmentCopyJobs(
|
||||
fileSize: number,
|
||||
initialOffset: number,
|
||||
destResumeSize: number,
|
||||
): SegmentCopyJob[] {
|
||||
const jobs: SegmentCopyJob[] = [];
|
||||
for (
|
||||
let offset = initialOffset;
|
||||
offset < fileSize;
|
||||
offset += SFTP_XFER_SEGMENT_SIZE
|
||||
) {
|
||||
const length = Math.min(SFTP_XFER_SEGMENT_SIZE, fileSize - offset);
|
||||
const segmentIndex = Math.floor(offset / SFTP_XFER_SEGMENT_SIZE);
|
||||
if (destResumeSize >= offset + length) {
|
||||
continue;
|
||||
}
|
||||
const start = destResumeSize > offset ? destResumeSize : offset;
|
||||
jobs.push({
|
||||
offset: start,
|
||||
length: offset + length - start,
|
||||
segmentIndex,
|
||||
});
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import {
|
||||
buildPathFromSegments,
|
||||
joinPath,
|
||||
normalizeSftpPath,
|
||||
splitPathSegments,
|
||||
} from "../transfer-paths.js";
|
||||
import { isRootOnlyPath } from "./transfer-host-utils.js";
|
||||
import {
|
||||
promisifySftpMkdir,
|
||||
promisifySftpReaddir,
|
||||
promisifySftpRmdir,
|
||||
promisifySftpStat,
|
||||
promisifySftpUnlink,
|
||||
} from "./sftp-promisify.js";
|
||||
|
||||
type SFTPWrapper = import("ssh2").SFTPWrapper;
|
||||
|
||||
export async function ensureDirectoryTreeSftp(
|
||||
sftp: SFTPWrapper,
|
||||
dirPath: string,
|
||||
created: Set<string> = new Set(),
|
||||
): Promise<void> {
|
||||
const normalized = normalizeSftpPath(dirPath);
|
||||
if (!normalized || isRootOnlyPath(normalized)) return;
|
||||
|
||||
const { root, segments } = splitPathSegments(normalized);
|
||||
if (segments.length === 0) return;
|
||||
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
const current = buildPathFromSegments(root, segments, i + 1);
|
||||
if (created.has(current)) continue;
|
||||
|
||||
try {
|
||||
await promisifySftpMkdir(sftp, current, 0o755);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code !== "EEXIST") {
|
||||
try {
|
||||
const stats = await promisifySftpStat(sftp, current);
|
||||
if (!stats.isDirectory()) throw err;
|
||||
} catch {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
created.add(current);
|
||||
}
|
||||
}
|
||||
|
||||
export async function deletePathSftp(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
): Promise<void> {
|
||||
let stats: import("ssh2").Stats;
|
||||
try {
|
||||
stats = await promisifySftpStat(sftp, path);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
const entries = await promisifySftpReaddir(sftp, path);
|
||||
for (const entry of entries) {
|
||||
if (entry.filename === "." || entry.filename === "..") continue;
|
||||
await deletePathSftp(sftp, joinPath(path, entry.filename));
|
||||
}
|
||||
await promisifySftpRmdir(sftp, path);
|
||||
return;
|
||||
}
|
||||
|
||||
if (stats.isFile()) {
|
||||
await promisifySftpUnlink(sftp, path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { performance } from "node:perf_hooks";
|
||||
|
||||
const TRANSFER_PROGRESS_INTERVAL_MS = 200;
|
||||
|
||||
export type TransferHopId =
|
||||
"source_read" | "dest_sftp_write" | "dest_local_write";
|
||||
|
||||
export interface TransferHopMetrics {
|
||||
id: TransferHopId;
|
||||
bytes: number;
|
||||
/** Wall-clock span from first I/O on this hop to last I/O complete. */
|
||||
spanMs: number;
|
||||
mbPerSec: number;
|
||||
}
|
||||
|
||||
export interface TransferTimings {
|
||||
prepareDestMs?: number;
|
||||
compressMs?: number;
|
||||
transferMs?: number;
|
||||
extractMs?: number;
|
||||
verifyMs?: number;
|
||||
directBenchmarkMs?: number;
|
||||
relayBenchmarkMs?: number;
|
||||
sourceDeleteMs?: number;
|
||||
totalMs?: number;
|
||||
transferBytes?: number;
|
||||
endToEndMbPerSec?: number;
|
||||
hops?: TransferHopMetrics[];
|
||||
}
|
||||
|
||||
export function elapsedMs(start: number): number {
|
||||
return Date.now() - start;
|
||||
}
|
||||
|
||||
export function computeTransferMbPerSec(
|
||||
bytes: number,
|
||||
ms: number,
|
||||
): number | undefined {
|
||||
if (ms <= 0 || bytes <= 0) return undefined;
|
||||
return ((bytes / ms) * 1000) / (1024 * 1024);
|
||||
}
|
||||
|
||||
export interface HopWallClock {
|
||||
firstAt: number | null;
|
||||
lastAt: number | null;
|
||||
}
|
||||
|
||||
export function createHopWallClock(): HopWallClock {
|
||||
return { firstAt: null, lastAt: null };
|
||||
}
|
||||
|
||||
export function noteHopStart(
|
||||
clock: HopWallClock,
|
||||
t: number = performance.now(),
|
||||
): void {
|
||||
if (clock.firstAt === null) clock.firstAt = t;
|
||||
}
|
||||
|
||||
export function noteHopEnd(
|
||||
clock: HopWallClock,
|
||||
t: number = performance.now(),
|
||||
): void {
|
||||
clock.lastAt = t;
|
||||
}
|
||||
|
||||
export function hopSpanMs(clock: HopWallClock): number {
|
||||
if (clock.firstAt === null || clock.lastAt === null) return 0;
|
||||
return Math.max(0, clock.lastAt - clock.firstAt);
|
||||
}
|
||||
|
||||
export function createThrottledProgress(onProgress?: (bytes: number) => void) {
|
||||
let pending = 0;
|
||||
let lastFlush = 0;
|
||||
|
||||
const flush = () => {
|
||||
if (pending > 0) {
|
||||
onProgress?.(pending);
|
||||
pending = 0;
|
||||
lastFlush = Date.now();
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
add(bytes: number) {
|
||||
pending += bytes;
|
||||
const now = Date.now();
|
||||
if (now - lastFlush >= TRANSFER_PROGRESS_INTERVAL_MS) {
|
||||
flush();
|
||||
}
|
||||
},
|
||||
flush,
|
||||
};
|
||||
}
|
||||
|
||||
export interface PipelinedXferStats {
|
||||
bytes: number;
|
||||
sourceReadSpanMs: number;
|
||||
destWriteSpanMs: number;
|
||||
destWriteKind: "sftp" | "local";
|
||||
}
|
||||
|
||||
export function createEmptyXferStats(): PipelinedXferStats {
|
||||
return {
|
||||
bytes: 0,
|
||||
sourceReadSpanMs: 0,
|
||||
destWriteSpanMs: 0,
|
||||
destWriteKind: "sftp",
|
||||
};
|
||||
}
|
||||
|
||||
export function mergeXferStats(
|
||||
target: PipelinedXferStats,
|
||||
source: PipelinedXferStats,
|
||||
): void {
|
||||
target.bytes += source.bytes;
|
||||
target.sourceReadSpanMs += source.sourceReadSpanMs;
|
||||
target.destWriteSpanMs += source.destWriteSpanMs;
|
||||
target.destWriteKind = source.destWriteKind;
|
||||
}
|
||||
|
||||
export function buildTransferHopTimings(
|
||||
stats: PipelinedXferStats,
|
||||
transferMs: number,
|
||||
): Pick<TransferTimings, "transferBytes" | "endToEndMbPerSec" | "hops"> {
|
||||
const hops: TransferHopMetrics[] = [];
|
||||
|
||||
const sourceRate = computeTransferMbPerSec(
|
||||
stats.bytes,
|
||||
stats.sourceReadSpanMs,
|
||||
);
|
||||
if (sourceRate !== undefined) {
|
||||
hops.push({
|
||||
id: "source_read",
|
||||
bytes: stats.bytes,
|
||||
spanMs: stats.sourceReadSpanMs,
|
||||
mbPerSec: sourceRate,
|
||||
});
|
||||
}
|
||||
|
||||
const destHopId: TransferHopId =
|
||||
stats.destWriteKind === "local" ? "dest_local_write" : "dest_sftp_write";
|
||||
const destRate = computeTransferMbPerSec(stats.bytes, stats.destWriteSpanMs);
|
||||
if (destRate !== undefined) {
|
||||
hops.push({
|
||||
id: destHopId,
|
||||
bytes: stats.bytes,
|
||||
spanMs: stats.destWriteSpanMs,
|
||||
mbPerSec: destRate,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
transferBytes: stats.bytes,
|
||||
endToEndMbPerSec: computeTransferMbPerSec(stats.bytes, transferMs),
|
||||
hops,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user