diff --git a/src/backend/database/database.ts b/src/backend/database/database.ts index f1f98312b..7011ceaf1 100644 --- a/src/backend/database/database.ts +++ b/src/backend/database/database.ts @@ -687,6 +687,11 @@ app.post("/database/export", authenticateJWT, async (req, res) => { }); const exportDb = new Database(tempPath); + // A scratch file: without these every row is its own synced commit, and + // plugin rows may point at hosts the export leaves out. + exportDb.pragma("journal_mode = OFF"); + exportDb.pragma("synchronous = OFF"); + exportDb.pragma("foreign_keys = OFF"); try { exportDb.exec(` @@ -990,9 +995,13 @@ app.post("/database/export", authenticateJWT, async (req, res) => { userId, filename, }); + }); + // "close" also fires when the client aborts mid download, which "end" + // does not, so the decrypted export never stays behind on disk. + fileStream.on("close", () => { fs.unlink(tempPath, (err) => { - if (err) { + if (err && err.code !== "ENOENT") { apiLogger.warn("Failed to clean up export file", { operation: "export_cleanup_failed", path: tempPath, @@ -1001,6 +1010,7 @@ app.post("/database/export", authenticateJWT, async (req, res) => { } }); }); + res.on("close", () => fileStream.destroy()); fileStream.pipe(res); } catch (error) { diff --git a/src/backend/plugins/user-data.ts b/src/backend/plugins/user-data.ts index e08af2db9..cdb9e6e0d 100644 --- a/src/backend/plugins/user-data.ts +++ b/src/backend/plugins/user-data.ts @@ -66,25 +66,37 @@ export function listUserOwnedTables(): UserOwnedTable[] { type Row = Record; +async function readOwnedRows( + owned: UserOwnedTable, + userId: string, +): Promise { + try { + return await selectRows( + sql`SELECT ${sql.join( + owned.columns.map((column) => sql.identifier(column)), + sql`, `, + )} FROM ${sql.identifier(owned.table)} WHERE ${sql.identifier( + owned.userColumn, + )} = ${userId}`, + ); + } catch { + // The plugin's migration has not created the table yet. + return []; + } +} + +const yieldToEventLoop = () => + new Promise((resolve) => setImmediate(resolve)); + /** The user's rows in every user-owned plugin table, keyed by table. */ export async function readUserPluginRows( userId: string, ): Promise> { const result: Record = {}; for (const owned of listUserOwnedTables()) { - try { - const rows = await selectRows( - sql`SELECT ${sql.join( - owned.columns.map((column) => sql.identifier(column)), - sql`, `, - )} FROM ${sql.identifier(owned.table)} WHERE ${sql.identifier( - owned.userColumn, - )} = ${userId}`, - ); - if (rows.length > 0) result[owned.table] = rows; - } catch { - // The plugin's migration has not created the table yet. - } + const rows = await readOwnedRows(owned, userId); + if (rows.length > 0) result[owned.table] = rows; + await yieldToEventLoop(); } return result; } @@ -95,18 +107,25 @@ interface SqliteLike { run(...params: unknown[]): unknown; all(...params: unknown[]): unknown[]; }; + transaction? unknown>(fn: T): T; } -/** Writes the user's plugin rows into a SQLite export file. */ +const INSERT_CHUNK = 2000; + +/** + * Writes the user's plugin rows into a SQLite export file. One table at a + * time, in chunked transactions with a yield between them: history tables + * (command history, health checks) can hold many thousands of rows, and the + * export runs on the thread that serves every other request. + */ export async function writeUserPluginTables( exportDb: SqliteLike, userId: string, ): Promise { - const rowsByTable = await readUserPluginRows(userId); let written = 0; for (const owned of listUserOwnedTables()) { - const rows = rowsByTable[owned.table]; - if (!rows?.length) continue; + const rows = await readOwnedRows(owned, userId); + if (!rows.length) continue; for (const statement of createTableSql( "sqlite", owned.pluginId, @@ -119,9 +138,19 @@ export async function writeUserPluginTables( .map((column) => `"${column}"`) .join(", ")}) VALUES (${owned.columns.map(() => "?").join(", ")})`, ); - for (const row of rows) { - insert.run(...owned.columns.map((column) => toSqlite(row[column]))); - written++; + const writeChunk = (chunk: Row[]) => { + for (const row of chunk) { + insert.run(...owned.columns.map((column) => toSqlite(row[column]))); + } + }; + const runChunk = exportDb.transaction + ? exportDb.transaction(writeChunk) + : writeChunk; + for (let start = 0; start < rows.length; start += INSERT_CHUNK) { + const chunk = rows.slice(start, start + INSERT_CHUNK); + runChunk(chunk as never); + written += chunk.length; + await yieldToEventLoop(); } } return written; diff --git a/src/backend/tests/plugins/user-data.test.ts b/src/backend/tests/plugins/user-data.test.ts index 4270ab164..d93bea668 100644 --- a/src/backend/tests/plugins/user-data.test.ts +++ b/src/backend/tests/plugins/user-data.test.ts @@ -117,6 +117,32 @@ describe("plugin user data", () => { expect(again).toMatchObject({ imported: 0, skipped: 1 }); }); + it("writes large tables in chunks and lets the event loop run between them", async () => { + const insert = sqlite.prepare( + "INSERT INTO p_fixture_bookmark (user_id, path) VALUES ('u1', ?)", + ); + sqlite.transaction(() => { + for (let i = 0; i < 4999; i++) insert.run(`/p/${i}`); + })(); + const file = new Database(":memory:"); + file.exec("CREATE TABLE users (id TEXT PRIMARY KEY)"); + file.exec("CREATE TABLE ssh_data (id INTEGER PRIMARY KEY)"); + file.exec("INSERT INTO users (id) VALUES ('u1')"); + const transaction = vi.spyOn(file, "transaction"); + let ticks = 0; + const ticker = setInterval(() => ticks++, 0); + + const written = await writeUserPluginTables(file, "u1"); + clearInterval(ticker); + + expect(written).toBe(5000); + expect(transaction).toHaveBeenCalled(); + expect(ticks).toBeGreaterThan(0); + expect( + file.prepare("SELECT COUNT(*) AS n FROM p_fixture_bookmark").get(), + ).toEqual({ n: 5000 }); + }); + it("reads a pre-2.9 export by the legacy table name", async () => { const file = new Database(":memory:"); file.exec(