fix: user data export freezing the server (#1393)

src/backend/database/database.ts

@@ -687,6 +687,11 @@ app.post("/database/export", authenticateJWT, async (req, res) => {
     });
 
     const exportDb = new Database(tempPath);
+    // A scratch file: without these every row is its own synced commit, and
+    // plugin rows may point at hosts the export leaves out.
+    exportDb.pragma("journal_mode = OFF");
+    exportDb.pragma("synchronous = OFF");
+    exportDb.pragma("foreign_keys = OFF");
 
     try {
       exportDb.exec(`
@@ -990,9 +995,13 @@ app.post("/database/export", authenticateJWT, async (req, res) => {
         userId,
         filename,
       });
+    });
 
+    // "close" also fires when the client aborts mid download, which "end"
+    // does not, so the decrypted export never stays behind on disk.
+    fileStream.on("close", () => {
       fs.unlink(tempPath, (err) => {
-        if (err) {
+        if (err && err.code !== "ENOENT") {
           apiLogger.warn("Failed to clean up export file", {
             operation: "export_cleanup_failed",
             path: tempPath,
@@ -1001,6 +1010,7 @@ app.post("/database/export", authenticateJWT, async (req, res) => {
         }
       });
     });
+    res.on("close", () => fileStream.destroy());
 
     fileStream.pipe(res);
   } catch (error) {

src/backend/plugins/user-data.ts

@@ -66,14 +66,12 @@ export function listUserOwnedTables(): UserOwnedTable[] {
 
 type Row = Record<string, unknown>;
 
-/** The user's rows in every user-owned plugin table, keyed by table. */
+async function readOwnedRows(
-export async function readUserPluginRows(
+  owned: UserOwnedTable,
   userId: string,
-): Promise<Record<string, Row[]>> {
+): Promise<Row[]> {
-  const result: Record<string, Row[]> = {};
-  for (const owned of listUserOwnedTables()) {
   try {
-      const rows = await selectRows<Row>(
+    return await selectRows<Row>(
       sql`SELECT ${sql.join(
         owned.columns.map((column) => sql.identifier(column)),
         sql`, `,
@@ -81,10 +79,24 @@ export async function readUserPluginRows(
         owned.userColumn,
       )} = ${userId}`,
     );
-      if (rows.length > 0) result[owned.table] = rows;
   } catch {
     // The plugin's migration has not created the table yet.
+    return [];
   }
+}
+
+const yieldToEventLoop = () =>
+  new Promise<void>((resolve) => setImmediate(resolve));
+
+/** The user's rows in every user-owned plugin table, keyed by table. */
+export async function readUserPluginRows(
+  userId: string,
+): Promise<Record<string, Row[]>> {
+  const result: Record<string, Row[]> = {};
+  for (const owned of listUserOwnedTables()) {
+    const rows = await readOwnedRows(owned, userId);
+    if (rows.length > 0) result[owned.table] = rows;
+    await yieldToEventLoop();
   }
   return result;
 }
@@ -95,18 +107,25 @@ interface SqliteLike {
     run(...params: unknown[]): unknown;
     all(...params: unknown[]): unknown[];
   };
+  transaction?<T extends (...args: never[]) => unknown>(fn: T): T;
 }
 
-/** Writes the user's plugin rows into a SQLite export file. */
+const INSERT_CHUNK = 2000;
+
+/**
+ * Writes the user's plugin rows into a SQLite export file. One table at a
+ * time, in chunked transactions with a yield between them: history tables
+ * (command history, health checks) can hold many thousands of rows, and the
+ * export runs on the thread that serves every other request.
+ */
 export async function writeUserPluginTables(
   exportDb: SqliteLike,
   userId: string,
 ): Promise<number> {
-  const rowsByTable = await readUserPluginRows(userId);
   let written = 0;
   for (const owned of listUserOwnedTables()) {
-    const rows = rowsByTable[owned.table];
+    const rows = await readOwnedRows(owned, userId);
-    if (!rows?.length) continue;
+    if (!rows.length) continue;
     for (const statement of createTableSql(
       "sqlite",
       owned.pluginId,
@@ -119,9 +138,19 @@ export async function writeUserPluginTables(
         .map((column) => `"${column}"`)
         .join(", ")}) VALUES (${owned.columns.map(() => "?").join(", ")})`,
     );
-    for (const row of rows) {
+    const writeChunk = (chunk: Row[]) => {
+      for (const row of chunk) {
         insert.run(...owned.columns.map((column) => toSqlite(row[column])));
-      written++;
+      }
+    };
+    const runChunk = exportDb.transaction
+      ? exportDb.transaction(writeChunk)
+      : writeChunk;
+    for (let start = 0; start < rows.length; start += INSERT_CHUNK) {
+      const chunk = rows.slice(start, start + INSERT_CHUNK);
+      runChunk(chunk as never);
+      written += chunk.length;
+      await yieldToEventLoop();
     }
   }
   return written;

src/backend/tests/plugins/user-data.test.ts

@@ -117,6 +117,32 @@ describe("plugin user data", () => {
     expect(again).toMatchObject({ imported: 0, skipped: 1 });
   });
 
+  it("writes large tables in chunks and lets the event loop run between them", async () => {
+    const insert = sqlite.prepare(
+      "INSERT INTO p_fixture_bookmark (user_id, path) VALUES ('u1', ?)",
+    );
+    sqlite.transaction(() => {
+      for (let i = 0; i < 4999; i++) insert.run(`/p/${i}`);
+    })();
+    const file = new Database(":memory:");
+    file.exec("CREATE TABLE users (id TEXT PRIMARY KEY)");
+    file.exec("CREATE TABLE ssh_data (id INTEGER PRIMARY KEY)");
+    file.exec("INSERT INTO users (id) VALUES ('u1')");
+    const transaction = vi.spyOn(file, "transaction");
+    let ticks = 0;
+    const ticker = setInterval(() => ticks++, 0);
+
+    const written = await writeUserPluginTables(file, "u1");
+    clearInterval(ticker);
+
+    expect(written).toBe(5000);
+    expect(transaction).toHaveBeenCalled();
+    expect(ticks).toBeGreaterThan(0);
+    expect(
+      file.prepare("SELECT COUNT(*) AS n FROM p_fixture_bookmark").get(),
+    ).toEqual({ n: 5000 });
+  });
+
   it("reads a pre-2.9 export by the legacy table name", async () => {
     const file = new Database(":memory:");
     file.exec(