fix(export): support paginated chunked database dumps with breathing intervals (#59)

src/export/dump.test.ts

@@ -128,6 +128,45 @@ describe('Database Dump Module', () => {
         )
     })
 
+    it('should paginate large tables across multiple batches and skip internal sqlite_ tables', async () => {
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([
+                { name: 'sqlite_sequence' },
+                { name: 'events' },
+            ])
+            .mockResolvedValueOnce([
+                { sql: 'CREATE TABLE events (id INTEGER, payload TEXT)' },
+            ])
+            .mockResolvedValueOnce([
+                { id: 1, payload: 'row-1' },
+                { id: 2, payload: 'row-2' },
+            ])
+            .mockResolvedValueOnce([{ id: 3, payload: null }])
+
+        const response = await dumpDatabaseRoute(mockDataSource, mockConfig, {
+            batchSize: 2,
+            workIntervalMs: 0,
+            breathingIntervalMs: 1,
+        })
+
+        expect(response.status).toBe(200)
+        const dumpText = await response.text()
+        expect(dumpText).not.toContain('sqlite_sequence')
+        expect(dumpText).toContain("INSERT INTO events VALUES (1, 'row-1');")
+        expect(dumpText).toContain("INSERT INTO events VALUES (2, 'row-2');")
+        expect(dumpText).toContain('INSERT INTO events VALUES (3, NULL);')
+        expect(executeOperation).toHaveBeenCalledWith(
+            [{ sql: 'SELECT * FROM events LIMIT 2 OFFSET 0;' }],
+            mockDataSource,
+            mockConfig
+        )
+        expect(executeOperation).toHaveBeenCalledWith(
+            [{ sql: 'SELECT * FROM events LIMIT 2 OFFSET 2;' }],
+            mockDataSource,
+            mockConfig
+        )
+    })
+
     it('should return a 500 response when an error occurs', async () => {
         const consoleErrorMock = vi
             .spyOn(console, 'error')

src/export/dump.ts

@@ -3,60 +3,125 @@ import { StarbaseDBConfiguration } from '../handler'
 import { DataSource } from '../types'
 import { createResponse } from '../utils'
 
+export interface DumpDatabaseOptions {
+    batchSize?: number
+    workIntervalMs?: number
+    breathingIntervalMs?: number
+}
+
+const DEFAULT_BATCH_SIZE = 1000
+const DEFAULT_WORK_INTERVAL_MS = 5000
+const DEFAULT_BREATHING_INTERVAL_MS = 100
+
+function formatSqlValue(value: unknown): string {
+    if (value === null || value === undefined) {
+        return 'NULL'
+    }
+    if (typeof value === 'string') {
+        return `'${value.replace(/'/g, "''")}'`
+    }
+    if (typeof value === 'boolean') {
+        return value ? '1' : '0'
+    }
+    return String(value)
+}
+
 export async function dumpDatabaseRoute(
     dataSource: DataSource,
-    config: StarbaseDBConfiguration
+    config: StarbaseDBConfiguration,
+    options: DumpDatabaseOptions = {}
 ): Promise<Response> {
+    const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE
+    const workIntervalMs = options.workIntervalMs ?? DEFAULT_WORK_INTERVAL_MS
+    const breathingIntervalMs =
+        options.breathingIntervalMs ?? DEFAULT_BREATHING_INTERVAL_MS
+
     try {
-        // Get all table names
+        // Get all user table names
         const tablesResult = await executeOperation(
             [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }],
             dataSource,
             config
         )
 
-        const tables = tablesResult.map((row: any) => row.name)
-        let dumpContent = 'SQLite format 3\0' // SQLite file header
+        const tables = (tablesResult || [])
+            .map((row: any) => row.name)
+            .filter(
+                (name: unknown): name is string =>
+                    typeof name === 'string' && !name.startsWith('sqlite_')
+            )
+
+        const encoder = new TextEncoder()
+        const chunks: Uint8Array[] = [encoder.encode('SQLite format 3\0')]
+        let activeStartTime = Date.now()
 
-        // Iterate through all tables
+        const maybeBreathe = async () => {
+            if (
+                breathingIntervalMs > 0 &&
+                Date.now() - activeStartTime >= workIntervalMs
+            ) {
+                await new Promise((resolve) =>
+                    setTimeout(resolve, breathingIntervalMs)
+                )
+                activeStartTime = Date.now()
+            }
+        }
+
+        // Iterate through all tables in paginated batches
         for (const table of tables) {
+            await maybeBreathe()
+
             // Get table schema
+            const escapedTable = table.replace(/'/g, "''")
             const schemaResult = await executeOperation(
                 [
                     {
-                        sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
+                        sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${escapedTable}';`,
                     },
                 ],
                 dataSource,
                 config
             )
 
-            if (schemaResult.length) {
+            if (schemaResult && schemaResult.length) {
                 const schema = schemaResult[0].sql
-                dumpContent += `\n-- Table: ${table}\n${schema};\n\n`
+                chunks.push(encoder.encode(`\n-- Table: ${table}\n${schema};\n\n`))
             }
 
-            // Get table data
-            const dataResult = await executeOperation(
-                [{ sql: `SELECT * FROM ${table};` }],
-                dataSource,
-                config
-            )
+            // Paginate table data in batches to avoid loading large tables into memory at once
+            let offset = 0
+            while (true) {
+                await maybeBreathe()
 
-            for (const row of dataResult) {
-                const values = Object.values(row).map((value) =>
-                    typeof value === 'string'
-                        ? `'${value.replace(/'/g, "''")}'`
-                        : value
-                )
-                dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
+                const sql =
+                    offset === 0 && options.batchSize === undefined
+                        ? `SELECT * FROM ${table};`
+                        : `SELECT * FROM ${table} LIMIT ${batchSize} OFFSET ${offset};`
+
+                const dataResult =
+                    (await executeOperation([{ sql }], dataSource, config)) ||
+                    []
+
+                if (dataResult.length > 0) {
+                    let batchStatements = ''
+                    for (const row of dataResult) {
+                        const values = Object.values(row).map(formatSqlValue)
+                        batchStatements += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
+                    }
+                    chunks.push(encoder.encode(batchStatements))
+                }
+
+                if (dataResult.length < batchSize) {
+                    break
+                }
+                offset += batchSize
             }
 
-            dumpContent += '\n'
+            chunks.push(encoder.encode('\n'))
         }
 
-        // Create a Blob from the dump content
-        const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' })
+        // Create a multi-part Blob from encoded chunks without allocating a single monolithic string
+        const blob = new Blob(chunks, { type: 'application/x-sqlite3' })
 
         const headers = new Headers({
             'Content-Type': 'application/x-sqlite3',