diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..aaace42 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -128,6 +128,45 @@ describe('Database Dump Module', () => { ) }) + it('should paginate large tables across multiple batches and skip internal sqlite_ tables', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([ + { name: 'sqlite_sequence' }, + { name: 'events' }, + ]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE events (id INTEGER, payload TEXT)' }, + ]) + .mockResolvedValueOnce([ + { id: 1, payload: 'row-1' }, + { id: 2, payload: 'row-2' }, + ]) + .mockResolvedValueOnce([{ id: 3, payload: null }]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig, { + batchSize: 2, + workIntervalMs: 0, + breathingIntervalMs: 1, + }) + + expect(response.status).toBe(200) + const dumpText = await response.text() + expect(dumpText).not.toContain('sqlite_sequence') + expect(dumpText).toContain("INSERT INTO events VALUES (1, 'row-1');") + expect(dumpText).toContain("INSERT INTO events VALUES (2, 'row-2');") + expect(dumpText).toContain('INSERT INTO events VALUES (3, NULL);') + expect(executeOperation).toHaveBeenCalledWith( + [{ sql: 'SELECT * FROM events LIMIT 2 OFFSET 0;' }], + mockDataSource, + mockConfig + ) + expect(executeOperation).toHaveBeenCalledWith( + [{ sql: 'SELECT * FROM events LIMIT 2 OFFSET 2;' }], + mockDataSource, + mockConfig + ) + }) + it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..5b629a2 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -3,60 +3,125 @@ import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +export interface DumpDatabaseOptions { + batchSize?: number + workIntervalMs?: number + breathingIntervalMs?: number +} + +const DEFAULT_BATCH_SIZE = 1000 +const DEFAULT_WORK_INTERVAL_MS = 5000 +const DEFAULT_BREATHING_INTERVAL_MS = 100 + +function formatSqlValue(value: unknown): string { + if (value === null || value === undefined) { + return 'NULL' + } + if (typeof value === 'string') { + return `'${value.replace(/'/g, "''")}'` + } + if (typeof value === 'boolean') { + return value ? '1' : '0' + } + return String(value) +} + export async function dumpDatabaseRoute( dataSource: DataSource, - config: StarbaseDBConfiguration + config: StarbaseDBConfiguration, + options: DumpDatabaseOptions = {} ): Promise { + const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE + const workIntervalMs = options.workIntervalMs ?? DEFAULT_WORK_INTERVAL_MS + const breathingIntervalMs = + options.breathingIntervalMs ?? DEFAULT_BREATHING_INTERVAL_MS + try { - // Get all table names + // Get all user table names const tablesResult = await executeOperation( [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], dataSource, config ) - const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header + const tables = (tablesResult || []) + .map((row: any) => row.name) + .filter( + (name: unknown): name is string => + typeof name === 'string' && !name.startsWith('sqlite_') + ) + + const encoder = new TextEncoder() + const chunks: Uint8Array[] = [encoder.encode('SQLite format 3\0')] + let activeStartTime = Date.now() - // Iterate through all tables + const maybeBreathe = async () => { + if ( + breathingIntervalMs > 0 && + Date.now() - activeStartTime >= workIntervalMs + ) { + await new Promise((resolve) => + setTimeout(resolve, breathingIntervalMs) + ) + activeStartTime = Date.now() + } + } + + // Iterate through all tables in paginated batches for (const table of tables) { + await maybeBreathe() + // Get table schema + const escapedTable = table.replace(/'/g, "''") const schemaResult = await executeOperation( [ { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${escapedTable}';`, }, ], dataSource, config ) - if (schemaResult.length) { + if (schemaResult && schemaResult.length) { const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` + chunks.push(encoder.encode(`\n-- Table: ${table}\n${schema};\n\n`)) } - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) + // Paginate table data in batches to avoid loading large tables into memory at once + let offset = 0 + while (true) { + await maybeBreathe() - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + const sql = + offset === 0 && options.batchSize === undefined + ? `SELECT * FROM ${table};` + : `SELECT * FROM ${table} LIMIT ${batchSize} OFFSET ${offset};` + + const dataResult = + (await executeOperation([{ sql }], dataSource, config)) || + [] + + if (dataResult.length > 0) { + let batchStatements = '' + for (const row of dataResult) { + const values = Object.values(row).map(formatSqlValue) + batchStatements += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + } + chunks.push(encoder.encode(batchStatements)) + } + + if (dataResult.length < batchSize) { + break + } + offset += batchSize } - dumpContent += '\n' + chunks.push(encoder.encode('\n')) } - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + // Create a multi-part Blob from encoded chunks without allocating a single monolithic string + const blob = new Blob(chunks, { type: 'application/x-sqlite3' }) const headers = new Headers({ 'Content-Type': 'application/x-sqlite3',