Skip to content

Commit ade1466

Browse files
danielebriggidanielebriggi
committed
fix: socket.io limit to maxAttachments (blobs)
1 parent bc55280 commit ade1466

6 files changed

Lines changed: 275 additions & 28 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@sqlitecloud/drivers",
3-
"version": "1.0.871",
3+
"version": "1.0.872",
44
"description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients",
55
"main": "./lib/index.js",
66
"types": "./lib/index.d.ts",

‎src/drivers/connection-ws.ts‎

Lines changed: 67 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,50 @@ import { io, Socket } from 'socket.io-client'
66
import { Decoder as SocketIODecoder, Encoder as SocketIOEncoder } from 'socket.io-parser'
77
import { SQLiteCloudConnection } from './connection'
88
import { SQLiteCloudRowset } from './rowset'
9-
import { ErrorCallback, ResultsCallback, SQLiteCloudCommand, SQLiteCloudConfig, SQLiteCloudError } from './types'
10-
import { decodeBigIntMarkers, encodeBigIntMarkers } from './utilities'
9+
import {
10+
DEFAULT_WEBSOCKET_BLOB_TRANSFER_FORMAT,
11+
ErrorCallback,
12+
ResultsCallback,
13+
SQLiteCloudCommand,
14+
SQLiteCloudConfig,
15+
SQLiteCloudError,
16+
SQLiteCloudWebsocketBlobTransferFormat
17+
} from './types'
18+
import { decodeBigIntMarkers, decodeWebsocketRowsetData, encodeBigIntMarkers, parseWebsocketBlobTransferFormat, parseWebsocketMaxAttachments } from './utilities'
1119

12-
const MAX_SOCKET_IO_ATTACHMENTS = 100000
1320
const SocketIODecoderBase = SocketIODecoder as unknown as new (...args: any[]) => { opts?: { maxAttachments?: number } }
1421

15-
class SQLiteCloudSocketIODecoder extends SocketIODecoderBase {
16-
constructor(opts?: any) {
17-
super(typeof opts === 'function' ? opts : opts?.reviver)
18-
19-
if (this.opts) {
20-
this.opts.maxAttachments = Math.max(this.opts.maxAttachments ?? 0, MAX_SOCKET_IO_ATTACHMENTS)
22+
function createSocketIOParser(maxAttachments: number) {
23+
class SQLiteCloudSocketIODecoder extends SocketIODecoderBase {
24+
constructor(opts?: any) {
25+
const decoderOptions = typeof opts === 'function' ? { reviver: opts } : opts
26+
super(decoderOptions?.reviver)
27+
this.opts ||= {}
28+
this.opts.maxAttachments = Math.max(this.opts.maxAttachments ?? decoderOptions?.maxAttachments ?? 0, maxAttachments)
2129
}
2230
}
31+
32+
return {
33+
Encoder: SocketIOEncoder,
34+
Decoder: SQLiteCloudSocketIODecoder
35+
}
2336
}
2437

25-
const sqliteCloudSocketIOParser = {
26-
Encoder: SocketIOEncoder,
27-
Decoder: SQLiteCloudSocketIODecoder
38+
function getResponseBlobTransferFormat(response: any): SQLiteCloudWebsocketBlobTransferFormat | undefined {
39+
return parseWebsocketBlobTransferFormat(response?.capabilities?.blobTransferFormat || response?.blobTransferFormat, undefined)
40+
}
41+
42+
function getAttachmentLimitError(description: unknown, limit: number): SQLiteCloudError | undefined {
43+
const descriptionMessage = description instanceof Error ? description.message : typeof description === 'string' ? description : ''
44+
if (/illegal attachments/i.test(descriptionMessage)) {
45+
return new SQLiteCloudError(
46+
`WebSocket blob response exceeded the configured Socket.IO attachment limit (${limit}). Use websocketBlobFormat=base64-blobs-v1 or increase websocketMaxAttachments.`,
47+
{
48+
errorCode: 'ERR_WEBSOCKET_MAX_ATTACHMENTS_EXCEEDED',
49+
cause: description as Error | string
50+
}
51+
)
52+
}
2853
}
2954

3055
/**
@@ -50,20 +75,36 @@ export class SQLiteCloudWebsocketConnection extends SQLiteCloudConnection {
5075
if (!this.socket) {
5176
this.config = config
5277
const connectionstring = this.config.connectionstring as string
78+
const websocketMaxAttachments = parseWebsocketMaxAttachments(this.config.websocketMaxAttachments)
5379
const gatewayUrl = this.config?.gatewayurl || `${this.config.host === 'localhost' ? 'ws' : 'wss'}://${this.config.host as string}:443`
54-
this.socket = io(gatewayUrl, { auth: { token: connectionstring }, parser: sqliteCloudSocketIOParser })
80+
this.socket = io(gatewayUrl, {
81+
auth: { token: connectionstring },
82+
parser: createSocketIOParser(websocketMaxAttachments)
83+
})
5584

5685
this.socket.on('connect', () => {
5786
callback?.call(this, null)
5887
})
5988

60-
this.socket.on('disconnect', reason => {
89+
this.socket.on('disconnect', (reason, description) => {
6190
this.close()
62-
callback?.call(this, new SQLiteCloudError('Disconnected', { errorCode: 'ERR_CONNECTION_ENDED', cause: reason }))
91+
callback?.call(
92+
this,
93+
(reason === 'parse error' && getAttachmentLimitError(description, websocketMaxAttachments)) ||
94+
new SQLiteCloudError('Disconnected', { errorCode: 'ERR_CONNECTION_ENDED', cause: reason })
95+
)
6396
})
6497

6598
this.socket.on('connect_error', (error: any) => {
6699
this.close()
100+
if (error?.message === 'parse error' || error?.cause === 'parse error') {
101+
callback?.call(
102+
this,
103+
getAttachmentLimitError(error?.description || error?.data || error?.cause, websocketMaxAttachments) ||
104+
new SQLiteCloudError('Connection error', { errorCode: 'ERR_CONNECTION_ERROR', cause: error })
105+
)
106+
return
107+
}
67108
let message = error.message || 'Connection error'
68109
if (typeof error.context == 'object' && error.context.responseText) {
69110
try {
@@ -106,16 +147,24 @@ export class SQLiteCloudWebsocketConnection extends SQLiteCloudConnection {
106147
sql: commands.query,
107148
bind: encodeBigIntMarkers(commands.parameters),
108149
row: 'array',
109-
safe_integer_mode: this.config.safe_integer_mode
150+
safe_integer_mode: this.config.safe_integer_mode,
151+
capabilities: {
152+
blobTransferFormat: this.config.websocketBlobFormat || DEFAULT_WEBSOCKET_BLOB_TRANSFER_FORMAT
153+
}
110154
},
111155
(response: any) => {
112156
if (response?.error) {
113157
const error = new SQLiteCloudError(response.error.detail, { ...response.error })
114158
callback?.call(this, error)
115159
} else {
116160
const { metadata } = response
117-
const data = decodeBigIntMarkers(response?.data, this.config.safe_integer_mode)
118-
if (data && metadata) {
161+
const blobTransferFormat = getResponseBlobTransferFormat(response)
162+
const data =
163+
metadata && metadata.numberOfRows !== undefined && metadata.numberOfColumns !== undefined && metadata.columns !== undefined
164+
? decodeWebsocketRowsetData(response?.data, metadata, this.config.safe_integer_mode, blobTransferFormat)
165+
: decodeBigIntMarkers(response?.data, this.config.safe_integer_mode)
166+
167+
if (data !== undefined && metadata) {
119168
if (metadata.numberOfRows !== undefined && metadata.numberOfColumns !== undefined && metadata.columns !== undefined) {
120169
console.assert(Array.isArray(data), 'SQLiteCloudWebsocketConnection.transportCommands - data is not an array')
121170
// we can recreate a SQLiteCloudRowset from the response which we know to be an array of arrays

‎src/drivers/types.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,11 @@ export const DEFAULT_PORT = 8860
2020
* mixed - use BigInt and Number types depending on the value size
2121
*/
2222
export type SQLiteCloudSafeIntegerMode = 'number' | 'bigint' | 'mixed'
23+
export type SQLiteCloudWebsocketBlobTransferFormat = 'base64-blobs-v1' | 'socketio-blobs-v1'
2324

2425
export let SAFE_INTEGER_MODE: SQLiteCloudSafeIntegerMode = 'number'
26+
export const DEFAULT_WEBSOCKET_BLOB_TRANSFER_FORMAT: SQLiteCloudWebsocketBlobTransferFormat = 'base64-blobs-v1'
27+
export const DEFAULT_WEBSOCKET_MAX_ATTACHMENTS = 100000
2528
if (typeof process !== 'undefined') {
2629
const mode = process.env['SAFE_INTEGER_MODE']?.toLowerCase()
2730
if (mode === 'bigint' || mode === 'mixed' || mode === 'number') {
@@ -94,6 +97,10 @@ export interface SQLiteCloudConfig {
9497
usewebsocket?: boolean
9598
/** Url where we can connect to a SQLite Cloud Gateway that has a socket.io deamon waiting to connect, eg. wss://host:443 */
9699
gatewayurl?: string
100+
/** Preferred blob transfer format when using websocket transport. Defaults to base64-blobs-v1 for new clients. */
101+
websocketBlobFormat?: SQLiteCloudWebsocketBlobTransferFormat
102+
/** Maximum number of socket.io binary attachments accepted by the websocket parser. */
103+
websocketMaxAttachments?: number
97104

98105
/** Optional identifier used for verbose logging */
99106
clientid?: string

‎src/drivers/utilities.ts‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,19 @@
33
//
44

55
import {
6+
DEFAULT_WEBSOCKET_BLOB_TRANSFER_FORMAT,
7+
DEFAULT_WEBSOCKET_MAX_ATTACHMENTS,
68
DEFAULT_PORT,
79
DEFAULT_TIMEOUT,
10+
ErrorCallback,
811
SAFE_INTEGER_MODE,
12+
SQLCloudRowsetMetadata,
913
SQLiteCloudArrayType,
1014
SQLiteCloudConfig,
1115
SQLiteCloudDataTypes,
1216
SQLiteCloudError,
13-
SQLiteCloudSafeIntegerMode
17+
SQLiteCloudSafeIntegerMode,
18+
SQLiteCloudWebsocketBlobTransferFormat
1419
} from './types'
1520
import { getSafeURL } from './safe-imports'
1621
import { Buffer } from 'buffer'
@@ -187,6 +192,8 @@ export function validateConfiguration(config: SQLiteCloudConfig): SQLiteCloudCon
187192
config.noblob = parseBoolean(config.noblob)
188193
config.compression = config.compression != undefined && config.compression != null ? parseBoolean(config.compression) : true // default: true
189194
config.safe_integer_mode = parseSafeIntegerMode(config.safe_integer_mode || SAFE_INTEGER_MODE)
195+
config.websocketBlobFormat = parseWebsocketBlobTransferFormat(config.websocketBlobFormat)
196+
config.websocketMaxAttachments = parseWebsocketMaxAttachments(config.websocketMaxAttachments)
190197

191198
config.create = parseBoolean(config.create)
192199
config.non_linearizable = parseBoolean(config.non_linearizable)
@@ -257,6 +264,8 @@ export function parseconnectionstring(connectionstring: string): SQLiteCloudConf
257264
maxrowset: options.maxrowset ? parseInt(options.maxrowset) : undefined,
258265
safe_integer_mode: options.safe_integer_mode ? parseSafeIntegerMode(options.safe_integer_mode) : undefined,
259266
usewebsocket: options.usewebsocket ? parseBoolean(options.usewebsocket) : undefined,
267+
websocketBlobFormat: options.websocket_blob_format ? parseWebsocketBlobTransferFormat(options.websocket_blob_format, undefined) : undefined,
268+
websocketMaxAttachments: options.websocket_max_attachments ? parseWebsocketMaxAttachments(options.websocket_max_attachments) : undefined,
260269
verbose: options.verbose ? parseBoolean(options.verbose) : undefined
261270
}
262271

@@ -302,7 +311,29 @@ export function parseSafeIntegerMode(value: string | SQLiteCloudSafeIntegerMode
302311
return 'number'
303312
}
304313

314+
/** Parse websocket BLOB transport format, falling back to the driver default for new websocket clients. */
315+
export function parseWebsocketBlobTransferFormat(
316+
value: string | SQLiteCloudWebsocketBlobTransferFormat | null | undefined,
317+
fallback: SQLiteCloudWebsocketBlobTransferFormat | undefined = DEFAULT_WEBSOCKET_BLOB_TRANSFER_FORMAT
318+
): SQLiteCloudWebsocketBlobTransferFormat | undefined {
319+
const format = value?.toLowerCase()
320+
if (format === 'base64-blobs-v1' || format === 'socketio-blobs-v1') {
321+
return format
322+
}
323+
return fallback
324+
}
325+
326+
/** Parse the maximum number of socket.io binary attachments allowed for a websocket response. */
327+
export function parseWebsocketMaxAttachments(value: string | number | null | undefined): number {
328+
const parsed = typeof value === 'string' ? Number.parseInt(value, 10) : value
329+
if (Number.isSafeInteger(parsed) && (parsed as number) > 0) {
330+
return parsed as number
331+
}
332+
return DEFAULT_WEBSOCKET_MAX_ATTACHMENTS
333+
}
334+
305335
const BIGINT_MARKER_RE = /^-?\d+n$/
336+
const BLOB_COLUMN_TYPE_RE = /\bblob\b/i
306337

307338
/** Convert values that JSON cannot represent losslessly into sqlitecloud-js bigint markers. */
308339
export function encodeBigIntMarkers(value: any): any {
@@ -349,3 +380,31 @@ export function decodeBigIntMarkers(value: any, safeIntegerMode?: SQLiteCloudSaf
349380

350381
return value
351382
}
383+
384+
/** Decode websocket rowset cells using metadata-aware rules for bigint markers and negotiated BLOB transport. */
385+
export function decodeWebsocketRowsetData(
386+
data: any,
387+
metadata: SQLCloudRowsetMetadata,
388+
safeIntegerMode?: SQLiteCloudSafeIntegerMode,
389+
blobTransferFormat?: SQLiteCloudWebsocketBlobTransferFormat
390+
): any {
391+
if (!Array.isArray(data)) {
392+
return decodeBigIntMarkers(data, safeIntegerMode)
393+
}
394+
395+
const blobColumnIndexes = new Set(
396+
metadata.columns.flatMap((column, index) => (column.type && BLOB_COLUMN_TYPE_RE.test(column.type) ? [index] : []))
397+
)
398+
const decodeCell = (value: any, columnIndex: number) => {
399+
if (blobTransferFormat === 'base64-blobs-v1' && blobColumnIndexes.has(columnIndex) && typeof value === 'string') {
400+
return Buffer.from(value, 'base64')
401+
}
402+
return decodeBigIntMarkers(value, safeIntegerMode)
403+
}
404+
405+
if (data.every(row => !Array.isArray(row))) {
406+
return data.map((value, index) => decodeCell(value, index % metadata.numberOfColumns))
407+
}
408+
409+
return data.map(row => (Array.isArray(row) ? row.map((value, columnIndex) => decodeCell(value, columnIndex)) : decodeBigIntMarkers(row, safeIntegerMode)))
410+
}

0 commit comments

Comments
 (0)