fix(tables): rank CSV delimiter by row consistency and bound the peek buffer

Addresses review findings on the delimiter sniffer:

- Rank candidates by row-width consistency first (modal column count), then column
  count. Ranking on column count alone let a comma that appears in a semicolon
  file's unquoted header win over the real separator; the wide-but-ragged split
  now loses to the uniform one.
- Move the partial-trailing-line trim into detectCsvDelimiter so every path
  (stream, client preview, parseFileRows) prepares the sniff sample identically
  and can't disagree on the delimiter.
- Cap the stream peeker's detection copy at the sniff window via Buffer.concat's
  length arg and replay the buffered chunks by reference, so an oversized source
  chunk can't break the bounded-memory contract.

Author: waleedlatif1

apps/sim/lib/table/csv-delimiter-stream.ts

@@ -18,9 +18,13 @@ export interface SniffedCsvStream {
  * Reads the head of a CSV/TSV stream, sniffs its field separator, then returns a
  * stream that replays the buffered head followed by the untouched remainder.
  *
- * Only {@link CSV_DELIMITER_SNIFF_BYTES} are ever held in memory, so this stays
- * safe on multi-GB imports. The buffered head is trimmed to its last newline
- * before sniffing so a mid-record cut can't skew the column counts.
+ * Memory is bounded: it buffers only the source chunks needed to reach the
+ * {@link CSV_DELIMITER_SNIFF_BYTES} window (one chunk past it, at worst), and the
+ * detection sample it copies is capped at exactly that window regardless of how
+ * large a single upstream chunk is. Those buffered chunks are then replayed
+ * *by reference* — never re-copied — so a multi-GB import stays O(sniff window)
+ * plus the single in-flight chunk. {@link detectCsvDelimiter} drops any partial
+ * trailing line, so no newline trimming is needed here.
  */
 export async function sniffCsvDelimiterFromStream(
   source: Readable,
@@ -42,18 +46,15 @@ export async function sniffCsvDelimiterFromStream(
     size += chunk.length
   }
 
-  const head = Buffer.concat(chunks)
-  let sample = head
-  if (!exhausted) {
-    const lastNewline = head.lastIndexOf(0x0a)
-    if (lastNewline > 0) sample = head.subarray(0, lastNewline + 1)
-  }
-
+  // Copy at most the sniff window for detection — `Buffer.concat`'s length arg truncates,
+  // so an oversized final chunk can't inflate this allocation past CSV_DELIMITER_SNIFF_BYTES.
+  const sample = Buffer.concat(chunks, Math.min(size, CSV_DELIMITER_SNIFF_BYTES))
   const delimiter = await detectCsvDelimiter(sample, fallback)
 
   const stream = Readable.from(
     (async function* replay() {
-      if (head.length > 0) yield head
+      // Replay the already-read chunks by reference (no re-copy), then drain the rest.
+      for (const chunk of chunks) yield chunk
       if (exhausted) return
       while (true) {
         const next = await reader.next()

apps/sim/lib/table/import.test.ts

@@ -376,6 +376,23 @@ describe('import', () => {
       expect(await detectCsvDelimiter('text\n"hello, world"\n', ';')).toBe(';')
     })
 
+    it('prefers the consistent delimiter over one that only widens the header row', async () => {
+      // The header splits into more fields on comma, but only semicolon yields a uniform
+      // column count across the data rows — consistency must win over raw field count.
+      expect(await detectCsvDelimiter('a,b;c,d;e,f\n1;2;3\n4;5;6\n')).toBe(';')
+    })
+
+    it('detects the real delimiter when the header itself has quoted commas', async () => {
+      const csv = '"last, first";age;city\n"Doe, John";30;NYC\n"Roe, Jane";25;LA\n'
+      expect(await detectCsvDelimiter(csv)).toBe(';')
+    })
+
+    it('drops a partial trailing line so a mid-record byte cut cannot skew detection', async () => {
+      // Simulates a fixed byte-window slice that ends mid-record; the last (partial) line
+      // is ignored, so the complete rows above still drive the result.
+      expect(await detectCsvDelimiter('a;b;c\n1;2;3\n4;5;')).toBe(';')
+    })
+
     it('returns the fallback for empty input', async () => {
       expect(await detectCsvDelimiter('', ';')).toBe(';')
     })
@@ -406,6 +423,18 @@ describe('import', () => {
       expect((await collect(stream)).equals(full)).toBe(true)
     })
 
+    it('detects and replays exactly when a single chunk far exceeds the sniff window', async () => {
+      // One >1 MiB chunk arrives before the size check — detection must still cap its copy
+      // and the replay must emit the original bytes unchanged.
+      const rows = ['a;b;c']
+      for (let i = 0; i < 40000; i++) rows.push(`${i};${'x'.repeat(20)};y`)
+      const full = Buffer.from(`${rows.join('\n')}\n`)
+      expect(full.length).toBeGreaterThan(1024 * 1024)
+      const { delimiter, stream } = await sniffCsvDelimiterFromStream(Readable.from([full]))
+      expect(delimiter).toBe(';')
+      expect((await collect(stream)).equals(full)).toBe(true)
+    })
+
     it('reassembles data split across many small chunks', async () => {
       const parts = ['na', 'me,ag', 'e\nfo', 'o,1\nba', 'r,2\n'].map((s) => Buffer.from(s))
       const { delimiter, stream } = await sniffCsvDelimiterFromStream(Readable.from(parts))

apps/sim/lib/table/import.ts

@@ -81,22 +81,37 @@ export function decodeCsvText(input: Buffer | Uint8Array | string): string {
 
 /**
  * Sniffs the field separator by trial-parsing the sample with each candidate and
- * keeping the one that yields the most columns.
+ * keeping the one whose column count is most *consistent* across rows.
  *
  * A real parse (rather than counting raw characters) is what makes this safe on
  * files whose quoted cells contain other candidates — `alarms.csv` exports a
  * semicolon-separated file whose `raw_text` cells are full of commas and
- * newlines, and a naive frequency count picks the comma. Ties are broken by how
- * consistently the records match the header width, then by candidate order, so a
- * genuinely single-column file falls back to `fallback` rather than latching onto
- * a separator that appears inside its values.
+ * newlines, and a naive frequency count picks the comma.
+ *
+ * Ranking is **consistency first, then column count**. The true delimiter yields
+ * a uniform column count across every row; a spurious one (e.g. a comma that
+ * happens to appear in an unquoted header of a semicolon file) produces a wide
+ * first row but ragged data rows. Ranking on column count alone would wrongly
+ * prefer that wide-but-inconsistent split, so consistency dominates and column
+ * count only breaks ties. The per-candidate column count is the *modal* (most
+ * common) row width, so one stray ragged row doesn't distort it. A single-column
+ * file (no candidate reaches two columns) falls back to `fallback`.
+ *
+ * All callers funnel through here, so the sample is prepared identically for
+ * every import path: a possibly-partial trailing line (from slicing a fixed byte
+ * window out of a larger file) is dropped before parsing so a mid-record cut
+ * can't skew the counts.
  */
 export async function detectCsvDelimiter(
   input: Buffer | Uint8Array | string,
   fallback: CsvDelimiter = ','
 ): Promise<CsvDelimiter> {
   const { parse } = await import('csv-parse/sync')
-  const text = decodeCsvText(input)
+  const decoded = decodeCsvText(input)
+  // Drop a partial final line when the sample is a prefix of a larger file; keep the
+  // whole thing when it's a single line (no newline) so a tiny full file still parses.
+  const lastNewline = decoded.lastIndexOf('\n')
+  const text = lastNewline > 0 ? decoded.slice(0, lastNewline + 1) : decoded
   if (text.trim() === '') return fallback
 
   let best: { delimiter: CsvDelimiter; fields: number; consistency: number } | null = null
@@ -118,16 +133,28 @@ export async function detectCsvDelimiter(
     }
 
     if (records.length === 0) continue
-    const fields = records[0].length
+
+    // Modal row width: the most frequent column count, tie-broken toward the wider one.
+    const widthCounts = new Map<number, number>()
+    for (const record of records) {
+      widthCounts.set(record.length, (widthCounts.get(record.length) ?? 0) + 1)
+    }
+    let fields = 0
+    let modalFreq = 0
+    for (const [width, freq] of widthCounts) {
+      if (freq > modalFreq || (freq === modalFreq && width > fields)) {
+        fields = width
+        modalFreq = freq
+      }
+    }
     if (fields < 2) continue
 
-    const matching = records.reduce((n, r) => (r.length === fields ? n + 1 : n), 0)
-    const consistency = matching / records.length
+    const consistency = modalFreq / records.length
 
     if (
       !best ||
-      fields > best.fields ||
-      (fields === best.fields && consistency > best.consistency)
+      consistency > best.consistency ||
+      (consistency === best.consistency && fields > best.fields)
     ) {
       best = { delimiter, fields, consistency }
     }