fix(knowledge): bound the workbook preview to the rows it emits

`sheet_to_json` allocates from a worksheet's DECLARED `!ref` range rather than
its populated cells, and Excel routinely writes an inflated range from stray
formatting. The 1,000-row preview cap was applied to the result, so it bounded
the emitted string while the allocation it was meant to bound had already
happened. An 880 KB workbook exhausted an 8 GB worker; the same content
exhausted 16 GB when this ran inside the connector sync. No machine size fixes
that, because the allocation scales with a number the file declares about
itself — fleet p99 for this task is 691 MB against 8 GB, so this is a cliff, not
pressure.

Passing the window into the conversion is what makes the cap real. `defval` goes
with it: defaulting every cell in the range made each row dense, so allocation
scaled with columns x declared rows rather than with populated cells, and
because no row was left empty it silently defeated the `blankrows: false` beside
it. Reported totals still come from the declared range, so bounding the
conversion does not change what the metadata says the workbook holds.

The eleven documents killed this way recorded `attempt_count = 1`: `maxAttempts`
does not cover `TASK_PROCESS_OOM_KILLED`, which Trigger.dev retries only when a
larger preset is named. Adding that escalation is a safety net rather than the
fix, and the same gap the dispatcher had.

Also corrects the machine comment, which claimed `large-1x` was 2 vCPU / 2 GB.
It is 4 vCPU / 8 GB, and believing the stale figure makes a resize look like the
answer when the parser is what is unbounded.

Author: waleedlatif1

apps/sim/background/knowledge-processing.test.ts

@@ -139,3 +139,17 @@ describe('knowledge processing worker', () => {
     )
   })
 })
+
+describe('knowledge-process-document task configuration', () => {
+  /**
+   * `maxAttempts` does not cover an out-of-memory kill — Trigger.dev retries
+   * `TASK_PROCESS_OOM_KILLED` only when a larger preset is named. Eleven
+   * documents were killed in one afternoon and every one recorded
+   * `attempt_count = 1`, so each was left `failed` having never been retried.
+   */
+  it('escalates to a larger machine on an out-of-memory kill', async () => {
+    const { processDocument } = await import('@/background/knowledge-processing')
+
+    expect(processDocument.retry?.outOfMemory?.machine).toBe('large-2x')
+  })
+})

apps/sim/background/knowledge-processing.ts

@@ -56,12 +56,22 @@ export async function runDocumentProcessing(rawPayload: DocumentProcessingPayloa
 export const processDocument = task({
   id: 'knowledge-process-document',
   maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
-  machine: 'large-1x', // 2 vCPU, 2GB RAM - needed for large PDF processing
+  machine: 'large-1x', // 4 vCPU, 8GB RAM - needed for large PDF processing
   retry: {
     maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
     factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),
     minTimeoutInMs: envNumber(env.KB_CONFIG_MIN_TIMEOUT, 1000),
     maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
+    /**
+     * `maxAttempts` does not cover an out-of-memory kill — Trigger.dev retries
+     * `TASK_PROCESS_OOM_KILLED` only when a larger preset is named here. Eleven
+     * documents were killed in one afternoon and every one recorded
+     * `attempt_count = 1`, so each was left `failed` with no retry at all. The
+     * escalation is a safety net, not the fix: the workbook parser's allocation
+     * no longer scales with a sheet's declared range, and fleet p99 memory is
+     * 691 MB against this machine's 8 GB.
+     */
+    outOfMemory: { machine: 'large-2x' },
   },
   queue: {
     concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),

apps/sim/lib/file-parsers/xlsx-parser.ts

@@ -86,24 +86,45 @@ export class XlsxParser implements FileParser {
 
       logger.info(`Processing sheet: ${sheetName} with ${rowCount} rows`)
 
-      // Convert to JSON with header row
+      /**
+       * Converted over a bounded window rather than the whole sheet.
+       *
+       * `sheet_to_json` allocates from the worksheet's DECLARED `!ref` range,
+       * not from its populated cells, and Excel routinely writes an inflated
+       * range from stray formatting — a sheet claiming hundreds of thousands of
+       * rows materializes that many arrays whatever it actually contains. The
+       * row cap below used to be applied after the conversion, so it bounded the
+       * emitted string while the allocation it was meant to bound had already
+       * happened: an 880 KB workbook exhausted an 8 GB worker, and the same
+       * content exhausted 16 GB when this ran inside the connector sync. No
+       * machine size fixes that, because the allocation scales with a number the
+       * file declares about itself.
+       *
+       * `defval` is gone with it. Defaulting every cell in the range made each
+       * row dense — allocation proportional to columns x rows rather than to
+       * populated cells — and, because no row was left empty, it also silently
+       * defeated the `blankrows: false` beside it.
+       */
+      const lastPreviewRow = Math.min(range.e.r, range.s.r + CONFIG.MAX_PREVIEW_ROWS - 1)
       const sheetData = XLSX.utils.sheet_to_json(worksheet, {
         header: 1,
-        defval: '', // Default value for empty cells
         blankrows: false, // Skip blank rows
+        range: { s: { r: range.s.r, c: range.s.c }, e: { r: lastPreviewRow, c: range.e.c } },
       })
 
+      // Reported from the declared range, as before, so bounding the conversion
+      // does not change what the metadata says the workbook holds.
       const actualRowCount = sheetData.length
-      totalRows += actualRowCount
+      totalRows += rowCount
 
       // Store limited sample for metadata
       if (sampledData.length < CONFIG.MAX_SAMPLE_ROWS) {
         const sampleSize = Math.min(CONFIG.MAX_SAMPLE_ROWS - sampledData.length, actualRowCount)
         sampledData.push(...sheetData.slice(0, sampleSize))
       }
 
-      // Only process limited rows for preview
-      const rowsToProcess = Math.min(actualRowCount, CONFIG.MAX_PREVIEW_ROWS)
+      // Already bounded by the conversion window above.
+      const rowsToProcess = actualRowCount
       const cleanSheetName = sanitizeTextForUTF8(sheetName)
 
       // Add sheet header

apps/sim/lib/file-parsers/xlsx-preview-bound.test.ts

@@ -0,0 +1,70 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import * as XLSX from 'xlsx'
+import { XlsxParser } from '@/lib/file-parsers/xlsx-parser'
+
+/**
+ * A workbook holding three populated rows while declaring a range of 200,000 —
+ * the shape Excel writes when stray formatting inflates `!ref`, and the shape
+ * that exhausted an 8 GB worker.
+ */
+function inflatedRangeWorkbook(): Buffer {
+  const sheet = XLSX.utils.aoa_to_sheet([
+    ['header-a', 'header-b'],
+    ['row-1-a', 'row-1-b'],
+    ['row-2-a', 'row-2-b'],
+  ])
+  sheet['!ref'] = 'A1:B200000'
+  const book = XLSX.utils.book_new()
+  XLSX.utils.book_append_sheet(book, sheet, 'Sheet1')
+  return XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+}
+
+describe('XlsxParser preview bound', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('converts only the preview window, not the declared range', async () => {
+    const toJson = vi.spyOn(XLSX.utils, 'sheet_to_json')
+
+    await new XlsxParser().parseBuffer(inflatedRangeWorkbook())
+
+    expect(toJson).toHaveBeenCalled()
+    const options = toJson.mock.calls[0][1] as {
+      range?: { s: { r: number }; e: { r: number } }
+      defval?: unknown
+    }
+
+    /**
+     * The row cap used to be applied AFTER conversion, so it bounded the emitted
+     * string while the allocation it was meant to bound had already happened.
+     * Passing the window into the conversion is what makes the cap real.
+     */
+    expect(options.range).toBeDefined()
+    const rowsRequested =
+      (options.range as { s: { r: number }; e: { r: number } }).e.r -
+      (options.range as { s: { r: number }; e: { r: number } }).s.r +
+      1
+    expect(rowsRequested).toBeLessThanOrEqual(1000)
+
+    /**
+     * `defval` made every cell in the range materialize, so allocation scaled
+     * with columns x declared rows — and, because no row was left empty, it
+     * silently defeated the `blankrows: false` sitting beside it.
+     */
+    expect(options.defval).toBeUndefined()
+  })
+
+  it('still reports the workbook the sheet declares', async () => {
+    const result = await new XlsxParser().parseBuffer(inflatedRangeWorkbook())
+
+    // Bounding the conversion must not change what the metadata claims the
+    // workbook holds, only how much of it is materialized to say so.
+    expect(result.metadata?.totalRows).toBe(200000)
+    expect(result.content).toContain('header-a')
+    expect(result.content).toContain('row-2-b')
+  })
+})