fix(knowledge): harden ingestion pipeline (#7077)

* fix(knowledge): harden ingestion pipeline

* fix(knowledge): correct bounded ingestion edge cases

* fix(knowledge): tighten redaction and chunk validation

* fix(knowledge): close bounded ingestion gaps

* fix(files): preserve parser complexity limits

Author: waleedlatif1

apps/docs/openapi-v2-knowledge.json

@@ -5278,6 +5278,13 @@
             "maximum": 9007199254740991,
             "description": "Documents unchanged."
           },
+          "docsSkipped": {
+            "default": 0,
+            "description": "Documents intentionally skipped because they could not be indexed safely.",
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991
+          },
           "docsFailed": {
             "type": "integer",
             "minimum": 0,
@@ -5306,6 +5313,7 @@
           "docsUpdated",
           "docsDeleted",
           "docsUnchanged",
+          "docsSkipped",
           "docsFailed",
           "errorMessage"
         ],

apps/sim/app/api/files/parse/route.test.ts

@@ -16,6 +16,7 @@ import {
 } from '@sim/testing'
 import { NextRequest } from 'next/server'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { FileParserError } from '@/lib/file-parsers/errors'
 
 const {
   mockVerifyFileAccess,
@@ -310,6 +311,30 @@ describe('File Parse API Route', () => {
     expect(data.output.content).toBe('plain text content')
   })
 
+  it('should reject parser complexity limits instead of returning raw text', async () => {
+    setupFileApiMocks({
+      cloudEnabled: true,
+      storageProvider: 's3',
+      authenticated: true,
+    })
+    storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('{"value":true}'))
+    mockParseBuffer.mockRejectedValueOnce(
+      new FileParserError('complexity_limit', 'JSON document exceeds the complexity limit')
+    )
+
+    const req = createMockRequest('POST', {
+      filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/data.json',
+    })
+
+    const response = await POST(req)
+    const data = await response.json()
+
+    expect(response.status).toBe(200)
+    expect(data.success).toBe(false)
+    expect(data.error).toContain('complexity limit')
+    expect(data).not.toHaveProperty('output')
+  })
+
   it('should handle multiple files', async () => {
     setupFileApiMocks({
       cloudEnabled: false,

apps/sim/app/api/files/parse/route.ts

@@ -13,8 +13,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
 import { sanitizeUrlForLog } from '@/lib/core/utils/logging'
 import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { isSupportedFileType, parseFile } from '@/lib/file-parsers'
-import { isHtmlComplexityError } from '@/lib/file-parsers/html-parser'
-import { isYamlComplexityError } from '@/lib/file-parsers/yaml-parser'
+import { isFileParserError } from '@/lib/file-parsers/errors'
 import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
 import { uploadExecutionFile } from '@/lib/uploads/contexts/execution'
 import {
@@ -1050,10 +1049,9 @@ async function handleGenericTextBuffer(
       }
     } catch (parserError) {
       if (isPayloadSizeLimitError(parserError)) throw parserError
-      // Fail closed on a resource-exhaustion rejection instead of silently
-      // storing the crafted document as raw text.
-      if (isYamlComplexityError(parserError)) throw parserError
-      if (isHtmlComplexityError(parserError)) throw parserError
+      if (isFileParserError(parserError) && parserError.code === 'complexity_limit') {
+        throw parserError
+      }
 
       logger.warn('Specialized parser failed, falling back to generic parsing:', parserError)
     }

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -89,6 +89,20 @@ function makeResult(id: string, distance = 0.1): SearchResult {
   }
 }
 
+const TEST_EMBEDDING = [0.1, 0.2, 0.3, ...Array.from({ length: 1533 }, () => 0)]
+
+function mockNextEmbeddingResponse(): void {
+  vi.mocked(fetch).mockResolvedValueOnce(
+    new Response(
+      JSON.stringify({
+        data: [{ embedding: TEST_EMBEDDING, index: 0 }],
+        usage: { prompt_tokens: 1, total_tokens: 1 },
+      }),
+      { status: 200, headers: { 'Content-Type': 'application/json' } }
+    )
+  )
+}
+
 describe('Knowledge Search Utils', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -553,12 +567,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       const result = await generateSearchEmbedding('test query')
 
@@ -570,7 +579,7 @@ describe('Knowledge Search Utils', () => {
           }),
         })
       )
-      expect(result.embedding).toEqual([0.1, 0.2, 0.3])
+      expect(result.embedding).toEqual(TEST_EMBEDDING)
 
       // Clean up
       Object.keys(env).forEach((key) => delete (env as any)[key])
@@ -583,12 +592,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       const result = await generateSearchEmbedding('test query')
 
@@ -600,7 +604,7 @@ describe('Knowledge Search Utils', () => {
           }),
         })
       )
-      expect(result.embedding).toEqual([0.1, 0.2, 0.3])
+      expect(result.embedding).toEqual(TEST_EMBEDDING)
 
       // Clean up
       Object.keys(env).forEach((key) => delete (env as any)[key])
@@ -616,12 +620,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query')
 
@@ -645,12 +644,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
@@ -733,12 +727,7 @@ describe('Knowledge Search Utils', () => {
         KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query')
 
@@ -764,12 +753,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
@@ -792,12 +776,7 @@ describe('Knowledge Search Utils', () => {
     it('projects verified provenance only in the model-bound embedding payload', async () => {
       Object.keys(env).forEach((key) => delete (env as any)[key])
       Object.assign(env, { OPENAI_API_KEY: 'test-openai-key' })
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       const registry = new ResolvedSecretTraceRegistry([
         { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'encrypted-token' },

apps/sim/app/api/knowledge/utils.test.ts

@@ -110,17 +110,24 @@ vi.mock('@/lib/knowledge/documents/document-processor', () => ({
   }),
 }))
 
-function createEmbeddingFetchMock() {
-  return vi.fn().mockResolvedValue({
-    ok: true,
-    json: async () => ({
-      data: [
-        { embedding: [0.1, 0.2], index: 0 },
-        { embedding: [0.3, 0.4], index: 1 },
-      ],
-      usage: { prompt_tokens: 2, total_tokens: 2 },
+const TEST_EMBEDDING_DIMENSION = 1536
+
+function createTestEmbedding(value: number): number[] {
+  return Array.from({ length: TEST_EMBEDDING_DIMENSION }, () => value)
+}
+
+function createEmbeddingResponse(values: number[]): Response {
+  return new Response(
+    JSON.stringify({
+      data: values.map((value, index) => ({ embedding: createTestEmbedding(value), index })),
+      usage: { prompt_tokens: values.length, total_tokens: values.length },
     }),
-  })
+    { status: 200, headers: { 'Content-Type': 'application/json' } }
+  )
+}
+
+function createEmbeddingFetchMock() {
+  return vi.fn().mockResolvedValue(createEmbeddingResponse([0.1, 0.3]))
 }
 
 vi.stubGlobal('fetch', createEmbeddingFetchMock())
@@ -166,6 +173,10 @@ describe('Knowledge Utils', () => {
           embeddingModel: 'text-embedding-3-small',
           billedAccountUserId: 'billing-user-1',
           uploadedBy: null,
+          filename: 'file.txt',
+          fileUrl: 'https://example.com/file.txt',
+          fileSize: 10,
+          mimeType: 'text/plain',
         },
       ])
       /** Legacy untracked documents have exact-empty provenance. */
@@ -301,13 +312,7 @@ describe('Knowledge Utils', () => {
       })
 
       const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
-          data: [{ embedding: [0.1, 0.2], index: 0 }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        }),
-      } as any)
+      fetchSpy.mockResolvedValueOnce(createEmbeddingResponse([0.1]))
 
       await generateEmbeddings(['test text'])
 
@@ -331,13 +336,7 @@ describe('Knowledge Utils', () => {
       })
 
       const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
-          data: [{ embedding: [0.1, 0.2], index: 0 }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        }),
-      } as any)
+      fetchSpy.mockResolvedValueOnce(createEmbeddingResponse([0.1]))
 
       await generateEmbeddings(['test text'])
 

apps/sim/app/api/tools/embeddings/route.test.ts

@@ -4,13 +4,26 @@
 import { createMockRequest, hybridAuthMockFns } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockEmbed, mockEmbedOpenRouter, mockGetOpenRouterEmbeddingModelMetadata } = vi.hoisted(
-  () => ({
+const {
+  MockEmbeddingOutputLimitError,
+  mockEmbed,
+  mockEmbedOpenRouter,
+  mockGetOpenRouterEmbeddingModelMetadata,
+} = vi.hoisted(() => {
+  class MockEmbeddingOutputLimitError extends Error {
+    constructor(message: string) {
+      super(message)
+      this.name = 'EmbeddingOutputLimitError'
+    }
+  }
+
+  return {
+    MockEmbeddingOutputLimitError,
     mockEmbed: vi.fn(),
     mockEmbedOpenRouter: vi.fn(),
     mockGetOpenRouterEmbeddingModelMetadata: vi.fn(),
-  })
-)
+  }
+})
 
 vi.mock('@/lib/embeddings/openrouter-model-catalog.server', () => ({
   getOpenRouterEmbeddingModelMetadata: mockGetOpenRouterEmbeddingModelMetadata,
@@ -27,6 +40,7 @@ vi.mock('@/lib/embeddings', async () => {
   return {
     embed: mockEmbed,
     embedOpenRouter: mockEmbedOpenRouter,
+    EmbeddingOutputLimitError: MockEmbeddingOutputLimitError,
     DEFAULT_OPENROUTER_EMBEDDING_MODEL: 'openrouter/openai/text-embedding-3-small',
     findEmbeddingModelInfo: catalog.findEmbeddingModelInfo,
     getModelsForProvider: catalog.getModelsForProvider,
@@ -230,6 +244,15 @@ describe('POST /api/tools/embeddings', () => {
     expect((await response.json()).error).toContain('429')
   })
 
+  it('returns 413 when the requested embedding output exceeds the safe aggregate limit', async () => {
+    mockEmbed.mockRejectedValue(
+      new MockEmbeddingOutputLimitError('Embedding output exceeds the safe aggregate limit')
+    )
+    const response = await post(baseBody)
+    expect(response.status).toBe(413)
+    expect((await response.json()).error).toContain('safe aggregate limit')
+  })
+
   it('splits a JSON-array input into separate texts', async () => {
     await post({ ...baseBody, input: '["alpha","beta"]' })
     expect(mockEmbed).toHaveBeenCalledWith(['alpha', 'beta'], expect.anything())

apps/sim/app/api/tools/embeddings/route.ts

@@ -12,6 +12,7 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
   DEFAULT_MODEL_BY_PROVIDER,
   DEFAULT_OPENROUTER_EMBEDDING_MODEL,
+  EmbeddingOutputLimitError,
   embed,
   embedOpenRouter,
   findEmbeddingModelInfo,
@@ -227,6 +228,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     })
   } catch (error) {
     const message = getErrorMessage(error, 'Embedding generation failed')
+    if (error instanceof EmbeddingOutputLimitError) {
+      logger.warn('Embedding output exceeds safe limit', { error: message })
+      return NextResponse.json({ success: false, error: message }, { status: 413 })
+    }
     logger.error('Embedding generation failed', { error: message })
     return NextResponse.json({ success: false, error: message }, { status: 502 })
   }