New doc setup

Author: Sg312

apps/sim/app/api/files/serve/[...path]/route.ts

@@ -5,6 +5,12 @@ import type { NextRequest } from 'next/server'
 import { NextResponse } from 'next/server'
 import { fileServeParamsSchema, fileServeQuerySchema } from '@/lib/api/contracts/storage-transfer'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
+import {
+  DocCompileUserError,
+  getE2BDocFormat,
+  loadCompiledDocByExt,
+} from '@/lib/copilot/tools/server/files/doc-compile'
+import { isE2BDocEnabled } from '@/lib/core/config/feature-flags'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
 import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads'
@@ -72,14 +78,55 @@ async function compileDocumentIfNeeded(
   if (raw) return { buffer, contentType: getContentType(filename) }
 
   const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase()
+  const extNoDot = ext.replace(/^\./, '')
   const format = COMPILABLE_FORMATS[ext]
-  if (!format) return { buffer, contentType: getContentType(filename) }
 
-  const magicLen = format.magic.length
-  if (buffer.length >= magicLen && buffer.subarray(0, magicLen).equals(format.magic)) {
+  // Already a binary file (uploaded or pre-compiled)? Serve as-is.
+  if (format) {
+    const magicLen = format.magic.length
+    if (buffer.length >= magicLen && buffer.subarray(0, magicLen).equals(format.magic)) {
+      return { buffer, contentType: getContentType(filename) }
+    }
+  }
+
+  // .xlsx is a ZIP container with no JS compile path. An uploaded/binary xlsx
+  // must short-circuit here (it isn't in COMPILABLE_FORMATS) — otherwise every
+  // xlsx open would utf-8-decode the whole binary and do an always-miss S3 GET.
+  // Only a Python-source xlsx (UTF-8 text, no ZIP magic) falls through.
+  if (
+    extNoDot === 'xlsx' &&
+    buffer.length >= ZIP_MAGIC.length &&
+    buffer.subarray(0, ZIP_MAGIC.length).equals(ZIP_MAGIC)
+  ) {
     return { buffer, contentType: getContentType(filename) }
   }
 
+  // Generated docs render from a content-addressed compiled binary that is built
+  // exactly ONCE per edit_content/create (at write time) and stored in S3. Serve
+  // only LOADS it — it must never compile, or it would re-run E2B on every preview
+  // fetch, including against the incomplete source mid-generation. A hit returns
+  // the (possibly partial) committed doc; a miss in the E2B regime means the doc
+  // is still being generated → 409, and the client polls until the artifact lands.
+  if (workspaceId && (format || extNoDot === 'xlsx')) {
+    const source = buffer.toString('utf-8')
+    // Load the prebuilt artifact directly from S3 (content-addressed). No extra
+    // in-memory layer here: the store is the source of truth, the client (react
+    // query) already caches the bytes, and this branch never recomputes.
+    const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot)
+    if (stored) {
+      return { buffer: stored.buffer, contentType: stored.contentType }
+    }
+
+    if (isE2BDocEnabled && getE2BDocFormat(filename)) {
+      // Artifact not built yet (still generating, or the source didn't compile at
+      // write time). Signal "not ready" without compiling — handled as 409.
+      throw new DocCompileUserError('Document is still being generated')
+    }
+  }
+
+  if (!format) return { buffer, contentType: getContentType(filename) }
+
+  // E2B disabled and no stored artifact → compile JS source via isolated-vm.
   const code = buffer.toString('utf-8')
   const cacheKey = sha256Hex(`${ext}${code}${workspaceId ?? ''}`)
   const cached = compiledDocCache.get(cacheKey)
@@ -164,6 +211,17 @@ export const GET = withRouteHandler(
 
       return await handleLocalFile(cloudKey, userId, raw, request.signal)
     } catch (error) {
+      // An in-progress/incomplete doc source fails to compile — this is expected
+      // mid-generation, not a server fault. Return 409 (not 500) so it isn't an
+      // alarming error; the client re-fetches once the doc finishes (the serve
+      // URL is busted on the file's updatedAt).
+      if (error instanceof DocCompileUserError) {
+        logger.info('Serve: document still compiling, returning 409', {
+          message: error.message,
+        })
+        return NextResponse.json({ error: 'Document is still being generated' }, { status: 409 })
+      }
+
       logger.error('Error serving file:', error)
 
       if (error instanceof FileNotFoundError) {

apps/sim/app/api/mcp/copilot/route.ts

@@ -31,6 +31,7 @@ import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access'
 import { prepareExecutionContext } from '@/lib/copilot/tools/handlers/context'
 import { DIRECT_TOOL_DEFS, SUBAGENT_TOOL_DEFS } from '@/lib/copilot/tools/mcp/definitions'
 import { env } from '@/lib/core/config/env'
+import { isE2BDocEnabled } from '@/lib/core/config/feature-flags'
 import { RateLimiter } from '@/lib/core/rate-limiter'
 import { getBaseUrl } from '@/lib/core/utils/urls'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
@@ -600,6 +601,7 @@ async function handleBuildToolCall(
       ...(resolvedWorkflowName ? { workflowName: resolvedWorkflowName } : {}),
       ...(resolvedWorkspaceId ? { workspaceId: resolvedWorkspaceId } : {}),
       ...(workspaceContext ? { workspaceContext } : {}),
+      ...(isE2BDocEnabled ? { docCompiler: 'python' } : {}),
       userId,
       model: DEFAULT_COPILOT_MODEL,
       mode: 'agent',

apps/sim/app/api/mothership/execute/route.ts

@@ -14,6 +14,7 @@ import {
 import { runHeadlessCopilotLifecycle } from '@/lib/copilot/request/lifecycle/headless'
 import { requestExplicitStreamAbort } from '@/lib/copilot/request/session/explicit-abort'
 import type { StreamEvent } from '@/lib/copilot/request/types'
+import { isE2BDocEnabled } from '@/lib/core/config/feature-flags'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { buildMothershipToolsForRequest } from '@/lib/mothership/settings/runtime'
 import {
@@ -132,6 +133,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
       messageId,
       isHosted: true,
       workspaceContext,
+      ...(isE2BDocEnabled ? { docCompiler: 'python' } : {}),
       ...(userMetadata ? { userMetadata } : {}),
       ...(fileAttachments && fileAttachments.length > 0 ? { fileAttachments } : {}),
       ...(integrationTools.length > 0 ? { integrationTools } : {}),