fix(execution): treat an undetermined lease as a fallback, not a denial

The distributed owner lease is a cross-process fairness check, not a
correctness lock. A round trip that did not answer before its deadline was
reported as a hard failure and rejected the execution, even though the
per-process pool and the per-owner active/queued limits still bound the work.

- Fall back to the local limits when the lease is undetermined. Only
  `limit_exceeded` denies an execution, since it is an actual answer.
- Rename that outcome from `unavailable` to `undetermined` so the absence of
  an answer is not read as a negative one, and log it at warn.
- Make the round-trip deadline configurable and raise its default. This
  deadline and the client's `commandTimeout` are both plain timers, so a value
  near normal event-loop latency misreads a scheduling pause as an unreachable
  dependency.
- Skip the release round trip when no lease was ever registered.

Author: waleedlatif1

apps/sim/lib/core/config/env.ts

@@ -432,6 +432,7 @@ export const env = createEnv({
     IVM_MAX_OWNER_WEIGHT:                  z.string().optional().default('5'),      // Max accepted weight for weighted owner scheduling
     IVM_DISTRIBUTED_MAX_INFLIGHT_PER_OWNER:z.string().optional().default('2200'),   // Max owner in-flight leases across replicas
     IVM_DISTRIBUTED_LEASE_MIN_TTL_MS:      z.string().optional().default('120000'), // Min TTL for distributed in-flight leases (ms)
+    IVM_LEASE_REDIS_DEADLINE_MS:           z.string().optional().default('1000'),   // Deadline for one distributed lease round trip (ms)
     IVM_QUEUE_TIMEOUT_MS:                  z.string().optional().default('300000'), // Max queue wait before rejection (ms)
     IVM_MAX_EXECUTIONS_PER_WORKER:         z.string().optional().default('200'),    // Max lifetime executions before worker is recycled
     IVM_MAX_BROKER_ARGS_JSON_CHARS:        z.string().optional().default('262144'),  // Max JSON payload size for sandbox task broker args (isolate→host)

apps/sim/lib/execution/isolated-vm.test.ts

@@ -498,7 +498,7 @@ describe('isolated-vm scheduler', () => {
     expect(result.error?.message).toContain('Too many concurrent')
   })
 
-  it('fails closed when Redis is configured but unavailable', async () => {
+  it('falls back to local limits when no Redis client is available', async () => {
     const { executeInIsolatedVM } = await loadExecutionModule({
       envOverrides: {
         REDIS_URL: 'redis://localhost:6379',
@@ -516,14 +516,11 @@ describe('isolated-vm scheduler', () => {
       ownerKey: 'user:redis-down',
     })
 
-    expect(result.error).toMatchObject({
-      isSystemError: true,
-      message: 'Code execution coordination is temporarily unavailable. Please try again later.',
-    })
-    expect(result.result).toBeNull()
+    expect(result.error).toBeUndefined()
+    expect(result.result).toBe('ok')
   })
 
-  it('fails closed when Redis lease evaluation errors', async () => {
+  it('falls back to local limits when the lease evaluation errors', async () => {
     const { executeInIsolatedVM } = await loadExecutionModule({
       envOverrides: {
         REDIS_URL: 'redis://localhost:6379',
@@ -548,10 +545,68 @@ describe('isolated-vm scheduler', () => {
       ownerKey: 'user:redis-error',
     })
 
-    expect(result.error).toMatchObject({
-      isSystemError: true,
-      message: 'Code execution coordination is temporarily unavailable. Please try again later.',
+    expect(result.error).toBeUndefined()
+    expect(result.result).toBe('ok')
+  })
+
+  it('falls back to local limits when the lease round trip exceeds its deadline', async () => {
+    const { executeInIsolatedVM } = await loadExecutionModule({
+      envOverrides: {
+        REDIS_URL: 'redis://localhost:6379',
+        IVM_LEASE_REDIS_DEADLINE_MS: '5',
+      },
+      spawns: [() => createReadyProc('ok')],
+      redisEvalImpl: (...args: unknown[]) => {
+        const script = String(args[0] ?? '')
+        // Never settles, so only the deadline can decide the outcome.
+        if (script.includes('ZREMRANGEBYSCORE')) {
+          return new Promise<number>(() => {})
+        }
+        return 1
+      },
+    })
+
+    const result = await executeInIsolatedVM({
+      code: 'return "ok"',
+      params: {},
+      envVars: {},
+      contextVariables: {},
+      timeoutMs: 100,
+      requestId: 'req-9',
+      ownerKey: 'user:redis-slow',
+    })
+
+    expect(result.error).toBeUndefined()
+    expect(result.result).toBe('ok')
+  })
+
+  it('still rejects when Redis answers that the owner is over its lease limit', async () => {
+    const { executeInIsolatedVM } = await loadExecutionModule({
+      envOverrides: {
+        IVM_DISTRIBUTED_MAX_INFLIGHT_PER_OWNER: '1',
+        REDIS_URL: 'redis://localhost:6379',
+      },
+      spawns: [() => createReadyProc('ok')],
+      redisEvalImpl: (...args: unknown[]) => {
+        const script = String(args[0] ?? '')
+        if (script.includes('ZREMRANGEBYSCORE')) {
+          return 0
+        }
+        return 1
+      },
+    })
+
+    const result = await executeInIsolatedVM({
+      code: 'return "ok"',
+      params: {},
+      envVars: {},
+      contextVariables: {},
+      timeoutMs: 100,
+      requestId: 'req-10',
+      ownerKey: 'user:over-limit',
     })
+
+    expect(result.error?.message).toContain('Too many concurrent')
     expect(result.result).toBeNull()
   })
 

apps/sim/lib/execution/isolated-vm.ts

@@ -140,7 +140,16 @@ const MAX_EXECUTIONS_PER_WORKER = Number.parseInt(env.IVM_MAX_EXECUTIONS_PER_WOR
 const MAX_BROKER_ARGS_JSON_CHARS = Number.parseInt(env.IVM_MAX_BROKER_ARGS_JSON_CHARS) || 262_144
 const MAX_BROKERS_PER_EXECUTION = Number.parseInt(env.IVM_MAX_BROKERS_PER_EXECUTION) || 1000
 const DISTRIBUTED_KEY_PREFIX = 'ivm:fair:v1:owner'
-const LEASE_REDIS_DEADLINE_MS = 200
+/**
+ * Deadline for a single lease round trip, kept below the shared Redis client's
+ * `commandTimeout` so this race still resolves first.
+ *
+ * Both this deadline and `commandTimeout` are plain `setTimeout`s, so what they
+ * actually measure is event-loop scheduling, not Redis. A value near normal loop
+ * latency therefore reports a healthy Redis as unreachable whenever a garbage
+ * collection pause lands on the call. Keep it well clear of that floor.
+ */
+const LEASE_REDIS_DEADLINE_MS = Number.parseInt(env.IVM_LEASE_REDIS_DEADLINE_MS) || 1000
 const QUEUE_RETRY_DELAY_MS = 1000
 const DISTRIBUTED_LEASE_GRACE_MS = 30000
 
@@ -347,7 +356,16 @@ function ownerRedisKey(ownerKey: string): string {
   return `${DISTRIBUTED_KEY_PREFIX}:${ownerKey}`
 }
 
-type LeaseAcquireResult = 'acquired' | 'limit_exceeded' | 'unavailable'
+/**
+ * Outcome of one distributed lease acquisition.
+ *
+ * `limit_exceeded` is an answer from Redis — the owner is genuinely over its
+ * share — and is the only outcome that denies an execution. `undetermined`
+ * means no answer arrived before the deadline, which is not a denial and must
+ * never be projected as one: the local admission limits below still bound the
+ * work, so the caller falls back to them.
+ */
+type LeaseAcquireResult = 'acquired' | 'limit_exceeded' | 'undetermined'
 
 async function tryAcquireDistributedLease(
   ownerKey: string,
@@ -358,10 +376,10 @@ async function tryAcquireDistributedLease(
 
   const redis = getRedisClient()
   if (!redis) {
-    logger.error('Redis is configured but unavailable for distributed lease acquisition', {
+    logger.warn('No Redis client for distributed lease acquisition; using local limits', {
       ownerKey,
     })
-    return 'unavailable'
+    return 'undetermined'
   }
 
   const now = Date.now()
@@ -407,11 +425,12 @@ async function tryAcquireDistributedLease(
     ])
     return Number(result) === 1 ? 'acquired' : 'limit_exceeded'
   } catch (error) {
-    logger.error('Failed to acquire distributed owner lease; execution will be rejected', {
+    logger.warn('Distributed owner lease undetermined; using local limits', {
       ownerKey,
+      deadlineMs: LEASE_REDIS_DEADLINE_MS,
       error,
     })
-    return 'unavailable'
+    return 'undetermined'
   } finally {
     clearTimeout(deadlineTimer)
   }
@@ -1416,26 +1435,16 @@ export async function executeInIsolatedVM(
       },
     }
   }
-  if (leaseAcquireResult === 'unavailable') {
-    logger.error('Isolated-vm execution rejected because its distributed lease is unavailable', {
-      ownerKey,
-    })
-    maybeCleanupOwner(ownerKey)
-    return {
-      result: null,
-      stdout: '',
-      error: {
-        message: 'Code execution coordination is temporarily unavailable. Please try again later.',
-        name: 'Error',
-        isSystemError: true,
-      },
-    }
-  }
+  // An undetermined lease cannot reject the execution: the per-process pool and
+  // the per-owner active/queued limits above still bound this work.
 
   let settled = false
+  const holdsDistributedLease = leaseAcquireResult === 'acquired'
   const releaseLease = () => {
     if (settled) return
     settled = true
+    // Nothing was registered when the lease was undetermined; skip the round trip.
+    if (!holdsDistributedLease) return
     releaseDistributedLease(ownerKey, distributedLeaseId).catch((error) => {
       logger.error('Failed to release distributed lease', { ownerKey, error })
     })