fix(agent-core-v2): detect and retry stalled LLM response streams

.changeset/llm-stream-stall-watchdog.md

@@ -0,0 +1,5 @@
+---
+"@moonshot-ai/kimi-code": patch
+---
+
+Fix model requests hanging indefinitely when a provider response stream stalls; stalled requests are now detected and retried automatically with a bounded budget. Retrying a failed step no longer leaves the previous attempt's partial output stuck as a running step in the transcript.

docs/en/configuration/config-files.md

@@ -283,17 +283,20 @@ Configuration errors fail loudly instead of falling back silently: session creat
 
 ## `loop_control`
 
-`loop_control` governs the step count limit, the per-step attempt limit, and the threshold that triggers automatic context compaction in the Agent execution loop.
+`loop_control` governs the step count limit, the per-step attempt limits, the stall detection timeouts for model requests, and the threshold that triggers automatic context compaction in the Agent execution loop.
 
 | Field | Type | Default | Description |
 | --- | --- | --- | --- |
 | `max_steps_per_turn` | `integer` | — | Maximum steps per turn; unset or `0` means unlimited |
 | `max_attempts_per_step` | `integer` | `10` | Maximum total attempts for a failing step, including the initial attempt |
+| `first_output_timeout_ms` | `integer` | `180000` | Maximum time in milliseconds to wait for the first output of a model request; a request that still has no output past this limit is treated as stalled and retried. `0` disables the check |
+| `stream_idle_timeout_ms` | `integer` | `120000` | Maximum time in milliseconds a streaming model response may produce no new output before it is treated as stalled and retried. `0` disables the check |
+| `max_stall_attempts_per_step` | `integer` | `3` | Maximum total attempts for a step whose model request keeps stalling, including the initial attempt |
 | `reserved_context_size` | `integer` | — | Number of tokens reserved for model output; automatic compaction is triggered when the remaining context window falls below this value |
 
-`max_steps_per_turn` can be overridden by the `KIMI_LOOP_MAX_STEPS_PER_TURN` environment variable, and `max_attempts_per_step` by `KIMI_LOOP_MAX_ATTEMPTS_PER_STEP`; both take higher priority than the config file. The former `KIMI_LOOP_MAX_RETRIES_PER_STEP` variable is deprecated but still honored (with a startup warning) when the new one is unset.
+`max_steps_per_turn` can be overridden by the `KIMI_LOOP_MAX_STEPS_PER_TURN` environment variable, and `max_attempts_per_step` by `KIMI_LOOP_MAX_ATTEMPTS_PER_STEP`; both take higher priority than the config file. The same applies to `first_output_timeout_ms` (`KIMI_LOOP_FIRST_OUTPUT_TIMEOUT_MS`), `stream_idle_timeout_ms` (`KIMI_LOOP_STREAM_IDLE_TIMEOUT_MS`), and `max_stall_attempts_per_step` (`KIMI_LOOP_MAX_STALL_ATTEMPTS_PER_STEP`). The former `KIMI_LOOP_MAX_RETRIES_PER_STEP` variable is deprecated but still honored (with a startup warning) when the new one is unset.
 
-Retries only apply to transient failures — connection errors, timeouts, HTTP 429 rate limits, and 5xx server errors. A 429 caused by an exhausted quota or insufficient account balance is not retried and fails immediately, since it cannot succeed until the account is recharged.
+Retries only apply to transient failures — connection errors, timeouts, HTTP 429 rate limits, and 5xx server errors. A 429 caused by an exhausted quota or insufficient account balance is not retried and fails immediately, since it cannot succeed until the account is recharged. A model request that stops making progress — no first output within `first_output_timeout_ms`, or a stream that stays idle beyond `stream_idle_timeout_ms` — fails as a stall and is retried with the separate `max_stall_attempts_per_step` budget instead of `max_attempts_per_step`.
 
 ## `token_counting`
 

docs/en/configuration/env-vars.md

@@ -138,6 +138,9 @@ Switches that control the behavior of subsystems such as telemetry, background t
 | `KIMI_MCP_TOOL_TIMEOUT_MS` | Global default single tool-call timeout (ms) for all MCP servers; takes higher priority than `[mcp] tool_timeout_ms` in `config.toml`, but a per-server `toolTimeoutMs` in `mcp.json` still wins (default `60000`) | Integer from `1` to `2147483647`; invalid values are ignored |
 | `KIMI_LOOP_MAX_STEPS_PER_TURN` | Maximum Agent steps per turn; takes higher priority than `[loop_control] max_steps_per_turn` in `config.toml` (unset or `0` means unlimited) | Non-negative integer; invalid values are ignored |
 | `KIMI_LOOP_MAX_ATTEMPTS_PER_STEP` | Maximum total attempts for a failing step (including the initial attempt); takes higher priority than `[loop_control] max_attempts_per_step` in `config.toml` (default `10`). The deprecated `KIMI_LOOP_MAX_RETRIES_PER_STEP` is still honored with a warning when this variable is unset | Non-negative integer; invalid values are ignored |
+| `KIMI_LOOP_FIRST_OUTPUT_TIMEOUT_MS` | Maximum time in milliseconds to wait for the first output of a model request before treating it as stalled; takes higher priority than `[loop_control] first_output_timeout_ms` in `config.toml` (default `180000`; `0` disables the check) | Non-negative integer; invalid values are ignored |
+| `KIMI_LOOP_STREAM_IDLE_TIMEOUT_MS` | Maximum time in milliseconds a streaming model response may produce no new output before being treated as stalled; takes higher priority than `[loop_control] stream_idle_timeout_ms` in `config.toml` (default `120000`; `0` disables the check) | Non-negative integer; invalid values are ignored |
+| `KIMI_LOOP_MAX_STALL_ATTEMPTS_PER_STEP` | Maximum total attempts for a step whose model request keeps stalling (including the initial attempt); takes higher priority than `[loop_control] max_stall_attempts_per_step` in `config.toml` (default `3`) | Non-negative integer; invalid values are ignored |
 | `KIMI_TOKEN_COUNTING_STRATEGY` | Which context token count is reported externally (the context-size display); takes higher priority than `[token_counting] strategy` in `config.toml` (default `measured+estimated`) | `measured+estimated`, `measured`, `estimated` (case-insensitive); invalid values are ignored |
 | `KIMI_WEB_SEARCH_BASE_URL` | API URL of the web search (`WebSearch`) service; takes higher priority than `[services.moonshot_search] base_url` in `config.toml`, and enables the service without that config section. Persisted credentials and custom headers are not forwarded to an env-selected endpoint | Non-blank string; blank values are ignored |
 | `KIMI_WEB_SEARCH_API_KEY` | API key of the web search (`WebSearch`) service; replaces both the configured API key and OAuth credential when set | Non-blank string; blank values are ignored |

docs/zh/configuration/config-files.md

@@ -282,17 +282,20 @@ k3-max = "同一模型的 max Thinking 档位。适合最难的子任务。"
 
 ## `loop_control`
 
-`loop_control` 控制 Agent 执行循环的步数上限、单步尝试次数上限，以及触发上下文自动压缩的阈值。
+`loop_control` 控制 Agent 执行循环的步数上限、单步尝试次数上限、模型请求的停滞检测超时，以及触发上下文自动压缩的阈值。
 
 | 字段 | 类型 | 默认值 | 说明 |
 | --- | --- | --- | --- |
 | `max_steps_per_turn` | `integer` | — | 单轮最大步数；不设或设为 `0` 则无上限 |
 | `max_attempts_per_step` | `integer` | `10` | 单步失败后的最大总尝试次数（含首次尝试） |
+| `first_output_timeout_ms` | `integer` | `180000` | 等待模型请求首个输出的最长时间（毫秒）；超过此时间仍无任何输出的请求会被判定为停滞并重试。设为 `0` 关闭该检测 |
+| `stream_idle_timeout_ms` | `integer` | `120000` | 流式模型响应允许没有任何新输出的最长时间（毫秒），超时即判定为停滞并重试。设为 `0` 关闭该检测 |
+| `max_stall_attempts_per_step` | `integer` | `3` | 模型请求持续停滞时单步的最大总尝试次数（含首次尝试） |
 | `reserved_context_size` | `integer` | — | 预留给模型输出的 token 数；上下文窗口剩余量低于此值时触发自动压缩 |
 
-`max_steps_per_turn` 可被环境变量 `KIMI_LOOP_MAX_STEPS_PER_TURN` 覆盖，`max_attempts_per_step` 可被 `KIMI_LOOP_MAX_ATTEMPTS_PER_STEP` 覆盖，优先级均高于配置文件。旧的 `KIMI_LOOP_MAX_RETRIES_PER_STEP` 已废弃，但在新变量未设置时仍生效（启动时会给出警告）。
+`max_steps_per_turn` 可被环境变量 `KIMI_LOOP_MAX_STEPS_PER_TURN` 覆盖，`max_attempts_per_step` 可被 `KIMI_LOOP_MAX_ATTEMPTS_PER_STEP` 覆盖，优先级均高于配置文件。`first_output_timeout_ms`、`stream_idle_timeout_ms`、`max_stall_attempts_per_step` 同理，分别对应 `KIMI_LOOP_FIRST_OUTPUT_TIMEOUT_MS`、`KIMI_LOOP_STREAM_IDLE_TIMEOUT_MS`、`KIMI_LOOP_MAX_STALL_ATTEMPTS_PER_STEP`。旧的 `KIMI_LOOP_MAX_RETRIES_PER_STEP` 已废弃，但在新变量未设置时仍生效（启动时会给出警告）。
 
-重试仅针对瞬时故障——连接错误、超时、HTTP 429 限流和 5xx 服务端错误。账户额度耗尽或余额不足导致的 429 不会重试，会立即失败：在充值之前重试不可能成功。
+重试仅针对瞬时故障——连接错误、超时、HTTP 429 限流和 5xx 服务端错误。账户额度耗尽或余额不足导致的 429 不会重试，会立即失败：在充值之前重试不可能成功。模型请求停止推进——超过 `first_output_timeout_ms` 仍无首个输出，或流式响应超过 `stream_idle_timeout_ms` 没有新输出——会按停滞失败，并使用独立的 `max_stall_attempts_per_step` 预算重试，而不是 `max_attempts_per_step`。
 
 ## `token_counting`
 

docs/zh/configuration/env-vars.md

@@ -138,6 +138,9 @@ kimi
 | `KIMI_MCP_TOOL_TIMEOUT_MS` | 所有 MCP server 的全局默认单次工具调用超时（毫秒）；优先级高于 `config.toml` 的 `[mcp] tool_timeout_ms`，但低于 `mcp.json` 中单个 server 的 `toolTimeoutMs`（默认 `60000`） | `1` 到 `2147483647` 的整数；非法值被忽略 |
 | `KIMI_LOOP_MAX_STEPS_PER_TURN` | Agent 单轮最大步数；优先级高于 `config.toml` 的 `[loop_control] max_steps_per_turn`（不设或 `0` 表示无上限） | 非负整数；非法值被忽略 |
 | `KIMI_LOOP_MAX_ATTEMPTS_PER_STEP` | 单步失败后的最大总尝试次数（含首次尝试）；优先级高于 `config.toml` 的 `[loop_control] max_attempts_per_step`（默认 `10`）。旧的 `KIMI_LOOP_MAX_RETRIES_PER_STEP` 已废弃，但在本变量未设置时仍生效并给出警告 | 非负整数；非法值被忽略 |
+| `KIMI_LOOP_FIRST_OUTPUT_TIMEOUT_MS` | 等待模型请求首个输出的最长时间（毫秒），超时即判定为停滞；优先级高于 `config.toml` 的 `[loop_control] first_output_timeout_ms`（默认 `180000`；`0` 表示关闭该检测） | 非负整数；非法值被忽略 |
+| `KIMI_LOOP_STREAM_IDLE_TIMEOUT_MS` | 流式模型响应允许没有任何新输出的最长时间（毫秒），超时即判定为停滞；优先级高于 `config.toml` 的 `[loop_control] stream_idle_timeout_ms`（默认 `120000`；`0` 表示关闭该检测） | 非负整数；非法值被忽略 |
+| `KIMI_LOOP_MAX_STALL_ATTEMPTS_PER_STEP` | 模型请求持续停滞时单步的最大总尝试次数（含首次尝试）；优先级高于 `config.toml` 的 `[loop_control] max_stall_attempts_per_step`（默认 `3`） | 非负整数；非法值被忽略 |
 | `KIMI_TOKEN_COUNTING_STRATEGY` | 对外上报的上下文 token 计数（上下文大小显示）；优先级高于 `config.toml` 的 `[token_counting] strategy`（默认 `measured+estimated`） | `measured+estimated`、`measured`、`estimated`（不区分大小写）；非法值被忽略 |
 | `KIMI_WEB_SEARCH_BASE_URL` | 网页搜索（`WebSearch`）服务的 API URL；优先级高于 `config.toml` 的 `[services.moonshot_search] base_url`，未写配置段时也可启用服务。文件中持久化的凭据和自定义 header 不会发送到环境变量指定的端点 | 非空字符串；空白值被忽略 |
 | `KIMI_WEB_SEARCH_API_KEY` | 网页搜索（`WebSearch`）服务的 API 密钥；设置后同时替换配置中的 API 密钥和 OAuth 凭据 | 非空字符串；空白值被忽略 |

packages/agent-core-v2/docs/config-manifest.toml

@@ -186,6 +186,9 @@ extra_skill_dirs = []
 #   env:
 #     max_steps_per_turn <- KIMI_LOOP_MAX_STEPS_PER_TURN (custom parse)
 #     max_attempts_per_step <- KIMI_LOOP_MAX_ATTEMPTS_PER_STEP (custom parse; deprecated fallback KIMI_LOOP_MAX_RETRIES_PER_STEP)
+#     first_output_timeout_ms <- KIMI_LOOP_FIRST_OUTPUT_TIMEOUT_MS (custom parse)
+#     stream_idle_timeout_ms <- KIMI_LOOP_STREAM_IDLE_TIMEOUT_MS (custom parse)
+#     max_stall_attempts_per_step <- KIMI_LOOP_MAX_STALL_ATTEMPTS_PER_STEP (custom parse)
 # ##########################################################################
 
 [loop_control]
@@ -194,6 +197,9 @@ extra_skill_dirs = []
 # max_ralph_iterations: integer
 # reserved_context_size: integer
 # compaction_trigger_ratio: number
+# first_output_timeout_ms: integer
+# stream_idle_timeout_ms: integer
+# max_stall_attempts_per_step: integer
 
 # ##########################################################################
 # mcp

packages/agent-core-v2/docs/state-manifest.d.ts

@@ -27,7 +27,7 @@
 // references become '(circular)', and class instances collapse to a '(ClassName)'
 // marker — the wire shape of an entry is the JSON projection of the type here.
 //
-// Index (App: 0 keys · Workspace: 6 keys · Session: 18 keys · Agent: 98 keys)
+// Index (App: 0 keys · Workspace: 6 keys · Session: 18 keys · Agent: 99 keys)
 //   App
 //   Workspace
 //     workspaceDirs.ephemeralDirs          src/workspace/workspaceDirs/workspaceDirsService.ts
@@ -127,6 +127,7 @@
 //     shellCommand.tasks                              src/agent/shellCommand/shellCommandService.ts
 //     skill                                           src/agent/skill/skillOps.ts
 //     stepRetry.failedAttempts                        src/agent/stepRetry/stepRetryService.ts
+//     stepRetry.failedStallAttempts                   src/agent/stepRetry/stepRetryService.ts
 //     stepRetry.lastFailedDriverId                    src/agent/stepRetry/stepRetryService.ts
 //     swarm                                           src/features/swarm/swarmOps.ts
 //     task                                            src/agent/task/taskOps.ts
@@ -1286,6 +1287,8 @@ export interface AgentStateSnapshot {
       readonly usedContextTokens?: number;
       readonly maxContextTokens?: number;
       readonly onTraceId?: (traceId: string | null) => void;
+      readonly firstOutputTimeoutMs?: number;
+      readonly streamIdleTimeoutMs?: number;
     };
     readonly systemPrompt: string;
   }>;
@@ -1422,6 +1425,7 @@ export interface AgentStateSnapshot {
   'skill': null;
   // src/agent/stepRetry/stepRetryService.ts
   'stepRetry.failedAttempts': number;
+  'stepRetry.failedStallAttempts': number;
   'stepRetry.lastFailedDriverId': string | undefined;
   // src/agent/task/taskOps.ts
   // replayable · durable — folds: TaskStarted, TaskTerminated

packages/agent-core-v2/src/_base/utils/abort.ts

@@ -1,3 +1,5 @@
+import { MAX_TIMER_DELAY_MS, systemTimeoutScheduler, type TimeoutScheduler } from './timer';
+
 export function abortError(message = 'Aborted'): Error {
   const error = new Error(message);
   error.name = 'AbortError';
@@ -69,6 +71,13 @@ export interface DeadlineAbortSignal {
   readonly clear: () => void;
 }
 
+export interface IdleTimeoutAbortSignal {
+  readonly signal: AbortSignal;
+  readonly idleTimedOut: () => boolean;
+  readonly touch: (timeoutMs?: number) => void;
+  readonly clear: () => void;
+}
+
 export function createDeadlineAbortSignal(
   source: AbortSignal,
   timeoutMs: number,
@@ -91,3 +100,44 @@ export function createDeadlineAbortSignal(
     },
   };
 }
+
+export function createIdleTimeoutAbortSignal(
+  source: AbortSignal | undefined,
+  timeoutMs: number,
+  scheduler: TimeoutScheduler = systemTimeoutScheduler,
+): IdleTimeoutAbortSignal {
+  const controller = new AbortController();
+  const unlinkAbortSignal =
+    source === undefined ? undefined : linkAbortSignal(source, controller);
+  let didIdleTimeout = false;
+  let cleared = false;
+  let currentTimeoutMs = timeoutMs;
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+
+  const arm = () => {
+    if (timeout !== undefined) scheduler.clear(timeout);
+    timeout = undefined;
+    if (cleared || didIdleTimeout || controller.signal.aborted || currentTimeoutMs <= 0) return;
+    timeout = scheduler.set(() => {
+      timeout = undefined;
+      didIdleTimeout = true;
+      controller.abort(abortError());
+    }, Math.min(currentTimeoutMs, MAX_TIMER_DELAY_MS));
+  };
+  arm();
+
+  return {
+    signal: controller.signal,
+    idleTimedOut: () => didIdleTimeout,
+    touch: (nextTimeoutMs?: number) => {
+      if (nextTimeoutMs !== undefined) currentTimeoutMs = nextTimeoutMs;
+      arm();
+    },
+    clear: () => {
+      cleared = true;
+      if (timeout !== undefined) scheduler.clear(timeout);
+      timeout = undefined;
+      unlinkAbortSignal?.();
+    },
+  };
+}

packages/agent-core-v2/src/_base/utils/timer.ts

@@ -9,6 +9,20 @@ export function setClampedTimeout(
   return setTimeout(callback, Math.min(timeoutMs, MAX_TIMER_DELAY_MS));
 }
 
+export interface TimeoutScheduler {
+  now(): number;
+  set(callback: () => void, timeoutMs: number): ReturnType<typeof setTimeout>;
+  clear(handle: ReturnType<typeof setTimeout>): void;
+}
+
+export const systemTimeoutScheduler: TimeoutScheduler = {
+  now: () => Date.now(),
+  set: (callback, timeoutMs) => setTimeout(callback, timeoutMs),
+  clear: (handle) => {
+    clearTimeout(handle);
+  },
+};
+
 export interface IntervalTimerOptions {
   readonly unref?: boolean;
 }

packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts

@@ -10,6 +10,12 @@ import {
 } from '#/agent/contextProjector/contextProjector';
 import { IAgentTokenCountingService } from '#/agent/tokenCounting/tokenCounting';
 import { IAgentProfileService, type ProfileModelContext } from '#/agent/profile/profile';
+import {
+  DEFAULT_FIRST_OUTPUT_TIMEOUT_MS,
+  DEFAULT_STREAM_IDLE_TIMEOUT_MS,
+  LOOP_CONTROL_SECTION,
+  type LoopControl,
+} from '#/agent/loop/configSection';
 import { IAgentStateService } from '#/agent/state/agentState';
 import { IAgentToolRegistryService } from '#/agent/toolRegistry/toolRegistry';
 import { IAgentToolSelectService } from '#/agent/toolSelect/toolSelect';
@@ -589,12 +595,20 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
           : undefined,
     });
     const requester = this.modelCatalog.getRequester(resolved.modelAlias);
+    const loopControl = this.config.get<LoopControl>(LOOP_CONTROL_SECTION);
 
     const messages = overrides.messages ?? this.context.get();
     return {
       requester,
       model: requester.model,
-      params: { ...baseParams, ...budgetParams },
+      params: {
+        ...baseParams,
+        ...budgetParams,
+        firstOutputTimeoutMs:
+          loopControl?.firstOutputTimeoutMs ?? DEFAULT_FIRST_OUTPUT_TIMEOUT_MS,
+        streamIdleTimeoutMs:
+          loopControl?.streamIdleTimeoutMs ?? DEFAULT_STREAM_IDLE_TIMEOUT_MS,
+      },
       modelAlias: resolved.modelAlias,
       thinkingEffort: resolved.thinkingLevel,
       systemPrompt: overrides.systemPrompt ?? turnConfig?.systemPrompt ?? this.profile.getSystemPrompt(),