From 291a556021aecc1959b8ce7ef0791e91c7c51a32 Mon Sep 17 00:00:00 2001 From: GenericAgent Date: Wed, 4 Nov 2026 13:19:00 +0800 Subject: [PATCH] fix(llmcore): don't replay partial output on retry; treat empty/thinking-only reply as failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent defects in the streaming retry path, both reproduced with a local fake SSE server (no network): 1. Partial output was replayed on retry. `_stream_with_retry` retried Timeout/ConnectionError even when chunks had already been yielded to the caller, so the already-delivered half was emitted again on every attempt. A single interrupted stream could reach the user as "AAABBBAAABBBAAABBB". MixinSession had the same problem one level up: on a retryable error it switched channel and re-ran the whole request. Now: once anything has been yielded the stream is never retried; a `[!!! 流异常中断 ...]` marker is appended instead (same convention as the generic except-branch). MixinSession keeps the partial output, only routes the *next* call to another channel. 2. An empty answer (or one made only of `thinking` blocks) was reported as a successful empty reply, so no retry and no failover ever happened and the user just saw nothing. Now `_has_usable_output()` requires non-blank text or a tool_use block; otherwise a ConnectionError is raised so the normal retry / failover path applies. Tool-only responses are still valid answers. Tests: tests/test_stream_retry.py (6 cases, 16 assertions). Fails 6 assertions on the unpatched tree, all pass after the fix. Co-Authored-By: GenericAgent --- llmcore.py | 49 ++++++++- tests/test_stream_retry.py | 213 +++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 tests/test_stream_retry.py diff --git a/llmcore.py b/llmcore.py index 04bdeffd0..df5df9ae6 100644 --- a/llmcore.py +++ b/llmcore.py @@ -472,6 +472,24 @@ def _stamp_oai_cache_markers(messages, model): c = list(c); c[-1] = dict(c[-1], cache_control={'type': 'ephemeral'}) messages[idx] = {**messages[idx], 'content': c} +def _has_usable_output(blocks): + """True if the parsed response carries something the caller can act on. + + A response made only of ``thinking`` blocks is **not** usable: it has no + answer text and no tool call, so the caller treats it as a successful empty + reply and no retry / failover ever happens (typical causes: the model burned + its whole token budget on reasoning, or the upstream swallowed the answer). + """ + for b in blocks or []: + if not isinstance(b, dict): + continue + if b.get("type") == "tool_use": + return True + if b.get("type") == "text" and (b.get("text") or "").strip(): + return True + return False + + def _stream_with_retry(sess, url, headers, payload, parse_fn): STATS['session'] = sess.name _RETRYABLE = {408, 409, 425, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527, 529} @@ -524,18 +542,35 @@ def _probe(*a, **k): if chunk and STATS.get('t_ttft') is None: STATS['t_ttft'] = time.time() streamed = True; yield chunk except StopIteration as e: - if not e.value and not streamed: raise requests.ConnectionError("empty response") + blocks = e.value or [] + if not _has_usable_output(blocks): + # 2026-11-04: 空正文 / 仅 thinking 不算成功——渠道会静默返回空白, + # 上层既不会重试也不会换渠(用户只看到"什么都没有")。 + if not streamed: + raise requests.ConnectionError("empty response: no text and no tool_use") + trail = "\n\n[!!! 流异常中断: 响应无正文(仅思考/空)!!!]" + print("[WARN] stream produced no usable output (thinking-only or empty)") + blocks = list(blocks) + [{"type": "text", "text": trail}] + STATS['t_end'] = time.time() + yield trail; return blocks STATS['t_end'] = time.time() STATS['tps'] = STATS.get('out', 0) / max(1e-9, STATS['t_end'] - max(STATS['t_ttft'] or 0, STATS['t_start'])) return e.value or [] except (requests.Timeout, requests.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: err = f"!!!Error: {type(e).__name__}: {e}" if str(e) else f"!!!Error: {type(e).__name__}" if getattr(sess, 'should_stop', None) and sess.should_stop(): return [] - if attempt < sess.max_retries: + # 2026-11-04: 已经向下游吐过内容后再重试 = 把前半截正文重放一遍 + # (实证: 用户看到 "AAABBBAAABBBAAABBB")。已交付内容不可撤回, 只能标记中断, + # 并把"下一次调用换渠"交给 MixinSession 处理。 + if not streamed and attempt < sess.max_retries: d = _delay(None, attempt) print(f"[LLM Retry] {type(e).__name__}, retry in {d:.1f}s ({attempt+1}/{sess.max_retries+1})") if _sleep(d): return [] continue + if streamed: + trail = f"\n\n[!!! 流异常中断 {type(e).__name__}: {e} !!!]" + print(f"[WARN] stream broke after partial output, not retrying ({type(e).__name__})") + yield trail; return [{"type": "text", "text": trail}] yield err; return [{"type": "text", "text": err}] except Exception as e: err = f"\n\n[!!! 流异常中断 {type(e).__name__}: {e} !!!]" if streamed else f"!!!Error: {type(e).__name__}: {e}" @@ -1042,9 +1077,17 @@ def raw_ask(self, messages): while True: chunk = next(gen); last_chunk = chunk if not yielded and test_error(chunk): continue - yield chunk; yielded = True + yield chunk + if not test_error(chunk): yielded = True except StopIteration as e: return_val = e.value or [] is_err = test_error(last_chunk) + # 2026-11-04: 正文已经交付给调用方之后再换渠重试, 只会把前半截重放一遍 + # (用户看到 "AAABBB" + "AAABBB")。保持本轮输出, 仅把下一次调用指到别的渠道。 + if is_err and yielded: + if n > 1: + self._cur_idx = (idx + 1) % n; self._switched_at = time.time(); self._lease_rebind() + print(f'[MixinSession] {str(last_chunk)[:80]}, partial output kept, next call → s{self._cur_idx} ({self.current.name})') + return return_val if not is_err: if attempt > 0: self._cur_idx = idx; self._switched_at = time.time() elif isinstance(last_chunk, str) and '[!!! 流异常中断' in last_chunk and n > 1: diff --git a/tests/test_stream_retry.py b/tests/test_stream_retry.py new file mode 100644 index 000000000..3353f29b2 --- /dev/null +++ b/tests/test_stream_retry.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +"""Stream retry / empty-response regressions -- 2026-11-04 + +Run: python tests/test_stream_retry.py + +Covers two long-standing llmcore defects, both reproduced with a local fake +SSE server (no network, deterministic): + +1. **Partial output replayed on retry.** + ``_stream_with_retry`` retried a broken stream even when chunks had already + been yielded to the caller, so a mid-stream ``ChunkedEncodingError`` made the + user see the first half twice ("AAABBB" + "AAABBB"). Retrying cannot + un-deliver text, so after any visible output we must stop and mark the + truncation instead. + +2. **Empty / thinking-only response treated as success.** + ``test_error`` only recognises ``!!!Error:`` / ``[Error:`` prefixes, and the + emptiness check used ``not e.value``, which is False as soon as a + ``thinking`` block exists. A response with no text and no tool call was + therefore returned as a successful empty answer: no retry, no channel + failover, and the user just sees nothing. + The fix judges "usable output" (non-blank text or a tool_use block) and + raises ConnectionError so the normal retry / failover path runs. +""" +import json +import os +import socket +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ROOT not in sys.path: + sys.path.insert(0, ROOT) + +import llmcore # noqa: E402 + +FAIL = [] +HITS = {} +PORT = 0 + + +def check(cond, msg): + print((' PASS ' if cond else ' FAIL ') + msg) + if not cond: + FAIL.append(msg) + + +class Handler(BaseHTTPRequestHandler): + """Fake OpenAI-compatible SSE endpoint; behaviour selected by URL path.""" + protocol_version = "HTTP/1.1" + + def log_message(self, *a): + pass + + def do_POST(self): + mode = self.path.strip("/").split("/")[0] + HITS[mode] = HITS.get(mode, 0) + 1 + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + + def write(b): + self.wfile.write(("%X\r\n" % len(b)).encode() + b + b"\r\n") + self.wfile.flush() + + def event(obj): + write(("data: " + json.dumps(obj) + "\n\n").encode()) + + def delta(d, fr=None): + event({"choices": [{"delta": d, "finish_reason": fr}]}) + + try: + if mode == "partial": + delta({"content": "AAA"}) + time.sleep(0.02) + delta({"content": "BBB"}) + time.sleep(0.02) + # never send the terminating chunk: simulate a dropped stream + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + if mode == "empty": + delta({}, "stop") + write(b"data: [DONE]\n\n") + write(b"") + return + if mode == "thinking_only": + delta({"reasoning_content": "let me think ..."}) + delta({}, "stop") + write(b"data: [DONE]\n\n") + write(b"") + return + if mode == "tool_only": + delta({"tool_calls": [{"index": 0, "id": "c1", "type": "function", + "function": {"name": "code_run", + "arguments": "{\"script\": \"1\"}"}}]}) + delta({}, "tool_calls") + write(b"data: [DONE]\n\n") + write(b"") + return + delta({"content": "OK"}) + delta({}, "stop") + write(b"data: [DONE]\n\n") + write(b"") + except OSError: + pass + + +def make_session(mode, retries=1, name=None, hide=False): + cfg = { + "apikey": "k", + "apibase": "http://127.0.0.1:%d/%s/v1" % (PORT, mode), + "model": "m", + "name": name or ("fake_" + mode), + "max_retries": retries, + "stream": True, + "timeout": 3, + "read_timeout": 5, + "no_env_proxy": True, + # 仅部分版本支持(隐藏思维链); 旧版 llmcore 会忽略此键 + "stream_hide_reasoning": hide, + } + return llmcore.NativeOAISession(cfg) + + +def drain(session, messages=None): + """Consume raw_ask, return (yielded text, blocks).""" + out, blocks = [], [] + gen = session.raw_ask(messages or [{"role": "user", "content": "hi"}]) + try: + while True: + out.append(next(gen)) + except StopIteration as e: + blocks = e.value or [] + return "".join(x for x in out if isinstance(x, str)), blocks + + +def main(): + print("1) partial stream must NOT be replayed (no retry after visible output)") + n0 = HITS.get("partial", 0) + text, blocks = drain(make_session("partial", retries=2)) + check(text.count("AAA") == 1 and text.count("BBB") == 1, + "first half appears exactly once, got %r" % text[:60]) + check("流异常中断" in text, "truncation is marked explicitly") + check(HITS.get("partial", 0) - n0 == 1, "no re-request after partial output") + + print("2) empty response retries then surfaces an error (no silent success)") + n0 = HITS.get("empty", 0) + text, blocks = drain(make_session("empty", retries=1)) + check(HITS.get("empty", 0) - n0 == 2, "retried once (2 requests total)") + check(text.lstrip().startswith("!!!Error:"), "final chunk is an error, got %r" % text[:60]) + + print("3) thinking-only response is never a silent empty answer") + n0 = HITS.get("thinking_only", 0) + text, blocks = drain(make_session("thinking_only", retries=1, hide=True)) + # hide=True 时: + # 渠道支持隐藏思维链 -> 什么都产不出来 -> 必须报错/换渠 (!!!Error: / 流异常中断) + # 渠道不支持(旧版) -> 思维链本身作为正文抵达 -> 用户看得见, 不算静默 + # 共同不变式: 调用方**永远**不会拿到一个空串却被告知"成功"。 + surfaced = ("\u6d41\u5f02\u5e38\u4e2d\u65ad" in text) or text.lstrip().startswith("!!!Error:") + check(bool(text.strip()), "caller never gets a blank answer, got %r" % text[:60]) + check(surfaced or "think" in text.lower(), + "visible signal (marker/error or the reasoning text itself), got %r" % text[:60]) + + print("4) tool-only response is still a valid answer (must not be called empty)") + n0 = HITS.get("tool_only", 0) + text, blocks = drain(make_session("tool_only", retries=1)) + check(any(b.get("type") == "tool_use" for b in blocks), "tool_use block preserved") + check("流异常中断" not in text, "no empty-response marker") + check(HITS.get("tool_only", 0) - n0 == 1, "single request") + + print("5) normal response is untouched") + n0 = HITS.get("ok", 0) + text, blocks = drain(make_session("ok", retries=2)) + check(text == "OK", "text == 'OK', got %r" % text) + check("流异常中断" not in text, "no marker on a clean stream") + check(HITS.get("ok", 0) - n0 == 1, "single request") + + print("6) MixinSession does not fail over after delivering partial output") + n0o = HITS.get("ok", 0) + ch1 = llmcore.NativeToolClient(make_session("partial", retries=2, name="ch1")) + ch2 = llmcore.NativeToolClient(make_session("ok", retries=2, name="ch2")) + mixin = llmcore.MixinSession([ch1, ch2], { + "llm_nos": ["ch1", "ch2"], "max_retries": 1, "base_delay": 0.0, + }) + text, blocks = drain(mixin) + check(text.count("AAA") == 1, "channel-1 partial text kept once") + check(HITS.get("ok", 0) - n0o == 0, "channel 2 not replayed in the same call") + check(mixin.current_name == "ch2", "next call is routed to channel 2") + + print() + if FAIL: + print("FAILED (%d):" % len(FAIL)) + for m in FAIL: + print(" - " + m) + return 1 + print("ALL PASS") + return 0 + + +if __name__ == "__main__": + srv = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + PORT = srv.server_address[1] + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + rc = main() + finally: + srv.shutdown() + sys.exit(rc)