Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions llmcore.py
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,24 @@ def _stamp_oai_cache_markers(messages, model):
c = list(c); c[-1] = dict(c[-1], cache_control={'type': 'ephemeral'})
messages[idx] = {**messages[idx], 'content': c}

def _has_usable_output(blocks):
"""True if the parsed response carries something the caller can act on.

A response made only of ``thinking`` blocks is **not** usable: it has no
answer text and no tool call, so the caller treats it as a successful empty
reply and no retry / failover ever happens (typical causes: the model burned
its whole token budget on reasoning, or the upstream swallowed the answer).
"""
for b in blocks or []:
if not isinstance(b, dict):
continue
if b.get("type") == "tool_use":
return True
if b.get("type") == "text" and (b.get("text") or "").strip():
return True
return False


def _stream_with_retry(sess, url, headers, payload, parse_fn):
STATS['session'] = sess.name
_RETRYABLE = {408, 409, 425, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527, 529}
Expand Down Expand Up @@ -524,18 +542,35 @@ def _probe(*a, **k):
if chunk and STATS.get('t_ttft') is None: STATS['t_ttft'] = time.time()
streamed = True; yield chunk
except StopIteration as e:
if not e.value and not streamed: raise requests.ConnectionError("empty response")
blocks = e.value or []
if not _has_usable_output(blocks):
# 2026-11-04: 空正文 / 仅 thinking 不算成功——渠道会静默返回空白,
# 上层既不会重试也不会换渠(用户只看到"什么都没有")。
if not streamed:
raise requests.ConnectionError("empty response: no text and no tool_use")
trail = "\n\n[!!! 流异常中断: 响应无正文(仅思考/空)!!!]"
print("[WARN] stream produced no usable output (thinking-only or empty)")
blocks = list(blocks) + [{"type": "text", "text": trail}]
STATS['t_end'] = time.time()
yield trail; return blocks
STATS['t_end'] = time.time()
STATS['tps'] = STATS.get('out', 0) / max(1e-9, STATS['t_end'] - max(STATS['t_ttft'] or 0, STATS['t_start']))
return e.value or []
except (requests.Timeout, requests.ConnectionError, requests.exceptions.ChunkedEncodingError) as e:
err = f"!!!Error: {type(e).__name__}: {e}" if str(e) else f"!!!Error: {type(e).__name__}"
if getattr(sess, 'should_stop', None) and sess.should_stop(): return []
if attempt < sess.max_retries:
# 2026-11-04: 已经向下游吐过内容后再重试 = 把前半截正文重放一遍
# (实证: 用户看到 "AAABBBAAABBBAAABBB")。已交付内容不可撤回, 只能标记中断,
# 并把"下一次调用换渠"交给 MixinSession 处理。
if not streamed and attempt < sess.max_retries:
d = _delay(None, attempt)
print(f"[LLM Retry] {type(e).__name__}, retry in {d:.1f}s ({attempt+1}/{sess.max_retries+1})")
if _sleep(d): return []
continue
if streamed:
trail = f"\n\n[!!! 流异常中断 {type(e).__name__}: {e} !!!]"
print(f"[WARN] stream broke after partial output, not retrying ({type(e).__name__})")
yield trail; return [{"type": "text", "text": trail}]
yield err; return [{"type": "text", "text": err}]
except Exception as e:
err = f"\n\n[!!! 流异常中断 {type(e).__name__}: {e} !!!]" if streamed else f"!!!Error: {type(e).__name__}: {e}"
Expand Down Expand Up @@ -1042,9 +1077,17 @@ def raw_ask(self, messages):
while True:
chunk = next(gen); last_chunk = chunk
if not yielded and test_error(chunk): continue
yield chunk; yielded = True
yield chunk
if not test_error(chunk): yielded = True
except StopIteration as e: return_val = e.value or []
is_err = test_error(last_chunk)
# 2026-11-04: 正文已经交付给调用方之后再换渠重试, 只会把前半截重放一遍
# (用户看到 "AAABBB" + "AAABBB")。保持本轮输出, 仅把下一次调用指到别的渠道。
if is_err and yielded:
if n > 1:
self._cur_idx = (idx + 1) % n; self._switched_at = time.time(); self._lease_rebind()
print(f'[MixinSession] {str(last_chunk)[:80]}, partial output kept, next call → s{self._cur_idx} ({self.current.name})')
return return_val
if not is_err:
if attempt > 0: self._cur_idx = idx; self._switched_at = time.time()
elif isinstance(last_chunk, str) and '[!!! 流异常中断' in last_chunk and n > 1:
Expand Down
213 changes: 213 additions & 0 deletions tests/test_stream_retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
# -*- coding: utf-8 -*-
"""Stream retry / empty-response regressions -- 2026-11-04

Run: python tests/test_stream_retry.py

Covers two long-standing llmcore defects, both reproduced with a local fake
SSE server (no network, deterministic):

1. **Partial output replayed on retry.**
``_stream_with_retry`` retried a broken stream even when chunks had already
been yielded to the caller, so a mid-stream ``ChunkedEncodingError`` made the
user see the first half twice ("AAABBB" + "AAABBB"). Retrying cannot
un-deliver text, so after any visible output we must stop and mark the
truncation instead.

2. **Empty / thinking-only response treated as success.**
``test_error`` only recognises ``!!!Error:`` / ``[Error:`` prefixes, and the
emptiness check used ``not e.value``, which is False as soon as a
``thinking`` block exists. A response with no text and no tool call was
therefore returned as a successful empty answer: no retry, no channel
failover, and the user just sees nothing.
The fix judges "usable output" (non-blank text or a tool_use block) and
raises ConnectionError so the normal retry / failover path runs.
"""
import json
import os
import socket
import sys
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if ROOT not in sys.path:
sys.path.insert(0, ROOT)

import llmcore # noqa: E402

FAIL = []
HITS = {}
PORT = 0


def check(cond, msg):
print((' PASS ' if cond else ' FAIL ') + msg)
if not cond:
FAIL.append(msg)


class Handler(BaseHTTPRequestHandler):
"""Fake OpenAI-compatible SSE endpoint; behaviour selected by URL path."""
protocol_version = "HTTP/1.1"

def log_message(self, *a):
pass

def do_POST(self):
mode = self.path.strip("/").split("/")[0]
HITS[mode] = HITS.get(mode, 0) + 1
self.rfile.read(int(self.headers.get("Content-Length") or 0))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Transfer-Encoding", "chunked")
self.end_headers()

def write(b):
self.wfile.write(("%X\r\n" % len(b)).encode() + b + b"\r\n")
self.wfile.flush()

def event(obj):
write(("data: " + json.dumps(obj) + "\n\n").encode())

def delta(d, fr=None):
event({"choices": [{"delta": d, "finish_reason": fr}]})

try:
if mode == "partial":
delta({"content": "AAA"})
time.sleep(0.02)
delta({"content": "BBB"})
time.sleep(0.02)
# never send the terminating chunk: simulate a dropped stream
self.connection.shutdown(socket.SHUT_RDWR)
self.connection.close()
return
if mode == "empty":
delta({}, "stop")
write(b"data: [DONE]\n\n")
write(b"")
return
if mode == "thinking_only":
delta({"reasoning_content": "let me think ..."})
delta({}, "stop")
write(b"data: [DONE]\n\n")
write(b"")
return
if mode == "tool_only":
delta({"tool_calls": [{"index": 0, "id": "c1", "type": "function",
"function": {"name": "code_run",
"arguments": "{\"script\": \"1\"}"}}]})
delta({}, "tool_calls")
write(b"data: [DONE]\n\n")
write(b"")
return
delta({"content": "OK"})
delta({}, "stop")
write(b"data: [DONE]\n\n")
write(b"")
except OSError:
pass


def make_session(mode, retries=1, name=None, hide=False):
cfg = {
"apikey": "k",
"apibase": "http://127.0.0.1:%d/%s/v1" % (PORT, mode),
"model": "m",
"name": name or ("fake_" + mode),
"max_retries": retries,
"stream": True,
"timeout": 3,
"read_timeout": 5,
"no_env_proxy": True,
# 仅部分版本支持(隐藏思维链); 旧版 llmcore 会忽略此键
"stream_hide_reasoning": hide,
}
return llmcore.NativeOAISession(cfg)


def drain(session, messages=None):
"""Consume raw_ask, return (yielded text, blocks)."""
out, blocks = [], []
gen = session.raw_ask(messages or [{"role": "user", "content": "hi"}])
try:
while True:
out.append(next(gen))
except StopIteration as e:
blocks = e.value or []
return "".join(x for x in out if isinstance(x, str)), blocks


def main():
print("1) partial stream must NOT be replayed (no retry after visible output)")
n0 = HITS.get("partial", 0)
text, blocks = drain(make_session("partial", retries=2))
check(text.count("AAA") == 1 and text.count("BBB") == 1,
"first half appears exactly once, got %r" % text[:60])
check("流异常中断" in text, "truncation is marked explicitly")
check(HITS.get("partial", 0) - n0 == 1, "no re-request after partial output")

print("2) empty response retries then surfaces an error (no silent success)")
n0 = HITS.get("empty", 0)
text, blocks = drain(make_session("empty", retries=1))
check(HITS.get("empty", 0) - n0 == 2, "retried once (2 requests total)")
check(text.lstrip().startswith("!!!Error:"), "final chunk is an error, got %r" % text[:60])

print("3) thinking-only response is never a silent empty answer")
n0 = HITS.get("thinking_only", 0)
text, blocks = drain(make_session("thinking_only", retries=1, hide=True))
# hide=True 时:
# 渠道支持隐藏思维链 -> 什么都产不出来 -> 必须报错/换渠 (!!!Error: / 流异常中断)
# 渠道不支持(旧版) -> 思维链本身作为正文抵达 -> 用户看得见, 不算静默
# 共同不变式: 调用方**永远**不会拿到一个空串却被告知"成功"。
surfaced = ("\u6d41\u5f02\u5e38\u4e2d\u65ad" in text) or text.lstrip().startswith("!!!Error:")
check(bool(text.strip()), "caller never gets a blank answer, got %r" % text[:60])
check(surfaced or "think" in text.lower(),
"visible signal (marker/error or the reasoning text itself), got %r" % text[:60])

print("4) tool-only response is still a valid answer (must not be called empty)")
n0 = HITS.get("tool_only", 0)
text, blocks = drain(make_session("tool_only", retries=1))
check(any(b.get("type") == "tool_use" for b in blocks), "tool_use block preserved")
check("流异常中断" not in text, "no empty-response marker")
check(HITS.get("tool_only", 0) - n0 == 1, "single request")

print("5) normal response is untouched")
n0 = HITS.get("ok", 0)
text, blocks = drain(make_session("ok", retries=2))
check(text == "OK", "text == 'OK', got %r" % text)
check("流异常中断" not in text, "no marker on a clean stream")
check(HITS.get("ok", 0) - n0 == 1, "single request")

print("6) MixinSession does not fail over after delivering partial output")
n0o = HITS.get("ok", 0)
ch1 = llmcore.NativeToolClient(make_session("partial", retries=2, name="ch1"))
ch2 = llmcore.NativeToolClient(make_session("ok", retries=2, name="ch2"))
mixin = llmcore.MixinSession([ch1, ch2], {
"llm_nos": ["ch1", "ch2"], "max_retries": 1, "base_delay": 0.0,
})
text, blocks = drain(mixin)
check(text.count("AAA") == 1, "channel-1 partial text kept once")
check(HITS.get("ok", 0) - n0o == 0, "channel 2 not replayed in the same call")
check(mixin.current_name == "ch2", "next call is routed to channel 2")

print()
if FAIL:
print("FAILED (%d):" % len(FAIL))
for m in FAIL:
print(" - " + m)
return 1
print("ALL PASS")
return 0


if __name__ == "__main__":
srv = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
PORT = srv.server_address[1]
threading.Thread(target=srv.serve_forever, daemon=True).start()
try:
rc = main()
finally:
srv.shutdown()
sys.exit(rc)