diff --git a/frontends/tests/test_dsml_tool_markup.py b/frontends/tests/test_dsml_tool_markup.py new file mode 100644 index 000000000..b8b2b9e30 --- /dev/null +++ b/frontends/tests/test_dsml_tool_markup.py @@ -0,0 +1,140 @@ +"""伪工具调用(DSML / XML 风格)救回机制验收测试。 + +关键约束: + A. 危险例(正文里被讨论/引用的标签)—— 绝不能执行 + B. 漏检例(真实泄漏形态)—— 必须救回 + C. DSML 标记变体(| | ‖ 单/双/三、开/闭标签、跨 chunk) + D. arguments 侧污染(上游 #846-B) + E. 散文/代码区否决,不产生假重试 +""" +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(ROOT)) +import llmcore as L + +NAMES = {'code_run', 'code_exec', 'ask_user', 'web_scan', 'web_execute_js', + 'file_read', 'file_write', 'file_patch', 'update_working_checkpoint'} + +DANGER = { + '讨论句带 cwd': '引擎不会输出 这种标记,请放心。', + '反引号裸标签': '发现 `` 抢救有误判风险', + '反引号带属性': '真实形态:`` 这种', + '反引号含完整调用': '看到 `print(1)` 这种就说明泄漏了', + '围栏代码块': '用法:\n```\nrm -rf /tmp/x\n```\n就这样。', + '函数名提及': '你可以用 code_run 来跑脚本,参数是 script。', + '行内提及 dsml': '最近的 commit 加了 DSML 检测,但还不够保底。', +} + +LEAK = [ + ('行内真泄漏', '查一下\n先跑 ls -l 看结果', 'code_run', {'cwd': '/tmp'}), + ('行首裸脚本', '查一下\n\nimport os\nprint(os.getcwd())\n', 'code_run', None), + ('调用后带说明', 'ls -l\n\n以上命令用于列出目录。', 'code_run', None), + ('调用后带 summary', 'ls -l\n已执行', 'code_run', None), + ('JSON 体', '{"script": "ls -l"}', 'code_run', None), + ('压平单行', 'script: echo hi', 'code_run', None), + ('裸子标签 ask_user', '\nwhich one?\nab\n', 'ask_user', None), + ('标签名后多引号', 'ls -l', 'code_run', None), +] + + +def calls_of(text): + return L.salvage_leaked_tool_calls(text, NAMES)[0] + + +@pytest.mark.parametrize('label,text', list(DANGER.items())) +def test_danger_cases_never_execute(label, text): + """正文里被讨论/引用的标签,一个都不许执行。""" + calls = calls_of(text) + assert not calls, f"{label}: 误执行 {[(c.function.name, c.function.arguments[:50]) for c in calls]}" + + +@pytest.mark.parametrize('label,text,want_name,want_args', LEAK) +def test_leaked_forms_are_salvaged(label, text, want_name, want_args): + """真实泄漏形态必须救回成工具调用。""" + calls = calls_of(text) + assert calls, f"{label}: 漏检" + assert calls[0].function.name == want_name, f"{label}: 工具名 {calls[0].function.name}" + if want_args: + args = json.loads(calls[0].function.arguments) + for k, v in want_args.items(): + assert args.get(k) == v, f"{label}: {k}={args.get(k)!r}" + + +def test_json_body_not_double_wrapped(): + """体内直接是 JSON 参数时,不能二次包装成 {"script": "{\"script\": ...}"}。""" + calls = calls_of('{"script": "ls -l"}') + assert json.loads(calls[0].function.arguments).get('script') == 'ls -l' + + +def test_bare_children_multi_value_becomes_list(): + calls = calls_of('\nwhich one?\n' + 'ab\n') + args = json.loads(calls[0].function.arguments) + assert args == {'question': 'which one?', 'candidates': ['a', 'b']} + + +@pytest.mark.parametrize('label,raw,want', [ + ('单ASCII竖线', f'AB', 'AB'), + ('全角竖线', f'AB', 'AB'), + ('双ASCII竖线', 'AB', 'AB'), + ('双全角竖线', f'AB', 'AB'), + ('U+2016双竖线', f'A<{chr(0x2016)}DSML{chr(0x2016)}tool_calls>B', 'AB'), + ('开标签带属性', f'A<{chr(124)}DSML{chr(124)}invoke name="x">B', 'AB'), + ('dsml_下划线式', 'AB', 'AB'), + ('无标记不误伤', 'A < b and c > d', 'A < b and c > d'), +]) +def test_dsml_marker_variants(label, raw, want): + assert L._strip_dsml_tags(raw) == want, label + + +def test_stream_filter_strips_across_chunks(): + f = L.DsmlStreamFilter() + out = f.feed('hello<|') + f.feed('|DSML||tool_calls>') + f.feed('world') + f.flush() + assert out == 'helloworld' + + +def test_stream_filter_keeps_trailing_angle_bracket(): + f = L.DsmlStreamFilter() + assert f.feed('a<') + f.flush() == 'a<' + + +def test_arguments_side_marker_stripped(): + raw = '{"path": "/tmp/a.txt"}' + assert L._try_parse_tool_args(raw)[0].get('path') == '/tmp/a.txt' + + +def test_arguments_with_lt_untouched(): + assert L._try_parse_tool_args('{"script": "if a < b: pass"}')[0].get('script') == 'if a < b: pass' + + +def test_bad_json_falls_back_without_raising(): + assert '_raw' in L._try_parse_tool_args('{bad json')[0] + + +def test_prose_mention_is_not_leak(): + assert L.leaked_markup_kind('不会输出 这种', NAMES) == '' + + +def test_fenced_example_is_not_leak(): + assert L.leaked_markup_kind('```\nx\n```', NAMES) == '' + + +def test_real_leak_is_detected(): + assert L.leaked_markup_kind('ls', NAMES) != '' + + +def test_prose_mention_is_not_incomplete(): + assert L.leaked_markup_incomplete('不会输出 这种标记') is False + + +def test_truncated_fragment_is_incomplete(): + assert L.leaked_markup_incomplete('') is True + + +def test_closed_call_is_not_incomplete(): + assert L.leaked_markup_incomplete('ls') is False diff --git a/ga.py b/ga.py index 15fdb71bf..7f4a6232e 100644 --- a/ga.py +++ b/ga.py @@ -478,8 +478,46 @@ def do_no_tool(self, args, response): if '[!!! 流异常中断' in content[-100:] or '!!!Error:' in content[50:][-100:] or (content.endswith('') and len(content) < 100): return self._retry_or_exit("[ERROR] Incomplete response. Regenerate and tooluse.") tail = content.rstrip()[-50:] - if 'DSML' in tail and '|' in tail and tail.endswith('>'): - return self._retry_or_exit("[ERROR] Tool call format invalid; no tool was called. Regenerate using proper tool calls.") + # 1. 工具调用协议泄漏(DSML / 等):全文级检测,不只尾部 50 字。 + # 能被完整解析 → 直接执行(不浪费一轮);被截断/残缺 → 要求重生成。 + from llmcore import leaked_markup_kind, salvage_leaked_tool_calls, leaked_markup_incomplete, salvaged_call_problem + _names = {m[3:] for m in dir(self) if m.startswith('do_')} - {'no_tool'} + kind = leaked_markup_kind(content, _names) + if kind: + calls, cleaned = salvage_leaked_tool_calls(content, _names) + calls = [c for c in calls if getattr(c.function, 'name', '') and c.function.name != 'no_tool'] + problems = [p for p in (salvaged_call_problem(c) for c in calls) if p] + if calls and not leaked_markup_incomplete(content) and not problems: + yield f"[Warn] Tool-call markup leaked into text ({kind}); salvaged {len(calls)} call(s) and executing them.\n" + data, prompts, result_lines = [], [], [] + for c in calls: + try: c_args = json.loads(c.function.arguments or '{}') + except Exception: c_args = {} + if not isinstance(c_args, dict): c_args = {} + gen = self.dispatch(c.function.name, c_args, response) + try: + v = next(gen) + if v: yield v + outcome = yield from gen + except StopIteration as e: + outcome = e.value or StepOutcome(None) + if outcome.data is not None: + data.append(outcome.data) + try: _ds = json.dumps(outcome.data, ensure_ascii=False, default=str) + except Exception: _ds = str(outcome.data) + result_lines.append(f'\n{_ds}\n') + if outcome.next_prompt: prompts.append(outcome.next_prompt) + if outcome.should_exit: return StepOutcome(data, should_exit=True) + # no_tool 支线的结果不会被 agent_loop 回灌(它跳过 no_tool 的 data),必须自带上 + head = "[System] 上一轮你的工具调用协议标记泄漏进了正文,已自动救回并执行,结果如下:\n" + "\n".join(result_lines) + "\n请据此继续。\n" + return StepOutcome(data or {"result": "salvaged"}, next_prompt=head + ('\n'.join(prompts) or self._get_anchor_prompt())) + # 残缺/无法还原:绝不执行半截脚本,要求整条重发 + why = ('; '.join(problems)) if problems else ('内容被截断' if calls else '无法解析出任何调用') + yield f"[Warn] Tool-call markup leaked into text ({kind}) and could not be recovered: {why}\n" + return self._retry_or_exit( + "[ERROR] 上一轮没有产生有效的工具调用:正文里混入了残缺的工具调用协议标记(DSML / 等)," + "或其中内容被截断。请重新生成,直接使用正常工具调用通道(不要输出任何 DSML/invoke/parameter 标记;" + "脚本较长时拆成多个小步骤,避免被 max_tokens 截断)。") if 'max_tokens !!!]' in content[-100:]: return self._retry_or_exit("[ERROR] max_tokens limit reached. Use multi small steps to do it.") diff --git a/llmcore.py b/llmcore.py index 611e83556..26cba9641 100644 --- a/llmcore.py +++ b/llmcore.py @@ -1,4 +1,4 @@ -import os, json, re, time, requests, sys, threading, urllib3, base64, importlib, uuid, pathlib, copy +import os, json, re, time, requests, sys, threading, urllib3, base64, importlib, uuid, pathlib, copy, ast from datetime import datetime urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) _INFLIGHT = {} # thread ident -> live socket; lets abort() close it even before response headers arrive @@ -257,8 +257,14 @@ def _parse_claude_sse(resp_lines): def _try_parse_tool_args(raw): """Parse tool args string; split concatenated JSON objects like {..}{..} if needed. - Returns list of parsed dicts.""" + Returns list of parsed dicts. + + DSML markers can leak into the arguments string too (upstream #846-B), e.g. + '{"path": "/a.txt"}'. Strip them first; the markers only + ever contain 'DSML', so ordinary scripts with '<' are untouched. + """ if not raw: return [{}] + raw = _strip_dsml_tags(raw) try: return [json.loads(raw)] except: pass parts = re.split(r'(?<=\})(?=\{)', raw) @@ -276,6 +282,7 @@ def _parse_openai_sse(resp_lines, api_mode="chat_completions"): content_block: {type:'text', text:str} | {type:'tool_use', id:str, name:str, input:dict} """ content_text = "" + _dsml = DsmlStreamFilter() # 流式剥离 DSML 标记(跨 chunk 也能剥,见 _DsmlStreamFilter) if api_mode == "responses": seen_delta = False; fc_buf = {}; current_fc_idx = None; reasoning_text = "" for line in resp_lines: @@ -289,9 +296,12 @@ def _parse_openai_sse(resp_lines, api_mode="chat_completions"): etype = evt.get("type", "") if etype == "response.output_text.delta": delta = evt.get("delta", "") - if delta: seen_delta = True; content_text += delta; yield delta + if delta: + seen_delta = True + clean = _dsml.feed(delta) + if clean: content_text += clean; yield clean elif etype == "response.output_text.done" and not seen_delta: - text = evt.get("text", "") + text = _strip_dsml_tags(evt.get("text", "")) if text: content_text += text; yield text elif etype == "response.reasoning_text.delta": delta = evt.get("delta", "") @@ -339,6 +349,8 @@ def _parse_openai_sse(resp_lines, api_mode="chat_completions"): _raise_if_retryable_overload(emsg) if emsg: content_text += f"!!!Error: {emsg}"; yield f"!!!Error: {emsg}" break + tail = _dsml.flush() + if tail: content_text += tail; yield tail blocks = [] if reasoning_text: blocks.append({"type": "thinking", "thinking": reasoning_text}) if content_text: blocks.append({"type": "text", "text": content_text}) @@ -366,7 +378,8 @@ def _parse_openai_sse(resp_lines, api_mode="chat_completions"): if rc := delta.get("reasoning_content") or delta.get("reasoning", ""): reasoning_text += rc; yield rc if delta.get("content"): - text = delta["content"]; content_text += text; yield text + text = _dsml.feed(delta["content"]) + if text: content_text += text; yield text for tc in (delta.get("tool_calls") or []): idx = tc.get("index", 0) has_name = bool(tc.get("function", {}).get("name")) @@ -378,6 +391,8 @@ def _parse_openai_sse(resp_lines, api_mode="chat_completions"): if tc.get("id") and not tc_buf[idx]["id"]: tc_buf[idx]["id"] = tc["id"] usage = evt.get("usage") if usage: _record_usage(usage, api_mode) + tail = _dsml.flush() + if tail: content_text += tail; yield tail blocks = [] if reasoning_text: blocks.append({"type": "thinking", "thinking": reasoning_text}) if content_text: blocks.append({"type": "text", "text": content_text}) @@ -429,8 +444,9 @@ def _parse_openai_json(data, api_mode="chat_completions"): if p.get("type") in ("reasoning_text", "summary_text") and p.get("text"): blocks.append({"type": "thinking", "thinking": p["text"]}) elif item.get("type") == "function_call": - try: args = json.loads(item.get("arguments", "")) if item.get("arguments") else {} - except: args = {"_raw": item.get("arguments", "")} + _a = _strip_dsml_tags(item.get("arguments", "")) + try: args = json.loads(_a) if _a else {} + except: args = {"_raw": _a} blocks.append({"type": "tool_use", "id": item.get("call_id", item.get("id", "")), "name": item.get("name", ""), "input": args}) status = (data.get("status") or "").lower() @@ -449,13 +465,14 @@ def _parse_openai_json(data, api_mode="chat_completions"): reasoning = msg.get("reasoning_content") or msg.get("reasoning", "") if reasoning: blocks.append({"type": "thinking", "thinking": reasoning}) - content = msg.get("content", "") + content = _strip_dsml_tags(msg.get("content", "")) if content: blocks.append({"type": "text", "text": content}); yield content for tc in (msg.get("tool_calls") or []): fn = tc.get("function", {}) - try: args = json.loads(fn.get("arguments", "")) if fn.get("arguments") else {} - except: args = {"_raw": fn.get("arguments", "")} + _a = _strip_dsml_tags(fn.get("arguments", "")) + try: args = json.loads(_a) if _a else {} + except: args = {"_raw": _a} blocks.append({"type": "tool_use", "id": tc.get("id", ""), "name": fn.get("name", ""), "input": args}) return blocks @@ -733,7 +750,14 @@ def _ask_gen(): if block.get('type', '') == 'tool_use': tu = {'name': block.get('name', ''), 'arguments': block.get('input', {})} yield f'{json.dumps(tu, ensure_ascii=False)}' - if content.strip() and not content.startswith("!!!Error:"): self.history.append({"role": "assistant", "content": [{"type": "text", "text": content}]}) + if content.strip() and not content.startswith("!!!Error:"): + _hist = content + if leaked_markup_kind(_hist): + _calls, _rest = salvage_leaked_tool_calls(_hist) + _txt = _rest.strip() + if not _txt: _txt = '' + self.history.append({"role": "assistant", "content": [{"type": "text", "text": _txt}]}) if _txt else None + else: self.history.append({"role": "assistant", "content": [{"type": "text", "text": _hist}]}) return _ask_gen() def _keep_claude_block(b): return not isinstance(b, dict) or b.get("type") != "thinking" or b.get("signature") @@ -884,6 +908,15 @@ def ask(self, msg): if content_blocks and not (len(content_blocks) == 1 and content_blocks[0].get("text", "").startswith("!!!Error:")): history_blocks = content_blocks if self.omit_thinking: history_blocks = [b for b in content_blocks if b.get("type") != "thinking"] + # 净化:把泄漏进正文的协议标记从历史里剥掉,避免模型照抄/自我强化 + _h2 = [] + for b in history_blocks: + if b.get("type") == "text" and leaked_markup_kind(b.get("text", "")): + _txt = salvage_leaked_tool_calls(b.get("text", ""))[1].strip() + if not _txt: continue + b = dict(b, text=_txt) + _h2.append(b) + history_blocks = _h2 self.history.append({"role": "assistant", "content": history_blocks}) text_parts = [b["text"] for b in content_blocks if b.get("type") == "text"] content = "\n".join(text_parts).strip() @@ -933,6 +966,434 @@ def __init__(self, thinking, content, tool_calls, raw, stop_reason='end_turn'): def __repr__(self): return f"" +_DSML_B = chr(0xFF5C) # 全角竖线,DSML 标记的分隔符 +_CODE_TAGS = ('code_run', 'code_exec', 'bash', 'shell', 'run') + +# 行首的 code 标签;容忍 "<\n\ncode_run>"(模型把标签拆行)与属性写法 +_CODE_OPEN = re.compile(r'(?m)^[ \t]*<[ \t\r\n]*(code_run|code_exec|bash|shell|run)\b([^<>]*)>', re.I) +_REG_OPEN = re.compile(r'(?m)^[ \t]*<[ \t\r\n]*(code|execute|exec|terminal)\b([^<>]*)>', re.I) +# 任意位置的 code 标签(是否算真调用由 _code_candidates 的证据规则判定) +_CODE_ANY = re.compile(r'<[ \t\r\n]*(code_run|code_exec|bash|shell|run)\b([^<>]*)>', re.I) + + + +def _strip_code_spans(text): + """去掉代码块/行内代码,避免把正常讲解里的示例标记当成泄漏。""" + t = re.sub(r'```[\s\S]*?```', ' ', text) + t = re.sub(r'`[^`\n]*`', ' ', t) + return t + + +def _canon_dsml(text): + """把 DSML 的各种畸形变体统一成 ,交给同一个解析器处理: + <|DSML| invoke> / <|DSML| ...> / <||DSML|| ...> / antml:invoke / 裸 /。""" + t = text + # 1) 分隔符变体 → |DSML|(单/双/三、全角/半角、混排、带空格) + t = re.sub('[' + _DSML_B + r'|]{1,3}\s*DSML\s*[' + _DSML_B + r'|]{1,3}', '|DSML|', t, flags=re.I) + + def _tag(m): + closing, name, attrs = m.group(1), m.group(2).lower(), m.group(3) or '' + name = 'calls' if name == 'function_calls' else name + return ('' % name) if closing else ('' % (name, attrs)) + + t = re.sub(r'<\s*(/?)\s*\|DSML\|\s*(function_calls|calls|invoke|parameter|args?|value)\b([^<>]*)>', _tag, t, flags=re.I) + # 2) antml:invoke / antml:parameter(Claude 官方文本协议) + t = re.sub(r'<\s*(/?)\s*antml\s*:\s*(function_calls|calls|invoke|parameter)\b([^<>]*)>', _tag, t, flags=re.I) + + # 3) 裸标签(无前缀的 / / ) + def _bare(m): + closing, name, attrs = m.group(1), m.group(2).lower(), m.group(3) or '' + if not closing and not re.search(r'\bname\s*=', attrs): return m.group(0) # 没 name 的多半是正文 + name = 'calls' if name == 'function_calls' else name + return ('' % name) if closing else ('' % (name, attrs)) + + t = re.sub(r'<\s*(/?)\s*(function_calls|invoke|parameter)\b([^<>]*)>', _bare, t, flags=re.I) + return t + + +def _attrs(s): + """解析标签属性:支持 name="x" / name='x' / name=x。""" + d = {} + for m in re.finditer(r'([A-Za-z_][\w-]*)\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s"\'>]+))', s or ''): + d[m.group(1).lower()] = next((g for g in m.groups()[1:] if g is not None), '') + return d + + +def _maybe_unescape(s): + """模型把脚本整体转义成一行时,救回真实换行/引号。""" + if not isinstance(s, str) or '\\' not in s: return s + if '\n' in s: return s + for a, b in (('\\r\\n', '\n'), ('\\n', '\n'), ('\\t', '\t'), ('\\r', '\r'), ('\\"', '"'), ("\\'", "'")): + s = s.replace(a, b) + return s + + +def _jsonish(v): + if not isinstance(v, str): return v + v = (v or '').strip() + try: return json.loads(v) + except Exception: pass + low = v.lower() + if low in ('true', 'false', 'null', 'none'): return {'true': True, 'false': False}.get(low, None) + if re.fullmatch(r'-?\d+', v): return int(v) + if re.fullmatch(r'-?\d*\.\d+', v): return float(v) + return v + + +def _clean_value(v): + v = v if isinstance(v, str) else str(v) + if v.startswith('\n'): v = v[1:] + if v.endswith('\n'): v = v[:-1] + return _maybe_unescape(v) + + +def _dsml_params(body, spans=None): + """取出全部 ;最后一个即使未闭合也收(截断场景)。""" + args = {} + for m in re.finditer(r']*)>([\s\S]*?)(?=|||\Z)', body): + kv, val = _attrs(m.group(1)), m.group(2) + name = (kv.get('name') or '').strip() + if not name: continue + args[name] = _jsonish(val) if str(kv.get('string', '')).strip().lower() == 'false' else _clean_value(val) + return args + + +def _strip_fence(code): + """去掉模型在标签体内又套的一层 ```lang 围栏,并返回它声明的语言。""" + lang = '' + m = re.match(r'[ \t]*```([A-Za-z0-9_+-]*)[ \t]*\n([\s\S]*?)(?:\n[ \t]*```[ \t]*)?$', code) + if m: + lang, code = (m.group(1) or '').lower(), m.group(2) + else: + code = re.sub(r'(?m)^[ \t]*```[A-Za-z0-9_+-]*[ \t]*$', '', code) + return code, lang + +def _split_code_block(body, tag): + """把 code 标签的体内内容切成 (code_体, 尾巴)。尾巴 = 闭合标签之后到下一个块之前。""" + cm = re.search(r'<[ \t\r\n]*/[ \t\r\n]*' + tag + r'[ \t\r\n]*>', body, re.I) + if cm: return body[:cm.start()], body[cm.end():] + cut = re.search(r'(?m)^[ \t]*<[ \t\r\n]*/?[ \t\r\n]*(?:dsml_|(?:%s)\b)' % '|'.join(_CODE_TAGS), body, re.I) + return (body[:cut.start()], body[cut.start():]) if cut else (body, '') + + +_DSML_PIPES = chr(124) + _DSML_B + chr(0x2016) # | | ‖ +# 一次覆盖:单/双/三竖线、开/闭标签、带空格、截断到 > 之前的长属性(最多 200 字) +_DSML_TAG_RE = re.compile( + '<' + '[ \t]*/?' + '[ \t]*' + '[' + _DSML_PIPES + ']{1,3}' + '[ \t]*DSML[ \t]*' + + '[' + _DSML_PIPES + ']{1,3}' + '[^<>]{0,200}?>', re.I) + + +def _strip_dsml_tags(s): + if not s or 'dsml' not in s.lower(): return s + s = re.sub(r']*>', '', s) + return _DSML_TAG_RE.sub('', s) + + +class DsmlStreamFilter: + """分块安全地剥掉 DSML 标记:把一个还没等到 '>' 的尾部 '<...' 扣住, + 这样被 SSE chunk 切开的标记也能整段删除(单块正则做不到)。""" + HOLD_MAX = 64 + def __init__(self): self.buf = '' + def feed(self, chunk): + if not chunk: return '' + self.buf += chunk + cut = self.buf.rfind('<') + hold = '' + if cut >= 0 and '>' not in self.buf[cut:] and len(self.buf) - cut <= self.HOLD_MAX: + hold, self.buf = self.buf[cut:], self.buf[:cut] + out = _strip_dsml_tags(self.buf) + self.buf = hold + return out + def flush(self): + out = _strip_dsml_tags(self.buf); self.buf = '' + return out + + +_CODE_SIGNAL = re.compile( + r'(?:^|[\s:(=;&|])(?:import|from|def|class|print|echo|ls|cd|cat|grep|curl|pip|npm|git|' + r'python|bash|sh|sudo|rm|cp|mv|mkdir|export|for|if|while|return|await|async)\b' + r'|[=(){}\[\]$#`\\;|&<>]', re.I) + + +def _looks_like_prose(body): + """体内是"讲解文字"而不是代码 → True。用于否决正文里被讨论的标签。 + + 真实误判样本:`引擎不会输出 这种标记…` + (带 cwd 属性,但体内全是中文,绝不能当脚本执行)。真实代码很少出现大段中文散文。 + """ + s = (body or '').strip() + if not s: return False + cjk = sum(1 for ch in s if '\u4e00' <= ch <= '\u9fff') + nonspace = sum(1 for ch in s if not ch.isspace()) + if not nonspace or cjk / nonspace <= 0.5: return False + # 有代码信号(括号/赋值/命令词)就不算散文 + return not _CODE_SIGNAL.search(s) + + +def _code_span_ranges(text): + """行内代码 / 围栏代码块的区间。这些区域里的标签是"被引用的示例",不是调用。""" + spans = [] + for m in re.finditer(r'```[\s\S]*?```', text): spans.append((m.start(), m.end())) + for m in re.finditer(r'`[^`\n]*`', text): spans.append((m.start(), m.end())) + return spans + + +def _in_spans(pos, spans): + return any(s <= pos < e for s, e in spans) + + +def _code_candidates(t): + """找出所有"像真调用"的 code 标签候选。 + + 不能只认行首:真实日志里既有行首的,也有 "...先跑一段 " 这种 + 跟在正文后面的;也不能全认:正文里"你可以用 来跑脚本"这种讲解必须放过。 + 判定证据(任一即可):行首 / 带 script|type|cwd|lang 属性 / 体内有 ` ``);② 体内大段中文散文。 + """ + out = [] + spans = _code_span_ranges(t) + for m in _CODE_ANY.finditer(t): + if _in_spans(m.start(), spans): continue # 被引用的示例,不执行 + tag = m.group(1).lower() + attrs = _attrs(m.group(2) or '') + line_start = not t[t.rfind('\n', 0, m.start()) + 1:m.start()].strip() + body_end = _CODE_ANY.search(t, m.end()) + body_all = t[m.end():body_end.start() if body_end else len(t)] + body, _tail = _split_code_block(body_all, tag) + closed = bool(re.search(r'<[ \t\r\n]*/[ \t\r\n]*' + tag + r'[ \t\r\n]*>', body_all, re.I)) + evid = line_start or any(k in attrs for k in ('script', 'type', 'lang', 'language', 'cwd')) + if not evid and re.search(r'<[ \t\r\n]*parameter\b[^<>]*name\s*=', body, re.I): evid = True + if not evid and re.match(r'[ \t\r\n]*(?:script|python|bash|shell)\s*[::]', body, re.I): evid = True + if not evid and closed and '\n' in body.strip(): evid = True + # 行内且未闭合、也没有 :这是正文里"被讨论的标签"(`… 不会输出 这种`) + if evid and not line_start and not closed and '{"script": "ls -l"} + # (不能当脚本执行,否则会被二次包装成 {"script": "{\"script\": ...}"}) + stripped_body = _strip_dsml_tags(body).strip() + if stripped_body.startswith('{') and not args: + try: + obj = json.loads(stripped_body) + if isinstance(obj, dict) and obj: + args = obj + body = '' + except Exception: pass + raw = _strip_dsml_tags(body) + raw = re.sub(r'^[ \t\r\n]*(?:script|python|bash|shell)\s*[::][ \t]*', '', raw, flags=re.I) + inner = re.search(r'<[ \t]*script\b[^<>]*>([\s\S]*?)(?:<[ \t]*/[ \t]*script[ \t]*>|$)', body, re.I) + code, fence_lang = _strip_fence(inner.group(1) if inner else raw) + code = _maybe_unescape(code).strip('\n') + src = args.get('script') or args.get('code') + if isinstance(src, str) and src.strip(): code = _maybe_unescape(src).strip('\n') + if not code.strip(): continue + lang = (at.get('type') or at.get('lang') or at.get('language') or '').strip().lower() + if (at.get('script') or '').strip().lower() in ('python', 'py', 'bash', 'shell', 'sh', 'zsh', 'powershell'): + lang = at['script'].strip().lower() + claimed = str(args.get('type') or '').strip().lower() + if claimed in ('python', 'py', 'bash', 'shell', 'sh', 'zsh', 'powershell', 'ps1'): lang = claimed + if not lang: lang = fence_lang or _guess_lang(code) + out = {'script': code} + if lang not in ('python', 'py'): + out['type'] = 'bash' if lang in ('bash', 'shell', 'sh', 'zsh') else ('powershell' if lang in ('powershell', 'ps1', 'pwsh') else ('bash' if _guess_lang(code) == 'shell' else lang)) + for k in ('cwd', 'timeout', 'maxlen'): + if args.get(k) not in (None, ''): out[k] = _jsonish(args[k]) if k != 'cwd' else args[k] + if at.get('cwd'): out['cwd'] = at['cwd'] + if str(at.get('timeout', '')).strip().isdigit() and 'timeout' not in out: out['timeout'] = int(at['timeout']) + calls.append(MockToolCall('code_run', out)) + return calls + + +_BARE_TAG_RE = re.compile(r'<([A-Za-z_][\w-]*)\b[^<>]*?(?:/>|>([\s\S]*?))') + + +def _bare_child_args(body): + """把"裸子标签"形态的参数体转成 dict(上游 #837 第5种形态)。 + + + which? + ab + + → {'question': 'which?', 'candidates': ['a', 'b']} + + 只认"整体就是标签对"的体:去掉所有标签后不应有非空白文本, + 否则(例如裸 Python 脚本里恰好有 <...>)直接放弃,避免把代码当参数。 + """ + if not body or not body.lstrip().startswith('<'): return {} + if 'ab → ['a','b'] + else: + val = sub + if key in out: + out[key] = (out[key] if isinstance(out[key], list) else [out[key]]) + [val] + else: + out[key] = val + if not seen or body[pos:].strip(): return {} + return out + + +_DEFAULT_TOOL_NAMES = frozenset({ + 'code_run', 'code_exec', 'ask_user', 'web_scan', 'web_execute_js', + 'file_read', 'file_write', 'file_patch', 'update_working_checkpoint', + 'start_long_term_update', +}) + + +def _bare_tag_calls(t, names): + """扫"工具名 + 裸子标签参数"的调用(..)。 + + 只认已知工具名,且参数体必须是纯净的标签对结构(见 _bare_child_args), + 这样正文里出现的 HTML 片段、裸脚本都不会被误判。容忍标签名后的多余引号 + ( 这种真实泄漏形态)。 + """ + calls = [] + for name in names: + pat = re.compile(r'<[ \t]*' + re.escape(name) + r'[ \t]*["\']?[ \t]*>([\s\S]*?)<[ \t]*/[ \t]*' + + re.escape(name) + r'[ \t]*>', re.I) + for m in pat.finditer(t): + body = m.group(1) + args = _bare_child_args(body) + if args: calls.append((m.start(), m.end(), MockToolCall(name, args))) + return calls + + +def _strip_xml_code(t): + """删掉已被抢救的 code 块(从开标签到闭合标签/块尾,连带尾巴上挂的 dsml 参数)。""" + spans = [(c['start'], c['body_end']) for c in _code_candidates(t)] + for s, e in reversed(spans): t = t[:s] + t[e:] + return t + + +def salvage_leaked_tool_calls(text, names=None): + """从泄漏进正文的工具标记里抢救真正的工具调用。返回 (calls, 清理后的正文)。 + + names: 已知工具名集合(可选)。给了才能识别"工具名 + 裸子标签参数"这种形态。 + """ + if not text or len(text) < 8: return [], text + if not leaked_markup_kind(text, names): return [], text # 标记只在代码块/行内代码里 → 保留原样 + calls, t = [], text + try: + c = _canon_dsml(t) + if ']*)>([\s\S]*?)(?=]*>[\s\S]*?(?=.. + bare = _bare_tag_calls(t, names or _DEFAULT_TOOL_NAMES) + if bare: + before = len(calls) + for s, e, call in sorted(bare, key=lambda x: x[0], reverse=True): + calls.append(call) + t = t[:s] + t[e:] + if len(calls) > before: + t = _strip_dsml_tags(t) + t = _strip_dsml_tags(t) + if calls: + t = re.sub(r'(?m)^[ \t]*[{}][ \t]*$', '', t) # 残留的 JSON 花括号 + t = re.sub(r'(?m)^[ \t]*(?:```|~~~)[ \t]*$', '', t) + t = re.sub(r'\n{3,}', '\n\n', t).strip() + except Exception: + return [], text + return calls, t + + +def leaked_markup_incomplete(text): + """参数是否被截断(max_tokens / 流中断)。只有"最后一个参数体没有闭合"才算, + 因为缺 只要能取到完整 parameter 就不影响执行。""" + if not text: return True + t = _canon_dsml(text) + opens = list(re.finditer(r']*>', t)) + if opens: + return '' not in t[opens[-1].end():] + cands = _code_candidates(t) # 已排除行内/围栏引用与中文散文 + if cands: + found = cands[-1] + body = t[found['end']:found['body_end']] + if re.search(r'<[ \t\r\n]*/[ \t\r\n]*' + found['tag'] + r'[ \t\r\n]*>', body, re.I): + return False + return not re.search(r'<[ \t\r\n]*/[ \t\r\n]*' + found['tag'] + r'[ \t\r\n]*>', + t[found['body_end']:], re.I) + return False + + +def leaked_markup_kind(text, names=None): + """正文里是否残留工具调用标记(代码块/行内代码内不算)。返回类别字符串,无泄漏返回 ''。""" + if not text: return '' + body = _strip_code_spans(text) + if re.search('[' + _DSML_B + r'|]{1,3}\s*DSML\s*[' + _DSML_B + r'|]{1,3}', body, re.I): return 'DSML' + if re.search(r'antml\s*:\s*(?:invoke|parameter|function_calls)\b', body, re.I): return 'antml' + if _bare_tag_calls(body, names or _DEFAULT_TOOL_NAMES): return '<%s>' % next( + iter(n for n in (names or _DEFAULT_TOOL_NAMES) + if re.search(r'<' + re.escape(n) + r'[ \t]*["\']?[ \t]*>', body, re.I)), 'tool') + m = _CODE_OPEN.search(body) or _REG_OPEN.search(body) + if m: + tag = m.group(1).lower() + if re.search(r'<[ \t\r\n]*/[ \t\r\n]*' + tag + r'[ \t\r\n]*>', body, re.I): return '<%s>' % tag + if len(body[m.end():].strip()) > 20: return '<%s> (unclosed)' % tag + # 行内真泄漏("先跑一段 ls"):仅当存在可执行候选时才算泄漏, + # 否则正文里被讨论/引用的标签会引发假重试。 + if _code_candidates(body): return '' + if re.search(r'<[ \t\r\n]*(function_calls|tool_calls)\b', body, re.I): return '' + if re.search(r'<[ \t\r\n]*invoke\b[^<>]*name\s*=', body, re.I): return '' + if re.search(r'<[ \t\r\n]*parameter\b[^<>]*name\s*=', body, re.I): return '' + return '' + + +def salvaged_call_problem(call): + """抢救出的调用是否可安全执行。返回错误描述,'' 表示没问题。 + 宁可让模型重发,也不能把压平成一行/半截的脚本直接跑起来。""" + name = getattr(getattr(call, 'function', None), 'name', '') or '' + if not name: return '工具名为空' + try: args = json.loads(call.function.arguments or '{}') + except Exception as e: return '参数不是合法 JSON: %s' % e + if not isinstance(args, dict): return '参数不是对象' + if name in ('code_run', 'code_exec'): + code = args.get('script') or args.get('code') or '' + if not str(code).strip(): return 'script 为空' + if (args.get('type') or 'python') in ('python', 'py'): + try: ast.parse(code) + except SyntaxError as e: return 'python 语法错误(第 %s 行: %s)' % (e.lineno, e.msg) + elif name == 'file_read': + if not args.get('path'): return 'path 为空' + return '' + + class ToolClient: def __init__(self, backend, auto_save_tokens=True): self.backend = backend @@ -1064,6 +1525,9 @@ def _parse_text_tool_calls(content): if name: tcs.append(MockToolCall(name, args)) except: pass if tcs: content = re.sub(_xp, "", content, flags=re.DOTALL).strip() + # 最后一道:救回泄漏成正文的 DSML / 标记 + _salv, _rest = salvage_leaked_tool_calls(content) + if _salv: tcs.extend(_salv); content = _rest return tcs, content def _ensure_text_block(blocks):