From 2584bb868856e53cbf120a9fb3b5fc5397697e58 Mon Sep 17 00:00:00 2001 From: nonoqing Date: Wed, 28 Oct 2026 17:25:08 +0800 Subject: [PATCH] fix(router): handle decoded logprobs and report probe failures Accept decoded classifier candidates while preserving token alignment, two-class confidence calculation, and strict rejection of invalid probabilities. Validate the Router API format before constructing a classifier endpoint and show safe localized failure details in the model-routing connection check. Keep the Smart mode release gate disabled; exclude the local dev override. Validation: Router adapter 11 tests, focused Core 6 tests, frontend routing 211 tests, i18n audit, type checking, and diff checks passed. The broader check:web audit is blocked by an existing untracked MiniApp registry mismatch. --- .../adapters/ai-adapters/src/round_router.rs | 144 +++++++++++++++++- .../round_model_routing/configuration.rs | 38 +++++ .../components/ModelRouterSettings.test.tsx | 75 +++++++++ .../config/components/ModelRouterSettings.tsx | 30 +++- .../src/locales/en-US/settings/models.json | 9 ++ .../src/locales/zh-CN/settings/models.json | 9 ++ .../src/locales/zh-TW/settings/models.json | 9 ++ 7 files changed, 309 insertions(+), 5 deletions(-) diff --git a/src/crates/adapters/ai-adapters/src/round_router.rs b/src/crates/adapters/ai-adapters/src/round_router.rs index ead3e03260..c2ff981954 100644 --- a/src/crates/adapters/ai-adapters/src/round_router.rs +++ b/src/crates/adapters/ai-adapters/src/round_router.rs @@ -384,10 +384,28 @@ fn token_index_at_byte_offset( fn find_token_logprob(candidates: &[OpenAiTokenLogprob], token_id: u32) -> Option { let expected = format!("token_id:{token_id}"); - candidates + // Some compatible services decode top candidates despite return_tokens_as_token_ids. + // Accept only the classifier's two known token spellings; the generated + // token ID and semantic label position are still validated by the caller. + let decoded = match token_id { + SIMPLE_TOKEN_ID => "simple", + NON_SIMPLE_FIRST_TOKEN_ID => "non", + _ => return None, + }; + let mut observed_logprob = None; + for candidate in candidates .iter() - .find(|candidate| candidate.token == expected) - .map(|candidate| candidate.logprob) + .filter(|candidate| candidate.token == expected || candidate.token == decoded) + { + if !candidate.logprob.is_finite() || candidate.logprob > 0.0 { + return None; + } + if observed_logprob.is_some_and(|previous| previous != candidate.logprob) { + return None; + } + observed_logprob = Some(candidate.logprob); + } + observed_logprob } fn parse_router_route(raw_output: &str) -> Result<(bool, bool), RouterClientError> { @@ -758,4 +776,124 @@ mod tests { assert!(extract_simple_probability(&choice, raw_output, true).is_none()); } + + fn classifier_choice(candidates: &[(&str, f64)]) -> OpenAiChatChoice { + OpenAiChatChoice { + message: OpenAiResponseMessage { content: None }, + logprobs: Some(OpenAiChatLogprobs { + content: Some(vec![ + OpenAiChatLogprobContent { + bytes: Some(br#"{"simple_type":""#.to_vec()), + top_logprobs: Vec::new(), + }, + OpenAiChatLogprobContent { + bytes: Some(b"simple".to_vec()), + top_logprobs: candidates + .iter() + .map(|(token, logprob)| OpenAiTokenLogprob { + token: (*token).into(), + logprob: *logprob, + }) + .collect(), + }, + OpenAiChatLogprobContent { + bytes: Some(br#""}"#.to_vec()), + top_logprobs: Vec::new(), + }, + ]), + }), + token_ids: Some(vec![1, SIMPLE_TOKEN_ID, 2]), + } + } + + #[test] + fn decoded_classifier_candidates_preserve_two_class_confidence() { + for candidates in [ + vec![("simple", -0.2), ("non", -1.4)], + vec![("token_id:22944", -0.2), ("token_id:6280", -1.4)], + vec![("simple", -0.2), ("token_id:6280", -1.4)], + vec![ + ("simple", -0.2), + ("token_id:22944", -0.2), + ("non", -1.4), + ("non", -1.4), + ], + ] { + let choice = classifier_choice(&candidates); + let probability = + extract_simple_probability(&choice, r#"{"simple_type":"simple"}"#, true).unwrap(); + assert!((probability - 0.768_524_783_5).abs() < 1e-9); + } + let mut choice = classifier_choice(&[("simple", -1.4), ("non", -0.2)]); + choice.token_ids.as_mut().unwrap()[1] = NON_SIMPLE_FIRST_TOKEN_ID; + let tokens = choice.logprobs.as_mut().unwrap().content.as_mut().unwrap(); + tokens[1].bytes = Some(b"non".to_vec()); + tokens[2].bytes = Some(br#"_simple"}"#.to_vec()); + let probability = + extract_simple_probability(&choice, r#"{"simple_type":"non_simple"}"#, false).unwrap(); + assert!((probability - 0.231_475_216_5).abs() < 1e-9); + } + + #[test] + fn decoded_classifier_candidates_require_generated_id_and_label_position() { + let raw = r#"{"simple_type":"simple"}"#; + let mut wrong_id = classifier_choice(&[("simple", -0.2), ("non", -1.4)]); + wrong_id.token_ids.as_mut().unwrap()[1] = NON_SIMPLE_FIRST_TOKEN_ID; + assert!(extract_simple_probability(&wrong_id, raw, true).is_none()); + + let mut wrong_position = classifier_choice(&[("simple", -0.2), ("non", -1.4)]); + let tokens = wrong_position + .logprobs + .as_mut() + .unwrap() + .content + .as_mut() + .unwrap(); + tokens[0].top_logprobs = std::mem::take(&mut tokens[1].top_logprobs); + assert!(extract_simple_probability(&wrong_position, raw, true).is_none()); + + let mut wrong_alignment = classifier_choice(&[("simple", -0.2), ("non", -1.4)]); + wrong_alignment.token_ids.as_mut().unwrap().pop(); + assert!(extract_simple_probability(&wrong_alignment, raw, true).is_none()); + } + + #[test] + fn decoded_classifier_candidates_require_both_exact_classes() { + for candidates in [ + vec![("simple", -0.2)], + vec![("non", -1.4)], + vec![("simple", -0.2), ("non_simple", -1.4)], + vec![(" simple", -0.2), ("non", -1.4)], + vec![("simple", -0.2), ("token_id:999", -1.4)], + ] { + let choice = classifier_choice(&candidates); + assert!( + extract_simple_probability(&choice, r#"{"simple_type":"simple"}"#, true).is_none() + ); + } + } + + #[test] + fn classifier_candidates_reject_conflicting_and_invalid_probabilities() { + for candidates in [ + vec![("simple", -0.2), ("simple", -0.3), ("non", -1.4)], + vec![("simple", -0.2), ("token_id:22944", -0.3), ("non", -1.4)], + vec![("simple", -0.2), ("non", -1.4), ("token_id:6280", -1.5)], + vec![ + ("token_id:22944", -0.2), + ("token_id:22944", -0.3), + ("token_id:6280", -1.4), + ], + vec![("simple", f64::NAN), ("non", -1.4)], + vec![("simple", -0.2), ("non", f64::INFINITY)], + vec![("simple", f64::NEG_INFINITY), ("non", -1.4)], + vec![("simple", 0.1), ("non", -1.4)], + vec![("token_id:22944", -0.2), ("token_id:6280", 0.1)], + ] { + let choice = classifier_choice(&candidates); + assert!( + extract_simple_probability(&choice, r#"{"simple_type":"simple"}"#, true).is_none() + ); + } + } } diff --git a/src/crates/assembly/core/src/agentic/execution/round_model_routing/configuration.rs b/src/crates/assembly/core/src/agentic/execution/round_model_routing/configuration.rs index a45c4dc72a..175c97fbfe 100644 --- a/src/crates/assembly/core/src/agentic/execution/round_model_routing/configuration.rs +++ b/src/crates/assembly/core/src/agentic/execution/round_model_routing/configuration.rs @@ -123,6 +123,14 @@ fn product_service_config(config: &AIConfig) -> OpenBitFunResult Value { json!({"choices": [{ "message": {"content": "{\"simple_type\":\"simple\"}"}, @@ -682,6 +718,7 @@ mod tests { config.models.push(AIModelConfig { id: "router-service".into(), enabled: true, + provider: " OpenAI ".into(), base_url: "https://router.example.invalid/v1/".into(), model_name: "router-fixture".into(), api_key: "fixture-token".into(), @@ -735,6 +772,7 @@ mod tests { config.models.push(AIModelConfig { id: "router-service".into(), enabled: true, + provider: "openai".into(), base_url: "https://router.example.invalid/v1".into(), model_name: "router-fixture".into(), ..Default::default() diff --git a/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.test.tsx b/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.test.tsx index eb68832a88..6a1f97dbc4 100644 --- a/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.test.tsx +++ b/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.test.tsx @@ -399,12 +399,87 @@ describe('Router setup', () => { expect(document.body.textContent).not.toContain('router.connected'); expect(onClose).not.toHaveBeenCalled(); }); + it('prefers a Router connection issue over a separate readiness issue', async () => { + await render(); + vi.mocked(agentAPI.getModelRoutingStatus).mockResolvedValue({ + ...routingStatus(false), + connectionIssue: 'Router returned HTTP 401: api_key=private-secret', + issue: 'Configure an enabled economy model before using smart mode', + }); + await check(); + expectState('router', 'failed'); + expect(row('router').textContent).toContain('router.connectionFailures.authentication'); + expect(row('router').textContent).not.toContain('router.connectionChecks.unconfigured'); + expect(document.body.textContent).not.toContain('private-secret'); + }); + + it.each([ + ['Router requires OpenAI Chat Completions API format; select an OpenAI-compatible classifier service', 'protocol'], + ['Router protocol mismatch: classifier token IDs and logprobs are required', 'protocol'], + ['Router response lacks classifier token logprobs', 'protocol'], + ['Failed to decode Router response: invalid response body', 'protocol'], + ['Router returned HTTP 401', 'authentication'], + ['Router returned HTTP 403', 'authentication'], + ['Router returned HTTP 404', 'addressOrModel'], + ['Invalid Router URL', 'addressOrModel'], + ['Router returned HTTP 429', 'rateLimited'], + ['Router returned HTTP 500', 'serviceUnavailable'], + ['Router returned HTTP 503', 'serviceUnavailable'], + ['Router returned HTTP 418', 'http'], + ['Router deadline expired during classification after queue admission', 'network'], + ['Router request failed: connection refused', 'network'], + ])('localizes a legacy Router issue without displaying its raw contents: %s', async (issue, category) => { + await render(); + vi.mocked(agentAPI.getModelRoutingStatus).mockResolvedValue({ ...routingStatus(false), issue }); + await check(); + expectState('router', 'failed'); + expect(row('router').textContent).toContain(`router.connectionFailures.${category}`); + expect(document.body.textContent).not.toContain(issue); + }); + + it('reports an unconfigured Router without exposing the host diagnostic', async () => { + await render(); + const issue = 'Configure a Router service model in model settings'; + vi.mocked(agentAPI.getModelRoutingStatus).mockResolvedValue({ ...routingStatus(false), issue }); + await check(); + expectState('router', 'failed'); + expect(row('router').textContent).toContain('router.connectionChecks.unconfigured'); + expect(document.body.textContent).not.toContain(issue); + }); + + it('gives safe retry guidance for an unknown connection issue without substituting a readiness issue', async () => { + await render(); + vi.mocked(agentAPI.getModelRoutingStatus).mockResolvedValue({ + ...routingStatus(false), connectionIssue: 'provider body api_key=private-secret /private/host/config', + issue: 'Router returned HTTP 404', + }); + await check(); + expectState('router', 'failed'); + expect(row('router').textContent).toContain('router.checkFailed'); + expect(row('router').textContent).not.toContain('router.connectionFailures.addressOrModel'); + expect(document.body.textContent).not.toContain('private-secret'); + expect(document.body.textContent).not.toContain('/private/host/config'); + }); + + it('does not display stale failure details when the explicit Router probe succeeds', async () => { + await render(); + vi.mocked(agentAPI.getModelRoutingStatus).mockResolvedValue({ + ...routingStatus(true), connectionIssue: 'Router returned HTTP 401', issue: 'private-old-failure', + }); + await check(); + expectState('router', 'connected'); + expect(row('router').querySelector('.model-router-settings__check-detail')).toBeNull(); + expect(row('router').textContent).not.toContain('router.connectionFailures'); + expect(document.body.textContent).not.toContain('private-old-failure'); + }); + it('localizes a rejected protocol check containing a tokenizer file error', async () => { await render(); vi.mocked(agentAPI.getModelRoutingStatus).mockResolvedValueOnce({ apiVersion: 1, available: true, estimatedTokenBudgetSupported: true } as Awaited>) .mockRejectedValueOnce(new Error('Failed to load Router tokenizer from /host/tokenizer.json')); await act(async () => button('router.test').click()); expectState('router', 'failed'); + expect(row('router').textContent).toContain('router.checkFailed'); expect(document.body.textContent).not.toContain('Failed to load Router tokenizer'); expect(onClose).not.toHaveBeenCalled(); }); diff --git a/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.tsx b/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.tsx index 22c7c14d8b..69e66b6a77 100644 --- a/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.tsx +++ b/src/web-ui/src/infrastructure/config/components/ModelRouterSettings.tsx @@ -144,12 +144,38 @@ export function ModelRouterSettings({ models, onClose, onSaved, onManageModels, log.warn('Smart-mode model connection check failed', { role }); } }); + const routerFailureDetail = (issue: unknown): string => { + // Host/provider diagnostics may contain credentials or paths. Only select + // fixed translated advice; never render any part of the diagnostic itself. + const reason = typeof issue === 'string' ? issue : ''; + if (/Router requires OpenAI Chat Completions API format|Router protocol mismatch|Router response lacks classifier token logprobs|Failed to decode Router response|Router (?:output is invalid JSON|response did not contain)/i.test(reason)) { + return t('router.connectionFailures.protocol'); + } + const httpStatus = /\bRouter returned HTTP ([1-5]\d{2})\b/i.exec(reason)?.[1]; + if (httpStatus === '401' || httpStatus === '403') return t('router.connectionFailures.authentication'); + if (httpStatus === '404' || /Invalid Router URL|Router URL must use HTTP\(S\)/i.test(reason)) { + return t('router.connectionFailures.addressOrModel'); + } + if (httpStatus === '429') return t('router.connectionFailures.rateLimited'); + if (httpStatus?.startsWith('5')) return t('router.connectionFailures.serviceUnavailable'); + if (httpStatus) return t('router.connectionFailures.http'); + if (/Router (?:request failed|deadline expired)|\b(?:connection refused|connect error|dns error|timed out)\b/i.test(reason)) { + return t('router.connectionFailures.network'); + } + if (/Configure a Router service model|Configure an enabled (?:primary|economy) model/i.test(reason)) { + return t('router.connectionChecks.unconfigured'); + } + return t('router.checkFailed'); + }; const routerCheck = async () => { try { const status = await agentAPI.getModelRoutingStatus({ probe: true }); - update('router', { state: status.connected ? 'connected' : 'failed' }); + update('router', { + state: status.connected ? 'connected' : 'failed', + detail: status.connected ? undefined : routerFailureDetail(status.connectionIssue ?? status.issue), + }); } catch { - update('router', { state: 'failed' }); + update('router', { state: 'failed', detail: t('router.checkFailed') }); log.warn('Smart-mode Router connection check failed'); } }; diff --git a/src/web-ui/src/locales/en-US/settings/models.json b/src/web-ui/src/locales/en-US/settings/models.json index 58d883394b..dc9010dbce 100644 --- a/src/web-ui/src/locales/en-US/settings/models.json +++ b/src/web-ui/src/locales/en-US/settings/models.json @@ -843,6 +843,15 @@ "failed": "Connection failed", "unconfigured": "No available model configured" }, + "connectionFailures": { + "protocol": "Choose an OpenAI Chat Completions classifier service that returns token IDs and logprobs", + "authentication": "Router authentication failed; check the API key and service permissions", + "addressOrModel": "Check the Router service address and model name; the endpoint or model may not exist", + "rateLimited": "Router requests are rate limited; wait and retry", + "serviceUnavailable": "The Router service is temporarily unavailable; wait and retry", + "http": "The Router returned an unsuccessful HTTP response; check service settings and retry", + "network": "The Router could not be reached or timed out; check the service, network, and timeout settings" + }, "notReady": "Router not ready", "unsupported": "This execution host does not support smart routing", "tokenBudgetUnsupported": "This execution host still requires a tokenizer file; upgrade it before configuring smart routing", diff --git a/src/web-ui/src/locales/zh-CN/settings/models.json b/src/web-ui/src/locales/zh-CN/settings/models.json index 25da907638..f8f44bff1f 100644 --- a/src/web-ui/src/locales/zh-CN/settings/models.json +++ b/src/web-ui/src/locales/zh-CN/settings/models.json @@ -843,6 +843,15 @@ "failed": "连接失败", "unconfigured": "未配置可用模型" }, + "connectionFailures": { + "protocol": "请选择支持 OpenAI Chat Completions 且返回 token IDs 和 logprobs 的分类服务", + "authentication": "Router 认证失败，请检查 API Key 和服务访问权限", + "addressOrModel": "请检查 Router 服务地址和模型名称，接口或模型可能不存在", + "rateLimited": "Router 请求受到限流，请稍后重试", + "serviceUnavailable": "Router 服务暂时不可用，请稍后重试", + "http": "Router 返回非成功的 HTTP 响应，请检查服务设置后重试", + "network": "无法连接 Router 或请求超时，请检查服务、网络和超时设置" + }, "notReady": "路由未就绪", "unsupported": "当前执行端不支持智能路由", "tokenBudgetUnsupported": "当前执行主机仍需要分词表文件，请升级后再配置智能路由", diff --git a/src/web-ui/src/locales/zh-TW/settings/models.json b/src/web-ui/src/locales/zh-TW/settings/models.json index 1d64937ca2..7460e89766 100644 --- a/src/web-ui/src/locales/zh-TW/settings/models.json +++ b/src/web-ui/src/locales/zh-TW/settings/models.json @@ -843,6 +843,15 @@ "failed": "連線失敗", "unconfigured": "尚未設定可用模型" }, + "connectionFailures": { + "protocol": "請選擇支援 OpenAI Chat Completions 且回傳 token IDs 和 logprobs 的分類服務", + "authentication": "Router 驗證失敗，請檢查 API Key 和服務存取權限", + "addressOrModel": "請檢查 Router 服務位址和模型名稱，介面或模型可能不存在", + "rateLimited": "Router 請求受到速率限制，請稍後重試", + "serviceUnavailable": "Router 服務暫時無法使用，請稍後重試", + "http": "Router 回傳非成功的 HTTP 回應，請檢查服務設定後重試", + "network": "無法連線至 Router 或請求逾時，請檢查服務、網路和逾時設定" + }, "notReady": "路由未就緒", "unsupported": "目前執行端不支援智慧路由", "tokenBudgetUnsupported": "目前執行主機仍需要分詞表檔案，請升級後再設定智慧路由",