From 7dca5bad4890ff5e2425117cab521ad7d5309ebd Mon Sep 17 00:00:00 2001
From: yimingll <769197572@qq.com>
Date: Sun, 12 Jul 2026 21:50:20 +0800
Subject: [PATCH] fix(search): recover ideographic (CJK) query terms in
 extractSearchTerms
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`explore` returned "No relevant code found" for a query written in Chinese /
Japanese / Korean, even though those symbols are indexed and `query` / `node`
find them.

Root cause: extractSearchTerms() tokenizes with an ASCII-only split
(`normalised.split(/[^a-zA-Z0-9]+/)`). For an all-ideographic query every
character is a separator, so it returns []. context/index.ts then skips the
text search (`if (searchTerms.length > 0)`) and explore surfaces nothing.

Fix: after the existing ASCII pass (left untouched), additively recover each
ideographic run (Han/Hiragana/Katakana/Hangul) as a term. FTS5's default
unicode61 tokenizer already stores such a run as a single token, so a
whole-run term matches exactly the way query/node do. Two-char floor since an
ideographic word is 1-2 characters ("寻路" = pathfinding).

Adds __tests__/extract-search-terms-cjk.test.ts.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 __tests__/extract-search-terms-cjk.test.ts | 56 ++++++++++++++++++++++
 src/search/query-utils.ts                  | 22 +++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 __tests__/extract-search-terms-cjk.test.ts

diff --git a/__tests__/extract-search-terms-cjk.test.ts b/__tests__/extract-search-terms-cjk.test.ts
new file mode 100644
index 000000000..7bd4ea0a4
--- /dev/null
+++ b/__tests__/extract-search-terms-cjk.test.ts
@@ -0,0 +1,56 @@
+/**
+ * `extractSearchTerms` must surface terms from ideographic-script queries.
+ *
+ * Regression for: the ASCII-only `[^a-zA-Z0-9]` word split erased every
+ * character of a Chinese/Japanese/Korean query, so it returned `[]` and
+ * `explore` reported "No relevant code found" for symbols named in those
+ * scripts — even though the FTS index (default unicode61 tokenizer) stores
+ * them and `query`/`node` find them. The ASCII path is intentionally left
+ * unchanged; these cases are recovered additively.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { extractSearchTerms } from '../src/search/query-utils';
+
+describe('extractSearchTerms — ideographic (CJK) queries', () => {
+  it('surfaces a whole Chinese symbol name (was [] → explore found nothing)', () => {
+    expect(extractSearchTerms('寻路服务')).toContain('寻路服务');
+  });
+
+  it('keeps a two-character Chinese word (below the ASCII ≥3 floor)', () => {
+    expect(extractSearchTerms('寻路')).toContain('寻路');
+  });
+
+  it('splits whitespace-separated Chinese terms into separate terms', () => {
+    const terms = extractSearchTerms('寻路服务 自动寻路模块');
+    expect(terms).toContain('寻路服务');
+    expect(terms).toContain('自动寻路模块');
+  });
+
+  it('pulls the ideographic run out of a mixed Latin+Han identifier', () => {
+    const terms = extractSearchTerms('NavMeshAgent类型');
+    expect(terms).toContain('navmeshagent'); // ASCII compound — unchanged
+    expect(terms).toContain('类型');          // Han run — previously dropped
+  });
+
+  it('supports Japanese (Kanji) and Korean (Hangul) runs', () => {
+    expect(extractSearchTerms('経路探索')).toContain('経路探索');
+    expect(extractSearchTerms('길찾기')).toContain('길찾기');
+  });
+});
+
+describe('extractSearchTerms — ASCII behaviour is unchanged', () => {
+  it('splits camelCase into its parts', () => {
+    const terms = extractSearchTerms('getUserName');
+    expect(terms).toContain('user');
+    expect(terms).toContain('name');
+  });
+
+  it('still drops stop words and sub-3-char tokens', () => {
+    const terms = extractSearchTerms('the id of a UserService');
+    expect(terms).not.toContain('the');
+    expect(terms).not.toContain('of');
+    expect(terms).not.toContain('id');
+    expect(terms).toContain('service');
+  });
+});
diff --git a/src/search/query-utils.ts b/src/search/query-utils.ts
index 1a7b121fc..98b7601c1 100644
--- a/src/search/query-utils.ts
+++ b/src/search/query-utils.ts
@@ -141,6 +141,13 @@ export function getStemVariants(term: string): string[] {
   return [...variants].filter(v => v.length >= 3 && v !== t);
 }
 
+/**
+ * Ideographic scripts (Han, Hiragana, Katakana, Hangul) are not space-delimited,
+ * so a run of them is one "word" that the ASCII whitespace/underscore/camelCase
+ * tokenization below can't see. Matched as maximal runs and surfaced as terms.
+ */
+const IDEOGRAPHIC_RUN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+
 /**
  * Extract meaningful search terms from a natural language query.
  * Splits camelCase, PascalCase, snake_case, SCREAMING_SNAKE, and dot.notation
@@ -193,6 +200,21 @@ export function extractSearchTerms(query: string, options?: { stems?: boolean })
     tokens.add(lower);
   }
 
+  // The split above is ASCII-only ([^a-zA-Z0-9]), so a query written in an
+  // ideographic script — Chinese, Japanese, Korean — treats EVERY character as a
+  // separator and yields zero terms. That made `explore` return "No relevant code
+  // found" for symbols named in those scripts, even though they ARE indexed and
+  // `query`/`node` (which pass the raw string to FTS) find them. Recover each
+  // ideographic run as its own term: FTS5's default unicode61 tokenizer keeps
+  // such a run as a single token, so a whole-run term matches exactly the way
+  // query/node already do. Floor of 2 rather than the ≥3 used to strip short
+  // ASCII noise — an ideographic word is 1–2 characters (e.g. "寻路" = pathfinding).
+  for (const run of query.match(IDEOGRAPHIC_RUN_RE) ?? []) {
+    const lower = run.toLowerCase();
+    if (lower.length < 2) continue;
+    tokens.add(lower);
+  }
+
   // Generate stem variants for broader FTS matching.
   // "caching" → "cache" finds CacheBuilder; "eviction" → "evict" finds evictEntries.
   // Also enables co-occurrence dampening by increasing term count above 1.