fix(search): make explore find symbols named in CJK (Chinese/Japanese/Korean)

__tests__/extract-search-terms-cjk.test.ts

@@ -0,0 +1,56 @@
+/**
+ * `extractSearchTerms` must surface terms from ideographic-script queries.
+ *
+ * Regression for: the ASCII-only `[^a-zA-Z0-9]` word split erased every
+ * character of a Chinese/Japanese/Korean query, so it returned `[]` and
+ * `explore` reported "No relevant code found" for symbols named in those
+ * scripts — even though the FTS index (default unicode61 tokenizer) stores
+ * them and `query`/`node` find them. The ASCII path is intentionally left
+ * unchanged; these cases are recovered additively.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { extractSearchTerms } from '../src/search/query-utils';
+
+describe('extractSearchTerms — ideographic (CJK) queries', () => {
+  it('surfaces a whole Chinese symbol name (was [] → explore found nothing)', () => {
+    expect(extractSearchTerms('寻路服务')).toContain('寻路服务');
+  });
+
+  it('keeps a two-character Chinese word (below the ASCII ≥3 floor)', () => {
+    expect(extractSearchTerms('寻路')).toContain('寻路');
+  });
+
+  it('splits whitespace-separated Chinese terms into separate terms', () => {
+    const terms = extractSearchTerms('寻路服务 自动寻路模块');
+    expect(terms).toContain('寻路服务');
+    expect(terms).toContain('自动寻路模块');
+  });
+
+  it('pulls the ideographic run out of a mixed Latin+Han identifier', () => {
+    const terms = extractSearchTerms('NavMeshAgent类型');
+    expect(terms).toContain('navmeshagent'); // ASCII compound — unchanged
+    expect(terms).toContain('类型');          // Han run — previously dropped
+  });
+
+  it('supports Japanese (Kanji) and Korean (Hangul) runs', () => {
+    expect(extractSearchTerms('経路探索')).toContain('経路探索');
+    expect(extractSearchTerms('길찾기')).toContain('길찾기');
+  });
+});
+
+describe('extractSearchTerms — ASCII behaviour is unchanged', () => {
+  it('splits camelCase into its parts', () => {
+    const terms = extractSearchTerms('getUserName');
+    expect(terms).toContain('user');
+    expect(terms).toContain('name');
+  });
+
+  it('still drops stop words and sub-3-char tokens', () => {
+    const terms = extractSearchTerms('the id of a UserService');
+    expect(terms).not.toContain('the');
+    expect(terms).not.toContain('of');
+    expect(terms).not.toContain('id');
+    expect(terms).toContain('service');
+  });
+});

src/search/query-utils.ts

@@ -141,6 +141,13 @@ export function getStemVariants(term: string): string[] {
   return [...variants].filter(v => v.length >= 3 && v !== t);
 }
 
+/**
+ * Ideographic scripts (Han, Hiragana, Katakana, Hangul) are not space-delimited,
+ * so a run of them is one "word" that the ASCII whitespace/underscore/camelCase
+ * tokenization below can't see. Matched as maximal runs and surfaced as terms.
+ */
+const IDEOGRAPHIC_RUN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+
 /**
  * Extract meaningful search terms from a natural language query.
  * Splits camelCase, PascalCase, snake_case, SCREAMING_SNAKE, and dot.notation
@@ -193,6 +200,21 @@ export function extractSearchTerms(query: string, options?: { stems?: boolean })
     tokens.add(lower);
   }
 
+  // The split above is ASCII-only ([^a-zA-Z0-9]), so a query written in an
+  // ideographic script — Chinese, Japanese, Korean — treats EVERY character as a
+  // separator and yields zero terms. That made `explore` return "No relevant code
+  // found" for symbols named in those scripts, even though they ARE indexed and
+  // `query`/`node` (which pass the raw string to FTS) find them. Recover each
+  // ideographic run as its own term: FTS5's default unicode61 tokenizer keeps
+  // such a run as a single token, so a whole-run term matches exactly the way
+  // query/node already do. Floor of 2 rather than the ≥3 used to strip short
+  // ASCII noise — an ideographic word is 1–2 characters (e.g. "寻路" = pathfinding).
+  for (const run of query.match(IDEOGRAPHIC_RUN_RE) ?? []) {
+    const lower = run.toLowerCase();
+    if (lower.length < 2) continue;
+    tokens.add(lower);
+  }
+
   // Generate stem variants for broader FTS matching.
   // "caching" → "cache" finds CacheBuilder; "eviction" → "evict" finds evictEntries.
   // Also enables co-occurrence dampening by increasing term count above 1.