diff --git a/__tests__/extract-search-terms-cjk.test.ts b/__tests__/extract-search-terms-cjk.test.ts new file mode 100644 index 000000000..7bd4ea0a4 --- /dev/null +++ b/__tests__/extract-search-terms-cjk.test.ts @@ -0,0 +1,56 @@ +/** + * `extractSearchTerms` must surface terms from ideographic-script queries. + * + * Regression for: the ASCII-only `[^a-zA-Z0-9]` word split erased every + * character of a Chinese/Japanese/Korean query, so it returned `[]` and + * `explore` reported "No relevant code found" for symbols named in those + * scripts — even though the FTS index (default unicode61 tokenizer) stores + * them and `query`/`node` find them. The ASCII path is intentionally left + * unchanged; these cases are recovered additively. + */ + +import { describe, it, expect } from 'vitest'; +import { extractSearchTerms } from '../src/search/query-utils'; + +describe('extractSearchTerms — ideographic (CJK) queries', () => { + it('surfaces a whole Chinese symbol name (was [] → explore found nothing)', () => { + expect(extractSearchTerms('寻路服务')).toContain('寻路服务'); + }); + + it('keeps a two-character Chinese word (below the ASCII ≥3 floor)', () => { + expect(extractSearchTerms('寻路')).toContain('寻路'); + }); + + it('splits whitespace-separated Chinese terms into separate terms', () => { + const terms = extractSearchTerms('寻路服务 自动寻路模块'); + expect(terms).toContain('寻路服务'); + expect(terms).toContain('自动寻路模块'); + }); + + it('pulls the ideographic run out of a mixed Latin+Han identifier', () => { + const terms = extractSearchTerms('NavMeshAgent类型'); + expect(terms).toContain('navmeshagent'); // ASCII compound — unchanged + expect(terms).toContain('类型'); // Han run — previously dropped + }); + + it('supports Japanese (Kanji) and Korean (Hangul) runs', () => { + expect(extractSearchTerms('経路探索')).toContain('経路探索'); + expect(extractSearchTerms('길찾기')).toContain('길찾기'); + }); +}); + +describe('extractSearchTerms — ASCII behaviour is unchanged', () => { + it('splits camelCase into its parts', () => { + const terms = extractSearchTerms('getUserName'); + expect(terms).toContain('user'); + expect(terms).toContain('name'); + }); + + it('still drops stop words and sub-3-char tokens', () => { + const terms = extractSearchTerms('the id of a UserService'); + expect(terms).not.toContain('the'); + expect(terms).not.toContain('of'); + expect(terms).not.toContain('id'); + expect(terms).toContain('service'); + }); +}); diff --git a/src/search/query-utils.ts b/src/search/query-utils.ts index 1a7b121fc..98b7601c1 100644 --- a/src/search/query-utils.ts +++ b/src/search/query-utils.ts @@ -141,6 +141,13 @@ export function getStemVariants(term: string): string[] { return [...variants].filter(v => v.length >= 3 && v !== t); } +/** + * Ideographic scripts (Han, Hiragana, Katakana, Hangul) are not space-delimited, + * so a run of them is one "word" that the ASCII whitespace/underscore/camelCase + * tokenization below can't see. Matched as maximal runs and surfaced as terms. + */ +const IDEOGRAPHIC_RUN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu; + /** * Extract meaningful search terms from a natural language query. * Splits camelCase, PascalCase, snake_case, SCREAMING_SNAKE, and dot.notation @@ -193,6 +200,21 @@ export function extractSearchTerms(query: string, options?: { stems?: boolean }) tokens.add(lower); } + // The split above is ASCII-only ([^a-zA-Z0-9]), so a query written in an + // ideographic script — Chinese, Japanese, Korean — treats EVERY character as a + // separator and yields zero terms. That made `explore` return "No relevant code + // found" for symbols named in those scripts, even though they ARE indexed and + // `query`/`node` (which pass the raw string to FTS) find them. Recover each + // ideographic run as its own term: FTS5's default unicode61 tokenizer keeps + // such a run as a single token, so a whole-run term matches exactly the way + // query/node already do. Floor of 2 rather than the ≥3 used to strip short + // ASCII noise — an ideographic word is 1–2 characters (e.g. "寻路" = pathfinding). + for (const run of query.match(IDEOGRAPHIC_RUN_RE) ?? []) { + const lower = run.toLowerCase(); + if (lower.length < 2) continue; + tokens.add(lower); + } + // Generate stem variants for broader FTS matching. // "caching" → "cache" finds CacheBuilder; "eviction" → "evict" finds evictEntries. // Also enables co-occurrence dampening by increasing term count above 1.