diff --git a/.changeset/tidy-python-parameters.md b/.changeset/tidy-python-parameters.md new file mode 100644 index 0000000..e9e544f --- /dev/null +++ b/.changeset/tidy-python-parameters.md @@ -0,0 +1,5 @@ +--- +"@vortexlabs/kirari": patch +--- + +Keep Python annotated parameters after commas and unparenthesized lambda parameters inside dictionaries highlighted as variables rather than dictionary keys, including when streaming multiline signatures. diff --git a/packages/kirari/src/languages/python.ts b/packages/kirari/src/languages/python.ts index a275b31..2499415 100644 --- a/packages/kirari/src/languages/python.ts +++ b/packages/kirari/src/languages/python.ts @@ -1,16 +1,17 @@ import { definitionsFor } from '../grammars/definitions.js' import type { FunctionSyntax } from '../grammars/definitions.js' import { scan } from '../tokenize.js' +import type { ScanContext } from '../tokenize.js' import { isSignificant } from '../types.js' import type { Language, Token } from '../types.js' import { resumableScan } from '../resumable.js' -import type { ResumableScan } from '../resumable.js' +import type { PythonScope, ResumableScan } from '../resumable.js' const tripleString = { type: 'string', pattern: /[rRuUbBfF]{0,2}(?:"""(?:\\(?:[\s\S]|$)|(?!""")[^\\])*(?:"""|$)|'''(?:\\(?:[\s\S]|$)|(?!''')[^\\])*(?:'''|$))/ } const fstringOpening = { type: 'python-fstring', pattern: /(?:[fF][rR]?|[rR][fF])(?:"""|'''|"|')/ } type Literal = { kind: 'literal'; quote: string; raw: boolean; format: boolean } -type PythonFrame = { kind: 'expression'; depth: number; root?: true } | Literal +type PythonFrame = { kind: 'expression'; brackets?: PythonScope | undefined; root?: true } | Literal // Scans literal text in an f-string, stopping at a replacement field or the closing quote. const literalChunk = (source: string, start: number, frame: Literal): { end: number; closed: boolean } => { @@ -62,8 +63,9 @@ const scanLanguage = (language: Language): Language => { // Resume points are whitespace at unescaped line breaks, outside f-strings, with more source after them. const scanPython: ResumableScan = (source, language, from, checkpoint) => { const tokens: Token[] = [] - const frames: PythonFrame[] = [{ kind: 'expression', depth: 0, root: true }] - const scanner = scan(source, scanLanguage(language), from?.offset, from?.previous) + const frames: PythonFrame[] = [{ kind: 'expression', brackets: from?.brackets, root: true }] + const context: ScanContext = { parentheses: [] } + const scanner = scan(source, scanLanguage(language), from?.offset, from?.previous, undefined, [], context) let position = from?.offset ?? 0 let resume: number | undefined // Mirrors the scanner's own `previous`, which resuming replaces with a placeholder element. @@ -87,7 +89,7 @@ const scanPython: ResumableScan = (source, language, from, checkpoint) => { if (closed) frames.pop() else if (source[position] === '{') { emit('punctuation', position + 1) - frames.push({ kind: 'expression', depth: 0 }) + frames.push({ kind: 'expression' }) } else if (frame.format && source[position] === '}') { emit('punctuation', position + 1) frames.pop() @@ -97,6 +99,7 @@ const scanPython: ResumableScan = (source, language, from, checkpoint) => { } if (resume !== undefined && yielded) previous = { type: 'tag', value: 'element', start: yielded.start, end: resume } + context.objectKeys = frame.brackets?.value === '{' const result = scanner.next(resume) resume = undefined if (result.done) break @@ -108,7 +111,7 @@ const scanPython: ResumableScan = (source, language, from, checkpoint) => { const merged = last?.type === 'text' // A backslash in source text continues the line; one inside a comment does not. if (!merged || !last.value.endsWith('\\')) { - checkpoint({ offset: merged ? last.start : position, previous }, merged ? tokens.length - 1 : tokens.length) + checkpoint({ offset: merged ? last.start : position, previous, brackets: frame.brackets }, merged ? tokens.length - 1 : tokens.length) } } const opening = token.type === fstringOpening.type && /^(f|fr|rf)("""|'''|"|')$/i.exec(token.value) @@ -117,7 +120,7 @@ const scanPython: ResumableScan = (source, language, from, checkpoint) => { frames.push({ kind: 'literal', quote: opening[2]!, raw: /r/i.test(opening[1]!), format: false }) continue } - if (!frame.root && frame.depth === 0) { + if (!frame.root && !frame.brackets) { if (token.value === '}') { emit('punctuation', token.end) frames.pop() @@ -136,10 +139,9 @@ const scanPython: ResumableScan = (source, language, from, checkpoint) => { continue } } - if (token.type === 'punctuation') { - if (/^[([{]$/.test(token.value)) frame.depth++ - else if (/^[)\]}]$/.test(token.value) && frame.depth > 0) frame.depth-- - } + if (token.type === 'keyword' && token.value === 'lambda' || token.type === 'punctuation' && /^[([{]$/.test(token.value)) { + frame.brackets = { value: token.value, parent: frame.brackets } + } else if (token.type === 'punctuation' && (/^[)\]}]$/.test(token.value) || token.value === ':' && frame.brackets?.value === 'lambda')) frame.brackets = frame.brackets?.parent emit(token.type, token.end) } return tokens diff --git a/packages/kirari/src/resumable.ts b/packages/kirari/src/resumable.ts index 10869e3..1076936 100644 --- a/packages/kirari/src/resumable.ts +++ b/packages/kirari/src/resumable.ts @@ -1,10 +1,14 @@ import type { ScanContext } from './tokenize.js' import type { Language, Token } from './types.js' +export type PythonScope = { readonly value: string; readonly parent: PythonScope | undefined } + /** A root-level point where scanning can restart with the same results as a full scan. */ export type ResumePoint = { readonly offset: number readonly previous: Token | undefined + /** Immutable Python delimiters and lambda parameter scopes shared with later checkpoints. */ + readonly brackets?: PythonScope | undefined /** Script scan state at the point, for grammars that embed JavaScript. */ readonly context?: ScanContext } diff --git a/packages/kirari/src/tokenize.ts b/packages/kirari/src/tokenize.ts index b84c822..5e66f7e 100644 --- a/packages/kirari/src/tokenize.ts +++ b/packages/kirari/src/tokenize.ts @@ -21,9 +21,11 @@ import type { FunctionLookupCache } from './grammars/functions.js' import { canStartRegex, escapedIdentifierEnd, nextVisibleChar, regexEnd, templateChunk } from './scanner/lexical.js' export type Frame = { kind: 'template'; opening: boolean } | { kind: 'expression'; depth: number } -/** Lexical state for JavaScript-like languages, carried between scanner steps. */ +/** Lexical state carried between scanner steps, including context supplied by embedded grammars. */ export type ScanContext = { parentheses: { control: boolean; braces: number }[] + /** Unset keeps the generic comma-separated key heuristic. */ + objectKeys?: boolean braces?: number afterControl?: boolean forAwait?: boolean @@ -240,7 +242,7 @@ export function* scan(source: string, language: Language, start = 0, previous?: else if (types.has(name) || previous?.value === 'class' || typed && (context.annotation || previous?.type === 'keyword' && /^(type|interface|enum)$/.test(previous.value))) type = 'type' else { const following = nextVisibleChar(source, end) - if (following === ':' && (previous?.value === '{' || previous?.value === ',' || typed && method && previous?.value === ';')) type = 'property' // { name: value } or TypeScript members + if (following === ':' && (previous?.value === '{' || previous?.value === ',' && context.objectKeys !== false || typed && method && previous?.value === ';')) type = 'property' // { name: value } or TypeScript members else if (following === '(') type = 'function' // name(…) } // Look ahead for definitions such as `name = () => …` or `name: function () {}`. diff --git a/packages/kirari/test/corpus/fixtures.ts b/packages/kirari/test/corpus/fixtures.ts index f8e6494..319f031 100644 --- a/packages/kirari/test/corpus/fixtures.ts +++ b/packages/kirari/test/corpus/fixtures.ts @@ -268,6 +268,26 @@ export const corpus: readonly CorpusFixture[] = [ { text: 'rows', type: 'attribute' }, ], }, + { + id: 'python-typed-parameters-and-dictionary-keys', language: 'python', + source: 'def typed(\n first: tuple[int, str],\n second: str,\n defaults={key1: {inner: 1}, key2: 2},\n):\n return defaults.member\n', + assertions: [ + { text: 'typed', type: 'function' }, { text: 'first', type: 'variable' }, + { text: 'second', type: 'variable' }, { text: 'defaults', type: 'variable', occurrence: 0 }, + { text: 'key1', type: 'property' }, { text: 'inner', type: 'property' }, + { text: 'key2', type: 'property' }, { text: 'member', type: 'property' }, + ], + }, + { + id: 'python-dictionary-lambda-parameters', language: 'python', + source: 'handlers = {\n first: lambda left,\n right: 1,\n second: lambda value={key: 2},\n other: 3,\n}\n', + assertions: [ + { text: 'first', type: 'property' }, { text: 'second', type: 'property' }, + { text: 'left', type: 'variable' }, { text: 'right', type: 'variable' }, + { text: 'value', type: 'variable' }, { text: 'other', type: 'variable' }, + { text: 'key', type: 'property' }, + ], + }, { id: 'python-function-and-values', language: 'python', source: 'def select_route(routes):\n for route in routes:\n if route.enabled:\n return True\n return None # no match\n', diff --git a/packages/kirari/test/language-performance.test.ts b/packages/kirari/test/language-performance.test.ts index 56a64f6..cb844f9 100644 --- a/packages/kirari/test/language-performance.test.ts +++ b/packages/kirari/test/language-performance.test.ts @@ -5,6 +5,8 @@ import { csharp } from '../src/languages/csharp.js' import { php } from '../src/languages/php.js' import { css } from '../src/languages/css.js' import { javascript } from '../src/languages/javascript.js' +import { python } from '../src/languages/python.js' +import { resumableScanOf } from '../src/resumable.js' import { jsx } from '../src/languages/jsx.js' import { makefile } from '../src/languages/makefile.js' import { toml } from '../src/languages/toml.js' @@ -12,6 +14,16 @@ import { yaml } from '../src/languages/yaml.js' import { createMarkdown } from '../src/markdown.js' import { createRegistry } from '../src/registry.js' +test('Python checkpoints avoid copying growing delimiter stacks on incomplete input', () => { + const source = '(\n'.repeat(75_000) + 'value' + let checkpoints = 0 + const start = performance.now() + const tokens = resumableScanOf(python)!(source, python, undefined, () => checkpoints++) + expect(tokens.map(token => token.value).join('')).toBe(source) + expect(checkpoints).toBe(75_000) + expect(performance.now() - start).toBeLessThan(1500) +}, 10_000) + test('JSX expression lookups do not repeatedly search the full source for absent line separators', () => { const source = '
' + '{x}\n'.repeat(64_000) + '
' const start = performance.now() diff --git a/packages/kirari/test/python-accuracy.test.ts b/packages/kirari/test/python-accuracy.test.ts index 25f038f..ef37c54 100644 --- a/packages/kirari/test/python-accuracy.test.ts +++ b/packages/kirari/test/python-accuracy.test.ts @@ -4,6 +4,49 @@ import { python } from '../src/languages/python.js' const pairs = (source: string) => tokenize(source, python).map(token => [token.type, token.value]) +test.each([ + 'def f(a: str, b: str): pass', + 'def f(a, b: str): pass', + 'def f(a: str, b): pass', + 'async def f(a: tuple[int, str], b: str): pass', + 'def f(a={k1: 1, k2: 2}, b: str): pass', + 'def f(\n a: str, # first\n b: str,\n): pass', +])('Python keeps annotated parameters as variables in %j', source => { + const result = pairs(source) + for (const name of ['a', 'b']) { + expect(result).toContainEqual(['variable', name]) + expect(result).not.toContainEqual(['property', name]) + } +}) + +test('Python distinguishes dictionary keys, members, and lambda parameters in nested expressions', () => { + const result = pairs('def f(a={k1: {inner: 1}, k2: (lambda x, y: obj.attr)}, b: str): pass') + for (const name of ['k1', 'inner', 'k2', 'attr']) expect(result).toContainEqual(['property', name]) + for (const name of ['a', 'b', 'x', 'y']) expect(result).toContainEqual(['variable', name]) +}) + +test.each([ + "x = {'f': lambda a, b: 1, next: 2}", + "x = {'f': lambda a={key: 1}, b: 1, next: 2}", + "x = {'f': lambda a=lambda c, d: 1, b: 2, next: 3}", + "x = {'f': lambda a,\n b: 1,\n next: 2}", + 'f"{ {key: lambda a, b: 1, next: 2} }"', +])('Python keeps unparenthesized lambda parameters out of dictionary keys in %j', source => { + const result = pairs(source) + expect(result).toContainEqual(['variable', 'b']) + expect(result).not.toContainEqual(['property', 'b']) + expect(result).toContainEqual(['property', 'next']) + if (source.includes('lambda c')) expect(result).toContainEqual(['variable', 'd']) + if (source.includes('key:')) expect(result).toContainEqual(['property', 'key']) +}) + +test('Python bracket context preserves explicit property rules before identifier classification', () => { + const language = { ...python, rules: [{ type: 'property', pattern: /SPECIAL/ }, ...python.rules] } + const tokens = tokenize('def f(a, SPECIAL: str, b: str): pass', language) + expect(tokens).toContainEqual({ type: 'property', value: 'SPECIAL', start: 9, end: 16 }) + expect(tokens).toContainEqual({ type: 'variable', value: 'b', start: 23, end: 24 }) +}) + test('Python f-strings expose expression variables, members, calls, and operators', () => { const result = pairs('label = f"Arrival {station.name}: {count + 1} {format(delay)} 🚉"') for (const pair of [['variable', 'station'], ['property', 'name'], ['variable', 'count'], diff --git a/packages/kirari/test/stream.test.ts b/packages/kirari/test/stream.test.ts index f2a16ff..8502d38 100644 --- a/packages/kirari/test/stream.test.ts +++ b/packages/kirari/test/stream.test.ts @@ -110,6 +110,24 @@ test('Python streams keep definitions that later lines can still change revisabl } }) +test('Python line checkpoints bound rescanning inside multiline brackets', () => { + for (const source of [ + 'def f(\n' + ' arg: str,\n'.repeat(200) + '): pass\n', + 'values = {\n' + ' key: (lambda a, b: b),\n'.repeat(200) + '}\n', + ]) { + const stream = createTokenStream(python) + const tokens: Token[] = [] + let work = 0 + for (let start = 0; start < source.length; start += 4) { + const patch = stream.write(source.slice(start, start + 4)) + work += patch.length - patch.from + apply(tokens, patch) + } + expect(tokens).toEqual(tokenize(source, python)) + expect(work).toBeLessThan(source.length * 20) + } +}) + test('Python line checkpoints bound rescanning', () => { const blocks = [ 'def greet(name: str) -> str:\n return f"Hello, {name}!" # note\n', diff --git a/sites/docs/changelog/kirari/1.1.4.md b/sites/docs/changelog/kirari/1.1.4.md new file mode 100644 index 0000000..65fdce9 --- /dev/null +++ b/sites/docs/changelog/kirari/1.1.4.md @@ -0,0 +1,10 @@ +--- +package: kirari +version: 1.1.4 +date: 2026-10-22 +--- + +### Bug fixes + +- Python annotated parameters after commas, such as `b` in `def f(a: str, b: str): pass`, are highlighted as variables instead of dictionary keys. Multiline signatures keep the same highlighting when streamed. +- Unparenthesized lambda parameters inside dictionaries, such as `b` in `{'f': lambda a, b: 1}`, keep their variable highlighting, including across streamed line breaks.