diff --git a/src/emit-portable.ts b/src/emit-portable.ts index 245ced2..7b3be7a 100644 --- a/src/emit-portable.ts +++ b/src/emit-portable.ts @@ -120,6 +120,15 @@ export type TplCfg = { interpRule: string; // the rule that parses each `${…}` hole (the Pratt expression rule) }; +// Newline-only mode (gen-lexer.ts L236–253, L677–747): engine-emitted NEWLINE tokens at +// significant line boundaries; flowOpen/flowClose suspend; comment-only lines are skipped. +export type NewlineCfg = { + token: string; + flowOpen: string[]; + flowClose: string[]; + comment: string | null; +}; + export type ParserIR = { grammarName: string; entry: string; @@ -128,6 +137,7 @@ export type ParserIR = { rules: RuleIR[]; regexCtx: RegexCtx | null; // null unless the grammar has a regex token with context tpl: TplCfg | null; // null unless the grammar has a template token + newlineCfg: NewlineCfg | null; // null unless the grammar declares `newline` }; // The target-agnostic parse plan for a grammar. Applies the [Await]/[Yield] context fork @@ -140,11 +150,21 @@ export function portableIR(grammar: CstGrammar): ParserIR { // ── buildIR: grammar + analysis → the target-agnostic parse plan ── +function isNeverPat(p: TokenPattern): boolean { + return typeof p !== 'string' && p.type === 'never'; +} + function buildIR(grammar: CstGrammar): ParserIR { + if (grammar.indent) { + throw new Error('portable: indent-sensitive grammars are out of scope (use the interpreter path); declare `newline` without `indent` for line-boundary-only mode'); + } + const a = analyzeGrammar(grammar); const tokenNames = a.tokenNames; - const tokens: LexTok[] = grammar.tokens.map((t) => lexTok(t)); + // Engine-emitted tokens (`never()` pattern — NEWLINE in newline mode) are excluded from the + // char scanner; the target lexer state machine emits them instead. + const tokens: LexTok[] = grammar.tokens.filter((t) => !isNeverPat(t.pattern)).map((t) => lexTok(t)); const lits = new Set(); for (const r of grammar.rules) for (const l of collectLiterals(r.body)) lits.add(l); for (const lv of grammar.precs) for (const o of lv.operators) lits.add(o.value); @@ -335,7 +355,27 @@ function buildIR(grammar: CstGrammar): ParserIR { } if (tpl) tpl.interpRule = san(tpl.interpRule); - return { grammarName: grammar.name ?? 'grammar', entry: san(findEntryRule(grammar)), tokens, puncts, rules, regexCtx, tpl }; + let newlineCfg: NewlineCfg | null = null; + if (grammar.newline) { + const nc = grammar.newline; + newlineCfg = { + token: nc.token, + flowOpen: [...(nc.flowOpen ?? [])], + flowClose: [...(nc.flowClose ?? [])], + comment: nc.comment ?? null, + }; + // Prefix-ambiguous rd alts (e.g. Ident vs Ident '(' … ')') are ordered longest-first so + // first-match backtracking agrees with the interpreter's longest-match rd parse (gen-parser.ts). + for (const r of rules) { + if (r.kind !== 'rd' || r.alts.length < 2) continue; + const order = r.alts.map((_, i) => i).sort((a, b) => r.alts[b].length - r.alts[a].length); + if (order.every((v, i) => v === i)) continue; + r.alts = order.map((i) => r.alts[i]); + r.altFirst = order.map((i) => r.altFirst[i]); + } + } + + return { grammarName: grammar.name ?? 'grammar', entry: san(findEntryRule(grammar)), tokens, puncts, rules, regexCtx, tpl, newlineCfg }; } // Classify a token: a fast-path shape (run/string/line/block) when one cleanly matches, diff --git a/src/target-go.ts b/src/target-go.ts index 3e6beac..7bb6989 100644 --- a/src/target-go.ts +++ b/src/target-go.ts @@ -9,7 +9,7 @@ // stack. A node is an int32 index, never a heap pointer. Backtracking truncates the three // slices to saved lengths; the slices keep their capacity across parses (reset to len 0), so a // warmed parser allocates ~nothing per parse. -import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, FirstSig } from './emit-portable.ts'; +import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts'; import { portableIR } from './emit-portable.ts'; import type { Target } from './emit.ts'; import type { TokenPattern, CstGrammar } from './types.ts'; @@ -51,9 +51,8 @@ function compilePat(p: TokenPattern, defs: string[]): string { return name; } -function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): string { +function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, tplTok?: string): string { const name = (t as { name: string }).name; - const stateful = rxTok !== undefined || tplTok !== undefined; if (tplTok !== undefined && name === tplTok) return ''; // template token scanned by the state machine const push = (endE: string) => (t.skip ? `if strings.ContainsAny(src[pos:${endE}], "\\n\\r\\u2028\\u2029") { pendingNl = true }; ` : stateful ? `emit(${J(name)}, src[pos:${endE}], pos, ${endE}); ` : `pushTok(${J(name)}, src[pos:${endE}], pos, ${endE}); `); const gate = rxTok !== undefined && name === rxTok ? '!prevIsValue() && ' : ''; @@ -82,12 +81,59 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st return `\t\tif ${gate ? gate + 'true' : 'true'} { if e := ${m}(pos); e > pos { ${push('e')}pos = e; continue } }`; } +function newlinePartsGo(nl: NewlineCfg, pushFn: string): { state: string; boundary: string; ws: string; hooks: string } { + const commentSkip = nl.comment + ? `\t\tif strings.HasPrefix(src[p:], ${J(nl.comment)}) { e := p; for e < n && src[e] != 10 { e++ }; pos = e; continue }\n` + : ''; + return { + state: `\tlineStart, emittedContent, flowDepth := true, false, 0 +\t_flowOpen := map[string]bool{${nl.flowOpen.map((x) => `${J(x)}: true`).join(', ')}} +\t_flowClose := map[string]bool{${nl.flowClose.map((x) => `${J(x)}: true`).join(', ')}} +\tconst _nlTok = ${J(nl.token)} +`, + boundary: `\t\tif flowDepth == 0 && lineStart { +\t\t\tp := pos +\t\t\tfor p < n && src[p] == 32 { p++ } +\t\t\tif p >= n { pos = p; lineStart = false; continue } +\t\t\tch := int(src[p]) +\t\t\tif ch == 10 || ch == 13 { +\t\t\t\tpos = p + 1; if ch == 13 && pos < n && src[pos] == 10 { pos++ }; continue +\t\t\t} +\t\t\tif ch == 9 { +\t\t\t\tb := p +\t\t\t\tfor b < n && (src[b] == 32 || src[b] == 9) { b++ } +\t\t\t\tif b >= n { pos = b; continue } +\t\t\t\tbc := int(src[b]) +\t\t\t\tif bc == 10 || bc == 13 { +\t\t\t\t\tpos = b + 1; if bc == 13 && pos < n && src[pos] == 10 { pos++ }; continue +\t\t\t\t} +\t\t\t} +${commentSkip}\t\t\tpos = p +\t\t\tif emittedContent { ${pushFn}(_nlTok, "", pos, pos) } +\t\t\tlineStart = false +\t\t\tcontinue +\t\t} +`, + ws: `\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue } +\t\tif c == 10 || c == 13 { +\t\t\tpos++; if c == 13 && pos < n && src[pos] == 10 { pos++ } +\t\t\tif flowDepth == 0 { lineStart = true } else { pendingNl = true } +\t\t\tcontinue +\t\t} +`, + hooks: `\t\tif kind != _nlTok { emittedContent = true } +\t\tif kind == "" && _flowOpen[text] { flowDepth++ } else if kind == "" && _flowClose[text] { if flowDepth > 0 { flowDepth-- } } +`, + }; +} + function lexer(ir: ParserIR): string { const defs: string[] = []; const rx = ir.regexCtx; const tpl = ir.tpl; - const stateful = !!(rx || tpl); - const toks = ir.tokens.map((t) => scanTok(t, defs, rx?.regexToken, tpl?.token)).join('\n'); + const nl = ir.newlineCfg; + const stateful = !!(rx || tpl || nl); + const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); const pushPunct = stateful ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`; const puncts = ir.puncts.map((p) => `\t\tif strings.HasPrefix(src[pos:], ${J(p)}) { ${pushPunct(p)}; pos += ${p.length}; continue }`).join('\n'); @@ -132,6 +178,7 @@ function lexer(ir: ParserIR): string { \t\t} \t\tif _pav[text] { lastBang = prevIsValue() }` : '', tpl ? `\t\tif len(templateStack) > 0 { if text == ${J(tpl.braceOpen)} { templateStack[len(templateStack)-1]++ } else if text == ${J(tpl.interpClose)} { templateStack[len(templateStack)-1]-- } }` : '', + nl ? newlinePartsGo(nl, 'emit').hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? `\n\t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true` : ''; const emitFn = stateful ? `\temit := func(kind, text string, off, end int) { @@ -152,19 +199,29 @@ ${emitHooks} \t\t\tpos = e; continue \t\t} ` : ''; - const pushTokFn = stateful ? '' : `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`; + const nlState = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').state : ''; + const nlBoundary = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').boundary : ''; + const nlWs = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').ws : `\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8) +\t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue } // LF/CR +\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue } +`; + const pushHooks = nl && !stateful ? newlinePartsGo(nl, 'pushTok').hooks : ''; + const pushTokFn = stateful ? '' : nl + ? `\tpushTok := func(kind, text string, off, end int) { +${pushHooks}\t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false +\t} +\t_ = pushTok +` + : `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`; return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lex(src string) []Tok { \ttoks := toks[:0] \tn := len(src) \tpos := 0 \tpendingNl := false \t_ = pendingNl -${rxState}${tplState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\n' : ''}\tfor pos < n { -\t\tc := int(src[pos]) -\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8) -\t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue } // LF/CR -\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue } -${tplDispatch}${toks} +${rxState}${tplState}${nlState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\n' : ''}\tfor pos < n { +${nlBoundary}\t\tc := int(src[pos]) +${nlWs}${tplDispatch}${toks} ${puncts} \t\tpanic(fmt.Sprintf("lex error at %d", pos)) \t} diff --git a/src/target-rust.ts b/src/target-rust.ts index 34104b1..c39fd6b 100644 --- a/src/target-rust.ts +++ b/src/target-rust.ts @@ -11,7 +11,7 @@ // parser allocates ~nothing per parse. Rule fns return `i32` (-1 = fail); sub-sequence // combinators take non-capturing `fn(&mut Parser) -> bool` pointers (the kids vec is now on // the Parser as `scratch`, so the second param the old owned-tree version threaded is gone). -import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, FirstSig } from './emit-portable.ts'; +import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts'; import { portableIR } from './emit-portable.ts'; import type { Target } from './emit.ts'; import type { TokenPattern, CstGrammar } from './types.ts'; @@ -55,9 +55,8 @@ function compilePat(p: TokenPattern, defs: string[]): string { return name; } -function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): string { +function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, tplTok?: string): string { const name = (t as { name: string }).name; - const stateful = rxTok !== undefined || tplTok !== undefined; if (tplTok !== undefined && name === tplTok) return ''; // template token scanned by the state machine const nlVar = stateful ? 'st.pending_nl' : 'pending_nl'; const push = (endE: string) => (t.skip ? `if src[pos..${endE}].chars().any(|c| matches!(c, '\\n' | '\\r' | '\\u{2028}' | '\\u{2029}')) { ${nlVar} = true; } ` : stateful ? `st.emit(${J(name)}, &src[pos..${endE}], pos, ${endE}); ` : `toks.push(Tok { kind: ${J(name)}, text: &src[pos..${endE}], off: pos, end: ${endE}, nl: pending_nl }); pending_nl = false; `); @@ -87,12 +86,62 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st return ` if ${gate}true { let e = ${m}(src, pos as i64); if e > pos as i64 { let e = e as usize; ${push('e')}pos = e; continue; } }`; } +function newlinePartsRs(nl: NewlineCfg): { consts: string; fields: string; init: string; boundary: string; ws: string; hooks: string } { + const commentSkip = nl.comment + ? ` if src[p..].starts_with(${J(nl.comment)}) { let mut e = p; while e < n && b[e] != 10 { e += 1; } pos = e; continue; }\n` + : ''; + return { + consts: `const _NLTOK: &str = ${J(nl.token)}; +const _FLOW_OPEN: &[&str] = ${`&[${nl.flowOpen.map(J).join(', ')}]`}; +const _FLOW_CLOSE: &[&str] = ${`&[${nl.flowClose.map(J).join(', ')}]`}; +`, + fields: 'line_start: bool, emitted_content: bool, flow_depth: i64', + init: 'line_start: true, emitted_content: false, flow_depth: 0', + boundary: ` if st.flow_depth == 0 && st.line_start { + let mut p = pos; + while p < n && b[p] == 32 { p += 1; } + if p >= n { pos = p; st.line_start = false; continue; } + let ch = b[p] as u32; + if ch == 10 || ch == 13 { + pos = p + 1; if ch == 13 && pos < n && b[pos] == 10 { pos += 1; } continue; + } + if ch == 9 { + let mut bb = p; + while bb < n && (b[bb] == 32 || b[bb] == 9) { bb += 1; } + if bb >= n { pos = bb; continue; } + let bc = b[bb] as u32; + if bc == 10 || bc == 13 { + pos = bb + 1; if bc == 13 && pos < n && b[pos] == 10 { pos += 1; } continue; + } + } +${commentSkip} pos = p; + if st.emitted_content { st.emit(_NLTOK, &src[pos..pos], pos, pos); } + st.line_start = false; + continue; + } +`, + ws: ` if c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos += 1; continue; } + if c == 10 || c == 13 { + pos += 1; if c == 13 && pos < n && b[pos] == 10 { pos += 1; } + if st.flow_depth == 0 { st.line_start = true; } else { st.pending_nl = true; } + continue; + } +`, + hooks: ` if kind != _NLTOK { self.emitted_content = true; } + if kind == "" && _in(_FLOW_OPEN, text) { self.flow_depth += 1; } + else if kind == "" && _in(_FLOW_CLOSE, text) { self.flow_depth = (self.flow_depth - 1).max(0); } +`, + }; +} + function lexer(ir: ParserIR): string { const defs: string[] = []; const rx = ir.regexCtx; const tpl = ir.tpl; - const stateful = !!(rx || tpl); - const toks = ir.tokens.map((t) => scanTok(t, defs, rx?.regexToken, tpl?.token)).join('\n'); + const nl = ir.newlineCfg; + const nlRs = nl ? newlinePartsRs(nl) : null; + const stateful = !!(rx || tpl || nl); + const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); const puncts = ir.puncts.map((p) => ` if src[pos..].starts_with(${J(p)}) { ${stateful ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n'); const rsArr = (a: string[]) => `&[${a.map(J).join(', ')}]`; @@ -106,7 +155,8 @@ const _MEM: &[&str] = ${rsArr(rx.memberAccess)}; const _PAV: &[&str] = ${rsArr(rx.postfixAfterValue)}; const _IDENT: &str = ${J(rx.identToken)}; fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) } -` : ''; +${nlRs ? nlRs.consts : ''}` : (nlRs ? `fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) } +${nlRs.consts}` : ''); const tplFn = tpl ? `fn _scan_tpl_span(s: &str, mut p: usize) -> (bool, usize) { let n = s.len(); while p < n { @@ -120,7 +170,8 @@ fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) } ` : ''; const fields = ['toks: Vec>', 'pending_nl: bool', rx ? 'prev_text: &\'a str, prev_kind: &\'static str, bp_text: &\'a str, has_prev: bool, has_prev2: bool, paren_head: Vec, last_close: bool, last_bang: bool' : '', - tpl ? 'template_stack: Vec' : ''].filter(Boolean).join(', '); + tpl ? 'template_stack: Vec' : '', + nlRs ? nlRs.fields : ''].filter(Boolean).join(', '); const prevIsValue = rx ? ` fn prev_is_value(&self) -> bool { if !self.has_prev { return false; } if _in(_PAV, self.prev_text) { return self.last_bang; } @@ -134,6 +185,7 @@ fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) } else if text == ")" { self.last_close = self.paren_head.pop().unwrap_or(false); } if _in(_PAV, text) { self.last_bang = self.prev_is_value(); }` : '', tpl ? ` if !self.template_stack.is_empty() { if text == ${J(tpl.braceOpen)} { *self.template_stack.last_mut().unwrap() += 1; } else if text == ${J(tpl.interpClose)} { *self.template_stack.last_mut().unwrap() -= 1; } }` : '', + nlRs ? nlRs.hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? ` self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true;` : ''; @@ -147,7 +199,8 @@ ${emitHooks} ` : ''; const initFields = ['toks: Vec::new()', 'pending_nl: false', rx ? 'prev_text: "", prev_kind: "", bp_text: "", has_prev: false, has_prev2: false, paren_head: Vec::new(), last_close: false, last_bang: false' : '', - tpl ? 'template_stack: Vec::new()' : ''].filter(Boolean).join(', '); + tpl ? 'template_stack: Vec::new()' : '', + nlRs ? nlRs.init : ''].filter(Boolean).join(', '); const open = stateful ? ` let mut st = LexState { ${initFields} };` : ` let mut toks: Vec = Vec::new();\n let mut pending_nl = false;`; const nlVar = stateful ? 'st.pending_nl' : 'pending_nl'; const tplDispatch = tpl ? ` if !st.template_stack.is_empty() && src[pos..].starts_with(${J(tpl.interpClose)}) && *st.template_stack.last().unwrap() == 0 { @@ -162,17 +215,19 @@ ${emitHooks} pos = e; continue; } ` : ''; + const nlBoundary = nlRs ? nlRs.boundary : ''; + const nlWs = nlRs ? nlRs.ws : ` if c == 32 || c == 9 { pos += 1; continue; } + if pos + 2 < n && b[pos] == 0xE2 && b[pos + 1] == 0x80 && (b[pos + 2] == 0xA8 || b[pos + 2] == 0xA9) { ${nlVar} = true; pos += 3; continue; } // LS/PS (UTF-8) + if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; } // LF/CR +`; return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${stateImpl}fn lex<'a>(src: &'a str) -> Vec> { let b = src.as_bytes(); let n = b.len(); ${open} let mut pos = 0usize; while pos < n { - let c = b[pos] as u32; - if c == 32 || c == 9 { pos += 1; continue; } - if pos + 2 < n && b[pos] == 0xE2 && b[pos + 1] == 0x80 && (b[pos + 2] == 0xA8 || b[pos + 2] == 0xA9) { ${nlVar} = true; pos += 3; continue; } // LS/PS (UTF-8) - if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; } // LF/CR -${tplDispatch}${toks} +${nlBoundary} let c = b[pos] as u32; +${nlWs}${tplDispatch}${toks} ${puncts} panic!("lex error at {}", pos); } diff --git a/src/target-ts.ts b/src/target-ts.ts index edf55a4..ec8f5d6 100644 --- a/src/target-ts.ts +++ b/src/target-ts.ts @@ -4,7 +4,7 @@ // index LEDs), and a CST→JSON printer over stdin. It is the reference rendering — its CST // is checked byte-for-byte against the interpreter (createParser), so a divergence in the // portable logic surfaces here before Go/Rust are compiled. -import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, FirstSig } from './emit-portable.ts'; +import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts'; import { portableIR } from './emit-portable.ts'; import type { Target } from './emit.ts'; import type { CstGrammar } from './types.ts'; @@ -49,9 +49,8 @@ function compilePat(p: TokenPattern, defs: string[]): string { return name; } -function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): string { +function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, tplTok?: string): string { const name = (t as { name: string }).name; - const stateful = rxTok !== undefined || tplTok !== undefined; if (tplTok !== undefined && name === tplTok) return ''; // template token is scanned by the state machine // `emit(...)` threads the lexer state in stateful mode; a plain push otherwise. A skipped // token (comment) still records a newline it spans, so `sameLine` sees it. @@ -82,12 +81,63 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st return ` if (${gate}true) { const e = ${m}(pos); if (e > pos) { ${push('e')}pos = e; continue; } }`; } +function newlineParts(nl: NewlineCfg, pushFn: string): { state: string; boundary: string; ws: string; hooks: string } { + const commentSkip = nl.comment + ? ` if (src.startsWith(${J(nl.comment)}, p)) { let e = p; while (e < n && src.charCodeAt(e) !== 10) e++; pos = e; continue; }\n` + : ''; + return { + state: ` let lineStart = true, emittedContent = false, flowDepth = 0; + const _flowOpen = new Set([${nl.flowOpen.map(J).join(', ')}]); + const _flowClose = new Set([${nl.flowClose.map(J).join(', ')}]); + const _nlTok = ${J(nl.token)}; +`, + boundary: ` if (flowDepth === 0 && lineStart) { + let p = pos; + while (p < n && src.charCodeAt(p) === 32) p++; + if (p >= n) { pos = p; lineStart = false; continue; } + const ch = src.charCodeAt(p); + if (ch === 10 || ch === 13) { // LF/CR only — the interpreter's newline mode rejects LS/PS (gen-lexer.ts blank-line check) + pos = p + 1; if (ch === 13 && pos < n && src.charCodeAt(pos) === 10) pos++; + continue; + } + if (ch === 9) { + let b = p; + while (b < n && (src.charCodeAt(b) === 32 || src.charCodeAt(b) === 9)) b++; + if (b >= n) { pos = b; continue; } + const bc = src.charCodeAt(b); + if (bc === 10 || bc === 13) { + pos = b + 1; if (bc === 13 && pos < n && src.charCodeAt(pos) === 10) pos++; + continue; + } + } +${commentSkip} pos = p; + if (emittedContent) ${pushFn}(_nlTok, '', pos, pos); + lineStart = false; + continue; + } +`, + ws: ` if (c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; } + if (c === 10 || c === 13) { // LF/CR only — LS/PS fall through to the unexpected-character throw, matching the interpreter + pos++; if (c === 13 && pos < n && src.charCodeAt(pos) === 10) pos++; + if (flowDepth === 0) lineStart = true; + else pendingNl = true; + continue; + } +`, + hooks: ` if (kind !== _nlTok) emittedContent = true; + if (kind === '' && _flowOpen.has(text)) flowDepth++; + else if (kind === '' && _flowClose.has(text)) flowDepth = Math.max(0, flowDepth - 1); +`, + }; +} + function lexer(ir: ParserIR): string { const defs: string[] = []; const rx = ir.regexCtx; const tpl = ir.tpl; - const stateful = !!(rx || tpl); - const toks = ir.tokens.map((t) => scanTok(t, defs, rx?.regexToken, tpl?.token)).join('\n'); + const nl = ir.newlineCfg; + const stateful = !!(rx || tpl || nl); + const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); const pushFn = stateful ? 'emit' : 'push'; const puncts = ir.puncts.map((p) => ` if (src.startsWith(${J(p)}, pos)) { ${pushFn}('', ${J(p)}, pos, pos + ${p.length}); pos += ${p.length}; continue; }`).join('\n'); @@ -123,6 +173,7 @@ function lexer(ir: ParserIR): string { else if (text === ')') { lastClose = parenHead.pop() ?? false; } if (_pav.has(text)) lastBang = prevIsValue();` : '', tpl ? ` if (templateStack.length > 0) { if (text === ${J(tpl.braceOpen)}) templateStack[templateStack.length - 1]++; else if (text === ${J(tpl.interpClose)}) templateStack[templateStack.length - 1]--; }` : '', + nl ? newlineParts(nl, 'emit').hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? `\n bpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true;` : ''; const emitFn = stateful ? ` function emit(kind: string, text: string, off: number, end: number): void { @@ -145,17 +196,27 @@ ${emitHooks} pos = sp.end; continue; } ` : ''; + const nlState = nl ? newlineParts(nl, stateful ? 'emit' : 'push').state : ''; + const nlBoundary = nl ? newlineParts(nl, stateful ? 'emit' : 'push').boundary : ''; + const nlWs = nl ? newlineParts(nl, stateful ? 'emit' : 'push').ws : ` if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; } + if (c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; } +`; + const pushHooks = nl && !stateful ? newlineParts(nl, 'push').hooks : ''; + const pushFnDef = stateful ? '' : nl + ? ` const push = (kind: string, text: string, off: number, end: number) => { +${pushHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; + }; +` + : ' const push = (kind: string, text: string, off: number, end: number) => { toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; };\n'; return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lex(src: string): Tok[] { const toks: Tok[] = []; const n = src.length; let pos = 0; let pendingNl = false; -${defs.length ? ' _s = src;\n' : ''}${rxState}${tplState}${stateful ? emitFn : ' const push = (kind: string, text: string, off: number, end: number) => { toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; };\n'} while (pos < n) { - const c = src.charCodeAt(pos); +${defs.length ? ' _s = src;\n' : ''}${rxState}${tplState}${nlState}${stateful ? emitFn : pushFnDef} while (pos < n) { +${nlBoundary} const c = src.charCodeAt(pos); // JS line terminators LF/CR/LS/PS set newline-before, matching the interpreter (gen-lexer.ts). - if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; } - if (c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; } -${tplDispatch}${toks} +${nlWs}${tplDispatch}${toks} ${puncts} throw new Error('lex error at ' + pos + ': ' + JSON.stringify(src[pos])); } diff --git a/test/fixtures/envspec.ts b/test/fixtures/envspec.ts new file mode 100644 index 0000000..e4f0605 --- /dev/null +++ b/test/fixtures/envspec.ts @@ -0,0 +1,26 @@ +// Isolated newline-mode grammar (dotenv / env-spec flavour) for portable-targets gate. +// Exercises engine-emitted NEWLINE tokens: line-boundary separators, flow ( … ) suspension, +// blank-line folding, comment-only lines — with NO indent stack (see test/newline-mode.ts). +import { + token, rule, defineGrammar, many, opt, sep, seq, plus, oneOf, range, star, noneOf, never, +} from '../../src/api.ts'; +import type { NewlineConfig } from '../../src/types.ts'; + +const Newline = token(never(), {}); +const Ident = token(plus(oneOf(range('a', 'z'), range('A', 'Z'), range('0', '9'), '_')), { identifier: true }); +const Comment = token(seq('#', star(noneOf('\n'))), { skip: true }); + +const Value = rule(($: any) => [Ident, [Ident, '(', sep($, ','), ')']]); +const Stmt = rule(() => [[Ident, '=', Value]]); +const Program = rule(() => [[opt(Stmt), many(Newline, opt(Stmt))]]); + +const newline: NewlineConfig = { token: 'Newline', flowOpen: ['('], flowClose: [')'], comment: '#' }; + +export default defineGrammar({ + name: 'envspec', + scopeName: 'source.envspec', + tokens: { Comment, Ident, Newline }, + rules: { Value, Stmt, Program }, + entry: Program, + newline, +}); diff --git a/test/portable-targets.ts b/test/portable-targets.ts index bdca05a..5d55d73 100644 --- a/test/portable-targets.ts +++ b/test/portable-targets.ts @@ -91,6 +91,15 @@ const CASES: Case[] = [ ], reject: ['var x = `${ }`;', 'var y = `${a`;', '`${a} ${}`;'], }, + { + // Newline-only mode (engine-emitted NEWLINE at significant line boundaries; flow suspension). + grammar: 'envspec', path: './fixtures/envspec.ts', + accept: [ + 'A=1', 'A=1\nB=2', 'A=1\n', 'A=1\n# c\nB=2', 'A=fn(1,\n2)\nB=3', + 'A=1\n\n\nB=2', '\n\nA=1', 'A=fn(1,\n2)', + ], + reject: ['A B', 'A=', 'A=1 B=2', 'A=fn(1,'], + }, { // General (non-literal) inline alt: object keys are alt(Ident | Str | Number) — a // backtracking alternation of token refs inside a rule sequence.