Skip to content

Portable targets: newline-only lexer mode (env-spec use case) #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
johnsoncodehk wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Portable targets: newline-only lexer mode (env-spec use case) #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
15e9c78
emit-portable: add newline-only mode for TS/Go/Rust portable targets
johnsoncodehk Jul 10, 2026
4193aa6
emit-portable: reject LS/PS in newline mode, matching the interpreter
johnsoncodehk Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions src/emit-portable.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,15 @@ export type TplCfg = {
interpRule: string; // the rule that parses each `${…}` hole (the Pratt expression rule)
};

// Newline-only mode (gen-lexer.ts L236–253, L677–747): engine-emitted NEWLINE tokens at
// significant line boundaries; flowOpen/flowClose suspend; comment-only lines are skipped.
export type NewlineCfg = {
token: string;
flowOpen: string[];
flowClose: string[];
comment: string | null;
};

export type ParserIR = {
grammarName: string;
entry: string;
Expand All @@ -128,6 +137,7 @@ export type ParserIR = {
rules: RuleIR[];
regexCtx: RegexCtx | null; // null unless the grammar has a regex token with context
tpl: TplCfg | null; // null unless the grammar has a template token
newlineCfg: NewlineCfg | null; // null unless the grammar declares `newline`
};

// The target-agnostic parse plan for a grammar. Applies the [Await]/[Yield] context fork
Expand All @@ -140,11 +150,21 @@ export function portableIR(grammar: CstGrammar): ParserIR {

// ── buildIR: grammar + analysis → the target-agnostic parse plan ──

function isNeverPat(p: TokenPattern): boolean {
return typeof p !== 'string' && p.type === 'never';
}

function buildIR(grammar: CstGrammar): ParserIR {
if (grammar.indent) {
throw new Error('portable: indent-sensitive grammars are out of scope (use the interpreter path); declare `newline` without `indent` for line-boundary-only mode');
}

const a = analyzeGrammar(grammar);
const tokenNames = a.tokenNames;

const tokens: LexTok[] = grammar.tokens.map((t) => lexTok(t));
// Engine-emitted tokens (`never()` pattern — NEWLINE in newline mode) are excluded from the
// char scanner; the target lexer state machine emits them instead.
const tokens: LexTok[] = grammar.tokens.filter((t) => !isNeverPat(t.pattern)).map((t) => lexTok(t));
const lits = new Set<string>();
for (const r of grammar.rules) for (const l of collectLiterals(r.body)) lits.add(l);
for (const lv of grammar.precs) for (const o of lv.operators) lits.add(o.value);
Expand Down Expand Up @@ -335,7 +355,27 @@ function buildIR(grammar: CstGrammar): ParserIR {
}
if (tpl) tpl.interpRule = san(tpl.interpRule);

return { grammarName: grammar.name ?? 'grammar', entry: san(findEntryRule(grammar)), tokens, puncts, rules, regexCtx, tpl };
let newlineCfg: NewlineCfg | null = null;
if (grammar.newline) {
const nc = grammar.newline;
newlineCfg = {
token: nc.token,
flowOpen: [...(nc.flowOpen ?? [])],
flowClose: [...(nc.flowClose ?? [])],
comment: nc.comment ?? null,
};
// Prefix-ambiguous rd alts (e.g. Ident vs Ident '(' … ')') are ordered longest-first so
// first-match backtracking agrees with the interpreter's longest-match rd parse (gen-parser.ts).
for (const r of rules) {
if (r.kind !== 'rd' || r.alts.length < 2) continue;
const order = r.alts.map((_, i) => i).sort((a, b) => r.alts[b].length - r.alts[a].length);
if (order.every((v, i) => v === i)) continue;
r.alts = order.map((i) => r.alts[i]);
r.altFirst = order.map((i) => r.altFirst[i]);
}
}

return { grammarName: grammar.name ?? 'grammar', entry: san(findEntryRule(grammar)), tokens, puncts, rules, regexCtx, tpl, newlineCfg };
}

// Classify a token: a fast-path shape (run/string/line/block) when one cleanly matches,
Expand Down
81 changes: 69 additions & 12 deletions src/target-go.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
// stack. A node is an int32 index, never a heap pointer. Backtracking truncates the three
// slices to saved lengths; the slices keep their capacity across parses (reset to len 0), so a
// warmed parser allocates ~nothing per parse.
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, FirstSig } from './emit-portable.ts';
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts';
import { portableIR } from './emit-portable.ts';
import type { Target } from './emit.ts';
import type { TokenPattern, CstGrammar } from './types.ts';
Expand Down Expand Up @@ -51,9 +51,8 @@ function compilePat(p: TokenPattern, defs: string[]): string {
return name;
}

function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): string {
function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, tplTok?: string): string {
const name = (t as { name: string }).name;
const stateful = rxTok !== undefined || tplTok !== undefined;
if (tplTok !== undefined && name === tplTok) return ''; // template token scanned by the state machine
const push = (endE: string) => (t.skip ? `if strings.ContainsAny(src[pos:${endE}], "\\n\\r\\u2028\\u2029") { pendingNl = true }; ` : stateful ? `emit(${J(name)}, src[pos:${endE}], pos, ${endE}); ` : `pushTok(${J(name)}, src[pos:${endE}], pos, ${endE}); `);
const gate = rxTok !== undefined && name === rxTok ? '!prevIsValue() && ' : '';
Expand Down Expand Up @@ -82,12 +81,59 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st
return `\t\tif ${gate ? gate + 'true' : 'true'} { if e := ${m}(pos); e > pos { ${push('e')}pos = e; continue } }`;
}

function newlinePartsGo(nl: NewlineCfg, pushFn: string): { state: string; boundary: string; ws: string; hooks: string } {
const commentSkip = nl.comment
? `\t\tif strings.HasPrefix(src[p:], ${J(nl.comment)}) { e := p; for e < n && src[e] != 10 { e++ }; pos = e; continue }\n`
: '';
return {
state: `\tlineStart, emittedContent, flowDepth := true, false, 0
\t_flowOpen := map[string]bool{${nl.flowOpen.map((x) => `${J(x)}: true`).join(', ')}}
\t_flowClose := map[string]bool{${nl.flowClose.map((x) => `${J(x)}: true`).join(', ')}}
\tconst _nlTok = ${J(nl.token)}
`,
boundary: `\t\tif flowDepth == 0 && lineStart {
\t\t\tp := pos
\t\t\tfor p < n && src[p] == 32 { p++ }
\t\t\tif p >= n { pos = p; lineStart = false; continue }
\t\t\tch := int(src[p])
\t\t\tif ch == 10 || ch == 13 {
\t\t\t\tpos = p + 1; if ch == 13 && pos < n && src[pos] == 10 { pos++ }; continue
\t\t\t}
\t\t\tif ch == 9 {
\t\t\t\tb := p
\t\t\t\tfor b < n && (src[b] == 32 || src[b] == 9) { b++ }
\t\t\t\tif b >= n { pos = b; continue }
\t\t\t\tbc := int(src[b])
\t\t\t\tif bc == 10 || bc == 13 {
\t\t\t\t\tpos = b + 1; if bc == 13 && pos < n && src[pos] == 10 { pos++ }; continue
\t\t\t\t}
\t\t\t}
${commentSkip}\t\t\tpos = p
\t\t\tif emittedContent { ${pushFn}(_nlTok, "", pos, pos) }
\t\t\tlineStart = false
\t\t\tcontinue
\t\t}
`,
ws: `\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue }
\t\tif c == 10 || c == 13 {
\t\t\tpos++; if c == 13 && pos < n && src[pos] == 10 { pos++ }
\t\t\tif flowDepth == 0 { lineStart = true } else { pendingNl = true }
\t\t\tcontinue
\t\t}
`,
hooks: `\t\tif kind != _nlTok { emittedContent = true }
\t\tif kind == "" && _flowOpen[text] { flowDepth++ } else if kind == "" && _flowClose[text] { if flowDepth > 0 { flowDepth-- } }
`,
};
}

function lexer(ir: ParserIR): string {
const defs: string[] = [];
const rx = ir.regexCtx;
const tpl = ir.tpl;
const stateful = !!(rx || tpl);
const toks = ir.tokens.map((t) => scanTok(t, defs, rx?.regexToken, tpl?.token)).join('\n');
const nl = ir.newlineCfg;
const stateful = !!(rx || tpl || nl);
const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n');
const pushPunct = stateful ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`;
const puncts = ir.puncts.map((p) =>
`\t\tif strings.HasPrefix(src[pos:], ${J(p)}) { ${pushPunct(p)}; pos += ${p.length}; continue }`).join('\n');
Expand Down Expand Up @@ -132,6 +178,7 @@ function lexer(ir: ParserIR): string {
\t\t}
\t\tif _pav[text] { lastBang = prevIsValue() }` : '',
tpl ? `\t\tif len(templateStack) > 0 { if text == ${J(tpl.braceOpen)} { templateStack[len(templateStack)-1]++ } else if text == ${J(tpl.interpClose)} { templateStack[len(templateStack)-1]-- } }` : '',
nl ? newlinePartsGo(nl, 'emit').hooks : '',
].filter(Boolean).join('\n');
const emitTail = rx ? `\n\t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true` : '';
const emitFn = stateful ? `\temit := func(kind, text string, off, end int) {
Expand All @@ -152,19 +199,29 @@ ${emitHooks}
\t\t\tpos = e; continue
\t\t}
` : '';
const pushTokFn = stateful ? '' : `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`;
const nlState = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').state : '';
const nlBoundary = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').boundary : '';
const nlWs = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').ws : `\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8)
\t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue } // LF/CR
\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue }
`;
const pushHooks = nl && !stateful ? newlinePartsGo(nl, 'pushTok').hooks : '';
const pushTokFn = stateful ? '' : nl
? `\tpushTok := func(kind, text string, off, end int) {
${pushHooks}\t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false
\t}
\t_ = pushTok
`
: `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`;
return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lex(src string) []Tok {
\ttoks := toks[:0]
\tn := len(src)
\tpos := 0
\tpendingNl := false
\t_ = pendingNl
${rxState}${tplState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\n' : ''}\tfor pos < n {
\t\tc := int(src[pos])
\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8)
\t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue } // LF/CR
\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue }
${tplDispatch}${toks}
${rxState}${tplState}${nlState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\n' : ''}\tfor pos < n {
${nlBoundary}\t\tc := int(src[pos])
${nlWs}${tplDispatch}${toks}
${puncts}
\t\tpanic(fmt.Sprintf("lex error at %d", pos))
\t}
Expand Down
81 changes: 68 additions & 13 deletions src/target-rust.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
// parser allocates ~nothing per parse. Rule fns return `i32` (-1 = fail); sub-sequence
// combinators take non-capturing `fn(&mut Parser) -> bool` pointers (the kids vec is now on
// the Parser as `scratch`, so the second param the old owned-tree version threaded is gone).
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, FirstSig } from './emit-portable.ts';
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts';
import { portableIR } from './emit-portable.ts';
import type { Target } from './emit.ts';
import type { TokenPattern, CstGrammar } from './types.ts';
Expand Down Expand Up @@ -55,9 +55,8 @@ function compilePat(p: TokenPattern, defs: string[]): string {
return name;
}

function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): string {
function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, tplTok?: string): string {
const name = (t as { name: string }).name;
const stateful = rxTok !== undefined || tplTok !== undefined;
if (tplTok !== undefined && name === tplTok) return ''; // template token scanned by the state machine
const nlVar = stateful ? 'st.pending_nl' : 'pending_nl';
const push = (endE: string) => (t.skip ? `if src[pos..${endE}].chars().any(|c| matches!(c, '\\n' | '\\r' | '\\u{2028}' | '\\u{2029}')) { ${nlVar} = true; } ` : stateful ? `st.emit(${J(name)}, &src[pos..${endE}], pos, ${endE}); ` : `toks.push(Tok { kind: ${J(name)}, text: &src[pos..${endE}], off: pos, end: ${endE}, nl: pending_nl }); pending_nl = false; `);
Expand Down Expand Up @@ -87,12 +86,62 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st
return ` if ${gate}true { let e = ${m}(src, pos as i64); if e > pos as i64 { let e = e as usize; ${push('e')}pos = e; continue; } }`;
}

function newlinePartsRs(nl: NewlineCfg): { consts: string; fields: string; init: string; boundary: string; ws: string; hooks: string } {
const commentSkip = nl.comment
? ` if src[p..].starts_with(${J(nl.comment)}) { let mut e = p; while e < n && b[e] != 10 { e += 1; } pos = e; continue; }\n`
: '';
return {
consts: `const _NLTOK: &str = ${J(nl.token)};
const _FLOW_OPEN: &[&str] = ${`&[${nl.flowOpen.map(J).join(', ')}]`};
const _FLOW_CLOSE: &[&str] = ${`&[${nl.flowClose.map(J).join(', ')}]`};
`,
fields: 'line_start: bool, emitted_content: bool, flow_depth: i64',
init: 'line_start: true, emitted_content: false, flow_depth: 0',
boundary: ` if st.flow_depth == 0 && st.line_start {
let mut p = pos;
while p < n && b[p] == 32 { p += 1; }
if p >= n { pos = p; st.line_start = false; continue; }
let ch = b[p] as u32;
if ch == 10 || ch == 13 {
pos = p + 1; if ch == 13 && pos < n && b[pos] == 10 { pos += 1; } continue;
}
if ch == 9 {
let mut bb = p;
while bb < n && (b[bb] == 32 || b[bb] == 9) { bb += 1; }
if bb >= n { pos = bb; continue; }
let bc = b[bb] as u32;
if bc == 10 || bc == 13 {
pos = bb + 1; if bc == 13 && pos < n && b[pos] == 10 { pos += 1; } continue;
}
}
${commentSkip} pos = p;
if st.emitted_content { st.emit(_NLTOK, &src[pos..pos], pos, pos); }
st.line_start = false;
continue;
}
`,
ws: ` if c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos += 1; continue; }
if c == 10 || c == 13 {
pos += 1; if c == 13 && pos < n && b[pos] == 10 { pos += 1; }
if st.flow_depth == 0 { st.line_start = true; } else { st.pending_nl = true; }
continue;
}
`,
hooks: ` if kind != _NLTOK { self.emitted_content = true; }
if kind == "" && _in(_FLOW_OPEN, text) { self.flow_depth += 1; }
else if kind == "" && _in(_FLOW_CLOSE, text) { self.flow_depth = (self.flow_depth - 1).max(0); }
`,
};
}

function lexer(ir: ParserIR): string {
const defs: string[] = [];
const rx = ir.regexCtx;
const tpl = ir.tpl;
const stateful = !!(rx || tpl);
const toks = ir.tokens.map((t) => scanTok(t, defs, rx?.regexToken, tpl?.token)).join('\n');
const nl = ir.newlineCfg;
const nlRs = nl ? newlinePartsRs(nl) : null;
const stateful = !!(rx || tpl || nl);
const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n');
const puncts = ir.puncts.map((p) =>
` if src[pos..].starts_with(${J(p)}) { ${stateful ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n');
const rsArr = (a: string[]) => `&[${a.map(J).join(', ')}]`;
Expand All @@ -106,7 +155,8 @@ const _MEM: &[&str] = ${rsArr(rx.memberAccess)};
const _PAV: &[&str] = ${rsArr(rx.postfixAfterValue)};
const _IDENT: &str = ${J(rx.identToken)};
fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) }
` : '';
${nlRs ? nlRs.consts : ''}` : (nlRs ? `fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) }
${nlRs.consts}` : '');
const tplFn = tpl ? `fn _scan_tpl_span(s: &str, mut p: usize) -> (bool, usize) {
let n = s.len();
while p < n {
Expand All @@ -120,7 +170,8 @@ fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) }
` : '';
const fields = ['toks: Vec<Tok<\'a>>', 'pending_nl: bool',
rx ? 'prev_text: &\'a str, prev_kind: &\'static str, bp_text: &\'a str, has_prev: bool, has_prev2: bool, paren_head: Vec<bool>, last_close: bool, last_bang: bool' : '',
tpl ? 'template_stack: Vec<i64>' : ''].filter(Boolean).join(', ');
tpl ? 'template_stack: Vec<i64>' : '',
nlRs ? nlRs.fields : ''].filter(Boolean).join(', ');
const prevIsValue = rx ? ` fn prev_is_value(&self) -> bool {
if !self.has_prev { return false; }
if _in(_PAV, self.prev_text) { return self.last_bang; }
Expand All @@ -134,6 +185,7 @@ fn _in(set: &[&str], x: &str) -> bool { set.iter().any(|s| *s == x) }
else if text == ")" { self.last_close = self.paren_head.pop().unwrap_or(false); }
if _in(_PAV, text) { self.last_bang = self.prev_is_value(); }` : '',
tpl ? ` if !self.template_stack.is_empty() { if text == ${J(tpl.braceOpen)} { *self.template_stack.last_mut().unwrap() += 1; } else if text == ${J(tpl.interpClose)} { *self.template_stack.last_mut().unwrap() -= 1; } }` : '',
nlRs ? nlRs.hooks : '',
].filter(Boolean).join('\n');
const emitTail = rx ? `
self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true;` : '';
Expand All @@ -147,7 +199,8 @@ ${emitHooks}
` : '';
const initFields = ['toks: Vec::new()', 'pending_nl: false',
rx ? 'prev_text: "", prev_kind: "", bp_text: "", has_prev: false, has_prev2: false, paren_head: Vec::new(), last_close: false, last_bang: false' : '',
tpl ? 'template_stack: Vec::new()' : ''].filter(Boolean).join(', ');
tpl ? 'template_stack: Vec::new()' : '',
nlRs ? nlRs.init : ''].filter(Boolean).join(', ');
const open = stateful ? ` let mut st = LexState { ${initFields} };` : ` let mut toks: Vec<Tok> = Vec::new();\n let mut pending_nl = false;`;
const nlVar = stateful ? 'st.pending_nl' : 'pending_nl';
const tplDispatch = tpl ? ` if !st.template_stack.is_empty() && src[pos..].starts_with(${J(tpl.interpClose)}) && *st.template_stack.last().unwrap() == 0 {
Expand All @@ -162,17 +215,19 @@ ${emitHooks}
pos = e; continue;
}
` : '';
const nlBoundary = nlRs ? nlRs.boundary : '';
const nlWs = nlRs ? nlRs.ws : ` if c == 32 || c == 9 { pos += 1; continue; }
if pos + 2 < n && b[pos] == 0xE2 && b[pos + 1] == 0x80 && (b[pos + 2] == 0xA8 || b[pos + 2] == 0xA9) { ${nlVar} = true; pos += 3; continue; } // LS/PS (UTF-8)
if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; } // LF/CR
`;
return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${stateImpl}fn lex<'a>(src: &'a str) -> Vec<Tok<'a>> {
let b = src.as_bytes();
let n = b.len();
${open}
let mut pos = 0usize;
while pos < n {
let c = b[pos] as u32;
if c == 32 || c == 9 { pos += 1; continue; }
if pos + 2 < n && b[pos] == 0xE2 && b[pos + 1] == 0x80 && (b[pos + 2] == 0xA8 || b[pos + 2] == 0xA9) { ${nlVar} = true; pos += 3; continue; } // LS/PS (UTF-8)
if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; } // LF/CR
${tplDispatch}${toks}
${nlBoundary} let c = b[pos] as u32;
${nlWs}${tplDispatch}${toks}
${puncts}
panic!("lex error at {}", pos);
}
Expand Down
Loading
Loading