From 59f354a07a2efd44f576c005e0bd322814b1b8f1 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 10 Sep 2026 22:41:30 -0700 Subject: [PATCH 01/10] Add API for user defined grmtools section entries in GrammarAST --- cfgrammar/src/lib/header.rs | 2 +- cfgrammar/src/lib/markmap.rs | 2 +- cfgrammar/src/lib/yacc/ast.rs | 295 ++++++++++++++++++++++++++++++- cfgrammar/src/lib/yacc/parser.rs | 3 +- 4 files changed, 298 insertions(+), 4 deletions(-) diff --git a/cfgrammar/src/lib/header.rs b/cfgrammar/src/lib/header.rs index 645277f43..05a9ab19f 100644 --- a/cfgrammar/src/lib/header.rs +++ b/cfgrammar/src/lib/header.rs @@ -48,7 +48,7 @@ impl Spanned for HeaderError { // This is essentially a tuple that needs a newtype so we can implement `From` for it. // Thus we aren't worried about it being `pub`. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] #[doc(hidden)] pub struct HeaderValue(pub T, pub Value); diff --git a/cfgrammar/src/lib/markmap.rs b/cfgrammar/src/lib/markmap.rs index b2f910290..bf367751f 100644 --- a/cfgrammar/src/lib/markmap.rs +++ b/cfgrammar/src/lib/markmap.rs @@ -21,7 +21,7 @@ use std::fmt; /// /// Merge behaviors configure how the merge operator handles cases where both `MarkMaps` being merged /// contain a particular key. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] #[doc(hidden)] pub struct MarkMap { default_merge_behavior: MergeBehavior, diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index fb1ef5cb2..f42735b05 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -14,7 +14,7 @@ use super::{ use crate::{ Span, - header::{GrmtoolsSectionParser, HeaderError, HeaderErrorKind, HeaderValue}, + header::{GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Value}, yacc::YaccOriginalActionKind, }; @@ -178,6 +178,7 @@ pub struct GrammarAST { // The set of symbol names that, if unused in a // grammar, will not cause a warning or error. pub expect_unused: Vec, + pub grmtools_section: Option>, } #[derive(Debug, Clone)] @@ -255,6 +256,7 @@ impl GrammarAST { parse_generics: None, programs: None, expect_unused: Vec::new(), + grmtools_section: None, } } @@ -543,6 +545,39 @@ impl GrammarAST { }), ) } + + /// Performs a lookup in the grmtools section for an entry with the key `crate_name.key_name` and returns it. + /// If the entry is found it marks the key as `used`, for the purposes of `unused_grmtools_section_keys_for_crate`. + pub fn grmtools_section_value_for_crate( + &mut self, + crate_name: &str, + key_name: &str, + ) -> Option<(Span, &Value)> { + let key = format!("{crate_name}.{key_name}"); + if let Some(HeaderValue(span, value)) = self.grmtools_section.as_mut().and_then(|map| { + map.mark_used(&key); + map.get(&key) + }) { + Some((*span, value)) + } else { + None + } + } + + pub fn unused_grmtools_section_keys_for_crate(&self, crate_name: &str) -> Vec { + if let Some(map) = &self.grmtools_section { + map.unused() + .iter() + .filter(|key_name| { + let crate_prefix = format!("{crate_name}."); + key_name.starts_with(&crate_prefix) + }) + .cloned() + .collect::>() + } else { + vec![] + } + } } #[cfg(test)] @@ -984,4 +1019,262 @@ start -> () : "a" {$;;;; }; }] ); } + + #[test] + fn test_grmtools_section_values() { + use super::*; + use crate::header::Value; + let src = r#" +%grmtools { + yacckind: Grmtools, + lrpar.recoverer: CPCTPlus, + test.Flag, + !test.Negative, + test.string: "Foo", + test.vec: ["Aaaa", "Bbbb"], + test.num: 1234, + test.unused: 5678 +} +%token a +%% +start -> () : "a" { () }; +"#; + let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); + let test_flag_span = src.find_span("test.Flag"); + let test_neg_span = src.find_span("test.Negative"); + let test_neg_val_span = src.find_span("!test.Negative"); + let test_string_span = src.find_span("test.string"); + let test_string_val_span = src.find_span("Foo"); + let test_vec_span = src.find_span("test.vec"); + let test_vec_a_span = src.find_span("Aaaa"); + let test_vec_b_span = src.find_span("Bbbb"); + let test_vec_val_span = src.find_span("[\"Aaaa\", \"Bbbb\"]"); + let test_num_span = src.find_span("test.num"); + let test_num_val_span = src.find_span("1234"); + let mut test_crate_expected = HashMap::new(); + test_crate_expected.insert( + "Flag".to_string(), + (test_flag_span, Value::Bool(true, test_flag_span)), + ); + test_crate_expected.insert( + "Negative".to_string(), + (test_neg_span, Value::Bool(false, test_neg_val_span)), + ); + test_crate_expected.insert( + "string".to_string(), + ( + test_string_span, + Value::String("Foo".to_string(), test_string_val_span), + ), + ); + test_crate_expected.insert( + "vec".to_string(), + ( + test_vec_span, + Value::Array( + vec![ + Value::String("Aaaa".to_string(), test_vec_a_span), + Value::String("Bbbb".to_string(), test_vec_b_span), + ], + test_vec_val_span, + ), + ), + ); + test_crate_expected.insert( + "num".to_string(), + (test_num_span, Value::Num(1234, test_num_val_span)), + ); + for (key, (expected_span, expected_value)) in test_crate_expected { + let value = ast_validity + .ast + .grmtools_section_value_for_crate("test", &key); + assert_eq!(value, Some((expected_span, &expected_value))); + } + assert_eq!( + ast_validity + .ast + .unused_grmtools_section_keys_for_crate("test"), + vec!["test.unused"] + ); + + let mut cfgrammar_crate_expected = HashMap::new(); + let yacckind_span = src.find_span("yacckind"); + let yacckind_val_span = src.find_span("Grmtools"); + cfgrammar_crate_expected.insert( + "yacckind".to_string(), + ( + yacckind_span, + // The actual value we receive has been lower cased + Value::Namespaced("Grmtools".to_string(), yacckind_val_span), + ), + ); + for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { + let value = ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", &key); + assert_eq!(value, Some((expected_span, &expected_value))); + } + assert!( + ast_validity + .ast + .unused_grmtools_section_keys_for_crate("cfgrammar") + .is_empty() + ); + + let mut lrpar_crate_expected = HashMap::new(); + let recoverer_span = src.find_span("lrpar.recoverer"); + let recoverer_val_span = src.find_span("CPCTPlus"); + lrpar_crate_expected.insert( + "recoverer".to_string(), + ( + recoverer_span, + Value::Namespaced("CPCTPlus".to_string(), recoverer_val_span), + ), + ); + for (key, (expected_span, expected_value)) in lrpar_crate_expected { + let value = ast_validity + .ast + .grmtools_section_value_for_crate("lrpar", &key); + assert_eq!(value, Some((expected_span, &expected_value))); + } + assert!( + ast_validity + .ast + .unused_grmtools_section_keys_for_crate("lrpar") + .is_empty() + ); + } + + #[test] + fn test_grmtools_section_values2() { + use super::*; + let src = r#" +%grmtools { + yacckind: Original(YaccOriginalActionKind::UserAction), +} +%token a +%actiontype () +%% +start: "a" { () }; +"#; + let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); + let mut cfgrammar_crate_expected = HashMap::new(); + let yacckind_span = src.find_span("yacckind"); + let yacckind_val_span = src.find_span("Original(YaccOriginalActionKind::UserAction)"); + cfgrammar_crate_expected.insert( + "yacckind".to_string(), + ( + yacckind_span, + Value::Namespaced( + "Original(YaccOriginalActionKind::UserAction)".to_string(), + yacckind_val_span, + ), + ), + ); + for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { + let value = ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", &key); + assert_eq!(value, Some((expected_span, &expected_value))); + } + assert!( + ast_validity + .ast + .unused_grmtools_section_keys_for_crate("cfgrammar") + .is_empty() + ); + } + + #[test] + fn test_grmtools_section_values3() { + use super::*; + let src = r#" +%grmtools { + yacckind: YaccKind::Original(YaccOriginalActionKind::UserAction), +} +%token a +%actiontype () +%% +start: "a" { () }; +"#; + let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); + let mut cfgrammar_crate_expected = HashMap::new(); + let yacckind_span = src.find_span("yacckind"); + let yacckind_val_span = + src.find_span("YaccKind::Original(YaccOriginalActionKind::UserAction)"); + cfgrammar_crate_expected.insert( + "yacckind".to_string(), + ( + yacckind_span, + Value::Namespaced( + "YaccKind::Original(YaccOriginalActionKind::UserAction)".to_string(), + yacckind_val_span, + ), + ), + ); + for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { + let value = ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", &key); + assert_eq!(value, Some((expected_span, &expected_value))); + } + assert!( + ast_validity + .ast + .unused_grmtools_section_keys_for_crate("cfgrammar") + .is_empty() + ); + } + + #[test] + fn test_grmtools_section_values4() { + use super::*; + let src = r#" +%grmtools { + yacckind: YaccKind::Original(UserAction), +} +%token a +%actiontype () +%% +start: "a" { () }; +"#; + let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); + let mut cfgrammar_crate_expected = HashMap::new(); + let yacckind_span = src.find_span("yacckind"); + let yacckind_val_span = src.find_span("YaccKind::Original(UserAction)"); + cfgrammar_crate_expected.insert( + "yacckind".to_string(), + ( + yacckind_span, + Value::Namespaced( + "YaccKind::Original(UserAction)".to_string(), + yacckind_val_span, + ), + ), + ); + for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { + let value = ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", &key); + assert_eq!(value, Some((expected_span, &expected_value))); + } + assert!( + ast_validity + .ast + .unused_grmtools_section_keys_for_crate("cfgrammar") + .is_empty() + ); + } + + trait FindSpan { + fn find_span(&self, s: &str) -> Span; + } + + impl FindSpan for &'_ str { + #[track_caller] + fn find_span(&self, s: &str) -> Span { + let start_pos = self.find(s).unwrap(); + Span::new(start_pos, start_pos + s.len()) + } + } } diff --git a/cfgrammar/src/lib/yacc/parser.rs b/cfgrammar/src/lib/yacc/parser.rs index 4ee8eec99..13197929b 100644 --- a/cfgrammar/src/lib/yacc/parser.rs +++ b/cfgrammar/src/lib/yacc/parser.rs @@ -337,9 +337,10 @@ impl YaccParser<'_> { pub(crate) fn parse(&mut self) -> YaccGrammarResult { let mut errs = Vec::new(); - let (_, pos) = GrmtoolsSectionParser::new(self.src, false) + let (header, pos) = GrmtoolsSectionParser::new(self.src, false) .parse() .map_err(|mut errs| errs.drain(..).map(|e| e.into()).collect::>())?; + self.ast.grmtools_section = Some(header); // We pass around an index into the *bytes* of self.src. We guarantee that at all times // this points to the beginning of a UTF-8 character (since multibyte characters exist, not // every byte within the string is also a valid character). From eff44808165a9d16215938ae0840d3e267a48501 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 10 Sep 2026 23:49:12 -0700 Subject: [PATCH 02/10] Clean up new test cases --- cfgrammar/src/lib/yacc/ast.rs | 204 ++++++++++++++-------------------- 1 file changed, 83 insertions(+), 121 deletions(-) diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index f42735b05..bf30e6ff6 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -1040,51 +1040,49 @@ start -> () : "a" {$;;;; }; start -> () : "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); - let test_flag_span = src.find_span("test.Flag"); - let test_neg_span = src.find_span("test.Negative"); - let test_neg_val_span = src.find_span("!test.Negative"); - let test_string_span = src.find_span("test.string"); - let test_string_val_span = src.find_span("Foo"); - let test_vec_span = src.find_span("test.vec"); - let test_vec_a_span = src.find_span("Aaaa"); - let test_vec_b_span = src.find_span("Bbbb"); - let test_vec_val_span = src.find_span("[\"Aaaa\", \"Bbbb\"]"); - let test_num_span = src.find_span("test.num"); - let test_num_val_span = src.find_span("1234"); - let mut test_crate_expected = HashMap::new(); - test_crate_expected.insert( - "Flag".to_string(), - (test_flag_span, Value::Bool(true, test_flag_span)), - ); - test_crate_expected.insert( - "Negative".to_string(), - (test_neg_span, Value::Bool(false, test_neg_val_span)), - ); - test_crate_expected.insert( - "string".to_string(), + for (key, (expected_span, expected_value)) in [ ( - test_string_span, - Value::String("Foo".to_string(), test_string_val_span), + "Flag".to_string(), + ( + src.find_span("test.Flag"), + Value::Bool(true, src.find_span("test.Flag")), + ), ), - ); - test_crate_expected.insert( - "vec".to_string(), ( - test_vec_span, - Value::Array( - vec![ - Value::String("Aaaa".to_string(), test_vec_a_span), - Value::String("Bbbb".to_string(), test_vec_b_span), - ], - test_vec_val_span, + "Negative".to_string(), + ( + src.find_span("test.Negative"), + Value::Bool(false, src.find_span("!test.Negative")), ), ), - ); - test_crate_expected.insert( - "num".to_string(), - (test_num_span, Value::Num(1234, test_num_val_span)), - ); - for (key, (expected_span, expected_value)) in test_crate_expected { + ( + "string".to_string(), + ( + src.find_span("test.string"), + Value::String("Foo".to_string(), src.find_span("Foo")), + ), + ), + ( + "vec".to_string(), + ( + src.find_span("test.vec"), + Value::Array( + vec![ + Value::String("Aaaa".to_string(), src.find_span("Aaaa")), + Value::String("Bbbb".to_string(), src.find_span("Bbbb")), + ], + src.find_span("[\"Aaaa\", \"Bbbb\"]"), + ), + ), + ), + ( + "num".to_string(), + ( + src.find_span("test.num"), + Value::Num(1234, src.find_span("1234")), + ), + ), + ] { let value = ast_validity .ast .grmtools_section_value_for_crate("test", &key); @@ -1096,24 +1094,16 @@ start -> () : "a" { () }; .unused_grmtools_section_keys_for_crate("test"), vec!["test.unused"] ); - - let mut cfgrammar_crate_expected = HashMap::new(); - let yacckind_span = src.find_span("yacckind"); - let yacckind_val_span = src.find_span("Grmtools"); - cfgrammar_crate_expected.insert( - "yacckind".to_string(), - ( - yacckind_span, - // The actual value we receive has been lower cased - Value::Namespaced("Grmtools".to_string(), yacckind_val_span), - ), - ); - for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { - let value = ast_validity + assert_eq!( + ast_validity .ast - .grmtools_section_value_for_crate("cfgrammar", &key); - assert_eq!(value, Some((expected_span, &expected_value))); - } + .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + Some(( + src.find_span("yacckind"), + &Value::Namespaced("Grmtools".to_string(), src.find_span("Grmtools")) + )) + ); + assert!( ast_validity .ast @@ -1121,22 +1111,16 @@ start -> () : "a" { () }; .is_empty() ); - let mut lrpar_crate_expected = HashMap::new(); - let recoverer_span = src.find_span("lrpar.recoverer"); - let recoverer_val_span = src.find_span("CPCTPlus"); - lrpar_crate_expected.insert( - "recoverer".to_string(), - ( - recoverer_span, - Value::Namespaced("CPCTPlus".to_string(), recoverer_val_span), - ), - ); - for (key, (expected_span, expected_value)) in lrpar_crate_expected { - let value = ast_validity + assert_eq!( + ast_validity .ast - .grmtools_section_value_for_crate("lrpar", &key); - assert_eq!(value, Some((expected_span, &expected_value))); - } + .grmtools_section_value_for_crate("lrpar", "recoverer"), + Some(( + src.find_span("lrpar.recoverer"), + &Value::Namespaced("CPCTPlus".to_string(), src.find_span("CPCTPlus")) + )) + ); + assert!( ast_validity .ast @@ -1158,25 +1142,18 @@ start -> () : "a" { () }; start: "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); - let mut cfgrammar_crate_expected = HashMap::new(); - let yacckind_span = src.find_span("yacckind"); - let yacckind_val_span = src.find_span("Original(YaccOriginalActionKind::UserAction)"); - cfgrammar_crate_expected.insert( - "yacckind".to_string(), - ( - yacckind_span, - Value::Namespaced( + assert_eq!( + ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + Some(( + src.find_span("yacckind"), + &Value::Namespaced( "Original(YaccOriginalActionKind::UserAction)".to_string(), - yacckind_val_span, + src.find_span("Original(YaccOriginalActionKind::UserAction)"), ), - ), + )) ); - for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { - let value = ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", &key); - assert_eq!(value, Some((expected_span, &expected_value))); - } assert!( ast_validity .ast @@ -1198,26 +1175,18 @@ start: "a" { () }; start: "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); - let mut cfgrammar_crate_expected = HashMap::new(); - let yacckind_span = src.find_span("yacckind"); - let yacckind_val_span = - src.find_span("YaccKind::Original(YaccOriginalActionKind::UserAction)"); - cfgrammar_crate_expected.insert( - "yacckind".to_string(), - ( - yacckind_span, - Value::Namespaced( + assert_eq!( + ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + Some(( + src.find_span("yacckind"), + &Value::Namespaced( "YaccKind::Original(YaccOriginalActionKind::UserAction)".to_string(), - yacckind_val_span, + src.find_span("YaccKind::Original(YaccOriginalActionKind::UserAction)"), ), - ), + )) ); - for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { - let value = ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", &key); - assert_eq!(value, Some((expected_span, &expected_value))); - } assert!( ast_validity .ast @@ -1239,25 +1208,18 @@ start: "a" { () }; start: "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); - let mut cfgrammar_crate_expected = HashMap::new(); - let yacckind_span = src.find_span("yacckind"); - let yacckind_val_span = src.find_span("YaccKind::Original(UserAction)"); - cfgrammar_crate_expected.insert( - "yacckind".to_string(), - ( - yacckind_span, - Value::Namespaced( + assert_eq!( + ast_validity + .ast + .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + Some(( + src.find_span("yacckind"), + &Value::Namespaced( "YaccKind::Original(UserAction)".to_string(), - yacckind_val_span, + src.find_span("YaccKind::Original(UserAction)"), ), - ), + )) ); - for (key, (expected_span, expected_value)) in cfgrammar_crate_expected { - let value = ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", &key); - assert_eq!(value, Some((expected_span, &expected_value))); - } assert!( ast_validity .ast From 14537c3c3df7ab12215179f8a086c2081d408745 Mon Sep 17 00:00:00 2001 From: matt rice Date: Fri, 11 Sep 2026 00:03:43 -0700 Subject: [PATCH 03/10] Remove unneeded `to_string()` in test --- cfgrammar/src/lib/yacc/ast.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index bf30e6ff6..7742a8527 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -1042,28 +1042,28 @@ start -> () : "a" { () }; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); for (key, (expected_span, expected_value)) in [ ( - "Flag".to_string(), + "Flag", ( src.find_span("test.Flag"), Value::Bool(true, src.find_span("test.Flag")), ), ), ( - "Negative".to_string(), + "Negative", ( src.find_span("test.Negative"), Value::Bool(false, src.find_span("!test.Negative")), ), ), ( - "string".to_string(), + "string", ( src.find_span("test.string"), Value::String("Foo".to_string(), src.find_span("Foo")), ), ), ( - "vec".to_string(), + "vec", ( src.find_span("test.vec"), Value::Array( @@ -1076,7 +1076,7 @@ start -> () : "a" { () }; ), ), ( - "num".to_string(), + "num", ( src.find_span("test.num"), Value::Num(1234, src.find_span("1234")), @@ -1085,7 +1085,7 @@ start -> () : "a" { () }; ] { let value = ast_validity .ast - .grmtools_section_value_for_crate("test", &key); + .grmtools_section_value_for_crate("test", key); assert_eq!(value, Some((expected_span, &expected_value))); } assert_eq!( From a96e799d62cb5b83f8befbd47fa0f33cd24db537 Mon Sep 17 00:00:00 2001 From: matt rice Date: Fri, 11 Sep 2026 02:52:21 -0700 Subject: [PATCH 04/10] Move ownership of field to ASTWithValidityInfo --- cfgrammar/src/lib/yacc/ast.rs | 107 ++++++++++++------------------- cfgrammar/src/lib/yacc/parser.rs | 13 ++-- 2 files changed, 50 insertions(+), 70 deletions(-) diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index 7742a8527..0bab7a686 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -39,6 +39,7 @@ impl fmt::Display for ASTModificationError { pub struct ASTWithValidityInfo { yacc_kind: YaccKind, ast: GrammarAST, + grmtools_section: Header, errs: Vec, } @@ -52,18 +53,19 @@ impl ASTWithValidityInfo { /// already extracted the `YaccKind` if any. pub fn new(yacc_kind: YaccKind, s: &str) -> Self { let mut errs = Vec::new(); - let ast = { + let (ast, grmtools_section) = { let mut yp = YaccParser::new(yacc_kind, s); yp.parse().map_err(|e| errs.extend(e)).ok(); - let mut ast = yp.build(); + let (mut ast, grmtools_section) = yp.build(); ast.complete_and_validate(Some(yacc_kind)) .map_err(|e| errs.push(e)) .ok(); - ast + (ast, grmtools_section) }; ASTWithValidityInfo { ast, errs, + grmtools_section, yacc_kind, } } @@ -108,6 +110,34 @@ impl ASTWithValidityInfo { }) } } + + /// Performs a lookup in the grmtools section for an entry with the key `crate_name.key_name` and returns it. + /// If the entry is found it marks the key as `used`, for the purposes of `unused_grmtools_section_keys_for_crate`. + pub fn grmtools_section_value_for_crate( + &mut self, + crate_name: &str, + key_name: &str, + ) -> Option<(Span, &Value)> { + let key = format!("{crate_name}.{key_name}"); + self.grmtools_section.mark_used(&key); + if let Some(HeaderValue(span, value)) = self.grmtools_section.get(&key) { + Some((*span, value)) + } else { + None + } + } + + pub fn unused_grmtools_section_keys_for_crate(&self, crate_name: &str) -> Vec { + self.grmtools_section + .unused() + .iter() + .filter(|key_name| { + let crate_prefix = format!("{crate_name}."); + key_name.starts_with(&crate_prefix) + }) + .cloned() + .collect::>() + } } impl FromStr for ASTWithValidityInfo { @@ -124,7 +154,7 @@ impl FromStr for ASTWithValidityInfo { // We don't want to strip off the header so that span's will be correct. let mut yp = YaccParser::new(yacc_kind, src); yp.parse().map_err(|e| errs.extend(e)).ok(); - let mut ast = yp.build(); + let (mut ast, _) = yp.build(); ast.complete_and_validate(Some(yacc_kind)) .map_err(|e| errs.push(e)) .ok(); @@ -133,6 +163,7 @@ impl FromStr for ASTWithValidityInfo { Ok(ASTWithValidityInfo { ast, errs, + grmtools_section: header, yacc_kind, }) } else { @@ -178,7 +209,6 @@ pub struct GrammarAST { // The set of symbol names that, if unused in a // grammar, will not cause a warning or error. pub expect_unused: Vec, - pub grmtools_section: Option>, } #[derive(Debug, Clone)] @@ -256,7 +286,6 @@ impl GrammarAST { parse_generics: None, programs: None, expect_unused: Vec::new(), - grmtools_section: None, } } @@ -545,39 +574,6 @@ impl GrammarAST { }), ) } - - /// Performs a lookup in the grmtools section for an entry with the key `crate_name.key_name` and returns it. - /// If the entry is found it marks the key as `used`, for the purposes of `unused_grmtools_section_keys_for_crate`. - pub fn grmtools_section_value_for_crate( - &mut self, - crate_name: &str, - key_name: &str, - ) -> Option<(Span, &Value)> { - let key = format!("{crate_name}.{key_name}"); - if let Some(HeaderValue(span, value)) = self.grmtools_section.as_mut().and_then(|map| { - map.mark_used(&key); - map.get(&key) - }) { - Some((*span, value)) - } else { - None - } - } - - pub fn unused_grmtools_section_keys_for_crate(&self, crate_name: &str) -> Vec { - if let Some(map) = &self.grmtools_section { - map.unused() - .iter() - .filter(|key_name| { - let crate_prefix = format!("{crate_name}."); - key_name.starts_with(&crate_prefix) - }) - .cloned() - .collect::>() - } else { - vec![] - } - } } #[cfg(test)] @@ -1083,21 +1079,15 @@ start -> () : "a" { () }; ), ), ] { - let value = ast_validity - .ast - .grmtools_section_value_for_crate("test", key); + let value = ast_validity.grmtools_section_value_for_crate("test", key); assert_eq!(value, Some((expected_span, &expected_value))); } assert_eq!( - ast_validity - .ast - .unused_grmtools_section_keys_for_crate("test"), + ast_validity.unused_grmtools_section_keys_for_crate("test"), vec!["test.unused"] ); assert_eq!( - ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + ast_validity.grmtools_section_value_for_crate("cfgrammar", "yacckind"), Some(( src.find_span("yacckind"), &Value::Namespaced("Grmtools".to_string(), src.find_span("Grmtools")) @@ -1106,15 +1096,12 @@ start -> () : "a" { () }; assert!( ast_validity - .ast .unused_grmtools_section_keys_for_crate("cfgrammar") .is_empty() ); assert_eq!( - ast_validity - .ast - .grmtools_section_value_for_crate("lrpar", "recoverer"), + ast_validity.grmtools_section_value_for_crate("lrpar", "recoverer"), Some(( src.find_span("lrpar.recoverer"), &Value::Namespaced("CPCTPlus".to_string(), src.find_span("CPCTPlus")) @@ -1123,7 +1110,6 @@ start -> () : "a" { () }; assert!( ast_validity - .ast .unused_grmtools_section_keys_for_crate("lrpar") .is_empty() ); @@ -1143,9 +1129,7 @@ start: "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); assert_eq!( - ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + ast_validity.grmtools_section_value_for_crate("cfgrammar", "yacckind"), Some(( src.find_span("yacckind"), &Value::Namespaced( @@ -1156,7 +1140,6 @@ start: "a" { () }; ); assert!( ast_validity - .ast .unused_grmtools_section_keys_for_crate("cfgrammar") .is_empty() ); @@ -1176,9 +1159,7 @@ start: "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); assert_eq!( - ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + ast_validity.grmtools_section_value_for_crate("cfgrammar", "yacckind"), Some(( src.find_span("yacckind"), &Value::Namespaced( @@ -1189,7 +1170,6 @@ start: "a" { () }; ); assert!( ast_validity - .ast .unused_grmtools_section_keys_for_crate("cfgrammar") .is_empty() ); @@ -1209,9 +1189,7 @@ start: "a" { () }; "#; let mut ast_validity = ASTWithValidityInfo::from_str(src).unwrap(); assert_eq!( - ast_validity - .ast - .grmtools_section_value_for_crate("cfgrammar", "yacckind"), + ast_validity.grmtools_section_value_for_crate("cfgrammar", "yacckind"), Some(( src.find_span("yacckind"), &Value::Namespaced( @@ -1222,7 +1200,6 @@ start: "a" { () }; ); assert!( ast_validity - .ast .unused_grmtools_section_keys_for_crate("cfgrammar") .is_empty() ); diff --git a/cfgrammar/src/lib/yacc/parser.rs b/cfgrammar/src/lib/yacc/parser.rs index 13197929b..6604a03ae 100644 --- a/cfgrammar/src/lib/yacc/parser.rs +++ b/cfgrammar/src/lib/yacc/parser.rs @@ -16,7 +16,7 @@ use wincode::{SchemaRead, SchemaWrite}; use crate::{ Span, Spanned, - header::{GrmtoolsSectionParser, HeaderErrorKind}, + header::{GrmtoolsSectionParser, Header, HeaderErrorKind}, }; pub type YaccGrammarResult = Result>; @@ -294,6 +294,7 @@ pub(crate) struct YaccParser<'a> { src: &'a str, num_newlines: usize, ast: GrammarAST, + header: Option>, global_actiontype: Option<(String, Span)>, } @@ -331,6 +332,7 @@ impl YaccParser<'_> { src, num_newlines: 0, ast: GrammarAST::new(), + header: None, global_actiontype: None, } } @@ -340,7 +342,7 @@ impl YaccParser<'_> { let (header, pos) = GrmtoolsSectionParser::new(self.src, false) .parse() .map_err(|mut errs| errs.drain(..).map(|e| e.into()).collect::>())?; - self.ast.grmtools_section = Some(header); + self.header = Some(header); // We pass around an index into the *bytes* of self.src. We guarantee that at all times // this points to the beginning of a UTF-8 character (since multibyte characters exist, not // every byte within the string is also a valid character). @@ -372,8 +374,8 @@ impl YaccParser<'_> { } } - pub(crate) fn build(self) -> GrammarAST { - self.ast + pub(crate) fn build(self) -> (GrammarAST, Header) { + (self.ast, self.header.expect("set by parse()")) } fn parse_declarations( @@ -1084,7 +1086,8 @@ mod test { fn parse(yacc_kind: YaccKind, s: &str) -> Result> { let mut yp = YaccParser::new(yacc_kind, s); yp.parse()?; - Ok(yp.build()) + let (ast, _) = yp.build(); + Ok(ast) } fn rule(n: &str) -> Symbol { From 86a4457a22bda501212c28ee08f7800a23c8be2a Mon Sep 17 00:00:00 2001 From: matt rice Date: Fri, 11 Sep 2026 10:12:26 -0700 Subject: [PATCH 05/10] Use the grmtools section from `YaccParser::build` --- cfgrammar/src/lib/yacc/ast.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index 0bab7a686..8bc806c20 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -150,20 +150,20 @@ impl FromStr for ASTWithValidityInfo { .map_err(|mut errs| errs.drain(..).map(|e| e.into()).collect::>())?; if let Some(HeaderValue(_, yk_val)) = header.get("cfgrammar.yacckind") { let yacc_kind = YaccKind::try_from(yk_val).map_err(|e| vec![e.into()])?; - let ast = { + let (ast, grmtools_section) = { // We don't want to strip off the header so that span's will be correct. let mut yp = YaccParser::new(yacc_kind, src); yp.parse().map_err(|e| errs.extend(e)).ok(); - let (mut ast, _) = yp.build(); + let (mut ast, grmtools_section) = yp.build(); ast.complete_and_validate(Some(yacc_kind)) .map_err(|e| errs.push(e)) .ok(); - ast + (ast, grmtools_section) }; Ok(ASTWithValidityInfo { ast, errs, - grmtools_section: header, + grmtools_section, yacc_kind, }) } else { From adb16f4c6bd2c9ee6125dcad8f8b21317f845f35 Mon Sep 17 00:00:00 2001 From: matt rice Date: Sat, 12 Sep 2026 00:36:49 -0700 Subject: [PATCH 06/10] First attempt at allowing downstream crate keys in CTParserBuilder --- lrpar/src/lib/codegen.rs | 95 ++++++++++++++++++++++++++++++++++++-- lrpar/src/lib/ctbuilder.rs | 15 +++++- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index ecb342641..7651bb039 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -13,7 +13,7 @@ use crate::{ use cfgrammar::{ Location, RIdx, Span, Symbol, - header::{GrmtoolsSectionParser, Header, HeaderError, HeaderValue}, + header::{GrmtoolsSectionParser, Header, HeaderError, HeaderValue, RE_CRATE_DOT}, markmap::MergeError, yacc::{ YaccGrammar, YaccGrammarError, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo, @@ -32,6 +32,7 @@ const ACTIONS_KIND: &str = "__GtActionsKind"; const ACTIONS_KIND_PREFIX: &str = "Ak"; const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; +#[derive(Debug)] #[non_exhaustive] pub(crate) enum ParserSrcEnvError { GrmtoolsSectionParseError(Vec>), @@ -41,6 +42,7 @@ pub(crate) enum ParserSrcEnvError { MissingModName, } +#[derive(Debug)] #[non_exhaustive] pub(crate) enum ParserBuildEnvError where @@ -53,6 +55,7 @@ where GrmtoolsSectionMissingRequiredKeys(Vec), } +#[derive(Debug)] #[non_exhaustive] pub(crate) enum CodegenError { ProcMacro2Error(proc_macro2::LexError), @@ -456,8 +459,23 @@ where self.ast_with_validity_info.yacc_kind() } - pub(crate) fn check_unused_header_keys(&self) -> Result<(), ParserBuildEnvError> { - let unused_keys = self.header.unused(); + pub(crate) fn check_unused_header_keys_for_crate( + &self, + crate_name: Option<&str>, + ) -> Result<(), ParserBuildEnvError> { + let unused_keys = self + .header + .unused() + .iter() + .filter(|s| { + if let Some(crate_name) = crate_name { + s.starts_with(&format!("{crate_name}.")) + } else { + !RE_CRATE_DOT.is_match(s) + } + }) + .map(|s| s.to_string()) + .collect::>(); if !unused_keys.is_empty() { return Err(ParserBuildEnvError::GrmtoolsSectionUnusedKeys(unused_keys)); } @@ -1294,3 +1312,74 @@ pub(crate) fn make_generics(parse_generics: Option<&str>) -> Result)) } } + +#[cfg(test)] +mod test { + use crate::test_utils::TestLexerTypes; + use cfgrammar::{header::Header, span::Location}; + + use super::*; + #[test] + fn test_unused_crate_header_entry() { + let src = r#" + %grmtools{ + yacckind: Grmtools, + test.foo: "test crate value", + } + %% + start -> () : "A" { () }; + "#; + let empty_header = Header::::new(); + let src_env = ParserSrcEnv::::new_with_header(src, None, empty_header); + let build_env = src_env + .build_env(ParserBuildEnvArgs::new().mod_name(Some("test_module"))) + .unwrap(); + build_env + .check_unused_header_keys_for_crate(Some("cfgrammar")) + .unwrap(); + build_env + .check_unused_header_keys_for_crate(Some("lrpar")) + .unwrap(); + build_env + .check_unused_header_keys_for_crate(Some("lrlex")) + .unwrap(); + build_env.check_unused_header_keys_for_crate(None).unwrap(); + let codegen = build_env.code_generator("timestamp").unwrap(); + let out = codegen.generate(&build_env).unwrap(); + assert!(!out.is_empty()); + } + + #[test] + fn test_unused_header_entry() { + let src = r#" + %grmtools{ + yacckind: Grmtools, + testfoo: "values which do not specify a crate origin should show up as unused", + } + %% + start -> () : "A" { () }; + "#; + let empty_header = Header::::new(); + let src_env = ParserSrcEnv::::new_with_header(src, None, empty_header); + let build_env = src_env + .build_env(ParserBuildEnvArgs::new().mod_name(Some("test_module"))) + .unwrap(); + build_env + .check_unused_header_keys_for_crate(Some("cfgrammar")) + .unwrap(); + build_env + .check_unused_header_keys_for_crate(Some("lrpar")) + .unwrap(); + build_env + .check_unused_header_keys_for_crate(Some("lrlex")) + .unwrap(); + match build_env.check_unused_header_keys_for_crate(None) { + Err(ParserBuildEnvError::GrmtoolsSectionUnusedKeys(keys)) + if keys == vec!["testfoo".to_string()] => {} + _ => panic!("Unexpected return value for unused header keys check"), + } + let codegen = build_env.code_generator("timestamp").unwrap(); + let out = codegen.generate(&build_env).unwrap(); + assert!(!out.is_empty()); + } +} diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 0df742103..1d7a1feaf 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -747,10 +747,21 @@ where inspector_rt(build_env.header_mut(), rt, &rule_ids, grmp)? } + // Catch any typos in key names for cfgrammar or lrpar build_env - .check_unused_header_keys() + .check_unused_header_keys_for_crate(Some("cfgrammar")) + .map_err(|e| ErrorString(e.to_string()))?; + build_env + .check_unused_header_keys_for_crate(Some("lrpar")) + .map_err(|e| ErrorString(e.to_string()))?; + // Catch any stray lrlex keys that accidentally make their way into the parser src. + build_env + .check_unused_header_keys_for_crate(Some("lrlex")) + .map_err(|e| ErrorString(e.to_string()))?; + // Catch any stray keys without a crate prefix. + build_env + .check_unused_header_keys_for_crate(None) .map_err(|e| ErrorString(e.to_string()))?; - self.output_file( &code_gen, outp, From fd3fa865b61736efe9b8e7a4eda488a381e6755c Mon Sep 17 00:00:00 2001 From: matt rice Date: Sat, 12 Sep 2026 02:08:56 -0700 Subject: [PATCH 07/10] Use same naming convention as the codegen module --- cfgrammar/src/lib/yacc/ast.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index 8bc806c20..ae9a1fa22 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -112,7 +112,7 @@ impl ASTWithValidityInfo { } /// Performs a lookup in the grmtools section for an entry with the key `crate_name.key_name` and returns it. - /// If the entry is found it marks the key as `used`, for the purposes of `unused_grmtools_section_keys_for_crate`. + /// If the entry is found it marks the key as `used`, for the purposes of `unused_header_keys_for_crate`. pub fn grmtools_section_value_for_crate( &mut self, crate_name: &str, @@ -127,7 +127,7 @@ impl ASTWithValidityInfo { } } - pub fn unused_grmtools_section_keys_for_crate(&self, crate_name: &str) -> Vec { + pub fn unused_header_keys_for_crate(&self, crate_name: &str) -> Vec { self.grmtools_section .unused() .iter() @@ -1083,7 +1083,7 @@ start -> () : "a" { () }; assert_eq!(value, Some((expected_span, &expected_value))); } assert_eq!( - ast_validity.unused_grmtools_section_keys_for_crate("test"), + ast_validity.unused_header_keys_for_crate("test"), vec!["test.unused"] ); assert_eq!( @@ -1096,7 +1096,7 @@ start -> () : "a" { () }; assert!( ast_validity - .unused_grmtools_section_keys_for_crate("cfgrammar") + .unused_header_keys_for_crate("cfgrammar") .is_empty() ); @@ -1110,7 +1110,7 @@ start -> () : "a" { () }; assert!( ast_validity - .unused_grmtools_section_keys_for_crate("lrpar") + .unused_header_keys_for_crate("lrpar") .is_empty() ); } @@ -1140,7 +1140,7 @@ start: "a" { () }; ); assert!( ast_validity - .unused_grmtools_section_keys_for_crate("cfgrammar") + .unused_header_keys_for_crate("cfgrammar") .is_empty() ); } @@ -1170,7 +1170,7 @@ start: "a" { () }; ); assert!( ast_validity - .unused_grmtools_section_keys_for_crate("cfgrammar") + .unused_header_keys_for_crate("cfgrammar") .is_empty() ); } @@ -1200,7 +1200,7 @@ start: "a" { () }; ); assert!( ast_validity - .unused_grmtools_section_keys_for_crate("cfgrammar") + .unused_header_keys_for_crate("cfgrammar") .is_empty() ); } From fcba11b571ca53d7c7e49f91f25b6aae027f2dff Mon Sep 17 00:00:00 2001 From: matt rice Date: Sat, 12 Sep 2026 02:56:24 -0700 Subject: [PATCH 08/10] Preemptively call `mark_used` on grmtools crate entries in header --- cfgrammar/src/lib/yacc/parser.rs | 15 +++++- lrpar/src/lib/codegen.rs | 89 ++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/cfgrammar/src/lib/yacc/parser.rs b/cfgrammar/src/lib/yacc/parser.rs index 6604a03ae..39e8f60c9 100644 --- a/cfgrammar/src/lib/yacc/parser.rs +++ b/cfgrammar/src/lib/yacc/parser.rs @@ -16,7 +16,7 @@ use wincode::{SchemaRead, SchemaWrite}; use crate::{ Span, Spanned, - header::{GrmtoolsSectionParser, Header, HeaderErrorKind}, + header::{CRATE_KEY_MAP, GrmtoolsSectionParser, Header, HeaderErrorKind}, }; pub type YaccGrammarResult = Result>; @@ -375,7 +375,18 @@ impl YaccParser<'_> { } pub(crate) fn build(self) -> (GrammarAST, Header) { - (self.ast, self.header.expect("set by parse()")) + let mut header = self.header.expect("set by parse()"); + // Preemptively mark the keys for lrpar and cfgrammar as used in the header. + // If a downstream crate checks the keys in the ast. The lrpar crate works on a + // local instance which merges the keys from ast with keys from the `CTBuilder`. + // + // It is difficult to do later due to shared references. + for (key_name, crate_name) in CRATE_KEY_MAP.iter() { + if ["cfgrammar", "lrpar"].contains(crate_name) { + header.mark_used(&format!("{crate_name}.{key_name}")); + } + } + (self.ast, header) } fn parse_declarations( diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 7651bb039..8cf2cde65 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -1334,15 +1334,33 @@ mod test { let build_env = src_env .build_env(ParserBuildEnvArgs::new().mod_name(Some("test_module"))) .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("cfgrammar") + .is_empty() + ); build_env .check_unused_header_keys_for_crate(Some("cfgrammar")) .unwrap(); build_env .check_unused_header_keys_for_crate(Some("lrpar")) .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("lrpar") + .is_empty() + ); build_env .check_unused_header_keys_for_crate(Some("lrlex")) .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("lrpar") + .is_empty() + ); build_env.check_unused_header_keys_for_crate(None).unwrap(); let codegen = build_env.code_generator("timestamp").unwrap(); let out = codegen.generate(&build_env).unwrap(); @@ -1367,12 +1385,30 @@ mod test { build_env .check_unused_header_keys_for_crate(Some("cfgrammar")) .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("cfgrammar") + .is_empty() + ); build_env .check_unused_header_keys_for_crate(Some("lrpar")) .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("lrpar") + .is_empty() + ); build_env .check_unused_header_keys_for_crate(Some("lrlex")) .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("lrlex") + .is_empty() + ); match build_env.check_unused_header_keys_for_crate(None) { Err(ParserBuildEnvError::GrmtoolsSectionUnusedKeys(keys)) if keys == vec!["testfoo".to_string()] => {} @@ -1382,4 +1418,57 @@ mod test { let out = codegen.generate(&build_env).unwrap(); assert!(!out.is_empty()); } + + #[test] + fn test_unused_grmtools_header_entry() { + let src = r#" + %grmtools{ + yacckind: Grmtools, + cfgrammar.unknown: "should be unused", + lrpar.unknown: "should be unused", + + } + %% + start -> () : "A" { () }; + "#; + let empty_header = Header::::new(); + let src_env = ParserSrcEnv::::new_with_header(src, None, empty_header); + let build_env = src_env + .build_env(ParserBuildEnvArgs::new().mod_name(Some("test_module"))) + .unwrap(); + assert!( + build_env + .check_unused_header_keys_for_crate(Some("cfgrammar")) + .is_err() + ); + assert_eq!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("cfgrammar"), + vec!["cfgrammar.unknown"] + ); + assert!( + build_env + .check_unused_header_keys_for_crate(Some("lrpar")) + .is_err() + ); + assert_eq!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("lrpar"), + vec!["lrpar.unknown"] + ); + build_env + .check_unused_header_keys_for_crate(Some("lrlex")) + .unwrap(); + assert!( + build_env + .ast_with_validity_info() + .unused_header_keys_for_crate("lrlex") + .is_empty() + ); + let codegen = build_env.code_generator("timestamp").unwrap(); + let out = codegen.generate(&build_env).unwrap(); + assert!(!out.is_empty()); + } } From 751a4d4dcacaaadc704117ef24fde7eedc17eb75 Mon Sep 17 00:00:00 2001 From: matt rice Date: Sat, 12 Sep 2026 02:58:32 -0700 Subject: [PATCH 09/10] stray whitespace --- lrpar/src/lib/codegen.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 8cf2cde65..cbf180335 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -1426,7 +1426,6 @@ mod test { yacckind: Grmtools, cfgrammar.unknown: "should be unused", lrpar.unknown: "should be unused", - } %% start -> () : "A" { () }; From e8012c93efd8c71ef2657628599a3ef2f7db97a2 Mon Sep 17 00:00:00 2001 From: matt rice Date: Sat, 12 Sep 2026 03:26:08 -0700 Subject: [PATCH 10/10] Make unused header entries check return spans --- cfgrammar/src/lib/markmap.rs | 11 +++++++---- cfgrammar/src/lib/yacc/ast.rs | 13 ++++++++----- lrlex/src/lib/ctbuilder.rs | 7 ++++++- lrlex/src/main.rs | 6 +++++- lrpar/src/lib/codegen.rs | 23 +++++++++++++++++++---- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/cfgrammar/src/lib/markmap.rs b/cfgrammar/src/lib/markmap.rs index bf367751f..9e2a3d245 100644 --- a/cfgrammar/src/lib/markmap.rs +++ b/cfgrammar/src/lib/markmap.rs @@ -484,12 +484,15 @@ impl MarkMap { } /// Returns a `Vec` containing all the keys that are not marked as used. - pub fn unused(&self) -> Vec { + pub fn unused(&self) -> Vec<(K, V)> + where + V: Clone, + { let mut ret = Vec::new(); for (k, mark, v) in &self.contents { let used_mark = Mark::Used.repr(); if v.is_some() && mark & used_mark == 0 { - ret.push(k.to_owned()) + ret.push((k.to_owned(), v.as_ref().unwrap().clone())) } } ret @@ -711,7 +714,7 @@ mod test { assert!(mm.insert("a", "test").is_none()); mm.mark_used(&"a"); assert_eq!(mm.get_mark(&"a"), Some(Mark::Used.repr())); - let empty: &[&String] = &[]; + let empty: &[(&str, &str)] = &[]; assert_eq!(mm.unused().as_slice(), empty); } @@ -722,7 +725,7 @@ mod test { assert!(mm.insert("b", "unused").is_none()); assert_eq!(mm.get_mark(&"a"), Some(Mark::Used.repr())); assert_eq!(mm.get_mark(&"b"), Some(0)); - assert_eq!(mm.unused().as_slice(), &["b"]); + assert_eq!(mm.unused().as_slice(), &[("b", "unused")]); } } diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs index ae9a1fa22..f9a8bc92e 100644 --- a/cfgrammar/src/lib/yacc/ast.rs +++ b/cfgrammar/src/lib/yacc/ast.rs @@ -127,15 +127,18 @@ impl ASTWithValidityInfo { } } - pub fn unused_header_keys_for_crate(&self, crate_name: &str) -> Vec { + pub fn unused_header_keys_for_crate(&self, crate_name: &str) -> Vec<(String, Span)> { self.grmtools_section .unused() .iter() - .filter(|key_name| { + .filter_map(|(key_name, HeaderValue(key_span, _))| { let crate_prefix = format!("{crate_name}."); - key_name.starts_with(&crate_prefix) + if key_name.starts_with(&crate_prefix) { + Some((key_name.clone(), *key_span)) + } else { + None + } }) - .cloned() .collect::>() } } @@ -1084,7 +1087,7 @@ start -> () : "a" { () }; } assert_eq!( ast_validity.unused_header_keys_for_crate("test"), - vec!["test.unused"] + vec![("test.unused".to_string(), src.find_span("test.unused"))] ); assert_eq!( ast_validity.grmtools_section_value_for_crate("cfgrammar", "yacckind"), diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 7b6983e46..6a1eea4bd 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -520,7 +520,12 @@ where None }; - let unused_header_values = build_env.header().unused(); + let unused_header_values = build_env + .header() + .unused() + .iter() + .map(|(s, _)| s.to_string()) + .collect::>(); if !unused_header_values.is_empty() { return Err( format!("Unused header values: {}", unused_header_values.join(", ")).into(), diff --git a/lrlex/src/main.rs b/lrlex/src/main.rs index 30ebbb326..bb87599f2 100644 --- a/lrlex/src/main.rs +++ b/lrlex/src/main.rs @@ -123,7 +123,11 @@ fn main() -> Result<(), Box> { } }; { - let unused_header_values = header.unused(); + let unused_header_values = header + .unused() + .iter() + .map(|(s, _)| s.to_string()) + .collect::>(); if !unused_header_values.is_empty() { Err(ErrorString(format!( "Unused header values: {}", diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index cbf180335..c589a6643 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -467,14 +467,14 @@ where .header .unused() .iter() - .filter(|s| { + .filter(|(s, _)| { if let Some(crate_name) = crate_name { s.starts_with(&format!("{crate_name}.")) } else { !RE_CRATE_DOT.is_match(s) } }) - .map(|s| s.to_string()) + .map(|(s, _)| s.to_string()) .collect::>(); if !unused_keys.is_empty() { return Err(ParserBuildEnvError::GrmtoolsSectionUnusedKeys(unused_keys)); @@ -1444,7 +1444,10 @@ mod test { build_env .ast_with_validity_info() .unused_header_keys_for_crate("cfgrammar"), - vec!["cfgrammar.unknown"] + vec![( + "cfgrammar.unknown".to_string(), + src.find_span("cfgrammar.unknown") + )] ); assert!( build_env @@ -1455,7 +1458,7 @@ mod test { build_env .ast_with_validity_info() .unused_header_keys_for_crate("lrpar"), - vec!["lrpar.unknown"] + vec![("lrpar.unknown".to_string(), src.find_span("lrpar.unknown"))] ); build_env .check_unused_header_keys_for_crate(Some("lrlex")) @@ -1470,4 +1473,16 @@ mod test { let out = codegen.generate(&build_env).unwrap(); assert!(!out.is_empty()); } + + trait FindSpan { + fn find_span(&self, s: &str) -> Span; + } + + impl FindSpan for &'_ str { + #[track_caller] + fn find_span(&self, s: &str) -> Span { + let start_pos = self.find(s).unwrap(); + Span::new(start_pos, start_pos + s.len()) + } + } }