diff --git a/Cargo.lock b/Cargo.lock index 754dca177d3f..2d61c534f0c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3251,6 +3251,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.9", + "yeast-schema", ] [[package]] diff --git a/shared/yeast-macros/BUILD.bazel b/shared/yeast-macros/BUILD.bazel index 71bc6eb288ad..855e3962e0cc 100644 --- a/shared/yeast-macros/BUILD.bazel +++ b/shared/yeast-macros/BUILD.bazel @@ -8,5 +8,7 @@ rust_proc_macro( srcs = glob(["src/**/*.rs"]), aliases = aliases(), visibility = ["//visibility:public"], - deps = all_crate_deps(), + deps = all_crate_deps() + [ + "//shared/yeast-schema", + ], ) diff --git a/shared/yeast-macros/Cargo.toml b/shared/yeast-macros/Cargo.toml index d8ea435093e4..1d038714bda7 100644 --- a/shared/yeast-macros/Cargo.toml +++ b/shared/yeast-macros/Cargo.toml @@ -10,3 +10,4 @@ proc-macro = true proc-macro2 = "1.0" quote = "1.0" syn = { version = "3.0", features = ["full"] } +yeast-schema = { path = "../yeast-schema" } diff --git a/shared/yeast-macros/src/ast.rs b/shared/yeast-macros/src/ast.rs index f400257fd1d4..869ec001c632 100644 --- a/shared/yeast-macros/src/ast.rs +++ b/shared/yeast-macros/src/ast.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use proc_macro2::{Ident, Literal}; /// A complete rewrite rule such as `(input) where guard => (output)`. @@ -117,6 +119,8 @@ pub(crate) enum Content { Static(Literal), /// Computed leaf content such as `#{name}` in `(identifier #{name})`. Computed(syn::Block), + /// Value inherited from the node matched by the enclosing rule. + Inherited, } pub(crate) enum Field { @@ -137,3 +141,117 @@ pub(crate) enum Field { block: syn::Block, }, } + +impl Rule { + pub(crate) fn resolve_bare_leaf_content( + &mut self, + leaf_kinds: &BTreeSet, + ) -> syn::Result<()> { + let Replacement::Templates(templates) = &mut self.replacement else { + return Ok(()); + }; + for template in templates { + template.resolve_bare_leaf_content(leaf_kinds, true)?; + } + Ok(()) + } +} + +impl Template { + fn resolve_bare_leaf_content( + &mut self, + leaf_kinds: &BTreeSet, + is_rule_result: bool, + ) -> syn::Result<()> { + let Template::Node(node) = self else { + return Ok(()); + }; + + for field in &mut node.fields { + if let Field::Named { value, .. } = field { + value.resolve_bare_leaf_content(leaf_kinds, false)?; + } + } + + if node.content.is_none() + && node.fields.is_empty() + && leaf_kinds.contains(node.kind.to_string().as_str()) + { + if !is_rule_result { + return Err(syn::Error::new_spanned( + &node.kind, + format!( + "bare leaf `{}` must be a top-level rule result or have explicit content", + node.kind + ), + )); + } + node.content = Some(Content::Inherited); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use quote::quote; + + fn output_leaf_kinds() -> BTreeSet { + yeast_schema::node_types_yaml::leaf_kinds_from_yaml( + r#" +named: + assignment: + left: identifier + identifier: +"#, + ) + .unwrap() + } + + #[test] + fn marks_top_level_bare_leaves_as_inherited() { + let mut rule = crate::rule_parse::parse_rule(quote!((source) => (identifier))).unwrap(); + rule.resolve_bare_leaf_content(&output_leaf_kinds()) + .unwrap(); + + let Replacement::Templates(templates) = rule.replacement else { + panic!("expected template replacement"); + }; + let [Template::Node(node)] = templates.as_slice() else { + panic!("expected one output node"); + }; + assert!(matches!(node.content, Some(Content::Inherited))); + } + + #[test] + fn rejects_nested_bare_leaves() { + let mut rule = + crate::rule_parse::parse_rule(quote!((source) => (assignment left: (identifier)))) + .unwrap(); + let error = rule + .resolve_bare_leaf_content(&output_leaf_kinds()) + .unwrap_err(); + + assert_eq!( + error.to_string(), + "bare leaf `identifier` must be a top-level rule result or have explicit content" + ); + } + + #[test] + fn leaves_fieldless_structural_nodes_unchanged() { + let mut rule = crate::rule_parse::parse_rule(quote!((source) => (assignment))).unwrap(); + rule.resolve_bare_leaf_content(&output_leaf_kinds()) + .unwrap(); + + let Replacement::Templates(templates) = rule.replacement else { + panic!("expected template replacement"); + }; + let [Template::Node(node)] = templates.as_slice() else { + panic!("expected one output node"); + }; + assert!(node.content.is_none()); + } +} diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 7e7521a8b76f..2a3b379da7a1 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -201,8 +201,18 @@ pub fn rule(input: TokenStream) -> TokenStream { /// Paths are resolved relative to the consuming crate's `CARGO_MANIFEST_DIR` /// (the same convention `include_str!` uses for relative paths). The /// resolved paths are also emitted as `include_str!` references so the -/// consuming crate gets invalidated when a schema YAML changes, prepping -/// the ground for compile-time type-checking against those schemas. +/// consuming crate gets invalidated when a schema YAML changes. +/// +/// A bare output leaf used as a top-level rule result inherits the matched +/// input node's value: +/// +/// ```text +/// rule!((superExpr) => (super_expr)) +/// ``` +/// +/// Whether the output kind is a leaf is determined from the output schema. +/// Nested bare leaves are rejected at compile time because they have no +/// unambiguous input node from which to inherit a value. #[proc_macro] pub fn rules(input: TokenStream) -> TokenStream { let input2: TokenStream2 = input.into(); diff --git a/shared/yeast-macros/src/lower.rs b/shared/yeast-macros/src/lower.rs index e127423910d5..8321a64dee5a 100644 --- a/shared/yeast-macros/src/lower.rs +++ b/shared/yeast-macros/src/lower.rs @@ -380,6 +380,7 @@ impl Node { } } } + Content::Inherited => quote! { #ctx.inherited_literal(#kind_str) }, }; quote! { let __node: yeast::OutputId = #node; } } else { diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index becdd56831f7..ccc8b036ff4e 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -51,6 +51,25 @@ pub fn parse_rules_top(input: TokenStream) -> Result { }; let input_abs = resolve_path(&input_path.value); let output_abs = resolve_path(&output_path.value); + let output_yaml = std::fs::read_to_string(&output_abs).map_err(|error| { + syn::Error::new( + output_path.literal_span, + format!( + "rules!: failed to read output schema `{}`: {error}", + output_abs.display() + ), + ) + })?; + let output_leaf_kinds = yeast_schema::node_types_yaml::leaf_kinds_from_yaml(&output_yaml) + .map_err(|error| { + syn::Error::new( + output_path.literal_span, + format!( + "rules!: failed to parse output schema `{}`: {error}", + output_abs.display() + ), + ) + })?; let list = expect_group(&mut tokens, proc_macro2::Delimiter::Bracket)?; if let Some(token) = tokens.next() { @@ -63,8 +82,15 @@ pub fn parse_rules_top(input: TokenStream) -> Result { let emitted_items = split_top_level_commas(list.stream()) .into_iter() .map(|item| { - if has_top_level_arrow(&item) { - Ok(crate::rule_parse::parse_rule(item)?.lower()) + let rule_body = if has_top_level_arrow(&item) { + Some(item.clone()) + } else { + explicit_rule_body(&item) + }; + if let Some(rule_body) = rule_body { + let mut rule = crate::rule_parse::parse_rule(rule_body)?; + rule.resolve_bare_leaf_content(&output_leaf_kinds)?; + Ok(rule.lower()) } else { Ok(item) } @@ -84,6 +110,7 @@ pub fn parse_rules_top(input: TokenStream) -> Result { struct NamedString { value: String, + literal_span: Span, } fn parse_named_string_arg( @@ -109,7 +136,10 @@ fn parse_named_string_arg( format!("`{expected_name}` must be a string literal path"), ) })?; - Ok(NamedString { value }) + Ok(NamedString { + value, + literal_span: literal.span(), + }) } fn string_literal_value(literal: &proc_macro2::Literal) -> Option { @@ -152,6 +182,17 @@ fn has_top_level_arrow(stream: &TokenStream) -> bool { }) } +fn explicit_rule_body(stream: &TokenStream) -> Option { + let expression = syn::parse2::(stream.clone()).ok()?; + let is_rule = expression + .mac + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "rule"); + is_rule.then_some(expression.mac.tokens) +} + type Tokens = std::iter::Peekable; fn expect_ident(tokens: &mut Tokens, message: &str) -> Result { @@ -212,4 +253,12 @@ mod tests { "e! { { match x { 1 => 2, _ => 3 } } } )); } + + #[test] + fn recognizes_explicit_rule_macros() { + assert!(explicit_rule_body("e! { rule!((a) => (b)) }).is_some()); + assert!(explicit_rule_body("e! { yeast::rule!((a) => (b)) }).is_some()); + assert!(explicit_rule_body("e! { other!((a) => (b)) }).is_none()); + assert!(explicit_rule_body("e! { make_rule() }).is_none()); + } } diff --git a/shared/yeast-schema/src/node_types_yaml.rs b/shared/yeast-schema/src/node_types_yaml.rs index 2bdcc02945e9..7f2c2d6b3fe3 100644 --- a/shared/yeast-schema/src/node_types_yaml.rs +++ b/shared/yeast-schema/src/node_types_yaml.rs @@ -469,6 +469,22 @@ pub fn schema_from_yaml(yaml_input: &str) -> Result Result, String> { + let yaml: YamlNodeTypes = + serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; + Ok(yaml + .named + .into_iter() + .filter_map(|(name, fields)| { + fields + .as_ref() + .map_or(true, BTreeMap::is_empty) + .then_some(name) + }) + .collect()) +} + // --------------------------------------------------------------------------- // JSON → YAML conversion // --------------------------------------------------------------------------- @@ -810,6 +826,24 @@ unnamed: assert_eq!(end["named"], false); } + #[test] + fn test_leaf_kinds_from_yaml() { + let leaf_kinds = leaf_kinds_from_yaml( + r#" +named: + identifier: + empty_fields: {} + call: + callee: identifier +"#, + ) + .unwrap(); + + assert!(leaf_kinds.contains("identifier")); + assert!(leaf_kinds.contains("empty_fields")); + assert!(!leaf_kinds.contains("call")); + } + #[test] fn test_explicit_unnamed_disambiguation() { let yaml = r#" diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs index 6f5d3fe7d0a1..3ac2779703d9 100644 --- a/shared/yeast-schema/src/schema.rs +++ b/shared/yeast-schema/src/schema.rs @@ -277,13 +277,8 @@ impl Schema { .insert((parent_kind.to_string(), field_id), node_types); } - pub fn field_types( - &self, - parent_kind: &str, - field_id: FieldId, - ) -> Option<&Vec> { - self.field_types - .get(&(parent_kind.to_string(), field_id)) + pub fn field_types(&self, parent_kind: &str, field_id: FieldId) -> Option<&Vec> { + self.field_types.get(&(parent_kind.to_string(), field_id)) } /// Record the declared (named) field order for a node kind, as authored in diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index f3075973be41..a4dfa9157b6f 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -236,6 +236,27 @@ impl<'a, C> BuildCtx<'a, C> { ) } + /// Create a leaf whose value is inherited from the current rule's match. + #[doc(hidden)] + pub fn inherited_literal(&mut self, kind: &'static str) -> OutputId { + let source = self + .translator + .as_ref() + .expect("inherited leaf content requires a rule match") + .matched_root(); + let node = self + .ast + .get_node(source) + .expect("rule source node must exist"); + let stored = node.opt_string_content(); + let has_children = node.fields.values().any(|children| !children.is_empty()); + let content = match stored { + Some(value) if !value.is_empty() || !has_children => value, + _ => self.ast.source_text(source), + }; + self.create_named_token_with_range(kind, content, None) + } + /// Create a leaf node with a fixed string content. pub fn literal(&mut self, kind: &'static str, value: &str) -> OutputId { self.create_named_token_with_range(kind, value.to_string(), None) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 142bbc7b6254..d8f368a5f224 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -1117,6 +1117,12 @@ impl Clone for TranslatorHandle<'_, C> { } } +impl TranslatorHandle<'_, C> { + pub(crate) fn matched_root(&self) -> Id { + self.matched_root + } +} + impl<'a, C: Clone> TranslatorHandle<'a, C> { /// Recursively apply the current phase's rules to `id` and return the /// resulting node ids. diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index f53b3e22615a..08cc85c163e3 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -109,6 +109,21 @@ fn run_and_ast(input: &str, rules: Vec) -> OutputAst { runner.run(input).unwrap() } +fn run_ast_with_rules(ast: Ast, rules: Vec) -> OutputAst { + let schema = yeast::node_types_yaml::schema_from_yaml(OUTPUT_SCHEMA_YAML).unwrap(); + let phases = [Phase::new("test", rules)]; + let runner: Runner = Runner::with_schema_no_language(&schema, &phases); + runner.run_from_ast(ast).unwrap() +} + +fn synthetic_integer_ast(content: &str) -> Ast { + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let mut ast = Ast::with_schema(yeast::schema::from_language(&lang)); + let root = ast.create_named_token("integer", content.to_owned()); + ast.set_root(root); + ast +} + /// Helper: translate Ruby source with multiple rule phases and return its dump. fn run_phased_and_dump(input: &str, phases: Vec) -> String { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); @@ -681,6 +696,74 @@ fn test_forward_scan_preserves_order() { // ---- Tree builder tests ---- +#[test] +fn test_bare_leaf_rule_result_inherits_parsed_source_text() { + let rules: Vec = yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + (integer) => (identifier), + ] + }; + let ast = run_and_ast("x = 1", rules); + let dump = dump_ast(&ast, ast.get_root(), "x = 1"); + + assert_dump_eq( + &dump, + r#" + program + assignment + left: identifier "x" + right: identifier "1" + "#, + ); + assert!(ast.reachable_node_ids().into_iter().any(|id| { + let node = ast.get_node(id).unwrap(); + node.kind_name() == "identifier" && node.opt_string_content().as_deref() == Some("1") + })); +} + +#[test] +fn test_bare_leaf_rule_result_inherits_synthetic_content() { + let ast = run_ast_with_rules( + synthetic_integer_ast("synthetic"), + yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + rule!((integer) => (identifier)), + ] + }, + ); + let dump = dump_ast(&ast, ast.get_root(), ""); + + assert_dump_eq( + &dump, + r#" + identifier "synthetic" + "#, + ); +} + +#[test] +fn test_bare_leaf_rule_result_inherits_empty_content() { + let ast = run_ast_with_rules( + synthetic_integer_ast(""), + yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + (integer) => (identifier), + ] + }, + ); + + assert!(ast.reachable_node_ids().into_iter().any(|id| { + let node = ast.get_node(id).unwrap(); + node.kind_name() == "identifier" && node.opt_string_content().as_deref() == Some("") + })); +} + #[test] fn test_tree_builder() { let input = "x = 1"; diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index a1dab86b626e..a9894272ce6a 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -1,5 +1,5 @@ use codeql_extractor::extractor::desugaring; -use yeast::{ConcreteDesugarer, DesugaringConfig, Rule, rule, tree}; +use yeast::{ConcreteDesugarer, DesugaringConfig, Rule, tree}; /// User context propagated from outer rules down to the inner rules that /// emit the corresponding output declarations, so that each emitted node @@ -143,7 +143,10 @@ fn member_chain( } fn translation_rules() -> Vec> { - vec![ + yeast::rules! { + input: "swift_node_types.yml", + output: "ast_types.yml", + [ // ---- Top-level ---- // These rules translate the swift-syntax AST (camelCase kind names), // produced by the sibling `adapter` module from swift-syntax JSON. @@ -619,8 +622,9 @@ fn translation_rules() -> Vec> { // A type-casting pattern (`case is T`). Not yet supported, so it is // mapped to `unsupported_node` — an explicit reminder that this needs // handling in the future. (Redundant with the catch-all fallback, but - // kept as a signpost.) - rule!((isTypePattern) => (unsupported_node)), + // kept as a signpost.) Its explicit empty content preserves the + // existing representation instead of inheriting the matched source. + rule!((isTypePattern) => (unsupported_node "")), // A wildcard *binding* pattern (`let _ = x`, `for _ in xs`). swift-syntax // models this as a `wildcardPattern`, distinct from the `_` match form // handled by the context-aware `discardAssignmentExpr` rule. @@ -1218,7 +1222,7 @@ fn translation_rules() -> Vec> { ), // Preserve the `super` keyword as a dedicated expression, normally used // as the base of a member access (`super.foo`). - rule!((superExpr superKeyword: @@keyword) => (super_expr #{keyword})), + rule!((superExpr) => (super_expr)), // Type expressions. A generic type applied with explicit arguments // (`Set`) becomes a `generic_type_expr` whose `base` is the type // name and whose `type_argument`s are the (structured) arguments — the @@ -1520,35 +1524,37 @@ fn translation_rules() -> Vec> { // These kinds can reach translation independently, but we do not // currently map them to the unified AST. Syntax nested inside one of // these roots is discarded with its parent and needs no separate rule. - rule!((actorDecl) => (unsupported_node)), - rule!((borrowExpr) => (unsupported_node)), - rule!((classRestrictionType) => (unsupported_node)), - rule!((compositionType) => (unsupported_node)), - rule!((consumeExpr) => (unsupported_node)), - rule!((copyExpr) => (unsupported_node)), - rule!((discardStmt) => (unsupported_node)), - rule!((fallThroughStmt) => (unsupported_node)), - rule!((ifConfigDecl) => (unsupported_node)), - rule!((implicitlyUnwrappedOptionalType) => (unsupported_node)), - rule!((inlineArrayType) => (unsupported_node)), - rule!((keyPathExpr) => (unsupported_node)), - rule!((macroDecl) => (unsupported_node)), - rule!((metatypeType) => (unsupported_node)), - rule!((namedOpaqueReturnType) => (unsupported_node)), - rule!((operatorDecl) => (unsupported_node)), - rule!((packElementExpr) => (unsupported_node)), - rule!((packElementType) => (unsupported_node)), - rule!((packExpansionExpr) => (unsupported_node)), - rule!((packExpansionType) => (unsupported_node)), - rule!((postfixIfConfigExpr) => (unsupported_node)), - rule!((poundSourceLocation) => (unsupported_node)), - rule!((precedenceGroupDecl) => (unsupported_node)), - rule!((someOrAnyType) => (unsupported_node)), - rule!((subscriptDecl) => (unsupported_node)), - rule!((suppressedType) => (unsupported_node)), - rule!((typeExpr) => (unsupported_node)), - rule!((unsafeExpr) => (unsupported_node)), - rule!((yieldStmt) => (unsupported_node)), + // Explicit empty content preserves their existing representation + // instead of inheriting the matched source text. + rule!((actorDecl) => (unsupported_node "")), + rule!((borrowExpr) => (unsupported_node "")), + rule!((classRestrictionType) => (unsupported_node "")), + rule!((compositionType) => (unsupported_node "")), + rule!((consumeExpr) => (unsupported_node "")), + rule!((copyExpr) => (unsupported_node "")), + rule!((discardStmt) => (unsupported_node "")), + rule!((fallThroughStmt) => (unsupported_node "")), + rule!((ifConfigDecl) => (unsupported_node "")), + rule!((implicitlyUnwrappedOptionalType) => (unsupported_node "")), + rule!((inlineArrayType) => (unsupported_node "")), + rule!((keyPathExpr) => (unsupported_node "")), + rule!((macroDecl) => (unsupported_node "")), + rule!((metatypeType) => (unsupported_node "")), + rule!((namedOpaqueReturnType) => (unsupported_node "")), + rule!((operatorDecl) => (unsupported_node "")), + rule!((packElementExpr) => (unsupported_node "")), + rule!((packElementType) => (unsupported_node "")), + rule!((packExpansionExpr) => (unsupported_node "")), + rule!((packExpansionType) => (unsupported_node "")), + rule!((postfixIfConfigExpr) => (unsupported_node "")), + rule!((poundSourceLocation) => (unsupported_node "")), + rule!((precedenceGroupDecl) => (unsupported_node "")), + rule!((someOrAnyType) => (unsupported_node "")), + rule!((subscriptDecl) => (unsupported_node "")), + rule!((suppressedType) => (unsupported_node "")), + rule!((typeExpr) => (unsupported_node "")), + rule!((unsafeExpr) => (unsupported_node "")), + rule!((yieldStmt) => (unsupported_node "")), // Anything reaching this final generic handler has neither a // dedicated rule nor an explicit unsupported entry. rule!( @@ -1568,7 +1574,8 @@ fn translation_rules() -> Vec> { tree!((unhandled_node)) } ), - ] + ] + } } pub fn language_spec(desugared_ast_schema: &'static str) -> desugaring::LanguageSpec {