X++: add extractor and QL library generated from the compiler's own AST

xpp/.gitignore

@@ -0,0 +1,4 @@
+extractor/**/bin/*
+extractor/**/obj/*
+ql/build/
+extractor-pack/

xpp/codegen.conf

@@ -0,0 +1,10 @@
+# configuration file for X++ code generation default options
+--schema=schema
+--generate=dbscheme,ql
+--dbscheme=ql/lib/xpp.dbscheme
+--ql-output=ql/lib/codeql/xpp/elements/internal/generated
+--ql-stub-output=ql/lib/codeql/xpp/elements
+# codegen requires a test output directory, but its per-node-type placeholders are not worth
+# committing; real tests live in ql/test/library-tests. This path is gitignored.
+--ql-test-output=ql/build/extractor-tests
+--script-name=xpp/tools/generate-schema.sh

xpp/codeql-extractor.yml

@@ -0,0 +1,12 @@
+name: "xpp"
+display_name: "X++"
+version: 0.0.1
+column_kind: "utf16"
+legacy_qltest_extraction: true
+build_modes:
+  - none
+file_types:
+  - name: xpp
+    display_name: X++ metadata objects
+    extensions:
+      - .xml

xpp/extractor/Xpp.Extraction/AstSequence.cs

@@ -0,0 +1,68 @@
+using System.Collections;
+
+namespace Xpp.Extraction;
+
+/// <summary>
+/// Flattens the collection shapes the X++ compiler uses for child nodes.
+/// </summary>
+public static class AstSequence
+{
+    /// <summary>
+    /// The elements of <paramref name="value"/>, with dictionary entries reduced to their values.
+    /// </summary>
+    /// <remarks>
+    /// The compiler holds some children in dictionaries keyed by name, such as a class's fields
+    /// and methods. Iterating one as a bare <see cref="IEnumerable"/> yields
+    /// <see cref="KeyValuePair{TKey,TValue}"/> rather than the child, so the entries have to be
+    /// unwrapped or those subtrees are labelled wrongly and never traversed.
+    /// </remarks>
+    public static IEnumerable<object> Elements(object? value)
+    {
+        switch (value)
+        {
+            case null:
+            case string:
+                yield break;
+
+            // Dictionary<,> and the compiler's LinkedDictionary<,> both implement the
+            // non-generic interface, which exposes the values directly.
+            case IDictionary dictionary:
+                foreach (var item in dictionary.Values)
+                {
+                    if (item is not null)
+                        yield return item;
+                }
+
+                yield break;
+
+            case IEnumerable sequence:
+                foreach (var item in sequence)
+                {
+                    var unwrapped = Unwrap(item);
+                    if (unwrapped is not null)
+                        yield return unwrapped;
+                }
+
+                yield break;
+        }
+    }
+
+    /// <summary>
+    /// The value of a key/value entry, or the item itself when it is not one.
+    /// </summary>
+    /// <remarks>
+    /// A dictionary typed only as <c>IEnumerable&lt;KeyValuePair&lt;,&gt;&gt;</c> does not reach
+    /// the <see cref="IDictionary"/> case above, so entries are also unwrapped here.
+    /// </remarks>
+    private static object? Unwrap(object? item)
+    {
+        if (item is null)
+            return null;
+
+        var type = item.GetType();
+        if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(KeyValuePair<,>))
+            return type.GetProperty("Value")?.GetValue(item);
+
+        return item;
+    }
+}

xpp/extractor/Xpp.Extraction/CompilerPackageResolver.cs

@@ -0,0 +1,57 @@
+using System.Reflection;
+using System.Runtime.Loader;
+
+namespace Xpp.Extraction;
+
+/// <summary>
+/// Resolves the X++ compiler assemblies from an extracted
+/// <c>Microsoft.Dynamics.AX.Platform.CompilerPackage</c> at run time.
+/// </summary>
+/// <remarks>
+/// The package is proprietary, so it is neither committed nor copied into build output. The
+/// extractor locates it through <c>XPP_COMPILER_PACKAGE</c> instead, which also keeps a single
+/// copy shared between the extractor and any other tool that needs it.
+/// </remarks>
+public static class CompilerPackageResolver
+{
+    public const string PackageVariable = "XPP_COMPILER_PACKAGE";
+
+    private static bool installed;
+
+    /// <summary>
+    /// Starts resolving compiler assemblies from <paramref name="packageDirectory"/>, or from
+    /// <c>XPP_COMPILER_PACKAGE</c> when it is null.
+    /// </summary>
+    /// <exception cref="DirectoryNotFoundException">
+    /// The package directory is unset or does not exist.
+    /// </exception>
+    public static void Install(string? packageDirectory = null)
+    {
+        if (installed)
+            return;
+
+        packageDirectory ??= Environment.GetEnvironmentVariable(PackageVariable);
+
+        if (string.IsNullOrEmpty(packageDirectory))
+        {
+            throw new DirectoryNotFoundException(
+                $"{PackageVariable} is not set. Point it at an extracted " +
+                "Microsoft.Dynamics.AX.Platform.CompilerPackage.");
+        }
+
+        if (!Directory.Exists(packageDirectory))
+        {
+            throw new DirectoryNotFoundException(
+                $"{PackageVariable} does not exist: {packageDirectory}");
+        }
+
+        var root = packageDirectory;
+        AssemblyLoadContext.Default.Resolving += (context, name) =>
+        {
+            var candidate = Path.Combine(root, name.Name + ".dll");
+            return File.Exists(candidate) ? context.LoadFromAssemblyPath(candidate) : null;
+        };
+
+        installed = true;
+    }
+}

xpp/extractor/Xpp.Extraction/Extractor.cs

@@ -0,0 +1,217 @@
+using System.Runtime.CompilerServices;
+using System.Xml.Linq;
+using Microsoft.Dynamics.AX.Framework.Xlnt.XppParser;
+using Microsoft.Dynamics.AX.Metadata.XppCompiler;
+using Xpp.Extraction.Generated;
+
+namespace Xpp.Extraction;
+
+/// <summary>Outcome of extracting a single metadata object.</summary>
+public sealed record ExtractionResult(
+    string Path,
+    int Blocks,
+    int Nodes,
+    int Unsupported,
+    IReadOnlyList<string> Errors);
+
+/// <summary>
+/// Parses the X++ in a metadata object and writes the resulting AST as TRAP.
+/// </summary>
+public sealed class Extractor
+{
+    /// <summary>
+    /// Method bodies parse without metadata; class and interface headers need a provider in
+    /// order to look up the type named by `extends`.
+    /// </summary>
+    private readonly IXppcMetadataProvider metadata = NullMetadataProvider.Create();
+
+    public ExtractionResult Extract(string path, ITrapFile trap)
+    {
+        var text = File.ReadAllText(path);
+        var errors = new List<string>();
+
+        XDocument document;
+        try
+        {
+            document = XppSourceFile.Parse(text);
+        }
+        catch (Exception e)
+        {
+            return new ExtractionResult(path, 0, 0, 0, [$"malformed XML: {e.Message}"]);
+        }
+
+        var name = XppSourceFile.ObjectName(document) ?? Path.GetFileNameWithoutExtension(path);
+        var elementType = ElementType(path);
+
+        var fileLabel = trap.FreshLabel();
+        trap.Tuple("files", fileLabel, path);
+
+        var blocks = 0;
+        var nodes = 0;
+        var unsupported = 0;
+
+        foreach (var block in XppSourceFile.Blocks(document))
+        {
+            blocks++;
+            var context = new ParserContext(elementType, name);
+            var diagnostics = new DiagnosticsHandler();
+            var macros = new MacroLibrary();
+            var lineOffset = block.LineOffset;
+
+            Ast? unit;
+            try
+            {
+                // The declaration block holds the class or interface header; everything else is
+                // a method body.
+                unit = block.Name is null
+                    ? Pass1.ParseClassOrInterface(
+                        context, block.Source, metadata, diagnostics, macros,
+                        ref lineOffset, 0, [])
+                    : Pass1.ParseMethod(
+                        context, block.Source, metadata, diagnostics, macros, ref lineOffset, 0);
+            }
+            catch (Exception e)
+            {
+                errors.Add($"{name}.{block.Name ?? "<declaration>"}: {e.GetType().Name}: {e.Message}");
+                continue;
+            }
+
+            if (unit is null)
+                continue;
+
+            EmitTree(trap, fileLabel, unit, ref nodes, ref unsupported);
+        }
+
+        return new ExtractionResult(path, blocks, nodes, unsupported, errors);
+    }
+
+    /// <summary>
+    /// The parser's name for the kind of object being parsed, derived from the metadata
+    /// directory. The directories carry an `Ax` prefix that the parser does not accept.
+    /// </summary>
+    private static string ElementType(string path)
+    {
+        var directory = Path.GetFileName(Path.GetDirectoryName(path)) ?? "AxClass";
+        return directory.StartsWith("Ax", StringComparison.Ordinal)
+            ? directory["Ax".Length..]
+            : directory;
+    }
+
+    /// <summary>Walks the AST, writing each node's tuples exactly once.</summary>
+    private static void EmitTree(
+        ITrapFile trap, Label file, object root, ref int nodes, ref int unsupported)
+    {
+        var seen = new HashSet<object>(ReferenceEqualityComparer.Instance);
+        var pending = new Stack<object>();
+        pending.Push(root);
+
+        while (pending.Count > 0)
+        {
+            var node = pending.Pop();
+            if (!seen.Add(node))
+                continue;
+
+            nodes++;
+            var id = trap.Label(node);
+            if (!AstTrapEmitter.Emit(trap, id, node))
+                unsupported++;
+
+            EmitLocation(trap, file, id, node);
+
+            foreach (var child in Children(node))
+                pending.Push(child);
+        }
+    }
+
+    /// <summary>
+    /// Records where <paramref name="node"/> came from.
+    /// </summary>
+    /// <remarks>
+    /// The parser was given the source block's line offset, so these positions are already
+    /// relative to the containing file rather than to the extracted fragment.
+    /// </remarks>
+    private static void EmitLocation(ITrapFile trap, Label file, Label id, object node)
+    {
+        if (node is not Ast ast)
+            return;
+
+        var position = ast.Position;
+
+        // Nodes the parser synthesises carry no extent; recording a zero span would put alerts
+        // at the top of the file.
+        if (position.StartLine <= 0)
+            return;
+
+        var location = trap.FreshLabel();
+        trap.Tuple(
+            "locations", location, file,
+            position.StartLine, position.StartCol, position.EndLine, position.EndCol);
+        trap.Tuple("locatable_locations", id, location);
+    }
+
+    /// <summary>The AST nodes directly reachable from <paramref name="node"/>.</summary>
+    private static IEnumerable<object> Children(object node)
+    {
+        foreach (var property in node.GetType().GetProperties())
+        {
+            if (property.Name is "Parent" or "ParentAst")
+                continue;
+
+            object? value;
+            try
+            {
+                value = property.GetValue(node);
+            }
+            catch
+            {
+                continue;
+            }
+
+            foreach (var child in Reachable(value))
+                yield return child;
+        }
+    }
+
+    /// <summary>
+    /// The AST nodes held by a property value.
+    /// </summary>
+    /// <remarks>
+    /// The compiler groups some children in CLR tuples, such as a `catch` with its handler body
+    /// or a switch case with its statements, so tuple slots have to be looked through or those
+    /// subtrees are never visited.
+    /// </remarks>
+    private static IEnumerable<object> Reachable(object? value)
+    {
+        switch (value)
+        {
+            case null:
+            case string:
+                yield break;
+
+            case Ast child:
+                yield return child;
+                break;
+
+            case ITuple tuple:
+                for (var i = 0; i < tuple.Length; i++)
+                {
+                    foreach (var nested in Reachable(tuple[i]))
+                        yield return nested;
+                }
+
+                break;
+
+            case System.Collections.IEnumerable:
+                // AstSequence reduces dictionary entries to their values; iterating the raw
+                // collection would yield KeyValuePair, which is neither an Ast nor a tuple and
+                // would silently drop the subtree.
+                foreach (var item in AstSequence.Elements(value))
+                {
+                    foreach (var nested in Reachable(item))
+                        yield return nested;
+                }
+
+                break;
+        }
+    }
+}