Port LCM domain-model codegen to a Roslyn source generator

AGENTS.md

@@ -65,13 +65,15 @@ No command timeouts were observed.
 - src/SIL.LCModel: main LCM library (net462; netstandard2.0).
 - src/SIL.LCModel.Core: core utilities and ICU data generation (netstandard2.0; net462; net8.0).
 - src/SIL.LCModel.Utils: shared utilities (net462; netstandard2.0).
-- src/SIL.LCModel.Build.Tasks: MSBuild tasks used for code generation.
+- src/SIL.LCModel.Build.Tasks: the IdlImp (kernel-interface codegen) and LcmGenerate (single-template rendering, e.g. FieldWorks C++ headers) MSBuild tasks.
+- src/SIL.LCModel.ModelGeneration: the shared NVelocity engine (LcmGenerateImpl + model wrappers) that renders LCM model templates; referenced by both SourceGenerators and Build.Tasks. NVelocity-only dependency.
+- src/SIL.LCModel.SourceGenerators: Roslyn source generator for the LCM domain model (MasterLCModel.xml → Generated*.cs), with the NVelocity templates embedded from its Templates/vm folder.
 - src/SIL.LCModel.FixData: data-fix utilities.
 - src/CSTools: auxiliary tools (pg/lg/Tools).
 
-Code generation targets to know about:
-- SIL.LCModel: GenerateModel (MasterLCModel.xml → Generated*.cs).
-- SIL.LCModel.Core: GenerateKernelCs, GenerateIcuData.
+Code generation to know about:
+- SIL.LCModel: the LcmModelGenerator source generator turns MasterLCModel.xml (with HandGenerated.xml and IntPropTypeOverrides.xml, wired in as AdditionalFiles) into the domain-model Generated*.cs sources at compile time.
+- SIL.LCModel.Core: GenerateKernelCs (IdlImp task), GenerateIcuData.
 
 ### Tests (tests/)
 - SIL.LCModel.Tests

CHANGELOG.md

@@ -42,6 +42,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
+- [SIL.LCModel] The domain-model `Generated*.cs` sources are now produced at compile time by the new `SIL.LCModel.SourceGenerators` Roslyn source generator, instead of the `LcmGenerate` MSBuild task running the NVelocity templates in a child MSBuild process. The NVelocity engine and model wrappers moved to a shared `SIL.LCModel.ModelGeneration` project used by both the source generator and the (retained) `LcmGenerate` task, which still renders other templates from the model (e.g. the FieldWorks C++ cellar-constants header). Templates and generated output are byte-for-byte identical. No public API change.
 - [SIL.LCModel] Replaced the internal StructureMap (`structuremap.patched`) IoC container with `Microsoft.Extensions.DependencyInjection` (8.x). No public API change.
 - [SIL.LCModel] Trim 12 overwordy semantic domain descriptions and fix 22 punctuation/whitespace issues in the SemDom.xml template, matching sillsdev/FwLocalizations#5 and sillsdev/FwLocalizations#7
 - [SIL.LCModel] `FileUtils.IsFileUriOrPath` checks for the presence of "file:" rather than the absence of known non-file URI schemes

LCM.sln

@@ -53,6 +53,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.LCModel.FixData", "src\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestHelper", "tests\TestHelper\TestHelper.csproj", "{99270559-03D8-49F7-B61B-22E9DBC5D42B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.LCModel.SourceGenerators", "src\SIL.LCModel.SourceGenerators\SIL.LCModel.SourceGenerators.csproj", "{688D1F13-8FAC-4FED-A908-1274B576D81B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.LCModel.ModelGeneration", "src\SIL.LCModel.ModelGeneration\SIL.LCModel.ModelGeneration.csproj", "{5F120076-4431-480D-84F9-EFB60DE4667B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -111,6 +115,14 @@ Global
 		{99270559-03D8-49F7-B61B-22E9DBC5D42B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{99270559-03D8-49F7-B61B-22E9DBC5D42B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{99270559-03D8-49F7-B61B-22E9DBC5D42B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{688D1F13-8FAC-4FED-A908-1274B576D81B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{688D1F13-8FAC-4FED-A908-1274B576D81B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{688D1F13-8FAC-4FED-A908-1274B576D81B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{688D1F13-8FAC-4FED-A908-1274B576D81B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F120076-4431-480D-84F9-EFB60DE4667B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F120076-4431-480D-84F9-EFB60DE4667B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F120076-4431-480D-84F9-EFB60DE4667B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F120076-4431-480D-84F9-EFB60DE4667B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -130,6 +142,8 @@ Global
 		{8ECAF4C5-E651-4EA0-91FF-9251428E3890} = {B9B859EC-6E0D-447E-A0CD-ABF16E7278B1}
 		{8E5F8AF6-6ED0-4205-826C-6BD80D04BDD1} = {040CD95F-B637-493B-9B1C-2989DF3A0D91}
 		{99270559-03D8-49F7-B61B-22E9DBC5D42B} = {B9B859EC-6E0D-447E-A0CD-ABF16E7278B1}
+		{688D1F13-8FAC-4FED-A908-1274B576D81B} = {040CD95F-B637-493B-9B1C-2989DF3A0D91}
+		{5F120076-4431-480D-84F9-EFB60DE4667B} = {040CD95F-B637-493B-9B1C-2989DF3A0D91}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {C625E82E-381D-4590-A382-0B4EF1EF916E}

src/SIL.LCModel.Build.Tasks/LcmGenerate.cs

@@ -1,4 +1,4 @@
-// Copyright (c) 2006-2015 SIL International
+// Copyright (c) 2006-2025 SIL International
 // This software is licensed under the LGPL, version 2.1 or later
 // (http://www.gnu.org/licenses/lgpl-2.1.html)
 
@@ -8,217 +8,116 @@
 using System.Xml;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using SIL.LCModel.ModelGeneration;
 
 namespace SIL.LCModel.Build.Tasks
 {
 	/// ----------------------------------------------------------------------------------------
 	/// <summary>
-	///
+	/// Renders a single NVelocity template (e.g. a C++ header such as CellarConstants.vm.h) from
+	/// the LCM model. The LCM domain-model C# is produced by the SIL.LCModel.SourceGenerators
+	/// source generator; this task remains for consumers (notably the FieldWorks native build)
+	/// that render other artifacts from the same model through the shared engine.
 	/// </summary>
 	/// ----------------------------------------------------------------------------------------
-	public class LcmGenerate: Task
+	public class LcmGenerate : Task
 	{
-		/// ------------------------------------------------------------------------------------
-		/// <summary>
-		/// Gets or sets the XML file.
-		/// </summary>
-		/// <value>The XML file.</value>
-		/// ------------------------------------------------------------------------------------
+		/// <summary>The model XML file (MasterLCModel.xml).</summary>
 		[Required]
 		public string XmlFile { get; set; }
 
-		/// ------------------------------------------------------------------------------------
-		/// <summary>
-		/// Gets or sets the base dir for the output.
-		/// </summary>
-		/// <value>The output directory.</value>
-		/// ------------------------------------------------------------------------------------
+		/// <summary>The base directory for the output.</summary>
 		[Required]
 		public string OutputDir { get; set; }
 
-		/// ------------------------------------------------------------------------------------
-		/// <summary>
-		/// Gets or sets the file for the output.
-		/// </summary>
-		/// <value>The output file name.</value>
-		/// ------------------------------------------------------------------------------------
+		/// <summary>The output file, relative to <see cref="OutputDir"/>.</summary>
 		[Required]
 		public string OutputFile { get; set; }
 
-		/// ------------------------------------------------------------------------------------
-		/// <summary>
-		/// Gets or sets the template file.
-		/// </summary>
-		/// <value>The template file.</value>
-		/// ------------------------------------------------------------------------------------
+		/// <summary>The template file to render.</summary>
 		[Required]
 		public string TemplateFile { get; set; }
 
-		/// ------------------------------------------------------------------------------------
 		/// <summary>
-		/// Gets or sets the template file.
+		/// Retained for backwards compatibility with existing call sites; no longer used now that
+		/// the domain model is produced by the source generator.
 		/// </summary>
-		/// <value>The template file.</value>
-		/// ------------------------------------------------------------------------------------
 		public string BackendTemplateFiles { get; set; }
 
 		/// <summary>
-		/// Gets or sets the working directory.
+		/// Retained for backwards compatibility with existing call sites. The engine no longer
+		/// relies on the current directory (see <c>EmbeddedTemplateLoader</c>), so this is ignored.
 		/// </summary>
-		/// <value>The working directory.</value>
 		public string WorkingDirectory { get; set; }
 
 		/// <summary>
-		/// Gets or sets the directory that contains HandGenerated.xml and IntPropTypeOverrides.xml
+		/// Retained for backwards compatibility with existing call sites; not needed for
+		/// single-template rendering.
 		/// </summary>
 		public string HandGeneratedDir { get; set; }
 
-		/// ------------------------------------------------------------------------------------
-		/// <summary>
-		/// Executes the task.
-		/// </summary>
-		/// ------------------------------------------------------------------------------------
 		public override bool Execute()
 		{
-			string origDir = Directory.GetCurrentDirectory();
-			string oldDir;
-			if (!String.IsNullOrEmpty(WorkingDirectory))
-				oldDir = WorkingDirectory;
-			else
-				oldDir = origDir;
 			try
 			{
-				var doc = new XmlDocument();
-				string xmlPath = XmlFile;
-				if (!Path.IsPathRooted(xmlPath))
-					xmlPath = Path.Combine(oldDir, XmlFile);
-				try
-				{
-					Log.LogMessage(MessageImportance.Low, "Loading XML file {0}.", xmlPath);
-					doc.Load(xmlPath);
-				}
-				catch (XmlException e)
-				{
-					Log.LogMessage(MessageImportance.High, $"Error loading XML file {xmlPath} {e.Message}");
-					return false;
-				}
+				var baseDir = string.IsNullOrEmpty(WorkingDirectory)
+					? Directory.GetCurrentDirectory()
+					: WorkingDirectory;
 
-				var handGeneratedFilesDir = string.IsNullOrEmpty(HandGeneratedDir)
-					? Path.Combine(oldDir, "LcmGenerate")
-					: HandGeneratedDir;
+				var xmlPath = Path.IsPathRooted(XmlFile) ? XmlFile : Path.Combine(baseDir, XmlFile);
+				var templatePath = Path.IsPathRooted(TemplateFile) ? TemplateFile : Path.Combine(baseDir, TemplateFile);
+				var outputDirPath = Path.IsPathRooted(OutputDir) ? OutputDir : Path.Combine(baseDir, OutputDir);
 
-				var config = new XmlDocument();
-				var handGeneratedClasses = new Dictionary<string, List<string>>();
-				try
-				{
-					var handGeneratedFile = Path.Combine(handGeneratedFilesDir, "HandGenerated.xml");
-					Log.LogMessage(MessageImportance.Low, $"Loading hand generated classes from \"{handGeneratedFile}\".");
-					config.Load(handGeneratedFile);
-					foreach (XmlElement node in config.GetElementsByTagName("Class"))
-					{
-						var props = new List<string>();
-// ReSharper disable PossibleNullReferenceException
-						foreach (XmlNode propertyNode in node.SelectNodes("property"))
-// ReSharper restore PossibleNullReferenceException
-						{
-							props.Add(propertyNode.Attributes["name"].Value);
-						}
-						if (props.Count > 0)
-						{
-							handGeneratedClasses.Add(node.Attributes["id"].Value, props);
-						}
-					}
-				}
-				catch (XmlException e)
-				{
-					Log.LogMessage(MessageImportance.High, $"Error loading hand generated classes {e.Message}");
-					return false;
-				}
-
-				// Dictionary<ClassName, Property>
-				var intPropTypeOverridesClasses = new Dictionary<string, Dictionary<string, string>>();
-				try
-				{
-					var handGeneratedFile = Path.Combine(handGeneratedFilesDir, "IntPropTypeOverrides.xml");
-					Log.LogMessage(MessageImportance.Low,
-						$"Loading hand generated classes from \"{handGeneratedFile}\".");
-					config.Load(handGeneratedFile);
-					foreach (XmlElement node in config.GetElementsByTagName("Class"))
-					{
-						// Dictionary<PropertyName, PropertyType>
-						var props = new Dictionary<string, string>();
-// ReSharper disable PossibleNullReferenceException
-						foreach (XmlNode propertyNode in node.SelectNodes("property"))
-// ReSharper restore PossibleNullReferenceException
-						{
-							props.Add(propertyNode.Attributes["name"].Value,
-								propertyNode.Attributes["type"].Value);
-						}
-						if (props.Count > 0)
-						{
-							intPropTypeOverridesClasses.Add(node.Attributes["id"].Value, props);
-						}
-					}
-				}
-				catch (XmlException e)
-				{
-					Log.LogMessage(MessageImportance.High, $"Error loading IntPropTypeOverrides classes {e.Message}");
-					return false;
-				}
+				var doc = new XmlDocument();
+				Log.LogMessage(MessageImportance.Low, "Loading XML file {0}.", xmlPath);
+				doc.Load(xmlPath);
 
+				// The engine resolves templates (including any #parse siblings) by bare file name,
+				// so key the map on the template's file name and include its directory siblings.
+				var templateName = Path.GetFileName(templatePath);
+				var templates = LoadTemplates(templatePath);
 
-				try
-				{
-					// Remember current directory.
-					var originalCurrentDirectory = Directory.GetCurrentDirectory();
-
-					Log.LogMessage(MessageImportance.Low, "Processing template {0}.", TemplateFile);
-					string outputDirPath = OutputDir;
-					if (!Path.IsPathRooted(OutputDir))
-						outputDirPath = Path.Combine(oldDir, OutputDir);
-					var lcmGenerate = new LcmGenerateImpl(doc, outputDirPath)
-										{
-											Overrides = handGeneratedClasses,
-											IntPropTypeOverrides = intPropTypeOverridesClasses
-										};
-					string outputPath = OutputFile;
-					if (!Path.IsPathRooted(outputPath))
-						outputPath = Path.Combine(outputDirPath, OutputFile);
-					// Generate the main code.
-					if (Path.GetDirectoryName(TemplateFile).Length > 0)
-						Directory.SetCurrentDirectory(Path.GetDirectoryName(TemplateFile));
-					lcmGenerate.SetOutput(outputPath);
-					lcmGenerate.Process(Path.GetFileName(TemplateFile));
-
-					// Generate the backend provider(s) code.
-					if (!string.IsNullOrEmpty(BackendTemplateFiles))
-					{
-						foreach (var backendDir in BackendTemplateFiles.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
-						{
-							var beDir = backendDir.Trim();
-							if (beDir == string.Empty) continue;
-
-							var curDir = Path.Combine(Path.Combine(OutputDir, "LcmGenerate"), beDir);
-							Directory.SetCurrentDirectory(curDir);
-							lcmGenerate.SetOutput(Path.Combine(beDir, beDir + @"Generated.cs"));
-							lcmGenerate.Process("Main" + beDir + ".vm.cs");
-						}
-					}
-
-					// Restore original directory.
-					Directory.SetCurrentDirectory(originalCurrentDirectory);
-				}
-				catch (Exception e)
+				var impl = new LcmGenerateImpl(doc, templates)
 				{
-					Log.LogMessage(MessageImportance.High, "Error processing template" + " " + e.Message);
-					return false;
-				}
+					// Empty override maps: the rendered artifacts (e.g. C++ headers) do not use the
+					// hand-generated / int-type-override metadata, but set them so the model
+					// wrappers never dereference a null.
+					Overrides = new Dictionary<string, List<string>>(),
+					IntPropTypeOverrides = new Dictionary<string, Dictionary<string, string>>()
+				};
+
+				Log.LogMessage(MessageImportance.Low, "Processing template {0}.", templateName);
+				impl.SetOutput(OutputFile);
+				impl.Process(templateName);
+
+				Directory.CreateDirectory(outputDirPath);
+				File.WriteAllText(Path.Combine(outputDirPath, OutputFile), impl.Outputs[OutputFile]);
+				return true;
 			}
-			finally
+			catch (Exception e)
+			{
+				Log.LogError("LcmGenerate failed: {0}", e.Message);
+				return false;
+			}
+		}
+
+		/// <summary>
+		/// Loads the target template plus any sibling "*.vm.*" templates in the same directory,
+		/// keyed by bare file name, so <c>#parse</c> references resolve without touching the
+		/// current directory.
+		/// </summary>
+		private static IReadOnlyDictionary<string, string> LoadTemplates(string templatePath)
+		{
+			var result = new Dictionary<string, string>(StringComparer.Ordinal);
+			var dir = Path.GetDirectoryName(templatePath);
+			if (!string.IsNullOrEmpty(dir) && Directory.Exists(dir))
 			{
-				Directory.SetCurrentDirectory(origDir);
+				foreach (var file in Directory.GetFiles(dir, "*.vm.*"))
+					result[Path.GetFileName(file)] = File.ReadAllText(file);
 			}
-			return true;
+			// Ensure the target template is present even if its extension is not "*.vm.*".
+			result[Path.GetFileName(templatePath)] = File.ReadAllText(templatePath);
+			return result;
 		}
 	}
 }

src/SIL.LCModel.Build.Tasks/SIL.LCModel.Build.Tasks.csproj

@@ -4,7 +4,7 @@
     <RootNamespace>SIL.LCModel.Build.Tasks</RootNamespace>
     <TargetFrameworks>net462;netstandard2.0</TargetFrameworks>
     <Description>The liblcm library is the core FieldWorks model for linguistic analyses of languages. Tools in this library provide the ability to store and interact with language and culture data, including anthropological, text corpus, and linguistics data.
-SIL.LCModel.Build.Tasks provides msbuild tasks for generating C# classes for the FieldWorks model: IdlImp and LcmGenerate.</Description>
+SIL.LCModel.Build.Tasks provides the IdlImp and LcmGenerate msbuild tasks for generating C# kernel interfaces and rendering LCM model templates (e.g. FieldWorks C++ headers). The LCM domain model itself is generated by the SIL.LCModel.SourceGenerators Roslyn source generator.</Description>
     <BuildOutputTargetFolder>tools/$(TargetFramework)</BuildOutputTargetFolder>
 	<IsPackable>true</IsPackable>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
@@ -13,11 +13,17 @@ SIL.LCModel.Build.Tasks provides msbuild tasks for generating C# classes for the
   <ItemGroup>
     <PackageReference Include="GitVersion.MsBuild" Version="5.6.10" PrivateAssets="All" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
-    <PackageReference Include="NVelocity" Version="1.2.0" PrivateAssets="All" />
     <PackageReference Include="SIL.IdlImporter" Version="3.0.1" PrivateAssets="All" />
     <PackageReference Include="SIL.ReleaseTasks" Version="2.5.0" PrivateAssets="All" />
   </ItemGroup>
 
+  <!-- The LcmGenerate task reuses the shared NVelocity engine. PrivateAssets=all keeps
+       ModelGeneration (which is not itself packed) out of this package's dependency list; the
+       engine assembly is instead packed directly into tools/ below. -->
+  <ItemGroup>
+    <ProjectReference Include="..\SIL.LCModel.ModelGeneration\SIL.LCModel.ModelGeneration.csproj" PrivateAssets="all" />
+  </ItemGroup>
+
   <Target Name="CopyTargetsAndProps" AfterTargets="Build">
     <MakeDir Directories="$(OutputPath)/lib/net462" />
     <!-- Create empty files -->
@@ -40,8 +46,9 @@ SIL.LCModel.Build.Tasks provides msbuild tasks for generating C# classes for the
       <None Include="$(TFMArtifactFolder)/IDLImporter.dll" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
       <None Include="$(TFMArtifactFolder)/IDLImporter.xml" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
       <None Include="$(TFMArtifactFolder)/Newtonsoft.Json.dll" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
-      <None Include="$(TFMArtifactFolder)/NVelocity.dll" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
       <None Include="$(TFMArtifactFolder)/System.CodeDom.dll" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
+      <None Include="$(TFMArtifactFolder)/NVelocity.dll" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
+      <None Include="$(TFMArtifactFolder)/SIL.LCModel.ModelGeneration.dll" Pack="true" PackagePath="tools/%(_TargetFrameworksItem.Identity)/" />
       <None Include="$(BuildArtifactsFolder)/lib/**/_._" Pack="true" PackagePath="lib/" />
     </ItemGroup>
   </Target>