Improve linux compatibility for test purposes

.github/CONTRIBUTING.md

@@ -20,7 +20,7 @@ If that function isn't on a public API, we could choose a more specific type if
 If that function is on a public API, we could make internal callers use a different method with a specific type, and have the public method just call through to that method. We may want to flag such cases with an attribute, comment or #warning pragma encouraging refactoring to remove the "Object" overload if it's not required by public API consumers.
 
 ## Test types
-See [Tests/Readme.md](https://github.com/icsharpcode/CodeConverter/blob/master/Tests/Readme.md)
+See [Tests/Readme.md](https://github.com/icsharpcode/CodeConverter/blob/master/Tests/Readme.md), including [how to keep tests and code cross-platform](https://github.com/icsharpcode/CodeConverter/blob/master/Tests/Readme.md#cross-platform-windows-and-linux) (line endings: git + tests; path separators: production code)
 
 ## How to get started changing code
 In ProjectConversion, you'll see there's a separate entry point for the online snippet converter which:

CodeConv/MsBuildWorkspaceConverter.cs

@@ -85,6 +85,8 @@ public async Task<Solution> AnalyzeSolutionAsync(IProgress<string> progress, str
             using var workspace = MSBuildWorkspace.Create(properties);
 
             Solution solution = await workspace.OpenSolutionAsync(_solutionFilePath);
+            solution = await RestoreVisualBasicDefinesLostOnUnixAsync(solution, properties);
+            solution = await ReplaceProjectReferencesWithEmittedMetadataOnUnixAsync(solution);
 
             var errorString = await GetCompilationErrorsAsync(workspace, solution.Projects);
             if (string.IsNullOrEmpty(errorString)) return solution;
@@ -105,6 +107,64 @@ ValidationException CreateException(string mainMessage, string fullDetail)
             }
         }
 
+        /// <summary>
+        /// On Unix, MSBuild turns the backslashes escaping the quotes in VB string defines (e.g. CONFIG=\"Debug\") into slashes
+        /// when returning the compiler arguments. Roslyn then fails to parse the whole /define argument, so every conditional
+        /// compilation symbol is lost, including _MyType which then defaults to a My template that can't compile.
+        /// This re-reads the defines from the project's evaluated FinalDefineConstants property.
+        /// </summary>
+        private static async Task<Solution> RestoreVisualBasicDefinesLostOnUnixAsync(Solution solution, IDictionary<string, string> globalProperties)
+        {
+            if (System.IO.Path.DirectorySeparatorChar == '\\') return solution;
+
+            foreach (var project in solution.Projects.Where(p => p.Language == LanguageNames.VisualBasic && p.FilePath != null)) {
+                if (project.ParseOptions is not Microsoft.CodeAnalysis.VisualBasic.VisualBasicParseOptions parseOptions) continue;
+                if (parseOptions.PreprocessorSymbolNames.Contains("CONFIG")) continue;
+
+                var propertyArgs = string.Join(" ", globalProperties.Select(kv => $"\"-p:{kv.Key}={kv.Value}\""));
+                var psi = new ProcessStartInfo("dotnet", $"msbuild \"{project.FilePath}\" -nologo -t:AddImplicitDefineConstants -getProperty:FinalDefineConstants,DefineConstants {propertyArgs}");
+                var (exitCode, stdOut, _) = await psi.GetOutputAsync();
+                if (exitCode != 0) continue;
+
+                var finalDefines = System.Text.Json.JsonDocument.Parse(stdOut).RootElement
+                    .GetProperty("Properties").GetProperty("FinalDefineConstants").GetString();
+                if (string.IsNullOrWhiteSpace(finalDefines)) continue;
+
+                var defines = Microsoft.CodeAnalysis.VisualBasic.VisualBasicCommandLineParser.ParseConditionalCompilationSymbols(finalDefines, out _);
+                var merged = parseOptions.PreprocessorSymbols.Concat(defines).GroupBy(kv => kv.Key).Select(g => g.Last());
+                var newParseOptions = parseOptions.WithPreprocessorSymbols(merged);
+                // The VB compilation options carry their own copy of the parse options, which is where the compiler reads _MyType from
+                var newCompilationOptions = ((Microsoft.CodeAnalysis.VisualBasic.VisualBasicCompilationOptions)project.CompilationOptions!).WithParseOptions(newParseOptions);
+                solution = solution.WithProjectParseOptions(project.Id, newParseOptions).WithProjectCompilationOptions(project.Id, newCompilationOptions);
+            }
+            return solution;
+        }
+
+        /// <summary>
+        /// Roslyn's workspace-generated reference to another project (used when its output hasn't been built) isn't readable on Unix,
+        /// so the referencing compilation reports "not a valid assembly". Emit the metadata directly and reference that instead.
+        /// </summary>
+        private static async Task<Solution> ReplaceProjectReferencesWithEmittedMetadataOnUnixAsync(Solution solution)
+        {
+            if (System.IO.Path.DirectorySeparatorChar == '\\') return solution;
+
+            foreach (var projectId in solution.GetProjectDependencyGraph().GetTopologicallySortedProjects()) {
+                var project = solution.GetProject(projectId)!;
+                foreach (var projectReference in project.ProjectReferences.ToArray()) {
+                    var referenced = solution.GetProject(projectReference.ProjectId);
+                    var compilation = referenced == null ? null : await referenced.GetCompilationAsync();
+                    if (compilation == null) continue;
+
+                    using var stream = new System.IO.MemoryStream();
+                    if (!compilation.Emit(stream, options: new Microsoft.CodeAnalysis.Emit.EmitOptions(metadataOnly: true)).Success) continue;
+
+                    var reference = MetadataReference.CreateFromImage(stream.ToArray(), new MetadataReferenceProperties(aliases: projectReference.Aliases, embedInteropTypes: projectReference.EmbedInteropTypes));
+                    solution = solution.RemoveProjectReference(projectId, projectReference).AddMetadataReference(projectId, reference);
+                }
+            }
+            return solution;
+        }
+
         private async Task<string> GetCompilationErrorsAsync(MSBuildWorkspace workspace, IEnumerable<Project> projectsToConvert)
         {
             var workspaceErrors = workspace.Diagnostics.GetErrorString();

CodeConverter/CSharp/ProjectExtensions.cs

@@ -1,18 +1,34 @@
-namespace ICSharpCode.CodeConverter.CSharp;
+using Microsoft.CodeAnalysis.Emit;
+
+namespace ICSharpCode.CodeConverter.CSharp;
 
 internal static class ProjectExtensions
 {
     private static readonly char[] DirSeparators = { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar };
 
-    public static Project CreateReferenceOnlyProjectFromAnyOptions(this Project project, CompilationOptions baseOptions, ParseOptions parseOptions)
+    /// <summary>
+    /// Creates a project in the other language which can see the types of <paramref name="project"/> (including internal ones).
+    /// It references an image emitted directly from the project's compilation, since Roslyn's workspace-generated
+    /// reference to the project isn't readable on all platforms (e.g. Linux) which silently hides all of its types.
+    /// </summary>
+    public static async Task<Project> CreateReferenceOnlyProjectFromAnyOptionsAsync(this Project project, CompilationOptions baseOptions, ParseOptions parseOptions, CancellationToken cancellationToken)
     {
         var options = baseOptions.WithMetadataImportOptions(MetadataImportOptions.All);
         var viewerId = ProjectId.CreateNewId();
-        var projectReferences = project.ProjectReferences.Concat(new[] {new ProjectReference(project.Id)});
-        var viewerProjectInfo = project.ToProjectInfo(viewerId, project.Name + viewerId, options,
-            projectReferences, parseOptions);
-        var csharpViewOfVbProject = project.Solution.AddProject(viewerProjectInfo).GetProject(viewerId);
-        return csharpViewOfVbProject;
+
+        var compilation = await project.GetCompilationAsync(cancellationToken);
+        using var stream = new MemoryStream();
+        var emitResult = compilation.Emit(stream, options: new EmitOptions(metadataOnly: true, includePrivateMembers: false), cancellationToken: cancellationToken);
+        if (!emitResult.Success) {
+            // Fall back to letting the workspace generate the reference to the project
+            var projectReferences = project.ProjectReferences.Concat(new[] { new ProjectReference(project.Id) });
+            var fallbackProjectInfo = project.ToProjectInfo(viewerId, project.Name + viewerId, options, projectReferences, parseOptions);
+            return project.Solution.AddProject(fallbackProjectInfo).GetProject(viewerId);
+        }
+
+        var viewerProjectInfo = project.ToProjectInfo(viewerId, project.Name + viewerId, options, project.ProjectReferences, parseOptions);
+        var reference = MetadataReference.CreateFromImage(stream.ToArray(), new MetadataReferenceProperties());
+        return project.Solution.AddProject(viewerProjectInfo).GetProject(viewerId).AddMetadataReference(reference);
     }
 
     public static ProjectInfo ToProjectInfo(this Project project, ProjectId projectId, string projectName,

CodeConverter/CSharp/ProjectMergedDeclarationExtensions.cs

@@ -37,7 +37,8 @@ private static IEnumerable<string> GetAllLocaleResxFilePaths(string projDir, str
         var projXml = XDocument.Load(vbProject.FilePath);
         var xmlNs = projXml.Root?.GetDefaultNamespace();
         foreach (var resx in projXml.Descendants().Where(IsStandaloneGeneratedResource)) {
-            string relativePath = GetIncludeOrUpdateAttribute(resx).Value;
+            // Project files written on Windows use backslashes, whatever OS we are running on
+            string relativePath = GetIncludeOrUpdateAttribute(resx).Value.Replace('\\', Path.DirectorySeparatorChar);
             string directoryName = Path.GetDirectoryName(relativePath);
             if (directoryName != null && resx.Element(xmlNs + "LastGenOutput") is {Value: {} lastGenOutput}) {
                 yield return (relativePath, Path.Combine(directoryName, lastGenOutput));

CodeConverter/CSharp/VBToCSProjectContentsConverter.cs

@@ -43,7 +43,7 @@ public async Task InitializeSourceAsync(Project project)
         project = await ClashingMemberRenamer.RenameClashingSymbolsAsync(project, _cancellationToken);
         var cSharpCompilationOptions = CSharpCompiler.CreateCompilationOptions();
         _convertedCsProject = project.ToProjectFromAnyOptions(cSharpCompilationOptions, CSharpCompiler.ParseOptions);
-        _csharpReferenceProject = project.CreateReferenceOnlyProjectFromAnyOptions(cSharpCompilationOptions, CSharpCompiler.ParseOptions);
+        _csharpReferenceProject = await project.CreateReferenceOnlyProjectFromAnyOptionsAsync(cSharpCompilationOptions, CSharpCompiler.ParseOptions, _cancellationToken);
         _csharpViewOfVbSymbols = (CSharpCompilation)await _csharpReferenceProject.GetCompilationAsync(_cancellationToken);
         _designerToResxRelativePath = project.ReadVbEmbeddedResources().ToDictionary(r => r.LastGenOutput, r => r.RelativePath);
         SourceProject = await WithProjectLevelWinformsAdjustmentsAsync(project);
@@ -121,8 +121,8 @@ private static string RebaseResxPaths(string projDirPath, string resxDirPath, st
             .Select(d => d.Element(xmlNs + "value"));
         foreach (var fileRefValue in fileRefValues) {
             var origValueParts = fileRefValue.Value.Split(';');
-            string newRelativePath = GetPathRelativeToProject(projDirPath, Path.GetFullPath(Path.Combine(resxDirPath, origValueParts[0])));
-            fileRefValue.Value = string.Join(";", newRelativePath.Yield().Concat(origValueParts.Skip(1)));
+            string newRelativePath = GetPathRelativeToProject(projDirPath, Path.GetFullPath(Path.Combine(resxDirPath, origValueParts[0].Replace('\\', Path.DirectorySeparatorChar))));
+            fileRefValue.Value = string.Join(";", newRelativePath.Replace('/', '\\').Yield().Concat(origValueParts.Skip(1)));
         }
         return xml.Declaration + Environment.NewLine + xml;
     }

CodeConverter/Common/PathConverter.cs

@@ -38,4 +38,8 @@ public static string GetRelativePath(string relativeTo, string path)
 
         return rel;
     }
+
+    /// <summary>Regex-escapes a path, allowing either directory separator wherever the path has one</summary>
+    public static string EscapeSeparatorAgnostic(string path) =>
+        System.Text.RegularExpressions.Regex.Replace(System.Text.RegularExpressions.Regex.Escape(path), @"\\\\|/", @"[\\/]");
 }

CodeConverter/Common/ProjectConversion.cs

@@ -14,7 +14,7 @@ public class ProjectConversion
     private readonly ILanguageConversion _languageConversion;
     private readonly bool _showCompilationErrors;
     private readonly bool _returnSelectedNode;
-    private static readonly string[] BannedPaths = { ".AssemblyAttributes.", "\\bin\\", "\\obj\\" };
+    private static readonly string[] BannedPaths = { ".AssemblyAttributes.", "/bin/", "/obj/" };
     private readonly CancellationToken _cancellationToken;
 
     private ProjectConversion(IProjectContentsConverter projectContentsConverter, IEnumerable<Document> documentsToConvert, IEnumerable<TextDocument> additionalDocumentsToConvert,
@@ -118,7 +118,8 @@ private static async IAsyncEnumerable<ConversionResult> WithProjectFileAsync(IPr
             }
         }
 
-        var sourceTargetReplacements = sourceToTargetMap.Select(m => (Regex.Escape(m.Item1), m.Item2));
+        // Project files may use either directory separator, whatever OS this is running on
+        var sourceTargetReplacements = sourceToTargetMap.Select(m => (PathConverter.EscapeSeparatorAgnostic(m.Item1), m.Item2.Replace('/', '\\')));
         var languageSpecificReplacements = sourceTargetReplacements.Concat(languageConversion.GetProjectFileReplacementRegexes()).Concat(languageConversion.GetProjectTypeGuidMappings())
             .Select(m => (m.Item1, m.Item2, false));
 
@@ -166,7 +167,7 @@ private static async IAsyncEnumerable<ConversionResult> ConvertProjectContentsAs
         IProgress<ConversionProgress> progress, [EnumeratorCancellation] CancellationToken cancellationToken)
     {
         var documentsWithLengths = await projectContentsConverter.SourceProject.Documents
-            .Where(d => !BannedPaths.Any(d.FilePath.Contains))
+            .Where(d => !BannedPaths.Any(d.FilePath.Replace('\\', '/').Contains))
             .SelectAsync(async d => (Doc: d, (await d.GetTextAsync(cancellationToken)).Length));
 
         //Perf heuristic: Decrease memory pressure on the simplification phase by converting large files first https://github.com/icsharpcode/CodeConverter/issues/524#issuecomment-590301594

CodeConverter/Common/SolutionFileTextEditor.cs

@@ -16,19 +16,9 @@ public class SolutionFileTextEditor : ISolutionFileTextEditor
         foreach (var relativeProjPath in relativeProjPaths)
         {
             // Add replacements for both backslash and forward-slash variants so .slnx files using either separator are handled
-            var nativeVariant = relativeProjPath;
-            var altVariant = relativeProjPath.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
-
-            // native (likely backslashes on Windows)
-            var escapedNative = Regex.Escape(nativeVariant);
-            var newNative = PathConverter.TogglePathExtension(nativeVariant);
-            projectReferenceReplacements.Add((escapedNative, newNative, false));
-
-            // alternate (forward slashes)
-            if (altVariant != nativeVariant) {
-                var escapedAlt = Regex.Escape(altVariant);
-                var newAlt = PathConverter.TogglePathExtension(altVariant);
-                projectReferenceReplacements.Add((escapedAlt, newAlt, false));
+            var variants = new[] { relativeProjPath.Replace('/', '\\'), relativeProjPath.Replace('\\', '/') }.Distinct();
+            foreach (var variant in variants) {
+                projectReferenceReplacements.Add((Regex.Escape(variant), PathConverter.TogglePathExtension(variant), false));
             }
         }
 
@@ -44,8 +34,9 @@ public class SolutionFileTextEditor : ISolutionFileTextEditor
         var projectReferenceReplacements = new List<(string Find, string Replace, bool FirstOnly)>();
         foreach ((string name, string relativeProjPath) in projTuples)
         {
-            var newProjPath = @"""" + PathConverter.TogglePathExtension(relativeProjPath) + @""", ";
-            var projPathEscaped = @"""" + Regex.Escape(relativeProjPath);
+            // Solution files always use backslashes, whatever the OS we're running on
+            var newProjPath = @"""" + PathConverter.TogglePathExtension(relativeProjPath.Replace('/', '\\')) + @""", ";
+            var projPathEscaped = @"""" + PathConverter.EscapeSeparatorAgnostic(relativeProjPath);
             var projName = GetProjNameWithoutFramework(name);
 
             (string oldType, string newType) = GetProjectTypeReplacement(projTypeGuidMappings, projName, projPathEscaped,
@@ -70,14 +61,16 @@ public class SolutionFileTextEditor : ISolutionFileTextEditor
         var projectReferenceReplacements = new List<(string Find, string Replace, bool FirstOnly)>();
         foreach ((string _, string relativeProjPath, string projContents) in projTuples)
         {
-            var escapedProjPath = Regex.Escape(relativeProjPath);
+            var escapedProjPath = PathConverter.EscapeSeparatorAgnostic(relativeProjPath);
             var projRefRegex = new Regex(@"(\\|"")" + escapedProjPath);
 
             var projRefMatch = projRefRegex.Match(projContents);
             var characterBeforePath = projRefMatch.Groups[1].Value;
 
-            var extendedEscProjPath = Regex.Escape(characterBeforePath) + escapedProjPath;
-            var newProjPath = characterBeforePath + PathConverter.TogglePathExtension(relativeProjPath);
+            // Keep whichever directory separators each referencing file already uses
+            var (findPath, replacePath) = SeparatorPreservingReplacement(relativeProjPath, PathConverter.TogglePathExtension(relativeProjPath));
+            var extendedEscProjPath = Regex.Escape(characterBeforePath) + findPath;
+            var newProjPath = characterBeforePath + replacePath;
 
             projectReferenceReplacements.Add((extendedEscProjPath, newProjPath, false));
             if (string.IsNullOrWhiteSpace(sourceSolutionContents)) {
@@ -90,6 +83,20 @@ public class SolutionFileTextEditor : ISolutionFileTextEditor
         return projectReferenceReplacements;
     }
 
+    /// <summary>
+    /// Regex find and replacement strings which match a path with either directory separator, and keep the separators found when replacing.
+    /// Both paths must have the same number of separators.
+    /// </summary>
+    private static (string Find, string Replace) SeparatorPreservingReplacement(string path, string newPath)
+    {
+        var separators = new[] { '\\', '/' };
+        var oldSegments = path.Split(separators);
+        var newSegments = newPath.Split(separators);
+        var find = string.Join(@"([\\/])", oldSegments.Select(Regex.Escape));
+        var replace = string.Concat(newSegments.Select((segment, i) => (i > 0 ? "${" + i + "}" : "") + segment.Replace("$", "$$")));
+        return (find, replace);
+    }
+
     // Gets the project name without the framework specifier (in case of multiple target frameworks)
     private static string GetProjNameWithoutFramework(string projName)
     {

CodeConverter/Util/GeneratedCodeAnalysisExtensions.cs

@@ -9,10 +9,10 @@ public static class GeneratedCodeAnalysisExtensions
     /// Apache-2.0 license
     /// </summary>
     public static bool IsGeneratedFile(this string filePath) =>
-        Regex.IsMatch(filePath, @"(\\service|\\TemporaryGeneratedFile_.*|\\assemblyinfo|\\assemblyattributes|\.(g\.i|g|designer|generated|assemblyattributes))\.(cs|vb)$",
+        Regex.IsMatch(filePath, @"([\\/]service|[\\/]TemporaryGeneratedFile_.*|[\\/]assemblyinfo|[\\/]assemblyattributes|\.(g\.i|g|designer|generated|assemblyattributes))\.(cs|vb)$",
             RegexOptions.IgnoreCase);
 
     public static bool IsTempFile(this string filePath) =>
-        Regex.IsMatch(filePath, @"(\\service|\\obj|\\TemporaryGeneratedFile_.*|\\assemblyattributes|\.(g\.i|g|assemblyattributes))\.(cs|vb)$",
+        Regex.IsMatch(filePath, @"([\\/]service|[\\/]obj|[\\/]TemporaryGeneratedFile_.*|[\\/]assemblyattributes|\.(g\.i|g|assemblyattributes))\.(cs|vb)$",
             RegexOptions.IgnoreCase);
 }