diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 827b97f33ac..e19d3824dad 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -509,6 +509,64 @@ stages: continueOnError: true condition: failed() + # Windows Apex VS integration tests (non-gating — published for signal only) + - job: WindowsApexIntegration + variables: + - name: _configuration + value: Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 90 + steps: + - checkout: self + clean: true + + - powershell: eng\SetupVSHive.ps1 + displayName: Setup VS Hive + + # Build + deploy the F# VSIX into the RoslynDev hive, then run the Apex tests. The Apex host + # resolves the installed VS via LocateVisualStudio (set as VisualStudio.InstallationUnderTest.Path + # by the -testApex path), so it does not depend on the local-only 'Canary' milestone. + # continueOnError keeps this leg non-gating: failures surface as signal but do not fail the PR. + - script: eng\CIBuildNoPublish.cmd -configuration $(_configuration) -deployExtensions -testApex + env: + NativeToolsOnMachine: true + displayName: Build, deploy extension and run Apex integration tests + continueOnError: true + + - task: PublishTestResults@2 + displayName: Publish Apex Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: WindowsApexIntegration + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Apex Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' + ArtifactName: Windows $(_configuration) Apex test logs + publishLocation: Container + continueOnError: true + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Apex Build Logs + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)' + ArtifactName: Windows $(_configuration) Apex build logs + ArtifactType: Container + parallel: true + # Windows With Compressed Metadata Desktop (split into 3 batches) - job: WindowsCompressedMetadata_Desktop strategy: diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 41a52df6395..6149e223f33 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -60,6 +60,7 @@ param ( [switch]$testIntegration, [switch]$testScripting, [switch]$testVs, + [switch]$testApex, [switch]$testAll, [switch]$testAllButIntegration, [switch]$testAllButIntegrationAndAot, @@ -128,6 +129,7 @@ function Print-Usage() { Write-Host " -testIntegration Run F# integration tests" Write-Host " -testScripting Run Scripting tests" Write-Host " -testVs Run F# editor unit tests" + Write-Host " -testApex Run F# Apex VS integration tests (requires -deployExtensions)" Write-Host " -testpack Verify built packages" Write-Host " -testAOT Run AOT/Trimming tests" Write-Host " -testEditor Run VS Editor tests" @@ -402,6 +404,245 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str Exec-Console $dotnetExe $test_args } +# Runs the Apex VS integration tests. These are a classic MSTest (VSTest) project, NOT part of the +# repo's Microsoft.Testing.Platform 'dotnet test' path, and they drive a real VS instance via the Apex +# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS). +# The project is intentionally excluded from VisualFSharp.slnx (its dependencies must version-match the +# VS it drives), so this builds it explicitly. It also must match a specific VS version +# (MicrosoftTestApexVisualStudioVersion); if the installed VS does not match, the run is skipped rather +# than producing confusing Apex/VS-mismatch failures. The F# VSIX must be deployed into the RoslynDev +# hive first (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev. + +# ===== BEGIN Apex CI diagnostics helpers (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== +# On CI the Apex host failed with "Timed out ... trying get running object": devenv started but never +# registered its DTE automation object in the ROT. These helpers (a) make sure devenv has an interactive +# console desktop (the usual reason DTE never registers) and (b) collect the RoslynDev-hive logs so we can +# tell an interactive-desktop problem apart from an F# VSIX / package load failure. +function Capture-ApexScreenshot([string] $path) { + # Returns $true only if a screenshot could be taken, which implies an interactive desktop is attached. + try { + Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop + Add-Type -AssemblyName System.Drawing -ErrorAction Stop + $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen + $bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height + $gfx = [System.Drawing.Graphics]::FromImage($bmp) + try { + $gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) + $bmp.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + Write-Host "Captured screenshot to '$path'." + return $true + } finally { + $gfx.Dispose(); $bmp.Dispose() + } + } catch { + Write-Host "Screenshot capture failed (no interactive desktop?): $($_.Exception.Message)" + return $false + } +} + +function Prepare-ApexInteractiveSession([string] $screenshotPath) { + # Mirrors dotnet/roslyn's Setup-IntegrationTestRun: if the desktop is not interactive, reconnect the + # session to the console so devenv can publish its DTE automation object; then minimize other windows. + if (-not (Capture-ApexScreenshot $screenshotPath)) { + Write-Host "No interactive desktop detected; attempting to reconnect the session to the console." + try { + $quserItems = ((quser $env:USERNAME | Select-Object -Skip 1) -split '\s+') + $sessionId = $quserItems[2] + if ($sessionId -eq 'Disc') { $sessionId = $quserItems[1] } + Write-Host "tscon $sessionId /dest:console" + tscon $sessionId /dest:console + Start-Sleep -Seconds 3 + [void](Capture-ApexScreenshot $screenshotPath) + } catch { + Write-Host "##vso[task.logissue type=warning]Failed to reconnect session to console: $($_.Exception.Message)" + } + } + + try { + (New-Object -ComObject "Shell.Application").MinimizeAll() + } catch { + Write-Host "MinimizeAll failed: $($_.Exception.Message)" + } +} + +function Collect-ApexDiagnostics([string] $screenshotPath, [string] $destination) { + # After-run screenshot plus the RoslynDev experimental-hive logs. ActivityLog.xml records whether the + # F# package failed to load; ComponentModelCache\*.err records MEF composition failures. + [void](Capture-ApexScreenshot $screenshotPath) + + $roamingVs = Join-Path $env:USERPROFILE "AppData\Roaming\Microsoft\VisualStudio" + $localVs = Join-Path $env:USERPROFILE "AppData\Local\Microsoft\VisualStudio" + + if (Test-Path $roamingVs) { + foreach ($hiveDir in @(Get-ChildItem -Path $roamingVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) { + foreach ($name in @("ActivityLog.xml", "ActivityLog.xsl")) { + $src = Join-Path $hiveDir.FullName $name + if (Test-Path $src) { + Copy-Item $src (Join-Path $destination "$($hiveDir.Name)-$name") -Force -ErrorAction SilentlyContinue + Write-Host "Collected $src" + } + } + } + } + if (Test-Path $localVs) { + foreach ($hiveDir in @(Get-ChildItem -Path $localVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) { + $mefErr = Join-Path $hiveDir.FullName "ComponentModelCache\Microsoft.VisualStudio.Default.err" + if (Test-Path $mefErr) { + Copy-Item $mefErr (Join-Path $destination "$($hiveDir.Name)-Microsoft.VisualStudio.Default.err") -Force -ErrorAction SilentlyContinue + Write-Host "Collected $mefErr" + } + } + } +} +# ===== END Apex CI diagnostics helpers (Phase 2) ===== + +function TestApexUsingVSTest([string] $targetFramework) { + $projectName = "FSharp.Editor.Apex.IntegrationTests" + $apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj" + + # Create the results directory up-front so the CI 'publish test logs' step never fails on a missing + # path, even when the run is skipped below (a marker file records the skip reason). + $testResultsDir = "$ArtifactsDir\TestResults\$configuration" + Create-Directory $testResultsDir + + # The Apex package (MicrosoftTestApexVisualStudioVersion) is pinned to a VS 18.x build. Because the + # installed VS differs by machine (CI's image trails a dev's local install and they rarely share the + # same minor), gate on the MAJOR version by default so any VS 18.x can run the tests; set + # FSHARP_APEX_VS_VERSION to a stricter prefix (e.g. "18.9") to force an exact minor. + $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18" } + + # Determine which VS will run the tests, and gate on its version. An explicit + # VisualStudio.InstallationUnderTest.Path override is trusted (gate skipped); otherwise enumerate + # ALL installed VS instances (not just the latest) and pick one whose version starts with the target + # prefix, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors. + $overridePath = ${env:VisualStudio.InstallationUnderTest.Path} + if ($overridePath) { + Write-Host "Using explicit VisualStudio.InstallationUnderTest.Path=$overridePath (VS version gate skipped)." + $vsDir = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $overridePath)) + } + else { + $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + # Query the properties separately (parallel arrays in a stable order) instead of parsing + # `-format json`: Windows PowerShell 5.1's ConvertFrom-Json mishandles vswhere's multi-instance + # JSON array (it collapses the instances into one object with array-valued properties), which + # would make every version comparison fail. Build.ps1 runs under Windows PowerShell 5.1. + $versions = @() + $paths = @() + $names = @() + if (Test-Path $vswhere) { + $versions = @(& $vswhere -all -prerelease -products * -property installationVersion) + $paths = @(& $vswhere -all -prerelease -products * -property installationPath) + $names = @(& $vswhere -all -prerelease -products * -property displayName) + } + $found = $versions -join ', ' + + # Print the full VS inventory for diagnostics (helps explain a skip on CI). Always logged. + Write-Host "Visual Studio instances found by vswhere ($($versions.Count)) [vswhere: $vswhere]:" + if ($versions.Count -eq 0) { + Write-Host " (none)" + } + for ($k = 0; $k -lt $versions.Count; $k++) { + $mm = ($versions[$k] -split '\.')[0..1] -join '.' + $hasIde = Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe") + $name = if ($k -lt $names.Count) { $names[$k] } else { "" } + Write-Host (" [{0}] version={1} (major.minor {2}) devenv={3} name='{4}' path={5}" -f $k, $versions[$k], $mm, $hasIde, $name, $paths[$k]) + } + + # Among installs whose version starts with the target prefix (major, or an explicit major.minor) + # and that actually have an IDE (devenv.exe — excludes Build Tools / incomplete installs), pick + # the LOWEST version: it is closest to the pinned Apex minor, and "Apex older than VS" is the + # safer cross-minor direction than the reverse. + $vsDir = $null + $selectedVersion = $null + $versionMatchPath = $null + $candidates = for ($k = 0; $k -lt $versions.Count; $k++) { + if ($versions[$k] -like "$expectedVsVersion.*") { + [PSCustomObject]@{ Version = $versions[$k]; Path = $paths[$k] } + } + } + $candidates = @($candidates | Sort-Object { [version]$_.Version }) + foreach ($c in $candidates) { + if (-not $versionMatchPath) { $versionMatchPath = $c.Path } + if (Test-Path (Join-Path $c.Path "Common7\IDE\devenv.exe")) { + $vsDir = $c.Path.TrimEnd("\") + $selectedVersion = $c.Version + break + } + } + + if (-not $vsDir) { + if ($versionMatchPath) { + Write-Host "##vso[task.logissue type=warning]Found VS $expectedVsVersion ($versionMatchPath) but no devenv.exe under Common7\IDE (incomplete install, Build Tools, or a VS update in progress); skipping Apex integration tests (non-gating)." + Write-Host "Skipping Apex tests: VS $expectedVsVersion present but devenv.exe missing at [$versionMatchPath]. If VS was updating, retry once it finishes." + } + else { + Write-Host "##vso[task.logissue type=warning]No installed VS matches the Apex target $expectedVsVersion (found: $found); skipping Apex integration tests (non-gating)." + Write-Host "Skipping Apex tests: no VS $expectedVsVersion found among installs [$found]. Set FSHARP_APEX_VS_VERSION or VisualStudio.InstallationUnderTest.Path to run against a different install." + } + Set-Content -Path (Join-Path $testResultsDir "apex-tests-skipped.txt") -Value "Apex integration tests skipped: no usable VS matching target $expectedVsVersion (installed: $found)." + return + } + Write-Host "Selected VS $selectedVersion at '$vsDir' for the Apex tests (target $expectedVsVersion)." + # This process env var propagates to the child vstest.console -> testhost -> Apex processes. + ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe" + } + + $vstestConsole = @( + (Join-Path $vsDir "Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"), + (Join-Path $vsDir "Common7\IDE\Extensions\TestPlatform\vstest.console.exe") + ) | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $vstestConsole) { + throw "vstest.console.exe not found under '$vsDir' (looked in CommonExtensions\Microsoft\TestWindow and Extensions\TestPlatform)." + } + + # Build the Apex test project explicitly (it is intentionally excluded from VisualFSharp.slnx). + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + $buildBinLog = "$LogDir\${projectName}_$targetFramework.binlog" + Exec-Console $dotnetExe "build `"$apexProject`" -c $configuration /bl:`"$buildBinLog`"" + + $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll" + if (-not (Test-Path $testAssembly)) { + throw "Apex test assembly not found at '$testAssembly' after build." + } + + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $trxFileName = "$projectName.$targetFramework.$jobName.trx" + + $vstestArgs = @( + $testAssembly, + "/Platform:x64", + "/Framework:.NETFramework,Version=v4.7.2", + "/logger:trx;LogFileName=$trxFileName", + "/ResultsDirectory:$testResultsDir" + ) + + Write-Host "$vstestConsole $($vstestArgs -join ' ')" + + # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== + if ($ci) { + Prepare-ApexInteractiveSession (Join-Path $testResultsDir "apex-before-run.png") + } + # ===== END Apex CI diagnostics (Phase 2) ===== + + & $vstestConsole @vstestArgs + $vstestExitCode = $LASTEXITCODE + + # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== + if ($ci) { + Collect-ApexDiagnostics (Join-Path $testResultsDir "apex-after-run.png") $testResultsDir + } + # ===== END Apex CI diagnostics (Phase 2) ===== + + # This leg is non-gating (signal only): a test failure must NOT fail the build. Surface it as a + # warning and let the published TRX / Tests tab carry the signal. Genuine infra errors (missing + # assembly / vstest.console) already threw above and remain hard failures. + if ($vstestExitCode -ne 0) { + Write-Host "##vso[task.logissue type=warning]Apex integration tests reported failures (vstest.console exit code $vstestExitCode). This leg is non-gating; see the published TRX and the Tests tab." + Write-Host "Apex tests exit code $vstestExitCode (non-gating; not failing the build)." + } +} + function Prepare-TempDir() { Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir @@ -669,6 +910,10 @@ try { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } + if ($testApex) { + TestApexUsingVSTest -targetFramework $script:desktopTargetFramework + } + if ($testAOT) { Push-Location "$RepoRoot\tests\AheadOfTime" ./check.ps1 diff --git a/eng/SetupVSHive.ps1 b/eng/SetupVSHive.ps1 index 50dcf7531d0..f88f3e4a488 100644 --- a/eng/SetupVSHive.ps1 +++ b/eng/SetupVSHive.ps1 @@ -12,6 +12,24 @@ $vsRegEdit = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'VSRegEdit $hive = "RoslynDev" &$vsRegEdit set "$vsDir" $hive HKCU "Roslyn\Internal\OnOff\Features" OOP64Bit dword 0 +# Suppress modal dialogs and online-profile prompts that block UI automation (Apex) and cause the +# integration tests to hang until they time out. These mirror the settings dotnet/roslyn applies to its +# RoslynDev hive before running VS integration tests. + +# Disable the editor "report exceptions" dialog: fail silently and keep the test running instead of +# popping a modal that stalls automation. +&$vsRegEdit set "$vsDir" $hive HKCU "Text Editor" "Report Exceptions" dword 0 + +# Disable roaming settings so an online user profile / sign-in prompt can't interfere on CI. +&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio" RoamingEnabled string "1*System.Boolean*False" + +# Disable background download UI to avoid toasts appearing over the IDE during a run. +&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Setup\BackgroundDownload" Value dword 0 + +# Disable targeted (remote settings) notifications. This one can't be set via VsRegEdit, so write it +# directly to the user hive (it is not hive-suffix specific). +reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1 | Out-Null + Write-Host "-- VS Info --" $isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini' Get-Content $isolationIni | Write-Host diff --git a/eng/Versions.props b/eng/Versions.props index 60486fabcaa..adb4f0006ba 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -130,6 +130,8 @@ $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) 17.14.0 + 18.7.0-preview-1-11715-043 + 3.6.4 0.1.800-beta $(MicrosoftVisualStudioExtensibilityTestingVersion) diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs new file mode 100644 index 00000000000..453cac96bb6 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs. The project is created from +// the .NET SDK class-library template (via `dotnet new`, matching the source tests; Apex's +// ProjectTemplate.ClassLibrary is the legacy .NET Framework template). The template auto-opens the +// default source file, so the source is set through the open document and saved to disk (an +// out-of-band disk write would be superseded by the still-open template buffer, and the compiler would +// build the valid template instead of the intended code). Rather than parsing the "Build: 1 succeeded, +// 0 failed ..." output-pane summary string, these assert on the idiomatic Apex build result: +// BuildManager.Succeeded and BuildManager.Verify.HasFailedWithErrors for the error case. + +using System.IO; +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class BuildProjectTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify a well-formed F# project builds successfully.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void SuccessfulBuild() + { + var code = @"module Test + +let answer = 42"; + + this.SetLibraryContent(code); + + Assert.IsTrue(this.BuildSolutionSucceeded(), "Build was expected to succeed."); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify an F# project with an incomplete binding fails to build with FS0010.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void FailedBuild() + { + var code = @"module Test + +let answer ="; + + this.SetLibraryContent(code); + + this.BuildSolution(); + + var buildManager = this.Library.VisualStudio.ObjectModel.Solution.BuildManager; + Assert.IsFalse(buildManager.Succeeded, "Build was expected to fail."); + + // The Apex error-list verifier matches the error DESCRIPTION (the message text); the code + // 'FS0010' lives in a separate Code column, so match on the FS0010 message instead. + Assert.IsTrue( + buildManager.Verify.HasFailedWithErrors( + new[] { "Incomplete structured construct" }, alsoVerifyInOutputWindow: false), + "Build was expected to fail with the FS0010 'Incomplete structured construct' error."); + } + + /// + /// Creates an F# class library and replaces its default Library.fs with + /// through the open document (saved to disk), verifying the file on disk matches before building. + /// + private void SetLibraryContent(string code) + { + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Library.fs"); + document.ReplaceAllAndSave(code); + + AssertSourceEquals(code, File.ReadAllText(document.FilePath)); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs new file mode 100644 index 00000000000..515cd14883f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/CodeActionTests.cs. Each case sets up the same +// triggering F# source, places the caret where that test does, and verifies the same code/error fix +// is offered on the light bulb (asserting on the action's display text). The Apex light-bulb model +// exposes a flat action list, so the CodeFix/ErrorFix category asserted by the source tests is not +// mirrored here. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class CodeActionTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Remove unused open declarations' code fix is offered on an unused open.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void UnusedOpenDeclarations() + { + var code = @"module Library + +open System + +let x = 42"; + + this.AssertCodeActionOffered(code, "open System", "Remove unused open declarations"); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Add missing 'fun' keyword' error fix is offered on a lambda missing 'fun'.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void AddMissingFunKeyword() + { + var code = @"module Library + +let original = [] +let transformed = original |> List.map (x -> x)"; + + this.AssertCodeActionOffered(code, "->", "Add missing 'fun' keyword"); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Add 'new' keyword' code fix is offered when constructing a disposable.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void AddNewKeywordToDisposables() + { + var code = @"module Library + +let sr = System.IO.StreamReader("""")"; + + this.AssertCodeActionOffered(code, "let sr", "Add 'new' keyword"); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs new file mode 100644 index 00000000000..1aa66153b95 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Ported verbatim from the TypeScript-VS repository +// (VS/LanguageService/Tests/Integration/CodeFixes/SuggestionCodeFix.cs) and then adjusted so that +// the setup is tailored for the F# extension. A single test case is kept, exercising an F# code fix +// (Wrap expression in parentheses, FS0597). The triggering code is taken from the F# code-fix unit +// tests: vsintegration/tests/FSharp.Editor.Tests/CodeFixes/WrapExpressionInParenthesesTests.fs. + +using System; +using System.Linq; +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class SuggestionCodeFix : FSharpLanguageServiceApexTest + { + #region Private members + protected override bool AutomaticallyDismissMessageBoxes => false; + private const string code = @"module Library + +let rng = System.Random() + +printfn ""Hello %d"" rng.Next(5)"; + private const string expectedCode = @"module Library + +let rng = System.Random() + +printfn ""Hello %d"" (rng.Next(5))"; + #endregion + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify code-fix offers suggestion to wrap an expression in parentheses in an F# file.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void SuggestionCodeFixInFSharpFile() + { + var document = this.OpenFSharpDocument(code, "rng.Next"); + + this.InvokeCodeFix(document, "rng.Next"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + // Poll until the fix rewrites the buffer, then assert it actually applied. Without the + // assertion a fix that never fires would leave the original source and the test would still + // pass. Compare on normalized source so incidental line-ending differences do not race the poll. + this.Library.Synchronization.TryWaitForCondition( + () => NormalizeSource(document.Contents) == NormalizeSource(expectedCode), + TimeSpan.FromSeconds(15)); + + AssertSourceEquals(expectedCode, document.Contents); + } + + #region Helps + public void InvokeCodeFix(TextDocumentView document, string expression) + { + var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, expression); + var globalScopeAction = lightBulb.Actions.Single(action => + action.Text.IndexOf("Wrap", StringComparison.OrdinalIgnoreCase) >= 0 + ); + + globalScopeAction.Invoke(); + } + #endregion + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs new file mode 100644 index 00000000000..4ce393246a3 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/CreateProjectTests.cs. Each case creates an F# +// project from a .NET SDK template (via `dotnet new`, matching the SDK templates the source tests +// target — Apex's ProjectTemplate enum maps to the legacy .NET Framework F# templates with different +// filenames and content) and asserts the auto-generated default source. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class CreateProjectTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# class library project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void ClassLibrary() + { + var expectedCode = @"namespace Library + +module Say = + let hello name = + printfn ""Hello %s"" name"; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Library.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# console application project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void ConsoleApp() + { + var expectedCode = @"// For more information see https://aka.ms/fsharp-console-apps +printfn ""Hello from F#"""; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("console", "ConsoleApp"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Program.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# xUnit test project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void XUnitTestProject() + { + var expectedCode = @"module Tests + +open System +open Xunit + +[] +let ``My test`` () = + Assert.True(true)"; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("xunit", "Tests"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Tests.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj new file mode 100644 index 00000000000..924a74297f4 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj @@ -0,0 +1,41 @@ + + + + + net472 + preview + disable + Library + false + + true + $(NoWarn);CS1591;VSTHRD200 + + + + + + + + + + + + + + + + diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs new file mode 100644 index 00000000000..5eff72aa107 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs. Go To Definition is +// invoked idiomatically via the editor caret (IVisualStudioCaretTestExtension.GoToDefinition), and the +// result is read from the active document (current line + window caption) rather than the async +// SolutionExplorer/Editor/Shell helpers used by the xUnit harness. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.Test.Apex.VisualStudio.Solution; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class GoToDefinitionTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify Go To Definition navigates from a use of a binding to its definition.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void GoesToDefinition() + { + var code = @"module Test + +let add x y = x + y + +let increment = add 1"; + + var document = this.OpenFSharpDocument(code, "add 1"); + + // Resolve references so the file type-checks and Go To Definition can bind the symbol. + this.BuildSolution(); + + this.GoToDefinitionAndWait(document, "add 1", line => line.Contains("let add x y = x + y")); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify Go To Definition stays within the signature file and the implementation file respectively.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void FsiAndFsFilesGoToCorrespondentDefinitions() + { + var fsi = @"module Module + +type SomeType = +| Number of int +| Letter of char + +val id: t: SomeType -> SomeType"; + + var fs = @"module Module + +type SomeType = + | Number of int + | Letter of char + +let id (t: SomeType) = t"; + + var project = this.Library.ProjectCreation.CreateFSharpProject(ProjectTemplate.ClassLibrary, "Library"); + + // The signature file must precede the implementation file in F# compile order. When a file is + // added, VS inserts it alphabetically, so adding "Module.fsi" then "Module.fs" would place the + // implementation first ("Module.fs" sorts before "Module.fsi") and the signature would not + // apply. Add the signature under a name that sorts first, then rename it — the same approach as + // the source IdeFact test. + var fsiItem = this.Library.ProjectCreation.AddProjectItemFromContent(project, "AModule.fsi", fsi); + this.Library.ProjectCreation.AddProjectItemFromContent(project, "Module.fs", fs); + fsiItem.Rename("Module.fsi"); + this.Library.Synchronization.WaitForSolutionCrawler(); + this.BuildSolution(); + + // From the signature file, Go To Definition on the type usage stays in Module.fsi. + var fsiDocument = this.Library.OpenProjectFile(project, "Module.fsi"); + this.GoToDefinitionAndWait(fsiDocument, "SomeType ->", line => line.Trim() == "type SomeType =", "Module.fsi"); + + // From the implementation file, Go To Definition on the type usage stays in Module.fs. + var fsDocument = this.Library.OpenProjectFile(project, "Module.fs"); + this.GoToDefinitionAndWait(fsDocument, "SomeType)", line => line.Trim() == "type SomeType =", "Module.fs"); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs new file mode 100644 index 00000000000..4583a7b321f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs @@ -0,0 +1,317 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Web.Script.Serialization; +using Microsoft.Test.Apex.Editor; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Base test class for F# language service tests built using the Apex framework. + /// Compact, self-contained equivalent of the TypeScript-VS LanguageServiceApexTest, tailored for + /// the F# extension and without the JavaScript/TypeScript BrowserTools layer. + /// + [TestClass] + public abstract class FSharpLanguageServiceApexTest : VisualStudioHostTest + { + protected const int ShortTimeoutMS = 1000 * 60 * 12; + protected const int LongTimeoutMS = 1000 * 60 * 20; + + // Apex launches the Visual Studio instance pointed to by this environment variable, if set. + private const string InstallationUnderTestPathVariable = "VisualStudio.InstallationUnderTest.Path"; + + private FSharpLanguageServiceLibrary libraryCache; + + /// + /// Gets the F# language service test library. + /// + public FSharpLanguageServiceLibrary Library + => this.libraryCache ??= new FSharpLanguageServiceLibrary(this.VisualStudio, this.Operations); + + /// + /// Whether message boxes shown during the test run should be dismissed automatically. + /// + protected virtual bool AutomaticallyDismissMessageBoxes => true; + + /// + /// The registry root suffix (experimental hive) of the Visual Studio instance to launch. + /// Defaults to "RoslynDev", where the locally-built F# extension is deployed. + /// + protected virtual string RootSuffix => "RoslynDev"; + + /// + /// The product milestone of the installed Visual Studio to launch, as reported by vswhere's + /// catalog.productMilestone. Defaults to "Canary" (the IntCanary channel), which selects the + /// Canary install rather than Insiders when both are present locally. On CI the installed VS is + /// a different channel (e.g. "Preview"), so this can be overridden with the + /// FSHARP_APEX_VS_MILESTONE environment variable; setting VisualStudio.InstallationUnderTest.Path + /// directly takes precedence over milestone resolution entirely. + /// + protected virtual string TargetProductMilestone + => Environment.GetEnvironmentVariable("FSHARP_APEX_VS_MILESTONE") is string milestone + && !string.IsNullOrEmpty(milestone) + ? milestone + : "Canary"; + + protected override VisualStudioHostConfiguration GetVisualStudioHostConfiguration() + { + this.EnsureTargetInstallationSelected(); + + var config = base.GetVisualStudioHostConfiguration(); + config.AutomaticallyDismissMessageBoxes = this.AutomaticallyDismissMessageBoxes; + config.RootSuffix = this.RootSuffix; + + // On a fresh experimental hive (e.g. CI), devenv shows a full-screen "Sign in to Visual Studio" + // first-launch dialog that blocks the UI thread, so it never registers its DTE automation + // object and Apex times out waiting for it. The devenv '/NoSigninPrompt' switch (registered by + // Microsoft.VisualStudio.Shell.Connected under AppCommandLine) suppresses that dialog. + config.CommandLineArguments = + string.IsNullOrEmpty(config.CommandLineArguments) + ? "/NoSigninPrompt" + : config.CommandLineArguments + " /NoSigninPrompt"; + + return config; + } + + /// + /// Points Apex at the Visual Studio install matching + /// (e.g. Canary). An explicit VisualStudio.InstallationUnderTest.Path override is respected. + /// + private void EnsureTargetInstallationSelected() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(InstallationUnderTestPathVariable))) + { + return; + } + + string devenvPath = ResolveDevenvByMilestone(this.TargetProductMilestone); + if (!string.IsNullOrEmpty(devenvPath)) + { + Environment.SetEnvironmentVariable(InstallationUnderTestPathVariable, devenvPath); + } + } + + private static string ResolveDevenvByMilestone(string milestone) + { + string programFilesX86 = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + string vswhere = Path.Combine(programFilesX86, "Microsoft Visual Studio", "Installer", "vswhere.exe"); + if (!File.Exists(vswhere)) + { + return null; + } + + var startInfo = new ProcessStartInfo(vswhere, "-all -prerelease -format json -utf8") + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + string json; + using (var process = Process.Start(startInfo)) + { + json = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + } + + if (string.IsNullOrWhiteSpace(json) + || !(new JavaScriptSerializer { MaxJsonLength = int.MaxValue }.DeserializeObject(json) is object[] installations)) + { + return null; + } + + foreach (var installation in installations.OfType>()) + { + if (installation.TryGetValue("catalog", out var catalogObject) + && catalogObject is Dictionary catalog + && catalog.TryGetValue("productMilestone", out var installedMilestone) + && string.Equals(installedMilestone as string, milestone, StringComparison.OrdinalIgnoreCase) + && installation.TryGetValue("productPath", out var productPath)) + { + return productPath as string; + } + } + + return null; + } + + /// + /// Moves the caret onto , triggers the code-fix request and + /// expands the resulting light bulb. + /// + protected ILightBulbTestExtension MoveToExpressionAndExpandLightBulb(TextDocumentView document, string expression) + { + // Nudging the caret into the diagnostic range triggers a request for code fixes. + document.MoveToExpression(expression); + document.MoveRight(); + document.MoveLeft(); + + this.Library.Synchronization.WaitForLightBulb(); + + Assert.IsTrue( + this.Library.Editor.LightBulb.Verify.IsLightBulbPresent(TimeSpan.FromSeconds(60)), + $"Expected a light bulb to be present at expression '{expression}'."); + + var lightBulb = this.Library.Editor.LightBulb.GetActiveLightBulb(); + lightBulb.Expand(); + + // Wait for the suggested actions to populate. Poll the already-expanded session directly: + // re-querying GetActiveLightBulb() here returns null once the light bulb is expanded into + // its flyout (LightBulbBroker no longer reports it as the active session), and the flyout + // taking focus can also null out Library.Editor (ActiveDocumentWindowAsTextEditor) — either + // of which previously threw a NullReferenceException inside the poll. LightBulbTestExtension + // .Actions reads the captured session and returns an empty (never null) set while pending. + this.Library.Synchronization.TryWaitForCondition(() => lightBulb.Actions.Any()); + + return lightBulb; + } + + /// + /// Creates a fresh single-project F# solution whose Library.fs contains + /// and opens it in the editor. Waits for the project to finish loading before editing (so the + /// F# project system does not reload the empty file over the insert) and confirms + /// reached the buffer before returning, so downstream caret + /// searches are not racing document initialization. + /// + protected TextDocumentView OpenFSharpDocument(string code, string expectedContent) + { + var project = this.Library.ProjectCreation.CreateFSharpLibrary(); + var documentItem = this.Library.ProjectCreation.AddProjectItemFromEmptyFile(project, "Library.fs"); + var document = this.Library.OpenDocument(documentItem); + + this.Library.Synchronization.WaitForSolutionCrawler(); + + document.InsertText(code); + + Assert.IsTrue( + this.Library.Synchronization.TryWaitForCondition( + () => document.Contents.Contains(expectedContent), TimeSpan.FromSeconds(15)), + $"Inserted source did not appear in the document buffer (looking for '{expectedContent}')."); + + return document; + } + + /// + /// Builds the current solution and waits for it to finish. Building performs a NuGet restore and + /// resolves references — which the F# language service needs before it can type-check and surface + /// SEMANTIC diagnostics (e.g. the unused-open analyzer, or FS0760 for disposables). This is the + /// Apex counterpart of the source IdeFact tests' RestoreNuGetPackagesAsync + WaitForProjectSystem. + /// Parse-based fixes (FS0597, FS0010) don't need it. Build success is not asserted: some fixture + /// sources intentionally contain errors, and a failed compile still restores references. + /// + protected void BuildSolution() + { + this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Build(waitForBuildToFinish: true); + this.Library.Synchronization.WaitForSolutionCrawler(); + } + + /// + /// Builds the current solution and returns whether it succeeded. + /// + protected bool BuildSolutionSucceeded() + { + this.BuildSolution(); + return this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Succeeded; + } + + /// + /// Places the caret on , focuses the editor and invokes Go To + /// Definition, then waits for navigation to settle. The result is read from the active document + /// afterwards (which may be the same or a different file). + /// + protected void GoToDefinition(TextDocumentView document, string expression) + { + document.MoveToExpression(expression); + document.Focus(); + document.GoToDefinition(); + this.Library.Synchronization.WaitForSolutionCrawler(); + } + + /// + /// Invokes Go To Definition on and polls until the active document + /// settles on a line matching and, when given, a window caption + /// equal to . Go To Definition navigates asynchronously (it may + /// open/activate another document and only then move the caret), so — exactly like the light-bulb + /// helper waits for its actions — we poll for the end state instead of reading it immediately. + /// Reading it immediately is the fire-and-assert race that makes the ported navigation tests flaky. + /// + protected void GoToDefinitionAndWait( + TextDocumentView document, + string expression, + Func lineMatches, + string expectedCaption = null) + { + this.GoToDefinition(document, expression); + + string lastLine = null; + string lastCaption = null; + bool landed = this.Library.Synchronization.TryWaitForCondition( + () => + { + lastLine = this.Library.ActiveDocumentCurrentLineText; + lastCaption = this.Library.ActiveDocumentCaption; + return lastLine != null + && lineMatches(lastLine) + && (expectedCaption == null || string.Equals(lastCaption, expectedCaption, StringComparison.Ordinal)); + }, + TimeSpan.FromSeconds(30)); + + Assert.IsTrue( + landed, + $"Go To Definition on '{expression}' did not settle on the expected location. " + + (expectedCaption != null ? $"Expected caption '{expectedCaption}', actual '{lastCaption}'. " : string.Empty) + + $"Actual current line: '{lastLine}'."); + } + + /// + /// Asserts two pieces of F# source are equal, ignoring line-ending style and any trailing + /// newline. Template output and editor buffers differ in those incidental ways across SDKs, so + /// normalizing avoids brittle failures while still comparing the meaningful content exactly. + /// + protected static void AssertSourceEquals(string expected, string actual) + { + Assert.AreEqual(NormalizeSource(expected), NormalizeSource(actual)); + } + + /// + /// Normalizes F# source for comparison by unifying line-ending style and trimming any trailing + /// newline, so comparisons ignore those incidental differences. + /// + protected static string NormalizeSource(string source) + => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n'); + + /// + /// Verifies that placing the caret on in the given F# source + /// offers exactly one light-bulb action whose text contains . + /// Mirrors the display-text assertions in FSharp.Editor.IntegrationTests CodeActionTests (the Apex + /// light-bulb model exposes a flat action list, so the code-fix/error-fix category is not checked). + /// + protected void AssertCodeActionOffered(string code, string caretExpression, string expectedActionText) + { + var document = this.OpenFSharpDocument(code, caretExpression); + + // Restore/resolve references so the F# language service can type-check and offer semantic + // fixes (the unused-open analyzer, FS0760, ...); without this their light bulb never appears. + this.BuildSolution(); + + var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, caretExpression); + + var matchCount = lightBulb.Actions.Count(action => + action.Text.IndexOf(expectedActionText, StringComparison.OrdinalIgnoreCase) >= 0); + + Assert.AreEqual( + 1, + matchCount, + $"Expected exactly one code action containing '{expectedActionText}'. Offered actions: " + + string.Join(", ", lightBulb.Actions.Select(action => $"'{action.Text}'"))); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs new file mode 100644 index 00000000000..a3b98ab410f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs @@ -0,0 +1,82 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Linq; +using Microsoft.Test.Apex; +using Microsoft.Test.Apex.Services; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.Test.Apex.VisualStudio.Editor; +using Microsoft.Test.Apex.VisualStudio.Shell; +using Microsoft.Test.Apex.VisualStudio.Solution; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Compact test library for the F# Apex integration tests. Provides the small facade over the + /// Apex that the code-action tests use (project creation, + /// synchronization, the active editor and document opening), tailored for the F# extension. + /// + public sealed class FSharpLanguageServiceLibrary + { + public FSharpLanguageServiceLibrary(VisualStudioHost visualStudio, IOperations operations) + { + this.VisualStudio = visualStudio; + this.Synchronization = new SynchronizationHelper(visualStudio, operations.Get()); + this.ProjectCreation = new ProjectCreationHelper(visualStudio); + } + + /// The Apex Visual Studio host. + public VisualStudioHost VisualStudio { get; } + + /// Synchronization helpers used to wait for background work. + public SynchronizationHelper Synchronization { get; } + + /// F# project and project-item creation helpers. + public ProjectCreationHelper ProjectCreation { get; } + + /// The editor of the active document window, or null if none is active. + public IVisualStudioTextEditorTestExtension Editor + => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindowAsTextEditor?.Editor; + + /// + /// Opens the given project item in the text editor and returns a view over it. + /// + public TextDocumentView OpenDocument(ProjectItemTestExtension documentItem) + { + var window = documentItem.Open(); + return new TextDocumentView(window); + } + + /// + /// Finds under , opens it in the text + /// editor and returns a view over it. The project's item tree is populated lazily, so the lookup + /// is polled (TryFindChild is single-shot); on timeout it throws listing the project's actual + /// top-level item names, which makes a template-filename surprise obvious. + /// + public TextDocumentView OpenProjectFile(ProjectTestExtension project, string fileName) + { + ProjectItemTestExtension item = null; + this.Synchronization.TryWaitForCondition( + () => (item = project.TryFindChild(fileName, true)) != null); + + if (item == null) + { + var actualNames = string.Join(", ", project.ProjectItems.Select(child => $"'{child.Name}'")); + throw new InvalidOperationException( + $"Project item '{fileName}' was not found in project '{project.Name}'. Items present: {actualNames}."); + } + + return this.OpenDocument(item); + } + + /// The short caption (file name) of the active document window, or null if none is active. + public string ActiveDocumentCaption + => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindow?.Caption; + + /// The text of the caret's current line in the active document editor, or null if none is active. + public string ActiveDocumentCurrentLineText + => this.Editor?.Caret.GetCurrentLineText(); + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs new file mode 100644 index 00000000000..11a837bd233 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs @@ -0,0 +1,117 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.Test.Apex.VisualStudio.Solution; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Creates F# projects and project items for the Apex integration tests. + /// Tailored for the F# extension: the TypeScript-VS ProjectCreationHelper created ASP.NET Core + /// web apps for JavaScript/TypeScript; this one creates plain F# class libraries. + /// + public sealed class ProjectCreationHelper + { + private readonly VisualStudioHost visualStudio; + private int projectCounter = 0; + + public ProjectCreationHelper(VisualStudioHost visualStudio) + { + this.visualStudio = visualStudio; + } + + /// + /// Creates a new F# class library in a fresh solution. + /// + public ProjectTestExtension CreateFSharpLibrary() + { + string projectName = $"FSharpLibrary_{++this.projectCounter}"; + return this.CreateFSharpProject(ProjectTemplate.ClassLibrary, projectName); + } + + /// + /// Creates a new F# project of the given built-in template with the given name. + /// + public ProjectTestExtension CreateFSharpProject(ProjectTemplate template, string projectName) + => this.visualStudio.ObjectModel.Solution.CreateProject( + ProjectLanguage.FSharp, + template, + projectName); + + /// + /// Creates a fresh single-project solution from an F# SDK template (e.g. "classlib", "console", + /// "xunit") scaffolded on disk with dotnet new. Apex's enum + /// maps to the legacy .NET Framework F# templates (which produce Library1.fs/Script.fsx), whereas + /// the source FSharp.Editor.IntegrationTests target the .NET SDK templates; using dotnet new + /// makes the project (filenames and default source) match those. An empty solution is created + /// first because AddProject throws when no solution is open. + /// + public ProjectTestExtension CreateFSharpProjectFromSdkTemplate(string sdkTemplateName, string projectName) + { + string projectDirectory = Path.Combine( + Path.GetTempPath(), "FSharpApexTemplates", Guid.NewGuid().ToString("N"), projectName); + Directory.CreateDirectory(projectDirectory); + + RunDotNet($"new {sdkTemplateName} --language \"F#\" --name \"{projectName}\" --output \"{projectDirectory}\""); + + string projectPath = Path.Combine(projectDirectory, projectName + ".fsproj"); + this.visualStudio.ObjectModel.Solution.CreateEmptySolution(); + return this.visualStudio.ObjectModel.Solution.AddProject(projectPath); + } + + /// + /// Creates an empty file on disk and adds it to the given project or project item. + /// + public ProjectItemTestExtension AddProjectItemFromEmptyFile(IProjectItemsContainer container, string fileName) + { + string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath; + string filePath = Path.Combine(containerDirectory, fileName); + File.WriteAllText(filePath, string.Empty); + return container.AddProjectItemFromFile(filePath); + } + + /// + /// Creates a file with the given content on disk and adds it to the given project or item. + /// + public ProjectItemTestExtension AddProjectItemFromContent(IProjectItemsContainer container, string fileName, string content) + { + string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath; + string filePath = Path.Combine(containerDirectory, fileName); + File.WriteAllText(filePath, content); + return container.AddProjectItemFromFile(filePath); + } + + private static void RunDotNet(string arguments) + { + var startInfo = new ProcessStartInfo("dotnet", arguments) + { + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"Failed to start 'dotnet {arguments}'."); + + // Drain both redirected streams concurrently. Reading one to completion before the other can + // deadlock: if the child fills the stdout pipe buffer while we block on stderr's EOF (which + // only arrives at process exit), neither side can make progress. + Task standardErrorTask = process.StandardError.ReadToEndAsync(); + process.StandardOutput.ReadToEnd(); + string standardError = standardErrorTask.GetAwaiter().GetResult(); + process.WaitForExit(); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"'dotnet {arguments}' failed with exit code {process.ExitCode}: {standardError}"); + } + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs new file mode 100644 index 00000000000..a5f834042ec --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Drop-in replacement for that re-runs a test up to + /// maxAttempts times, passing as soon as one attempt passes. These Apex tests drive a real + /// Visual Studio through UI automation and are occasionally flaky (focus, background analysis and + /// build/restore timing); a bounded retry keeps them reliable without masking a genuine failure — + /// every attempt has to fail before the test is reported as failed. + /// + /// Each attempt goes through the full per-test pipeline (a fresh test-class instance plus + /// TestInitialize/TestCleanup, i.e. a fresh VS session for the Apex host), so a retry starts from a + /// clean state rather than reusing the dirtied state of the previous attempt. + /// + /// MSTest's own [Retry] attribute is only available from 3.8; this repo pins 3.6.4, so the + /// classic "derive from TestMethodAttribute and override Execute" idiom is used instead. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class RetryTestMethodAttribute : TestMethodAttribute + { + private readonly int maxAttempts; + + public RetryTestMethodAttribute(int maxAttempts = 2) + { + if (maxAttempts < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "At least one attempt is required."); + } + + this.maxAttempts = maxAttempts; + } + + public override TestResult[] Execute(ITestMethod testMethod) + { + TestResult[] results = null; + + for (var attempt = 1; attempt <= this.maxAttempts; attempt++) + { + results = base.Execute(testMethod); + + if (results.All(result => result.Outcome == UnitTestOutcome.Passed)) + { + return results; + } + + if (attempt < this.maxAttempts) + { + Trace.WriteLine( + $"[RetryTestMethod] '{testMethod.TestMethodName}' failed on attempt {attempt} of {this.maxAttempts}; retrying."); + } + } + + return results; + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs new file mode 100644 index 00000000000..80969ffd74c --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs @@ -0,0 +1,58 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using Microsoft.Test.Apex.Services; +using Microsoft.Test.Apex.VisualStudio; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Synchronization helper for the F# Apex integration tests. + /// This is a compact, self-contained equivalent of the TypeScript-VS SynchronizationHelper: + /// where that type delegated to the Roslyn "WaitForFeatures" async-operation waiter, this one + /// polls via the Apex so the harness has no dependency on + /// the JavaScript/TypeScript BrowserTools layer. + /// + public sealed class SynchronizationHelper + { + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30); + private static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(1); + + private readonly VisualStudioHost visualStudio; + private readonly ISynchronizationService synchronizationService; + + public SynchronizationHelper(VisualStudioHost visualStudio, ISynchronizationService synchronizationService) + { + this.visualStudio = visualStudio; + this.synchronizationService = synchronizationService; + } + + /// + /// Waits for the solution to be fully loaded and for background analysis to settle. + /// + public void WaitForSolutionCrawler() + { + this.visualStudio.ObjectModel.Solution.WaitForFullyLoaded(); + this.Settle(); + } + + /// + /// Waits for the light bulb / code-action pipeline to settle before the light bulb is queried. + /// + public void WaitForLightBulb() => this.Settle(); + + /// + /// Polls until it returns true or the timeout elapses. + /// + /// True if the condition became true within the timeout; otherwise false. + public bool TryWaitForCondition(Func condition, TimeSpan? timeout = null) + { + return this.synchronizationService.TryWaitFor(timeout ?? DefaultTimeout, () => condition()); + } + + private void Settle() + => this.synchronizationService.TryWaitFor(DefaultInterval, () => false); + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs new file mode 100644 index 00000000000..47e756ea87c --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs @@ -0,0 +1,132 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Test.Apex.VisualStudio.Editor; +using Microsoft.Test.Apex.VisualStudio.Shell; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Represents the text view of an open VS document. Compact, self-contained equivalent of the + /// TypeScript-VS TextDocumentView / TextDocumentViewBase, wrapping the Apex text editor extension + /// with the small set of operations the code-action tests need. + /// + public sealed class TextDocumentView + { + private readonly TextEditorDocumentWindowTestExtension window; + + public TextDocumentView(TextEditorDocumentWindowTestExtension window) + { + this.window = window; + } + + private IVisualStudioTextEditorTestExtension editor => this.window.Editor; + + /// The full text content of the document. + public string Contents => this.editor.Contents; + + /// The path of the file backing this document. + public string FilePath => this.window.FilePath; + + /// + /// Inserts text verbatim at the current caret position via a direct text-buffer edit. + /// Unlike InsertTextWithReturn, this does not simulate per-character typing, so the F# + /// editor's brace/quote auto-completion, IntelliSense auto-commit and smart indentation do not + /// fire and rewrite the buffer. That keeps the inserted source exactly as given, which the + /// code-fix tests rely on when locating expressions afterwards. + /// + public void InsertText(string text) => this.editor.Edit.InsertTextInBuffer(text); + + /// + /// Moves the caret to the first (or -th) occurrence of an + /// expression. Apex's caret search treats the argument as a regular expression, but every caller + /// passes a literal source fragment (which may contain regex metacharacters such as ')' in + /// "SomeType)"), so the input is escaped to match it literally. + /// + public void MoveToExpression(string expression, int matchIndex = 1) + => this.editor.Caret.MoveToExpression(Regex.Escape(expression), matchIndex: matchIndex); + + /// Moves the caret one character to the left. + public void MoveLeft() => this.editor.Caret.MoveLeft(); + + /// Moves the caret one character to the right. + public void MoveRight() => this.editor.Caret.MoveRight(); + + /// Gives this document's editor keyboard focus (needed before caret-driven commands). + public void Focus() => this.editor.Focus(); + + /// The text of the line the caret is currently on (untrimmed). + public string CurrentLineText => this.editor.Caret.GetCurrentLineText(); + + /// Invokes Go To Definition from the current caret position. + public void GoToDefinition() => this.editor.Caret.GoToDefinition(); + + /// + /// Replaces the entire document with and saves it to disk. Build tests + /// need this: the template auto-opens the default file, so the source must be set through the + /// open document (buffer) and saved, otherwise an out-of-band disk write is superseded by the + /// still-open (valid template) buffer and the compiler never sees the intended code. + /// + /// The template content loads into the buffer asynchronously after the document opens, so this + /// first waits for that initial content to arrive and then applies the whole-buffer replacement + /// with verify-and-retry: without this a replacement done too early is overwritten by the + /// late-arriving template content, leaving the original source on disk. The selection is deleted + /// before a direct buffer insert (which does not itself replace a selection and, unlike simulated + /// typing, avoids brace/quote auto-completion). + /// + public void ReplaceAllAndSave(string text) + { + this.editor.Focus(); + + if (!WaitUntil(() => !string.IsNullOrEmpty(this.editor.Contents))) + { + throw new InvalidOperationException("Document did not load its initial content before editing."); + } + + var replaced = WaitUntil(() => + { + this.editor.Selection.SelectAll(); + this.editor.Edit.DeleteCharNext(1); + this.editor.Edit.InsertTextInBuffer(text); + return Normalize(this.editor.Contents) == Normalize(text); + }); + + if (!replaced) + { + throw new InvalidOperationException( + $"Failed to set document content. Expected:\n{text}\nActual:\n{this.editor.Contents}"); + } + + this.window.Save(); + + if (!this.window.TryWaitForNotDirty(TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(100))) + { + throw new InvalidOperationException("Document remained dirty after saving."); + } + } + + private static bool WaitUntil(Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); + do + { + if (condition()) + { + return true; + } + + Thread.Sleep(200); + } + while (DateTime.UtcNow < deadline); + + return false; + } + + private static string Normalize(string source) + => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n'); + } +}