VS integration tests running on CI

azure-pipelines-PR.yml

@@ -1,4 +1,4 @@
 # CI and PR triggers
 trigger:
   branches:
     include:
@@ -509,6 +509,64 @@
             continueOnError: true
             condition: failed()
 
+        # Windows Apex VS integration tests (non-gating — published for signal only)
+        - job: WindowsApexIntegration
+          variables:
+          - name: _configuration
+            value: Release
+          - name: __VSNeverShowWhatsNew
+            value: 1
+          pool:
+            name: $(DncEngPublicBuildPool)
+            demands: ImageOverride -equals $(_WindowsMachineQueueName)
+          timeoutInMinutes: 90
+          steps:
+          - checkout: self
+            clean: true
+
+          - powershell: eng\SetupVSHive.ps1
+            displayName: Setup VS Hive
+
+          # Build + deploy the F# VSIX into the RoslynDev hive, then run the Apex tests. The Apex host
+          # resolves the installed VS via LocateVisualStudio (set as VisualStudio.InstallationUnderTest.Path
+          # by the -testApex path), so it does not depend on the local-only 'Canary' milestone.
+          # continueOnError keeps this leg non-gating: failures surface as signal but do not fail the PR.
+          - script: eng\CIBuildNoPublish.cmd -configuration $(_configuration) -deployExtensions -testApex
+            env:
+              NativeToolsOnMachine: true
+            displayName: Build, deploy extension and run Apex integration tests
+            continueOnError: true
+
+          - task: PublishTestResults@2
+            displayName: Publish Apex Test Results
+            inputs:
+              testResultsFormat: 'VSTest'
+              testRunTitle: WindowsApexIntegration
+              mergeTestResults: true
+              testResultsFiles: '*.trx'
+              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)'
+            continueOnError: true
+            condition: succeededOrFailed()
+
+          - task: PublishBuildArtifacts@1
+            displayName: Publish Apex Test Logs
+            inputs:
+              PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)'
+              ArtifactName: Windows $(_configuration) Apex test logs
+              publishLocation: Container
+            continueOnError: true
+            condition: always()
+
+          - task: PublishBuildArtifacts@1
+            displayName: Publish Apex Build Logs
+            condition: always()
+            continueOnError: true
+            inputs:
+                PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)'
+                ArtifactName: Windows $(_configuration) Apex build logs
+                ArtifactType: Container
+                parallel: true
+
           # Windows With Compressed Metadata Desktop (split into 3 batches)
         - job: WindowsCompressedMetadata_Desktop
           strategy:

eng/Build.ps1

@@ -60,6 +60,7 @@ param (
     [switch]$testIntegration,
     [switch]$testScripting,
     [switch]$testVs,
+    [switch]$testApex,
     [switch]$testAll,
     [switch]$testAllButIntegration,
     [switch]$testAllButIntegrationAndAot,
@@ -128,6 +129,7 @@ function Print-Usage() {
     Write-Host "  -testIntegration              Run F# integration tests"
     Write-Host "  -testScripting                Run Scripting tests"
     Write-Host "  -testVs                       Run F# editor unit tests"
+    Write-Host "  -testApex                     Run F# Apex VS integration tests (requires -deployExtensions)"
     Write-Host "  -testpack                     Verify built packages"
     Write-Host "  -testAOT                      Run AOT/Trimming tests"
     Write-Host "  -testEditor                   Run VS Editor tests"
@@ -402,6 +404,245 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     Exec-Console $dotnetExe $test_args
 }
 
+# Runs the Apex VS integration tests. These are a classic MSTest (VSTest) project, NOT part of the
+# repo's Microsoft.Testing.Platform 'dotnet test' path, and they drive a real VS instance via the Apex
+# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS).
+# The project is intentionally excluded from VisualFSharp.slnx (its dependencies must version-match the
+# VS it drives), so this builds it explicitly. It also must match a specific VS version
+# (MicrosoftTestApexVisualStudioVersion); if the installed VS does not match, the run is skipped rather
+# than producing confusing Apex/VS-mismatch failures. The F# VSIX must be deployed into the RoslynDev
+# hive first (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev.
+
+# ===== BEGIN Apex CI diagnostics helpers (Phase 2, TEMPORARY — revert once the CI hang is understood) =====
+# On CI the Apex host failed with "Timed out ... trying get running object": devenv started but never
+# registered its DTE automation object in the ROT. These helpers (a) make sure devenv has an interactive
+# console desktop (the usual reason DTE never registers) and (b) collect the RoslynDev-hive logs so we can
+# tell an interactive-desktop problem apart from an F# VSIX / package load failure.
+function Capture-ApexScreenshot([string] $path) {
+    # Returns $true only if a screenshot could be taken, which implies an interactive desktop is attached.
+    try {
+        Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop
+        Add-Type -AssemblyName System.Drawing -ErrorAction Stop
+        $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen
+        $bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
+        $gfx = [System.Drawing.Graphics]::FromImage($bmp)
+        try {
+            $gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
+            $bmp.Save($path, [System.Drawing.Imaging.ImageFormat]::Png)
+            Write-Host "Captured screenshot to '$path'."
+            return $true
+        } finally {
+            $gfx.Dispose(); $bmp.Dispose()
+        }
+    } catch {
+        Write-Host "Screenshot capture failed (no interactive desktop?): $($_.Exception.Message)"
+        return $false
+    }
+}
+
+function Prepare-ApexInteractiveSession([string] $screenshotPath) {
+    # Mirrors dotnet/roslyn's Setup-IntegrationTestRun: if the desktop is not interactive, reconnect the
+    # session to the console so devenv can publish its DTE automation object; then minimize other windows.
+    if (-not (Capture-ApexScreenshot $screenshotPath)) {
+        Write-Host "No interactive desktop detected; attempting to reconnect the session to the console."
+        try {
+            $quserItems = ((quser $env:USERNAME | Select-Object -Skip 1) -split '\s+')
+            $sessionId = $quserItems[2]
+            if ($sessionId -eq 'Disc') { $sessionId = $quserItems[1] }
+            Write-Host "tscon $sessionId /dest:console"
+            tscon $sessionId /dest:console
+            Start-Sleep -Seconds 3
+            [void](Capture-ApexScreenshot $screenshotPath)
+        } catch {
+            Write-Host "##vso[task.logissue type=warning]Failed to reconnect session to console: $($_.Exception.Message)"
+        }
+    }
+
+    try {
+        (New-Object -ComObject "Shell.Application").MinimizeAll()
+    } catch {
+        Write-Host "MinimizeAll failed: $($_.Exception.Message)"
+    }
+}
+
+function Collect-ApexDiagnostics([string] $screenshotPath, [string] $destination) {
+    # After-run screenshot plus the RoslynDev experimental-hive logs. ActivityLog.xml records whether the
+    # F# package failed to load; ComponentModelCache\*.err records MEF composition failures.
+    [void](Capture-ApexScreenshot $screenshotPath)
+
+    $roamingVs = Join-Path $env:USERPROFILE "AppData\Roaming\Microsoft\VisualStudio"
+    $localVs   = Join-Path $env:USERPROFILE "AppData\Local\Microsoft\VisualStudio"
+
+    if (Test-Path $roamingVs) {
+        foreach ($hiveDir in @(Get-ChildItem -Path $roamingVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) {
+            foreach ($name in @("ActivityLog.xml", "ActivityLog.xsl")) {
+                $src = Join-Path $hiveDir.FullName $name
+                if (Test-Path $src) {
+                    Copy-Item $src (Join-Path $destination "$($hiveDir.Name)-$name") -Force -ErrorAction SilentlyContinue
+                    Write-Host "Collected $src"
+                }
+            }
+        }
+    }
+    if (Test-Path $localVs) {
+        foreach ($hiveDir in @(Get-ChildItem -Path $localVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) {
+            $mefErr = Join-Path $hiveDir.FullName "ComponentModelCache\Microsoft.VisualStudio.Default.err"
+            if (Test-Path $mefErr) {
+                Copy-Item $mefErr (Join-Path $destination "$($hiveDir.Name)-Microsoft.VisualStudio.Default.err") -Force -ErrorAction SilentlyContinue
+                Write-Host "Collected $mefErr"
+            }
+        }
+    }
+}
+# ===== END Apex CI diagnostics helpers (Phase 2) =====
+
+function TestApexUsingVSTest([string] $targetFramework) {
+    $projectName = "FSharp.Editor.Apex.IntegrationTests"
+    $apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj"
+
+    # Create the results directory up-front so the CI 'publish test logs' step never fails on a missing
+    # path, even when the run is skipped below (a marker file records the skip reason).
+    $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
+    Create-Directory $testResultsDir
+
+    # The Apex package (MicrosoftTestApexVisualStudioVersion) is pinned to a VS 18.x build. Because the
+    # installed VS differs by machine (CI's image trails a dev's local install and they rarely share the
+    # same minor), gate on the MAJOR version by default so any VS 18.x can run the tests; set
+    # FSHARP_APEX_VS_VERSION to a stricter prefix (e.g. "18.9") to force an exact minor.
+    $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18" }
+
+    # Determine which VS will run the tests, and gate on its version. An explicit
+    # VisualStudio.InstallationUnderTest.Path override is trusted (gate skipped); otherwise enumerate
+    # ALL installed VS instances (not just the latest) and pick one whose version starts with the target
+    # prefix, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors.
+    $overridePath = ${env:VisualStudio.InstallationUnderTest.Path}
+    if ($overridePath) {
+        Write-Host "Using explicit VisualStudio.InstallationUnderTest.Path=$overridePath (VS version gate skipped)."
+        $vsDir = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $overridePath))
+    }
+    else {
+        $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"
+        # Query the properties separately (parallel arrays in a stable order) instead of parsing
+        # `-format json`: Windows PowerShell 5.1's ConvertFrom-Json mishandles vswhere's multi-instance
+        # JSON array (it collapses the instances into one object with array-valued properties), which
+        # would make every version comparison fail. Build.ps1 runs under Windows PowerShell 5.1.
+        $versions = @()
+        $paths = @()
+        $names = @()
+        if (Test-Path $vswhere) {
+            $versions = @(& $vswhere -all -prerelease -products * -property installationVersion)
+            $paths    = @(& $vswhere -all -prerelease -products * -property installationPath)
+            $names    = @(& $vswhere -all -prerelease -products * -property displayName)
+        }
+        $found = $versions -join ', '
+
+        # Print the full VS inventory for diagnostics (helps explain a skip on CI). Always logged.
+        Write-Host "Visual Studio instances found by vswhere ($($versions.Count)) [vswhere: $vswhere]:"
+        if ($versions.Count -eq 0) {
+            Write-Host "  (none)"
+        }
+        for ($k = 0; $k -lt $versions.Count; $k++) {
+            $mm = ($versions[$k] -split '\.')[0..1] -join '.'
+            $hasIde = Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe")
+            $name = if ($k -lt $names.Count) { $names[$k] } else { "" }
+            Write-Host ("  [{0}] version={1} (major.minor {2}) devenv={3} name='{4}' path={5}" -f $k, $versions[$k], $mm, $hasIde, $name, $paths[$k])
+        }
+
+        # Among installs whose version starts with the target prefix (major, or an explicit major.minor)
+        # and that actually have an IDE (devenv.exe — excludes Build Tools / incomplete installs), pick
+        # the LOWEST version: it is closest to the pinned Apex minor, and "Apex older than VS" is the
+        # safer cross-minor direction than the reverse.
+        $vsDir = $null
+        $selectedVersion = $null
+        $versionMatchPath = $null
+        $candidates = for ($k = 0; $k -lt $versions.Count; $k++) {
+            if ($versions[$k] -like "$expectedVsVersion.*") {
+                [PSCustomObject]@{ Version = $versions[$k]; Path = $paths[$k] }
+            }
+        }
+        $candidates = @($candidates | Sort-Object { [version]$_.Version })
+        foreach ($c in $candidates) {
+            if (-not $versionMatchPath) { $versionMatchPath = $c.Path }
+            if (Test-Path (Join-Path $c.Path "Common7\IDE\devenv.exe")) {
+                $vsDir = $c.Path.TrimEnd("\")
+                $selectedVersion = $c.Version
+                break
+            }
+        }
+
+        if (-not $vsDir) {
+            if ($versionMatchPath) {
+                Write-Host "##vso[task.logissue type=warning]Found VS $expectedVsVersion ($versionMatchPath) but no devenv.exe under Common7\IDE (incomplete install, Build Tools, or a VS update in progress); skipping Apex integration tests (non-gating)."
+                Write-Host "Skipping Apex tests: VS $expectedVsVersion present but devenv.exe missing at [$versionMatchPath]. If VS was updating, retry once it finishes."
+            }
+            else {
+                Write-Host "##vso[task.logissue type=warning]No installed VS matches the Apex target $expectedVsVersion (found: $found); skipping Apex integration tests (non-gating)."
+                Write-Host "Skipping Apex tests: no VS $expectedVsVersion found among installs [$found]. Set FSHARP_APEX_VS_VERSION or VisualStudio.InstallationUnderTest.Path to run against a different install."
+            }
+            Set-Content -Path (Join-Path $testResultsDir "apex-tests-skipped.txt") -Value "Apex integration tests skipped: no usable VS matching target $expectedVsVersion (installed: $found)."
+            return
+        }
+        Write-Host "Selected VS $selectedVersion at '$vsDir' for the Apex tests (target $expectedVsVersion)."
+        # This process env var propagates to the child vstest.console -> testhost -> Apex processes.
+        ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe"
+    }
+
+    $vstestConsole = @(
+        (Join-Path $vsDir "Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"),
+        (Join-Path $vsDir "Common7\IDE\Extensions\TestPlatform\vstest.console.exe")
+    ) | Where-Object { Test-Path $_ } | Select-Object -First 1
+    if (-not $vstestConsole) {
+        throw "vstest.console.exe not found under '$vsDir' (looked in CommonExtensions\Microsoft\TestWindow and Extensions\TestPlatform)."
+    }
+
+    # Build the Apex test project explicitly (it is intentionally excluded from VisualFSharp.slnx).
+    $dotnetPath = InitializeDotNetCli
+    $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+    $buildBinLog = "$LogDir\${projectName}_$targetFramework.binlog"
+    Exec-Console $dotnetExe "build `"$apexProject`" -c $configuration /bl:`"$buildBinLog`""
+
+    $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll"
+    if (-not (Test-Path $testAssembly)) {
+        throw "Apex test assembly not found at '$testAssembly' after build."
+    }
+
+    $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
+    $trxFileName = "$projectName.$targetFramework.$jobName.trx"
+
+    $vstestArgs = @(
+        $testAssembly,
+        "/Platform:x64",
+        "/Framework:.NETFramework,Version=v4.7.2",
+        "/logger:trx;LogFileName=$trxFileName",
+        "/ResultsDirectory:$testResultsDir"
+    )
+
+    Write-Host "$vstestConsole $($vstestArgs -join ' ')"
+
+    # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) =====
+    if ($ci) {
+        Prepare-ApexInteractiveSession (Join-Path $testResultsDir "apex-before-run.png")
+    }
+    # ===== END Apex CI diagnostics (Phase 2) =====
+
+    & $vstestConsole @vstestArgs
+    $vstestExitCode = $LASTEXITCODE
+
+    # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) =====
+    if ($ci) {
+        Collect-ApexDiagnostics (Join-Path $testResultsDir "apex-after-run.png") $testResultsDir
+    }
+    # ===== END Apex CI diagnostics (Phase 2) =====
+
+    # This leg is non-gating (signal only): a test failure must NOT fail the build. Surface it as a
+    # warning and let the published TRX / Tests tab carry the signal. Genuine infra errors (missing
+    # assembly / vstest.console) already threw above and remain hard failures.
+    if ($vstestExitCode -ne 0) {
+        Write-Host "##vso[task.logissue type=warning]Apex integration tests reported failures (vstest.console exit code $vstestExitCode). This leg is non-gating; see the published TRX and the Tests tab."
+        Write-Host "Apex tests exit code $vstestExitCode (non-gating; not failing the build)."
+    }
+}
+
 function Prepare-TempDir() {
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir
@@ -669,6 +910,10 @@ try {
         TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
     }
 
+    if ($testApex) {
+        TestApexUsingVSTest -targetFramework $script:desktopTargetFramework
+    }
+
     if ($testAOT) {
         Push-Location "$RepoRoot\tests\AheadOfTime"
         ./check.ps1

eng/SetupVSHive.ps1

@@ -12,6 +12,24 @@ $vsRegEdit = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'VSRegEdit
 $hive = "RoslynDev"
 &$vsRegEdit set "$vsDir" $hive HKCU "Roslyn\Internal\OnOff\Features" OOP64Bit dword 0
 
+# Suppress modal dialogs and online-profile prompts that block UI automation (Apex) and cause the
+# integration tests to hang until they time out. These mirror the settings dotnet/roslyn applies to its
+# RoslynDev hive before running VS integration tests.
+
+# Disable the editor "report exceptions" dialog: fail silently and keep the test running instead of
+# popping a modal that stalls automation.
+&$vsRegEdit set "$vsDir" $hive HKCU "Text Editor" "Report Exceptions" dword 0
+
+# Disable roaming settings so an online user profile / sign-in prompt can't interfere on CI.
+&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio" RoamingEnabled string "1*System.Boolean*False"
+
+# Disable background download UI to avoid toasts appearing over the IDE during a run.
+&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Setup\BackgroundDownload" Value dword 0
+
+# Disable targeted (remote settings) notifications. This one can't be set via VsRegEdit, so write it
+# directly to the user hive (it is not hive-suffix specific).
+reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1 | Out-Null
+
 Write-Host "-- VS Info --"
 $isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini'
 Get-Content $isolationIni | Write-Host