[6.1] Port Flaky Test Separation

build.proj

@@ -37,6 +37,8 @@
       The provided path should be ended to a `\` character without white spaces: Ex. C:\x86\
     -->
     <DotnetPath></DotnetPath>
+    <!-- Default filter for test runs. Override with -p:Filter=... -->
+    <Filter Condition="'$(Filter)' == ''">category!=failing&amp;category!=flaky</Filter>
     <!-- Using these properties to compile and pack netfx dll fixes nuget package explorer error "Compiler flags: missing" -->
     <NugetPackProperties>DebugType=portable;DebugSymbols=true;IncludeSymbols=true;SymbolPackageFormat=snupkg;PublishRepositoryUrl=true;RepositoryUrl=https://github.com/dotnet/sqlclient;RepositoryType=git;EmbedUnTrackedSources=true;Deterministic=true;</NugetPackProperties>
     <!-- TF_BUILD is enabled only within AzureDevOps pipeline to support continuous integation build. -->
@@ -226,13 +228,13 @@
         -p:TestTargetOS=Windows$(TargetGroup)
         --collect "Code coverage"
         --results-directory $(ResultsDirectory)
-        --filter "category!=failing%26category!=flaky"
+        --filter "$(Filter)"
         --logger:"trx;LogFilePrefix=Unit-Windows$(TargetGroup)-$(TestSet)"
       </TestCommand>
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
     </PropertyGroup>
     <Message Text=">>> Running unit tests for Windows via command: $(TestCommand)"/>
     <Exec ConsoleToMsBuild="true" Command="$(TestCommand)"/>
   </Target>
 
   <!-- Run all unit tests applicable to Unix. -->
@@ -247,7 +249,7 @@
         -p:TestTargetOS=Unixnetcoreapp
         --collect "Code coverage"
         --results-directory $(ResultsDirectory)
-        --filter "category!=failing%26category!=flaky"
+        --filter "$(Filter)"
         --logger:"trx;LogFilePrefix=Unit-Unixnetcoreapp-$(TestSet)"
     </TestCommand>
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
@@ -273,7 +275,7 @@
         -p:TestTargetOS=Windows$(TargetGroup)
         --collect "Code coverage"
         --results-directory $(ResultsDirectory)
-        --filter "category!=non$(TargetGroup)tests&amp;category!=failing&amp;category!=nonwindowstests"
+        --filter "$(Filter)&amp;category!=non$(TargetGroup)tests&amp;category!=nonwindowstests"
         --logger:"trx;LogFilePrefix=Functional-Windows$(TargetGroup)-$(TestSet)"
       </TestCommand>
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
@@ -296,7 +298,7 @@
         -p:TestTargetOS=Unixnetcoreapp
         --collect "Code coverage"
         --results-directory $(ResultsDirectory)
-        --filter "category!=nonnetcoreapptests&amp;category!=failing&amp;category!=nonlinuxtests&amp;category!=nonuaptests"
+        --filter "$(Filter)&amp;category!=nonnetcoreapptests&amp;category!=nonlinuxtests&amp;category!=nonuaptests"
         --logger:"trx;LogFilePrefix=Functional-Unixnetcoreapp-$(TestSet)"
       </TestCommand>
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
@@ -322,13 +324,13 @@
         -p:TestTargetOS=Windows$(TargetGroup)
         --collect "Code coverage"
         --results-directory $(ResultsDirectory)
-        --filter "category!=non$(TargetGroup)tests&amp;category!=failing&amp;category!=nonwindowstests"
+        --filter "$(Filter)&amp;category!=non$(TargetGroup)tests&amp;category!=nonwindowstests"
         --logger:"trx;LogFilePrefix=Manual-Windows$(TargetGroup)-$(TestSet)"
       </TestCommand>
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
     </PropertyGroup>
     <Message Text=">>> Running Manual test for Windows via command: $(TestCommand)" />
     <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
   </Target>
 
   <!-- Run all Manual tests applicable to Unix. -->
@@ -345,7 +347,7 @@
         -p:TestTargetOS=Unixnetcoreapp
         --collect "Code coverage"
         --results-directory $(ResultsDirectory)
-        --filter "category!=nonnetcoreapptests&amp;category!=failing&amp;category!=nonlinuxtests&amp;category!=nonuaptests"
+        --filter "$(Filter)&amp;category!=nonnetcoreapptests&amp;category!=nonlinuxtests&amp;category!=nonuaptests"
         --logger:"trx;LogFilePrefix=Manual-Unixnetcoreapp-$(TestSet)"
       </TestCommand>
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>

eng/pipelines/common/templates/steps/run-all-tests-step.yml

@@ -68,6 +68,22 @@ steps:
         condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
         retryCountOnTaskFailure: 1
 
+    - task: MSBuild@1
+      displayName: 'Run Flaky Unit Tests ${{ parameters.msbuildArchitecture }}'
+      inputs:
+        solution: build.proj
+        msbuildArchitecture: ${{parameters.msbuildArchitecture }}
+        platform: '${{parameters.platform }}'
+        configuration: '${{parameters.configuration }}'
+        ${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
+          msbuildArguments: '-t:RunUnitTests -p:TF=${{parameters.targetFramework }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:Filter="category=flaky"'
+        ${{ else }}: #x86
+          msbuildArguments: '-t:RunUnitTests -p:TF=${{parameters.targetFramework }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }} -p:Filter="category=flaky"'
+        verbosityRestore: Detailed
+        verbosityPack: Detailed
+      condition: succeededOrFailed()
+      continueOnError: true
+
   - task: MSBuild@1
     displayName: 'Run Functional Tests ${{parameters.msbuildArchitecture }}'
     inputs:
@@ -79,9 +95,23 @@ steps:
         msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
       ${{ else }}: # x86
         msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }}'
-    condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+    condition: succeededOrFailed()
     retryCountOnTaskFailure: 1
 
+  - task: MSBuild@1
+    displayName: 'Run Flaky Functional Tests ${{parameters.msbuildArchitecture }}'
+    inputs:
+      solution: build.proj
+      msbuildArchitecture: ${{parameters.msbuildArchitecture }}
+      platform: '${{parameters.platform }}'
+      configuration: '${{parameters.configuration }}'
+      ${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
+        msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:Filter="category=flaky"'
+      ${{ else }}: # x86
+        msbuildArguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }} -p:Filter="category=flaky"'
+    condition: succeededOrFailed()
+    continueOnError: true
+
   - task: MSBuild@1
     displayName: 'Run Manual Tests ${{parameters.msbuildArchitecture }}'
     inputs:
@@ -93,9 +123,23 @@ steps:
         msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
       ${{ else }}: # x86
         msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }}'
-    condition: eq(variables['Agent.OS'], 'Windows_NT')
+    condition: succeededOrFailed()
     retryCountOnTaskFailure: 2
 
+  - task: MSBuild@1
+    displayName: 'Run Flaky Manual Tests ${{parameters.msbuildArchitecture }}'
+    inputs:
+      solution: build.proj
+      msbuildArchitecture: ${{parameters.msbuildArchitecture }}
+      platform: '${{parameters.platform }}'
+      configuration: '${{parameters.configuration }}'
+      ${{ if eq(parameters.msbuildArchitecture, 'x64') }}:
+        msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:Filter="category=flaky"'
+      ${{ else }}: # x86
+        msbuildArguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:DotnetPath=${{parameters.dotnetx86RootPath }} -p:Filter="category=flaky"'
+    condition: succeededOrFailed()
+    continueOnError: true
+
 - ${{ else }}: # Linux or macOS
   - ${{if eq(parameters.referenceType, 'Project')}}:
     - task: DotNetCoreCLI@2
@@ -110,6 +154,18 @@ steps:
       retryCountOnTaskFailure: 1
       condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
 
+    - task: DotNetCoreCLI@2
+      displayName: 'Run Flaky Unit Tests'
+      inputs:
+        command: custom
+        projects: build.proj
+        custom: msbuild
+        arguments: '-t:RunUnitTests -p:TF=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:TestMicrosoftDataSqlClientVersion=${{ parameters.nugetPackageVersion }} -p:platform=${{ parameters.platform }} -p:Configuration=${{ parameters.configuration }} -p:Filter="category=flaky"'
+        verbosityRestore: Detailed
+        verbosityPack: Detailed
+      condition: succeededOrFailed()
+      continueOnError: true
+
   - task: DotNetCoreCLI@2
     displayName: 'Run Functional Tests'
     inputs:
@@ -120,7 +176,19 @@ steps:
       verbosityRestore: Detailed
       verbosityPack: Detailed
     retryCountOnTaskFailure: 1
-    condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
+    condition: succeededOrFailed()
+
+  - task: DotNetCoreCLI@2
+    displayName: 'Run Flaky Functional Tests'
+    inputs:
+      command: custom
+      projects: build.proj
+      custom: msbuild
+      arguments: '-t:RunFunctionalTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }} -p:Filter="category=flaky"'
+      verbosityRestore: Detailed
+      verbosityPack: Detailed
+    condition: succeededOrFailed()
+    continueOnError: true
 
   - task: DotNetCoreCLI@2
     displayName: 'Run Manual Tests'
@@ -132,4 +200,16 @@ steps:
       verbosityRestore: Detailed
       verbosityPack: Detailed
     retryCountOnTaskFailure: 2
-    condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
+    condition: succeededOrFailed()
+
+  - task: DotNetCoreCLI@2
+    displayName: 'Run Flaky Manual Tests'
+    inputs:
+      command: custom
+      projects: build.proj
+      custom: msbuild
+      arguments: '-t:RunManualTests -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }} -p:Filter="category=flaky"'
+      verbosityRestore: Detailed
+      verbosityPack: Detailed
+    condition: succeededOrFailed()
+    continueOnError: true

src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs

@@ -331,6 +331,7 @@ public static string CurrentTestName(ITestOutputHelper outputHelper)
             return match.Groups[1].Value;
         }
 
+        // @TODO: Depending on the test runner, ClassName may or may not be populated!
         private static readonly Regex TestNameRegex = new(
             // Capture the ClassName.MethodName portion, which may terminate
             // the name, or have (args...) appended.

src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/XEventsTracingTest.cs

@@ -13,6 +13,83 @@
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests
 {
+    // XEvent sessions may become orphaned on the Azure SQL Server, which leads to poor performance
+    // (query timeouts, deadlocks, etc) over time.  This class is instantiated once per test run and
+    // drops these orphaned sessions as part of every run to help mitigate this issue.
+    public class XEventCleaner
+    {
+        public XEventCleaner()
+        {
+            if (DataTestUtility.AreConnStringsSetup() &&
+                DataTestUtility.IsNotAzureSynapse() &&
+                DataTestUtility.IsNotManagedInstance())
+            {
+                using SqlConnection connection = new(DataTestUtility.TCPConnectionString);
+                connection.Open();
+
+                // Identify orphaned event sessions and generate DROP commands.
+                using SqlCommand command = new(
+                    """
+                    SELECT Sessions.name
+                    FROM sys.database_event_sessions Sessions
+                    LEFT JOIN sys.dm_xe_database_sessions Active ON Sessions.name = Active.name
+                    WHERE Active.name IS NULL;
+                    """,
+                    connection);
+
+                HashSet<string> orphanedSessions = new();
+                try
+                {
+                    using SqlDataReader reader = command.ExecuteReader();
+
+                    while (reader.Read())
+                    {
+                        string sessionName = reader.GetString(0);
+                        orphanedSessions.Add(sessionName);
+                    }
+                }
+                catch (SqlException)
+                {
+                    // Ignore exceptions - the next test run will attempt the cleanup.
+                }
+
+                if (orphanedSessions.Count == 0)
+                {
+                    return;
+                }
+
+                Console.WriteLine($"Identified {orphanedSessions.Count} orphaned XEvent sessions:");
+
+                // Drop them one at a time.
+                foreach (string sessionName in orphanedSessions)
+                {
+                    using SqlCommand dropCommand = new(
+                        $"DROP EVENT SESSION [{sessionName}] ON DATABASE;",
+                        connection);
+
+                    try
+                    {
+                        dropCommand.ExecuteNonQuery();
+                    }
+                    catch (SqlException)
+                    {
+                        // Ignore exceptions, as the session may have been cleaned up by another
+                        // test run at the same time.
+                    }
+
+                    Console.WriteLine($"  Dropped orphaned XEvent session: {sessionName}");
+                }
+            }
+        }
+    }
+
+    // This empty class is required by xUnit to tie the cleaner to the test classes.
+    [CollectionDefinition("XEventCleaner")]
+    public class XEventCleanerCollection : ICollectionFixture<XEventCleaner>
+    {
+    }
+
+    [Collection("XEventCleaner")]
     public class XEventsTracingTest
     {
         private readonly string _testName;
@@ -22,6 +99,7 @@ public XEventsTracingTest(ITestOutputHelper outputHelper)
             _testName = DataTestUtility.CurrentTestName(outputHelper);
         }
 
+        [Trait("Category", "flaky")]
         [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsNotManagedInstance))]
         [InlineData("SELECT @@VERSION", System.Data.CommandType.Text, "sql_statement_starting")]
         [InlineData("sp_help", System.Data.CommandType.StoredProcedure, "rpc_starting")]
@@ -39,7 +117,7 @@ public void XEventActivityIDConsistentWithTracing(string query, System.Data.Comm
 
             using SqlConnection xEventManagementConnection = new(DataTestUtility.TCPConnectionString);
             xEventManagementConnection.Open();
-            
+
             using DataTestUtility.XEventScope xEventSession = new(
                 _testName,
                 xEventManagementConnection,
@@ -84,7 +162,7 @@ private static string GetCommandActivityId(string commandText, string eventName,
             XPathNavigator? current = statementStartingQuery.Current;
             Assert.NotNull(current);
             XPathNavigator? activityIdElement = current.SelectSingleNode("action[@name='attach_activity_id_xfer']/value");
-            
+
             Assert.NotNull(activityIdElement);
             Assert.NotNull(activityIdElement.Value);
 

src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs

@@ -11,8 +11,8 @@
 
 namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests
 {
-    [Collection("SimulatedServerTests")]
     [Trait("Category", "flaky")]
+    [Collection("SimulatedServerTests")]
     public class ConnectionFailoverTests
     {
         //TODO parameterize for transient errors