Tests | Manual Test "Set" Trait

build2.proj

@@ -223,13 +223,13 @@
       TestSet
       Applies to:     TestMdsManual target
       Description:    Used to select a set of tests to run from the MDS manual tests project. If not
-                      specified, all tests will be executed.
+                      specified, all manual tests will be executed. In the manual test target, this
+                      value is converted into a Set trait filter and combined with any TestFilters.
       Default value:  [blank]
       Allowed values: ([1][2][3][AE])
       Examples:       "1", "12", "AE", "12AE"
     -->
     <TestSet Condition="'$(TestSet)' == ''" />
-    <TestSetArgument Condition="'$(TestSet)' != ''">-p:TestSet="$(TestSet)"</TestSetArgument>
   </PropertyGroup>
 
   <!-- ================================================================= -->
@@ -466,14 +466,24 @@
       <LogFilePrefix Condition="'$(TestFramework)' != ''">$(LogFilePrefix)-$(TestFramework)</LogFilePrefix>
       <LogFilePrefix Condition="'$(TestSet)' != ''">$(LogFilePrefix)-$(TestSet)</LogFilePrefix>
 
+      <!-- Join test sets together, if any were selected -->
+      <NormalizedTestSet>$(TestSet.ToUpper())</NormalizedTestSet>
+      <TestSetFilter Condition="$(NormalizedTestSet.Contains('1'))">Set=1</TestSetFilter>
+      <TestSetFilter Condition="$(NormalizedTestSet.Contains('2'))">$(TestSetFilter)|Set=2</TestSetFilter>
+      <TestSetFilter Condition="$(NormalizedTestSet.Contains('3'))">$(TestSetFilter)|Set=3</TestSetFilter>
+      <TestSetFilter Condition="$(TestSet.ToUpper().Contains('AE'))">$(TestSetFilter)|Set=AE</TestSetFilter>
+      <TestSetFilter Condition="$(TestSetFilter) != ''">($(TestSetFilter.Trim('|')))</TestSetFilter>
+
+      <!-- Combine existing filters with test set filters -->
+      <ManualTestFilters>$(TestFilters)&amp;$(TestSetFilter)</ManualTestFilters>
+      <ManualTestFilters>$(ManualTestFilters.Trim('&amp;'))</ManualTestFilters>
+
       <DotnetCommand>
         "$(DotnetPath)dotnet" test "$(MdsManualTestProjectPath)"
         -p:Configuration=$(Configuration)
         $(TestBlameArgument)
         $(TestCodeCoverageArgument)
-        $(TestFiltersArgument)
-        $(TestFrameworkArgument)
-        $(TestSetArgument)
+        --filter "$(ManualTestFilters)"
         --results-directory "$(TestResultsFolderPath)"
         --logger:"trx;LogFilePrefix=$(LogFilePrefix)"
 
@@ -487,7 +497,7 @@
       <DotnetCommand>$([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " "))</DotnetCommand>
     </PropertyGroup>
     <Message Text=">>> Running manual tests for MDS via command: $(DotnetCommand)" />
     <Exec ConsoleToMsBuild="true" Command="$(DotnetCommand)" />
   </Target>
 
   <!-- TestMdsUnit: Runs unit tests for MDS -->
@@ -515,6 +525,6 @@
       <DotnetCommand>$([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " "))</DotnetCommand>
     </PropertyGroup>
     <Message Text=">>> Running unit tests for MDS via command: $(DotnetCommand)" />
     <Exec ConsoleToMsBuild="true" Command="$(DotnetCommand)" />
   </Target>
 </Project>

src/Microsoft.Data.SqlClient.slnx

@@ -316,20 +316,11 @@
     <Project Path="Microsoft.Data.SqlClient/tests/StressTests/SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj" />
     <Project Path="Microsoft.Data.SqlClient/tests/StressTests/SqlClient.Stress.Tests/SqlClient.Stress.Tests.csproj" />
   </Folder>
-  <Folder Name="/test/TestClr/">
-    <Project Path="Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/UDTs/Address/Address.csproj">
-      <BuildDependency Project="Microsoft.Data.SqlClient/netcore/src/Microsoft.Data.SqlClient.csproj" />
-      <BuildDependency Project="Microsoft.Data.SqlClient/netfx/src/Microsoft.Data.SqlClient.csproj" />
-    </Project>
-    <Project Path="Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/UDTs/Circle/Circle.csproj">
-      <BuildDependency Project="Microsoft.Data.SqlClient/netcore/src/Microsoft.Data.SqlClient.csproj" />
-    </Project>
-    <Project Path="Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/UDTs/Shapes/Shapes.csproj">
-      <BuildDependency Project="Microsoft.Data.SqlClient/netcore/src/Microsoft.Data.SqlClient.csproj" />
-    </Project>
-    <Project Path="Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/UDTs/Utf8String/Utf8String.csproj">
-      <BuildDependency Project="Microsoft.Data.SqlClient/netcore/src/Microsoft.Data.SqlClient.csproj" />
-    </Project>
+  <Folder Name="/test/TestUdts/">
+    <Project Path="Microsoft.Data.SqlClient/tests/TestUdts/Address/Address.csproj" />
+    <Project Path="Microsoft.Data.SqlClient/tests/TestUdts/Circle/Circle.csproj" />
+    <Project Path="Microsoft.Data.SqlClient/tests/TestUdts/Shapes/Shapes.csproj" />
+    <Project Path="Microsoft.Data.SqlClient/tests/TestUdts/Utf8String/Utf8String.csproj" />
   </Folder>
   <Project Path="../doc/apps/AzureAuthentication/AzureAuthentication.csproj" />
 </Solution>

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/AKVTests.cs

@@ -12,6 +12,7 @@
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
 {
+    [Trait("Set", "AE")]
     public class AKVTest : IClassFixture<SQLSetupStrategyAzureKeyVault>
     {
         private readonly SQLSetupStrategyAzureKeyVault _fixture;
@@ -59,12 +60,12 @@ public void TestEncryptDecryptWithAKV()
         }
 
         /*
-         This unit test is going to assess an issue where a failed decryption leaves a connection in a bad state  
+         This unit test is going to assess an issue where a failed decryption leaves a connection in a bad state
          when it is returned to the connection pool. If a subsequent connection is retried it will result in an "Internal connection fatal error",
-         which causes that connection to be doomed, preventing it from being returned to the pool. 
+         which causes that connection to be doomed, preventing it from being returned to the pool.
          Consequently, retrying a third connection will encounter the same decryption error, leading to a repetitive failure cycle.
 
-         The purpose of this unit test is to simulate a decryption error and verify that the connection remains usable when returned to the pool. 
+         The purpose of this unit test is to simulate a decryption error and verify that the connection remains usable when returned to the pool.
          It aims to confirm that three consecutive connections will consistently fail with the "Failed to decrypt column" error.
         */
         [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringSetupForAE), nameof(DataTestUtility.IsAKVSetupAvailable))]
@@ -100,10 +101,10 @@ public void ForcedColumnDecryptErrorTestShouldFail()
             // does not leave the connection in a bad state.
             // In each try, when a "Failed to decrypt error" is thrown, the connection's TDS Parser state object buffer is drained of any
             // pending data so it does not interfere with future operations. In addition, the TDS parser state object's reader.DataReady flag
-            // is set to false so that the calling function that catches the exception will not continue to use the reader. Otherwise, it will 
+            // is set to false so that the calling function that catches the exception will not continue to use the reader. Otherwise, it will
             // timeout waiting to read data that doesn't exist. Also, the TDS Parser state object HasPendingData flag is also set to false
             // to indicate that the buffer has been cleared and to avoid it getting cleared again in SqlDataReader.TryCloseInternal function.
-            // Finally, after successfully handling the decryption error, the connection is then returned back to the connection pool without 
+            // Finally, after successfully handling the decryption error, the connection is then returned back to the connection pool without
             // an error. A proof that the connection's state object is clean is in the second connection being able to throw the same error.
             // The third connection is for making sure we test 3 times as the minimum number of connections to reproduce the issue previously.
             for (int i = 0; i < 3; i++)
@@ -182,7 +183,7 @@ public void TestLocalCekCacheIsScopedToProvider()
             sqlDataReader2.Close();
 
             // global cek cache and local cek cache are populated above
-            // when using a new per-command provider, it will only use its local cek cache 
+            // when using a new per-command provider, it will only use its local cek cache
             // the following query should fail due to an empty cek cache and invalid credentials
             customProvider[SqlColumnEncryptionAzureKeyVaultProvider.ProviderName] =
                 new SqlColumnEncryptionAzureKeyVaultProvider(new ClientSecretCredential("tenant", "client", "secret"));

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/AKVUnitTests.cs

@@ -12,6 +12,7 @@
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
 {
+    [Trait("Set", "AE")]
     public class AKVUnitTests : IClassFixture<AzureKeyVaultKeyFixture>
     {
         const string EncryptionAlgorithm = "RSA_OAEP";

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -21,6 +21,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
     /// <summary>
     /// Always Encrypted public API Manual tests.
     /// </summary>
+    [Trait("Set", "AE")]
     public sealed class ApiShould : IClassFixture<SQLSetupStrategyCertStoreProvider>, IDisposable
     {
         private SQLSetupStrategy _fixture;
@@ -1930,20 +1931,20 @@ public void TestExecuteXmlReader(string connection)
             IList<object> values = GetValues(dataHint: 60);
             int numberOfRows = 10;
 
-            // Insert a bunch of rows in to the table.	
+            // Insert a bunch of rows in to the table.
             int rowsAffected = InsertRows(tableName: _tableName, numberofRows: numberOfRows, values: values, connection: connection);
             Assert.True(rowsAffected == numberOfRows, "number of rows affected is unexpected.");
 
             using (SqlConnection sqlConnection = new SqlConnection(connection))
             {
                 sqlConnection.Open();
 
-                // select the set of rows that were inserted just now.	
+                // select the set of rows that were inserted just now.
                 using (SqlCommand sqlCommand = new SqlCommand($"SELECT LastName FROM [{_tableName}] WHERE FirstName = @FirstName AND CustomerId = @CustomerId FOR XML AUTO;", sqlConnection, transaction: null, columnEncryptionSetting: SqlCommandColumnEncryptionSetting.Enabled))
                 {
                     if (DataTestUtility.EnclaveEnabled)
                     {
-                        //Increase Time out for enclave-enabled server.	
+                        //Increase Time out for enclave-enabled server.
                         sqlCommand.CommandTimeout = 90;
                     }
                     sqlCommand.Parameters.Add(@"CustomerId", SqlDbType.Int);
@@ -2047,7 +2048,7 @@ public void TestSqlCommandCancel(string connection, string value)
                      * Use long-running tasks to create the thread. This enables any failed assertions to propagate, rather than
                      * allowing the exception to kill the thread and the process.
                      * These threads should progress in the sequence below:
-                     * 
+                     *
                      * Workload Thread                      | Cancel Thread
                      * ------------------------------------ | -------------
                      * Start thread                         | Start thread
@@ -2323,7 +2324,7 @@ public void TestConnectionCustomKeyStoreProviderDuringAeQuery(string connectionS
                     () => ExecuteQueryThatRequiresCustomKeyStoreProvider(connection));
                 AssertExceptionCausedByFailureToDecrypt(ex);
 
-                // not required provider will replace the previous entry so required provider will not be found 
+                // not required provider will replace the previous entry so required provider will not be found
                 connection.RegisterColumnEncryptionKeyStoreProvidersOnConnection(_notRequiredProvider);
                 ex = Assert.Throws<ArgumentException>(
                     () => ExecuteQueryThatRequiresCustomKeyStoreProvider(connection));
@@ -2464,7 +2465,7 @@ public void TestRetryWhenAEParameterMetadataCacheIsStale(string connectionString
             cmd.CommandText = string.Format(alterCekQueryFormatString, _tableName, table.columnEncryptionKey2.Name);
             cmd.ExecuteNonQuery();
 
-            // execute the select query again. it will attempt to use the stale cache entry, receive 
+            // execute the select query again. it will attempt to use the stale cache entry, receive
             // a retryable error from the server, remove the stale cache entry, retry and succeed
             cmd.CommandText = enclaveSelectQuery;
             cmd.Parameters.AddWithValue("@CustomerId", 0);
@@ -2578,7 +2579,7 @@ public void TestRetryWhenAEEnclaveCacheIsStale(string connectionString)
             readAsyncTask2.GetAwaiter().GetResult();
 #endif
             */
-            
+
             // revert the CEK change to the CustomerId column
             cmd.Parameters.Clear();
             cmd.CommandText = string.Format(alterCekQueryFormatString, _tableName, table.columnEncryptionKey1.Name);
@@ -2715,8 +2716,8 @@ private DataTable CreateDataTable(string tableName, int numberofRows)
             };
             table.Columns.Add(column);
 
-            // Create three new DataRow objects and add  
-            // them to the DataTable 
+            // Create three new DataRow objects and add
+            // them to the DataTable
             for (int i = 0; i < numberofRows; i++)
             {
                 row = table.NewRow();
@@ -2826,7 +2827,7 @@ private async Task<object> VerifyExecuteScalarAsync(SqlCommand sqlCommand)
         }
 
         /// <summary>
-        /// 
+        ///
         /// </summary>
         /// <param name="sqlCommand"></param>
         /// <param name="cancellationToken"></param>
@@ -2837,7 +2838,7 @@ private async Task ExecuteNonQueryAsync(SqlCommand sqlCommand, CancellationToken
         }
 
         /// <summary>
-        /// 
+        ///
         /// </summary>
         /// <param name="sqlCommand"></param>
         /// <returns></returns>

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/BulkCopyAE.cs

@@ -12,6 +12,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
     /// <summary>
     /// Always Encrypted public API Manual tests.
     /// </summary>
+    [Trait("Set", "AE")]
     public sealed class BulkCopyAE : IClassFixture<SQLSetupStrategyCertStoreProvider>, IDisposable
     {
         private SQLSetupStrategy fixture;

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/BulkCopyAEErrorMessage.cs

@@ -11,6 +11,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
     /// <summary>
     /// Always Encrypted public API Manual tests.
     /// </summary>
+    [Trait("Set", "AE")]
     public class BulkCopyAEErrorMessage : IClassFixture<SQLSetupStrategyCertStoreProvider>
     {
         private SQLSetupStrategy _fixture;

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ColumnDecryptErrorTests.cs

@@ -10,6 +10,7 @@
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
 {
+    [Trait("Set", "AE")]
     public sealed class ColumnDecryptErrorTests : IClassFixture<SQLSetupStrategyAzureKeyVault>, IDisposable
     {
         private SQLSetupStrategyAzureKeyVault fixture;
@@ -28,13 +29,13 @@ public ColumnDecryptErrorTests(SQLSetupStrategyAzureKeyVault context)
          * properly reset before being returned to the pool. If this doesn't happen, then random bytes
          * may be left in the connection's state. These can interfere with the next operation that utilizes
          * the connection.
-         * 
+         *
          * We test that state is properly reset by triggering the same error condition twice. Routing column key discovery
-         * away from AKV toward a dummy key store achieves this. Each connection pulls from a pool of max 
-         * size one to ensure we are using the same internal connection/socket both times. We expect to 
+         * away from AKV toward a dummy key store achieves this. Each connection pulls from a pool of max
+         * size one to ensure we are using the same internal connection/socket both times. We expect to
          * receive the "Failed to decrypt column" exception twice. If the state were not cleaned properly,
          * the second error would be different because the TDS stream would be unintelligible.
-         * 
+         *
          * Finally, we assert that restoring the connection to AKV allows a successful query.
          */
         [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTargetReadyForAeWithKeyStore), nameof(DataTestUtility.IsAKVSetupAvailable))]
@@ -75,7 +76,7 @@ public void TestCleanConnectionAfterDecryptFail(string connString, string select
 
                 using SqlCommand sqlCommand = new SqlCommand(string.Format(selectQuery, tableName),
                                                             sqlConnection, null, SqlCommandColumnEncryptionSetting.Enabled);
-                
+
                 using SqlDataReader sqlDataReader = sqlCommand.ExecuteReader();
 
                 Assert.True(sqlDataReader.HasRows, "FAILED: Select statement did not return any rows.");

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ConversionTests.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.using System;
 
@@ -18,6 +18,7 @@
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted
 {
+    [Trait("Set", "AE")]
     public sealed class ConversionTests : IDisposable, IClassFixture<ColumnMasterKeyCertificateFixture>
     {
 
@@ -621,7 +622,7 @@ private void AdjustSizePrecisionAndScale(ref ColumnMetaData largeColumnMeta, ref
 
             if (TypeHasSize(largeColumnMeta.ColumnType))
             {
-                // 20% of the time use (max) as the length.                
+                // 20% of the time use (max) as the length.
                 largeColumnMeta.UseMax = (largeColumnMeta.ColumnType is SqlDbType.VarChar ||
                     largeColumnMeta.ColumnType is SqlDbType.NVarChar ||
                     largeColumnMeta.ColumnType is SqlDbType.VarBinary) &&
@@ -683,7 +684,7 @@ largeColumnMeta.ColumnType is SqlDbType.NVarChar ||
                 {
                     smallColumnMeta.Precision = 0;
 
-                    // For Time / DateTime2 / DateTimeOffset types, actual scale is set to 7 when parameter.scale is zero. 
+                    // For Time / DateTime2 / DateTimeOffset types, actual scale is set to 7 when parameter.scale is zero.
                     // Active Issue in SQLParameter.cs when user wants to specify zero as the actual scale.
                     smallColumnMeta.Scale = random.Next(minScale, largeColumnMeta.Scale);
                 }