diff --git a/src/Core/Models/RestRequestContexts/RestRequestContext.cs b/src/Core/Models/RestRequestContexts/RestRequestContext.cs index 70d6a371b5..e9987730a0 100644 --- a/src/Core/Models/RestRequestContexts/RestRequestContext.cs +++ b/src/Core/Models/RestRequestContexts/RestRequestContext.cs @@ -77,6 +77,12 @@ protected RestRequestContext(string entityName, DatabaseObject dbo) /// public NameValueCollection ParsedQueryString { get; set; } = new(); + /// + /// Raw query string from the HTTP request (URL-encoded). + /// Used to preserve encoding for special characters in query parameters. + /// + public string RawQueryString { get; set; } = string.Empty; + /// /// String holds information needed for pagination. /// Based on request this property may or may not be populated. diff --git a/src/Core/Parsers/RequestParser.cs b/src/Core/Parsers/RequestParser.cs index 6402ce4ecb..081018e820 100644 --- a/src/Core/Parsers/RequestParser.cs +++ b/src/Core/Parsers/RequestParser.cs @@ -113,14 +113,32 @@ public static void ParseQueryString(RestRequestContext context, ISqlMetadataProv context.FieldsToBeReturned = context.ParsedQueryString[key]!.Split(",").ToList(); break; case FILTER_URL: - // save the AST that represents the filter for the query - // ?$filter= - string filterQueryString = $"?{FILTER_URL}={context.ParsedQueryString[key]}"; - context.FilterClauseInUrl = sqlMetadataProvider.GetODataParser().GetFilterClause(filterQueryString, $"{context.EntityName}.{context.DatabaseObject.FullName}"); + // Use raw (URL-encoded) filter value to preserve special characters like & + string? rawFilterValue = ExtractRawQueryParameter(context.RawQueryString, FILTER_URL); + // If key exists in ParsedQueryString but not in RawQueryString, something is wrong + if (rawFilterValue is null) + { + throw new DataApiBuilderException( + message: $"Unable to extract {FILTER_URL} parameter from query string.", + statusCode: HttpStatusCode.BadRequest, + subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest); + } + + context.FilterClauseInUrl = sqlMetadataProvider.GetODataParser().GetFilterClause($"?{FILTER_URL}={rawFilterValue}", $"{context.EntityName}.{context.DatabaseObject.FullName}"); break; case SORT_URL: - string sortQueryString = $"?{SORT_URL}={context.ParsedQueryString[key]}"; - (context.OrderByClauseInUrl, context.OrderByClauseOfBackingColumns) = GenerateOrderByLists(context, sqlMetadataProvider, sortQueryString); + // Use raw (URL-encoded) orderby value to preserve special characters + string? rawSortValue = ExtractRawQueryParameter(context.RawQueryString, SORT_URL); + // If key exists in ParsedQueryString but not in RawQueryString, something is wrong + if (rawSortValue is null) + { + throw new DataApiBuilderException( + message: $"Unable to extract {SORT_URL} parameter from query string.", + statusCode: HttpStatusCode.BadRequest, + subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest); + } + + (context.OrderByClauseInUrl, context.OrderByClauseOfBackingColumns) = GenerateOrderByLists(context, sqlMetadataProvider, $"?{SORT_URL}={rawSortValue}"); break; case AFTER_URL: context.After = context.ParsedQueryString[key]; @@ -283,5 +301,38 @@ private static bool IsNull(string value) { return string.IsNullOrWhiteSpace(value) || string.Equals(value, "null", StringComparison.OrdinalIgnoreCase); } + + /// + /// Extracts the raw (URL-encoded) value of a query parameter from a query string. + /// Preserves special characters like & in filter values (e.g., %26 stays as %26). + /// + /// IMPORTANT: This method assumes the input queryString is a raw, URL-encoded query string + /// where special characters in parameter values are encoded (e.g., & is %26, space is %20). + /// It splits on unencoded '&' characters which are parameter separators in the URL standard. + /// If the queryString has already been decoded, this method will not work correctly. + /// + /// Raw URL-encoded query string (e.g., "?$filter=title%20eq%20%27A%26B%27") + /// The parameter name to extract (e.g., "$filter") + /// The raw encoded value of the parameter, or null if not found + internal static string? ExtractRawQueryParameter(string queryString, string parameterName) + { + if (string.IsNullOrWhiteSpace(queryString)) + { + return null; + } + + // Split on '&' which are parameter separators in properly URL-encoded query strings. + // Any '&' characters within parameter values will be encoded as %26. + foreach (string param in queryString.TrimStart('?').Split('&')) + { + int idx = param.IndexOf('='); + if (idx >= 0 && param.Substring(0, idx).Equals(parameterName, StringComparison.OrdinalIgnoreCase)) + { + return idx < param.Length - 1 ? param.Substring(idx + 1) : string.Empty; + } + } + + return null; + } } } diff --git a/src/Core/Services/RestService.cs b/src/Core/Services/RestService.cs index e63f23cc5f..72beb037c6 100644 --- a/src/Core/Services/RestService.cs +++ b/src/Core/Services/RestService.cs @@ -174,6 +174,7 @@ RequestValidator requestValidator if (!string.IsNullOrWhiteSpace(queryString)) { + context.RawQueryString = queryString; context.ParsedQueryString = HttpUtility.ParseQueryString(queryString); RequestParser.ParseQueryString(context, sqlMetadataProvider); } @@ -277,6 +278,7 @@ private void PopulateStoredProcedureContext( // So, $filter will be treated as any other parameter (inevitably will raise a Bad Request) if (!string.IsNullOrWhiteSpace(queryString)) { + context.RawQueryString = queryString; context.ParsedQueryString = HttpUtility.ParseQueryString(queryString); } diff --git a/src/Service.Tests/SqlTests/RestApiTests/Find/DwSqlFindApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Find/DwSqlFindApiTests.cs index 8c78a27061..dad77efb17 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Find/DwSqlFindApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Find/DwSqlFindApiTests.cs @@ -221,6 +221,18 @@ public class DwSqlFindApiTests : FindApiTestBase $"WHERE (NOT (id < 3) OR id < 4) OR NOT (title = 'Awesome book') " + $"FOR JSON PATH, INCLUDE_NULL_VALUES" }, + { + "FindTestWithFilterContainingSpecialCharacters", + $"SELECT * FROM { _integrationTableName } " + + $"WHERE title = 'SOME%CONN' " + + $"FOR JSON PATH, INCLUDE_NULL_VALUES" + }, + { + "FindTestWithOrderByContainingSpecialCharacters", + $"SELECT * FROM { _integrationTableName } " + + $"ORDER BY title desc " + + $"FOR JSON PATH, INCLUDE_NULL_VALUES" + }, { "FindTestWithPrimaryKeyContainingForeignKey", $"SELECT [id], [content] FROM reviews " + diff --git a/src/Service.Tests/SqlTests/RestApiTests/Find/FindApiTestBase.cs b/src/Service.Tests/SqlTests/RestApiTests/Find/FindApiTestBase.cs index 483d870d85..580b3389c8 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Find/FindApiTestBase.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Find/FindApiTestBase.cs @@ -693,6 +693,41 @@ await SetupAndRunRestApiTest( ); } + /// + /// Tests the REST Api for Find operation with a filter containing special characters + /// that need to be URL-encoded. Uses existing book with '%' character (SOME%CONN). + /// This validates that the fix for the double-decoding issue is working correctly. + /// + [TestMethod] + public async Task FindTestWithFilterContainingSpecialCharacters() + { + // Testing with SOME%CONN - the %25 is URL-encoded % + await SetupAndRunRestApiTest( + primaryKeyRoute: string.Empty, + queryString: "?$filter=title%20eq%20%27SOME%25CONN%27", + entityNameOrPath: _integrationEntityName, + sqlQuery: GetQuery(nameof(FindTestWithFilterContainingSpecialCharacters)) + ); + } + + /// + /// Tests the REST Api for Find operation with an $orderby clause containing URL-encoded spaces. + /// This validates that $orderby parameter extraction preserves URL encoding through the same + /// code path as $filter. + /// + [TestMethod] + public async Task FindTestWithOrderByContainingSpecialCharacters() + { + // Order by title desc - tests that $orderby parameter is extracted with URL encoding preserved + // The %20 represents space in "$orderby=title%20desc" + await SetupAndRunRestApiTest( + primaryKeyRoute: string.Empty, + queryString: "?$orderby=title%20desc", + entityNameOrPath: _integrationEntityName, + sqlQuery: GetQuery(nameof(FindTestWithOrderByContainingSpecialCharacters)) + ); + } + /// /// Tests the REST Api for Find operation where we compare one field /// to the bool returned from another comparison. diff --git a/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs index 6f43fb2073..4dbdbf40ae 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs @@ -228,6 +228,18 @@ public class MsSqlFindApiTests : FindApiTestBase $"WHERE (NOT (id < 3) OR id < 4) OR NOT (title = 'Awesome book') " + $"FOR JSON PATH, INCLUDE_NULL_VALUES" }, + { + "FindTestWithFilterContainingSpecialCharacters", + $"SELECT * FROM { _integrationTableName } " + + $"WHERE title = 'SOME%CONN' " + + $"FOR JSON PATH, INCLUDE_NULL_VALUES" + }, + { + "FindTestWithOrderByContainingSpecialCharacters", + $"SELECT * FROM { _integrationTableName } " + + $"ORDER BY title desc " + + $"FOR JSON PATH, INCLUDE_NULL_VALUES" + }, { "FindTestWithPrimaryKeyContainingForeignKey", $"SELECT [id], [content] FROM reviews " + diff --git a/src/Service.Tests/SqlTests/RestApiTests/Find/MySqlFindApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Find/MySqlFindApiTests.cs index f9a3fdb764..42d488557b 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Find/MySqlFindApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Find/MySqlFindApiTests.cs @@ -397,6 +397,29 @@ ORDER BY id asc ) AS subq " }, + { + "FindTestWithFilterContainingSpecialCharacters", + @" + SELECT JSON_ARRAYAGG(JSON_OBJECT('id', id, 'title', title, 'publisher_id', publisher_id)) AS data + FROM ( + SELECT * + FROM " + _integrationTableName + @" + WHERE title = 'SOME%CONN' + ORDER BY id asc + ) AS subq + " + }, + { + "FindTestWithOrderByContainingSpecialCharacters", + @" + SELECT JSON_ARRAYAGG(JSON_OBJECT('id', id, 'title', title, 'publisher_id', publisher_id)) AS data + FROM ( + SELECT * + FROM " + _integrationTableName + @" + ORDER BY title desc + ) AS subq + " + }, { "FindTestWithFilterQueryStringBoolResultFilter", @" diff --git a/src/Service.Tests/SqlTests/RestApiTests/Find/PostgreSqlFindApiTests.cs b/src/Service.Tests/SqlTests/RestApiTests/Find/PostgreSqlFindApiTests.cs index 9abcfe88c2..d30cd4816a 100644 --- a/src/Service.Tests/SqlTests/RestApiTests/Find/PostgreSqlFindApiTests.cs +++ b/src/Service.Tests/SqlTests/RestApiTests/Find/PostgreSqlFindApiTests.cs @@ -411,6 +411,27 @@ SELECT json_agg(to_jsonb(subq)) AS data ORDER BY id asc ) AS subq" }, + { + "FindTestWithFilterContainingSpecialCharacters", + @" + SELECT json_agg(to_jsonb(subq)) AS data + FROM ( + SELECT * + FROM " + _integrationTableName + @" + WHERE title = 'SOME%CONN' + ORDER BY id asc + ) AS subq" + }, + { + "FindTestWithOrderByContainingSpecialCharacters", + @" + SELECT json_agg(to_jsonb(subq)) AS data + FROM ( + SELECT * + FROM " + _integrationTableName + @" + ORDER BY title desc + ) AS subq" + }, { "FindTestWithPrimaryKeyContainingForeignKey", @" diff --git a/src/Service.Tests/UnitTests/RequestParserUnitTests.cs b/src/Service.Tests/UnitTests/RequestParserUnitTests.cs new file mode 100644 index 0000000000..4da3266271 --- /dev/null +++ b/src/Service.Tests/UnitTests/RequestParserUnitTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.DataApiBuilder.Core.Parsers; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Azure.DataApiBuilder.Service.Tests.UnitTests +{ + /// + /// Test class for RequestParser utility methods. + /// Specifically tests the ExtractRawQueryParameter method which preserves + /// URL encoding for special characters in query parameters. + /// + [TestClass] + public class RequestParserUnitTests + { + /// + /// Tests that ExtractRawQueryParameter correctly extracts URL-encoded + /// parameter values, preserving special characters like ampersand (&). + /// + [DataTestMethod] + [DataRow("?$filter=region%20eq%20%27filter%20%26%20test%27", "$filter", "region%20eq%20%27filter%20%26%20test%27", DisplayName = "Extract filter with encoded ampersand (&)")] + [DataRow("?$filter=title%20eq%20%27A%20%26%20B%27&$select=id", "$filter", "title%20eq%20%27A%20%26%20B%27", DisplayName = "Extract filter with ampersand and other params")] + [DataRow("?$select=id&$filter=name%20eq%20%27test%27", "$filter", "name%20eq%20%27test%27", DisplayName = "Extract filter when not first parameter")] + [DataRow("?$orderby=name%20asc", "$orderby", "name%20asc", DisplayName = "Extract orderby parameter")] + [DataRow("?param1=value1¶m2=value%26with%26ampersands", "param2", "value%26with%26ampersands", DisplayName = "Extract parameter with multiple ampersands")] + [DataRow("$filter=title%20eq%20%27test%27", "$filter", "title%20eq%20%27test%27", DisplayName = "Extract without leading question mark")] + [DataRow("?$filter=", "$filter", "", DisplayName = "Extract empty filter value")] + [DataRow("?$filter=name%20eq%20%27test%3D123%27", "$filter", "name%20eq%20%27test%3D123%27", DisplayName = "Extract filter with encoded equals sign (=)")] + [DataRow("?$filter=url%20eq%20%27http%3A%2F%2Fexample.com%3Fkey%3Dvalue%27", "$filter", "url%20eq%20%27http%3A%2F%2Fexample.com%3Fkey%3Dvalue%27", DisplayName = "Extract filter with encoded URL (: / ?)")] + [DataRow("?$filter=text%20eq%20%27A%2BB%27", "$filter", "text%20eq%20%27A%2BB%27", DisplayName = "Extract filter with encoded plus sign (+)")] + [DataRow("?$filter=value%20eq%20%2750%25%27", "$filter", "value%20eq%20%2750%25%27", DisplayName = "Extract filter with encoded percent sign (%)")] + [DataRow("?$filter=tag%20eq%20%27%23hashtag%27", "$filter", "tag%20eq%20%27%23hashtag%27", DisplayName = "Extract filter with encoded hash (#)")] + [DataRow("?$filter=expr%20eq%20%27a%3Cb%3Ed%27", "$filter", "expr%20eq%20%27a%3Cb%3Ed%27", DisplayName = "Extract filter with encoded less-than and greater-than (< >)")] + public void ExtractRawQueryParameter_PreservesEncoding(string queryString, string parameterName, string expectedValue) + { + // Call the internal method directly (no reflection needed) + string? result = RequestParser.ExtractRawQueryParameter(queryString, parameterName); + + Assert.AreEqual(expectedValue, result, + $"Expected '{expectedValue}' but got '{result}' for parameter '{parameterName}' in query '{queryString}'"); + } + + /// + /// Tests that ExtractRawQueryParameter returns null when parameter is not found. + /// + [DataTestMethod] + [DataRow("?$filter=test", "$orderby", DisplayName = "Parameter not in query string")] + [DataRow("", "$filter", DisplayName = "Empty query string")] + [DataRow(null, "$filter", DisplayName = "Null query string")] + [DataRow("?otherParam=value", "$filter", DisplayName = "Different parameter")] + public void ExtractRawQueryParameter_ReturnsNull_WhenParameterNotFound(string? queryString, string parameterName) + { + // Call the internal method directly (no reflection needed) + string? result = RequestParser.ExtractRawQueryParameter(queryString, parameterName); + + Assert.IsNull(result, + $"Expected null but got '{result}' for parameter '{parameterName}' in query '{queryString}'"); + } + + /// + /// Tests that ExtractRawQueryParameter handles edge cases correctly: + /// - Duplicate parameters (returns first occurrence) + /// - Case-insensitive parameter name matching + /// - Malformed query strings with unencoded ampersands + /// + [DataTestMethod] + [DataRow("?$filter=value&$filter=anothervalue", "$filter", "value", DisplayName = "Multiple same parameters - returns first")] + [DataRow("?$FILTER=value", "$filter", "value", DisplayName = "Case insensitive parameter matching")] + [DataRow("?param=value1&value2", "param", "value1", DisplayName = "Value with unencoded ampersand after parameter")] + public void ExtractRawQueryParameter_HandlesEdgeCases(string queryString, string parameterName, string expectedValue) + { + // Call the internal method directly (no reflection needed) + string? result = RequestParser.ExtractRawQueryParameter(queryString, parameterName); + + Assert.AreEqual(expectedValue, result, + $"Expected '{expectedValue}' but got '{result}' for parameter '{parameterName}' in query '{queryString}'"); + } + } +}