From f47cdb49123be4740728c4d0126d5c51b4172d7b Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Mon, 12 Oct 2026 21:23:32 -0500 Subject: [PATCH 01/26] Configure webhook delivery options --- docs/webhook-delivery.md | 11 + src/Exceptionless.Core/Bootstrapper.cs | 11 + .../Configuration/AppOptions.cs | 2 + .../Configuration/WebHookOptions.cs | 15 ++ .../Extensions/LoggerExtensions.cs | 23 ++- src/Exceptionless.Core/Jobs/WebHooksJob.cs | 43 ++-- .../Services/WebHookDestinationPolicy.cs | 62 ++++++ .../Api/Handlers/WebHookHandler.cs | 35 +++- src/Exceptionless.Web/ApmExtensions.cs | 4 +- .../Extensions/LoggerExtensions.cs | 6 +- .../Api/Endpoints/WebHookEndpointTests.cs | 89 +++++++- .../Jobs/WebHookDeliveryTests.cs | 195 ++++++++++++++++++ .../Services/WebHookDestinationPolicyTests.cs | 68 ++++++ tests/http/webhooks.http | 2 +- 14 files changed, 523 insertions(+), 43 deletions(-) create mode 100644 docs/webhook-delivery.md create mode 100644 src/Exceptionless.Core/Configuration/WebHookOptions.cs create mode 100644 src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs create mode 100644 tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs create mode 100644 tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs diff --git a/docs/webhook-delivery.md b/docs/webhook-delivery.md new file mode 100644 index 0000000000..d822b241a5 --- /dev/null +++ b/docs/webhook-delivery.md @@ -0,0 +1,11 @@ +Webhooks use HTTP or HTTPS destinations. Connections resolve and check the destination address at delivery time, including URLs saved before deployment. Redirects are not followed, and requests do not use a proxy or cookies. + +For a self-hosted private receiver, explicitly configure its smallest required network on every API and job host: + +```yaml +WebHooks: + AllowedPrivateNetworks: + - 10.20.30.40/32 +``` + +The default list is empty. Existing private receivers require this override before deploying. Their saved records are preserved. Delivery logs identify the destination origin; target paths, queries, credentials, and HTTP delivery traces are omitted. diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 5a4f2da256..6994500568 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -224,6 +224,17 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO s.GetRequiredService(), DecompressionMethods.None)); services.AddSingleton(); + services.AddSingleton(); + services.AddHttpClient(WebHookDestinationPolicy.HttpClientName) + .RemoveAllLoggers() + .ConfigurePrimaryHttpMessageHandler(provider => new SocketsHttpHandler + { + AllowAutoRedirect = false, + UseProxy = false, + UseCookies = false, + ActivityHeadersPropagator = null, + ConnectCallback = provider.GetRequiredService().ConnectAsync + }); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(provider => provider.GetRequiredService()); diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs index d7ebd91d72..16546635df 100644 --- a/src/Exceptionless.Core/Configuration/AppOptions.cs +++ b/src/Exceptionless.Core/Configuration/AppOptions.cs @@ -83,6 +83,7 @@ public class AppOptions public AuthOptions AuthOptions { get; internal set; } = null!; public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!; public SourceMapOptions SourceMapOptions { get; internal set; } = null!; + public WebHookOptions WebHookOptions { get; internal set; } = null!; public AssistantOptions AssistantOptions { get; internal set; } = null!; public static AppOptions ReadFromConfiguration(IConfiguration config) @@ -139,6 +140,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config) options.AuthOptions = AuthOptions.ReadFromConfiguration(config); options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config); options.SourceMapOptions = SourceMapOptions.ReadFromConfiguration(config); + options.WebHookOptions = WebHookOptions.ReadFromConfiguration(config); options.AssistantOptions = AssistantOptions.ReadFromConfiguration(config); return options; diff --git a/src/Exceptionless.Core/Configuration/WebHookOptions.cs b/src/Exceptionless.Core/Configuration/WebHookOptions.cs new file mode 100644 index 0000000000..1105ee3bc8 --- /dev/null +++ b/src/Exceptionless.Core/Configuration/WebHookOptions.cs @@ -0,0 +1,15 @@ +using System.Net; +using Microsoft.Extensions.Configuration; + +namespace Exceptionless.Core.Configuration; + +public sealed class WebHookOptions +{ + public IPNetwork[] AllowedPrivateNetworks { get; internal set; } = []; + + public static WebHookOptions ReadFromConfiguration(IConfiguration configuration) + => new() + { + AllowedPrivateNetworks = configuration.GetSection("WebHooks:AllowedPrivateNetworks").Get()?.Select(IPNetwork.Parse).ToArray() ?? [] + }; +} diff --git a/src/Exceptionless.Core/Extensions/LoggerExtensions.cs b/src/Exceptionless.Core/Extensions/LoggerExtensions.cs index 53237e4473..069490821c 100644 --- a/src/Exceptionless.Core/Extensions/LoggerExtensions.cs +++ b/src/Exceptionless.Core/Extensions/LoggerExtensions.cs @@ -1,5 +1,6 @@ using System.Net; using Microsoft.Extensions.Logging; +using Exceptionless.Core.Services; namespace Exceptionless.Core.Extensions; @@ -24,17 +25,17 @@ internal static class LoggerExtensions new EventId(2, nameof(WebHookCancelledBackoff)), "Web hook cancelled due to {FailureCount} consecutive failed attempts. Will be allowed to try again at {NextAttempt}."); - private static readonly Action _webHookTimeout = - LoggerMessage.Define( + private static readonly Action _webHookTimeout = + LoggerMessage.Define( LogLevel.Error, new EventId(3, nameof(WebHookTimeout)), - "Timeout calling web hook: status={Status} org={organization} project={ProjectId} url={Url}"); + "Timeout calling web hook: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination} failure={ExceptionType}"); - private static readonly Action _webHookError = - LoggerMessage.Define( + private static readonly Action _webHookError = + LoggerMessage.Define( LogLevel.Error, new EventId(4, nameof(WebHookError)), - "Error calling web hook: status={Status} org={organization} project={ProjectId} url={Url}"); + "Error calling web hook: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination} failure={ExceptionType}"); private static readonly Action _webHookComplete = LoggerMessage.Define( @@ -178,16 +179,16 @@ public static void WebHookDisabledTooManyErrors(this ILogger logger, string webH => _webHookDisabledTooManyErrors(logger, webHookId, null); public static void WebHookDisabledStatusCode(this ILogger logger, string webHookId, HttpStatusCode? statusCode, string organizationId, string projectId, string url) - => _webHookDisabledStatusCode(logger, webHookId, statusCode, organizationId, projectId, url, null); + => _webHookDisabledStatusCode(logger, webHookId, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), null); public static void WebHookComplete(this ILogger logger, HttpStatusCode? statusCode, string organizationId, string projectId, string url) - => _webHookComplete(logger, statusCode, organizationId, projectId, url, null); + => _webHookComplete(logger, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), null); public static void WebHookError(this ILogger logger, HttpStatusCode? statusCode, string organizationId, string projectId, string url, Exception exception) - => _webHookError(logger, statusCode, organizationId, projectId, url, exception); + => _webHookError(logger, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), exception.GetType().Name, null); public static void WebHookTimeout(this ILogger logger, HttpStatusCode? statusCode, string organizationId, string projectId, string url, Exception exception) - => _webHookTimeout(logger, statusCode, organizationId, projectId, url, exception); + => _webHookTimeout(logger, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), exception.GetType().Name, null); public static void WebHookCancelledBackoff(this ILogger logger, long consecutiveErrors, DateTime nextAttemptAllowedAt) => _webHookCancelledBackoff(logger, consecutiveErrors, nextAttemptAllowedAt, null); @@ -196,5 +197,5 @@ public static void WebHookCancelled(this ILogger logger) => _webHookCancelled(logger, null); public static void RecordWebHook(this ILogger logger, string id, string projectId, string url) - => _recordWebHook(logger, id, projectId, url, null); + => _recordWebHook(logger, id, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), null); } diff --git a/src/Exceptionless.Core/Jobs/WebHooksJob.cs b/src/Exceptionless.Core/Jobs/WebHooksJob.cs index d8cd0a3536..4ed15cbab7 100644 --- a/src/Exceptionless.Core/Jobs/WebHooksJob.cs +++ b/src/Exceptionless.Core/Jobs/WebHooksJob.cs @@ -40,15 +40,11 @@ public class WebHooksJob : QueueJobBase, IDisposable private readonly JsonSerializerOptions _versionOneJsonOptions; private readonly AppOptions _appOptions; - private HttpClient? _client; - - private HttpClient Client - { - get => _client ??= new HttpClient(); - } + private readonly HttpClient _client; + private readonly WebHookDestinationPolicy _destinationPolicy; public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, ITextSerializer serializer, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, - IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) + IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, WebHookDestinationPolicy destinationPolicy) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { _projectRepository = projectRepository; _slackService = slackService; @@ -58,16 +54,25 @@ public WebHooksJob(IQueue queue, IProjectRepository project _jsonOptions = jsonOptions; _versionOneJsonOptions = VersionOnePlugin.CreateJsonSerializerOptions(jsonOptions); _appOptions = appOptions; + _client = httpClientFactory.CreateClient(WebHookDestinationPolicy.HttpClientName); + _destinationPolicy = destinationPolicy; } protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) { var body = context.QueueEntry.Value; + string destination = WebHookDestinationPolicy.GetLoggingDestination(body.Url); bool shouldLog = body.ProjectId != _appOptions.InternalProjectId; using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) { - if (shouldLog) _logger.RecordWebHook(context.QueueEntry.Id, body.ProjectId, body.Url); + if (shouldLog) _logger.RecordWebHook(context.QueueEntry.Id, body.ProjectId, destination); + + if (!_destinationPolicy.IsValidDestination(body.Url)) + { + _logger.LogWarning("Web hook destination is not allowed for {WebHookId}", body.WebHookId); + return JobResult.Cancelled; + } if (!await IsEnabledAsync(body)) { @@ -101,7 +106,12 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex _ => _jsonOptions }; - response = await Client.PostAsJsonAsync(body.Url, body.Data, jsonOptions, postCancellationTokenSource.Token); + using var request = new HttpRequestMessage(HttpMethod.Post, body.Url) + { + Content = JsonContent.Create(body.Data, options: jsonOptions) + }; + request.Options.Set(WebHookDestinationPolicy.DeliveryRequest, true); + response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, postCancellationTokenSource.Token); if (!response.IsSuccessStatusCode) successful = false; else if (consecutiveErrors > 0) @@ -110,24 +120,24 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex catch (OperationCanceledException ex) { successful = false; - if (shouldLog) _logger.WebHookTimeout(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url, ex); + if (shouldLog) _logger.WebHookTimeout(response?.StatusCode, body.OrganizationId, body.ProjectId, destination, ex); return JobResult.Cancelled; } catch (Exception ex) { successful = false; - if (shouldLog) _logger.WebHookError(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url, ex); - return JobResult.FromException(ex); + if (shouldLog) _logger.WebHookError(response?.StatusCode, body.OrganizationId, body.ProjectId, destination, ex); + return JobResult.FromException(new HttpRequestException($"Web hook delivery failed ({ex.GetType().Name}).")); } finally { if (successful) { - _logger.WebHookComplete(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); + _logger.WebHookComplete(response?.StatusCode, body.OrganizationId, body.ProjectId, destination); } else if (response is not null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone)) { - _logger.WebHookDisabledStatusCode(body.Type == WebHookType.Slack ? "Slack" : body.WebHookId!, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url); + _logger.WebHookDisabledStatusCode(body.Type == WebHookType.Slack ? "Slack" : body.WebHookId!, response.StatusCode, body.OrganizationId, body.ProjectId, destination); await DisableIntegrationAsync(body); await cache.RemoveAllAsync(_cacheKeys); } @@ -161,6 +171,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex } } } + response?.Dispose(); } } @@ -174,7 +185,7 @@ private async Task IsEnabledAsync(WebHookNotification body) case WebHookType.General: if (body.WebHookId is null) { - _logger.LogWarning("WebHook notification is missing the web hook id. Organization: {OrganizationId}, Project: {ProjectId}, Url: {Url}", body.OrganizationId, body.ProjectId, body.Url); + _logger.LogWarning("WebHook notification is missing the web hook id. Organization: {OrganizationId}, Project: {ProjectId}", body.OrganizationId, body.ProjectId); return false; } @@ -221,6 +232,6 @@ private static string GetCacheKeyScope(WebHookNotification body) public void Dispose() { - _client?.Dispose(); + _client.Dispose(); } } diff --git a/src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs b/src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs new file mode 100644 index 0000000000..10dcba5136 --- /dev/null +++ b/src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs @@ -0,0 +1,62 @@ +using System.Net; +using System.Net.Sockets; +using Exceptionless.Core.Utility; + +namespace Exceptionless.Core.Services; + +public sealed class WebHookDestinationPolicy(AppOptions options) +{ + public const string HttpClientName = "WebHooks"; + public static readonly HttpRequestOptionsKey DeliveryRequest = new("Exceptionless.WebHookDelivery"); + + public bool IsAllowed(IPAddress address) + { + if (address.IsIPv4MappedToIPv6) + address = address.MapToIPv4(); + return PublicAddressPolicy.IsPublic(address) || options.WebHookOptions.AllowedPrivateNetworks.Any(network => network.Contains(address)); + } + + public static bool TryCreateUri(string? value, out Uri uri) + { + uri = null!; + return Uri.TryCreate(value, UriKind.Absolute, out uri!) + && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) + && !String.IsNullOrEmpty(uri.Host) && String.IsNullOrEmpty(uri.UserInfo) && String.IsNullOrEmpty(uri.Fragment); + } + + public bool IsValidDestination(string? value) + => TryCreateUri(value, out var uri) && (IPAddress.TryParse(uri.Host.Trim('[', ']'), out var address) + ? IsAllowed(address) + : !uri.IsLoopback || IsAllowed(IPAddress.Loopback) || IsAllowed(IPAddress.IPv6Loopback)); + + public async ValueTask ConnectAsync(SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + var addresses = await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken); + Exception? lastException = null; + foreach (var address in addresses.Where(IsAllowed)) + { + var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true }; + try + { + await socket.ConnectAsync(new IPEndPoint(address, context.DnsEndPoint.Port), cancellationToken); + return new NetworkStream(socket, ownsSocket: true); + } + catch (SocketException ex) + { + lastException = ex; + socket.Dispose(); + } + catch + { + socket.Dispose(); + throw; + } + } + throw new HttpRequestException("Web hook destination has no reachable allowed address.", lastException); + } + + public static string GetLoggingDestination(string? value) + => Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) + ? uri.GetComponents(UriComponents.SchemeAndServer, UriFormat.UriEscaped) + : "invalid"; +} diff --git a/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs b/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs index 6ceaa0ff84..ce80193183 100644 --- a/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs @@ -1,8 +1,11 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; using Exceptionless.Core.Authorization; using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; using Exceptionless.Web.Api.Results; @@ -19,6 +22,7 @@ public class WebHookHandler( IWebHookRepository repository, IProjectRepository projectRepository, BillingManager billingManager, + WebHookDestinationPolicy destinationPolicy, ApiMapper mapper, LinkGenerator linkGenerator, IHttpContextAccessor httpContextAccessor, @@ -86,8 +90,8 @@ public async Task> Handle(DeleteWebHooks message) public async Task> Handle(SubscribeWebHook message) { - string? eventType = message.Data.RootElement.TryGetProperty("event", out var eventProp) ? eventProp.GetString() : null; - string? url = message.Data.RootElement.TryGetProperty("target_url", out var urlProp) ? urlProp.GetString() : null; + string? eventType = GetStringProperty(message.Data.RootElement, "event"); + string? url = GetStringProperty(message.Data.RootElement, "target_url"); if (String.IsNullOrEmpty(eventType) || String.IsNullOrEmpty(url)) return Result.BadRequest("Webhook subscription event and target_url are required."); @@ -108,7 +112,7 @@ public async Task> Handle(DeleteWebHooks message) Version = new Version(message.ApiVersion >= 0 ? message.ApiVersion : 0, 0) }; - if (!webHook.Url.StartsWith("https://hooks.zapier.com")) + if (!IsZapierTarget(webHook.Url)) return Result.NotFound("Webhook target not found."); return await PostImplAsync(webHook); @@ -116,8 +120,8 @@ public async Task> Handle(DeleteWebHooks message) public async Task Handle(UnsubscribeWebHook message) { - string? targetUrl = message.Data.RootElement.TryGetProperty("target_url", out var urlProp) ? urlProp.GetString() : null; - if (targetUrl is null || !targetUrl.StartsWith("https://hooks.zapier.com")) + string? targetUrl = GetStringProperty(message.Data.RootElement, "target_url"); + if (!IsZapierTarget(targetUrl)) return Result.NotFound("Webhook target not found."); var results = await repository.GetByUrlAsync(targetUrl); @@ -127,13 +131,29 @@ public async Task Handle(UnsubscribeWebHook message) if (results.Documents.Any(h => h.OrganizationId != organizationId)) throw new ArgumentException("All OrganizationIds must be the same."); - _logger.RemovingZapierUrls(results.Documents.Count, targetUrl); + _logger.RemovingZapierUrls(results.Documents.Count, organizationId); await repository.RemoveAsync(results.Documents); } return Result.Success(); } + private static string? GetStringProperty(JsonElement data, string name) + => data.ValueKind == JsonValueKind.Object && data.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + + private static bool IsZapierTarget([NotNullWhen(true)] string? url) + { + return url is not null + && url.StartsWith("https://hooks.zapier.com", StringComparison.Ordinal) + && Uri.TryCreate(url, UriKind.Absolute, out var uri) + && uri.Scheme == Uri.UriSchemeHttps + && uri.Host.Equals("hooks.zapier.com", StringComparison.Ordinal) + && uri.IsDefaultPort + && String.IsNullOrEmpty(uri.UserInfo); + } + public Result Handle(TestWebHook message) { return new object[] { @@ -188,6 +208,9 @@ public Result Handle(TestWebHook message) if (!await billingManager.HasPremiumFeaturesAsync(project is not null ? project.OrganizationId : value.OrganizationId)) return Result.Invalid(ValidationError.Create(ApiValidationErrorIdentifiers.PlanLimit, "Please upgrade your plan to add integrations.")); + if (!destinationPolicy.IsValidDestination(value.Url)) + return Result.BadRequest("Please specify an allowed HTTP or HTTPS destination."); + return null; } diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index b83915989c..d55850550b 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Text.RegularExpressions; +using Exceptionless.Core.Services; using OpenTelemetry.Logs; using OpenTelemetry.Metrics; using OpenTelemetry.Resources; @@ -71,7 +72,8 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }; }); - b.AddHttpClientInstrumentation(); + b.AddHttpClientInstrumentation(o => o.FilterHttpRequestMessage = request => + !request.Options.TryGetValue(WebHookDestinationPolicy.DeliveryRequest, out bool delivery) || !delivery); b.AddSource("Exceptionless", "Exceptionless.Assistant", "Foundatio"); if (config.EnableRedis) diff --git a/src/Exceptionless.Web/Extensions/LoggerExtensions.cs b/src/Exceptionless.Web/Extensions/LoggerExtensions.cs index d4b8f36550..50a87d9a79 100644 --- a/src/Exceptionless.Web/Extensions/LoggerExtensions.cs +++ b/src/Exceptionless.Web/Extensions/LoggerExtensions.cs @@ -12,7 +12,7 @@ internal static class LoggerExtensions LoggerMessage.Define( LogLevel.Information, new EventId(1, nameof(RemovingZapierUrls)), - "Removing {Count} zapier urls matching: {Url}"); + "Removing {Count} Zapier webhooks for organization {OrganizationId}"); private static readonly Action _removedTokens = LoggerMessage.Define( @@ -113,8 +113,8 @@ internal static class LoggerExtensions public static void ProjectRouteDoesNotMatch(this ILogger logger, string? requestProjectId, string routeProjectId) => _projectRouteDoesNotMatch(logger, requestProjectId, routeProjectId, null); - public static void RemovingZapierUrls(this ILogger logger, int count, string targetUrl) - => _removingZapierUrls(logger, count, targetUrl, null); + public static void RemovingZapierUrls(this ILogger logger, int count, string organizationId) + => _removingZapierUrls(logger, count, organizationId, null); public static void RemovedTokens(this ILogger logger, long removedCount, string userId) => _removedTokens(logger, removedCount, userId, null); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs index 0ebddc9914..9fe7b5d332 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs @@ -166,7 +166,7 @@ public Task CanCreateNewWebHook() EventTypes = [WebHook.KnownEventTypes.StackPromoted], OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, - Url = "https://localhost/test" + Url = "https://example.com/test" }) .StatusCodeShouldBeCreated() ); @@ -181,7 +181,7 @@ public async Task PostAsync_WithUnauthorizedOrganization_ReturnsBadRequestProble EventTypes = [WebHook.KnownEventTypes.StackPromoted], OrganizationId = SampleDataService.FREE_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, - Url = "https://localhost/test" + Url = "https://example.com/test" }; // Act @@ -208,7 +208,7 @@ public async Task PostAsync_WithUnauthorizedProject_ReturnsBadRequestProblemDeta EventTypes = [WebHook.KnownEventTypes.StackPromoted], OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.FREE_PROJECT_ID, - Url = "https://localhost/test" + Url = "https://example.com/test" }; // Act @@ -248,7 +248,7 @@ public async Task PostAsync_WithoutOrganizationOrProject_ReturnsBadRequestProble var newWebHook = new NewWebHook { EventTypes = [WebHook.KnownEventTypes.StackPromoted], - Url = "https://localhost/test" + Url = "https://example.com/test" }; // Act @@ -278,7 +278,7 @@ public async Task CreateNewWebHookWithInvalidEventTypeFails() EventTypes = ["Invalid"], OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, - Url = "https://localhost/test" + Url = "https://example.com/test" }) .StatusCodeShouldBeUnprocessableEntity() ); @@ -564,10 +564,21 @@ public async Task Test_WithGetRequest_ReturnsZapierTestMessages() message.Id == 2 && String.Equals(message.Message, expectedMessages[1], StringComparison.Ordinal)); } + [Theory] + [InlineData("http://127.0.0.1/path")] + [InlineData("http://localhost/path")] + [InlineData("http://169.254.169.254/path")] + [InlineData("https://user:password@example.com/path")] + public Task PostAsync_DisallowedDestination_ReturnsBadRequest(string url) + => SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("webhooks") + .Content(new NewWebHook { ProjectId = SampleDataService.TEST_PROJECT_ID, Url = url, EventTypes = [WebHook.KnownEventTypes.NewError] }) + .StatusCodeShouldBeBadRequest()); + [Fact] public async Task UnsubscribeAsync_ExistingZapierHook_RemovesWebHook() { // Arrange - create a zapier hook via subscribe + var logger = Assert.IsType(GetService()); const string zapierUrl = "https://hooks.zapier.com/hooks/unsubtest"; var webHook = await SendRequestAsAsync(r => r .Post() @@ -597,6 +608,74 @@ await SendRequestAsync(r => r await RefreshDataAsync(); var results = await _webHookRepository.GetByUrlAsync(zapierUrl); Assert.Empty(results.Documents); + var unsubscribeLogs = logger.LogEntries.Where(entry => entry.EventId.Name == "RemovingZapierUrls").ToList(); + Assert.NotEmpty(unsubscribeLogs); + Assert.All(unsubscribeLogs, entry => + { + Assert.DoesNotContain(zapierUrl, entry.Message); + Assert.DoesNotContain(entry.Properties.Values, value => String.Equals(value?.ToString(), zapierUrl, StringComparison.Ordinal)); + }); + } + + [Theory] + [InlineData("api/v2/webhooks/subscribe", "[]")] + [InlineData("api/v2/webhooks/subscribe", "{\"event\":42,\"target_url\":\"https://hooks.zapier.com/hooks/test\"}")] + [InlineData("api/v2/webhooks/subscribe", "{\"event\":\"NewError\",\"target_url\":false}")] + [InlineData("api/v1/projecthook/subscribe", "[]")] + public async Task SubscribeAsync_MalformedShape_ReturnsBadRequest(string path, string json) + { + // Act + var response = await SendRequestAsync(request => request.Post().AsTestOrganizationClientUser() + .BaseUri(_server.BaseAddress).AppendPath(path).Content(json, "application/json").StatusCodeShouldBeBadRequest()); + + // Assert + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Theory] + [InlineData("api/v2/webhooks/unsubscribe", "[]")] + [InlineData("api/v2/webhooks/unsubscribe", "{\"target_url\":42}")] + [InlineData("api/v1/projecthook/unsubscribe", "[]")] + public async Task UnsubscribeAsync_MalformedShape_ReturnsNotFound(string path, string json) + { + // Act + var response = await SendRequestAsync(request => request.Post().BaseUri(_server.BaseAddress) + .AppendPath(path).Content(json, "application/json").StatusCodeShouldBeNotFound()); + + // Assert + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Theory] + [InlineData("https://hooks.zapier.com.evil.example/hooks/test")] + [InlineData("https://hooks.zapier.com@evil.example/hooks/test")] + [InlineData("https://hooks.zapier.com:444/hooks/test")] + public async Task SubscribeAsync_LookalikeZapierTarget_ReturnsNotFound(string targetUrl) + { + // Arrange + var subscription = new { @event = WebHook.KnownEventTypes.NewError, target_url = targetUrl }; + + // Act + var response = await SendRequestAsync(request => request.Post().AsTestOrganizationClientUser() + .AppendPath("webhooks/subscribe").Content(subscription).StatusCodeShouldBeNotFound()); + + // Assert + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + Assert.Empty((await _webHookRepository.GetByUrlAsync(targetUrl)).Documents); + } + + [Theory] + [InlineData("https://hooks.zapier.com.evil.example/hooks/test")] + [InlineData("https://hooks.zapier.com@evil.example/hooks/test")] + [InlineData("https://hooks.zapier.com:444/hooks/test")] + public async Task UnsubscribeAsync_LookalikeZapierTarget_ReturnsNotFound(string targetUrl) + { + // Act + var response = await SendRequestAsync(request => request.Post().AppendPath("webhooks/unsubscribe") + .Content(new { target_url = targetUrl }).StatusCodeShouldBeNotFound()); + + // Assert + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); } [Fact] diff --git a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs b/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs new file mode 100644 index 0000000000..1b08523de2 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs @@ -0,0 +1,195 @@ +using System.Net; +using System.Reflection; +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Resilience; +using Foundatio.Serializer; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public sealed class WebHookDeliveryTests(ITestOutputHelper output) : TestWithServices(output) +{ + [Fact] + public async Task ProcessQueueEntryAsync_SuccessfulDelivery_DoesNotBufferResponseBody() + { + var content = new TrackingContent(); + using var handler = new StaticResponseHandler(new HttpResponseMessage(HttpStatusCode.OK) { Content = content }); + using var client = new HttpClient(handler, disposeHandler: false); + var options = GetService(); + var hook = new WebHook + { + Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", + Url = "https://example.com/webhook", Version = WebHook.KnownVersions.Version2, + EventTypes = [WebHook.KnownEventTypes.NewError] + }; + var repository = DispatchProxy.Create(); + ((HookRepositoryProxy)(object)repository).Hook = hook; + using var queue = new InMemoryQueue(); + using var cache = new InMemoryCacheClient(); + using var logs = new CaptureLoggerFactory(); + using var job = new WebHooksJob(queue, GetService(), GetService(), repository, + cache, GetService(), GetService(), options, TimeProvider, + GetService(), logs, new SingleClientFactory(client), GetService()); + await queue.EnqueueAsync(new WebHookNotification + { + OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, + Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } + }); + + await job.RunUntilEmptyAsync(TestCancellationToken); + + Assert.Equal(1, handler.RequestCount); + Assert.False(content.WasSerialized); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProcessQueueEntryAsync_ExistingPrivateHook_RequiresExplicitOverrideAndDoesNotRedirect(bool allowPrivate) + { + int delivered = 0; + int redirected = 0; + var builder = WebApplication.CreateBuilder(); + builder.Logging.ClearProviders(); + builder.WebHost.UseKestrel().UseUrls("http://127.0.0.1:0"); + await using var server = builder.Build(); + server.MapPost("/private-capability", context => + { + Interlocked.Increment(ref delivered); + context.Response.StatusCode = 302; + context.Response.Headers.Location = "/redirected-capability"; + return Task.CompletedTask; + }); + server.Map("/redirected-capability", () => Interlocked.Increment(ref redirected)); + await server.StartAsync(TestCancellationToken); + string address = server.Services.GetRequiredService().Features.Get()!.Addresses.Single(); + var options = GetService(); + options.WebHookOptions.AllowedPrivateNetworks = allowPrivate ? [IPNetwork.Parse("127.0.0.1/32")] : []; + var policy = GetService(); + var hook = new WebHook + { + Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", + Url = address + "/private-capability?secret-query=value", Version = WebHook.KnownVersions.Version2, + EventTypes = [WebHook.KnownEventTypes.NewError] + }; + var repository = DispatchProxy.Create(); + ((HookRepositoryProxy)(object)repository).Hook = hook; + using var queue = new InMemoryQueue(); + using var cache = new InMemoryCacheClient(); + using var logs = new CaptureLoggerFactory(); + using var job = new WebHooksJob(queue, GetService(), GetService(), repository, + cache, GetService(), GetService(), options, TimeProvider, + GetService(), logs, GetService(), policy); + await queue.EnqueueAsync(new WebHookNotification + { + OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, + Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } + }); + await job.RunUntilEmptyAsync(TestCancellationToken); + Assert.Equal(allowPrivate ? 1 : 0, delivered); + Assert.Equal(0, redirected); + Assert.True(hook.IsEnabled); + Assert.DoesNotContain(logs.Messages, message => message.Contains("private-capability") || message.Contains("secret-query")); + await server.StopAsync(TestCancellationToken); + } + + [Fact] + public void DeliveryLogging_CapabilityInUrlAndException_OmitsSecretValues() + { + using var capture = new CaptureLoggerFactory(); + const string url = "https://user:password@example.com/private-capability?secret-query=value"; + var exception = new HttpRequestException(url); + Exceptionless.Core.Extensions.LoggerExtensions.RecordWebHook(capture, "hook", "project", url); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookComplete(capture, HttpStatusCode.OK, "organization", "project", url); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookError(capture, HttpStatusCode.BadGateway, "organization", "project", url, exception); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookTimeout(capture, null, "organization", "project", url, exception); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookDisabledStatusCode(capture, "hook", HttpStatusCode.Gone, "organization", "project", url); + Assert.Equal(5, capture.Messages.Count); + Assert.All(capture.Messages, message => + { + Assert.DoesNotContain("private-capability", message); + Assert.DoesNotContain("secret-query", message); + Assert.DoesNotContain("password", message); + }); + } + + private class HookRepositoryProxy : DispatchProxy + { + public WebHook Hook { get; set; } = null!; + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + { + Assert.Equal("GetByIdAsync", targetMethod!.Name); + return Task.FromResult(Hook); + } + } + + private sealed class CaptureLoggerFactory : ILoggerFactory, ILogger + { + public List Messages { get; } = []; + public ILogger CreateLogger(string categoryName) => this; + public void AddProvider(ILoggerProvider provider) { } + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + string values = state is IEnumerable> properties + ? String.Join(" ", properties.Select(p => p.Value)) : String.Empty; + Messages.Add(formatter(state, exception) + values + exception); + } + public void Dispose() { } + } + + private sealed class SingleClientFactory(HttpClient client) : IHttpClientFactory + { + public HttpClient CreateClient(string name) => client; + } + + private sealed class StaticResponseHandler(HttpResponseMessage response) : HttpMessageHandler + { + public int RequestCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestCount++; + return Task.FromResult(response); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + response.Dispose(); + base.Dispose(disposing); + } + } + + private sealed class TrackingContent : HttpContent + { + public bool WasSerialized { get; private set; } + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + { + WasSerialized = true; + return Task.CompletedTask; + } + + protected override bool TryComputeLength(out long length) + { + length = -1; + return false; + } + } +} diff --git a/tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs b/tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs new file mode 100644 index 0000000000..82a7cee87d --- /dev/null +++ b/tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs @@ -0,0 +1,68 @@ +using System.Net; +using Exceptionless.Core; +using Exceptionless.Core.Configuration; +using Exceptionless.Core.Services; +using Microsoft.Extensions.Configuration; +using Xunit; + +namespace Exceptionless.Tests.Services; + +public sealed class WebHookDestinationPolicyTests(ITestOutputHelper output) : TestWithServices(output) +{ + [Theory] + [InlineData("https://example.com/path?token=value", true)] + [InlineData("http://8.8.8.8/path", true)] + [InlineData("http://127.0.0.1/path", false)] + [InlineData("http://localhost/path", false)] + [InlineData("http://[::ffff:127.0.0.1]/path", false)] + [InlineData("http://169.254.169.254/path", false)] + [InlineData("https://user:password@example.com/path", false)] + [InlineData("https://example.com/path#fragment", false)] + [InlineData("ftp://example.com/path", false)] + public void IsValidDestination_DefaultConfiguration_UsesAllowedHttpDestinations(string url, bool allowed) + => Assert.Equal(allowed, GetService().IsValidDestination(url)); + + [Fact] + public void IsAllowed_PrivateOverride_OnlyAppliesToConfiguredNetwork() + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["WebHooks:AllowedPrivateNetworks:0"] = "10.20.30.0/24" + }).Build(); + var options = GetService(); + options.WebHookOptions = WebHookOptions.ReadFromConfiguration(configuration); + var policy = new WebHookDestinationPolicy(options); + Assert.True(policy.IsAllowed(IPAddress.Parse("10.20.30.1"))); + Assert.True(policy.IsAllowed(IPAddress.Parse("::ffff:10.20.30.1"))); + Assert.False(policy.IsAllowed(IPAddress.Parse("10.20.31.1"))); + Assert.False(policy.IsAllowed(IPAddress.Parse("169.254.169.254"))); + } + + [Fact] + public async Task NamedClient_LoopbackDnsDestination_DoesNotConnect() + { + using var client = GetService().CreateClient(WebHookDestinationPolicy.HttpClientName); + await Assert.ThrowsAsync(() => client.GetAsync("http://localhost:1/private", TestCancellationToken)); + } + + [Fact] + public void NamedClient_Transport_DisablesImplicitBehavior() + { + HttpMessageHandler handler = GetService().CreateHandler(WebHookDestinationPolicy.HttpClientName); + while (handler is DelegatingHandler delegating) + handler = Assert.IsAssignableFrom(delegating.InnerHandler); + var sockets = Assert.IsType(handler); + Assert.False(sockets.AllowAutoRedirect); + Assert.False(sockets.UseProxy); + Assert.False(sockets.UseCookies); + Assert.Null(sockets.ActivityHeadersPropagator); + Assert.NotNull(sockets.ConnectCallback); + } + + [Fact] + public void GetLoggingDestination_CapabilityUrl_OmitsCredentialsPathAndQuery() + { + string destination = WebHookDestinationPolicy.GetLoggingDestination("https://user:password@example.com/private-capability?secret-query=value#fragment"); + Assert.Equal("https://example.com", destination); + } +} diff --git a/tests/http/webhooks.http b/tests/http/webhooks.http index d9c844f80a..615eff17ff 100644 --- a/tests/http/webhooks.http +++ b/tests/http/webhooks.http @@ -2,7 +2,7 @@ @email = admin@exceptionless.test @password = tester @projectId = 537650f3b77efe23a47914f4 -# Set this to your webhook receiver. +# Use a public HTTP(S) receiver. Private receivers require WebHooks:AllowedPrivateNetworks in server configuration. @webhookUrl = https://example.com/exceptionless ### login to test account From 537354e5943a7c94d82bebadf40672e76a6ba404 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Mon, 12 Oct 2026 22:30:13 -0500 Subject: [PATCH 02/26] Preserve webhook destinations and exception diagnostics --- docs/webhook-delivery.md | 12 +-- src/Exceptionless.Core/Bootstrapper.cs | 10 +- .../Configuration/AppOptions.cs | 2 - .../Configuration/WebHookOptions.cs | 15 --- .../Extensions/LoggerExtensions.cs | 20 ++-- src/Exceptionless.Core/Jobs/WebHooksJob.cs | 29 ++---- .../Services/WebHookDelivery.cs | 12 +++ .../Services/WebHookDestinationPolicy.cs | 62 ------------- .../Api/Handlers/WebHookHandler.cs | 5 - src/Exceptionless.Web/ApmExtensions.cs | 2 +- .../Api/Endpoints/WebHookEndpointTests.cs | 14 +-- .../Jobs/WebHookDeliveryTests.cs | 91 +++++++++++++++---- .../Services/WebHookDestinationPolicyTests.cs | 68 -------------- tests/http/webhooks.http | 2 +- 14 files changed, 120 insertions(+), 224 deletions(-) delete mode 100644 src/Exceptionless.Core/Configuration/WebHookOptions.cs create mode 100644 src/Exceptionless.Core/Services/WebHookDelivery.cs delete mode 100644 src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs delete mode 100644 tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs diff --git a/docs/webhook-delivery.md b/docs/webhook-delivery.md index d822b241a5..812a935988 100644 --- a/docs/webhook-delivery.md +++ b/docs/webhook-delivery.md @@ -1,11 +1,5 @@ -Webhooks use HTTP or HTTPS destinations. Connections resolve and check the destination address at delivery time, including URLs saved before deployment. Redirects are not followed, and requests do not use a proxy or cookies. +Webhooks deliver to user-supplied HTTP or HTTPS URLs, including private receivers. No destination allowlist or private-network override is required. Redirects and proxy settings use the standard HTTP client behavior. Cookies are not shared between deliveries. -For a self-hosted private receiver, explicitly configure its smallest required network on every API and job host: +Delivery logs identify the destination origin instead of recording URL credentials, paths, or queries. Automatic HTTP delivery traces and client logs are omitted. Failures retain the original exception and message for troubleshooting; exception details are not redacted. -```yaml -WebHooks: - AllowedPrivateNetworks: - - 10.20.30.40/32 -``` - -The default list is empty. Existing private receivers require this override before deploying. Their saved records are preserved. Delivery logs identify the destination origin; target paths, queries, credentials, and HTTP delivery traces are omitted. +Only response headers are needed to determine delivery status. Response bodies are not buffered, and responses are disposed even when failure bookkeeping throws. diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 6994500568..a940cb8fe0 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -224,16 +224,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO s.GetRequiredService(), DecompressionMethods.None)); services.AddSingleton(); - services.AddSingleton(); - services.AddHttpClient(WebHookDestinationPolicy.HttpClientName) + services.AddHttpClient(WebHookDelivery.HttpClientName) .RemoveAllLoggers() - .ConfigurePrimaryHttpMessageHandler(provider => new SocketsHttpHandler + .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { - AllowAutoRedirect = false, - UseProxy = false, UseCookies = false, - ActivityHeadersPropagator = null, - ConnectCallback = provider.GetRequiredService().ConnectAsync + ActivityHeadersPropagator = null }); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs index 16546635df..d7ebd91d72 100644 --- a/src/Exceptionless.Core/Configuration/AppOptions.cs +++ b/src/Exceptionless.Core/Configuration/AppOptions.cs @@ -83,7 +83,6 @@ public class AppOptions public AuthOptions AuthOptions { get; internal set; } = null!; public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!; public SourceMapOptions SourceMapOptions { get; internal set; } = null!; - public WebHookOptions WebHookOptions { get; internal set; } = null!; public AssistantOptions AssistantOptions { get; internal set; } = null!; public static AppOptions ReadFromConfiguration(IConfiguration config) @@ -140,7 +139,6 @@ public static AppOptions ReadFromConfiguration(IConfiguration config) options.AuthOptions = AuthOptions.ReadFromConfiguration(config); options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config); options.SourceMapOptions = SourceMapOptions.ReadFromConfiguration(config); - options.WebHookOptions = WebHookOptions.ReadFromConfiguration(config); options.AssistantOptions = AssistantOptions.ReadFromConfiguration(config); return options; diff --git a/src/Exceptionless.Core/Configuration/WebHookOptions.cs b/src/Exceptionless.Core/Configuration/WebHookOptions.cs deleted file mode 100644 index 1105ee3bc8..0000000000 --- a/src/Exceptionless.Core/Configuration/WebHookOptions.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.Net; -using Microsoft.Extensions.Configuration; - -namespace Exceptionless.Core.Configuration; - -public sealed class WebHookOptions -{ - public IPNetwork[] AllowedPrivateNetworks { get; internal set; } = []; - - public static WebHookOptions ReadFromConfiguration(IConfiguration configuration) - => new() - { - AllowedPrivateNetworks = configuration.GetSection("WebHooks:AllowedPrivateNetworks").Get()?.Select(IPNetwork.Parse).ToArray() ?? [] - }; -} diff --git a/src/Exceptionless.Core/Extensions/LoggerExtensions.cs b/src/Exceptionless.Core/Extensions/LoggerExtensions.cs index 069490821c..b1e655a02d 100644 --- a/src/Exceptionless.Core/Extensions/LoggerExtensions.cs +++ b/src/Exceptionless.Core/Extensions/LoggerExtensions.cs @@ -11,7 +11,7 @@ internal static class LoggerExtensions LoggerMessage.Define( LogLevel.Trace, new EventId(0, nameof(RecordWebHook)), - "Process web hook call: id={Id} project={ProjectId} url={Url}"); + "Process web hook call: id={Id} project={ProjectId} destination={Destination}"); private static readonly Action _webHookCancelled = LoggerMessage.Define( @@ -29,25 +29,25 @@ internal static class LoggerExtensions LoggerMessage.Define( LogLevel.Error, new EventId(3, nameof(WebHookTimeout)), - "Timeout calling web hook: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination} failure={ExceptionType}"); + "Timeout calling web hook: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination}: {Message}"); private static readonly Action _webHookError = LoggerMessage.Define( LogLevel.Error, new EventId(4, nameof(WebHookError)), - "Error calling web hook: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination} failure={ExceptionType}"); + "Error calling web hook: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination}: {Message}"); private static readonly Action _webHookComplete = LoggerMessage.Define( LogLevel.Information, new EventId(5, nameof(WebHookError)), - "Web hook POST complete: status={Status} org={organization} project={ProjectId} url={Url}"); + "Web hook POST complete: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination}"); private static readonly Action _webHookDisabledStatusCode = LoggerMessage.Define( LogLevel.Warning, new EventId(6, nameof(WebHookDisabledStatusCode)), - "Disabling Web hook instance {WebHookId} due to status code: status={Status} org={organization} project={ProjectId} url={Url}"); + "Disabling Web hook instance {WebHookId} due to status code: status={Status} organization={OrganizationId} project={ProjectId} destination={Destination}"); private static readonly Action _webHookDisabledTooManyErrors = LoggerMessage.Define( @@ -179,16 +179,16 @@ public static void WebHookDisabledTooManyErrors(this ILogger logger, string webH => _webHookDisabledTooManyErrors(logger, webHookId, null); public static void WebHookDisabledStatusCode(this ILogger logger, string webHookId, HttpStatusCode? statusCode, string organizationId, string projectId, string url) - => _webHookDisabledStatusCode(logger, webHookId, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), null); + => _webHookDisabledStatusCode(logger, webHookId, statusCode, organizationId, projectId, WebHookDelivery.GetLoggingDestination(url), null); public static void WebHookComplete(this ILogger logger, HttpStatusCode? statusCode, string organizationId, string projectId, string url) - => _webHookComplete(logger, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), null); + => _webHookComplete(logger, statusCode, organizationId, projectId, WebHookDelivery.GetLoggingDestination(url), null); public static void WebHookError(this ILogger logger, HttpStatusCode? statusCode, string organizationId, string projectId, string url, Exception exception) - => _webHookError(logger, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), exception.GetType().Name, null); + => _webHookError(logger, statusCode, organizationId, projectId, WebHookDelivery.GetLoggingDestination(url), exception.Message, exception); public static void WebHookTimeout(this ILogger logger, HttpStatusCode? statusCode, string organizationId, string projectId, string url, Exception exception) - => _webHookTimeout(logger, statusCode, organizationId, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), exception.GetType().Name, null); + => _webHookTimeout(logger, statusCode, organizationId, projectId, WebHookDelivery.GetLoggingDestination(url), exception.Message, exception); public static void WebHookCancelledBackoff(this ILogger logger, long consecutiveErrors, DateTime nextAttemptAllowedAt) => _webHookCancelledBackoff(logger, consecutiveErrors, nextAttemptAllowedAt, null); @@ -197,5 +197,5 @@ public static void WebHookCancelled(this ILogger logger) => _webHookCancelled(logger, null); public static void RecordWebHook(this ILogger logger, string id, string projectId, string url) - => _recordWebHook(logger, id, projectId, WebHookDestinationPolicy.GetLoggingDestination(url), null); + => _recordWebHook(logger, id, projectId, WebHookDelivery.GetLoggingDestination(url), null); } diff --git a/src/Exceptionless.Core/Jobs/WebHooksJob.cs b/src/Exceptionless.Core/Jobs/WebHooksJob.cs index 4ed15cbab7..175ae2ab45 100644 --- a/src/Exceptionless.Core/Jobs/WebHooksJob.cs +++ b/src/Exceptionless.Core/Jobs/WebHooksJob.cs @@ -41,10 +41,9 @@ public class WebHooksJob : QueueJobBase, IDisposable private readonly AppOptions _appOptions; private readonly HttpClient _client; - private readonly WebHookDestinationPolicy _destinationPolicy; public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, ITextSerializer serializer, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, - IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, WebHookDestinationPolicy destinationPolicy) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) + IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { _projectRepository = projectRepository; _slackService = slackService; @@ -54,25 +53,17 @@ public WebHooksJob(IQueue queue, IProjectRepository project _jsonOptions = jsonOptions; _versionOneJsonOptions = VersionOnePlugin.CreateJsonSerializerOptions(jsonOptions); _appOptions = appOptions; - _client = httpClientFactory.CreateClient(WebHookDestinationPolicy.HttpClientName); - _destinationPolicy = destinationPolicy; + _client = httpClientFactory.CreateClient(WebHookDelivery.HttpClientName); } protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) { var body = context.QueueEntry.Value; - string destination = WebHookDestinationPolicy.GetLoggingDestination(body.Url); bool shouldLog = body.ProjectId != _appOptions.InternalProjectId; using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) { - if (shouldLog) _logger.RecordWebHook(context.QueueEntry.Id, body.ProjectId, destination); - - if (!_destinationPolicy.IsValidDestination(body.Url)) - { - _logger.LogWarning("Web hook destination is not allowed for {WebHookId}", body.WebHookId); - return JobResult.Cancelled; - } + if (shouldLog) _logger.RecordWebHook(context.QueueEntry.Id, body.ProjectId, body.Url); if (!await IsEnabledAsync(body)) { @@ -110,7 +101,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex { Content = JsonContent.Create(body.Data, options: jsonOptions) }; - request.Options.Set(WebHookDestinationPolicy.DeliveryRequest, true); + request.Options.Set(WebHookDelivery.DeliveryRequest, true); response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, postCancellationTokenSource.Token); if (!response.IsSuccessStatusCode) successful = false; @@ -120,24 +111,25 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex catch (OperationCanceledException ex) { successful = false; - if (shouldLog) _logger.WebHookTimeout(response?.StatusCode, body.OrganizationId, body.ProjectId, destination, ex); + if (shouldLog) _logger.WebHookTimeout(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url, ex); return JobResult.Cancelled; } catch (Exception ex) { successful = false; - if (shouldLog) _logger.WebHookError(response?.StatusCode, body.OrganizationId, body.ProjectId, destination, ex); - return JobResult.FromException(new HttpRequestException($"Web hook delivery failed ({ex.GetType().Name}).")); + if (shouldLog) _logger.WebHookError(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url, ex); + return JobResult.FromException(ex); } finally { + using var responseToDispose = response; if (successful) { - _logger.WebHookComplete(response?.StatusCode, body.OrganizationId, body.ProjectId, destination); + _logger.WebHookComplete(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } else if (response is not null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone)) { - _logger.WebHookDisabledStatusCode(body.Type == WebHookType.Slack ? "Slack" : body.WebHookId!, response.StatusCode, body.OrganizationId, body.ProjectId, destination); + _logger.WebHookDisabledStatusCode(body.Type == WebHookType.Slack ? "Slack" : body.WebHookId!, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url); await DisableIntegrationAsync(body); await cache.RemoveAllAsync(_cacheKeys); } @@ -171,7 +163,6 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex } } } - response?.Dispose(); } } diff --git a/src/Exceptionless.Core/Services/WebHookDelivery.cs b/src/Exceptionless.Core/Services/WebHookDelivery.cs new file mode 100644 index 0000000000..2a5a519c0f --- /dev/null +++ b/src/Exceptionless.Core/Services/WebHookDelivery.cs @@ -0,0 +1,12 @@ +namespace Exceptionless.Core.Services; + +public static class WebHookDelivery +{ + public const string HttpClientName = "WebHooks"; + public static readonly HttpRequestOptionsKey DeliveryRequest = new("Exceptionless.WebHookDelivery"); + + public static string GetLoggingDestination(string? value) + => Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) + ? uri.GetComponents(UriComponents.SchemeAndServer, UriFormat.UriEscaped) + : "invalid"; +} diff --git a/src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs b/src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs deleted file mode 100644 index 10dcba5136..0000000000 --- a/src/Exceptionless.Core/Services/WebHookDestinationPolicy.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System.Net; -using System.Net.Sockets; -using Exceptionless.Core.Utility; - -namespace Exceptionless.Core.Services; - -public sealed class WebHookDestinationPolicy(AppOptions options) -{ - public const string HttpClientName = "WebHooks"; - public static readonly HttpRequestOptionsKey DeliveryRequest = new("Exceptionless.WebHookDelivery"); - - public bool IsAllowed(IPAddress address) - { - if (address.IsIPv4MappedToIPv6) - address = address.MapToIPv4(); - return PublicAddressPolicy.IsPublic(address) || options.WebHookOptions.AllowedPrivateNetworks.Any(network => network.Contains(address)); - } - - public static bool TryCreateUri(string? value, out Uri uri) - { - uri = null!; - return Uri.TryCreate(value, UriKind.Absolute, out uri!) - && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) - && !String.IsNullOrEmpty(uri.Host) && String.IsNullOrEmpty(uri.UserInfo) && String.IsNullOrEmpty(uri.Fragment); - } - - public bool IsValidDestination(string? value) - => TryCreateUri(value, out var uri) && (IPAddress.TryParse(uri.Host.Trim('[', ']'), out var address) - ? IsAllowed(address) - : !uri.IsLoopback || IsAllowed(IPAddress.Loopback) || IsAllowed(IPAddress.IPv6Loopback)); - - public async ValueTask ConnectAsync(SocketsHttpConnectionContext context, CancellationToken cancellationToken) - { - var addresses = await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken); - Exception? lastException = null; - foreach (var address in addresses.Where(IsAllowed)) - { - var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true }; - try - { - await socket.ConnectAsync(new IPEndPoint(address, context.DnsEndPoint.Port), cancellationToken); - return new NetworkStream(socket, ownsSocket: true); - } - catch (SocketException ex) - { - lastException = ex; - socket.Dispose(); - } - catch - { - socket.Dispose(); - throw; - } - } - throw new HttpRequestException("Web hook destination has no reachable allowed address.", lastException); - } - - public static string GetLoggingDestination(string? value) - => Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) - ? uri.GetComponents(UriComponents.SchemeAndServer, UriFormat.UriEscaped) - : "invalid"; -} diff --git a/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs b/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs index ce80193183..4ee0f72eeb 100644 --- a/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs @@ -5,7 +5,6 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Services; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; using Exceptionless.Web.Api.Results; @@ -22,7 +21,6 @@ public class WebHookHandler( IWebHookRepository repository, IProjectRepository projectRepository, BillingManager billingManager, - WebHookDestinationPolicy destinationPolicy, ApiMapper mapper, LinkGenerator linkGenerator, IHttpContextAccessor httpContextAccessor, @@ -208,9 +206,6 @@ public Result Handle(TestWebHook message) if (!await billingManager.HasPremiumFeaturesAsync(project is not null ? project.OrganizationId : value.OrganizationId)) return Result.Invalid(ValidationError.Create(ApiValidationErrorIdentifiers.PlanLimit, "Please upgrade your plan to add integrations.")); - if (!destinationPolicy.IsValidDestination(value.Url)) - return Result.BadRequest("Please specify an allowed HTTP or HTTPS destination."); - return null; } diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index d55850550b..3bc33b9e7c 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -73,7 +73,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }); b.AddHttpClientInstrumentation(o => o.FilterHttpRequestMessage = request => - !request.Options.TryGetValue(WebHookDestinationPolicy.DeliveryRequest, out bool delivery) || !delivery); + !request.Options.TryGetValue(WebHookDelivery.DeliveryRequest, out bool delivery) || !delivery); b.AddSource("Exceptionless", "Exceptionless.Assistant", "Foundatio"); if (config.EnableRedis) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs index 9fe7b5d332..ff1ba34cf5 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs @@ -166,7 +166,7 @@ public Task CanCreateNewWebHook() EventTypes = [WebHook.KnownEventTypes.StackPromoted], OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, - Url = "https://example.com/test" + Url = "https://localhost/test" }) .StatusCodeShouldBeCreated() ); @@ -181,7 +181,7 @@ public async Task PostAsync_WithUnauthorizedOrganization_ReturnsBadRequestProble EventTypes = [WebHook.KnownEventTypes.StackPromoted], OrganizationId = SampleDataService.FREE_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, - Url = "https://example.com/test" + Url = "https://localhost/test" }; // Act @@ -208,7 +208,7 @@ public async Task PostAsync_WithUnauthorizedProject_ReturnsBadRequestProblemDeta EventTypes = [WebHook.KnownEventTypes.StackPromoted], OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.FREE_PROJECT_ID, - Url = "https://example.com/test" + Url = "https://localhost/test" }; // Act @@ -248,7 +248,7 @@ public async Task PostAsync_WithoutOrganizationOrProject_ReturnsBadRequestProble var newWebHook = new NewWebHook { EventTypes = [WebHook.KnownEventTypes.StackPromoted], - Url = "https://example.com/test" + Url = "https://localhost/test" }; // Act @@ -278,7 +278,7 @@ public async Task CreateNewWebHookWithInvalidEventTypeFails() EventTypes = ["Invalid"], OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, - Url = "https://example.com/test" + Url = "https://localhost/test" }) .StatusCodeShouldBeUnprocessableEntity() ); @@ -569,10 +569,10 @@ public async Task Test_WithGetRequest_ReturnsZapierTestMessages() [InlineData("http://localhost/path")] [InlineData("http://169.254.169.254/path")] [InlineData("https://user:password@example.com/path")] - public Task PostAsync_DisallowedDestination_ReturnsBadRequest(string url) + public Task PostAsync_UserSuppliedDestination_ReturnsCreated(string url) => SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("webhooks") .Content(new NewWebHook { ProjectId = SampleDataService.TEST_PROJECT_ID, Url = url, EventTypes = [WebHook.KnownEventTypes.NewError] }) - .StatusCodeShouldBeBadRequest()); + .StatusCodeShouldBeCreated()); [Fact] public async Task UnsubscribeAsync_ExistingZapierHook_RemovesWebHook() diff --git a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs b/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs index 1b08523de2..a6cc7022f3 100644 --- a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs @@ -14,7 +14,6 @@ using Foundatio.Serializer; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; using Xunit; @@ -23,8 +22,10 @@ namespace Exceptionless.Tests.Jobs; public sealed class WebHookDeliveryTests(ITestOutputHelper output) : TestWithServices(output) { - [Fact] - public async Task ProcessQueueEntryAsync_SuccessfulDelivery_DoesNotBufferResponseBody() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProcessQueueEntryAsync_Response_DisposesWithoutBufferingEvenWhenCompletionFails(bool failCompletion) { var content = new TrackingContent(); using var handler = new StaticResponseHandler(new HttpResponseMessage(HttpStatusCode.OK) { Content = content }); @@ -40,26 +41,59 @@ public async Task ProcessQueueEntryAsync_SuccessfulDelivery_DoesNotBufferRespons ((HookRepositoryProxy)(object)repository).Hook = hook; using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); - using var logs = new CaptureLoggerFactory(); + using var logs = new CaptureLoggerFactory { FailCompletion = failCompletion }; using var job = new WebHooksJob(queue, GetService(), GetService(), repository, cache, GetService(), GetService(), options, TimeProvider, - GetService(), logs, new SingleClientFactory(client), GetService()); + GetService(), logs, new SingleClientFactory(client)); await queue.EnqueueAsync(new WebHookNotification { OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } }); - await job.RunUntilEmptyAsync(TestCancellationToken); + var exception = await Record.ExceptionAsync(async () => { _ = await job.RunAsync(TestCancellationToken); }); Assert.Equal(1, handler.RequestCount); Assert.False(content.WasSerialized); + Assert.True(content.WasDisposed); + Assert.Equal(failCompletion ? "Completion logging failed" : null, exception?.Message); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ProcessQueueEntryAsync_ExistingPrivateHook_RequiresExplicitOverrideAndDoesNotRedirect(bool allowPrivate) + [Fact] + public async Task ProcessQueueEntryAsync_TransportFailure_PreservesOriginalException() + { + var failure = new HttpRequestException("Connection failed", new IOException("Socket closed")); + using var handler = new FailingHandler(failure); + using var client = new HttpClient(handler, disposeHandler: false); + var options = GetService(); + var hook = new WebHook + { + Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", + Url = "https://example.com/webhook", Version = WebHook.KnownVersions.Version2, + EventTypes = [WebHook.KnownEventTypes.NewError] + }; + var repository = DispatchProxy.Create(); + ((HookRepositoryProxy)(object)repository).Hook = hook; + using var queue = new InMemoryQueue(); + using var cache = new InMemoryCacheClient(); + using var logs = new CaptureLoggerFactory(); + using var job = new WebHooksJob(queue, GetService(), GetService(), repository, + cache, GetService(), GetService(), options, TimeProvider, + GetService(), logs, new SingleClientFactory(client)); + await queue.EnqueueAsync(new WebHookNotification + { + OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, + Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } + }); + + var result = await job.RunAsync(TestCancellationToken); + + Assert.Same(failure, result.Error); + Assert.Contains(logs.Exceptions, exception => ReferenceEquals(exception, failure)); + } + + [Fact] + public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsRedirectWithoutOverrides() { int delivered = 0; int redirected = 0; @@ -78,8 +112,6 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_RequiresExplicitOve await server.StartAsync(TestCancellationToken); string address = server.Services.GetRequiredService().Features.Get()!.Addresses.Single(); var options = GetService(); - options.WebHookOptions.AllowedPrivateNetworks = allowPrivate ? [IPNetwork.Parse("127.0.0.1/32")] : []; - var policy = GetService(); var hook = new WebHook { Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", @@ -93,32 +125,36 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_RequiresExplicitOve using var logs = new CaptureLoggerFactory(); using var job = new WebHooksJob(queue, GetService(), GetService(), repository, cache, GetService(), GetService(), options, TimeProvider, - GetService(), logs, GetService(), policy); + GetService(), logs, GetService()); await queue.EnqueueAsync(new WebHookNotification { OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } }); - await job.RunUntilEmptyAsync(TestCancellationToken); - Assert.Equal(allowPrivate ? 1 : 0, delivered); - Assert.Equal(0, redirected); + var result = await job.RunAsync(TestCancellationToken); + Assert.True(result.IsSuccess, result.Error?.ToString()); + Assert.Equal(1, delivered); + Assert.Equal(1, redirected); Assert.True(hook.IsEnabled); Assert.DoesNotContain(logs.Messages, message => message.Contains("private-capability") || message.Contains("secret-query")); await server.StopAsync(TestCancellationToken); } [Fact] - public void DeliveryLogging_CapabilityInUrlAndException_OmitsSecretValues() + public void DeliveryLogging_RedactsDestinationAndPreservesException() { using var capture = new CaptureLoggerFactory(); const string url = "https://user:password@example.com/private-capability?secret-query=value"; - var exception = new HttpRequestException(url); + var exception = new HttpRequestException("Connection failed", new IOException("Socket closed")); Exceptionless.Core.Extensions.LoggerExtensions.RecordWebHook(capture, "hook", "project", url); Exceptionless.Core.Extensions.LoggerExtensions.WebHookComplete(capture, HttpStatusCode.OK, "organization", "project", url); Exceptionless.Core.Extensions.LoggerExtensions.WebHookError(capture, HttpStatusCode.BadGateway, "organization", "project", url, exception); Exceptionless.Core.Extensions.LoggerExtensions.WebHookTimeout(capture, null, "organization", "project", url, exception); Exceptionless.Core.Extensions.LoggerExtensions.WebHookDisabledStatusCode(capture, "hook", HttpStatusCode.Gone, "organization", "project", url); Assert.Equal(5, capture.Messages.Count); + Assert.Equal(2, capture.Exceptions.Count); + Assert.All(capture.Exceptions, logged => Assert.Same(exception, logged)); + Assert.Contains(capture.Messages, message => message.Contains("Connection failed")); Assert.All(capture.Messages, message => { Assert.DoesNotContain("private-capability", message); @@ -139,13 +175,19 @@ private class HookRepositoryProxy : DispatchProxy private sealed class CaptureLoggerFactory : ILoggerFactory, ILogger { + public bool FailCompletion { get; init; } public List Messages { get; } = []; + public List Exceptions { get; } = []; public ILogger CreateLogger(string categoryName) => this; public void AddProvider(ILoggerProvider provider) { } public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { + if (FailCompletion && eventId.Id == 5) + throw new InvalidOperationException("Completion logging failed"); + if (exception is not null) + Exceptions.Add(exception); string values = state is IEnumerable> properties ? String.Join(" ", properties.Select(p => p.Value)) : String.Empty; Messages.Add(formatter(state, exception) + values + exception); @@ -158,6 +200,12 @@ private sealed class SingleClientFactory(HttpClient client) : IHttpClientFactory public HttpClient CreateClient(string name) => client; } + private sealed class FailingHandler(Exception exception) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromException(exception); + } + private sealed class StaticResponseHandler(HttpResponseMessage response) : HttpMessageHandler { public int RequestCount { get; private set; } @@ -179,6 +227,13 @@ protected override void Dispose(bool disposing) private sealed class TrackingContent : HttpContent { public bool WasSerialized { get; private set; } + public bool WasDisposed { get; private set; } + + protected override void Dispose(bool disposing) + { + WasDisposed = true; + base.Dispose(disposing); + } protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) { diff --git a/tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs b/tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs deleted file mode 100644 index 82a7cee87d..0000000000 --- a/tests/Exceptionless.Tests/Services/WebHookDestinationPolicyTests.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System.Net; -using Exceptionless.Core; -using Exceptionless.Core.Configuration; -using Exceptionless.Core.Services; -using Microsoft.Extensions.Configuration; -using Xunit; - -namespace Exceptionless.Tests.Services; - -public sealed class WebHookDestinationPolicyTests(ITestOutputHelper output) : TestWithServices(output) -{ - [Theory] - [InlineData("https://example.com/path?token=value", true)] - [InlineData("http://8.8.8.8/path", true)] - [InlineData("http://127.0.0.1/path", false)] - [InlineData("http://localhost/path", false)] - [InlineData("http://[::ffff:127.0.0.1]/path", false)] - [InlineData("http://169.254.169.254/path", false)] - [InlineData("https://user:password@example.com/path", false)] - [InlineData("https://example.com/path#fragment", false)] - [InlineData("ftp://example.com/path", false)] - public void IsValidDestination_DefaultConfiguration_UsesAllowedHttpDestinations(string url, bool allowed) - => Assert.Equal(allowed, GetService().IsValidDestination(url)); - - [Fact] - public void IsAllowed_PrivateOverride_OnlyAppliesToConfiguredNetwork() - { - var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary - { - ["WebHooks:AllowedPrivateNetworks:0"] = "10.20.30.0/24" - }).Build(); - var options = GetService(); - options.WebHookOptions = WebHookOptions.ReadFromConfiguration(configuration); - var policy = new WebHookDestinationPolicy(options); - Assert.True(policy.IsAllowed(IPAddress.Parse("10.20.30.1"))); - Assert.True(policy.IsAllowed(IPAddress.Parse("::ffff:10.20.30.1"))); - Assert.False(policy.IsAllowed(IPAddress.Parse("10.20.31.1"))); - Assert.False(policy.IsAllowed(IPAddress.Parse("169.254.169.254"))); - } - - [Fact] - public async Task NamedClient_LoopbackDnsDestination_DoesNotConnect() - { - using var client = GetService().CreateClient(WebHookDestinationPolicy.HttpClientName); - await Assert.ThrowsAsync(() => client.GetAsync("http://localhost:1/private", TestCancellationToken)); - } - - [Fact] - public void NamedClient_Transport_DisablesImplicitBehavior() - { - HttpMessageHandler handler = GetService().CreateHandler(WebHookDestinationPolicy.HttpClientName); - while (handler is DelegatingHandler delegating) - handler = Assert.IsAssignableFrom(delegating.InnerHandler); - var sockets = Assert.IsType(handler); - Assert.False(sockets.AllowAutoRedirect); - Assert.False(sockets.UseProxy); - Assert.False(sockets.UseCookies); - Assert.Null(sockets.ActivityHeadersPropagator); - Assert.NotNull(sockets.ConnectCallback); - } - - [Fact] - public void GetLoggingDestination_CapabilityUrl_OmitsCredentialsPathAndQuery() - { - string destination = WebHookDestinationPolicy.GetLoggingDestination("https://user:password@example.com/private-capability?secret-query=value#fragment"); - Assert.Equal("https://example.com", destination); - } -} diff --git a/tests/http/webhooks.http b/tests/http/webhooks.http index 615eff17ff..d9c844f80a 100644 --- a/tests/http/webhooks.http +++ b/tests/http/webhooks.http @@ -2,7 +2,7 @@ @email = admin@exceptionless.test @password = tester @projectId = 537650f3b77efe23a47914f4 -# Use a public HTTP(S) receiver. Private receivers require WebHooks:AllowedPrivateNetworks in server configuration. +# Set this to your webhook receiver. @webhookUrl = https://example.com/exceptionless ### login to test account From 08c6d4fee358dc0d7f24e972f30a570b78c87bea Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 13 Oct 2026 06:47:51 -0500 Subject: [PATCH 03/26] Simplify webhook delivery test repository mocks --- .../Jobs/WebHookDeliveryTests.cs | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs b/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs index a6cc7022f3..839991e3e0 100644 --- a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Jobs; @@ -7,15 +6,18 @@ using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; +using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Queues; +using Foundatio.Repositories; using Foundatio.Resilience; using Foundatio.Serializer; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; +using Moq; using Xunit; namespace Exceptionless.Tests.Jobs; @@ -27,22 +29,23 @@ public sealed class WebHookDeliveryTests(ITestOutputHelper output) : TestWithSer [InlineData(true)] public async Task ProcessQueueEntryAsync_Response_DisposesWithoutBufferingEvenWhenCompletionFails(bool failCompletion) { + // Arrange var content = new TrackingContent(); using var handler = new StaticResponseHandler(new HttpResponseMessage(HttpStatusCode.OK) { Content = content }); using var client = new HttpClient(handler, disposeHandler: false); var options = GetService(); var hook = new WebHook { - Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", + Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, Url = "https://example.com/webhook", Version = WebHook.KnownVersions.Version2, EventTypes = [WebHook.KnownEventTypes.NewError] }; - var repository = DispatchProxy.Create(); - ((HookRepositoryProxy)(object)repository).Hook = hook; + var repository = new Mock(MockBehavior.Strict); + repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); using var logs = new CaptureLoggerFactory { FailCompletion = failCompletion }; - using var job = new WebHooksJob(queue, GetService(), GetService(), repository, + using var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, cache, GetService(), GetService(), options, TimeProvider, GetService(), logs, new SingleClientFactory(client)); await queue.EnqueueAsync(new WebHookNotification @@ -51,8 +54,10 @@ await queue.EnqueueAsync(new WebHookNotification Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } }); + // Act var exception = await Record.ExceptionAsync(async () => { _ = await job.RunAsync(TestCancellationToken); }); + // Assert Assert.Equal(1, handler.RequestCount); Assert.False(content.WasSerialized); Assert.True(content.WasDisposed); @@ -62,22 +67,23 @@ await queue.EnqueueAsync(new WebHookNotification [Fact] public async Task ProcessQueueEntryAsync_TransportFailure_PreservesOriginalException() { + // Arrange var failure = new HttpRequestException("Connection failed", new IOException("Socket closed")); using var handler = new FailingHandler(failure); using var client = new HttpClient(handler, disposeHandler: false); var options = GetService(); var hook = new WebHook { - Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", + Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, Url = "https://example.com/webhook", Version = WebHook.KnownVersions.Version2, EventTypes = [WebHook.KnownEventTypes.NewError] }; - var repository = DispatchProxy.Create(); - ((HookRepositoryProxy)(object)repository).Hook = hook; + var repository = new Mock(MockBehavior.Strict); + repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); using var logs = new CaptureLoggerFactory(); - using var job = new WebHooksJob(queue, GetService(), GetService(), repository, + using var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, cache, GetService(), GetService(), options, TimeProvider, GetService(), logs, new SingleClientFactory(client)); await queue.EnqueueAsync(new WebHookNotification @@ -86,8 +92,11 @@ await queue.EnqueueAsync(new WebHookNotification Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } }); + // Act var result = await job.RunAsync(TestCancellationToken); + // Assert + Assert.Same(failure, result.Error); Assert.Contains(logs.Exceptions, exception => ReferenceEquals(exception, failure)); } @@ -95,6 +104,7 @@ await queue.EnqueueAsync(new WebHookNotification [Fact] public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsRedirectWithoutOverrides() { + // Arrange int delivered = 0; int redirected = 0; var builder = WebApplication.CreateBuilder(); @@ -114,16 +124,16 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR var options = GetService(); var hook = new WebHook { - Id = "222222222222222222222222", OrganizationId = "333333333333333333333333", ProjectId = "444444444444444444444444", + Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, Url = address + "/private-capability?secret-query=value", Version = WebHook.KnownVersions.Version2, EventTypes = [WebHook.KnownEventTypes.NewError] }; - var repository = DispatchProxy.Create(); - ((HookRepositoryProxy)(object)repository).Hook = hook; + var repository = new Mock(MockBehavior.Strict); + repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); using var logs = new CaptureLoggerFactory(); - using var job = new WebHooksJob(queue, GetService(), GetService(), repository, + using var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, cache, GetService(), GetService(), options, TimeProvider, GetService(), logs, GetService()); await queue.EnqueueAsync(new WebHookNotification @@ -131,7 +141,10 @@ await queue.EnqueueAsync(new WebHookNotification OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } }); + // Act var result = await job.RunAsync(TestCancellationToken); + + // Assert Assert.True(result.IsSuccess, result.Error?.ToString()); Assert.Equal(1, delivered); Assert.Equal(1, redirected); @@ -143,14 +156,18 @@ await queue.EnqueueAsync(new WebHookNotification [Fact] public void DeliveryLogging_RedactsDestinationAndPreservesException() { + // Arrange using var capture = new CaptureLoggerFactory(); const string url = "https://user:password@example.com/private-capability?secret-query=value"; var exception = new HttpRequestException("Connection failed", new IOException("Socket closed")); + // Act Exceptionless.Core.Extensions.LoggerExtensions.RecordWebHook(capture, "hook", "project", url); Exceptionless.Core.Extensions.LoggerExtensions.WebHookComplete(capture, HttpStatusCode.OK, "organization", "project", url); Exceptionless.Core.Extensions.LoggerExtensions.WebHookError(capture, HttpStatusCode.BadGateway, "organization", "project", url, exception); Exceptionless.Core.Extensions.LoggerExtensions.WebHookTimeout(capture, null, "organization", "project", url, exception); Exceptionless.Core.Extensions.LoggerExtensions.WebHookDisabledStatusCode(capture, "hook", HttpStatusCode.Gone, "organization", "project", url); + + // Assert Assert.Equal(5, capture.Messages.Count); Assert.Equal(2, capture.Exceptions.Count); Assert.All(capture.Exceptions, logged => Assert.Same(exception, logged)); @@ -163,16 +180,6 @@ public void DeliveryLogging_RedactsDestinationAndPreservesException() }); } - private class HookRepositoryProxy : DispatchProxy - { - public WebHook Hook { get; set; } = null!; - protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) - { - Assert.Equal("GetByIdAsync", targetMethod!.Name); - return Task.FromResult(Hook); - } - } - private sealed class CaptureLoggerFactory : ILoggerFactory, ILogger { public bool FailCompletion { get; init; } From ae5012b608ed1b12bf3addcd51436fe0159203b1 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 13 Oct 2026 22:36:57 -0500 Subject: [PATCH 04/26] Preserve safe webhook diagnostics and pooled client lifetimes --- docs/webhook-delivery.md | 6 +- src/Exceptionless.Core/Bootstrapper.cs | 4 +- .../Extensions/JsonExtensions.cs | 12 +++- src/Exceptionless.Core/Jobs/WebHooksJob.cs | 16 ++--- .../Services/WebHookDelivery.cs | 8 +++ .../Api/Handlers/WebHookHandler.cs | 12 +--- src/Exceptionless.Web/ApmExtensions.cs | 3 +- ...okDeliveryTests.cs => WebHooksJobTests.cs} | 72 ++++++++++--------- .../Logging/SensitiveDataLoggingTests.cs | 30 ++++++++ 9 files changed, 105 insertions(+), 58 deletions(-) rename tests/Exceptionless.Tests/Jobs/{WebHookDeliveryTests.cs => WebHooksJobTests.cs} (78%) diff --git a/docs/webhook-delivery.md b/docs/webhook-delivery.md index 812a935988..a376b74327 100644 --- a/docs/webhook-delivery.md +++ b/docs/webhook-delivery.md @@ -1,5 +1,7 @@ -Webhooks deliver to user-supplied HTTP or HTTPS URLs, including private receivers. No destination allowlist or private-network override is required. Redirects and proxy settings use the standard HTTP client behavior. Cookies are not shared between deliveries. +Webhook delivery uses the configured URL unchanged, including its path and query string. Existing private receivers, redirects, and proxy settings remain supported. Cookies returned by one receiver are not stored or sent with later deliveries. -Delivery logs identify the destination origin instead of recording URL credentials, paths, or queries. Automatic HTTP delivery traces and client logs are omitted. Failures retain the original exception and message for troubleshooting; exception details are not redacted. +The `Destination` log field and HTTP trace URL contain the origin (scheme, host, and port). Webhook paths and queries can contain credentials; for example, Slack treats its incoming webhook URL as a secret. Use the webhook identifier to find the complete configured URL when troubleshooting. + +Delivery logs retain organization/project context, status, and original exceptions. HTTP traces retain method, duration, status, and error information. Only duplicate framework HTTP client logs are disabled because their URL paths are not redacted by default. Exception details are preserved, so this is not a guarantee that every diagnostic string is redacted. Only response headers are needed to determine delivery status. Response bodies are not buffered, and responses are disposed even when failure bookkeeping throws. diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index a940cb8fe0..921ec0ad8d 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Text.Json; @@ -225,11 +226,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO DecompressionMethods.None)); services.AddSingleton(); services.AddHttpClient(WebHookDelivery.HttpClientName) + // Job logs retain delivery diagnostics without logging credential-bearing URL paths. .RemoveAllLoggers() .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { UseCookies = false, - ActivityHeadersPropagator = null + ActivityHeadersPropagator = DistributedContextPropagator.CreateNoOutputPropagator() }); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Core/Extensions/JsonExtensions.cs b/src/Exceptionless.Core/Extensions/JsonExtensions.cs index 3fdda1cf50..810e610500 100644 --- a/src/Exceptionless.Core/Extensions/JsonExtensions.cs +++ b/src/Exceptionless.Core/Extensions/JsonExtensions.cs @@ -1,4 +1,6 @@ -namespace Exceptionless.Core.Extensions; +using System.Text.Json; + +namespace Exceptionless.Core.Extensions; /// /// Extension methods for JSON operations using System.Text.Json. @@ -6,6 +8,14 @@ /// public static class JsonExtensions { + /// + /// Returns a string property, or null when the object, property, or value is not a string. + /// + public static string? GetStringProperty(this JsonElement data, string name) + => data.ValueKind == JsonValueKind.Object && data.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + /// /// Checks if a string contains JSON content (starts with { or [). /// diff --git a/src/Exceptionless.Core/Jobs/WebHooksJob.cs b/src/Exceptionless.Core/Jobs/WebHooksJob.cs index 175ae2ab45..2e53d7aeeb 100644 --- a/src/Exceptionless.Core/Jobs/WebHooksJob.cs +++ b/src/Exceptionless.Core/Jobs/WebHooksJob.cs @@ -19,7 +19,7 @@ namespace Exceptionless.Core.Jobs; [Job(Description = "Processes queued web hook messages.", InitialDelay = "5s")] -public class WebHooksJob : QueueJobBase, IDisposable +public class WebHooksJob : QueueJobBase { private const string ConsecutiveErrorsCacheKey = "errors"; private const string FirstAttemptCacheKey = "first-attempt"; @@ -40,7 +40,7 @@ public class WebHooksJob : QueueJobBase, IDisposable private readonly JsonSerializerOptions _versionOneJsonOptions; private readonly AppOptions _appOptions; - private readonly HttpClient _client; + private readonly IHttpClientFactory _httpClientFactory; public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, ITextSerializer serializer, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) @@ -53,7 +53,7 @@ public WebHooksJob(IQueue queue, IProjectRepository project _jsonOptions = jsonOptions; _versionOneJsonOptions = VersionOnePlugin.CreateJsonSerializerOptions(jsonOptions); _appOptions = appOptions; - _client = httpClientFactory.CreateClient(WebHookDelivery.HttpClientName); + _httpClientFactory = httpClientFactory; } protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) @@ -61,7 +61,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex var body = context.QueueEntry.Value; bool shouldLog = body.ProjectId != _appOptions.InternalProjectId; - using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) + using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId).Property("WebHookId", body.WebHookId))) { if (shouldLog) _logger.RecordWebHook(context.QueueEntry.Id, body.ProjectId, body.Url); @@ -102,7 +102,8 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex Content = JsonContent.Create(body.Data, options: jsonOptions) }; request.Options.Set(WebHookDelivery.DeliveryRequest, true); - response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, postCancellationTokenSource.Token); + using var client = _httpClientFactory.CreateClient(WebHookDelivery.HttpClientName); + response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, postCancellationTokenSource.Token); if (!response.IsSuccessStatusCode) successful = false; else if (consecutiveErrors > 0) @@ -220,9 +221,4 @@ private static string GetCacheKeyScope(WebHookNotification body) { return String.Concat("Project:", body.ProjectId, ":webhook:", body.Type == WebHookType.Slack ? "slack" : body.WebHookId); } - - public void Dispose() - { - _client.Dispose(); - } } diff --git a/src/Exceptionless.Core/Services/WebHookDelivery.cs b/src/Exceptionless.Core/Services/WebHookDelivery.cs index 2a5a519c0f..cc3bf804c1 100644 --- a/src/Exceptionless.Core/Services/WebHookDelivery.cs +++ b/src/Exceptionless.Core/Services/WebHookDelivery.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; + namespace Exceptionless.Core.Services; public static class WebHookDelivery @@ -5,6 +7,12 @@ public static class WebHookDelivery public const string HttpClientName = "WebHooks"; public static readonly HttpRequestOptionsKey DeliveryRequest = new("Exceptionless.WebHookDelivery"); + public static void EnrichActivity(Activity activity, HttpRequestMessage request) + { + if (request.Options.TryGetValue(DeliveryRequest, out bool delivery) && delivery) + activity.SetTag("url.full", GetLoggingDestination(request.RequestUri?.AbsoluteUri)); + } + public static string GetLoggingDestination(string? value) => Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) ? uri.GetComponents(UriComponents.SchemeAndServer, UriFormat.UriEscaped) diff --git a/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs b/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs index 4ee0f72eeb..a7bcdf3c87 100644 --- a/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/WebHookHandler.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; -using System.Text.Json; using Exceptionless.Core.Authorization; using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; @@ -88,8 +87,8 @@ public async Task> Handle(DeleteWebHooks message) public async Task> Handle(SubscribeWebHook message) { - string? eventType = GetStringProperty(message.Data.RootElement, "event"); - string? url = GetStringProperty(message.Data.RootElement, "target_url"); + string? eventType = message.Data.RootElement.GetStringProperty("event"); + string? url = message.Data.RootElement.GetStringProperty("target_url"); if (String.IsNullOrEmpty(eventType) || String.IsNullOrEmpty(url)) return Result.BadRequest("Webhook subscription event and target_url are required."); @@ -118,7 +117,7 @@ public async Task> Handle(DeleteWebHooks message) public async Task Handle(UnsubscribeWebHook message) { - string? targetUrl = GetStringProperty(message.Data.RootElement, "target_url"); + string? targetUrl = message.Data.RootElement.GetStringProperty("target_url"); if (!IsZapierTarget(targetUrl)) return Result.NotFound("Webhook target not found."); @@ -136,11 +135,6 @@ public async Task Handle(UnsubscribeWebHook message) return Result.Success(); } - private static string? GetStringProperty(JsonElement data, string name) - => data.ValueKind == JsonValueKind.Object && data.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String - ? value.GetString() - : null; - private static bool IsZapierTarget([NotNullWhen(true)] string? url) { return url is not null diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 3bc33b9e7c..fbc3842e74 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -72,8 +72,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }; }); - b.AddHttpClientInstrumentation(o => o.FilterHttpRequestMessage = request => - !request.Options.TryGetValue(WebHookDelivery.DeliveryRequest, out bool delivery) || !delivery); + b.AddHttpClientInstrumentation(o => o.EnrichWithHttpRequestMessage = WebHookDelivery.EnrichActivity); b.AddSource("Exceptionless", "Exceptionless.Assistant", "Foundatio"); if (config.EnableRedis) diff --git a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs b/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs similarity index 78% rename from tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs rename to tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs index 839991e3e0..c9a8c237f1 100644 --- a/tests/Exceptionless.Tests/Jobs/WebHookDeliveryTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs @@ -1,3 +1,5 @@ +using System.Collections.Concurrent; +using System.Diagnostics; using System.Net; using System.Text.Json; using Exceptionless.Core; @@ -18,11 +20,13 @@ using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; using Moq; +using OpenTelemetry; +using OpenTelemetry.Trace; using Xunit; namespace Exceptionless.Tests.Jobs; -public sealed class WebHookDeliveryTests(ITestOutputHelper output) : TestWithServices(output) +public sealed class WebHooksJobTests(ITestOutputHelper output) : TestWithServices(output) { [Theory] [InlineData(false)] @@ -45,7 +49,7 @@ public async Task ProcessQueueEntryAsync_Response_DisposesWithoutBufferingEvenWh using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); using var logs = new CaptureLoggerFactory { FailCompletion = failCompletion }; - using var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, + var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, cache, GetService(), GetService(), options, TimeProvider, GetService(), logs, new SingleClientFactory(client)); await queue.EnqueueAsync(new WebHookNotification @@ -83,7 +87,7 @@ public async Task ProcessQueueEntryAsync_TransportFailure_PreservesOriginalExcep using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); using var logs = new CaptureLoggerFactory(); - using var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, + var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, cache, GetService(), GetService(), options, TimeProvider, GetService(), logs, new SingleClientFactory(client)); await queue.EnqueueAsync(new WebHookNotification @@ -96,7 +100,6 @@ await queue.EnqueueAsync(new WebHookNotification var result = await job.RunAsync(TestCancellationToken); // Assert - Assert.Same(failure, result.Error); Assert.Contains(logs.Exceptions, exception => ReferenceEquals(exception, failure)); } @@ -107,6 +110,12 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR // Arrange int delivered = 0; int redirected = 0; + var cookies = new ConcurrentQueue(); + var activities = new ConcurrentQueue(); + using var tracing = Sdk.CreateTracerProviderBuilder() + .AddHttpClientInstrumentation(options => options.EnrichWithHttpRequestMessage = WebHookDelivery.EnrichActivity) + .AddProcessor(new CaptureActivityProcessor(activities)) + .Build(); var builder = WebApplication.CreateBuilder(); builder.Logging.ClearProviders(); builder.WebHost.UseKestrel().UseUrls("http://127.0.0.1:0"); @@ -114,6 +123,8 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR server.MapPost("/private-capability", context => { Interlocked.Increment(ref delivered); + cookies.Enqueue(context.Request.Headers.Cookie.ToString()); + context.Response.Headers.SetCookie = "receiver-session=private; Path=/"; context.Response.StatusCode = 302; context.Response.Headers.Location = "/redirected-capability"; return Task.CompletedTask; @@ -133,51 +144,46 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); using var logs = new CaptureLoggerFactory(); - using var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, + var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, cache, GetService(), GetService(), options, TimeProvider, GetService(), logs, GetService()); - await queue.EnqueueAsync(new WebHookNotification + var notification = new WebHookNotification { OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } - }); + }; + await queue.EnqueueAsync(notification); + // Act var result = await job.RunAsync(TestCancellationToken); + await queue.EnqueueAsync(notification); + var secondResult = await job.RunAsync(TestCancellationToken); // Assert Assert.True(result.IsSuccess, result.Error?.ToString()); - Assert.Equal(1, delivered); - Assert.Equal(1, redirected); + Assert.True(secondResult.IsSuccess, secondResult.Error?.ToString()); + Assert.Equal(2, delivered); + Assert.Equal(2, redirected); + Assert.All(cookies, cookie => Assert.Empty(cookie)); + var deliveries = activities.Where(activity => Equals(activity.GetTagItem("server.port"), new Uri(address).Port)).ToList(); + Assert.Equal(4, deliveries.Count); + Assert.Equal(2, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "POST", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 302))); + Assert.Equal(2, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "GET", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 200))); + Assert.All(deliveries, activity => + { + Assert.Equal(address, activity.GetTagItem("url.full")); + Assert.True(activity.Duration > TimeSpan.Zero); + Assert.DoesNotContain(activity.TagObjects, tag => tag.Value?.ToString()?.Contains("capability", StringComparison.Ordinal) is true); + Assert.DoesNotContain(activity.TagObjects, tag => tag.Value?.ToString()?.Contains("secret-query", StringComparison.Ordinal) is true); + }); Assert.True(hook.IsEnabled); Assert.DoesNotContain(logs.Messages, message => message.Contains("private-capability") || message.Contains("secret-query")); await server.StopAsync(TestCancellationToken); } - [Fact] - public void DeliveryLogging_RedactsDestinationAndPreservesException() + private sealed class CaptureActivityProcessor(ConcurrentQueue activities) : BaseProcessor { - // Arrange - using var capture = new CaptureLoggerFactory(); - const string url = "https://user:password@example.com/private-capability?secret-query=value"; - var exception = new HttpRequestException("Connection failed", new IOException("Socket closed")); - // Act - Exceptionless.Core.Extensions.LoggerExtensions.RecordWebHook(capture, "hook", "project", url); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookComplete(capture, HttpStatusCode.OK, "organization", "project", url); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookError(capture, HttpStatusCode.BadGateway, "organization", "project", url, exception); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookTimeout(capture, null, "organization", "project", url, exception); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookDisabledStatusCode(capture, "hook", HttpStatusCode.Gone, "organization", "project", url); - - // Assert - Assert.Equal(5, capture.Messages.Count); - Assert.Equal(2, capture.Exceptions.Count); - Assert.All(capture.Exceptions, logged => Assert.Same(exception, logged)); - Assert.Contains(capture.Messages, message => message.Contains("Connection failed")); - Assert.All(capture.Messages, message => - { - Assert.DoesNotContain("private-capability", message); - Assert.DoesNotContain("secret-query", message); - Assert.DoesNotContain("password", message); - }); + public override void OnEnd(Activity activity) => activities.Enqueue(activity); } private sealed class CaptureLoggerFactory : ILoggerFactory, ILogger diff --git a/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs b/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs index a4f646139f..48cedaf8bd 100644 --- a/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs +++ b/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs @@ -15,6 +15,36 @@ namespace Exceptionless.Tests.Logging; public class SensitiveDataLoggingTests { + [Fact] + public void WebHookLogging_Destination_OmitsCredentialsAndPreservesException() + { + // Arrange + var sink = new CollectingSink(); + using var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger(); + using var factory = new Serilog.Extensions.Logging.SerilogLoggerFactory(logger); + var deliveryLogger = factory.CreateLogger("WebHooks"); + const string url = "https://user:password@example.com/private-capability?secret-query=value"; + var exception = new HttpRequestException("Connection failed", new IOException("Socket closed")); + + // Act + Exceptionless.Core.Extensions.LoggerExtensions.RecordWebHook(deliveryLogger, "hook", "project", url); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookComplete(deliveryLogger, System.Net.HttpStatusCode.OK, "organization", "project", url); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookError(deliveryLogger, System.Net.HttpStatusCode.BadGateway, "organization", "project", url, exception); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookTimeout(deliveryLogger, null, "organization", "project", url, exception); + Exceptionless.Core.Extensions.LoggerExtensions.WebHookDisabledStatusCode(deliveryLogger, "hook", System.Net.HttpStatusCode.Gone, "organization", "project", url); + + // Assert + Assert.Equal(5, sink.Events.Count); + Assert.Equal(2, sink.Events.Count(entry => ReferenceEquals(entry.Exception, exception))); + Assert.All(sink.Events, entry => + { + Assert.Equal("https://example.com", Assert.IsType(entry.Properties["Destination"]).Value); + Assert.DoesNotContain("private-capability", entry.RenderMessage()); + Assert.DoesNotContain("secret-query", entry.RenderMessage()); + Assert.DoesNotContain("password", entry.RenderMessage()); + }); + } + [Fact] public void ApplySensitiveDataLogging_AppOptions_PreservesSafeValuesAndRedactsNestedSecrets() { From 86a229f4c01e0f7cec3d1e9597133155a93b2c5a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 13 Oct 2026 22:45:03 -0500 Subject: [PATCH 05/26] Remove webhook delivery doc and use JSON pattern matching --- docs/webhook-delivery.md | 7 ------- src/Exceptionless.Core/Extensions/JsonExtensions.cs | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 docs/webhook-delivery.md diff --git a/docs/webhook-delivery.md b/docs/webhook-delivery.md deleted file mode 100644 index a376b74327..0000000000 --- a/docs/webhook-delivery.md +++ /dev/null @@ -1,7 +0,0 @@ -Webhook delivery uses the configured URL unchanged, including its path and query string. Existing private receivers, redirects, and proxy settings remain supported. Cookies returned by one receiver are not stored or sent with later deliveries. - -The `Destination` log field and HTTP trace URL contain the origin (scheme, host, and port). Webhook paths and queries can contain credentials; for example, Slack treats its incoming webhook URL as a secret. Use the webhook identifier to find the complete configured URL when troubleshooting. - -Delivery logs retain organization/project context, status, and original exceptions. HTTP traces retain method, duration, status, and error information. Only duplicate framework HTTP client logs are disabled because their URL paths are not redacted by default. Exception details are preserved, so this is not a guarantee that every diagnostic string is redacted. - -Only response headers are needed to determine delivery status. Response bodies are not buffered, and responses are disposed even when failure bookkeeping throws. diff --git a/src/Exceptionless.Core/Extensions/JsonExtensions.cs b/src/Exceptionless.Core/Extensions/JsonExtensions.cs index 810e610500..a6bb6e78c9 100644 --- a/src/Exceptionless.Core/Extensions/JsonExtensions.cs +++ b/src/Exceptionless.Core/Extensions/JsonExtensions.cs @@ -12,7 +12,7 @@ public static class JsonExtensions /// Returns a string property, or null when the object, property, or value is not a string. /// public static string? GetStringProperty(this JsonElement data, string name) - => data.ValueKind == JsonValueKind.Object && data.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String + => data.ValueKind is JsonValueKind.Object && data.TryGetProperty(name, out var value) && value.ValueKind is JsonValueKind.String ? value.GetString() : null; From 0a1fe694877074ab5761256a0f747a1b38c292d2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 13 Oct 2026 23:00:44 -0500 Subject: [PATCH 06/26] Prevent OTel context propagation to webhook receivers --- src/Exceptionless.Core/Bootstrapper.cs | 3 +- .../Services/WebHookDelivery.cs | 24 +++++++++++ .../Jobs/WebHooksJobTests.cs | 42 ++++++++++++++----- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 921ec0ad8d..92c2ad4c43 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Text.Json; @@ -231,7 +230,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { UseCookies = false, - ActivityHeadersPropagator = DistributedContextPropagator.CreateNoOutputPropagator() + ActivityHeadersPropagator = WebHookDelivery.ContextPropagator }); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Core/Services/WebHookDelivery.cs b/src/Exceptionless.Core/Services/WebHookDelivery.cs index cc3bf804c1..f5dfe92e0c 100644 --- a/src/Exceptionless.Core/Services/WebHookDelivery.cs +++ b/src/Exceptionless.Core/Services/WebHookDelivery.cs @@ -6,6 +6,7 @@ public static class WebHookDelivery { public const string HttpClientName = "WebHooks"; public static readonly HttpRequestOptionsKey DeliveryRequest = new("Exceptionless.WebHookDelivery"); + public static DistributedContextPropagator ContextPropagator { get; } = new WebHookContextPropagator(); public static void EnrichActivity(Activity activity, HttpRequestMessage request) { @@ -17,4 +18,27 @@ public static string GetLoggingDestination(string? value) => Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) ? uri.GetComponents(UriComponents.SchemeAndServer, UriFormat.UriEscaped) : "invalid"; + + private sealed class WebHookContextPropagator : DistributedContextPropagator + { + private readonly DistributedContextPropagator _noOutput = CreateNoOutputPropagator(); + + public override IReadOnlyCollection Fields => _noOutput.Fields; + + public override void Inject(Activity? activity, object? carrier, PropagatorSetterCallback? setter) + { + if (carrier is not HttpRequestMessage request) + return; + + // OTel may already have injected headers before the runtime calls this propagator. + foreach (string field in Fields) + request.Headers.Remove(field); + } + + public override void ExtractTraceIdAndState(object? carrier, PropagatorGetterCallback? getter, out string? traceId, out string? traceState) + => _noOutput.ExtractTraceIdAndState(carrier, getter, out traceId, out traceState); + + public override IEnumerable>? ExtractBaggage(object? carrier, PropagatorGetterCallback? getter) + => _noOutput.ExtractBaggage(carrier, getter); + } } diff --git a/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs b/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs index c9a8c237f1..523a5055ed 100644 --- a/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs @@ -104,15 +104,19 @@ await queue.EnqueueAsync(new WebHookNotification Assert.Contains(logs.Exceptions, exception => ReferenceEquals(exception, failure)); } - [Fact] - public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsRedirectWithoutOverrides() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversWithSafeTelemetry(bool sampled) { // Arrange int delivered = 0; int redirected = 0; var cookies = new ConcurrentQueue(); + var propagationHeaders = new ConcurrentQueue(); var activities = new ConcurrentQueue(); using var tracing = Sdk.CreateTracerProviderBuilder() + .SetSampler(sampled ? new AlwaysOnSampler() : new AlwaysOffSampler()) .AddHttpClientInstrumentation(options => options.EnrichWithHttpRequestMessage = WebHookDelivery.EnrichActivity) .AddProcessor(new CaptureActivityProcessor(activities)) .Build(); @@ -124,12 +128,17 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR { Interlocked.Increment(ref delivered); cookies.Enqueue(context.Request.Headers.Cookie.ToString()); + propagationHeaders.Enqueue(String.Concat(context.Request.Headers["traceparent"], context.Request.Headers["tracestate"], context.Request.Headers["baggage"])); context.Response.Headers.SetCookie = "receiver-session=private; Path=/"; context.Response.StatusCode = 302; context.Response.Headers.Location = "/redirected-capability"; return Task.CompletedTask; }); - server.Map("/redirected-capability", () => Interlocked.Increment(ref redirected)); + server.Map("/redirected-capability", (HttpContext context) => + { + propagationHeaders.Enqueue(String.Concat(context.Request.Headers["traceparent"], context.Request.Headers["tracestate"], context.Request.Headers["baggage"])); + return Interlocked.Increment(ref redirected); + }); await server.StartAsync(TestCancellationToken); string address = server.Services.GetRequiredService().Features.Get()!.Addresses.Single(); var options = GetService(); @@ -154,10 +163,21 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR }; await queue.EnqueueAsync(notification); - // Act - var result = await job.RunAsync(TestCancellationToken); - await queue.EnqueueAsync(notification); - var secondResult = await job.RunAsync(TestCancellationToken); + JobResult result; + JobResult secondResult; + var previousBaggage = Baggage.Current; + Baggage.SetBaggage("webhook-test", "private-baggage"); + try + { + // Act + result = await job.RunAsync(TestCancellationToken); + await queue.EnqueueAsync(notification); + secondResult = await job.RunAsync(TestCancellationToken); + } + finally + { + Baggage.Current = previousBaggage; + } // Assert Assert.True(result.IsSuccess, result.Error?.ToString()); @@ -165,10 +185,12 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversAndFollowsR Assert.Equal(2, delivered); Assert.Equal(2, redirected); Assert.All(cookies, cookie => Assert.Empty(cookie)); + Assert.Equal(4, propagationHeaders.Count); + Assert.All(propagationHeaders, header => Assert.Empty(header)); var deliveries = activities.Where(activity => Equals(activity.GetTagItem("server.port"), new Uri(address).Port)).ToList(); - Assert.Equal(4, deliveries.Count); - Assert.Equal(2, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "POST", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 302))); - Assert.Equal(2, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "GET", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 200))); + Assert.Equal(sampled ? 4 : 0, deliveries.Count); + Assert.Equal(sampled ? 2 : 0, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "POST", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 302))); + Assert.Equal(sampled ? 2 : 0, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "GET", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 200))); Assert.All(deliveries, activity => { Assert.Equal(address, activity.GetTagItem("url.full")); From 54fd0c0b96706a99fddabb255fd3e6cf33c496f1 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 14 Oct 2026 09:28:42 -0500 Subject: [PATCH 07/26] Simplify webhook disposal and reuse shared test logging --- src/Exceptionless.Core/Jobs/WebHooksJob.cs | 11 +- .../Services/WebHookDelivery.cs | 2 + src/Exceptionless.Web/ApmExtensions.cs | 1 + .../Api/Endpoints/WebHookEndpointTests.cs | 36 +++- .../Jobs/WebHooksJobTests.cs | 202 +++++++++--------- .../Logging/SensitiveDataLoggingTests.cs | 30 --- 6 files changed, 134 insertions(+), 148 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WebHooksJob.cs b/src/Exceptionless.Core/Jobs/WebHooksJob.cs index 2e53d7aeeb..f9b096bc5c 100644 --- a/src/Exceptionless.Core/Jobs/WebHooksJob.cs +++ b/src/Exceptionless.Core/Jobs/WebHooksJob.cs @@ -101,6 +101,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex { Content = JsonContent.Create(body.Data, options: jsonOptions) }; + // Mark this request so shared HTTP instrumentation redacts webhook URLs only. request.Options.Set(WebHookDelivery.DeliveryRequest, true); using var client = _httpClientFactory.CreateClient(WebHookDelivery.HttpClientName); response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, postCancellationTokenSource.Token); @@ -123,14 +124,16 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex } finally { - using var responseToDispose = response; + var statusCode = response?.StatusCode; + response?.Dispose(); + if (successful) { - _logger.WebHookComplete(response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); + _logger.WebHookComplete(statusCode, body.OrganizationId, body.ProjectId, body.Url); } - else if (response is not null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone)) + else if (statusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.Gone) { - _logger.WebHookDisabledStatusCode(body.Type == WebHookType.Slack ? "Slack" : body.WebHookId!, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url); + _logger.WebHookDisabledStatusCode(body.Type == WebHookType.Slack ? "Slack" : body.WebHookId!, statusCode, body.OrganizationId, body.ProjectId, body.Url); await DisableIntegrationAsync(body); await cache.RemoveAllAsync(_cacheKeys); } diff --git a/src/Exceptionless.Core/Services/WebHookDelivery.cs b/src/Exceptionless.Core/Services/WebHookDelivery.cs index f5dfe92e0c..f43ddb9f26 100644 --- a/src/Exceptionless.Core/Services/WebHookDelivery.cs +++ b/src/Exceptionless.Core/Services/WebHookDelivery.cs @@ -19,6 +19,8 @@ public static string GetLoggingDestination(string? value) ? uri.GetComponents(UriComponents.SchemeAndServer, UriFormat.UriEscaped) : "invalid"; + // External webhook receivers must not receive internal trace context or potentially sensitive baggage. + // https://opentelemetry.io/docs/concepts/signals/baggage/#baggage-security-considerations private sealed class WebHookContextPropagator : DistributedContextPropagator { private readonly DistributedContextPropagator _noOutput = CreateNoOutputPropagator(); diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index fbc3842e74..d31bb43890 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -72,6 +72,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }; }); + // Only marked webhook requests are changed; other HTTP spans keep their normal tags. b.AddHttpClientInstrumentation(o => o.EnrichWithHttpRequestMessage = WebHookDelivery.EnrichActivity); b.AddSource("Exceptionless", "Exceptionless.Assistant", "Foundatio"); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs index ff1ba34cf5..a6b13e35ca 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/WebHookEndpointTests.cs @@ -7,6 +7,7 @@ using Exceptionless.Core.Repositories; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; using Exceptionless.Web.Models; using FluentRest; using Foundatio.Repositories; @@ -569,16 +570,28 @@ public async Task Test_WithGetRequest_ReturnsZapierTestMessages() [InlineData("http://localhost/path")] [InlineData("http://169.254.169.254/path")] [InlineData("https://user:password@example.com/path")] - public Task PostAsync_UserSuppliedDestination_ReturnsCreated(string url) - => SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("webhooks") - .Content(new NewWebHook { ProjectId = SampleDataService.TEST_PROJECT_ID, Url = url, EventTypes = [WebHook.KnownEventTypes.NewError] }) - .StatusCodeShouldBeCreated()); + public async Task PostAsync_UserSuppliedDestination_ReturnsCreated(string url) + { + // Arrange + var webHook = new NewWebHook + { + OrganizationId = TestConstants.OrganizationId, + ProjectId = TestConstants.ProjectId, + Url = url, + EventTypes = [WebHook.KnownEventTypes.NewError] + }; + + // Act + var response = await SendRequestAsync(request => request.Post().AsTestOrganizationUser().AppendPath("webhooks").Content(webHook)); + + // Assert + Assert.Equal(HttpStatusCode.Created, response.StatusCode); + } [Fact] public async Task UnsubscribeAsync_ExistingZapierHook_RemovesWebHook() { // Arrange - create a zapier hook via subscribe - var logger = Assert.IsType(GetService()); const string zapierUrl = "https://hooks.zapier.com/hooks/unsubtest"; var webHook = await SendRequestAsAsync(r => r .Post() @@ -608,12 +621,13 @@ await SendRequestAsync(r => r await RefreshDataAsync(); var results = await _webHookRepository.GetByUrlAsync(zapierUrl); Assert.Empty(results.Documents); - var unsubscribeLogs = logger.LogEntries.Where(entry => entry.EventId.Name == "RemovingZapierUrls").ToList(); + var unsubscribeLogs = Log.LogEntries.Where(entry => String.Equals(entry.EventId.Name, nameof(Exceptionless.Web.Extensions.LoggerExtensions.RemovingZapierUrls), StringComparison.Ordinal)).ToList(); Assert.NotEmpty(unsubscribeLogs); Assert.All(unsubscribeLogs, entry => { Assert.DoesNotContain(zapierUrl, entry.Message); - Assert.DoesNotContain(entry.Properties.Values, value => String.Equals(value?.ToString(), zapierUrl, StringComparison.Ordinal)); + var properties = Assert.IsAssignableFrom>>(entry.State); + Assert.DoesNotContain(properties, property => String.Equals(property.Value?.ToString(), zapierUrl, StringComparison.Ordinal)); }); } @@ -626,7 +640,7 @@ public async Task SubscribeAsync_MalformedShape_ReturnsBadRequest(string path, s { // Act var response = await SendRequestAsync(request => request.Post().AsTestOrganizationClientUser() - .BaseUri(_server.BaseAddress).AppendPath(path).Content(json, "application/json").StatusCodeShouldBeBadRequest()); + .BaseUri(_server.BaseAddress).AppendPath(path).Content(json, "application/json")); // Assert Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); @@ -640,7 +654,7 @@ public async Task UnsubscribeAsync_MalformedShape_ReturnsNotFound(string path, s { // Act var response = await SendRequestAsync(request => request.Post().BaseUri(_server.BaseAddress) - .AppendPath(path).Content(json, "application/json").StatusCodeShouldBeNotFound()); + .AppendPath(path).Content(json, "application/json")); // Assert Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); @@ -657,7 +671,7 @@ public async Task SubscribeAsync_LookalikeZapierTarget_ReturnsNotFound(string ta // Act var response = await SendRequestAsync(request => request.Post().AsTestOrganizationClientUser() - .AppendPath("webhooks/subscribe").Content(subscription).StatusCodeShouldBeNotFound()); + .AppendPath("webhooks/subscribe").Content(subscription)); // Assert Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); @@ -672,7 +686,7 @@ public async Task UnsubscribeAsync_LookalikeZapierTarget_ReturnsNotFound(string { // Act var response = await SendRequestAsync(request => request.Post().AppendPath("webhooks/unsubscribe") - .Content(new { target_url = targetUrl }).StatusCodeShouldBeNotFound()); + .Content(new { target_url = targetUrl })); // Assert Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); diff --git a/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs b/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs index 523a5055ed..b117ff0f3a 100644 --- a/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WebHooksJobTests.cs @@ -3,6 +3,7 @@ using System.Net; using System.Text.Json; using Exceptionless.Core; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; using Exceptionless.Core.Queues.Models; @@ -16,9 +17,9 @@ using Foundatio.Resilience; using Foundatio.Serializer; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; using Moq; using OpenTelemetry; using OpenTelemetry.Trace; @@ -28,35 +29,25 @@ namespace Exceptionless.Tests.Jobs; public sealed class WebHooksJobTests(ITestOutputHelper output) : TestWithServices(output) { + private const string DestinationUrl = "https://example.com/webhook"; + [Theory] [InlineData(false)] [InlineData(true)] - public async Task ProcessQueueEntryAsync_Response_DisposesWithoutBufferingEvenWhenCompletionFails(bool failCompletion) + public async Task ProcessQueueEntryAsync_Response_DisposesWithoutBufferingEvenWhenBookkeepingFails(bool failBookkeeping) { // Arrange var content = new TrackingContent(); - using var handler = new StaticResponseHandler(new HttpResponseMessage(HttpStatusCode.OK) { Content = content }); - using var client = new HttpClient(handler, disposeHandler: false); - var options = GetService(); - var hook = new WebHook - { - Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, - Url = "https://example.com/webhook", Version = WebHook.KnownVersions.Version2, - EventTypes = [WebHook.KnownEventTypes.NewError] - }; - var repository = new Mock(MockBehavior.Strict); - repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); - using var queue = new InMemoryQueue(); + var failure = new IOException("Failure bookkeeping failed"); + using var handler = new DelegateHandler((_, _) => Task.FromResult(new HttpResponseMessage(failBookkeeping ? HttpStatusCode.BadGateway : HttpStatusCode.OK) { Content = content })); using var cache = new InMemoryCacheClient(); - using var logs = new CaptureLoggerFactory { FailCompletion = failCompletion }; - var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, - cache, GetService(), GetService(), options, TimeProvider, - GetService(), logs, new SingleClientFactory(client)); - await queue.EnqueueAsync(new WebHookNotification - { - OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, - Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } - }); + var failingCache = new Mock(MockBehavior.Strict); + failingCache.Setup(client => client.GetAsync(It.IsAny())).Returns((string key) => cache.GetAsync(key)); + failingCache.Setup(client => client.SetAsync(It.IsAny(), It.IsAny(), It.IsAny())).ThrowsAsync(failure); + using var queue = new InMemoryQueue(); + var hook = CreateWebHook(DestinationUrl); + var job = CreateJob(queue, hook, failBookkeeping ? failingCache.Object : cache, CreateClientFactory(handler)); + await queue.EnqueueAsync(CreateNotification(hook)); // Act var exception = await Record.ExceptionAsync(async () => { _ = await job.RunAsync(TestCancellationToken); }); @@ -65,7 +56,7 @@ await queue.EnqueueAsync(new WebHookNotification Assert.Equal(1, handler.RequestCount); Assert.False(content.WasSerialized); Assert.True(content.WasDisposed); - Assert.Equal(failCompletion ? "Completion logging failed" : null, exception?.Message); + Assert.Same(failBookkeeping ? failure : null, exception); } [Fact] @@ -73,35 +64,49 @@ public async Task ProcessQueueEntryAsync_TransportFailure_PreservesOriginalExcep { // Arrange var failure = new HttpRequestException("Connection failed", new IOException("Socket closed")); - using var handler = new FailingHandler(failure); - using var client = new HttpClient(handler, disposeHandler: false); - var options = GetService(); - var hook = new WebHook - { - Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, - Url = "https://example.com/webhook", Version = WebHook.KnownVersions.Version2, - EventTypes = [WebHook.KnownEventTypes.NewError] - }; - var repository = new Mock(MockBehavior.Strict); - repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); + using var handler = new DelegateHandler((_, _) => Task.FromException(failure)); using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); - using var logs = new CaptureLoggerFactory(); - var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, - cache, GetService(), GetService(), options, TimeProvider, - GetService(), logs, new SingleClientFactory(client)); - await queue.EnqueueAsync(new WebHookNotification - { - OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, - Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } - }); + var hook = CreateWebHook(DestinationUrl); + var job = CreateJob(queue, hook, cache, CreateClientFactory(handler)); + await queue.EnqueueAsync(CreateNotification(hook)); // Act var result = await job.RunAsync(TestCancellationToken); // Assert Assert.Same(failure, result.Error); - Assert.Contains(logs.Exceptions, exception => ReferenceEquals(exception, failure)); + var entry = Assert.Single(Log.LogEntries, entry => String.Equals(entry.EventId.Name, nameof(Exceptionless.Core.Extensions.LoggerExtensions.WebHookError), StringComparison.Ordinal)); + Assert.Same(failure, entry.Exception); + } + + [Fact] + public void WebHookLogging_Destination_OmitsCredentialsAndPreservesException() + { + // Arrange + Log.DefaultLogLevel = LogLevel.Trace; + const string url = "https://user:password@example.com/private-capability?secret-query=value"; + var exception = new HttpRequestException("Connection failed", new IOException("Socket closed")); + + // Act + _logger.RecordWebHook(TestConstants.WebHookId, TestConstants.ProjectId, url); + _logger.WebHookComplete(HttpStatusCode.OK, TestConstants.OrganizationId, TestConstants.ProjectId, url); + _logger.WebHookError(HttpStatusCode.BadGateway, TestConstants.OrganizationId, TestConstants.ProjectId, url, exception); + _logger.WebHookTimeout(null, TestConstants.OrganizationId, TestConstants.ProjectId, url, exception); + _logger.WebHookDisabledStatusCode(TestConstants.WebHookId, HttpStatusCode.Gone, TestConstants.OrganizationId, TestConstants.ProjectId, url); + + // Assert + var entries = Log.LogEntries.Where(entry => String.Equals(entry.CategoryName, typeof(WebHooksJobTests).FullName, StringComparison.Ordinal)).ToList(); + Assert.Equal(5, entries.Count); + Assert.Equal(2, entries.Count(entry => ReferenceEquals(entry.Exception, exception))); + Assert.All(entries, entry => + { + var properties = Assert.IsAssignableFrom>>(entry.State).ToDictionary(property => property.Key, property => property.Value); + Assert.Equal("https://example.com", properties["Destination"]); + Assert.DoesNotContain("private-capability", entry.Message); + Assert.DoesNotContain("secret-query", entry.Message); + Assert.DoesNotContain("password", entry.Message); + }); } [Theory] @@ -114,6 +119,7 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversWithSafeTel int redirected = 0; var cookies = new ConcurrentQueue(); var propagationHeaders = new ConcurrentQueue(); + var internalHeaders = new ConcurrentQueue<(string TraceParent, string Baggage)>(); var activities = new ConcurrentQueue(); using var tracing = Sdk.CreateTracerProviderBuilder() .SetSampler(sampled ? new AlwaysOnSampler() : new AlwaysOffSampler()) @@ -139,32 +145,24 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversWithSafeTel propagationHeaders.Enqueue(String.Concat(context.Request.Headers["traceparent"], context.Request.Headers["tracestate"], context.Request.Headers["baggage"])); return Interlocked.Increment(ref redirected); }); + server.MapGet("/internal-metadata", (HttpContext context) => + { + internalHeaders.Enqueue((context.Request.Headers["traceparent"].ToString(), context.Request.Headers["baggage"].ToString())); + return Results.NoContent(); + }); await server.StartAsync(TestCancellationToken); string address = server.Services.GetRequiredService().Features.Get()!.Addresses.Single(); - var options = GetService(); - var hook = new WebHook - { - Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, - Url = address + "/private-capability?secret-query=value", Version = WebHook.KnownVersions.Version2, - EventTypes = [WebHook.KnownEventTypes.NewError] - }; - var repository = new Mock(MockBehavior.Strict); - repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); + var hook = CreateWebHook(address + "/private-capability?secret-query=value"); using var queue = new InMemoryQueue(); using var cache = new InMemoryCacheClient(); - using var logs = new CaptureLoggerFactory(); - var job = new WebHooksJob(queue, GetService(), GetService(), repository.Object, - cache, GetService(), GetService(), options, TimeProvider, - GetService(), logs, GetService()); - var notification = new WebHookNotification - { - OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, - Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } - }; + var factory = GetService(); + var job = CreateJob(queue, hook, cache, factory); + using var internalClient = factory.CreateClient(); + var notification = CreateNotification(hook); await queue.EnqueueAsync(notification); - JobResult result; JobResult secondResult; + HttpStatusCode internalStatus; var previousBaggage = Baggage.Current; Baggage.SetBaggage("webhook-test", "private-baggage"); try @@ -173,6 +171,8 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversWithSafeTel result = await job.RunAsync(TestCancellationToken); await queue.EnqueueAsync(notification); secondResult = await job.RunAsync(TestCancellationToken); + using var internalResponse = await internalClient.GetAsync(address + "/internal-metadata", TestCancellationToken); + internalStatus = internalResponse.StatusCode; } finally { @@ -187,75 +187,71 @@ public async Task ProcessQueueEntryAsync_ExistingPrivateHook_DeliversWithSafeTel Assert.All(cookies, cookie => Assert.Empty(cookie)); Assert.Equal(4, propagationHeaders.Count); Assert.All(propagationHeaders, header => Assert.Empty(header)); - var deliveries = activities.Where(activity => Equals(activity.GetTagItem("server.port"), new Uri(address).Port)).ToList(); + int port = new Uri(address).Port; + var clientActivities = activities.Where(activity => Equals(activity.GetTagItem("server.port"), port)).ToList(); + Assert.Equal(sampled ? 5 : 0, clientActivities.Count); + var deliveries = clientActivities.Where(activity => String.Equals(activity.GetTagItem("url.full") as string, address, StringComparison.Ordinal)).ToList(); Assert.Equal(sampled ? 4 : 0, deliveries.Count); Assert.Equal(sampled ? 2 : 0, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "POST", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 302))); Assert.Equal(sampled ? 2 : 0, deliveries.Count(activity => String.Equals(activity.GetTagItem("http.request.method") as string, "GET", StringComparison.Ordinal) && Equals(activity.GetTagItem("http.response.status_code"), 200))); Assert.All(deliveries, activity => { - Assert.Equal(address, activity.GetTagItem("url.full")); Assert.True(activity.Duration > TimeSpan.Zero); Assert.DoesNotContain(activity.TagObjects, tag => tag.Value?.ToString()?.Contains("capability", StringComparison.Ordinal) is true); Assert.DoesNotContain(activity.TagObjects, tag => tag.Value?.ToString()?.Contains("secret-query", StringComparison.Ordinal) is true); }); + Assert.Equal(HttpStatusCode.NoContent, internalStatus); + var headers = Assert.Single(internalHeaders); + Assert.NotEmpty(headers.TraceParent); + Assert.Contains("private-baggage", headers.Baggage); + var internalActivities = clientActivities.Where(activity => activity.GetTagItem("url.full") is string url && url.StartsWith(address + "/internal-metadata", StringComparison.Ordinal)).ToList(); + Assert.Equal(sampled ? 1 : 0, internalActivities.Count); Assert.True(hook.IsEnabled); - Assert.DoesNotContain(logs.Messages, message => message.Contains("private-capability") || message.Contains("secret-query")); + Assert.DoesNotContain(Log.LogEntries, entry => entry.Message.Contains("private-capability", StringComparison.Ordinal) || entry.Message.Contains("secret-query", StringComparison.Ordinal)); await server.StopAsync(TestCancellationToken); } - private sealed class CaptureActivityProcessor(ConcurrentQueue activities) : BaseProcessor + private WebHooksJob CreateJob(IQueue queue, WebHook hook, ICacheClient cache, IHttpClientFactory factory) { - public override void OnEnd(Activity activity) => activities.Enqueue(activity); + var repository = new Mock(MockBehavior.Strict); + repository.Setup(value => value.GetByIdAsync(hook.Id, It.IsAny>())).ReturnsAsync(hook); + return new WebHooksJob(queue, GetService(), GetService(), repository.Object, + cache, GetService(), GetService(), GetService(), TimeProvider, + GetService(), Log, factory); } - private sealed class CaptureLoggerFactory : ILoggerFactory, ILogger + private static WebHook CreateWebHook(string url) => new() { - public bool FailCompletion { get; init; } - public List Messages { get; } = []; - public List Exceptions { get; } = []; - public ILogger CreateLogger(string categoryName) => this; - public void AddProvider(ILoggerProvider provider) { } - public IDisposable? BeginScope(TState state) where TState : notnull => null; - public bool IsEnabled(LogLevel logLevel) => true; - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) - { - if (FailCompletion && eventId.Id == 5) - throw new InvalidOperationException("Completion logging failed"); - if (exception is not null) - Exceptions.Add(exception); - string values = state is IEnumerable> properties - ? String.Join(" ", properties.Select(p => p.Value)) : String.Empty; - Messages.Add(formatter(state, exception) + values + exception); - } - public void Dispose() { } - } + Id = TestConstants.WebHookId, OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, + Url = url, Version = WebHook.KnownVersions.Version2, EventTypes = [WebHook.KnownEventTypes.NewError] + }; + + private static WebHookNotification CreateNotification(WebHook hook) => new() + { + OrganizationId = hook.OrganizationId, ProjectId = hook.ProjectId, WebHookId = hook.Id, + Type = WebHookType.General, Url = hook.Url, Data = new { synthetic = "delivery-test" } + }; - private sealed class SingleClientFactory(HttpClient client) : IHttpClientFactory + private static IHttpClientFactory CreateClientFactory(HttpMessageHandler handler) { - public HttpClient CreateClient(string name) => client; + var factory = new Mock(MockBehavior.Strict); + factory.Setup(value => value.CreateClient(WebHookDelivery.HttpClientName)).Returns(() => new HttpClient(handler, disposeHandler: false)); + return factory.Object; } - private sealed class FailingHandler(Exception exception) : HttpMessageHandler + private sealed class CaptureActivityProcessor(ConcurrentQueue activities) : BaseProcessor { - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - => Task.FromException(exception); + public override void OnEnd(Activity activity) => activities.Enqueue(activity); } - private sealed class StaticResponseHandler(HttpResponseMessage response) : HttpMessageHandler + private sealed class DelegateHandler(Func> send) : HttpMessageHandler { public int RequestCount { get; private set; } protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { RequestCount++; - return Task.FromResult(response); - } - - protected override void Dispose(bool disposing) - { - if (disposing) - response.Dispose(); - base.Dispose(disposing); + return send(request, cancellationToken); } } diff --git a/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs b/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs index 48cedaf8bd..a4f646139f 100644 --- a/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs +++ b/tests/Exceptionless.Tests/Logging/SensitiveDataLoggingTests.cs @@ -15,36 +15,6 @@ namespace Exceptionless.Tests.Logging; public class SensitiveDataLoggingTests { - [Fact] - public void WebHookLogging_Destination_OmitsCredentialsAndPreservesException() - { - // Arrange - var sink = new CollectingSink(); - using var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger(); - using var factory = new Serilog.Extensions.Logging.SerilogLoggerFactory(logger); - var deliveryLogger = factory.CreateLogger("WebHooks"); - const string url = "https://user:password@example.com/private-capability?secret-query=value"; - var exception = new HttpRequestException("Connection failed", new IOException("Socket closed")); - - // Act - Exceptionless.Core.Extensions.LoggerExtensions.RecordWebHook(deliveryLogger, "hook", "project", url); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookComplete(deliveryLogger, System.Net.HttpStatusCode.OK, "organization", "project", url); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookError(deliveryLogger, System.Net.HttpStatusCode.BadGateway, "organization", "project", url, exception); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookTimeout(deliveryLogger, null, "organization", "project", url, exception); - Exceptionless.Core.Extensions.LoggerExtensions.WebHookDisabledStatusCode(deliveryLogger, "hook", System.Net.HttpStatusCode.Gone, "organization", "project", url); - - // Assert - Assert.Equal(5, sink.Events.Count); - Assert.Equal(2, sink.Events.Count(entry => ReferenceEquals(entry.Exception, exception))); - Assert.All(sink.Events, entry => - { - Assert.Equal("https://example.com", Assert.IsType(entry.Properties["Destination"]).Value); - Assert.DoesNotContain("private-capability", entry.RenderMessage()); - Assert.DoesNotContain("secret-query", entry.RenderMessage()); - Assert.DoesNotContain("password", entry.RenderMessage()); - }); - } - [Fact] public void ApplySensitiveDataLogging_AppOptions_PreservesSafeValuesAndRedactsNestedSecrets() { From 47f8904744b2701cf9a575e099b0105a2896fedf Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Mon, 12 Oct 2026 21:24:13 -0500 Subject: [PATCH 08/26] Configure trusted forwarded header peers --- docs/forwarded-headers.md | 14 ++++ .../ForwardedHeadersConfiguration.cs | 27 +++++++ .../Extensions/HttpExtensions.cs | 3 +- src/Exceptionless.Web/Program.cs | 7 +- .../ForwardedHeadersConfigurationTests.cs | 75 +++++++++++++++++++ .../Extensions/HttpExtensionsTests.cs | 12 +++ 6 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 docs/forwarded-headers.md create mode 100644 src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs create mode 100644 tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs diff --git a/docs/forwarded-headers.md b/docs/forwarded-headers.md new file mode 100644 index 0000000000..7e88ac6c35 --- /dev/null +++ b/docs/forwarded-headers.md @@ -0,0 +1,14 @@ +The API uses forwarded client addresses and schemes only from trusted peers. Framework loopback defaults remain enabled; non-loopback ingress peers must be configured before deployment. + +Configure the actual proxy addresses or their smallest required network and the number of forwarding hops: + +```yaml +ForwardedHeaders: + KnownProxies: + - 10.20.30.40 + KnownNetworks: + - 10.20.40.0/24 + ForwardLimit: 1 +``` + +Leave unused lists empty. The ingress must replace incoming forwarded headers rather than preserve client-supplied values. Public access should terminate at that ingress. With no additional configuration, headers from other peers are ignored. IPv4-mapped IPv6 client addresses use the same login window as their IPv4 equivalent. diff --git a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs new file mode 100644 index 0000000000..7f25e37988 --- /dev/null +++ b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs @@ -0,0 +1,27 @@ +using System.Net; +using Microsoft.AspNetCore.HttpOverrides; + +namespace Exceptionless.Web.Extensions; + +public static class ForwardedHeadersConfiguration +{ + public static void Configure(ForwardedHeadersOptions options, IConfiguration configuration) + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + options.RequireHeaderSymmetry = false; + options.ForwardLimit = configuration.GetValue("ForwardedHeaders:ForwardLimit", 1); + if (options.ForwardLimit < 1) + throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be positive."); + + // Retain the framework's loopback defaults and add explicitly trusted ingress peers. + foreach (string address in configuration.GetSection("ForwardedHeaders:KnownProxies").Get() ?? []) + options.KnownProxies.Add(IPAddress.Parse(address)); + foreach (string value in configuration.GetSection("ForwardedHeaders:KnownNetworks").Get() ?? []) + { + var network = System.Net.IPNetwork.Parse(value); + if (network.PrefixLength == 0) + throw new InvalidOperationException("ForwardedHeaders:KnownNetworks must identify trusted ingress networks."); + options.KnownIPNetworks.Add(network); + } + } +} diff --git a/src/Exceptionless.Web/Extensions/HttpExtensions.cs b/src/Exceptionless.Web/Extensions/HttpExtensions.cs index 80b4fb3fd5..c32cddb481 100644 --- a/src/Exceptionless.Web/Extensions/HttpExtensions.cs +++ b/src/Exceptionless.Web/Extensions/HttpExtensions.cs @@ -138,7 +138,8 @@ public static ICollection GetAssociatedOrganizationIds(this HttpRequest { ArgumentNullException.ThrowIfNull(request); - return request.HttpContext.Connection.RemoteIpAddress?.ToString(); + var address = request.HttpContext.Connection.RemoteIpAddress; + return address is { IsIPv4MappedToIPv6: true } ? address.MapToIPv4().ToString() : address?.ToString(); } public static string? GetClientKeyHash(this HttpRequest request) diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index b64a9adeef..ab124a9212 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -134,10 +134,7 @@ public static async Task Main(string[] args) builder.Services.Configure(o => { - o.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; - o.RequireHeaderSymmetry = false; - o.KnownIPNetworks.Clear(); - o.KnownProxies.Clear(); + ForwardedHeadersConfiguration.Configure(o, builder.Configuration); }); builder.Services.ConfigureHttpJsonOptions(o => @@ -231,6 +228,7 @@ ApplicationException applicationException when applicationException.Message.Cont } }); app.UseStatusCodePages(WriteProblemDetailsStatusCodeResponseAsync); + app.UseForwardedHeaders(); app.UseHealthChecks("/health", new HealthCheckOptions { @@ -350,7 +348,6 @@ ApplicationException applicationException when applicationException.Message.Cont app.UseMiddleware(); app.UseCors("AllowAny"); app.UseHttpMethodOverride(); - app.UseForwardedHeaders(); app.UseAuthentication(); app.UseAuthorization(); diff --git a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs new file mode 100644 index 0000000000..b4a80646f7 --- /dev/null +++ b/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs @@ -0,0 +1,75 @@ +using System.Net; +using Exceptionless.Web.Extensions; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Exceptionless.Tests.Extensions; + +public sealed class ForwardedHeadersConfigurationTests +{ + [Theory] + [InlineData(false, "198.51.100.20")] + [InlineData(true, "203.0.113.40")] + public async Task Invoke_ForwardedAddress_OnlyUsesConfiguredPeer(bool trusted, string expected) + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ForwardedHeaders:KnownProxies:0"] = trusted ? "198.51.100.20" : "192.0.2.10" + }).Build(); + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, configuration); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse("198.51.100.20"); + context.Request.Scheme = "http"; + context.Request.Headers["X-Forwarded-For"] = "203.0.113.40"; + context.Request.Headers["X-Forwarded-Proto"] = "https"; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + await middleware.Invoke(context); + Assert.Equal(expected, context.Request.GetClientIpAddress()); + Assert.Equal(trusted ? "https" : "http", context.Request.Scheme); + } + + [Fact] + public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer() + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ForwardedHeaders:KnownNetworks:0"] = "192.0.2.0/24", + ["ForwardedHeaders:ForwardLimit"] = "2" + }).Build(); + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, configuration); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse("192.0.2.10"); + context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40, 192.0.2.11"; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + await middleware.Invoke(context); + Assert.Equal("203.0.113.40", context.Request.GetClientIpAddress()); + } + + [Theory] + [InlineData("0.0.0.0/0")] + [InlineData("::/0")] + public void Configure_UnboundedTrustedNetwork_Throws(string network) + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ForwardedHeaders:KnownNetworks:0"] = network + }).Build(); + Assert.Throws(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + } + + [Fact] + public void Configure_Defaults_RetainsLoopbackTrustAndOneHopLimit() + { + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + Assert.NotEmpty(options.KnownProxies); + Assert.NotEmpty(options.KnownIPNetworks); + Assert.Equal(1, options.ForwardLimit); + } +} diff --git a/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs b/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs index 352e066be8..1f4a25b5bc 100644 --- a/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs +++ b/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs @@ -33,6 +33,18 @@ public void GetBasicAuth_InvalidHeader_ReturnsNull(string? authorization) Assert.Null(credentials); } + [Theory] + [InlineData("192.0.2.1", "192.0.2.1")] + [InlineData("::ffff:192.0.2.1", "192.0.2.1")] + [InlineData("2001:db8::1", "2001:db8::1")] + [InlineData(null, null)] + public void GetClientIpAddress_Address_NormalizesMappedIpv4(string? value, string? expected) + { + var request = new DefaultHttpContext().Request; + request.HttpContext.Connection.RemoteIpAddress = value is null ? null : System.Net.IPAddress.Parse(value); + Assert.Equal(expected, request.GetClientIpAddress()); + } + [Theory] [InlineData("Basic", "user@example.com", "password")] [InlineData("bAsIc", "user@example.com", "password")] From 9679428488c94ab7241d1f4e5fc40779c1cb1147 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Mon, 12 Oct 2026 22:23:07 -0500 Subject: [PATCH 09/26] Clarify forwarded header compatibility and align code style --- docs/forwarded-headers.md | 12 ++++++++++++ .../Extensions/ForwardedHeadersConfiguration.cs | 6 ++++-- src/Exceptionless.Web/Extensions/HttpExtensions.cs | 1 + 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/forwarded-headers.md b/docs/forwarded-headers.md index 7e88ac6c35..2768d9483a 100644 --- a/docs/forwarded-headers.md +++ b/docs/forwarded-headers.md @@ -12,3 +12,15 @@ ForwardedHeaders: ``` Leave unused lists empty. The ingress must replace incoming forwarded headers rather than preserve client-supplied values. Public access should terminate at that ingress. With no additional configuration, headers from other peers are ignored. IPv4-mapped IPv6 client addresses use the same login window as their IPv4 equivalent. + +## Compatibility and deployment + +This changes the previous trust-all behavior. Before upgrading a deployment behind a non-loopback proxy, configure the peer addresses as seen by the API and verify the actual forwarding chain. Otherwise, forwarded client IPs and HTTPS schemes are ignored: IP-based login, signup, password-reset and anonymous API limits can group users behind the proxy, and scheme-dependent redirects can behave incorrectly. Authenticated API throttling still uses its existing user or organization identifier. No endpoint routes, payload schemas, API keys, or WebSocket message formats change. + +Direct self-hosted installations do not need a proxy or additional configuration. `ForwardLimit: 1` is the maximum number of forwarded entries processed, not a minimum number of proxies required on each request. Requests without forwarded headers retain their connection IP and scheme. Multiple trusted proxies require a limit matching the intended chain and trust configuration for each traversed proxy. + +The current application configuration requires a positive integer. An explicit zero or negative limit prevents the middleware from initializing. This is stricter than ASP.NET Core itself: zero processes no forwarded entries, and a nullable limit can allow unlimited depth. This application does not currently expose the unlimited option. Supporting an explicit disabled mode or unlimited depth should be a separate, tested configuration decision; unlimited depth still requires restricted trusted peers. + +Forwarded headers run after exception/status-code handling and before HTTPS redirection, request logging, routing, MCP origin validation, authentication, and throttling. This lets downstream middleware use the validated client IP and scheme. Microsoft explicitly permits placement after diagnostics and error handling. See [Microsoft's proxy configuration guidance](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0). + +Before rollout, verify direct access, a trusted proxy, an untrusted peer, and the deployed multi-proxy chain in staging. Check the resulting client IP, HTTPS scheme, origin checks, and IP-based limits. The unit tests do not establish the SaaS ingress topology or validate a self-hoster's proxy configuration. diff --git a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs index 7f25e37988..c0f8067638 100644 --- a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs @@ -10,17 +10,19 @@ public static void Configure(ForwardedHeadersOptions options, IConfiguration con options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; options.RequireHeaderSymmetry = false; options.ForwardLimit = configuration.GetValue("ForwardedHeaders:ForwardLimit", 1); - if (options.ForwardLimit < 1) + if (options.ForwardLimit is < 1) throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be positive."); // Retain the framework's loopback defaults and add explicitly trusted ingress peers. foreach (string address in configuration.GetSection("ForwardedHeaders:KnownProxies").Get() ?? []) options.KnownProxies.Add(IPAddress.Parse(address)); + foreach (string value in configuration.GetSection("ForwardedHeaders:KnownNetworks").Get() ?? []) { var network = System.Net.IPNetwork.Parse(value); - if (network.PrefixLength == 0) + if (network.PrefixLength is 0) throw new InvalidOperationException("ForwardedHeaders:KnownNetworks must identify trusted ingress networks."); + options.KnownIPNetworks.Add(network); } } diff --git a/src/Exceptionless.Web/Extensions/HttpExtensions.cs b/src/Exceptionless.Web/Extensions/HttpExtensions.cs index c32cddb481..832573624d 100644 --- a/src/Exceptionless.Web/Extensions/HttpExtensions.cs +++ b/src/Exceptionless.Web/Extensions/HttpExtensions.cs @@ -139,6 +139,7 @@ public static ICollection GetAssociatedOrganizationIds(this HttpRequest ArgumentNullException.ThrowIfNull(request); var address = request.HttpContext.Connection.RemoteIpAddress; + return address is { IsIPv4MappedToIPv6: true } ? address.MapToIPv4().ToString() : address?.ToString(); } From 830924d1e1025ebba23dcc5e7b1c08e4fee962b3 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 13 Oct 2026 06:51:37 -0500 Subject: [PATCH 10/26] Support disabling forwarded headers and cover proxy boundaries --- docs/forwarded-headers.md | 2 +- .../ForwardedHeadersConfiguration.cs | 4 +- .../ForwardedHeadersConfigurationTests.cs | 83 ++++++++++++++++--- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/docs/forwarded-headers.md b/docs/forwarded-headers.md index 2768d9483a..e18c5b359f 100644 --- a/docs/forwarded-headers.md +++ b/docs/forwarded-headers.md @@ -19,7 +19,7 @@ This changes the previous trust-all behavior. Before upgrading a deployment behi Direct self-hosted installations do not need a proxy or additional configuration. `ForwardLimit: 1` is the maximum number of forwarded entries processed, not a minimum number of proxies required on each request. Requests without forwarded headers retain their connection IP and scheme. Multiple trusted proxies require a limit matching the intended chain and trust configuration for each traversed proxy. -The current application configuration requires a positive integer. An explicit zero or negative limit prevents the middleware from initializing. This is stricter than ASP.NET Core itself: zero processes no forwarded entries, and a nullable limit can allow unlimited depth. This application does not currently expose the unlimited option. Supporting an explicit disabled mode or unlimited depth should be a separate, tested configuration decision; unlimited depth still requires restricted trusted peers. +`ForwardLimit` defaults to `1`. Set it to `0` to disable forwarded-header processing, including headers from trusted proxies; requests retain the connection IP and scheme. Negative values are invalid and prevent the middleware from initializing. A positive value caps the number of entries processed, with processing also stopping at an unknown peer. This application requires a finite integer and does not expose ASP.NET Core's nullable/unlimited option. Direct self-hosters can leave the default unchanged or explicitly disable processing; deployments relying on a TLS-terminating proxy must keep processing enabled and configure their trusted peers. Forwarded headers run after exception/status-code handling and before HTTPS redirection, request logging, routing, MCP origin validation, authentication, and throttling. This lets downstream middleware use the validated client IP and scheme. Microsoft explicitly permits placement after diagnostics and error handling. See [Microsoft's proxy configuration guidance](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0). diff --git a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs index c0f8067638..3a1b8c4183 100644 --- a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs @@ -10,8 +10,8 @@ public static void Configure(ForwardedHeadersOptions options, IConfiguration con options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; options.RequireHeaderSymmetry = false; options.ForwardLimit = configuration.GetValue("ForwardedHeaders:ForwardLimit", 1); - if (options.ForwardLimit is < 1) - throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be positive."); + if (options.ForwardLimit is < 0) + throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be zero or positive."); // Retain the framework's loopback defaults and add explicitly trusted ingress peers. foreach (string address in configuration.GetSection("ForwardedHeaders:KnownProxies").Get() ?? []) diff --git a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs index b4a80646f7..466287fb1b 100644 --- a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs +++ b/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs @@ -12,13 +12,16 @@ namespace Exceptionless.Tests.Extensions; public sealed class ForwardedHeadersConfigurationTests { [Theory] - [InlineData(false, "198.51.100.20")] - [InlineData(true, "203.0.113.40")] - public async Task Invoke_ForwardedAddress_OnlyUsesConfiguredPeer(bool trusted, string expected) + [InlineData("192.0.2.10", "1", "198.51.100.20", "http")] + [InlineData("198.51.100.20", "1", "203.0.113.40", "https")] + [InlineData("198.51.100.20", "0", "198.51.100.20", "http")] + public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string trustedPeer, string limit, string expectedAddress, string expectedScheme) { + // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { - ["ForwardedHeaders:KnownProxies:0"] = trusted ? "198.51.100.20" : "192.0.2.10" + ["ForwardedHeaders:KnownProxies:0"] = trustedPeer, + ["ForwardedHeaders:ForwardLimit"] = limit }).Build(); var options = new ForwardedHeadersOptions(); ForwardedHeadersConfiguration.Configure(options, configuration); @@ -28,18 +31,26 @@ public async Task Invoke_ForwardedAddress_OnlyUsesConfiguredPeer(bool trusted, s context.Request.Headers["X-Forwarded-For"] = "203.0.113.40"; context.Request.Headers["X-Forwarded-Proto"] = "https"; var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act await middleware.Invoke(context); - Assert.Equal(expected, context.Request.GetClientIpAddress()); - Assert.Equal(trusted ? "https" : "http", context.Request.Scheme); + + // Assert + Assert.Equal(expectedAddress, context.Request.GetClientIpAddress()); + Assert.Equal(expectedScheme, context.Request.Scheme); } - [Fact] - public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer() + [Theory] + [InlineData("1", "192.0.2.11")] + [InlineData("2", "203.0.113.40")] + [InlineData("3", "203.0.113.40")] + public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer(string limit, string expectedAddress) { + // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["ForwardedHeaders:KnownNetworks:0"] = "192.0.2.0/24", - ["ForwardedHeaders:ForwardLimit"] = "2" + ["ForwardedHeaders:ForwardLimit"] = limit }).Build(); var options = new ForwardedHeadersOptions(); ForwardedHeadersConfiguration.Configure(options, configuration); @@ -47,8 +58,49 @@ public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer context.Connection.RemoteIpAddress = IPAddress.Parse("192.0.2.10"); context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40, 192.0.2.11"; var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act + await middleware.Invoke(context); + + // Assert + Assert.Equal(expectedAddress, context.Request.GetClientIpAddress()); + } + + [Theory] + [InlineData("http")] + [InlineData("https")] + public async Task Invoke_DirectRequest_PreservesConnectionAddressAndScheme(string scheme) + { + // Arrange + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse("203.0.113.40"); + context.Request.Scheme = scheme; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act await middleware.Invoke(context); + + // Assert Assert.Equal("203.0.113.40", context.Request.GetClientIpAddress()); + Assert.Equal(scheme, context.Request.Scheme); + } + + [Fact] + public void Configure_NegativeForwardLimit_Throws() + { + // Arrange + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ForwardedHeaders:ForwardLimit"] = "-1" + }).Build(); + + // Act + var exception = Assert.Throws(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + + // Assert + Assert.Contains("ForwardedHeaders:ForwardLimit", exception.Message, StringComparison.Ordinal); } [Theory] @@ -56,18 +108,29 @@ public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer [InlineData("::/0")] public void Configure_UnboundedTrustedNetwork_Throws(string network) { + // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["ForwardedHeaders:KnownNetworks:0"] = network }).Build(); - Assert.Throws(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + + // Act + var exception = Assert.Throws(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + + // Assert + Assert.Contains("ForwardedHeaders:KnownNetworks", exception.Message, StringComparison.Ordinal); } [Fact] public void Configure_Defaults_RetainsLoopbackTrustAndOneHopLimit() { + // Arrange var options = new ForwardedHeadersOptions(); + + // Act ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + + // Assert Assert.NotEmpty(options.KnownProxies); Assert.NotEmpty(options.KnownIPNetworks); Assert.Equal(1, options.ForwardLimit); From 5a726999ab6c1bdfac42d508c02adf59b3aa1a02 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 14 Oct 2026 23:45:03 -0500 Subject: [PATCH 11/26] Validate forwarded peer settings and clarify hosting migration --- docs/docs/self-hosting/docker.md | 2 + docs/docs/self-hosting/index.md | 32 +++++++ docs/docs/self-hosting/kubernetes.md | 14 ++- .../upgrading-self-hosted-instance.md | 4 + docs/forwarded-headers.md | 26 ----- .../ForwardedHeadersConfiguration.cs | 26 ++++- src/Exceptionless.Web/appsettings.yml | 7 ++ .../ForwardedHeadersConfigurationTests.cs | 94 ++++++++++++++----- .../Extensions/HttpExtensionsTests.cs | 11 ++- 9 files changed, 159 insertions(+), 57 deletions(-) delete mode 100644 docs/forwarded-headers.md diff --git a/docs/docs/self-hosting/docker.md b/docs/docs/self-hosting/docker.md index c701d41e45..2e277d27da 100644 --- a/docs/docs/self-hosting/docker.md +++ b/docs/docs/self-hosting/docker.md @@ -4,6 +4,8 @@ title: "Docker" # Docker +If you place a reverse proxy in front of the container, configure its [trusted forwarding peers](/docs/self-hosting/#reverse-proxies-and-https) before upgrading. The direct-port examples below do not require proxy trust settings. + If you would like to test Exceptionless locally, please follow this section. ## Requirements diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index 829ae550b0..9aadcdc22b 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -12,6 +12,38 @@ You can also use Kubernetes while self-hosting Exceptionless. We'll cover both t * [Kubernetes](/docs/self-hosting/kubernetes) * [Upgrading](/docs/self-hosting/upgrading-self-hosted-instance) +## Reverse proxies and HTTPS + +Direct HTTP/HTTPS hosting needs no forwarded-header configuration. The API accepts `X-Forwarded-For` and `X-Forwarded-Proto` only from loopback or explicitly trusted proxies. A known proxy is the **source IP of the connection reaching the API**, not its hostname, public frontend IP, Kubernetes Service IP, or proxy identifier. For Docker bridges or NAT, verify the address seen inside the API container. + +Before upgrading from unrestricted forwarding, configure every non-loopback proxy that must supply client addresses or HTTPS schemes. Otherwise requests use the proxy IP and backend scheme: unrelated users can share IP-based limits and HTTPS detection can be incorrect. Set the public website `EX_BaseURL` and API `EX_ApiUrl` separately; forwarded headers do not configure public origins. + +Use this section in the web application's `appsettings.Local.yml`, replacing the example address with the verified proxy source IP: + +```yaml +ForwardedHeaders: + KnownProxies: + - 192.0.2.10 + ForwardLimit: 1 +``` + +The equivalent Docker Compose `environment` entries, or Helm values under the chart's existing `config` map, are: + +```yaml +EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" +EX_ForwardedHeaders__ForwardLimit: "1" +``` + +For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup. + +`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero disables processing; negative or non-integer limits are invalid. Do not enable the unrestricted `ASPNETCORE_FORWARDEDHEADERS_ENABLED` shortcut. + +The public edge proxy must derive the scheme from its connection and replace untrusted forwarding headers, or append the observed client address so the configured hop limit cannot reach user-supplied entries. Configure Nginx, Caddy, Traefik, or other proxies accordingly. Loopback proxies and IIS out-of-process integration normally need no additional trust entries. A remote IIS/ARR proxy needs the same explicit peer configuration as other remote proxies. + +Environment variables override YAML by key; command-line values override both. Use double underscores for nesting and numbered entries for lists. Overriding index `0` does not remove other entries inherited from YAML; remove obsolete trust entries from their source and restart after changes. Avoid defining the same key with both prefixed and unprefixed environment names. + +The API processes IP/scheme headers before redirect, authentication, and throttling middleware. It does not consume `X-Forwarded-Host` or `X-Forwarded-Prefix`; preserve the public Host and root paths. Verify HTTPS, client IPs, redirects, and IP-based limits through the actual proxy chain before rollout. Enable `EX_Serilog__MinimumLevel__Override__Microsoft.AspNetCore.HttpOverrides: "Debug"` temporarily to diagnose unknown peers. See [Microsoft's proxy guidance](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0). + --- [Next > Docker](/docs/self-hosting/docker) diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index 667ab9a4de..a77d4224bc 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -25,7 +25,19 @@ The following section will cover how to configure exceptionless inside of a Kube All configuration options and settings can be found in the various option classes located [here](https://github.com/exceptionless/Exceptionless/tree/master/src/Exceptionless.Core/Configuration). -_Please note that if you are specifying configuration via `docker-compose`, then you will need to drop the `EX_` and `EX_ConnectionStrings__` prefixes._ +The same `EX_` environment keys work in Docker Compose. Nested keys use double underscores. + +### Trusted ingress peers + +Before deploying behind ingress, configure [trusted forwarding peers](/docs/self-hosting/#reverse-proxies-and-https) through the Helm chart's existing `config` map. For example, after replacing the documentation address with a verified proxy source address: + +```yaml +config: + EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" + EX_ForwardedHeaders__ForwardLimit: "1" +``` + +The repository's ingress templates use Azure Application Gateway for Containers, terminate TLS at ingress, and route `/` to the API Service on port 80 and container port 8080. The checked-in values do not identify its backend source addresses or association subnet. Verify those addresses and any intermediate proxies before setting trust; the ALB controller pod and Service address are not substitutes. No cluster-wide trust is inferred automatically. This configuration must ship with the application upgrade; the manifests alone do not prove live forwarding behavior. ## ConnectionStrings diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index 20068a2efc..efa7d8eaaa 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -4,6 +4,10 @@ title: "Upgrading" # Upgrading +## Trusted proxy migration + +Upgrading from unrestricted forwarding requires [explicit trust for non-loopback reverse proxies](/docs/self-hosting/#reverse-proxies-and-https). Configure and verify the actual ingress peers before rollout, or client-IP limits and HTTPS detection can change. Direct hosting needs no new settings. + **Please ensure that you have created backups before upgrading!** **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.** diff --git a/docs/forwarded-headers.md b/docs/forwarded-headers.md deleted file mode 100644 index e18c5b359f..0000000000 --- a/docs/forwarded-headers.md +++ /dev/null @@ -1,26 +0,0 @@ -The API uses forwarded client addresses and schemes only from trusted peers. Framework loopback defaults remain enabled; non-loopback ingress peers must be configured before deployment. - -Configure the actual proxy addresses or their smallest required network and the number of forwarding hops: - -```yaml -ForwardedHeaders: - KnownProxies: - - 10.20.30.40 - KnownNetworks: - - 10.20.40.0/24 - ForwardLimit: 1 -``` - -Leave unused lists empty. The ingress must replace incoming forwarded headers rather than preserve client-supplied values. Public access should terminate at that ingress. With no additional configuration, headers from other peers are ignored. IPv4-mapped IPv6 client addresses use the same login window as their IPv4 equivalent. - -## Compatibility and deployment - -This changes the previous trust-all behavior. Before upgrading a deployment behind a non-loopback proxy, configure the peer addresses as seen by the API and verify the actual forwarding chain. Otherwise, forwarded client IPs and HTTPS schemes are ignored: IP-based login, signup, password-reset and anonymous API limits can group users behind the proxy, and scheme-dependent redirects can behave incorrectly. Authenticated API throttling still uses its existing user or organization identifier. No endpoint routes, payload schemas, API keys, or WebSocket message formats change. - -Direct self-hosted installations do not need a proxy or additional configuration. `ForwardLimit: 1` is the maximum number of forwarded entries processed, not a minimum number of proxies required on each request. Requests without forwarded headers retain their connection IP and scheme. Multiple trusted proxies require a limit matching the intended chain and trust configuration for each traversed proxy. - -`ForwardLimit` defaults to `1`. Set it to `0` to disable forwarded-header processing, including headers from trusted proxies; requests retain the connection IP and scheme. Negative values are invalid and prevent the middleware from initializing. A positive value caps the number of entries processed, with processing also stopping at an unknown peer. This application requires a finite integer and does not expose ASP.NET Core's nullable/unlimited option. Direct self-hosters can leave the default unchanged or explicitly disable processing; deployments relying on a TLS-terminating proxy must keep processing enabled and configure their trusted peers. - -Forwarded headers run after exception/status-code handling and before HTTPS redirection, request logging, routing, MCP origin validation, authentication, and throttling. This lets downstream middleware use the validated client IP and scheme. Microsoft explicitly permits placement after diagnostics and error handling. See [Microsoft's proxy configuration guidance](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0). - -Before rollout, verify direct access, a trusted proxy, an untrusted peer, and the deployed multi-proxy chain in staging. Check the resulting client IP, HTTPS scheme, origin checks, and IP-based limits. The unit tests do not establish the SaaS ingress topology or validate a self-hoster's proxy configuration. diff --git a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs index 3a1b8c4183..96a66a2e93 100644 --- a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs @@ -14,16 +14,32 @@ public static void Configure(ForwardedHeadersOptions options, IConfiguration con throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be zero or positive."); // Retain the framework's loopback defaults and add explicitly trusted ingress peers. - foreach (string address in configuration.GetSection("ForwardedHeaders:KnownProxies").Get() ?? []) - options.KnownProxies.Add(IPAddress.Parse(address)); + foreach (var entry in GetEntries(configuration, "KnownProxies")) + { + if (!IPAddress.TryParse(entry.Value, out var address)) + throw new InvalidOperationException($"{entry.Path} must be an IP address without a port."); + + options.KnownProxies.Add(address); + } - foreach (string value in configuration.GetSection("ForwardedHeaders:KnownNetworks").Get() ?? []) + foreach (var entry in GetEntries(configuration, "KnownNetworks")) { - var network = System.Net.IPNetwork.Parse(value); + if (!System.Net.IPNetwork.TryParse(entry.Value, out var network)) + throw new InvalidOperationException($"{entry.Path} must be a valid IPv4 or IPv6 CIDR."); + if (network.PrefixLength is 0) - throw new InvalidOperationException("ForwardedHeaders:KnownNetworks must identify trusted ingress networks."); + throw new InvalidOperationException($"{entry.Path} must identify a trusted ingress network, not a /0 network."); options.KnownIPNetworks.Add(network); } } + + private static IEnumerable GetEntries(IConfiguration configuration, string name) + { + var section = configuration.GetSection($"ForwardedHeaders:{name}"); + if (section.Value is not null) + throw new InvalidOperationException($"{section.Path} must be a list. Use numbered environment keys such as EX_ForwardedHeaders__{name}__0."); + + return section.GetChildren(); + } } diff --git a/src/Exceptionless.Web/appsettings.yml b/src/Exceptionless.Web/appsettings.yml index d4c5606fbb..6362051d76 100644 --- a/src/Exceptionless.Web/appsettings.yml +++ b/src/Exceptionless.Web/appsettings.yml @@ -33,6 +33,13 @@ Serilog: # Optional public API URL for OAuth discovery and resource validation; defaults to BaseURL. ApiUrl: +# Reverse proxy trust: see docs/docs/self-hosting/index.md#reverse-proxies-and-https. +# Configure only verified API-visible peers; loopback is trusted by default. +ForwardedHeaders: + ForwardLimit: 1 + KnownProxies: [] + KnownNetworks: [] + Assistant: Endpoint: https://openrouter.ai/api/v1/chat/completions Model: "~deepseek/deepseek-v4-flash-latest" diff --git a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs index 466287fb1b..ab0748f383 100644 --- a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs +++ b/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs @@ -1,5 +1,8 @@ +using System.Collections; using System.Net; +using Exceptionless.Core.Configuration; using Exceptionless.Web.Extensions; +using Foundatio.Xunit; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Configuration; @@ -9,13 +12,15 @@ namespace Exceptionless.Tests.Extensions; -public sealed class ForwardedHeadersConfigurationTests +public sealed class ForwardedHeadersConfigurationTests(ITestOutputHelper output) : TestWithLoggingBase(output) { [Theory] - [InlineData("192.0.2.10", "1", "198.51.100.20", "http")] - [InlineData("198.51.100.20", "1", "203.0.113.40", "https")] - [InlineData("198.51.100.20", "0", "198.51.100.20", "http")] - public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string trustedPeer, string limit, string expectedAddress, string expectedScheme) + [InlineData("192.0.2.10", "198.51.100.20", "1", "198.51.100.20", "http")] + [InlineData("198.51.100.20", "198.51.100.20", "1", "203.0.113.40", "https")] + [InlineData("198.51.100.20", "198.51.100.20", "0", "198.51.100.20", "http")] + [InlineData("198.51.100.20", "::ffff:198.51.100.20", "1", "203.0.113.40", "https")] + [InlineData("2001:db8::20", "2001:db8::20", "1", "203.0.113.40", "https")] + public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string trustedPeer, string remoteAddress, string limit, string expectedAddress, string expectedScheme) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary @@ -26,7 +31,7 @@ public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string var options = new ForwardedHeadersOptions(); ForwardedHeadersConfiguration.Configure(options, configuration); var context = new DefaultHttpContext(); - context.Connection.RemoteIpAddress = IPAddress.Parse("198.51.100.20"); + context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); context.Request.Scheme = "http"; context.Request.Headers["X-Forwarded-For"] = "203.0.113.40"; context.Request.Headers["X-Forwarded-Proto"] = "https"; @@ -41,10 +46,11 @@ public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string } [Theory] - [InlineData("1", "192.0.2.11")] - [InlineData("2", "203.0.113.40")] - [InlineData("3", "203.0.113.40")] - public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer(string limit, string expectedAddress) + [InlineData("1", "192.0.2.10", "192.0.2.11", "http")] + [InlineData("2", "192.0.2.10", "203.0.113.40", "https")] + [InlineData("3", "192.0.2.10", "203.0.113.40", "https")] + [InlineData("2", "::ffff:192.0.2.10", "203.0.113.40", "https")] + public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer(string limit, string remoteAddress, string expectedAddress, string expectedScheme) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary @@ -55,8 +61,9 @@ public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer var options = new ForwardedHeadersOptions(); ForwardedHeadersConfiguration.Configure(options, configuration); var context = new DefaultHttpContext(); - context.Connection.RemoteIpAddress = IPAddress.Parse("192.0.2.10"); + context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40, 192.0.2.11"; + context.Request.Headers["X-Forwarded-Proto"] = "http, https, http"; var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); // Act @@ -64,6 +71,7 @@ public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer // Assert Assert.Equal(expectedAddress, context.Request.GetClientIpAddress()); + Assert.Equal(expectedScheme, context.Request.Scheme); } [Theory] @@ -97,42 +105,82 @@ public void Configure_NegativeForwardLimit_Throws() }).Build(); // Act - var exception = Assert.Throws(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + var exception = Record.Exception(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); // Assert + Assert.IsType(exception); Assert.Contains("ForwardedHeaders:ForwardLimit", exception.Message, StringComparison.Ordinal); } [Theory] - [InlineData("0.0.0.0/0")] - [InlineData("::/0")] - public void Configure_UnboundedTrustedNetwork_Throws(string network) + [InlineData("KnownNetworks:0", "0.0.0.0/0")] + [InlineData("KnownNetworks:0", "::/0")] + [InlineData("KnownNetworks:0", "192.0.999.0/24")] + [InlineData("KnownNetworks:0", "192.0.2.0/33")] + [InlineData("KnownProxies:0", "proxy.example.com")] + [InlineData("KnownProxies:0", "192.0.2.10:8080")] + [InlineData("KnownProxies:0", "")] + [InlineData("KnownProxies", "192.0.2.10,192.0.2.11")] + [InlineData("KnownNetworks", "192.0.2.0/24")] + public void Configure_InvalidPeerConfiguration_ThrowsWithKey(string key, string value) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { - ["ForwardedHeaders:KnownNetworks:0"] = network + [$"ForwardedHeaders:{key}"] = value }).Build(); // Act - var exception = Assert.Throws(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + var exception = Record.Exception(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); // Assert - Assert.Contains("ForwardedHeaders:KnownNetworks", exception.Message, StringComparison.Ordinal); + Assert.IsType(exception); + Assert.Contains($"ForwardedHeaders:{key}", exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("127.0.0.1")] + [InlineData("::1")] + [InlineData("::ffff:127.0.0.1")] + public async Task Invoke_Defaults_TrustsLoopbackAndProcessesOneHop(string remoteAddress) + { + // Arrange + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); + context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40"; + context.Request.Headers["X-Forwarded-Proto"] = "http, https"; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act + await middleware.Invoke(context); + + // Assert + Assert.Equal("203.0.113.40", context.Request.GetClientIpAddress()); + Assert.Equal("https", context.Request.Scheme); } [Fact] - public void Configure_Defaults_RetainsLoopbackTrustAndOneHopLimit() + public void Configure_PrefixedEnvironmentLists_ReadsNestedPeersAndLimit() { // Arrange + var provider = new CustomEnvironmentVariablesConfigurationProvider(); + using var configuration = new ConfigurationRoot([provider]); + provider.Load(new Hashtable + { + ["EX_ForwardedHeaders__KnownProxies__0"] = "192.0.2.10", + ["EX_ForwardedHeaders__KnownNetworks__0"] = "2001:db8::/64", + ["EX_ForwardedHeaders__ForwardLimit"] = "2" + }); var options = new ForwardedHeadersOptions(); // Act - ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + ForwardedHeadersConfiguration.Configure(options, configuration); // Assert - Assert.NotEmpty(options.KnownProxies); - Assert.NotEmpty(options.KnownIPNetworks); - Assert.Equal(1, options.ForwardLimit); + Assert.Contains(IPAddress.Parse("192.0.2.10"), options.KnownProxies); + Assert.Contains(System.Net.IPNetwork.Parse("2001:db8::/64"), options.KnownIPNetworks); + Assert.Equal(2, options.ForwardLimit); } } diff --git a/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs b/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs index 1f4a25b5bc..14275b27f8 100644 --- a/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs +++ b/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs @@ -1,11 +1,12 @@ using System.Text; using Exceptionless.Web.Extensions; +using Foundatio.Xunit; using Microsoft.AspNetCore.Http; using Xunit; namespace Exceptionless.Tests.Extensions; -public sealed class HttpExtensionsTests +public sealed class HttpExtensionsTests(ITestOutputHelper output) : TestWithLoggingBase(output) { [Theory] [InlineData(null)] @@ -40,9 +41,15 @@ public void GetBasicAuth_InvalidHeader_ReturnsNull(string? authorization) [InlineData(null, null)] public void GetClientIpAddress_Address_NormalizesMappedIpv4(string? value, string? expected) { + // Arrange var request = new DefaultHttpContext().Request; request.HttpContext.Connection.RemoteIpAddress = value is null ? null : System.Net.IPAddress.Parse(value); - Assert.Equal(expected, request.GetClientIpAddress()); + + // Act + string? address = request.GetClientIpAddress(); + + // Assert + Assert.Equal(expected, address); } [Theory] From 2b5137cbd753616d1e27af49b44c5a01f15e5ab2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 14 Oct 2026 23:58:45 -0500 Subject: [PATCH 12/26] Discover gateway trust during dev and prod deployment --- .github/workflows/build.yaml | 11 +- docs/docs/self-hosting/kubernetes.md | 19 ++- k8s/exceptionless/templates/config.yaml | 12 ++ k8s/exceptionless/values.yaml | 5 + k8s/scripts/discover-forwarded-headers.mjs | 66 +++++++++++ .../ForwardedHeadersConfiguration.cs | 2 +- .../ForwardedHeadersConfigurationTests.cs | 3 +- tests/ci/forwarded-headers-discovery.test.mjs | 109 ++++++++++++++++++ 8 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 k8s/scripts/discover-forwarded-headers.mjs rename src/Exceptionless.Web/{Extensions => Security}/ForwardedHeadersConfiguration.cs (97%) rename tests/Exceptionless.Tests/{Extensions => Security}/ForwardedHeadersConfigurationTests.cs (99%) create mode 100644 tests/ci/forwarded-headers-discovery.test.mjs diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 832b5b3de5..e4dc0d6ec5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -171,8 +171,11 @@ jobs: - name: Build run: dotnet build ./Exceptionless.slnx --no-restore --configuration Release + - name: Install Helm for offline deployment tests + uses: azure/setup-helm@v5 + - name: Verify CI test tooling - run: node --test tests/ci/shards.test.mjs + run: node --test tests/ci/shards.test.mjs tests/ci/forwarded-headers-discovery.test.mjs - name: Run .NET shard with coverage run: node tests/ci/backend-shards.mjs run --index ${{ matrix.shard }} --count 4 --output coverage --coverage @@ -723,7 +726,8 @@ jobs: if: ${{ needs.version.outputs.is_dev_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless + node k8s/scripts/discover-forwarded-headers.mjs ex-dev k8s/ex-dev-values.yaml "$RUNNER_TEMP/forwarded-headers.json" + helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless - name: Run Development Migrations if: ${{ needs.version.outputs.is_dev_deploy == 'true' }} @@ -842,7 +846,8 @@ jobs: if: ${{ needs.version.outputs.is_prod_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml ex-prod --namespace ex-prod ./k8s/exceptionless + node k8s/scripts/discover-forwarded-headers.mjs ex-prod k8s/ex-prod-values.yaml "$RUNNER_TEMP/forwarded-headers.json" + helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" ex-prod --namespace ex-prod ./k8s/exceptionless - name: Summary if: ${{ always() }} diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index a77d4224bc..c5014f5d1f 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -37,7 +37,24 @@ config: EX_ForwardedHeaders__ForwardLimit: "1" ``` -The repository's ingress templates use Azure Application Gateway for Containers, terminate TLS at ingress, and route `/` to the API Service on port 80 and container port 8080. The checked-in values do not identify its backend source addresses or association subnet. Verify those addresses and any intermediate proxies before setting trust; the ALB controller pod and Service address are not substitutes. No cluster-wide trust is inferred automatically. This configuration must ship with the application upgrade; the manifests alone do not prove live forwarding behavior. +The repository's ingress templates use Azure Application Gateway for Containers, terminate TLS at ingress, and route `/` to the API Service on port 80 and container port 8080. The hosted development and production deployment workflow discovers the gateway's association subnet before each Helm upgrade, so subnet addresses do not need to be duplicated in environment values files. + +The shared `k8s/scripts/discover-forwarded-headers.mjs` script renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. + +Discovery writes a `forwardedHeaders.knownNetworks` Helm list. The chart converts it into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys and a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining discovery with manual `ForwardedHeaders` entries in `config` is rejected; remove those old entries from saved release values before using discovery. + +For manual upgrades, run discovery from the repository root after selecting the intended Kubernetes context and Azure subscription, then pass the generated file **last** to Helm. For example, the development commands are: + +```sh +node k8s/scripts/discover-forwarded-headers.mjs ex-dev k8s/ex-dev-values.yaml /tmp/ex-dev-forwarded-headers.json +# Run only if discovery succeeded: +helm upgrade ex-dev k8s/exceptionless --namespace ex-dev --reuse-values \ + --values k8s/ex-dev-values.yaml --values /tmp/ex-dev-forwarded-headers.json +``` + +For production, use `ex-prod`, `k8s/ex-prod-values.yaml`, and a separate output file. The script only reads infrastructure and replaces the local values file; Helm performs the upgrade. It removes any previous output before discovery so a failed refresh cannot leave stale trust available for reuse. Automated workflows stop on discovery failure; manual operators must also stop. + +This assumes the gateway connects directly to the application. Verify the API-visible peer and any intermediate proxies before the first rollout; discovering Azure configuration does not prove the live traffic path. The ALB controller pod and Service address are not substitutes for the association subnet. Other ingress topologies should use explicit trusted peers in `config` as shown above. See Microsoft's [association subnet documentation](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/application-gateway-for-containers-components). ## ConnectionStrings diff --git a/k8s/exceptionless/templates/config.yaml b/k8s/exceptionless/templates/config.yaml index 6c53f373a8..ef78904f55 100644 --- a/k8s/exceptionless/templates/config.yaml +++ b/k8s/exceptionless/templates/config.yaml @@ -17,6 +17,18 @@ data: {{- end }} EX_DisabledPlugins: GeoPlugin,LocationPlugin EX_EnableSsl: "true" + {{- $knownNetworks := dig "forwardedHeaders" "knownNetworks" (list) .Values.AsMap }} + {{- if $knownNetworks }} + {{- range $key, $value := .Values.config }} + {{- if hasPrefix "forwardedheaders:" (lower (replace "__" ":" (trimPrefix "EX_" $key))) }} + {{- fail "Do not combine discovered forwardedHeaders.knownNetworks with ForwardedHeaders config overrides; remove the old overrides" }} + {{- end }} + {{- end }} + EX_ForwardedHeaders__ForwardLimit: "1" + {{- range $index, $network := $knownNetworks }} + EX_ForwardedHeaders__KnownNetworks__{{ $index }}: {{ $network | quote }} + {{- end }} + {{- end }} EX_ConnectionStrings__Elasticsearch: {{ default "server=http://elasticsearch:9200" .Values.elasticsearch.connectionString }} {{- if (empty .Values.redis.connectionString) }} EX_ConnectionStrings__Redis: {{ template "exceptionless.fullname" . }}-redis-master,abortConnect=false diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index b2439e84b2..1b65ec236f 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -22,6 +22,11 @@ jobs: config: {} # key: value +# Populated by k8s/scripts/discover-forwarded-headers.mjs for hosted deployments. +# A replacement list prevents stale subnet entries surviving --reuse-values. +forwardedHeaders: + knownNetworks: [] + ingress: enabled: false clusterIssuer: cert-manager-webhook-dnsimple-production diff --git a/k8s/scripts/discover-forwarded-headers.mjs b/k8s/scripts/discover-forwarded-headers.mjs new file mode 100644 index 0000000000..5080be26a9 --- /dev/null +++ b/k8s/scripts/discover-forwarded-headers.mjs @@ -0,0 +1,66 @@ +import { execFileSync } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { isIP } from 'node:net'; +import { pathToFileURL } from 'node:url'; + +const run = (command, args) => execFileSync(command, args, { encoding: 'utf8', timeout: 60_000 }); + +// Read the chart's rendered annotations so gateway selection cannot drift from ingress. +function annotation(manifest, name) { + const values = [...manifest.matchAll(new RegExp(`^\\s+alb\\.networking\\.azure\\.io/${name}:\\s*([a-z0-9.-]+)\\s*$`, 'gm'))] + .map(match => match[1]); + if (!values.length || new Set(values).size !== 1) + throw new Error(`Expected one consistent ingress annotation: ${name}`); + return values[0]; +} + +export function discoverForwardedHeaders(release, valuesFile, execute = run) { + const manifest = execute('helm', ['template', release, 'k8s/exceptionless', '--values', valuesFile, '--show-only', 'templates/api.yaml']); + const name = annotation(manifest, 'alb-name'); + const namespace = annotation(manifest, 'alb-namespace'); + const gateway = JSON.parse(execute('kubectl', ['get', 'applicationloadbalancers.alb.networking.azure.io', name, '--namespace', namespace, '--output', 'json'])); + const associations = gateway.spec?.associations; + if (!Array.isArray(associations) || !associations.length) + throw new Error('Gateway has no association subnet IDs; refusing to infer proxy trust'); + + const networks = []; + for (const id of new Set(associations)) { + if (typeof id !== 'string' || !/^\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.Network\/virtualNetworks\/[^/]+\/subnets\/[^/]+$/i.test(id)) + throw new Error('Gateway association is not an Azure subnet resource ID'); + const subnet = JSON.parse(execute('az', ['network', 'vnet', 'subnet', 'show', '--ids', id, '--output', 'json', '--only-show-errors'])); + if (subnet.id?.toLowerCase() !== id.toLowerCase() || + !subnet.delegations?.some(item => item.serviceName === 'Microsoft.ServiceNetworking/trafficControllers')) + throw new Error('Association subnet identity or Application Gateway for Containers delegation is missing'); + + const prefixes = [subnet.addressPrefix, ...(subnet.addressPrefixes ?? [])].filter(value => value != null); + if (!prefixes.length) + throw new Error('Association subnet has no address prefixes'); + for (const prefix of prefixes) { + const [address, length, extra] = typeof prefix === 'string' ? prefix.split('/') : []; + const family = isIP(address ?? ''); + if (!family || extra !== undefined || !/^[1-9][0-9]*$/.test(length ?? '') || Number(length) > (family === 4 ? 32 : 128)) + throw new Error(`Invalid or unrestricted association subnet prefix: ${prefix}`); + networks.push(prefix); + } + } + + // A list replaces the previous list on Helm upgrades, including --reuse-values. + // Individual config-map keys would retain stale higher-numbered entries. + return { forwardedHeaders: { knownNetworks: [...new Set(networks)].sort() } }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); + if (!release || !valuesFile || !outputFile || extra.length) + throw new Error('Usage: node k8s/scripts/discover-forwarded-headers.mjs (from repository root)'); + // A failed refresh must not leave an earlier trust file available for upgrade. + rmSync(outputFile, { force: true }); + const values = discoverForwardedHeaders(release, valuesFile); + writeFileSync(outputFile, `${JSON.stringify(values, null, 2)}\n`, { mode: 0o600 }); + console.log(`Discovered ${values.forwardedHeaders.knownNetworks.length} gateway subnet prefix(es)`); + } catch (error) { + console.error(`Forwarded-header discovery failed: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs similarity index 97% rename from src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs rename to src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs index 96a66a2e93..e9ee2614b4 100644 --- a/src/Exceptionless.Web/Extensions/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs @@ -1,7 +1,7 @@ using System.Net; using Microsoft.AspNetCore.HttpOverrides; -namespace Exceptionless.Web.Extensions; +namespace Exceptionless.Web.Security; public static class ForwardedHeadersConfiguration { diff --git a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs similarity index 99% rename from tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs rename to tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs index ab0748f383..a621afc27d 100644 --- a/tests/Exceptionless.Tests/Extensions/ForwardedHeadersConfigurationTests.cs +++ b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs @@ -2,6 +2,7 @@ using System.Net; using Exceptionless.Core.Configuration; using Exceptionless.Web.Extensions; +using Exceptionless.Web.Security; using Foundatio.Xunit; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpOverrides; @@ -10,7 +11,7 @@ using Microsoft.Extensions.Options; using Xunit; -namespace Exceptionless.Tests.Extensions; +namespace Exceptionless.Tests.Security; public sealed class ForwardedHeadersConfigurationTests(ITestOutputHelper output) : TestWithLoggingBase(output) { diff --git a/tests/ci/forwarded-headers-discovery.test.mjs b/tests/ci/forwarded-headers-discovery.test.mjs new file mode 100644 index 0000000000..e1cb730f69 --- /dev/null +++ b/tests/ci/forwarded-headers-discovery.test.mjs @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { discoverForwardedHeaders } from '../../k8s/scripts/discover-forwarded-headers.mjs'; + +const subnetId = '/subscriptions/test/resourceGroups/gateway/providers/Microsoft.Network/virtualNetworks/ingress/subnets/association'; +const manifest = ' alb.networking.azure.io/alb-name: public-alb\n alb.networking.azure.io/alb-namespace: kube-system\n'; +const gateway = { spec: { associations: [subnetId] } }; +const subnet = { id: subnetId, addressPrefix: '192.0.2.0/24', delegations: [{ serviceName: 'Microsoft.ServiceNetworking/trafficControllers' }] }; + +function mock({ rendered = manifest, resource = gateway, network = subnet, fail } = {}) { + const calls = []; + return { + calls, + execute(command, args) { + calls.push([command, args]); + if (command === fail) throw new Error('Access denied'); + if (command === 'helm') return rendered; + if (command === 'kubectl') return JSON.stringify(resource); + if (command === 'az') return JSON.stringify(network); + throw new Error(`Unexpected command: ${command}`); + } + }; +} + +for (const environment of ['dev', 'prod']) { + test(`${environment}: discovers the rendered gateway and passes prefixes through the real chart`, () => { + // Helm renders locally; Azure and Kubernetes commands are always mocked. + const valuesFile = `k8s/ex-${environment}-values.yaml`; + const release = `ex-${environment}`; + const rendered = execFileSync('helm', ['template', release, 'k8s/exceptionless', '--values', valuesFile, '--show-only', 'templates/api.yaml'], { encoding: 'utf8' }); + const stub = mock({ rendered }); + const values = discoverForwardedHeaders(release, valuesFile, stub.execute); + assert.deepEqual(values, { forwardedHeaders: { knownNetworks: ['192.0.2.0/24'] } }); + assert.deepEqual(stub.calls[1], ['kubectl', ['get', 'applicationloadbalancers.alb.networking.azure.io', 'public-alb', '--namespace', 'kube-system', '--output', 'json']]); + assert.deepEqual(stub.calls[2], ['az', ['network', 'vnet', 'subnet', 'show', '--ids', subnetId, '--output', 'json', '--only-show-errors']]); + + const directory = mkdtempSync(join(tmpdir(), 'forwarded-headers-')); + try { + const previous = join(directory, 'previous.json'); + const discovered = join(directory, 'discovered.json'); + writeFileSync(previous, JSON.stringify({ forwardedHeaders: { knownNetworks: ['198.51.100.0/24', '203.0.113.0/24'] } })); + writeFileSync(discovered, JSON.stringify(values)); + const args = ['template', release, 'k8s/exceptionless', '--values', valuesFile, '--values', previous, '--values', discovered, '--show-only', 'templates/config.yaml']; + const config = execFileSync('helm', args, { encoding: 'utf8' }); + assert.match(config, /EX_ForwardedHeaders__ForwardLimit: "1"/); + assert.match(config, /EX_ForwardedHeaders__KnownNetworks__0: "192\.0\.2\.0\/24"/); + assert.doesNotMatch(config, /KnownNetworks__1|198\.51\.100|203\.0\.113/); + for (const key of ['EX_ForwardedHeaders__KnownNetworks__2', 'ForwardedHeaders__KnownProxies__0', 'EX_ForwardedHeaders__ForwardLimit']) { + assert.throws(() => execFileSync('helm', [...args, '--set-string', `config.${key}=1`], { stdio: 'pipe' }), /Do not combine discovered/); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +} + +test('deduplicates association IDs and IPv4/IPv6 prefixes', () => { + const stub = mock({ resource: { spec: { associations: [subnetId, subnetId] } }, network: { ...subnet, addressPrefixes: ['192.0.2.0/24', '2001:db8::/64'] } }); + assert.deepEqual(discoverForwardedHeaders('ex-test', 'values.yaml', stub.execute).forwardedHeaders.knownNetworks, ['192.0.2.0/24', '2001:db8::/64']); + assert.equal(stub.calls.filter(([command]) => command === 'az').length, 1); +}); + +for (const [name, options, message] of [ + ['missing ingress', { rendered: '' }, /annotation/], + ['conflicting gateway', { rendered: `${manifest} alb.networking.azure.io/alb-name: other-alb\n` }, /annotation/], + ['missing association', { resource: { spec: {} } }, /no association/], + ['invalid association', { resource: { spec: { associations: ['--help'] } } }, /resource ID/], + ['wrong subnet', { network: { ...subnet, id: `${subnetId}-other` } }, /identity/], + ['missing delegation', { network: { ...subnet, delegations: [] } }, /delegation/], + ['missing prefix', { network: { ...subnet, addressPrefix: undefined } }, /no address prefixes/], + ...['0.0.0.0/0', '::/0', '192.0.2.0/33', 'bad/24', '192.0.2.0/24/extra'].map(prefix => [prefix, { network: { ...subnet, addressPrefix: prefix } }, /Invalid or unrestricted/]), + ['Kubernetes denied', { fail: 'kubectl' }, /Access denied/], + ['Azure denied', { fail: 'az' }, /Access denied/] +]) { + test(`refuses discovery: ${name}`, () => { + const stub = mock(options); + assert.throws(() => discoverForwardedHeaders('ex-test', 'values.yaml', stub.execute), message); + }); +} + +test('both deployment paths require discovery and consume its output', () => { + const workflow = readFileSync('.github/workflows/build.yaml', 'utf8'); + for (const environment of ['dev', 'prod']) { + assert.ok(workflow.includes(`node k8s/scripts/discover-forwarded-headers.mjs ex-${environment} k8s/ex-${environment}-values.yaml "$RUNNER_TEMP/forwarded-headers.json"`)); + assert.ok(workflow.includes(`--values ./k8s/ex-${environment}-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json"`)); + } +}); + +test('CLI failure exits nonzero and removes stale generated trust', () => { + const directory = mkdtempSync(join(tmpdir(), 'forwarded-headers-cli-')); + try { + // Restrict PATH to a failing fake Helm; no cluster or Azure tools can run. + writeFileSync(join(directory, 'helm'), '#!/bin/sh\nexit 7\n', { mode: 0o700 }); + const output = join(directory, 'values.json'); + writeFileSync(output, JSON.stringify({ forwardedHeaders: { knownNetworks: ['198.51.100.0/24'] } })); + const result = spawnSync(process.execPath, ['k8s/scripts/discover-forwarded-headers.mjs', 'ex-test', 'unused.yaml', output], { + encoding: 'utf8', env: { ...process.env, PATH: directory } + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Forwarded-header discovery failed/); + assert.equal(existsSync(output), false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); From a8368d337748f8ec6c9e6a7a03dd10d392d1a83c Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 15 Oct 2026 19:36:58 -0500 Subject: [PATCH 13/26] Remove CI tooling tests and use JavaScript modules by default --- .github/workflows/build.yaml | 16 +-- AGENTS.md | 2 + docs/docs/self-hosting/kubernetes.md | 4 +- k8s/exceptionless/values.yaml | 2 +- ...ders.mjs => discover-forwarded-headers.js} | 2 +- package.json | 4 + .../ClientApp/api-templates/schemas.ejs | 2 +- src/Exceptionless.Web/ClientApp/package.json | 4 +- .../{generate-api.mjs => generate-api.js} | 0 .../{resolve-url.mjs => resolve-url.js} | 0 tests/ci/README.md | 14 +-- .../{backend-shards.mjs => backend-shards.js} | 2 +- .../{browser-report.mjs => browser-report.js} | 2 +- tests/ci/forwarded-headers-discovery.test.mjs | 109 ------------------ tests/ci/shards.test.mjs | 98 ---------------- 15 files changed, 26 insertions(+), 235 deletions(-) rename k8s/scripts/{discover-forwarded-headers.mjs => discover-forwarded-headers.js} (97%) create mode 100644 package.json rename src/Exceptionless.Web/ClientApp/scripts/{generate-api.mjs => generate-api.js} (100%) rename src/Exceptionless.Web/ClientApp/scripts/{resolve-url.mjs => resolve-url.js} (100%) rename tests/ci/{backend-shards.mjs => backend-shards.js} (98%) rename tests/ci/{browser-report.mjs => browser-report.js} (98%) delete mode 100644 tests/ci/forwarded-headers-discovery.test.mjs delete mode 100644 tests/ci/shards.test.mjs diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e4dc0d6ec5..71b0b94284 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -171,14 +171,8 @@ jobs: - name: Build run: dotnet build ./Exceptionless.slnx --no-restore --configuration Release - - name: Install Helm for offline deployment tests - uses: azure/setup-helm@v5 - - - name: Verify CI test tooling - run: node --test tests/ci/shards.test.mjs tests/ci/forwarded-headers-discovery.test.mjs - - name: Run .NET shard with coverage - run: node tests/ci/backend-shards.mjs run --index ${{ matrix.shard }} --count 4 --output coverage --coverage + run: node tests/ci/backend-shards.js run --index ${{ matrix.shard }} --count 4 --output coverage --coverage - name: Upload .NET shard results if: ${{ !cancelled() }} @@ -213,7 +207,7 @@ jobs: path: coverage-shards - name: Verify complete shard coverage and report timings - run: node tests/ci/backend-shards.mjs report coverage-shards --count 4 --coverage --write-timings coverage-shards/backend-durations.json + run: node tests/ci/backend-shards.js report coverage-shards --count 4 --coverage --write-timings coverage-shards/backend-durations.json - name: Setup .NET Core uses: actions/setup-dotnet@v6 @@ -514,7 +508,7 @@ jobs: E2E_SHARD: report run: | npx playwright test --project=chromium --list --reporter=json > test-results/discovery.json - node ../../../tests/ci/browser-report.mjs test-results/discovery.json test-results/e2e-results.json + node ../../../tests/ci/browser-report.js test-results/discovery.json test-results/e2e-results.json - name: Upload combined Playwright report if: ${{ !cancelled() }} @@ -726,7 +720,7 @@ jobs: if: ${{ needs.version.outputs.is_dev_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - node k8s/scripts/discover-forwarded-headers.mjs ex-dev k8s/ex-dev-values.yaml "$RUNNER_TEMP/forwarded-headers.json" + node k8s/scripts/discover-forwarded-headers.js ex-dev k8s/ex-dev-values.yaml "$RUNNER_TEMP/forwarded-headers.json" helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless - name: Run Development Migrations @@ -846,7 +840,7 @@ jobs: if: ${{ needs.version.outputs.is_prod_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - node k8s/scripts/discover-forwarded-headers.mjs ex-prod k8s/ex-prod-values.yaml "$RUNNER_TEMP/forwarded-headers.json" + node k8s/scripts/discover-forwarded-headers.js ex-prod k8s/ex-prod-values.yaml "$RUNNER_TEMP/forwarded-headers.json" helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" ex-prod --namespace ex-prod ./k8s/exceptionless - name: Summary diff --git a/AGENTS.md b/AGENTS.md index cd3883591d..f0e2ac567f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,8 @@ tests/ # C# tests and HTTP samples ## Testing and Safety +- **Keep testing proportional:** Do not add tests or CI steps for routine script wiring, configuration, file moves, or straightforward tooling changes. Prefer syntax checks, builds, or a focused manual check. Add regression tests only for meaningful behavior or a demonstrated bug, and extend existing coverage before creating another suite. Do not create tests that merely restate commands, configuration, or implementation details. +- **JavaScript modules:** Use `.js` with `"type": "module"` in the applicable `package.json` for repository-owned Node scripts. Use `.mjs` only when a specific tool or execution context requires it, and document that requirement. Do not add dependencies or CI jobs just to validate small scripts. - **C# test conventions:** Use explicit `// Arrange`, `// Act`, and `// Assert` sections; keep assertions out of the action. Inherit the appropriate existing test base (`TestWithLoggingBase` for isolated tests, `TestWithServices` for DI, or `IntegrationTestsBase` with `AppWebHostFactory` for application integration). Use the inherited `TestCancellationToken` and idiomatic `using var` declarations. Reuse repository fixtures and HTTP helpers rather than duplicating application startup or service registration. Read `.agents/skills/backend-testing/SKILL.md` before adding or changing backend tests. - **Test value and runtime:** Before adding a test, identify the distinct failure it catches and check existing coverage. Extend an existing test or use parameterized cases when the setup and behavior are the same. Combine related assertions in one scenario when they share expensive setup; keep independent behaviors separately diagnosable. Do not add tests for trivial accessors, framework behavior, implementation details, or duplicate coverage merely to increase test counts. - Use the cheapest reliable layer: pure logic in unit tests, service/API contracts in integration tests, and browser tests for user journeys or behavior that requires a real browser. Keep a representative browser integration case when moving a data matrix to unit/component tests. diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index c5014f5d1f..340f2d8c77 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -39,14 +39,14 @@ config: The repository's ingress templates use Azure Application Gateway for Containers, terminate TLS at ingress, and route `/` to the API Service on port 80 and container port 8080. The hosted development and production deployment workflow discovers the gateway's association subnet before each Helm upgrade, so subnet addresses do not need to be duplicated in environment values files. -The shared `k8s/scripts/discover-forwarded-headers.mjs` script renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. +The shared `k8s/scripts/discover-forwarded-headers.js` script renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. Discovery writes a `forwardedHeaders.knownNetworks` Helm list. The chart converts it into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys and a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining discovery with manual `ForwardedHeaders` entries in `config` is rejected; remove those old entries from saved release values before using discovery. For manual upgrades, run discovery from the repository root after selecting the intended Kubernetes context and Azure subscription, then pass the generated file **last** to Helm. For example, the development commands are: ```sh -node k8s/scripts/discover-forwarded-headers.mjs ex-dev k8s/ex-dev-values.yaml /tmp/ex-dev-forwarded-headers.json +node k8s/scripts/discover-forwarded-headers.js ex-dev k8s/ex-dev-values.yaml /tmp/ex-dev-forwarded-headers.json # Run only if discovery succeeded: helm upgrade ex-dev k8s/exceptionless --namespace ex-dev --reuse-values \ --values k8s/ex-dev-values.yaml --values /tmp/ex-dev-forwarded-headers.json diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index 1b65ec236f..23de64fb31 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -22,7 +22,7 @@ jobs: config: {} # key: value -# Populated by k8s/scripts/discover-forwarded-headers.mjs for hosted deployments. +# Populated by k8s/scripts/discover-forwarded-headers.js for hosted deployments. # A replacement list prevents stale subnet entries surviving --reuse-values. forwardedHeaders: knownNetworks: [] diff --git a/k8s/scripts/discover-forwarded-headers.mjs b/k8s/scripts/discover-forwarded-headers.js similarity index 97% rename from k8s/scripts/discover-forwarded-headers.mjs rename to k8s/scripts/discover-forwarded-headers.js index 5080be26a9..fe337e724d 100644 --- a/k8s/scripts/discover-forwarded-headers.mjs +++ b/k8s/scripts/discover-forwarded-headers.js @@ -53,7 +53,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) try { const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); if (!release || !valuesFile || !outputFile || extra.length) - throw new Error('Usage: node k8s/scripts/discover-forwarded-headers.mjs (from repository root)'); + throw new Error('Usage: node k8s/scripts/discover-forwarded-headers.js (from repository root)'); // A failed refresh must not leave an earlier trust file available for upgrade. rmSync(outputFile, { force: true }); const values = discoverForwardedHeaders(release, valuesFile); diff --git a/package.json b/package.json new file mode 100644 index 0000000000..44e4a84a61 --- /dev/null +++ b/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs b/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs index 58935cf119..9a8f0728ad 100644 --- a/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs +++ b/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs @@ -58,7 +58,7 @@ const typeToZod = (rawType, isRequired, nullable, field) => { const fieldName = field && field.name; // Handle nullable unions: swagger-typescript-api outputs "null | X" for most nullable fields, - // but outputs "X | null" for nullable arrays fixed via generate-api.mjs workaround. + // but outputs "X | null" for nullable arrays fixed via generate-api.js workaround. if (type.startsWith('null | ')) { let innerType = type.replace('null | ', ''); if (innerType.startsWith('(') && innerType.endsWith(')') && !innerType.endsWith('[]')) { diff --git a/src/Exceptionless.Web/ClientApp/package.json b/src/Exceptionless.Web/ClientApp/package.json index fcac4ba567..3d00cba7a3 100644 --- a/src/Exceptionless.Web/ClientApp/package.json +++ b/src/Exceptionless.Web/ClientApp/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "cross-env NODE_OPTIONS=--trace-warnings vite dev", "dev:api": "cross-env API_HTTP=https://dev-collector.exceptionless.io vite dev", - "urls": "node scripts/resolve-url.mjs", + "urls": "node scripts/resolve-url.js", "build": "vite build", "preview": "vite preview", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", @@ -17,7 +17,7 @@ "format:eslint": "eslint . --fix", "format:prettier": "prettier --write .", "validate": "npm run format && npm run check && npm run lint", - "generate-models": "node scripts/generate-api.mjs", + "generate-models": "node scripts/generate-api.js", "generate-templates": "swagger-typescript-api generate-templates -o api-templates", "test:e2e": "playwright test", "test:e2e:ci": "playwright test --project=chromium", diff --git a/src/Exceptionless.Web/ClientApp/scripts/generate-api.mjs b/src/Exceptionless.Web/ClientApp/scripts/generate-api.js similarity index 100% rename from src/Exceptionless.Web/ClientApp/scripts/generate-api.mjs rename to src/Exceptionless.Web/ClientApp/scripts/generate-api.js diff --git a/src/Exceptionless.Web/ClientApp/scripts/resolve-url.mjs b/src/Exceptionless.Web/ClientApp/scripts/resolve-url.js similarity index 100% rename from src/Exceptionless.Web/ClientApp/scripts/resolve-url.mjs rename to src/Exceptionless.Web/ClientApp/scripts/resolve-url.js diff --git a/tests/ci/README.md b/tests/ci/README.md index c150a3b910..9cb2f21749 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -4,9 +4,8 @@ The normal build targets a roughly **10-minute critical path**. Track elapsed wo ## Execution -- `.NET`: four isolated runners build the suite and discover tests through Microsoft Testing Platform's JSON discovery. `backend-shards.mjs` groups whole classes, then distributes the longest classes first using `backend-durations.json`. New classes are included automatically; stale timing entries cannot select or exclude tests. Each runner retains the existing six collection workers and coverage collection. -- CI tooling uses the project's Node.js runtime (Node 24 in CI), built-in modules, and `node:test`, with no additional packages. Backend summaries read xUnit's native CTRF JSON output; TRX reports remain available as artifacts. -- Frontend CI runs `build/update-config.test.mjs` against the real configuration writer and container entrypoints. Process stubs check configuration failures, argument preservation, and app process replacement without starting containers. CSP input cases use fast unit tests, with a representative HTTP response-header test covering integration. +- `.NET`: four isolated runners build the suite and discover tests through Microsoft Testing Platform's JSON discovery. `backend-shards.js` groups whole classes, then distributes the longest classes first using `backend-durations.json`. New classes are included automatically; stale timing entries cannot select or exclude tests. Each runner retains the existing six collection workers and coverage collection. +- CI tooling uses the project's Node.js runtime (Node 24 in CI) and built-in modules, with no additional packages. Backend summaries read xUnit's native CTRF JSON output; TRX reports remain available as artifacts. - Browser: six isolated Aspire applications run Playwright's native `--shard=N/6` with `fullyParallel` enabled for sharding. Each runner uses **one worker** because scenarios can change the shared administrator's preferences. This splits large spec files without introducing concurrent mutations of that user. Shard IDs are included in generated data names. - Frontend unit/component tests actually execute in `test-client`, with two Vitest workers to bound memory. Previously the workflow only echoed the command. - `test-api` and `test-e2e` remain the required checks. They depend on every shard, reject missing/duplicate/inconsistent results, and fail if any shard fails, is cancelled, or is skipped. `fail-fast: false` retains diagnostics from the other shards. @@ -43,15 +42,14 @@ Example local commands from the repository root (PowerShell): ```powershell dotnet build Exceptionless.slnx --configuration Release -node tests/ci/backend-shards.mjs run --index 1 --count 4 --output "$env:TEMP/api-shard-1" --coverage -node --test tests/ci/shards.test.mjs +node tests/ci/backend-shards.js run --index 1 --count 4 --output "$env:TEMP/api-shard-1" --coverage ``` Run local .NET shards **sequentially**: separate test processes on one host reuse test index scopes. Hosted shards are isolated machines. To validate or rebalance from downloaded artifacts: ```powershell gh run download --pattern 'api-results-*' --dir "$env:TEMP/api-results" -node tests/ci/backend-shards.mjs report "$env:TEMP/api-results" --count 4 --coverage --write-timings tests/ci/backend-durations.json +node tests/ci/backend-shards.js report "$env:TEMP/api-results" --count 4 --coverage --write-timings tests/ci/backend-durations.json ``` For a browser shard, start Aspire, discover and health-check its current local App URL, then from `src/Exceptionless.Web/ClientApp`: @@ -62,7 +60,7 @@ $env:E2E_SHARD = '1' npm run test:e2e:ci -- --shard=1/6 ``` -Keep the matrix counts, runner arguments, and aggregate completeness checks in sync when resizing either suite. The small CI-tool tests exercise lost/duplicated tests, inconsistent discovery, missing coverage, and failed/skipped browser results; they protect against a falsely green build. +Keep the matrix counts, runner arguments, and aggregate completeness checks in sync when resizing either suite. ## October 2026 audit @@ -104,7 +102,7 @@ Checked the resolved npm lockfile, NuGet packages (including transitive vulnerab | Storybook packages / Svelte CSF / Chromatic addon | 10.6.1 / 5.1.5 / 5.4.0 | Updated the component and visual testing toolchain together. | | Vite / Svelte Vite plugin | 8.3.2 / 7.3.1 | Updated the shared test/build host and plugin. | -Compatibility was checked against the packages' Node, Vite, Svelte, and Storybook peer requirements. CI uses Node 24; no preview packages or new script dependencies were introduced. The scripts consume native xUnit CTRF JSON and use the built-in Node test runner. The existing GitHub logger supports MTP, and the Visual Studio adapter remains useful for IDE discovery; neither is a redundant legacy package. +Compatibility was checked against the packages' Node, Vite, Svelte, and Storybook peer requirements. CI uses Node 24; no preview packages or new script dependencies were introduced. The scripts consume native xUnit CTRF JSON. The existing GitHub logger supports MTP, and the Visual Studio adapter remains useful for IDE discovery; neither is a redundant legacy package. Release references: [Playwright](https://playwright.dev/docs/release-notes), [xUnit](https://xunit.net/releases/), [Vitest](https://github.com/vitest-dev/vitest/releases/tag/v5.0.3), [jsdom](https://github.com/jsdom/jsdom/releases/tag/v30.1.1), [Storybook](https://github.com/storybookjs/storybook/releases/tag/v10.6.1), [Svelte CSF](https://github.com/storybookjs/addon-svelte-csf/releases/tag/v5.1.5), [Chromatic addon](https://github.com/chromaui/addon-visual-tests/releases/tag/v5.4.0), [Vite](https://github.com/vitejs/vite/releases/tag/v8.3.2), and [Svelte Vite plugin](https://github.com/sveltejs/vite-plugin-svelte/releases/tag/@sveltejs%2Fvite-plugin-svelte@7.3.1). diff --git a/tests/ci/backend-shards.mjs b/tests/ci/backend-shards.js similarity index 98% rename from tests/ci/backend-shards.mjs rename to tests/ci/backend-shards.js index fb2406616d..e0d9d9fa3d 100644 --- a/tests/ci/backend-shards.mjs +++ b/tests/ci/backend-shards.js @@ -207,7 +207,7 @@ function main() { if (positionals[0] === 'run' && positionals.length === 1) return run(values); if (positionals[0] !== 'report' || positionals.length !== 2) { throw new Error( - 'Usage: backend-shards.mjs run --index N --count N --output DIR [--coverage] | report DIR [--count N] [--coverage] [--write-timings FILE]' + 'Usage: backend-shards.js run --index N --count N --output DIR [--coverage] | report DIR [--count N] [--coverage] [--write-timings FILE]' ); } const result = report({ directory: positionals[1], count: values.count === undefined ? undefined : Number(values.count), coverage: values.coverage }); diff --git a/tests/ci/browser-report.mjs b/tests/ci/browser-report.js similarity index 98% rename from tests/ci/browser-report.mjs rename to tests/ci/browser-report.js index 53fc73402f..9e9bda95d0 100644 --- a/tests/ci/browser-report.mjs +++ b/tests/ci/browser-report.js @@ -34,7 +34,7 @@ export function validate(expected, actual) { function main() { const { positionals } = parseArgs({ allowPositionals: true }); - if (positionals.length !== 2) throw new Error('Usage: browser-report.mjs DISCOVERY_JSON RESULTS_JSON'); + if (positionals.length !== 2) throw new Error('Usage: browser-report.js DISCOVERY_JSON RESULTS_JSON'); const [expected, actual] = positionals.map((path) => JSON.parse(readFileSync(path, 'utf8'))); const tests = cases(actual); const duration = (test) => test.results.reduce((total, result) => total + result.duration, 0); diff --git a/tests/ci/forwarded-headers-discovery.test.mjs b/tests/ci/forwarded-headers-discovery.test.mjs deleted file mode 100644 index e1cb730f69..0000000000 --- a/tests/ci/forwarded-headers-discovery.test.mjs +++ /dev/null @@ -1,109 +0,0 @@ -import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import test from 'node:test'; -import { discoverForwardedHeaders } from '../../k8s/scripts/discover-forwarded-headers.mjs'; - -const subnetId = '/subscriptions/test/resourceGroups/gateway/providers/Microsoft.Network/virtualNetworks/ingress/subnets/association'; -const manifest = ' alb.networking.azure.io/alb-name: public-alb\n alb.networking.azure.io/alb-namespace: kube-system\n'; -const gateway = { spec: { associations: [subnetId] } }; -const subnet = { id: subnetId, addressPrefix: '192.0.2.0/24', delegations: [{ serviceName: 'Microsoft.ServiceNetworking/trafficControllers' }] }; - -function mock({ rendered = manifest, resource = gateway, network = subnet, fail } = {}) { - const calls = []; - return { - calls, - execute(command, args) { - calls.push([command, args]); - if (command === fail) throw new Error('Access denied'); - if (command === 'helm') return rendered; - if (command === 'kubectl') return JSON.stringify(resource); - if (command === 'az') return JSON.stringify(network); - throw new Error(`Unexpected command: ${command}`); - } - }; -} - -for (const environment of ['dev', 'prod']) { - test(`${environment}: discovers the rendered gateway and passes prefixes through the real chart`, () => { - // Helm renders locally; Azure and Kubernetes commands are always mocked. - const valuesFile = `k8s/ex-${environment}-values.yaml`; - const release = `ex-${environment}`; - const rendered = execFileSync('helm', ['template', release, 'k8s/exceptionless', '--values', valuesFile, '--show-only', 'templates/api.yaml'], { encoding: 'utf8' }); - const stub = mock({ rendered }); - const values = discoverForwardedHeaders(release, valuesFile, stub.execute); - assert.deepEqual(values, { forwardedHeaders: { knownNetworks: ['192.0.2.0/24'] } }); - assert.deepEqual(stub.calls[1], ['kubectl', ['get', 'applicationloadbalancers.alb.networking.azure.io', 'public-alb', '--namespace', 'kube-system', '--output', 'json']]); - assert.deepEqual(stub.calls[2], ['az', ['network', 'vnet', 'subnet', 'show', '--ids', subnetId, '--output', 'json', '--only-show-errors']]); - - const directory = mkdtempSync(join(tmpdir(), 'forwarded-headers-')); - try { - const previous = join(directory, 'previous.json'); - const discovered = join(directory, 'discovered.json'); - writeFileSync(previous, JSON.stringify({ forwardedHeaders: { knownNetworks: ['198.51.100.0/24', '203.0.113.0/24'] } })); - writeFileSync(discovered, JSON.stringify(values)); - const args = ['template', release, 'k8s/exceptionless', '--values', valuesFile, '--values', previous, '--values', discovered, '--show-only', 'templates/config.yaml']; - const config = execFileSync('helm', args, { encoding: 'utf8' }); - assert.match(config, /EX_ForwardedHeaders__ForwardLimit: "1"/); - assert.match(config, /EX_ForwardedHeaders__KnownNetworks__0: "192\.0\.2\.0\/24"/); - assert.doesNotMatch(config, /KnownNetworks__1|198\.51\.100|203\.0\.113/); - for (const key of ['EX_ForwardedHeaders__KnownNetworks__2', 'ForwardedHeaders__KnownProxies__0', 'EX_ForwardedHeaders__ForwardLimit']) { - assert.throws(() => execFileSync('helm', [...args, '--set-string', `config.${key}=1`], { stdio: 'pipe' }), /Do not combine discovered/); - } - } finally { - rmSync(directory, { recursive: true, force: true }); - } - }); -} - -test('deduplicates association IDs and IPv4/IPv6 prefixes', () => { - const stub = mock({ resource: { spec: { associations: [subnetId, subnetId] } }, network: { ...subnet, addressPrefixes: ['192.0.2.0/24', '2001:db8::/64'] } }); - assert.deepEqual(discoverForwardedHeaders('ex-test', 'values.yaml', stub.execute).forwardedHeaders.knownNetworks, ['192.0.2.0/24', '2001:db8::/64']); - assert.equal(stub.calls.filter(([command]) => command === 'az').length, 1); -}); - -for (const [name, options, message] of [ - ['missing ingress', { rendered: '' }, /annotation/], - ['conflicting gateway', { rendered: `${manifest} alb.networking.azure.io/alb-name: other-alb\n` }, /annotation/], - ['missing association', { resource: { spec: {} } }, /no association/], - ['invalid association', { resource: { spec: { associations: ['--help'] } } }, /resource ID/], - ['wrong subnet', { network: { ...subnet, id: `${subnetId}-other` } }, /identity/], - ['missing delegation', { network: { ...subnet, delegations: [] } }, /delegation/], - ['missing prefix', { network: { ...subnet, addressPrefix: undefined } }, /no address prefixes/], - ...['0.0.0.0/0', '::/0', '192.0.2.0/33', 'bad/24', '192.0.2.0/24/extra'].map(prefix => [prefix, { network: { ...subnet, addressPrefix: prefix } }, /Invalid or unrestricted/]), - ['Kubernetes denied', { fail: 'kubectl' }, /Access denied/], - ['Azure denied', { fail: 'az' }, /Access denied/] -]) { - test(`refuses discovery: ${name}`, () => { - const stub = mock(options); - assert.throws(() => discoverForwardedHeaders('ex-test', 'values.yaml', stub.execute), message); - }); -} - -test('both deployment paths require discovery and consume its output', () => { - const workflow = readFileSync('.github/workflows/build.yaml', 'utf8'); - for (const environment of ['dev', 'prod']) { - assert.ok(workflow.includes(`node k8s/scripts/discover-forwarded-headers.mjs ex-${environment} k8s/ex-${environment}-values.yaml "$RUNNER_TEMP/forwarded-headers.json"`)); - assert.ok(workflow.includes(`--values ./k8s/ex-${environment}-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json"`)); - } -}); - -test('CLI failure exits nonzero and removes stale generated trust', () => { - const directory = mkdtempSync(join(tmpdir(), 'forwarded-headers-cli-')); - try { - // Restrict PATH to a failing fake Helm; no cluster or Azure tools can run. - writeFileSync(join(directory, 'helm'), '#!/bin/sh\nexit 7\n', { mode: 0o700 }); - const output = join(directory, 'values.json'); - writeFileSync(output, JSON.stringify({ forwardedHeaders: { knownNetworks: ['198.51.100.0/24'] } })); - const result = spawnSync(process.execPath, ['k8s/scripts/discover-forwarded-headers.mjs', 'ex-test', 'unused.yaml', output], { - encoding: 'utf8', env: { ...process.env, PATH: directory } - }); - assert.equal(result.status, 1); - assert.match(result.stderr, /Forwarded-header discovery failed/); - assert.equal(existsSync(output), false); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); diff --git a/tests/ci/shards.test.mjs b/tests/ci/shards.test.mjs deleted file mode 100644 index 935b891392..0000000000 --- a/tests/ci/shards.test.mjs +++ /dev/null @@ -1,98 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import { partition, report } from './backend-shards.mjs'; -import { validate } from './browser-report.mjs'; - -test('partition covers new classes once and is deterministic', () => { - const classes = ['slow', 'medium', 'fast', 'new', 'another-new']; - const timings = { slow: 90, medium: 60, fast: 30, deleted: 999 }; - const shards = partition(classes, timings, 3); - assert.deepEqual(shards.flat().sort(), [...classes].sort()); - assert.deepEqual(shards, partition([...classes].reverse(), timings, 3)); - assert.equal(new Set(['slow', 'medium', 'fast'].map((name) => shards.findIndex((shard) => shard.includes(name)))).size, 3); - for (const count of [0, 6, 1.5]) { - assert.throws(() => partition(classes, timings, count)); - } -}); - -test('backend gate rejects incomplete or inconsistent shards and reads xUnit JSON timings', () => { - for (const defect of [ - null, - 'missing', - 'duplicate', - 'discovery', - 'omitted', - 'execution', - 'coverage', - 'trx', - 'failed', - 'skipped', - 'pending', - 'global-error', - 'empty', - 'invalid-duration' - ]) { - const directory = mkdtempSync(join(tmpdir(), 'exceptionless-ci-')); - try { - for (const index of [1, 2]) { - const shard = join(directory, String(index)); - mkdirSync(shard); - let name = `Example.Test${index}`; - const manifest = { index, count: 2, discovery_hash: 'same', discovered_count: 2, classes: [name] }; - if (index === 2) { - if (defect === 'missing') continue; - if (defect === 'duplicate') manifest.index = 1; - if (defect === 'discovery') manifest.discovery_hash = 'different'; - if (defect === 'omitted') manifest.classes = []; - if (defect === 'execution') name = 'Unexpected.Test'; - } - writeFileSync(join(shard, 'manifest.json'), JSON.stringify(manifest)); - if (defect !== 'coverage' || index !== 2) writeFileSync(join(shard, 'coverage.cobertura.xml'), ''); - if (defect !== 'trx' || index !== 2) writeFileSync(join(shard, 'test-results.trx'), ''); - const status = index === 2 && ['failed', 'skipped', 'pending'].includes(defect) ? defect : 'passed'; - const result = { - reportFormat: 'CTRF', - results: { - tests: [{ name: `${name}.Case(" & \\"quoted\\"")`, status, duration: 1500, extra: { type: name } }], - extra: { suites: [{ errors: [] }] } - } - }; - if (index === 2) { - if (defect === 'global-error') result.results.extra.suites[0].errors.push({ message: 'Fixture failed' }); - if (defect === 'empty') result.results.tests = []; - if (defect === 'invalid-duration') result.results.tests[0].duration = 'invalid'; - } - writeFileSync(join(shard, 'test-results.json'), JSON.stringify(result)); - } - const args = { directory, count: 2, coverage: true }; - if ([null, 'failed', 'skipped', 'pending'].includes(defect)) { - const result = report(args); - assert.equal(result.exitCode, Number(['failed', 'pending'].includes(defect)), defect); - assert.deepEqual(result.durations, { 'Example.Test1': 1.5, 'Example.Test2': 1.5 }); - } else { - assert.throws(() => report(args), undefined, defect); - } - } finally { - rmSync(directory, { recursive: true, force: true }); - } - } -}); - -test('browser gate checks discovery, results, and retries', () => { - const browserTest = { projectName: 'chromium', status: 'expected', results: [{ duration: 500 }] }; - const expected = { suites: [{ specs: [{ id: 'case', title: 'browser behavior', tests: [browserTest] }] }] }; - for (const defect of [null, 'missing', 'duplicate', 'skipped', 'unexpected', 'no-result', 'global-error', 'flaky']) { - const actual = structuredClone(expected); - const spec = actual.suites[0].specs[0]; - if (defect === 'missing') actual.suites = []; - else if (defect === 'duplicate') actual.suites.push(structuredClone(actual.suites[0])); - else if (['skipped', 'unexpected', 'flaky'].includes(defect)) spec.tests[0].status = defect; - else if (defect === 'no-result') spec.tests[0].results = []; - else if (defect === 'global-error') actual.errors = [{ message: 'worker crashed' }]; - if (defect === null) validate(expected, actual); - else assert.throws(() => validate(expected, actual), undefined, defect); - } -}); From 370d284c2c97184fd5601a7d02faec9df8519bce Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 15 Oct 2026 19:45:09 -0500 Subject: [PATCH 14/26] Harden and clarify gateway trust discovery --- AGENTS.md | 1 + k8s/scripts/discover-forwarded-headers.js | 206 +++++++++++++++++----- 2 files changed, 167 insertions(+), 40 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f0e2ac567f..4ba5aec1e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,7 @@ tests/ # C# tests and HTTP samples - **Keep testing proportional:** Do not add tests or CI steps for routine script wiring, configuration, file moves, or straightforward tooling changes. Prefer syntax checks, builds, or a focused manual check. Add regression tests only for meaningful behavior or a demonstrated bug, and extend existing coverage before creating another suite. Do not create tests that merely restate commands, configuration, or implementation details. - **JavaScript modules:** Use `.js` with `"type": "module"` in the applicable `package.json` for repository-owned Node scripts. Use `.mjs` only when a specific tool or execution context requires it, and document that requirement. Do not add dependencies or CI jobs just to validate small scripts. +- **JavaScript control flow:** Always use braces for `if`, `for`, `while`, and similar blocks, even for one statement. Put block contents on separate lines and leave a blank line after a completed block unless it is followed by `else`, `catch`, or `finally`. - **C# test conventions:** Use explicit `// Arrange`, `// Act`, and `// Assert` sections; keep assertions out of the action. Inherit the appropriate existing test base (`TestWithLoggingBase` for isolated tests, `TestWithServices` for DI, or `IntegrationTestsBase` with `AppWebHostFactory` for application integration). Use the inherited `TestCancellationToken` and idiomatic `using var` declarations. Reuse repository fixtures and HTTP helpers rather than duplicating application startup or service registration. Read `.agents/skills/backend-testing/SKILL.md` before adding or changing backend tests. - **Test value and runtime:** Before adding a test, identify the distinct failure it catches and check existing coverage. Extend an existing test or use parameterized cases when the setup and behavior are the same. Combine related assertions in one scenario when they share expensive setup; keep independent behaviors separately diagnosable. Do not add tests for trivial accessors, framework behavior, implementation details, or duplicate coverage merely to increase test counts. - Use the cheapest reliable layer: pure logic in unit tests, service/API contracts in integration tests, and browser tests for user journeys or behavior that requires a real browser. Keep a representative browser integration case when moving a data matrix to unit/component tests. diff --git a/k8s/scripts/discover-forwarded-headers.js b/k8s/scripts/discover-forwarded-headers.js index fe337e724d..b81a1ef1bf 100644 --- a/k8s/scripts/discover-forwarded-headers.js +++ b/k8s/scripts/discover-forwarded-headers.js @@ -3,62 +3,188 @@ import { rmSync, writeFileSync } from 'node:fs'; import { isIP } from 'node:net'; import { pathToFileURL } from 'node:url'; -const run = (command, args) => execFileSync(command, args, { encoding: 'utf8', timeout: 60_000 }); +const APPLICATION_LOAD_BALANCER_RESOURCE = 'applicationloadbalancers.alb.networking.azure.io'; +const ASSOCIATION_SUBNET_ID = + /^\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.Network\/virtualNetworks\/[^/]+\/subnets\/[^/]+$/i; +const TRAFFIC_CONTROLLER_DELEGATION = 'Microsoft.ServiceNetworking/trafficControllers'; +const USAGE = + 'Usage: node k8s/scripts/discover-forwarded-headers.js (from repository root)'; + +function run(command, arguments_) { + return execFileSync(command, arguments_, { + encoding: 'utf8', + timeout: 60_000 + }); +} + +function runJson(command, arguments_, description) { + const output = run(command, arguments_); + + try { + return JSON.parse(output); + } catch (error) { + throw new Error(`${description} returned invalid JSON`, { cause: error }); + } +} // Read the chart's rendered annotations so gateway selection cannot drift from ingress. -function annotation(manifest, name) { - const values = [...manifest.matchAll(new RegExp(`^\\s+alb\\.networking\\.azure\\.io/${name}:\\s*([a-z0-9.-]+)\\s*$`, 'gm'))] - .map(match => match[1]); - if (!values.length || new Set(values).size !== 1) - throw new Error(`Expected one consistent ingress annotation: ${name}`); +function readIngressAnnotation(manifest, annotationName) { + const annotationPattern = new RegExp( + `^\\s+alb\\.networking\\.azure\\.io/${annotationName}:\\s*([a-z0-9.-]+)\\s*$`, + 'gm' + ); + const values = [...new Set([...manifest.matchAll(annotationPattern)].map(match => match[1]))]; + + if (values.length === 0) { + throw new Error(`Rendered API ingress is missing the ${annotationName} annotation`); + } + + if (values.length > 1) { + throw new Error(`Rendered API ingresses use conflicting ${annotationName} annotations: ${values.join(', ')}`); + } + return values[0]; } -export function discoverForwardedHeaders(release, valuesFile, execute = run) { - const manifest = execute('helm', ['template', release, 'k8s/exceptionless', '--values', valuesFile, '--show-only', 'templates/api.yaml']); - const name = annotation(manifest, 'alb-name'); - const namespace = annotation(manifest, 'alb-namespace'); - const gateway = JSON.parse(execute('kubectl', ['get', 'applicationloadbalancers.alb.networking.azure.io', name, '--namespace', namespace, '--output', 'json'])); +function readAssociationSubnetIds(gatewayName, gatewayNamespace) { + const gateway = runJson( + 'kubectl', + ['get', APPLICATION_LOAD_BALANCER_RESOURCE, gatewayName, '--namespace', gatewayNamespace, '--output', 'json'], + `ApplicationLoadBalancer ${gatewayNamespace}/${gatewayName}` + ); + + if (!gateway || typeof gateway !== 'object' || Array.isArray(gateway)) { + throw new Error(`ApplicationLoadBalancer ${gatewayNamespace}/${gatewayName} returned an invalid object`); + } + const associations = gateway.spec?.associations; - if (!Array.isArray(associations) || !associations.length) + + if (!Array.isArray(associations) || associations.length === 0) { throw new Error('Gateway has no association subnet IDs; refusing to infer proxy trust'); + } + + const hasInvalidAssociation = associations.some( + association => typeof association !== 'string' || !ASSOCIATION_SUBNET_ID.test(association) + ); + + if (hasInvalidAssociation) { + throw new Error('Gateway contains an invalid Azure association subnet resource ID'); + } + + return [...new Set(associations)]; +} + +function readSubnetPrefixes(subnetId) { + const subnet = runJson( + 'az', + [ + 'network', + 'vnet', + 'subnet', + 'show', + '--ids', + subnetId, + '--query', + '{id:id,addressPrefix:addressPrefix,addressPrefixes:addressPrefixes,delegations:delegations[].serviceName}', + '--output', + 'json', + '--only-show-errors' + ], + `Azure subnet ${subnetId}` + ); + + if (!subnet || typeof subnet !== 'object' || Array.isArray(subnet)) { + throw new Error(`Azure subnet ${subnetId} returned an invalid object`); + } + + if (typeof subnet.id !== 'string' || subnet.id.toLowerCase() !== subnetId.toLowerCase()) { + throw new Error(`Azure returned a different subnet for association ${subnetId}`); + } - const networks = []; - for (const id of new Set(associations)) { - if (typeof id !== 'string' || !/^\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.Network\/virtualNetworks\/[^/]+\/subnets\/[^/]+$/i.test(id)) - throw new Error('Gateway association is not an Azure subnet resource ID'); - const subnet = JSON.parse(execute('az', ['network', 'vnet', 'subnet', 'show', '--ids', id, '--output', 'json', '--only-show-errors'])); - if (subnet.id?.toLowerCase() !== id.toLowerCase() || - !subnet.delegations?.some(item => item.serviceName === 'Microsoft.ServiceNetworking/trafficControllers')) - throw new Error('Association subnet identity or Application Gateway for Containers delegation is missing'); - - const prefixes = [subnet.addressPrefix, ...(subnet.addressPrefixes ?? [])].filter(value => value != null); - if (!prefixes.length) - throw new Error('Association subnet has no address prefixes'); - for (const prefix of prefixes) { - const [address, length, extra] = typeof prefix === 'string' ? prefix.split('/') : []; - const family = isIP(address ?? ''); - if (!family || extra !== undefined || !/^[1-9][0-9]*$/.test(length ?? '') || Number(length) > (family === 4 ? 32 : 128)) - throw new Error(`Invalid or unrestricted association subnet prefix: ${prefix}`); - networks.push(prefix); + if (!Array.isArray(subnet.delegations) || !subnet.delegations.includes(TRAFFIC_CONTROLLER_DELEGATION)) { + throw new Error(`Association subnet ${subnetId} is not delegated to ${TRAFFIC_CONTROLLER_DELEGATION}`); + } + + const hasInvalidAddressPrefix = + subnet.addressPrefix !== null && subnet.addressPrefix !== undefined && typeof subnet.addressPrefix !== 'string'; + + if (hasInvalidAddressPrefix) { + throw new Error(`Association subnet ${subnetId} returned an invalid addressPrefix`); + } + + const hasInvalidAddressPrefixes = + subnet.addressPrefixes !== null && + subnet.addressPrefixes !== undefined && + (!Array.isArray(subnet.addressPrefixes) || + !subnet.addressPrefixes.every(prefix => typeof prefix === 'string')); + + if (hasInvalidAddressPrefixes) { + throw new Error(`Association subnet ${subnetId} returned invalid addressPrefixes`); + } + + const addressPrefixes = subnet.addressPrefixes ?? []; + const prefixes = [subnet.addressPrefix, ...addressPrefixes].filter(prefix => prefix !== null && prefix !== undefined); + + if (prefixes.length === 0) { + throw new Error(`Association subnet ${subnetId} has no address prefixes`); + } + + for (const prefix of prefixes) { + const [address, prefixLength, extra] = typeof prefix === 'string' ? prefix.split('/') : []; + const addressFamily = isIP(address ?? ''); + const maximumPrefixLength = addressFamily === 4 ? 32 : 128; + const hasValidPrefixLength = + /^[1-9][0-9]*$/.test(prefixLength ?? '') && Number(prefixLength) <= maximumPrefixLength; + + if (!addressFamily || extra !== undefined || !hasValidPrefixLength) { + throw new Error(`Association subnet ${subnetId} has an invalid or unrestricted prefix: ${prefix}`); } } + return prefixes; +} + +function discoverKnownNetworks(release, valuesFile) { + const manifest = run('helm', [ + 'template', + release, + 'k8s/exceptionless', + '--values', + valuesFile, + '--show-only', + 'templates/api.yaml' + ]); + const gatewayName = readIngressAnnotation(manifest, 'alb-name'); + const gatewayNamespace = readIngressAnnotation(manifest, 'alb-namespace'); + const subnetIds = readAssociationSubnetIds(gatewayName, gatewayNamespace); + const networks = subnetIds.flatMap(subnetId => readSubnetPrefixes(subnetId)); + // A list replaces the previous list on Helm upgrades, including --reuse-values. // Individual config-map keys would retain stale higher-numbered entries. - return { forwardedHeaders: { knownNetworks: [...new Set(networks)].sort() } }; + return [...new Set(networks)].sort(); +} + +function main() { + const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); + + if (!release || !valuesFile || !outputFile || extra.length > 0) { + throw new Error(USAGE); + } + + // A failed refresh must not leave an earlier trust file available for upgrade. + rmSync(outputFile, { force: true }); + + const knownNetworks = discoverKnownNetworks(release, valuesFile); + const values = { forwardedHeaders: { knownNetworks } }; + writeFileSync(outputFile, `${JSON.stringify(values, null, 2)}\n`, { mode: 0o600 }); + console.log(`Discovered ${knownNetworks.length} gateway subnet prefix(es)`); } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { +const isMainModule = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isMainModule) { try { - const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); - if (!release || !valuesFile || !outputFile || extra.length) - throw new Error('Usage: node k8s/scripts/discover-forwarded-headers.js (from repository root)'); - // A failed refresh must not leave an earlier trust file available for upgrade. - rmSync(outputFile, { force: true }); - const values = discoverForwardedHeaders(release, valuesFile); - writeFileSync(outputFile, `${JSON.stringify(values, null, 2)}\n`, { mode: 0o600 }); - console.log(`Discovered ${values.forwardedHeaders.knownNetworks.length} gateway subnet prefix(es)`); + main(); } catch (error) { console.error(`Forwarded-header discovery failed: ${error.message}`); process.exitCode = 1; From 6a79c55a5e140120a118c55324325507466efd78 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 15 Oct 2026 20:05:23 -0500 Subject: [PATCH 15/26] Harden forwarded-header discovery output --- k8s/scripts/discover-forwarded-headers.js | 47 ++++++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/k8s/scripts/discover-forwarded-headers.js b/k8s/scripts/discover-forwarded-headers.js index b81a1ef1bf..47eaee662e 100644 --- a/k8s/scripts/discover-forwarded-headers.js +++ b/k8s/scripts/discover-forwarded-headers.js @@ -1,11 +1,15 @@ import { execFileSync } from 'node:child_process'; -import { rmSync, writeFileSync } from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'; import { isIP } from 'node:net'; +import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; const APPLICATION_LOAD_BALANCER_RESOURCE = 'applicationloadbalancers.alb.networking.azure.io'; const ASSOCIATION_SUBNET_ID = /^\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.Network\/virtualNetworks\/[^/]+\/subnets\/[^/]+$/i; +const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; +const MAX_HELM_RELEASE_NAME_LENGTH = 53; const TRAFFIC_CONTROLLER_DELEGATION = 'Microsoft.ServiceNetworking/trafficControllers'; const USAGE = 'Usage: node k8s/scripts/discover-forwarded-headers.js (from repository root)'; @@ -164,6 +168,37 @@ function discoverKnownNetworks(release, valuesFile) { return [...new Set(networks)].sort(); } +function writeValuesFile(outputFile, values) { + const temporaryFile = `${outputFile}.${randomUUID()}.tmp`; + + try { + writeFileSync(temporaryFile, `${JSON.stringify(values, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600 + }); + renameSync(temporaryFile, outputFile); + } finally { + rmSync(temporaryFile, { force: true }); + } +} + +function pathsReferToSameFile(firstPath, secondPath) { + if (resolve(firstPath) === resolve(secondPath)) { + return true; + } + + try { + return realpathSync(firstPath) === realpathSync(secondPath); + } catch (error) { + if (error.code === 'ENOENT') { + return false; + } + + throw error; + } +} + function main() { const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); @@ -171,12 +206,20 @@ function main() { throw new Error(USAGE); } + if (release.length > MAX_HELM_RELEASE_NAME_LENGTH || !HELM_RELEASE_NAME.test(release)) { + throw new Error('The Helm release name is invalid'); + } + + if (pathsReferToSameFile(valuesFile, outputFile)) { + throw new Error('The output file must be different from the input values file'); + } + // A failed refresh must not leave an earlier trust file available for upgrade. rmSync(outputFile, { force: true }); const knownNetworks = discoverKnownNetworks(release, valuesFile); const values = { forwardedHeaders: { knownNetworks } }; - writeFileSync(outputFile, `${JSON.stringify(values, null, 2)}\n`, { mode: 0o600 }); + writeValuesFile(outputFile, values); console.log(`Discovered ${knownNetworks.length} gateway subnet prefix(es)`); } From d48b97a0a5e85d2d696db9813c1215d98f53e98c Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 16 Oct 2026 14:09:41 -0500 Subject: [PATCH 16/26] Enforce script braces and reject unrestricted forwarding --- docs/docs/self-hosting/index.md | 2 +- eslint.config.js | 1 + package.json | 5 ++- .../ClientApp/eslint.scripts.config.js | 21 ++++++++++ src/Exceptionless.Web/ClientApp/package.json | 5 ++- .../Security/ForwardedHeadersConfiguration.cs | 15 +++++++ .../ForwardedHeadersConfigurationTests.cs | 10 +++-- tests/ci/README.md | 1 + tests/ci/backend-shards.js | 41 +++++++++++++++++-- tests/ci/browser-report.js | 19 +++++++-- 10 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 eslint.config.js create mode 100644 src/Exceptionless.Web/ClientApp/eslint.scripts.config.js diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index 9aadcdc22b..daf9dec044 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -36,7 +36,7 @@ EX_ForwardedHeaders__ForwardLimit: "1" For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup. -`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero disables processing; negative or non-integer limits are invalid. Do not enable the unrestricted `ASPNETCORE_FORWARDEDHEADERS_ENABLED` shortcut. +`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero disables processing; negative or non-integer limits are invalid. Do not enable the unrestricted `ASPNETCORE_FORWARDEDHEADERS_ENABLED` shortcut. Startup rejects an active `ForwardedHeaders_Enabled=true` setting because the framework clears its trusted-peer lists when this setting is enabled. The public edge proxy must derive the scheme from its connection and replace untrusted forwarding headers, or append the observed client address so the configured hop limit cannot reach user-supplied entries. Configure Nginx, Caddy, Traefik, or other proxies accordingly. Loopback proxies and IIS out-of-process integration normally need no additional trust entries. A remote IIS/ARR proxy needs the same explicit peer configuration as other remote proxies. diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000000..90c62b2204 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1 @@ +export { default } from './src/Exceptionless.Web/ClientApp/eslint.scripts.config.js'; diff --git a/package.json b/package.json index 44e4a84a61..88f236b785 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,7 @@ { "private": true, - "type": "module" + "type": "module", + "scripts": { + "lint:scripts": "node src/Exceptionless.Web/ClientApp/node_modules/eslint/bin/eslint.js tests/ci k8s/scripts" + } } diff --git a/src/Exceptionless.Web/ClientApp/eslint.scripts.config.js b/src/Exceptionless.Web/ClientApp/eslint.scripts.config.js new file mode 100644 index 0000000000..7dd0a35e1a --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/eslint.scripts.config.js @@ -0,0 +1,21 @@ +import js from '@eslint/js'; +import stylistic from '@stylistic/eslint-plugin'; +import globals from 'globals'; + +export default [ + { + files: ['tests/ci/**/*.js', 'k8s/scripts/**/*.js'], + languageOptions: { + globals: globals.node + }, + plugins: { + '@stylistic': stylistic + }, + rules: { + ...js.configs.recommended.rules, + '@stylistic/brace-style': ['error', '1tbs', { allowSingleLine: false }], + curly: ['error', 'all'], + 'padding-line-between-statements': ['error', { blankLine: 'always', next: '*', prev: 'block-like' }] + } + } +]; diff --git a/src/Exceptionless.Web/ClientApp/package.json b/src/Exceptionless.Web/ClientApp/package.json index 16b559c515..cd2451f94b 100644 --- a/src/Exceptionless.Web/ClientApp/package.json +++ b/src/Exceptionless.Web/ClientApp/package.json @@ -11,10 +11,11 @@ "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "npm run lint:prettier && npm run lint:eslint", - "lint:eslint": "eslint .", + "lint:eslint": "eslint . && npm run lint:scripts", + "lint:scripts": "npm --prefix ../../.. run lint:scripts --", "lint:prettier": "prettier --check .", "format": "npm run format:prettier && npm run format:eslint", - "format:eslint": "eslint . --fix", + "format:eslint": "eslint . --fix && npm run lint:scripts -- --fix", "format:prettier": "prettier --write .", "validate": "npm run format && npm run check && npm run lint", "generate-models": "node scripts/generate-api.js", diff --git a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs index e9ee2614b4..a8aa640dc8 100644 --- a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs @@ -7,17 +7,26 @@ public static class ForwardedHeadersConfiguration { public static void Configure(ForwardedHeadersOptions options, IConfiguration configuration) { + if (String.Equals(configuration["ForwardedHeaders_Enabled"], "true", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("ForwardedHeaders_Enabled must not be enabled. Configure ForwardedHeaders:KnownProxies or ForwardedHeaders:KnownNetworks instead."); + } + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; options.RequireHeaderSymmetry = false; options.ForwardLimit = configuration.GetValue("ForwardedHeaders:ForwardLimit", 1); if (options.ForwardLimit is < 0) + { throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be zero or positive."); + } // Retain the framework's loopback defaults and add explicitly trusted ingress peers. foreach (var entry in GetEntries(configuration, "KnownProxies")) { if (!IPAddress.TryParse(entry.Value, out var address)) + { throw new InvalidOperationException($"{entry.Path} must be an IP address without a port."); + } options.KnownProxies.Add(address); } @@ -25,10 +34,14 @@ public static void Configure(ForwardedHeadersOptions options, IConfiguration con foreach (var entry in GetEntries(configuration, "KnownNetworks")) { if (!System.Net.IPNetwork.TryParse(entry.Value, out var network)) + { throw new InvalidOperationException($"{entry.Path} must be a valid IPv4 or IPv6 CIDR."); + } if (network.PrefixLength is 0) + { throw new InvalidOperationException($"{entry.Path} must identify a trusted ingress network, not a /0 network."); + } options.KnownIPNetworks.Add(network); } @@ -38,7 +51,9 @@ private static IEnumerable GetEntries(IConfiguration conf { var section = configuration.GetSection($"ForwardedHeaders:{name}"); if (section.Value is not null) + { throw new InvalidOperationException($"{section.Path} must be a list. Use numbered environment keys such as EX_ForwardedHeaders__{name}__0."); + } return section.GetChildren(); } diff --git a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs index a621afc27d..6cf1f3259c 100644 --- a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs +++ b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs @@ -96,13 +96,15 @@ public async Task Invoke_DirectRequest_PreservesConnectionAddressAndScheme(strin Assert.Equal(scheme, context.Request.Scheme); } - [Fact] - public void Configure_NegativeForwardLimit_Throws() + [Theory] + [InlineData("ForwardedHeaders:ForwardLimit", "-1")] + [InlineData("ForwardedHeaders_Enabled", "true")] + public void Configure_UnsafeScalarConfiguration_ThrowsWithKey(string key, string value) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { - ["ForwardedHeaders:ForwardLimit"] = "-1" + [key] = value }).Build(); // Act @@ -110,7 +112,7 @@ public void Configure_NegativeForwardLimit_Throws() // Assert Assert.IsType(exception); - Assert.Contains("ForwardedHeaders:ForwardLimit", exception.Message, StringComparison.Ordinal); + Assert.Contains(key, exception.Message, StringComparison.Ordinal); } [Theory] diff --git a/tests/ci/README.md b/tests/ci/README.md index 9cb2f21749..ec4cd1ceba 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -4,6 +4,7 @@ The normal build targets a roughly **10-minute critical path**. Track elapsed wo ## Execution +- Run `npm run lint:scripts` from the repository root after installing ClientApp dependencies. The existing ClientApp lint command also checks `tests/ci` and `k8s/scripts` for required braces, multiline blocks, and a blank line after blocks. - `.NET`: four isolated runners build the suite and discover tests through Microsoft Testing Platform's JSON discovery. `backend-shards.js` groups whole classes, then distributes the longest classes first using `backend-durations.json`. New classes are included automatically; stale timing entries cannot select or exclude tests. Each runner retains the existing six collection workers and coverage collection. - CI tooling uses the project's Node.js runtime (Node 24 in CI) and built-in modules, with no additional packages. Backend summaries read xUnit's native CTRF JSON output; TRX reports remain available as artifacts. - Browser: six isolated Aspire applications run Playwright's native `--shard=N/6` with `fullyParallel` enabled for sharding. Each runner uses **one worker** because scenarios can change the shared administrator's preferences. This splits large spec files without introducing concurrent mutations of that user. Shard IDs are included in generated data names. diff --git a/tests/ci/backend-shards.js b/tests/ci/backend-shards.js index e0d9d9fa3d..bcfafdb680 100644 --- a/tests/ci/backend-shards.js +++ b/tests/ci/backend-shards.js @@ -14,6 +14,7 @@ export function partition(classes, durations, count) { if (!Number.isInteger(count) || count < 1 || classes.length < count || new Set(classes).size !== classes.length) { throw new Error('Shards must be nonempty and discovered classes must be unique'); } + const shards = Array.from({ length: count }, () => []); const totals = Array(count).fill(0); // Stable sorting keeps equal-duration classes in name order. New classes are always included. @@ -25,6 +26,7 @@ export function partition(classes, durations, count) { shards[index].push(name); totals[index] += Math.max(durations[name] ?? 1, 0.01); } + return shards.map((shard) => shard.sort()); } @@ -33,13 +35,16 @@ export function readResults(path) { if (report.reportFormat !== 'CTRF' || !Array.isArray(report.results?.tests) || !report.results.tests.length) { throw new Error(`No CTRF test results in ${path}`); } + if (report.results.extra?.suites?.some((suite) => suite.errors?.length)) { throw new Error(`xUnit reported errors outside individual tests: ${path}`); } + return report.results.tests.map((test) => { if (!test.extra?.type || typeof test.name !== 'string' || typeof test.status !== 'string' || !Number.isFinite(test.duration) || test.duration < 0) { throw new Error(`Invalid xUnit CTRF test result in ${path}`); } + return { class: test.extra.type, name: test.name, seconds: test.duration / 1000, outcome: test.status }; }); } @@ -49,6 +54,7 @@ export function summarize(results) { for (const result of results) { durations.set(result.class, (durations.get(result.class) ?? 0) + result.seconds); } + return Object.fromEntries([...durations.keys()].sort().map((name) => [name, Number(durations.get(name).toFixed(3))])); } @@ -63,6 +69,7 @@ function run(args) { if (discovered.schemaVersion !== 1) { throw new Error('Unsupported Microsoft Testing Platform discovery schema'); } + const classes = [...new Set(discovered.tests.map((test) => `${test.type.namespace}.${test.type.typeName}`))].sort(); const count = Number(args.count); const index = Number(args.index); @@ -70,9 +77,11 @@ function run(args) { if (!Number.isInteger(index) || index < 1 || index > count) { throw new Error('Shard index must be between 1 and shard count'); } + if (!args.output) { throw new Error('The run command requires --output'); } + const selected = shards[index - 1]; const output = resolve(args.output); mkdirSync(output, { recursive: true }); @@ -111,14 +120,19 @@ function run(args) { 'cobertura' ); } + if (process.env.GITHUB_ACTIONS) { command.push('--report-github'); } + const result = spawnSync('dotnet', command, { stdio: 'inherit', env: { ...process.env, ASPNETCORE_ENVIRONMENT: 'Development' } }); - if (result.error) throw result.error; + if (result.error) { + throw result.error; + } + return result.status ?? 1; } @@ -144,28 +158,37 @@ export function report({ directory, count, coverage }) { ) { throw new Error('Missing or duplicate shard results/manifests'); } + if (new Set(manifests.map((m) => m.discovery_hash)).size !== 1 || manifests.some((m) => m.count !== count)) { throw new Error('Shards used different test discovery or shard counts'); } + const selected = manifests.flatMap((m) => m.classes); if (new Set(selected).size !== selected.length || manifests.some((m) => m.discovered_count !== selected.length)) { throw new Error('Shard plan duplicated or omitted discovered classes'); } + for (const { path, results } of reports) { const manifest = readJson(join(dirname(path), 'manifest.json')); if (!isDeepStrictEqual(new Set(results.map((result) => result.class)), new Set(manifest.classes))) { throw new Error(`Executed classes differ from shard plan: ${path}`); } + if (!existsSync(join(dirname(path), 'test-results.trx'))) { throw new Error(`Missing shard TRX report: ${path}`); } + if (coverage && !existsSync(join(dirname(path), 'coverage.cobertura.xml'))) { throw new Error(`Missing shard coverage: ${path}`); } } } + const results = reports.flatMap((report) => report.results); - if (!results.length) throw new Error('No test reports found'); + if (!results.length) { + throw new Error('No test reports found'); + } + const durations = summarize(results); const lines = [ '### .NET test timings', @@ -180,11 +203,13 @@ export function report({ directory, count, coverage }) { .slice(0, 15)) { lines.push(`| ${name} | ${seconds.toFixed(2)} |`); } + lines.push('', '| Slowest test | Seconds |', '| --- | ---: |'); for (const result of [...results].sort((a, b) => b.seconds - a.seconds).slice(0, 15)) { const name = result.name.replaceAll('|', '\\|').replaceAll('\n', ' '); lines.push(`| ${name} | ${result.seconds.toFixed(2)} |`); } + return { durations, summary: lines.join('\n') + '\n', @@ -204,18 +229,26 @@ function main() { 'write-timings': { type: 'string' } } }); - if (positionals[0] === 'run' && positionals.length === 1) return run(values); + if (positionals[0] === 'run' && positionals.length === 1) { + return run(values); + } + if (positionals[0] !== 'report' || positionals.length !== 2) { throw new Error( 'Usage: backend-shards.js run --index N --count N --output DIR [--coverage] | report DIR [--count N] [--coverage] [--write-timings FILE]' ); } + const result = report({ directory: positionals[1], count: values.count === undefined ? undefined : Number(values.count), coverage: values.coverage }); if (values['write-timings']) { writeFileSync(values['write-timings'], JSON.stringify(result.durations, null, 2) + '\n'); } + console.log(result.summary); - if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, result.summary); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, result.summary); + } + return result.exitCode; } diff --git a/tests/ci/browser-report.js b/tests/ci/browser-report.js index 9e9bda95d0..5d3f068ab6 100644 --- a/tests/ci/browser-report.js +++ b/tests/ci/browser-report.js @@ -12,9 +12,11 @@ export function cases(report) { yield [JSON.stringify([spec.id, test.projectName]), spec, test]; } } + yield* walk(suite.suites ?? []); } } + return [...walk(report.suites)]; } @@ -24,7 +26,11 @@ export function validate(expected, actual) { if (!planned.size || executed.length !== new Set(executed).size || !isDeepStrictEqual(new Set(executed), planned)) { throw new Error('Merged browser results omitted or duplicated discovered tests'); } - if (actual.errors?.length) throw new Error('Playwright reported errors outside individual tests'); + + if (actual.errors?.length) { + throw new Error('Playwright reported errors outside individual tests'); + } + for (const [, spec, test] of cases(actual)) { if (!test.results?.length || test.status !== 'expected') { throw new Error(`Browser test did not pass: ${spec.title} (${test.status})`); @@ -34,7 +40,10 @@ export function validate(expected, actual) { function main() { const { positionals } = parseArgs({ allowPositionals: true }); - if (positionals.length !== 2) throw new Error('Usage: browser-report.js DISCOVERY_JSON RESULTS_JSON'); + if (positionals.length !== 2) { + throw new Error('Usage: browser-report.js DISCOVERY_JSON RESULTS_JSON'); + } + const [expected, actual] = positionals.map((path) => JSON.parse(readFileSync(path, 'utf8'))); const tests = cases(actual); const duration = (test) => test.results.reduce((total, result) => total + result.duration, 0); @@ -52,9 +61,13 @@ function main() { const name = `${spec.file}: ${spec.title}`.replaceAll('|', '\\|').replaceAll('\n', ' '); lines.push(`| ${name} | ${(duration(test) / 1000).toFixed(2)} | ${test.results.length} |`); } + const summary = lines.join('\n') + '\n'; console.log(summary); - if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } + validate(expected, actual); } From 7408a771a15610baa77e5bd23ef7f2bfa36336bf Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 16 Oct 2026 15:05:59 -0500 Subject: [PATCH 17/26] Use JavaScript module extensions and shared script linting --- .github/workflows/build.yaml | 8 +- AGENTS.md | 3 + package.json | 7 + .../ClientApp/eslint.config.js | 15 +- src/Exceptionless.Web/ClientApp/package.json | 11 +- .../{generate-api.mjs => generate-api.js} | 0 .../{resolve-url.mjs => resolve-url.js} | 0 tests/ci/README.md | 10 +- .../{backend-shards.mjs => backend-shards.js} | 227 ++++++++++-------- .../{browser-report.mjs => browser-report.js} | 19 +- tests/ci/shards.test.js | 140 +++++++++++ tests/ci/shards.test.mjs | 98 -------- 12 files changed, 325 insertions(+), 213 deletions(-) create mode 100644 package.json rename src/Exceptionless.Web/ClientApp/scripts/{generate-api.mjs => generate-api.js} (100%) rename src/Exceptionless.Web/ClientApp/scripts/{resolve-url.mjs => resolve-url.js} (100%) rename tests/ci/{backend-shards.mjs => backend-shards.js} (88%) rename tests/ci/{browser-report.mjs => browser-report.js} (87%) create mode 100644 tests/ci/shards.test.js delete mode 100644 tests/ci/shards.test.mjs diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 832b5b3de5..751ff94325 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -172,10 +172,10 @@ jobs: run: dotnet build ./Exceptionless.slnx --no-restore --configuration Release - name: Verify CI test tooling - run: node --test tests/ci/shards.test.mjs + run: node --test tests/ci/shards.test.js - name: Run .NET shard with coverage - run: node tests/ci/backend-shards.mjs run --index ${{ matrix.shard }} --count 4 --output coverage --coverage + run: node tests/ci/backend-shards.js run --index ${{ matrix.shard }} --count 4 --output coverage --coverage - name: Upload .NET shard results if: ${{ !cancelled() }} @@ -210,7 +210,7 @@ jobs: path: coverage-shards - name: Verify complete shard coverage and report timings - run: node tests/ci/backend-shards.mjs report coverage-shards --count 4 --coverage --write-timings coverage-shards/backend-durations.json + run: node tests/ci/backend-shards.js report coverage-shards --count 4 --coverage --write-timings coverage-shards/backend-durations.json - name: Setup .NET Core uses: actions/setup-dotnet@v6 @@ -511,7 +511,7 @@ jobs: E2E_SHARD: report run: | npx playwright test --project=chromium --list --reporter=json > test-results/discovery.json - node ../../../tests/ci/browser-report.mjs test-results/discovery.json test-results/e2e-results.json + node ../../../tests/ci/browser-report.js test-results/discovery.json test-results/e2e-results.json - name: Upload combined Playwright report if: ${{ !cancelled() }} diff --git a/AGENTS.md b/AGENTS.md index cd3883591d..14020da182 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,9 @@ tests/ # C# tests and HTTP samples ## Testing and Safety +- **JavaScript modules:** Use `.js` with `"type": "module"` in the applicable `package.json` for repository-owned Node scripts. Use `.mjs` only when a specific tool or execution context requires it, and document that requirement. Do not add dependencies or CI jobs just to validate small scripts. +- **JavaScript control flow:** Always use braces for `if`, `for`, `while`, and similar blocks, even for one statement. Put block contents on separate lines and leave a blank line after a completed block unless it is followed by `else`, `catch`, or `finally`. + - **C# test conventions:** Use explicit `// Arrange`, `// Act`, and `// Assert` sections; keep assertions out of the action. Inherit the appropriate existing test base (`TestWithLoggingBase` for isolated tests, `TestWithServices` for DI, or `IntegrationTestsBase` with `AppWebHostFactory` for application integration). Use the inherited `TestCancellationToken` and idiomatic `using var` declarations. Reuse repository fixtures and HTTP helpers rather than duplicating application startup or service registration. Read `.agents/skills/backend-testing/SKILL.md` before adding or changing backend tests. - **Test value and runtime:** Before adding a test, identify the distinct failure it catches and check existing coverage. Extend an existing test or use parameterized cases when the setup and behavior are the same. Combine related assertions in one scenario when they share expensive setup; keep independent behaviors separately diagnosable. Do not add tests for trivial accessors, framework behavior, implementation details, or duplicate coverage merely to increase test counts. - Use the cheapest reliable layer: pure logic in unit tests, service/API contracts in integration tests, and browser tests for user journeys or behavior that requires a real browser. Keep a representative browser integration case when moving a data matrix to unit/component tests. diff --git a/package.json b/package.json new file mode 100644 index 0000000000..fdcdaf898b --- /dev/null +++ b/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "scripts": { + "lint:scripts": "node src/Exceptionless.Web/ClientApp/node_modules/eslint/bin/eslint.js --config src/Exceptionless.Web/ClientApp/eslint.config.js --no-error-on-unmatched-pattern \"tests/ci/**/*.js\" \"k8s/scripts/**/*.js\"" + } +} diff --git a/src/Exceptionless.Web/ClientApp/eslint.config.js b/src/Exceptionless.Web/ClientApp/eslint.config.js index 176ef4fd03..18ec403ec6 100644 --- a/src/Exceptionless.Web/ClientApp/eslint.config.js +++ b/src/Exceptionless.Web/ClientApp/eslint.config.js @@ -45,17 +45,28 @@ export default ts.config( } }, { - files: ['**/*.svelte', '**/*.svelte.ts'], + files: ['**/*.svelte', '**/*.svelte.ts', '**/scripts/**/*.js', 'tests/ci/**/*.js'], plugins: { '@stylistic': stylistic }, rules: { '@stylistic/brace-style': ['error', '1tbs', { allowSingleLine: false }], - '@stylistic/object-curly-newline': ['error', { ObjectExpression: { minProperties: 1 } }], curly: ['error', 'all'], 'padding-line-between-statements': ['error', { blankLine: 'always', next: ['if', 'while', 'for', 'do', 'return'], prev: 'block-like' }] } }, + { + files: ['**/scripts/**/*.js', 'tests/ci/**/*.js'], + rules: { + 'padding-line-between-statements': ['error', { blankLine: 'always', next: '*', prev: 'block-like' }] + } + }, + { + files: ['**/*.svelte', '**/*.svelte.ts'], + rules: { + '@stylistic/object-curly-newline': ['error', { ObjectExpression: { minProperties: 1 } }] + } + }, { files: ['**/*.svelte', '**/*.ts'], rules: { diff --git a/src/Exceptionless.Web/ClientApp/package.json b/src/Exceptionless.Web/ClientApp/package.json index 647ca3e1d1..e1f5389e20 100644 --- a/src/Exceptionless.Web/ClientApp/package.json +++ b/src/Exceptionless.Web/ClientApp/package.json @@ -5,19 +5,19 @@ "scripts": { "dev": "cross-env NODE_OPTIONS=--trace-warnings vite dev", "dev:api": "cross-env API_HTTP=https://dev-collector.exceptionless.io vite dev", - "urls": "node scripts/resolve-url.mjs", + "urls": "node scripts/resolve-url.js", "build": "vite build", "preview": "vite preview", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "npm run lint:prettier && npm run lint:eslint", - "lint:eslint": "eslint .", + "lint:eslint": "eslint . && npm run lint:scripts", "lint:prettier": "prettier --check .", "format": "npm run format:prettier && npm run format:eslint", - "format:eslint": "eslint . --fix", + "format:eslint": "eslint . --fix && npm run lint:scripts -- --fix", "format:prettier": "prettier --write .", "validate": "npm run format && npm run check && npm run lint", - "generate-models": "node scripts/generate-api.mjs", + "generate-models": "node scripts/generate-api.js", "generate-templates": "swagger-typescript-api generate-templates -o api-templates", "test:e2e": "playwright test", "test:e2e:ci": "playwright test --project=chromium", @@ -30,7 +30,8 @@ "build-storybook": "storybook build", "update": "ncu -i", "update:shadcn": "npx shadcn-svelte@latest update", - "update:skills": "npx skills update" + "update:skills": "npx skills update", + "lint:scripts": "npm --prefix ../../.. run lint:scripts --" }, "devDependencies": { "@chromatic-com/storybook": "^5.4.0", diff --git a/src/Exceptionless.Web/ClientApp/scripts/generate-api.mjs b/src/Exceptionless.Web/ClientApp/scripts/generate-api.js similarity index 100% rename from src/Exceptionless.Web/ClientApp/scripts/generate-api.mjs rename to src/Exceptionless.Web/ClientApp/scripts/generate-api.js diff --git a/src/Exceptionless.Web/ClientApp/scripts/resolve-url.mjs b/src/Exceptionless.Web/ClientApp/scripts/resolve-url.js similarity index 100% rename from src/Exceptionless.Web/ClientApp/scripts/resolve-url.mjs rename to src/Exceptionless.Web/ClientApp/scripts/resolve-url.js diff --git a/tests/ci/README.md b/tests/ci/README.md index c150a3b910..517dde26a4 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -4,7 +4,9 @@ The normal build targets a roughly **10-minute critical path**. Track elapsed wo ## Execution -- `.NET`: four isolated runners build the suite and discover tests through Microsoft Testing Platform's JSON discovery. `backend-shards.mjs` groups whole classes, then distributes the longest classes first using `backend-durations.json`. New classes are included automatically; stale timing entries cannot select or exclude tests. Each runner retains the existing six collection workers and coverage collection. +- Run `npm run lint:scripts` from the repository root after installing ClientApp dependencies. This reuses the ClientApp ESLint configuration and runs as part of its existing lint command. + +- `.NET`: four isolated runners build the suite and discover tests through Microsoft Testing Platform's JSON discovery. `backend-shards.js` groups whole classes, then distributes the longest classes first using `backend-durations.json`. New classes are included automatically; stale timing entries cannot select or exclude tests. Each runner retains the existing six collection workers and coverage collection. - CI tooling uses the project's Node.js runtime (Node 24 in CI), built-in modules, and `node:test`, with no additional packages. Backend summaries read xUnit's native CTRF JSON output; TRX reports remain available as artifacts. - Frontend CI runs `build/update-config.test.mjs` against the real configuration writer and container entrypoints. Process stubs check configuration failures, argument preservation, and app process replacement without starting containers. CSP input cases use fast unit tests, with a representative HTTP response-header test covering integration. - Browser: six isolated Aspire applications run Playwright's native `--shard=N/6` with `fullyParallel` enabled for sharding. Each runner uses **one worker** because scenarios can change the shared administrator's preferences. This splits large spec files without introducing concurrent mutations of that user. Shard IDs are included in generated data names. @@ -43,15 +45,15 @@ Example local commands from the repository root (PowerShell): ```powershell dotnet build Exceptionless.slnx --configuration Release -node tests/ci/backend-shards.mjs run --index 1 --count 4 --output "$env:TEMP/api-shard-1" --coverage -node --test tests/ci/shards.test.mjs +node tests/ci/backend-shards.js run --index 1 --count 4 --output "$env:TEMP/api-shard-1" --coverage +node --test tests/ci/shards.test.js ``` Run local .NET shards **sequentially**: separate test processes on one host reuse test index scopes. Hosted shards are isolated machines. To validate or rebalance from downloaded artifacts: ```powershell gh run download --pattern 'api-results-*' --dir "$env:TEMP/api-results" -node tests/ci/backend-shards.mjs report "$env:TEMP/api-results" --count 4 --coverage --write-timings tests/ci/backend-durations.json +node tests/ci/backend-shards.js report "$env:TEMP/api-results" --count 4 --coverage --write-timings tests/ci/backend-durations.json ``` For a browser shard, start Aspire, discover and health-check its current local App URL, then from `src/Exceptionless.Web/ClientApp`: diff --git a/tests/ci/backend-shards.mjs b/tests/ci/backend-shards.js similarity index 88% rename from tests/ci/backend-shards.mjs rename to tests/ci/backend-shards.js index fb2406616d..857192387b 100644 --- a/tests/ci/backend-shards.mjs +++ b/tests/ci/backend-shards.js @@ -1,7 +1,7 @@ // Run complete xUnit classes on balanced shards, using discovery as the source of truth. import { execFileSync, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { isDeepStrictEqual, parseArgs } from 'node:util'; @@ -14,6 +14,7 @@ export function partition(classes, durations, count) { if (!Number.isInteger(count) || count < 1 || classes.length < count || new Set(classes).size !== classes.length) { throw new Error('Shards must be nonempty and discovered classes must be unique'); } + const shards = Array.from({ length: count }, () => []); const totals = Array(count).fill(0); // Stable sorting keeps equal-duration classes in name order. New classes are always included. @@ -25,6 +26,7 @@ export function partition(classes, durations, count) { shards[index].push(name); totals[index] += Math.max(durations[name] ?? 1, 0.01); } + return shards.map((shard) => shard.sort()); } @@ -33,103 +35,21 @@ export function readResults(path) { if (report.reportFormat !== 'CTRF' || !Array.isArray(report.results?.tests) || !report.results.tests.length) { throw new Error(`No CTRF test results in ${path}`); } + if (report.results.extra?.suites?.some((suite) => suite.errors?.length)) { throw new Error(`xUnit reported errors outside individual tests: ${path}`); } + return report.results.tests.map((test) => { if (!test.extra?.type || typeof test.name !== 'string' || typeof test.status !== 'string' || !Number.isFinite(test.duration) || test.duration < 0) { throw new Error(`Invalid xUnit CTRF test result in ${path}`); } - return { class: test.extra.type, name: test.name, seconds: test.duration / 1000, outcome: test.status }; - }); -} - -export function summarize(results) { - const durations = new Map(); - for (const result of results) { - durations.set(result.class, (durations.get(result.class) ?? 0) + result.seconds); - } - return Object.fromEntries([...durations.keys()].sort().map((name) => [name, Number(durations.get(name).toFixed(3))])); -} -function run(args) { - const assembly = resolve(args.assembly ?? join(root, 'tests/Exceptionless.Tests/bin/Release/net10.0/Exceptionless.Tests.dll')); - const discovered = JSON.parse( - execFileSync('dotnet', [assembly, '--list-tests', 'json', '--no-ansi'], { - encoding: 'utf8', - maxBuffer: 64 * 1024 * 1024 - }) - ); - if (discovered.schemaVersion !== 1) { - throw new Error('Unsupported Microsoft Testing Platform discovery schema'); - } - const classes = [...new Set(discovered.tests.map((test) => `${test.type.namespace}.${test.type.typeName}`))].sort(); - const count = Number(args.count); - const index = Number(args.index); - const shards = partition(classes, readJson(timingsPath), count); - if (!Number.isInteger(index) || index < 1 || index > count) { - throw new Error('Shard index must be between 1 and shard count'); - } - if (!args.output) { - throw new Error('The run command requires --output'); - } - const selected = shards[index - 1]; - const output = resolve(args.output); - mkdirSync(output, { recursive: true }); - const manifest = { - index, - count, - discovery_hash: createHash('sha256').update(classes.join('\n')).digest('hex'), - discovered_count: classes.length, - classes: selected - }; - writeFileSync(join(output, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n'); - console.log(`Shard ${index}/${count}: ${selected.length} of ${classes.length} classes`); - const command = [ - assembly, - '--results-directory', - output, - '--report-xunit-trx', - '--report-xunit-trx-filename', - 'test-results.trx', - '--report-xunit-ctrf', - '--report-xunit-ctrf-filename', - 'test-results.json', - '--minimum-expected-tests', - '1', - '--filter-class', - ...selected - ]; - if (args.coverage) { - command.push( - '--coverage', - '--coverage-settings', - join(root, 'tests/CodeCoverage.config'), - '--coverage-output', - 'coverage.cobertura.xml', - '--coverage-output-format', - 'cobertura' - ); - } - if (process.env.GITHUB_ACTIONS) { - command.push('--report-github'); - } - const result = spawnSync('dotnet', command, { - stdio: 'inherit', - env: { ...process.env, ASPNETCORE_ENVIRONMENT: 'Development' } + return { class: test.extra.type, name: test.name, outcome: test.status, seconds: test.duration / 1000 }; }); - if (result.error) throw result.error; - return result.status ?? 1; -} - -function findFiles(directory, name) { - return readdirSync(directory, { recursive: true, withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name === name) - .map((entry) => join(entry.parentPath, entry.name)) - .sort(); } -export function report({ directory, count, coverage }) { +export function report({ count, coverage, directory }) { const reports = findFiles(directory, 'test-results.json').map((path) => ({ path, results: readResults(path) })); if (count !== undefined) { const manifests = findFiles(directory, 'manifest.json').map(readJson); @@ -144,28 +64,37 @@ export function report({ directory, count, coverage }) { ) { throw new Error('Missing or duplicate shard results/manifests'); } + if (new Set(manifests.map((m) => m.discovery_hash)).size !== 1 || manifests.some((m) => m.count !== count)) { throw new Error('Shards used different test discovery or shard counts'); } + const selected = manifests.flatMap((m) => m.classes); if (new Set(selected).size !== selected.length || manifests.some((m) => m.discovered_count !== selected.length)) { throw new Error('Shard plan duplicated or omitted discovered classes'); } + for (const { path, results } of reports) { const manifest = readJson(join(dirname(path), 'manifest.json')); if (!isDeepStrictEqual(new Set(results.map((result) => result.class)), new Set(manifest.classes))) { throw new Error(`Executed classes differ from shard plan: ${path}`); } + if (!existsSync(join(dirname(path), 'test-results.trx'))) { throw new Error(`Missing shard TRX report: ${path}`); } + if (coverage && !existsSync(join(dirname(path), 'coverage.cobertura.xml'))) { throw new Error(`Missing shard coverage: ${path}`); } } } + const results = reports.flatMap((report) => report.results); - if (!results.length) throw new Error('No test reports found'); + if (!results.length) { + throw new Error('No test reports found'); + } + const durations = summarize(results); const lines = [ '### .NET test timings', @@ -180,45 +109,149 @@ export function report({ directory, count, coverage }) { .slice(0, 15)) { lines.push(`| ${name} | ${seconds.toFixed(2)} |`); } + lines.push('', '| Slowest test | Seconds |', '| --- | ---: |'); for (const result of [...results].sort((a, b) => b.seconds - a.seconds).slice(0, 15)) { const name = result.name.replaceAll('|', '\\|').replaceAll('\n', ' '); lines.push(`| ${name} | ${result.seconds.toFixed(2)} |`); } + return { durations, - summary: lines.join('\n') + '\n', - exitCode: Number(results.some((result) => !['passed', 'skipped'].includes(result.outcome))) + exitCode: Number(results.some((result) => !['passed', 'skipped'].includes(result.outcome))), + summary: lines.join('\n') + '\n' }; } +export function summarize(results) { + const durations = new Map(); + for (const result of results) { + durations.set(result.class, (durations.get(result.class) ?? 0) + result.seconds); + } + + return Object.fromEntries([...durations.keys()].sort().map((name) => [name, Number(durations.get(name).toFixed(3))])); +} + +function findFiles(directory, name) { + return readdirSync(directory, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name === name) + .map((entry) => join(entry.parentPath, entry.name)) + .sort(); +} + function main() { - const { values, positionals } = parseArgs({ + const { positionals, values } = parseArgs({ allowPositionals: true, options: { assembly: { type: 'string' }, - index: { type: 'string' }, count: { type: 'string' }, - output: { type: 'string' }, coverage: { type: 'boolean' }, + index: { type: 'string' }, + output: { type: 'string' }, 'write-timings': { type: 'string' } } }); - if (positionals[0] === 'run' && positionals.length === 1) return run(values); + if (positionals[0] === 'run' && positionals.length === 1) { + return run(values); + } + if (positionals[0] !== 'report' || positionals.length !== 2) { throw new Error( - 'Usage: backend-shards.mjs run --index N --count N --output DIR [--coverage] | report DIR [--count N] [--coverage] [--write-timings FILE]' + 'Usage: backend-shards.js run --index N --count N --output DIR [--coverage] | report DIR [--count N] [--coverage] [--write-timings FILE]' ); } - const result = report({ directory: positionals[1], count: values.count === undefined ? undefined : Number(values.count), coverage: values.coverage }); + + const result = report({ count: values.count === undefined ? undefined : Number(values.count), coverage: values.coverage, directory: positionals[1] }); if (values['write-timings']) { writeFileSync(values['write-timings'], JSON.stringify(result.durations, null, 2) + '\n'); } + console.log(result.summary); - if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, result.summary); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, result.summary); + } + return result.exitCode; } +function run(args) { + const assembly = resolve(args.assembly ?? join(root, 'tests/Exceptionless.Tests/bin/Release/net10.0/Exceptionless.Tests.dll')); + const discovered = JSON.parse( + execFileSync('dotnet', [assembly, '--list-tests', 'json', '--no-ansi'], { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024 + }) + ); + if (discovered.schemaVersion !== 1) { + throw new Error('Unsupported Microsoft Testing Platform discovery schema'); + } + + const classes = [...new Set(discovered.tests.map((test) => `${test.type.namespace}.${test.type.typeName}`))].sort(); + const count = Number(args.count); + const index = Number(args.index); + const shards = partition(classes, readJson(timingsPath), count); + if (!Number.isInteger(index) || index < 1 || index > count) { + throw new Error('Shard index must be between 1 and shard count'); + } + + if (!args.output) { + throw new Error('The run command requires --output'); + } + + const selected = shards[index - 1]; + const output = resolve(args.output); + mkdirSync(output, { recursive: true }); + const manifest = { + classes: selected, + count, + discovered_count: classes.length, + discovery_hash: createHash('sha256').update(classes.join('\n')).digest('hex'), + index + }; + writeFileSync(join(output, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n'); + console.log(`Shard ${index}/${count}: ${selected.length} of ${classes.length} classes`); + const command = [ + assembly, + '--results-directory', + output, + '--report-xunit-trx', + '--report-xunit-trx-filename', + 'test-results.trx', + '--report-xunit-ctrf', + '--report-xunit-ctrf-filename', + 'test-results.json', + '--minimum-expected-tests', + '1', + '--filter-class', + ...selected + ]; + if (args.coverage) { + command.push( + '--coverage', + '--coverage-settings', + join(root, 'tests/CodeCoverage.config'), + '--coverage-output', + 'coverage.cobertura.xml', + '--coverage-output-format', + 'cobertura' + ); + } + + if (process.env.GITHUB_ACTIONS) { + command.push('--report-github'); + } + + const result = spawnSync('dotnet', command, { + env: { ...process.env, ASPNETCORE_ENVIRONMENT: 'Development' }, + stdio: 'inherit' + }); + if (result.error) { + throw result.error; + } + + return result.status ?? 1; +} + if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { try { process.exitCode = main(); diff --git a/tests/ci/browser-report.mjs b/tests/ci/browser-report.js similarity index 87% rename from tests/ci/browser-report.mjs rename to tests/ci/browser-report.js index 53fc73402f..5d3f068ab6 100644 --- a/tests/ci/browser-report.mjs +++ b/tests/ci/browser-report.js @@ -12,9 +12,11 @@ export function cases(report) { yield [JSON.stringify([spec.id, test.projectName]), spec, test]; } } + yield* walk(suite.suites ?? []); } } + return [...walk(report.suites)]; } @@ -24,7 +26,11 @@ export function validate(expected, actual) { if (!planned.size || executed.length !== new Set(executed).size || !isDeepStrictEqual(new Set(executed), planned)) { throw new Error('Merged browser results omitted or duplicated discovered tests'); } - if (actual.errors?.length) throw new Error('Playwright reported errors outside individual tests'); + + if (actual.errors?.length) { + throw new Error('Playwright reported errors outside individual tests'); + } + for (const [, spec, test] of cases(actual)) { if (!test.results?.length || test.status !== 'expected') { throw new Error(`Browser test did not pass: ${spec.title} (${test.status})`); @@ -34,7 +40,10 @@ export function validate(expected, actual) { function main() { const { positionals } = parseArgs({ allowPositionals: true }); - if (positionals.length !== 2) throw new Error('Usage: browser-report.mjs DISCOVERY_JSON RESULTS_JSON'); + if (positionals.length !== 2) { + throw new Error('Usage: browser-report.js DISCOVERY_JSON RESULTS_JSON'); + } + const [expected, actual] = positionals.map((path) => JSON.parse(readFileSync(path, 'utf8'))); const tests = cases(actual); const duration = (test) => test.results.reduce((total, result) => total + result.duration, 0); @@ -52,9 +61,13 @@ function main() { const name = `${spec.file}: ${spec.title}`.replaceAll('|', '\\|').replaceAll('\n', ' '); lines.push(`| ${name} | ${(duration(test) / 1000).toFixed(2)} | ${test.results.length} |`); } + const summary = lines.join('\n') + '\n'; console.log(summary); - if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } + validate(expected, actual); } diff --git a/tests/ci/shards.test.js b/tests/ci/shards.test.js new file mode 100644 index 0000000000..230a9f6571 --- /dev/null +++ b/tests/ci/shards.test.js @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; + +import { partition, report } from './backend-shards.js'; +import { validate } from './browser-report.js'; + +test('partition covers new classes once and is deterministic', () => { + const classes = ['slow', 'medium', 'fast', 'new', 'another-new']; + const timings = { deleted: 999, fast: 30, medium: 60, slow: 90 }; + const shards = partition(classes, timings, 3); + assert.deepEqual(shards.flat().sort(), [...classes].sort()); + assert.deepEqual(shards, partition([...classes].reverse(), timings, 3)); + assert.equal(new Set(['slow', 'medium', 'fast'].map((name) => shards.findIndex((shard) => shard.includes(name)))).size, 3); + for (const count of [0, 6, 1.5]) { + assert.throws(() => partition(classes, timings, count)); + } +}); + +test('backend gate rejects incomplete or inconsistent shards and reads xUnit JSON timings', () => { + for (const defect of [ + null, + 'missing', + 'duplicate', + 'discovery', + 'omitted', + 'execution', + 'coverage', + 'trx', + 'failed', + 'skipped', + 'pending', + 'global-error', + 'empty', + 'invalid-duration' + ]) { + const directory = mkdtempSync(join(tmpdir(), 'exceptionless-ci-')); + try { + for (const index of [1, 2]) { + const shard = join(directory, String(index)); + mkdirSync(shard); + let name = `Example.Test${index}`; + const manifest = { classes: [name], count: 2, discovered_count: 2, discovery_hash: 'same', index }; + if (index === 2) { + if (defect === 'missing') { + continue; + } + + if (defect === 'duplicate') { + manifest.index = 1; + } + + if (defect === 'discovery') { + manifest.discovery_hash = 'different'; + } + + if (defect === 'omitted') { + manifest.classes = []; + } + + if (defect === 'execution') { + name = 'Unexpected.Test'; + } + } + + writeFileSync(join(shard, 'manifest.json'), JSON.stringify(manifest)); + if (defect !== 'coverage' || index !== 2) { + writeFileSync(join(shard, 'coverage.cobertura.xml'), ''); + } + + if (defect !== 'trx' || index !== 2) { + writeFileSync(join(shard, 'test-results.trx'), ''); + } + + const status = index === 2 && ['failed', 'pending', 'skipped'].includes(defect) ? defect : 'passed'; + const result = { + reportFormat: 'CTRF', + results: { + extra: { suites: [{ errors: [] }] }, + tests: [{ duration: 1500, extra: { type: name }, name: `${name}.Case(" & \\"quoted\\"")`, status }] + } + }; + if (index === 2) { + if (defect === 'global-error') { + result.results.extra.suites[0].errors.push({ message: 'Fixture failed' }); + } + + if (defect === 'empty') { + result.results.tests = []; + } + + if (defect === 'invalid-duration') { + result.results.tests[0].duration = 'invalid'; + } + } + + writeFileSync(join(shard, 'test-results.json'), JSON.stringify(result)); + } + + const args = { count: 2, coverage: true, directory }; + if (['failed', null, 'pending', 'skipped'].includes(defect)) { + const result = report(args); + assert.equal(result.exitCode, Number(['failed', 'pending'].includes(defect)), defect); + assert.deepEqual(result.durations, { 'Example.Test1': 1.5, 'Example.Test2': 1.5 }); + } else { + assert.throws(() => report(args), undefined, defect); + } + } finally { + rmSync(directory, { force: true, recursive: true }); + } + } +}); + +test('browser gate checks discovery, results, and retries', () => { + const browserTest = { projectName: 'chromium', results: [{ duration: 500 }], status: 'expected' }; + const expected = { suites: [{ specs: [{ id: 'case', tests: [browserTest], title: 'browser behavior' }] }] }; + for (const defect of [null, 'missing', 'duplicate', 'skipped', 'unexpected', 'no-result', 'global-error', 'flaky']) { + const actual = structuredClone(expected); + const spec = actual.suites[0].specs[0]; + if (defect === 'missing') { + actual.suites = []; + } else if (defect === 'duplicate') { + actual.suites.push(structuredClone(actual.suites[0])); + } else if (['flaky', 'skipped', 'unexpected'].includes(defect)) { + spec.tests[0].status = defect; + } else if (defect === 'no-result') { + spec.tests[0].results = []; + } else if (defect === 'global-error') { + actual.errors = [{ message: 'worker crashed' }]; + } + + if (defect === null) { + validate(expected, actual); + } else { + assert.throws(() => validate(expected, actual), undefined, defect); + } + } +}); diff --git a/tests/ci/shards.test.mjs b/tests/ci/shards.test.mjs deleted file mode 100644 index 935b891392..0000000000 --- a/tests/ci/shards.test.mjs +++ /dev/null @@ -1,98 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import { partition, report } from './backend-shards.mjs'; -import { validate } from './browser-report.mjs'; - -test('partition covers new classes once and is deterministic', () => { - const classes = ['slow', 'medium', 'fast', 'new', 'another-new']; - const timings = { slow: 90, medium: 60, fast: 30, deleted: 999 }; - const shards = partition(classes, timings, 3); - assert.deepEqual(shards.flat().sort(), [...classes].sort()); - assert.deepEqual(shards, partition([...classes].reverse(), timings, 3)); - assert.equal(new Set(['slow', 'medium', 'fast'].map((name) => shards.findIndex((shard) => shard.includes(name)))).size, 3); - for (const count of [0, 6, 1.5]) { - assert.throws(() => partition(classes, timings, count)); - } -}); - -test('backend gate rejects incomplete or inconsistent shards and reads xUnit JSON timings', () => { - for (const defect of [ - null, - 'missing', - 'duplicate', - 'discovery', - 'omitted', - 'execution', - 'coverage', - 'trx', - 'failed', - 'skipped', - 'pending', - 'global-error', - 'empty', - 'invalid-duration' - ]) { - const directory = mkdtempSync(join(tmpdir(), 'exceptionless-ci-')); - try { - for (const index of [1, 2]) { - const shard = join(directory, String(index)); - mkdirSync(shard); - let name = `Example.Test${index}`; - const manifest = { index, count: 2, discovery_hash: 'same', discovered_count: 2, classes: [name] }; - if (index === 2) { - if (defect === 'missing') continue; - if (defect === 'duplicate') manifest.index = 1; - if (defect === 'discovery') manifest.discovery_hash = 'different'; - if (defect === 'omitted') manifest.classes = []; - if (defect === 'execution') name = 'Unexpected.Test'; - } - writeFileSync(join(shard, 'manifest.json'), JSON.stringify(manifest)); - if (defect !== 'coverage' || index !== 2) writeFileSync(join(shard, 'coverage.cobertura.xml'), ''); - if (defect !== 'trx' || index !== 2) writeFileSync(join(shard, 'test-results.trx'), ''); - const status = index === 2 && ['failed', 'skipped', 'pending'].includes(defect) ? defect : 'passed'; - const result = { - reportFormat: 'CTRF', - results: { - tests: [{ name: `${name}.Case(" & \\"quoted\\"")`, status, duration: 1500, extra: { type: name } }], - extra: { suites: [{ errors: [] }] } - } - }; - if (index === 2) { - if (defect === 'global-error') result.results.extra.suites[0].errors.push({ message: 'Fixture failed' }); - if (defect === 'empty') result.results.tests = []; - if (defect === 'invalid-duration') result.results.tests[0].duration = 'invalid'; - } - writeFileSync(join(shard, 'test-results.json'), JSON.stringify(result)); - } - const args = { directory, count: 2, coverage: true }; - if ([null, 'failed', 'skipped', 'pending'].includes(defect)) { - const result = report(args); - assert.equal(result.exitCode, Number(['failed', 'pending'].includes(defect)), defect); - assert.deepEqual(result.durations, { 'Example.Test1': 1.5, 'Example.Test2': 1.5 }); - } else { - assert.throws(() => report(args), undefined, defect); - } - } finally { - rmSync(directory, { recursive: true, force: true }); - } - } -}); - -test('browser gate checks discovery, results, and retries', () => { - const browserTest = { projectName: 'chromium', status: 'expected', results: [{ duration: 500 }] }; - const expected = { suites: [{ specs: [{ id: 'case', title: 'browser behavior', tests: [browserTest] }] }] }; - for (const defect of [null, 'missing', 'duplicate', 'skipped', 'unexpected', 'no-result', 'global-error', 'flaky']) { - const actual = structuredClone(expected); - const spec = actual.suites[0].specs[0]; - if (defect === 'missing') actual.suites = []; - else if (defect === 'duplicate') actual.suites.push(structuredClone(actual.suites[0])); - else if (['skipped', 'unexpected', 'flaky'].includes(defect)) spec.tests[0].status = defect; - else if (defect === 'no-result') spec.tests[0].results = []; - else if (defect === 'global-error') actual.errors = [{ message: 'worker crashed' }]; - if (defect === null) validate(expected, actual); - else assert.throws(() => validate(expected, actual), undefined, defect); - } -}); From 711caec44f7a6a9a6abfbfc72f684d5157d5781b Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 16 Oct 2026 15:19:23 -0500 Subject: [PATCH 18/26] Limit pull request to forwarded header security changes --- .github/workflows/build.yaml | 3 +++ AGENTS.md | 4 ---- .../ClientApp/api-templates/schemas.ejs | 2 +- tests/ci/README.md | 9 ++++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 71b0b94284..33a78435a4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -171,6 +171,9 @@ jobs: - name: Build run: dotnet build ./Exceptionless.slnx --no-restore --configuration Release + - name: Verify CI test tooling + run: node --test tests/ci/shards.test.js + - name: Run .NET shard with coverage run: node tests/ci/backend-shards.js run --index ${{ matrix.shard }} --count 4 --output coverage --coverage diff --git a/AGENTS.md b/AGENTS.md index 481ec43f82..cd3883591d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,10 +49,6 @@ tests/ # C# tests and HTTP samples ## Testing and Safety -- **Keep testing proportional:** Do not add tests or CI steps for routine script wiring, configuration, file moves, or straightforward tooling changes. Prefer syntax checks, builds, or a focused manual check. Add regression tests only for meaningful behavior or a demonstrated bug, and extend existing coverage before creating another suite. Do not create tests that merely restate commands, configuration, or implementation details. -- **JavaScript modules:** Use `.js` for repository-owned Node scripts. Node 24 detects ES module syntax; do not add package manifests just to select module mode. Use `.mjs` only when a specific tool or execution context requires it, and document that requirement. Do not add dependencies or CI jobs just to validate small scripts. -- **JavaScript control flow:** Always use braces for `if`, `for`, `while`, and similar blocks, even for one statement. Put block contents on separate lines and leave a blank line after a completed block unless it is followed by `else`, `catch`, or `finally`. - - **C# test conventions:** Use explicit `// Arrange`, `// Act`, and `// Assert` sections; keep assertions out of the action. Inherit the appropriate existing test base (`TestWithLoggingBase` for isolated tests, `TestWithServices` for DI, or `IntegrationTestsBase` with `AppWebHostFactory` for application integration). Use the inherited `TestCancellationToken` and idiomatic `using var` declarations. Reuse repository fixtures and HTTP helpers rather than duplicating application startup or service registration. Read `.agents/skills/backend-testing/SKILL.md` before adding or changing backend tests. - **Test value and runtime:** Before adding a test, identify the distinct failure it catches and check existing coverage. Extend an existing test or use parameterized cases when the setup and behavior are the same. Combine related assertions in one scenario when they share expensive setup; keep independent behaviors separately diagnosable. Do not add tests for trivial accessors, framework behavior, implementation details, or duplicate coverage merely to increase test counts. - Use the cheapest reliable layer: pure logic in unit tests, service/API contracts in integration tests, and browser tests for user journeys or behavior that requires a real browser. Keep a representative browser integration case when moving a data matrix to unit/component tests. diff --git a/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs b/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs index 9a8f0728ad..58935cf119 100644 --- a/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs +++ b/src/Exceptionless.Web/ClientApp/api-templates/schemas.ejs @@ -58,7 +58,7 @@ const typeToZod = (rawType, isRequired, nullable, field) => { const fieldName = field && field.name; // Handle nullable unions: swagger-typescript-api outputs "null | X" for most nullable fields, - // but outputs "X | null" for nullable arrays fixed via generate-api.js workaround. + // but outputs "X | null" for nullable arrays fixed via generate-api.mjs workaround. if (type.startsWith('null | ')) { let innerType = type.replace('null | ', ''); if (innerType.startsWith('(') && innerType.endsWith(')') && !innerType.endsWith('[]')) { diff --git a/tests/ci/README.md b/tests/ci/README.md index 34e5092a5a..d880955b9b 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -5,8 +5,10 @@ The normal build targets a roughly **10-minute critical path**. Track elapsed wo ## Execution - Run `npm run lint:scripts` from `src/Exceptionless.Web/ClientApp` after installing its dependencies. This reuses the ClientApp ESLint configuration and runs as part of its existing lint command. + - `.NET`: four isolated runners build the suite and discover tests through Microsoft Testing Platform's JSON discovery. `backend-shards.js` groups whole classes, then distributes the longest classes first using `backend-durations.json`. New classes are included automatically; stale timing entries cannot select or exclude tests. Each runner retains the existing six collection workers and coverage collection. -- CI tooling uses the project's Node.js runtime (Node 24 in CI) and built-in modules, with no additional packages. Backend summaries read xUnit's native CTRF JSON output; TRX reports remain available as artifacts. +- CI tooling uses the project's Node.js runtime (Node 24 in CI), built-in modules, and `node:test`, with no additional packages. Backend summaries read xUnit's native CTRF JSON output; TRX reports remain available as artifacts. +- Frontend CI runs `build/update-config.test.mjs` against the real configuration writer and container entrypoints. Process stubs check configuration failures, argument preservation, and app process replacement without starting containers. CSP input cases use fast unit tests, with a representative HTTP response-header test covering integration. - Browser: six isolated Aspire applications run Playwright's native `--shard=N/6` with `fullyParallel` enabled for sharding. Each runner uses **one worker** because scenarios can change the shared administrator's preferences. This splits large spec files without introducing concurrent mutations of that user. Shard IDs are included in generated data names. - Frontend unit/component tests actually execute in `test-client`, with two Vitest workers to bound memory. Previously the workflow only echoed the command. - `test-api` and `test-e2e` remain the required checks. They depend on every shard, reject missing/duplicate/inconsistent results, and fail if any shard fails, is cancelled, or is skipped. `fail-fast: false` retains diagnostics from the other shards. @@ -44,6 +46,7 @@ Example local commands from the repository root (PowerShell): ```powershell dotnet build Exceptionless.slnx --configuration Release node tests/ci/backend-shards.js run --index 1 --count 4 --output "$env:TEMP/api-shard-1" --coverage +node --test tests/ci/shards.test.js ``` Run local .NET shards **sequentially**: separate test processes on one host reuse test index scopes. Hosted shards are isolated machines. To validate or rebalance from downloaded artifacts: @@ -61,7 +64,7 @@ $env:E2E_SHARD = '1' npm run test:e2e:ci -- --shard=1/6 ``` -Keep the matrix counts, runner arguments, and aggregate completeness checks in sync when resizing either suite. +Keep the matrix counts, runner arguments, and aggregate completeness checks in sync when resizing either suite. The small CI-tool tests exercise lost/duplicated tests, inconsistent discovery, missing coverage, and failed/skipped browser results; they protect against a falsely green build. ## October 2026 audit @@ -103,7 +106,7 @@ Checked the resolved npm lockfile, NuGet packages (including transitive vulnerab | Storybook packages / Svelte CSF / Chromatic addon | 10.6.1 / 5.1.5 / 5.4.0 | Updated the component and visual testing toolchain together. | | Vite / Svelte Vite plugin | 8.3.2 / 7.3.1 | Updated the shared test/build host and plugin. | -Compatibility was checked against the packages' Node, Vite, Svelte, and Storybook peer requirements. CI uses Node 24; no preview packages or new script dependencies were introduced. The scripts consume native xUnit CTRF JSON. The existing GitHub logger supports MTP, and the Visual Studio adapter remains useful for IDE discovery; neither is a redundant legacy package. +Compatibility was checked against the packages' Node, Vite, Svelte, and Storybook peer requirements. CI uses Node 24; no preview packages or new script dependencies were introduced. The scripts consume native xUnit CTRF JSON and use the built-in Node test runner. The existing GitHub logger supports MTP, and the Visual Studio adapter remains useful for IDE discovery; neither is a redundant legacy package. Release references: [Playwright](https://playwright.dev/docs/release-notes), [xUnit](https://xunit.net/releases/), [Vitest](https://github.com/vitest-dev/vitest/releases/tag/v5.0.3), [jsdom](https://github.com/jsdom/jsdom/releases/tag/v30.1.1), [Storybook](https://github.com/storybookjs/storybook/releases/tag/v10.6.1), [Svelte CSF](https://github.com/storybookjs/addon-svelte-csf/releases/tag/v5.1.5), [Chromatic addon](https://github.com/chromaui/addon-visual-tests/releases/tag/v5.4.0), [Vite](https://github.com/vitejs/vite/releases/tag/v8.3.2), and [Svelte Vite plugin](https://github.com/sveltejs/vite-plugin-svelte/releases/tag/@sveltejs%2Fvite-plugin-svelte@7.3.1). From 8addb153485a0c8cb8de2f5f68b3fa32ad68df8f Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 16 Oct 2026 15:24:54 -0500 Subject: [PATCH 19/26] Group forwarded header settings and document framework keys --- docs/docs/self-hosting/index.md | 12 +++++++--- docs/docs/self-hosting/kubernetes.md | 3 ++- k8s/exceptionless/templates/config.yaml | 1 + .../Security/ForwardedHeadersConfiguration.cs | 17 +++++++++----- src/Exceptionless.Web/appsettings.yml | 1 + .../ForwardedHeadersConfigurationTests.cs | 22 ++++++++++++------- 6 files changed, 38 insertions(+), 18 deletions(-) diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index daf9dec044..c6a219e183 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -22,21 +22,27 @@ Use this section in the web application's `appsettings.Local.yml`, replacing the ```yaml ForwardedHeaders: + Enabled: true + ForwardLimit: 1 KnownProxies: - 192.0.2.10 - ForwardLimit: 1 ``` The equivalent Docker Compose `environment` entries, or Helm values under the chart's existing `config` map, are: ```yaml -EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" +EX_ForwardedHeaders__Enabled: "true" EX_ForwardedHeaders__ForwardLimit: "1" +EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" ``` For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup. -`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero disables processing; negative or non-integer limits are invalid. Do not enable the unrestricted `ASPNETCORE_FORWARDEDHEADERS_ENABLED` shortcut. Startup rejects an active `ForwardedHeaders_Enabled=true` setting because the framework clears its trusted-peer lists when this setting is enabled. +All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. `ForwardLimit`, `KnownProxies`, and `KnownNetworks` use the names of [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). Exceptionless supplies the configuration group and `Enabled` switch. + +`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero also disables processing for compatibility; negative or non-integer limits are invalid. + +Do not set the separate framework variable `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true`. [Microsoft documents that this shortcut clears the trusted-peer lists](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies). ASP.NET Core exposes it internally as [`ForwardedHeaders_Enabled`](https://github.com/dotnet/aspnetcore/blob/main/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs#L21); Exceptionless rejects the active shortcut at startup. This framework key is not part of the Exceptionless configuration group and cannot replace `ForwardedHeaders:Enabled`. The public edge proxy must derive the scheme from its connection and replace untrusted forwarding headers, or append the observed client address so the configured hop limit cannot reach user-supplied entries. Configure Nginx, Caddy, Traefik, or other proxies accordingly. Loopback proxies and IIS out-of-process integration normally need no additional trust entries. A remote IIS/ARR proxy needs the same explicit peer configuration as other remote proxies. diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index 340f2d8c77..a36c05e1e1 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -34,6 +34,7 @@ Before deploying behind ingress, configure [trusted forwarding peers](/docs/self ```yaml config: EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" + EX_ForwardedHeaders__Enabled: "true" EX_ForwardedHeaders__ForwardLimit: "1" ``` @@ -41,7 +42,7 @@ The repository's ingress templates use Azure Application Gateway for Containers, The shared `k8s/scripts/discover-forwarded-headers.js` script renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. -Discovery writes a `forwardedHeaders.knownNetworks` Helm list. The chart converts it into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys and a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining discovery with manual `ForwardedHeaders` entries in `config` is rejected; remove those old entries from saved release values before using discovery. +Discovery writes a `forwardedHeaders.knownNetworks` Helm list. The chart converts it into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys, enables forwarding, and sets a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining discovery with manual `ForwardedHeaders` entries in `config` is rejected; remove those old entries from saved release values before using discovery. For manual upgrades, run discovery from the repository root after selecting the intended Kubernetes context and Azure subscription, then pass the generated file **last** to Helm. For example, the development commands are: diff --git a/k8s/exceptionless/templates/config.yaml b/k8s/exceptionless/templates/config.yaml index ef78904f55..e464a81344 100644 --- a/k8s/exceptionless/templates/config.yaml +++ b/k8s/exceptionless/templates/config.yaml @@ -24,6 +24,7 @@ data: {{- fail "Do not combine discovered forwardedHeaders.knownNetworks with ForwardedHeaders config overrides; remove the old overrides" }} {{- end }} {{- end }} + EX_ForwardedHeaders__Enabled: "true" EX_ForwardedHeaders__ForwardLimit: "1" {{- range $index, $network := $knownNetworks }} EX_ForwardedHeaders__KnownNetworks__{{ $index }}: {{ $network | quote }} diff --git a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs index a8aa640dc8..4ccd97f4d4 100644 --- a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs @@ -7,21 +7,26 @@ public static class ForwardedHeadersConfiguration { public static void Configure(ForwardedHeadersOptions options, IConfiguration configuration) { + // ASP.NET Core owns this separate key and clears proxy trust when it is enabled. + // https://learn.microsoft.com/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies if (String.Equals(configuration["ForwardedHeaders_Enabled"], "true", StringComparison.OrdinalIgnoreCase)) { - throw new InvalidOperationException("ForwardedHeaders_Enabled must not be enabled. Configure ForwardedHeaders:KnownProxies or ForwardedHeaders:KnownNetworks instead."); + throw new InvalidOperationException("The ASP.NET Core ForwardedHeaders_Enabled shortcut must not be enabled. Use the ForwardedHeaders configuration section to enable forwarding and configure trusted peers."); } - options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + var section = configuration.GetSection("ForwardedHeaders"); + options.ForwardedHeaders = section.GetValue("Enabled", true) + ? ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto + : ForwardedHeaders.None; options.RequireHeaderSymmetry = false; - options.ForwardLimit = configuration.GetValue("ForwardedHeaders:ForwardLimit", 1); + options.ForwardLimit = section.GetValue("ForwardLimit", 1); if (options.ForwardLimit is < 0) { throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be zero or positive."); } // Retain the framework's loopback defaults and add explicitly trusted ingress peers. - foreach (var entry in GetEntries(configuration, "KnownProxies")) + foreach (var entry in GetEntries(section, "KnownProxies")) { if (!IPAddress.TryParse(entry.Value, out var address)) { @@ -31,7 +36,7 @@ public static void Configure(ForwardedHeadersOptions options, IConfiguration con options.KnownProxies.Add(address); } - foreach (var entry in GetEntries(configuration, "KnownNetworks")) + foreach (var entry in GetEntries(section, "KnownNetworks")) { if (!System.Net.IPNetwork.TryParse(entry.Value, out var network)) { @@ -49,7 +54,7 @@ public static void Configure(ForwardedHeadersOptions options, IConfiguration con private static IEnumerable GetEntries(IConfiguration configuration, string name) { - var section = configuration.GetSection($"ForwardedHeaders:{name}"); + var section = configuration.GetSection(name); if (section.Value is not null) { throw new InvalidOperationException($"{section.Path} must be a list. Use numbered environment keys such as EX_ForwardedHeaders__{name}__0."); diff --git a/src/Exceptionless.Web/appsettings.yml b/src/Exceptionless.Web/appsettings.yml index 6362051d76..45b903e9eb 100644 --- a/src/Exceptionless.Web/appsettings.yml +++ b/src/Exceptionless.Web/appsettings.yml @@ -36,6 +36,7 @@ ApiUrl: # Reverse proxy trust: see docs/docs/self-hosting/index.md#reverse-proxies-and-https. # Configure only verified API-visible peers; loopback is trusted by default. ForwardedHeaders: + Enabled: true ForwardLimit: 1 KnownProxies: [] KnownNetworks: [] diff --git a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs index 6cf1f3259c..b9d1828fe0 100644 --- a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs +++ b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs @@ -16,16 +16,18 @@ namespace Exceptionless.Tests.Security; public sealed class ForwardedHeadersConfigurationTests(ITestOutputHelper output) : TestWithLoggingBase(output) { [Theory] - [InlineData("192.0.2.10", "198.51.100.20", "1", "198.51.100.20", "http")] - [InlineData("198.51.100.20", "198.51.100.20", "1", "203.0.113.40", "https")] - [InlineData("198.51.100.20", "198.51.100.20", "0", "198.51.100.20", "http")] - [InlineData("198.51.100.20", "::ffff:198.51.100.20", "1", "203.0.113.40", "https")] - [InlineData("2001:db8::20", "2001:db8::20", "1", "203.0.113.40", "https")] - public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string trustedPeer, string remoteAddress, string limit, string expectedAddress, string expectedScheme) + [InlineData("192.0.2.10", "198.51.100.20", "1", null, "198.51.100.20", "http")] + [InlineData("198.51.100.20", "198.51.100.20", "1", null, "203.0.113.40", "https")] + [InlineData("198.51.100.20", "198.51.100.20", "0", null, "198.51.100.20", "http")] + [InlineData("198.51.100.20", "::ffff:198.51.100.20", "1", null, "203.0.113.40", "https")] + [InlineData("2001:db8::20", "2001:db8::20", "1", null, "203.0.113.40", "https")] + [InlineData("198.51.100.20", "198.51.100.20", "1", "false", "198.51.100.20", "http")] + public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string trustedPeer, string remoteAddress, string limit, string? enabled, string expectedAddress, string expectedScheme) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { + ["ForwardedHeaders:Enabled"] = enabled, ["ForwardedHeaders:KnownProxies:0"] = trustedPeer, ["ForwardedHeaders:ForwardLimit"] = limit }).Build(); @@ -164,14 +166,17 @@ public async Task Invoke_Defaults_TrustsLoopbackAndProcessesOneHop(string remote Assert.Equal("https", context.Request.Scheme); } - [Fact] - public void Configure_PrefixedEnvironmentLists_ReadsNestedPeersAndLimit() + [Theory] + [InlineData("true", ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto)] + [InlineData("false", ForwardedHeaders.None)] + public void Configure_PrefixedEnvironmentLists_ReadsNestedSettings(string enabled, ForwardedHeaders expectedHeaders) { // Arrange var provider = new CustomEnvironmentVariablesConfigurationProvider(); using var configuration = new ConfigurationRoot([provider]); provider.Load(new Hashtable { + ["EX_ForwardedHeaders__Enabled"] = enabled, ["EX_ForwardedHeaders__KnownProxies__0"] = "192.0.2.10", ["EX_ForwardedHeaders__KnownNetworks__0"] = "2001:db8::/64", ["EX_ForwardedHeaders__ForwardLimit"] = "2" @@ -185,5 +190,6 @@ public void Configure_PrefixedEnvironmentLists_ReadsNestedPeersAndLimit() Assert.Contains(IPAddress.Parse("192.0.2.10"), options.KnownProxies); Assert.Contains(System.Net.IPNetwork.Parse("2001:db8::/64"), options.KnownIPNetworks); Assert.Equal(2, options.ForwardLimit); + Assert.Equal(expectedHeaders, options.ForwardedHeaders); } } From 0e7e9254fb02f1ea8040d5f312812102ddd90c31 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 16 Oct 2026 15:29:57 -0500 Subject: [PATCH 20/26] Clarify explicit forwarded header configuration binding --- docs/docs/self-hosting/index.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index c6a219e183..d7d62cbc02 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -38,11 +38,13 @@ EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup. -All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. `ForwardLimit`, `KnownProxies`, and `KnownNetworks` use the names of [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). Exceptionless supplies the configuration group and `Enabled` switch. +All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. Exceptionless reads this section explicitly and configures `ForwardedHeadersOptions` through [.NET's options registration](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/options?view=aspnetcore-10.0). ASP.NET Core does not automatically bind this section. The group and `Enabled` switch are Exceptionless settings. + +`ForwardLimit` and `KnownProxies` match [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). The `KnownNetworks` configuration key retains the older framework name; Exceptionless parses its CIDR strings into .NET 10's [`KnownIPNetworks` property](https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.builder.forwardedheadersoptions.knownipnetworks?view=aspnetcore-10.0). Both address lists are parsed and validated explicitly while preserving the framework's loopback defaults. `ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero also disables processing for compatibility; negative or non-integer limits are invalid. -Do not set the separate framework variable `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true`. [Microsoft documents that this shortcut clears the trusted-peer lists](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies). ASP.NET Core exposes it internally as [`ForwardedHeaders_Enabled`](https://github.com/dotnet/aspnetcore/blob/main/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs#L21); Exceptionless rejects the active shortcut at startup. This framework key is not part of the Exceptionless configuration group and cannot replace `ForwardedHeaders:Enabled`. +Do not set the separate framework variable `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true`. [Microsoft documents that this shortcut clears the trusted-peer lists](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies). ASP.NET Core exposes it internally as [`ForwardedHeaders_Enabled`](https://github.com/dotnet/aspnetcore/blob/release/10.0/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs#L21); Exceptionless rejects the active shortcut at startup. This framework key is not part of the Exceptionless configuration group and cannot replace `ForwardedHeaders:Enabled`. The public edge proxy must derive the scheme from its connection and replace untrusted forwarding headers, or append the observed client address so the configured hop limit cannot reach user-supplied entries. Configure Nginx, Caddy, Traefik, or other proxies accordingly. Loopback proxies and IIS out-of-process integration normally need no additional trust entries. A remote IIS/ARR proxy needs the same explicit peer configuration as other remote proxies. From 6c3d290995ec436dc9442da07c508e0ac392e308 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 16 Oct 2026 15:42:18 -0500 Subject: [PATCH 21/26] Separate trusted proxy options from middleware configuration --- docs/docs/self-hosting/index.md | 4 +- src/Exceptionless.Web/Program.cs | 7 +- .../Security/ForwardedHeadersConfiguration.cs | 49 ++--------- .../TrustedForwardedHeadersOptions.cs | 86 +++++++++++++++++++ .../ForwardedHeadersConfigurationTests.cs | 33 +++---- 5 files changed, 114 insertions(+), 65 deletions(-) create mode 100644 src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index d7d62cbc02..e5c4a801b7 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -38,9 +38,9 @@ EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup. -All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. Exceptionless reads this section explicitly and configures `ForwardedHeadersOptions` through [.NET's options registration](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/options?view=aspnetcore-10.0). ASP.NET Core does not automatically bind this section. The group and `Enabled` switch are Exceptionless settings. +All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. Exceptionless reads and validates this section once at startup into `TrustedForwardedHeadersOptions`, then configures ASP.NET Core's `ForwardedHeadersOptions` from those typed values through [.NET's options registration](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/options?view=aspnetcore-10.0). ASP.NET Core does not automatically bind this section. The group and `Enabled` switch are Exceptionless settings. -`ForwardLimit` and `KnownProxies` match [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). The `KnownNetworks` configuration key retains the older framework name; Exceptionless parses its CIDR strings into .NET 10's [`KnownIPNetworks` property](https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.builder.forwardedheadersoptions.knownipnetworks?view=aspnetcore-10.0). Both address lists are parsed and validated explicitly while preserving the framework's loopback defaults. +`ForwardLimit` and `KnownProxies` match [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). The `KnownNetworks` configuration key retains the older framework name; Exceptionless parses its CIDR strings into .NET 10's [`KnownIPNetworks` property](https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.builder.forwardedheadersoptions.knownipnetworks?view=aspnetcore-10.0). Both address lists are parsed and validated explicitly while preserving the framework's loopback defaults. Boolean settings use .NET's typed `GetValue` conversion; malformed booleans fail startup. `ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero also disables processing for compatibility; negative or non-integer limits are invalid. diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index 9a478eec8c..77663e730d 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -123,6 +123,7 @@ public static async Task Main(string[] args) builder.Services.AddSingleton(configuration); builder.Services.AddSingleton(apmConfig); builder.Services.AddAppOptions(options); + builder.Services.AddSingleton(TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration)); builder.Services.AddHttpContextAccessor(); builder.Services.AddCsp(nonceByteAmount: 32); @@ -134,10 +135,8 @@ public static async Task Main(string[] args) .SetPreflightMaxAge(TimeSpan.FromMinutes(5)) .WithExposedHeaders("ETag", Headers.LegacyConfigurationVersion, Headers.ConfigurationVersion, HeaderNames.Link, Headers.RateLimit, Headers.RateLimitRemaining, Headers.ResultCount))); - builder.Services.Configure(o => - { - ForwardedHeadersConfiguration.Configure(o, builder.Configuration); - }); + builder.Services.AddOptions() + .Configure(ForwardedHeadersConfiguration.Configure); builder.Services.ConfigureHttpJsonOptions(o => { diff --git a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs index 4ccd97f4d4..053dbfe1a2 100644 --- a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs +++ b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs @@ -1,65 +1,26 @@ -using System.Net; using Microsoft.AspNetCore.HttpOverrides; namespace Exceptionless.Web.Security; public static class ForwardedHeadersConfiguration { - public static void Configure(ForwardedHeadersOptions options, IConfiguration configuration) + public static void Configure(ForwardedHeadersOptions options, TrustedForwardedHeadersOptions settings) { - // ASP.NET Core owns this separate key and clears proxy trust when it is enabled. - // https://learn.microsoft.com/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies - if (String.Equals(configuration["ForwardedHeaders_Enabled"], "true", StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException("The ASP.NET Core ForwardedHeaders_Enabled shortcut must not be enabled. Use the ForwardedHeaders configuration section to enable forwarding and configure trusted peers."); - } - - var section = configuration.GetSection("ForwardedHeaders"); - options.ForwardedHeaders = section.GetValue("Enabled", true) + options.ForwardedHeaders = settings.Enabled ? ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto : ForwardedHeaders.None; options.RequireHeaderSymmetry = false; - options.ForwardLimit = section.GetValue("ForwardLimit", 1); - if (options.ForwardLimit is < 0) - { - throw new InvalidOperationException("ForwardedHeaders:ForwardLimit must be zero or positive."); - } + options.ForwardLimit = settings.ForwardLimit; // Retain the framework's loopback defaults and add explicitly trusted ingress peers. - foreach (var entry in GetEntries(section, "KnownProxies")) + foreach (var address in settings.KnownProxies) { - if (!IPAddress.TryParse(entry.Value, out var address)) - { - throw new InvalidOperationException($"{entry.Path} must be an IP address without a port."); - } - options.KnownProxies.Add(address); } - foreach (var entry in GetEntries(section, "KnownNetworks")) + foreach (var network in settings.KnownNetworks) { - if (!System.Net.IPNetwork.TryParse(entry.Value, out var network)) - { - throw new InvalidOperationException($"{entry.Path} must be a valid IPv4 or IPv6 CIDR."); - } - - if (network.PrefixLength is 0) - { - throw new InvalidOperationException($"{entry.Path} must identify a trusted ingress network, not a /0 network."); - } - options.KnownIPNetworks.Add(network); } } - - private static IEnumerable GetEntries(IConfiguration configuration, string name) - { - var section = configuration.GetSection(name); - if (section.Value is not null) - { - throw new InvalidOperationException($"{section.Path} must be a list. Use numbered environment keys such as EX_ForwardedHeaders__{name}__0."); - } - - return section.GetChildren(); - } } diff --git a/src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs b/src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs new file mode 100644 index 0000000000..c1176bec70 --- /dev/null +++ b/src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs @@ -0,0 +1,86 @@ +using System.Net; + +namespace Exceptionless.Web.Security; + +public sealed class TrustedForwardedHeadersOptions +{ + private const string FrameworkEnabledKey = "ForwardedHeaders_Enabled"; + + public bool Enabled { get; private init; } = true; + public int ForwardLimit { get; private init; } = 1; + public IReadOnlyList KnownProxies { get; private init; } = []; + public IReadOnlyList KnownNetworks { get; private init; } = []; + + public static TrustedForwardedHeadersOptions ReadFromConfiguration(IConfiguration configuration) + { + // ASP.NET Core's separate shortcut clears proxy trust; it is not our nested Enabled setting. + // https://github.com/dotnet/aspnetcore/blob/release/10.0/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs + if (configuration.GetValue(FrameworkEnabledKey, false)) + { + throw new InvalidOperationException($"The ASP.NET Core {FrameworkEnabledKey} shortcut must not be enabled. Use the ForwardedHeaders configuration section to enable forwarding and configure trusted peers."); + } + + var section = configuration.GetSection("ForwardedHeaders"); + int forwardLimit = section.GetValue(nameof(ForwardLimit), 1); + if (forwardLimit < 0) + { + throw new InvalidOperationException($"{section.Path}:{nameof(ForwardLimit)} must be zero or positive."); + } + + return new TrustedForwardedHeadersOptions + { + Enabled = section.GetValue(nameof(Enabled), true), + ForwardLimit = forwardLimit, + KnownProxies = ReadKnownProxies(section), + KnownNetworks = ReadKnownNetworks(section) + }; + } + + private static IReadOnlyList ReadKnownProxies(IConfigurationSection section) + { + var addresses = new List(); + foreach (var entry in GetEntries(section, nameof(KnownProxies))) + { + if (!IPAddress.TryParse(entry.Value, out var address)) + { + throw new InvalidOperationException($"{entry.Path} must be an IP address without a port."); + } + + addresses.Add(address); + } + + return addresses.AsReadOnly(); + } + + private static IReadOnlyList ReadKnownNetworks(IConfigurationSection section) + { + var networks = new List(); + foreach (var entry in GetEntries(section, nameof(KnownNetworks))) + { + if (!IPNetwork.TryParse(entry.Value, out var network)) + { + throw new InvalidOperationException($"{entry.Path} must be a valid IPv4 or IPv6 CIDR."); + } + + if (network.PrefixLength is 0) + { + throw new InvalidOperationException($"{entry.Path} must identify a trusted ingress network, not a /0 network."); + } + + networks.Add(network); + } + + return networks.AsReadOnly(); + } + + private static IEnumerable GetEntries(IConfigurationSection configuration, string name) + { + var section = configuration.GetSection(name); + if (section.Value is not null) + { + throw new InvalidOperationException($"{section.Path} must be a list. Use numbered environment keys such as EX_ForwardedHeaders__{name}__0."); + } + + return section.GetChildren(); + } +} diff --git a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs index b9d1828fe0..066cd038bd 100644 --- a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs +++ b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs @@ -31,8 +31,9 @@ public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string ["ForwardedHeaders:KnownProxies:0"] = trustedPeer, ["ForwardedHeaders:ForwardLimit"] = limit }).Build(); + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration); var options = new ForwardedHeadersOptions(); - ForwardedHeadersConfiguration.Configure(options, configuration); + ForwardedHeadersConfiguration.Configure(options, settings); var context = new DefaultHttpContext(); context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); context.Request.Scheme = "http"; @@ -61,8 +62,9 @@ public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer ["ForwardedHeaders:KnownNetworks:0"] = "192.0.2.0/24", ["ForwardedHeaders:ForwardLimit"] = limit }).Build(); + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration); var options = new ForwardedHeadersOptions(); - ForwardedHeadersConfiguration.Configure(options, configuration); + ForwardedHeadersConfiguration.Configure(options, settings); var context = new DefaultHttpContext(); context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40, 192.0.2.11"; @@ -83,8 +85,9 @@ public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer public async Task Invoke_DirectRequest_PreservesConnectionAddressAndScheme(string scheme) { // Arrange + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(new ConfigurationBuilder().Build()); var options = new ForwardedHeadersOptions(); - ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + ForwardedHeadersConfiguration.Configure(options, settings); var context = new DefaultHttpContext(); context.Connection.RemoteIpAddress = IPAddress.Parse("203.0.113.40"); context.Request.Scheme = scheme; @@ -101,7 +104,7 @@ public async Task Invoke_DirectRequest_PreservesConnectionAddressAndScheme(strin [Theory] [InlineData("ForwardedHeaders:ForwardLimit", "-1")] [InlineData("ForwardedHeaders_Enabled", "true")] - public void Configure_UnsafeScalarConfiguration_ThrowsWithKey(string key, string value) + public void ReadFromConfiguration_UnsafeScalarConfiguration_ThrowsWithKey(string key, string value) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary @@ -110,7 +113,7 @@ public void Configure_UnsafeScalarConfiguration_ThrowsWithKey(string key, string }).Build(); // Act - var exception = Record.Exception(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + var exception = Record.Exception(() => TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration)); // Assert Assert.IsType(exception); @@ -127,7 +130,7 @@ public void Configure_UnsafeScalarConfiguration_ThrowsWithKey(string key, string [InlineData("KnownProxies:0", "")] [InlineData("KnownProxies", "192.0.2.10,192.0.2.11")] [InlineData("KnownNetworks", "192.0.2.0/24")] - public void Configure_InvalidPeerConfiguration_ThrowsWithKey(string key, string value) + public void ReadFromConfiguration_InvalidPeerConfiguration_ThrowsWithKey(string key, string value) { // Arrange var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary @@ -136,7 +139,7 @@ public void Configure_InvalidPeerConfiguration_ThrowsWithKey(string key, string }).Build(); // Act - var exception = Record.Exception(() => ForwardedHeadersConfiguration.Configure(new ForwardedHeadersOptions(), configuration)); + var exception = Record.Exception(() => TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration)); // Assert Assert.IsType(exception); @@ -150,8 +153,9 @@ public void Configure_InvalidPeerConfiguration_ThrowsWithKey(string key, string public async Task Invoke_Defaults_TrustsLoopbackAndProcessesOneHop(string remoteAddress) { // Arrange + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(new ConfigurationBuilder().Build()); var options = new ForwardedHeadersOptions(); - ForwardedHeadersConfiguration.Configure(options, new ConfigurationBuilder().Build()); + ForwardedHeadersConfiguration.Configure(options, settings); var context = new DefaultHttpContext(); context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40"; @@ -167,9 +171,9 @@ public async Task Invoke_Defaults_TrustsLoopbackAndProcessesOneHop(string remote } [Theory] - [InlineData("true", ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto)] - [InlineData("false", ForwardedHeaders.None)] - public void Configure_PrefixedEnvironmentLists_ReadsNestedSettings(string enabled, ForwardedHeaders expectedHeaders) + [InlineData("true", true)] + [InlineData("false", false)] + public void ReadFromConfiguration_PrefixedEnvironmentLists_ReadsNestedSettings(string enabled, bool expectedEnabled) { // Arrange var provider = new CustomEnvironmentVariablesConfigurationProvider(); @@ -181,15 +185,14 @@ public void Configure_PrefixedEnvironmentLists_ReadsNestedSettings(string enable ["EX_ForwardedHeaders__KnownNetworks__0"] = "2001:db8::/64", ["EX_ForwardedHeaders__ForwardLimit"] = "2" }); - var options = new ForwardedHeadersOptions(); // Act - ForwardedHeadersConfiguration.Configure(options, configuration); + var options = TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration); // Assert Assert.Contains(IPAddress.Parse("192.0.2.10"), options.KnownProxies); - Assert.Contains(System.Net.IPNetwork.Parse("2001:db8::/64"), options.KnownIPNetworks); + Assert.Contains(System.Net.IPNetwork.Parse("2001:db8::/64"), options.KnownNetworks); Assert.Equal(2, options.ForwardLimit); - Assert.Equal(expectedHeaders, options.ForwardedHeaders); + Assert.Equal(expectedEnabled, options.Enabled); } } From 9f5647b87f1919f437c53e747cb0b604f1098447 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 18 Oct 2026 11:10:01 -0500 Subject: [PATCH 22/26] Validate gateway readiness and use Node 24 LTS for deployment --- .github/workflows/build.yaml | 7 +++++- Dockerfile | 2 +- docs/docs/self-hosting/kubernetes.md | 17 ++++++++++++++- k8s/scripts/discover-forwarded-headers.js | 26 +++++++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 33a78435a4..477169db3e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -700,6 +700,11 @@ jobs: with: ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} + - name: Setup Node.js environment + uses: actions/setup-node@v7 + with: + node-version: 24 + - name: Install Helm uses: azure/setup-helm@v5 @@ -844,7 +849,7 @@ jobs: run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml node k8s/scripts/discover-forwarded-headers.js ex-prod k8s/ex-prod-values.yaml "$RUNNER_TEMP/forwarded-headers.json" - helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" ex-prod --namespace ex-prod ./k8s/exceptionless + helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" --wait --timeout 5m ex-prod --namespace ex-prod ./k8s/exceptionless - name: Summary if: ${{ always() }} diff --git a/Dockerfile b/Dockerfile index 2e4084f47d..a3fd7e30f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN dotnet build ./src/Exceptionless.Web/Exceptionless.Web.csproj -c Release --n FROM build AS build-node RUN apt-get update -yq \ && apt-get install -yq curl ca-certificates \ - && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ && apt-get install -yq nodejs \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index a36c05e1e1..d5574cef96 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -40,7 +40,9 @@ config: The repository's ingress templates use Azure Application Gateway for Containers, terminate TLS at ingress, and route `/` to the API Service on port 80 and container port 8080. The hosted development and production deployment workflow discovers the gateway's association subnet before each Helm upgrade, so subnet addresses do not need to be duplicated in environment values files. -The shared `k8s/scripts/discover-forwarded-headers.js` script renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. +The shared `k8s/scripts/discover-forwarded-headers.js` script requires Node 24 LTS or later. It renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. + +Discovery also requires the gateway's `Accepted` and `Deployment` conditions to be `True`, with each `observedGeneration` equal to `metadata.generation`. A missing, stale, or deleting gateway stops discovery before Helm changes the application. Wait for the ALB controller to finish and rerun deployment. See Microsoft's [successful gateway provisioning example](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/quickstart-create-application-gateway-for-containers-managed-by-alb-controller#validate-creation-of-the-application-gateway-for-containers-resources). Discovery writes a `forwardedHeaders.knownNetworks` Helm list. The chart converts it into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys, enables forwarding, and sets a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining discovery with manual `ForwardedHeaders` entries in `config` is rejected; remove those old entries from saved release values before using discovery. @@ -57,6 +59,19 @@ For production, use `ex-prod`, `k8s/ex-prod-values.yaml`, and a separate output This assumes the gateway connects directly to the application. Verify the API-visible peer and any intermediate proxies before the first rollout; discovering Azure configuration does not prove the live traffic path. The ALB controller pod and Service address are not substitutes for the association subnet. Other ingress topologies should use explicit trusted peers in `config` as shown above. See Microsoft's [association subnet documentation](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/application-gateway-for-containers-components). +### Rollout checks and configuration failures + +For hosted deployments, check the deployment identity's read permissions and the actual proxy path once per environment before the first rollout. Remove conflicting saved forwarding overrides if present. Repeat these checks when the deployment identity or network topology changes. Each subsequent deployment discovers the current subnet automatically; no copied subnet values are required. + +| Failure | Result and recovery | +| --- | --- | +| Discovery cannot read a ready gateway or a valid subnet | The workflow stops before the application Helm upgrade. Fix permissions or wait for reconciliation, then rerun. Earlier development infrastructure steps may already have run. | +| Invalid application settings, such as a malformed IP, boolean, or `/0` network | New application processes fail at startup. Correct the configuration and redeploy. Production Helm waits up to five minutes for readiness and reports failure; it does not automatically roll back. | +| A valid CIDR does not contain the actual proxy, or forwarding is disabled | The app can remain ready while it ignores forwarded headers. Users can share the proxy's IP limits, and the request scheme reflects the backend connection. Readiness checks do not detect this. | +| The trusted range includes other callers | Those callers can supply forged client IPs or schemes. Use only the dedicated gateway subnet or verified proxy IPs. | + +After the first authorized development rollout, verify client IPs, HTTPS detection, and rejection of spoofed forwarding headers before a production release. The existing version and readiness checks do not prove those behaviors. + ## ConnectionStrings ```yaml diff --git a/k8s/scripts/discover-forwarded-headers.js b/k8s/scripts/discover-forwarded-headers.js index 4a28dc5524..0568beaa02 100644 --- a/k8s/scripts/discover-forwarded-headers.js +++ b/k8s/scripts/discover-forwarded-headers.js @@ -34,6 +34,30 @@ function discoverKnownNetworks(release, valuesFile) { return [...new Set(networks)].sort(); } +function ensureGatewayReady(gateway) { + if (gateway.metadata?.deletionTimestamp) { + throw new Error('Gateway is being deleted; refusing to infer proxy trust'); + } + + const generation = gateway.metadata?.generation; + const conditions = gateway.status?.conditions; + + if (!Number.isSafeInteger(generation) || generation < 1 || !Array.isArray(conditions)) { + throw new Error('Gateway has no current readiness status; wait for the ALB controller and retry deployment'); + } + + // Both conditions must describe this spec, not the previous association subnet. + for (const type of ['Accepted', 'Deployment']) { + const condition = conditions.find(condition => condition?.type === type); + + if (condition?.status !== 'True' || condition.observedGeneration !== generation) { + throw new Error( + `Gateway ${type} is not ready for generation ${generation}; wait for the ALB controller and retry deployment` + ); + } + } +} + function main() { const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); @@ -85,6 +109,8 @@ function readAssociationSubnetIds(gatewayName, gatewayNamespace) { throw new Error(`ApplicationLoadBalancer ${gatewayNamespace}/${gatewayName} returned an invalid object`); } + ensureGatewayReady(gateway); + const associations = gateway.spec?.associations; if (!Array.isArray(associations) || associations.length === 0) { From 658638473b0bb42575f0be8829eceda8403afb86 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 18 Oct 2026 11:20:41 -0500 Subject: [PATCH 23/26] Expose effective request scheme in admin echo --- docs/docs/self-hosting/kubernetes.md | 31 +++++++++++++++++++ .../Api/Handlers/AdminHandler.cs | 3 +- .../Api/Endpoints/AdminEndpointTests.cs | 16 ++++++++-- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index d5574cef96..95f38a359c 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -72,6 +72,37 @@ For hosted deployments, check the deployment identity's read permissions and the After the first authorized development rollout, verify client IPs, HTTPS detection, and rejection of spoofed forwarding headers before a production release. The existing version and readiness checks do not prove those behaviors. +Use the existing global-admin-only `GET /api/v2/admin/echo` endpoint for these checks. Its `ip_address` and `scheme` fields report the values the application sees after forwarded-header processing. Do not share the full echo response: its `headers` field can contain credentials and cookies. + +| Step | Check | Pass condition | +| --- | --- | --- | +| Before merge | Required CI and deployment identity permissions | CI passes; the deployment identity can read the selected gateway and its association subnets. Merging main starts the dev deployment. | +| Discovery | Gateway conditions and generated `forwardedHeaders.knownNetworks` | Current-generation `Accepted` and `Deployment` conditions are true. Generated CIDRs match the dedicated gateway subnets. Discovery enforces readiness, identity, delegation, and prefix validation. | +| Configuration | Saved Helm values, rendered ConfigMap, and new app/API pod environment | `EX_ForwardedHeaders__Enabled=true`, `EX_ForwardedHeaders__ForwardLimit=1`, and indexed `EX_ForwardedHeaders__KnownNetworks__*` match discovery. No conflicting forwarding overrides remain. Inspect only these keys, not unrelated secrets. | +| Rollout | Pod readiness, startup logs, dev migrations, and `/api/v2/about` | New app/API pods are ready, no configuration errors occur, migrations finish, and the expected version is served. | +| Baseline request | Call admin echo through HTTPS ingress with no forwarding or `X-Original-*` headers supplied by the client | `ip_address` equals the caller's known public egress IP and `scheme` is `https`. The baseline `X-Original-For` identifies a gateway peer within the discovered subnet. Repeat on both app and API ingress. | +| Spoofed request | Repeat through the same ingress with `X-Forwarded-For: 203.0.113.123` and `X-Forwarded-Proto: http` | `ip_address` stays equal to the caller's actual egress IP and `scheme` stays `https`. Fail if either supplied value controls the result. | +| Direct untrusted request | In an authorized dev check, request the backend directly from a source outside trusted networks with the same fake headers | Forwarded values are ignored; `ip_address` is the direct peer and `scheme` is the backend transport scheme. Loopback or port-forwarding cannot prove this because loopback is trusted. | +| Production release | After dev passes, use the normal release-tag workflow and repeat baseline/spoof checks | Production serves the expected version with correct IP and scheme on both hosts. Stop promotion on any failed check; readiness alone is insufficient. | + +For each request, retain only the observed IP, scheme, host, and pass/fail result. A global administrator can make the requests with their normal authenticated client. An unauthenticated request must still return `401`, and a non-admin request must return `403`. + +For an authorized rollout, the following commands prompt for the administrator's password and print only the diagnostic fields. Set `BASE_URL` to the HTTPS ingress being checked and `ADMIN_EMAIL` to the administrator's email. Run the baseline first, then the spoof check; both must report the same real client IP and `scheme: "https"`. + +```sh +# Baseline: do not supply forwarding or X-Original-* headers. +curl --fail --silent --show-error --user "$ADMIN_EMAIL" \ + "$BASE_URL/api/v2/admin/echo" \ + | jq '{ip_address, scheme, gateway_peer: .headers["X-Original-For"]}' + +# Spoof check: these supplied values must not become the effective IP or scheme. +curl --fail --silent --show-error --user "$ADMIN_EMAIL" \ + --header 'X-Forwarded-For: 203.0.113.123' \ + --header 'X-Forwarded-Proto: http' \ + "$BASE_URL/api/v2/admin/echo" \ + | jq '{ip_address, scheme}' +``` + ## ConnectionStrings ```yaml diff --git a/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs b/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs index 9bf11786b4..c758517674 100644 --- a/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs @@ -248,7 +248,8 @@ public Task> Handle(GetAdminEcho message) return Task.FromResult>(new { httpContext.Request.Headers, - IpAddress = httpContext.Request.GetClientIpAddress() + IpAddress = httpContext.Request.GetClientIpAddress(), + httpContext.Request.Scheme }); } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs index 65f4c29ed4..b5fdb347b0 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs @@ -1124,12 +1124,24 @@ public Task GetSettings_WithoutAuth_ReturnsUnauthorized() } [Fact] - public Task EchoRequest_AsGlobalAdmin_ReturnsIpAddress() + public async Task EchoRequest_AsGlobalAdmin_ReturnsForwardedIpAddressAndScheme() { - return SendRequestAsync(r => r + // Arrange + const string clientIpAddress = "203.0.113.42"; + const string scheme = "https"; + + // Act + var response = await SendRequestAsAsync(r => r .AsGlobalAdminUser() .AppendPaths("admin", "echo") + .Header("X-Forwarded-For", clientIpAddress) + .Header("X-Forwarded-Proto", scheme) .StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(clientIpAddress, response.GetProperty("ip_address").GetString()); + Assert.Equal(scheme, response.GetProperty("scheme").GetString()); + Assert.Equal(JsonValueKind.Object, response.GetProperty("headers").ValueKind); } [Fact] From fa7932e82eff0cfe3ab56b9d0b7e676beac608c5 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 18 Oct 2026 13:57:05 -0500 Subject: [PATCH 24/26] Wait for the selected saved view in navigation test --- .../ClientApp/e2e/tests/operator-navigation.e2e.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/operator-navigation.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/operator-navigation.e2e.ts index 575f7c23af..262b7def8b 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/operator-navigation.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/operator-navigation.e2e.ts @@ -79,7 +79,8 @@ test('operator can navigate from event discovery to event and stack details thro await test.step('open stack details from the Stacks table', async () => { await navigateToSidebarView(page, 'Stacks', 'Most Frequent Errors'); - await expect(page.getByRole('heading', { name: 'Stacks' })).toBeVisible(); + await expect(page).toHaveURL(/\/stack\/most-frequent-errors(?:[?#]|$)/); + await expect(page.getByRole('heading', { exact: true, name: 'Most Frequent Errors' })).toBeVisible(); const stackListUrl = page.url(); const stackRow = getVisibleRow(page, journey.message); From fc93c8910fcbda5dd9bde957ed53fdccae45f336 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 18 Oct 2026 20:55:52 -0500 Subject: [PATCH 25/26] Fix gateway discovery resource and Windows Azure CLI invocation --- .github/workflows/build.yaml | 2 +- k8s/scripts/discover-forwarded-headers.js | 26 ++++- tests/ci/forwarded-headers.test.js | 114 ++++++++++++++++++++++ 3 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 tests/ci/forwarded-headers.test.js diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 477169db3e..5cfd47fb80 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -172,7 +172,7 @@ jobs: run: dotnet build ./Exceptionless.slnx --no-restore --configuration Release - name: Verify CI test tooling - run: node --test tests/ci/shards.test.js + run: node --test tests/ci/*.test.js - name: Run .NET shard with coverage run: node tests/ci/backend-shards.js run --index ${{ matrix.shard }} --count 4 --output coverage --coverage diff --git a/k8s/scripts/discover-forwarded-headers.js b/k8s/scripts/discover-forwarded-headers.js index 0568beaa02..bb3d01206a 100644 --- a/k8s/scripts/discover-forwarded-headers.js +++ b/k8s/scripts/discover-forwarded-headers.js @@ -5,7 +5,7 @@ import { isIP } from 'node:net'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -const APPLICATION_LOAD_BALANCER_RESOURCE = 'applicationloadbalancers.alb.networking.azure.io'; +const APPLICATION_LOAD_BALANCER_RESOURCE = 'applicationloadbalancer.alb.networking.azure.io'; const ASSOCIATION_SUBNET_ID = /^\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.Network\/virtualNetworks\/[^/]+\/subnets\/[^/]+$/i; const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; @@ -218,10 +218,28 @@ function readSubnetPrefixes(subnetId) { } function run(command, arguments_) { - return execFileSync(command, arguments_, { + const options = { encoding: 'utf8', - timeout: 60_000 - }); + timeout: 60_000, + windowsHide: true + }; + + // Azure CLI uses az.cmd on Windows, which requires the command interpreter. + // Reject shell syntax before quoting arguments, including the discovered subnet ID. + if (process.platform === 'win32' && command === 'az') { + // eslint-disable-next-line no-control-regex -- Control characters are unsafe in a command line. + if (arguments_.some(argument => /[\x00-\x1f"<>|&^%!\\]/.test(argument))) { + throw new Error('Azure CLI arguments contain unsupported Windows command-shell characters'); + } + + const commandLine = `"az.cmd ${arguments_.map(argument => `"${argument}"`).join(' ')}"`; + return execFileSync(process.env.ComSpec ?? 'cmd.exe', ['/d', '/v:off', '/s', '/c', commandLine], { + ...options, + windowsVerbatimArguments: true + }); + } + + return execFileSync(command, arguments_, options); } function runJson(command, arguments_, description) { diff --git a/tests/ci/forwarded-headers.test.js b/tests/ci/forwarded-headers.test.js new file mode 100644 index 0000000000..110f7f428e --- /dev/null +++ b/tests/ci/forwarded-headers.test.js @@ -0,0 +1,114 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const repositoryRoot = fileURLToPath(new URL('../../', import.meta.url)); +const discoveryScript = join(repositoryRoot, 'k8s/scripts/discover-forwarded-headers.js'); +const subnetId = '/subscriptions/subscription/resourceGroups/gateway-group/providers/Microsoft.Network/virtualNetworks/network/subnets/gateway'; + +function discover(directory, association = subnetId) { + const output = join(directory, 'forwarding.json'); + const azureFixture = join(directory, 'azure-fixture.mjs'); + writeFileSync(azureFixture, ` + import assert from 'node:assert/strict'; + assert.deepEqual(process.argv.slice(2), [ + 'network', 'vnet', 'subnet', 'show', '--ids', process.env.TEST_SUBNET, + '--query', '{id:id,addressPrefix:addressPrefix,addressPrefixes:addressPrefixes,delegations:delegations[].serviceName}', + '--output', 'json', '--only-show-errors' + ]); + console.log(JSON.stringify({ + id: process.env.TEST_SUBNET, + addressPrefix: '10.60.64.0/24', + delegations: ['Microsoft.ServiceNetworking/trafficControllers'] + })); + `); + // Exercise the real command interpreter and argument forwarding on Windows. + writeFileSync(join(directory, 'az.cmd'), `@"${process.execPath}" "${azureFixture}" %*\r\n`); + + const runner = join(directory, 'runner.mjs'); + writeFileSync(runner, ` + import assert from 'node:assert/strict'; + import childProcess from 'node:child_process'; + import { syncBuiltinESMExports } from 'node:module'; + import { pathToFileURL } from 'node:url'; + const execute = childProcess.execFileSync; + childProcess.execFileSync = (command, args, options) => { + if (command === 'helm') { + return ' alb.networking.azure.io/alb-name: public-alb\\n alb.networking.azure.io/alb-namespace: kube-system\\n'; + } + if (command === 'kubectl') { + assert.deepEqual(args, [ + 'get', 'applicationloadbalancer.alb.networking.azure.io', 'public-alb', + '--namespace', 'kube-system', '--output', 'json' + ]); + return JSON.stringify({ + metadata: { generation: 1 }, + spec: { associations: [process.env.TEST_SUBNET] }, + status: { conditions: ['Accepted', 'Deployment'].map(type => ({ type, status: 'True', observedGeneration: 1 })) } + }); + } + if (command === 'az') { + assert.notEqual(process.platform, 'win32', 'Windows must invoke az.cmd through its interpreter'); + return execute(process.execPath, [process.env.TEST_AZURE_FIXTURE, ...args], options); + } + return execute(command, args, options); + }; + syncBuiltinESMExports(); + process.argv = [process.execPath, process.env.TEST_DISCOVERY_SCRIPT, 'ex-test', 'values.yaml', process.env.TEST_OUTPUT]; + await import(pathToFileURL(process.env.TEST_DISCOVERY_SCRIPT)); + `); + + const result = spawnSync(process.execPath, [runner], { + cwd: directory, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${directory}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH}`, + TEST_AZURE_FIXTURE: azureFixture, + TEST_DISCOVERY_SCRIPT: discoveryScript, + TEST_OUTPUT: output, + TEST_SUBNET: association + }, + timeout: 15_000, + windowsHide: true + }); + + assert.ifError(result.error); + return { ...result, output }; +} + +test('discovery uses the registered gateway resource and preserves Azure CLI arguments', () => { + const directory = mkdtempSync(join(tmpdir(), 'forwarding test ')); + try { + const result = discover(directory); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(result.output, 'utf8')), { + forwardedHeaders: { knownNetworks: ['10.60.64.0/24'] } + }); + } finally { + rmSync(directory, { force: true, recursive: true }); + } +}); + +test('subnet data cannot expand Windows shell syntax and failed discovery removes stale output', () => { + const directory = mkdtempSync(join(tmpdir(), 'forwarding test ')); + try { + const output = join(directory, 'forwarding.json'); + writeFileSync(output, 'stale forwarding values'); + const result = discover(directory, `${subnetId}%PATH%`); + if (process.platform === 'win32') { + assert.equal(result.status, 1); + assert.match(result.stderr, /unsupported Windows command-shell characters/); + assert.equal(existsSync(output), false); + } else { + // Direct execution must pass the percent signs literally, without shell expansion. + assert.equal(result.status, 0, result.stderr); + } + } finally { + rmSync(directory, { force: true, recursive: true }); + } +}); From a7bec909f958634b42f3ba3165e42e387d0f969f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 18 Oct 2026 21:11:30 -0500 Subject: [PATCH 26/26] Configure the verified gateway subnet without deployment discovery --- .github/workflows/build.yaml | 13 +- Dockerfile | 2 +- docs/docs/self-hosting/kubernetes.md | 35 +-- k8s/ex-dev-values.yaml | 4 + k8s/ex-prod-values.yaml | 4 + k8s/exceptionless/templates/config.yaml | 2 +- k8s/exceptionless/values.yaml | 2 +- k8s/scripts/discover-forwarded-headers.js | 279 ---------------------- tests/ci/forwarded-headers.test.js | 114 --------- 9 files changed, 27 insertions(+), 428 deletions(-) delete mode 100644 k8s/scripts/discover-forwarded-headers.js delete mode 100644 tests/ci/forwarded-headers.test.js diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5cfd47fb80..64c0941bc9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -172,7 +172,7 @@ jobs: run: dotnet build ./Exceptionless.slnx --no-restore --configuration Release - name: Verify CI test tooling - run: node --test tests/ci/*.test.js + run: node --test tests/ci/shards.test.js - name: Run .NET shard with coverage run: node tests/ci/backend-shards.js run --index ${{ matrix.shard }} --count 4 --output coverage --coverage @@ -700,11 +700,6 @@ jobs: with: ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} - - name: Setup Node.js environment - uses: actions/setup-node@v7 - with: - node-version: 24 - - name: Install Helm uses: azure/setup-helm@v5 @@ -728,8 +723,7 @@ jobs: if: ${{ needs.version.outputs.is_dev_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - node k8s/scripts/discover-forwarded-headers.js ex-dev k8s/ex-dev-values.yaml "$RUNNER_TEMP/forwarded-headers.json" - helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless + helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless - name: Run Development Migrations if: ${{ needs.version.outputs.is_dev_deploy == 'true' }} @@ -848,8 +842,7 @@ jobs: if: ${{ needs.version.outputs.is_prod_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - node k8s/scripts/discover-forwarded-headers.js ex-prod k8s/ex-prod-values.yaml "$RUNNER_TEMP/forwarded-headers.json" - helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --values "$RUNNER_TEMP/forwarded-headers.json" --wait --timeout 5m ex-prod --namespace ex-prod ./k8s/exceptionless + helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --wait --timeout 5m ex-prod --namespace ex-prod ./k8s/exceptionless - name: Summary if: ${{ always() }} diff --git a/Dockerfile b/Dockerfile index a3fd7e30f3..2e4084f47d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN dotnet build ./src/Exceptionless.Web/Exceptionless.Web.csproj -c Release --n FROM build AS build-node RUN apt-get update -yq \ && apt-get install -yq curl ca-certificates \ - && curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ + && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && apt-get install -yq nodejs \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index 95f38a359c..8c551e28e7 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -38,34 +38,26 @@ config: EX_ForwardedHeaders__ForwardLimit: "1" ``` -The repository's ingress templates use Azure Application Gateway for Containers, terminate TLS at ingress, and route `/` to the API Service on port 80 and container port 8080. The hosted development and production deployment workflow discovers the gateway's association subnet before each Helm upgrade, so subnet addresses do not need to be duplicated in environment values files. +For a dedicated ingress subnet with a single proxy hop, configure the chart's `forwardedHeaders.knownNetworks` list instead. Replace this example with your verified proxy subnet: -The shared `k8s/scripts/discover-forwarded-headers.js` script requires Node 24 LTS or later. It renders the environment's API ingress locally, reads the selected `ApplicationLoadBalancer` through Kubernetes, and resolves its association subnet IDs through Azure. It requires the existing deployment identity to have Kubernetes read access to that resource and Azure `Microsoft.Network/virtualNetworks/subnets/read` access to its subnets. Missing associations, failed reads, missing `Microsoft.ServiceNetworking/trafficControllers` delegation, invalid prefixes, and unrestricted `/0` prefixes stop deployment. - -Discovery also requires the gateway's `Accepted` and `Deployment` conditions to be `True`, with each `observedGeneration` equal to `metadata.generation`. A missing, stale, or deleting gateway stops discovery before Helm changes the application. Wait for the ALB controller to finish and rerun deployment. See Microsoft's [successful gateway provisioning example](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/quickstart-create-application-gateway-for-containers-managed-by-alb-controller#validate-creation-of-the-application-gateway-for-containers-resources). - -Discovery writes a `forwardedHeaders.knownNetworks` Helm list. The chart converts it into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys, enables forwarding, and sets a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining discovery with manual `ForwardedHeaders` entries in `config` is rejected; remove those old entries from saved release values before using discovery. - -For manual upgrades, run discovery from the repository root after selecting the intended Kubernetes context and Azure subscription, then pass the generated file **last** to Helm. For example, the development commands are: - -```sh -node k8s/scripts/discover-forwarded-headers.js ex-dev k8s/ex-dev-values.yaml /tmp/ex-dev-forwarded-headers.json -# Run only if discovery succeeded: -helm upgrade ex-dev k8s/exceptionless --namespace ex-dev --reuse-values \ - --values k8s/ex-dev-values.yaml --values /tmp/ex-dev-forwarded-headers.json +```yaml +forwardedHeaders: + knownNetworks: + - 192.0.2.0/24 ``` -For production, use `ex-prod`, `k8s/ex-prod-values.yaml`, and a separate output file. The script only reads infrastructure and replaces the local values file; Helm performs the upgrade. It removes any previous output before discovery so a failed refresh cannot leave stale trust available for reuse. Automated workflows stop on discovery failure; manual operators must also stop. +The chart converts this list into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys, enables forwarding, and sets a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining a nonempty list with manual `ForwardedHeaders` entries in `config` is rejected; remove conflicting entries from saved release values. The reusable chart's list is empty by default. For multiple proxy hops or other forwarding options, use explicit settings in `config` instead. + +The hosted development and production values files explicitly trust `10.60.64.0/24`, the dedicated `ex-net-v6/ex-agc` association subnet for `kube-system/public-alb`. Update both files with any gateway subnet change. Deployments use these reviewed values without querying Azure for proxy trust. Self-hosted installations must configure their own proxy addresses. -This assumes the gateway connects directly to the application. Verify the API-visible peer and any intermediate proxies before the first rollout; discovering Azure configuration does not prove the live traffic path. The ALB controller pod and Service address are not substitutes for the association subnet. Other ingress topologies should use explicit trusted peers in `config` as shown above. See Microsoft's [association subnet documentation](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/application-gateway-for-containers-components). +The repository's Azure Application Gateway for Containers ingress terminates TLS and routes directly to application Services on port 80 and pods on port 8080. Verify the API-visible peer and any intermediate proxies before the first rollout and whenever the network changes. The ALB controller pod and Service address are not substitutes for the gateway's [association subnet](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/application-gateway-for-containers-components). ### Rollout checks and configuration failures -For hosted deployments, check the deployment identity's read permissions and the actual proxy path once per environment before the first rollout. Remove conflicting saved forwarding overrides if present. Repeat these checks when the deployment identity or network topology changes. Each subsequent deployment discovers the current subnet automatically; no copied subnet values are required. +Before the first hosted rollout, verify the configured subnet against the gateway's association subnet and actual proxy path, and remove conflicting saved forwarding overrides. Repeat these checks when the network configuration changes. The separate framework `ForwardedHeaders_Enabled` shortcut must be absent or false. | Failure | Result and recovery | | --- | --- | -| Discovery cannot read a ready gateway or a valid subnet | The workflow stops before the application Helm upgrade. Fix permissions or wait for reconciliation, then rerun. Earlier development infrastructure steps may already have run. | | Invalid application settings, such as a malformed IP, boolean, or `/0` network | New application processes fail at startup. Correct the configuration and redeploy. Production Helm waits up to five minutes for readiness and reports failure; it does not automatically roll back. | | A valid CIDR does not contain the actual proxy, or forwarding is disabled | The app can remain ready while it ignores forwarded headers. Users can share the proxy's IP limits, and the request scheme reflects the backend connection. Readiness checks do not detect this. | | The trusted range includes other callers | Those callers can supply forged client IPs or schemes. Use only the dedicated gateway subnet or verified proxy IPs. | @@ -76,11 +68,10 @@ Use the existing global-admin-only `GET /api/v2/admin/echo` endpoint for these c | Step | Check | Pass condition | | --- | --- | --- | -| Before merge | Required CI and deployment identity permissions | CI passes; the deployment identity can read the selected gateway and its association subnets. Merging main starts the dev deployment. | -| Discovery | Gateway conditions and generated `forwardedHeaders.knownNetworks` | Current-generation `Accepted` and `Deployment` conditions are true. Generated CIDRs match the dedicated gateway subnets. Discovery enforces readiness, identity, delegation, and prefix validation. | -| Configuration | Saved Helm values, rendered ConfigMap, and new app/API pod environment | `EX_ForwardedHeaders__Enabled=true`, `EX_ForwardedHeaders__ForwardLimit=1`, and indexed `EX_ForwardedHeaders__KnownNetworks__*` match discovery. No conflicting forwarding overrides remain. Inspect only these keys, not unrelated secrets. | +| Before merge | Required CI and configured gateway trust | CI passes; environment values match the verified dedicated gateway subnet and proxy path. Merging main starts the dev deployment. | +| Configuration | Saved Helm values, rendered ConfigMap, and new app/API pod environment | `EX_ForwardedHeaders__Enabled=true`, `EX_ForwardedHeaders__ForwardLimit=1`, and indexed `EX_ForwardedHeaders__KnownNetworks__*` match environment values. No conflicting forwarding overrides remain. Inspect only these keys, not unrelated secrets. | | Rollout | Pod readiness, startup logs, dev migrations, and `/api/v2/about` | New app/API pods are ready, no configuration errors occur, migrations finish, and the expected version is served. | -| Baseline request | Call admin echo through HTTPS ingress with no forwarding or `X-Original-*` headers supplied by the client | `ip_address` equals the caller's known public egress IP and `scheme` is `https`. The baseline `X-Original-For` identifies a gateway peer within the discovered subnet. Repeat on both app and API ingress. | +| Baseline request | Call admin echo through HTTPS ingress with no forwarding or `X-Original-*` headers supplied by the client | `ip_address` equals the caller's known public egress IP and `scheme` is `https`. The baseline `X-Original-For` identifies a gateway peer within the configured subnet. Repeat on both app and API ingress. | | Spoofed request | Repeat through the same ingress with `X-Forwarded-For: 203.0.113.123` and `X-Forwarded-Proto: http` | `ip_address` stays equal to the caller's actual egress IP and `scheme` stays `https`. Fail if either supplied value controls the result. | | Direct untrusted request | In an authorized dev check, request the backend directly from a source outside trusted networks with the same fake headers | Forwarded values are ignored; `ip_address` is the direct peer and `scheme` is the backend transport scheme. Loopback or port-forwarding cannot prove this because loopback is trusted. | | Production release | After dev passes, use the normal release-tag workflow and repeat baseline/spoof checks | Production serves the expected version with correct IP and scheme on both hosts. Stop promotion on any failed check; readiness alone is insufficient. | diff --git a/k8s/ex-dev-values.yaml b/k8s/ex-dev-values.yaml index 0641e5e2ff..341471ab7d 100644 --- a/k8s/ex-dev-values.yaml +++ b/k8s/ex-dev-values.yaml @@ -1,5 +1,9 @@ appMode: Staging appScope: dev +# Dedicated public-alb association subnet (ex-net-v6/ex-agc); update with gateway network changes. +forwardedHeaders: + knownNetworks: + - 10.60.64.0/24 app: image: repository: exceptionless/app-ci diff --git a/k8s/ex-prod-values.yaml b/k8s/ex-prod-values.yaml index 363760edeb..c46a23cdc3 100644 --- a/k8s/ex-prod-values.yaml +++ b/k8s/ex-prod-values.yaml @@ -1,5 +1,9 @@ appMode: Production appScope: prod +# Dedicated public-alb association subnet (ex-net-v6/ex-agc); update with gateway network changes. +forwardedHeaders: + knownNetworks: + - 10.60.64.0/24 app: minReplicaCount: 2 maxReplicaCount: 5 diff --git a/k8s/exceptionless/templates/config.yaml b/k8s/exceptionless/templates/config.yaml index e464a81344..2823ac70bc 100644 --- a/k8s/exceptionless/templates/config.yaml +++ b/k8s/exceptionless/templates/config.yaml @@ -21,7 +21,7 @@ data: {{- if $knownNetworks }} {{- range $key, $value := .Values.config }} {{- if hasPrefix "forwardedheaders:" (lower (replace "__" ":" (trimPrefix "EX_" $key))) }} - {{- fail "Do not combine discovered forwardedHeaders.knownNetworks with ForwardedHeaders config overrides; remove the old overrides" }} + {{- fail "Do not combine forwardedHeaders.knownNetworks with ForwardedHeaders config overrides; remove the old overrides" }} {{- end }} {{- end }} EX_ForwardedHeaders__Enabled: "true" diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index 23de64fb31..dc37340412 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -22,7 +22,7 @@ jobs: config: {} # key: value -# Populated by k8s/scripts/discover-forwarded-headers.js for hosted deployments. +# Verified, dedicated ingress subnets; configure them in environment values. # A replacement list prevents stale subnet entries surviving --reuse-values. forwardedHeaders: knownNetworks: [] diff --git a/k8s/scripts/discover-forwarded-headers.js b/k8s/scripts/discover-forwarded-headers.js deleted file mode 100644 index bb3d01206a..0000000000 --- a/k8s/scripts/discover-forwarded-headers.js +++ /dev/null @@ -1,279 +0,0 @@ -import { execFileSync } from 'node:child_process'; -import { randomUUID } from 'node:crypto'; -import { realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'; -import { isIP } from 'node:net'; -import { resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; - -const APPLICATION_LOAD_BALANCER_RESOURCE = 'applicationloadbalancer.alb.networking.azure.io'; -const ASSOCIATION_SUBNET_ID = - /^\/subscriptions\/[^/]+\/resourceGroups\/[^/]+\/providers\/Microsoft\.Network\/virtualNetworks\/[^/]+\/subnets\/[^/]+$/i; -const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; -const MAX_HELM_RELEASE_NAME_LENGTH = 53; -const TRAFFIC_CONTROLLER_DELEGATION = 'Microsoft.ServiceNetworking/trafficControllers'; -const USAGE = - 'Usage: node k8s/scripts/discover-forwarded-headers.js (from repository root)'; - -function discoverKnownNetworks(release, valuesFile) { - const manifest = run('helm', [ - 'template', - release, - 'k8s/exceptionless', - '--values', - valuesFile, - '--show-only', - 'templates/api.yaml' - ]); - const gatewayName = readIngressAnnotation(manifest, 'alb-name'); - const gatewayNamespace = readIngressAnnotation(manifest, 'alb-namespace'); - const subnetIds = readAssociationSubnetIds(gatewayName, gatewayNamespace); - const networks = subnetIds.flatMap(subnetId => readSubnetPrefixes(subnetId)); - - // A list replaces the previous list on Helm upgrades, including --reuse-values. - // Individual config-map keys would retain stale higher-numbered entries. - return [...new Set(networks)].sort(); -} - -function ensureGatewayReady(gateway) { - if (gateway.metadata?.deletionTimestamp) { - throw new Error('Gateway is being deleted; refusing to infer proxy trust'); - } - - const generation = gateway.metadata?.generation; - const conditions = gateway.status?.conditions; - - if (!Number.isSafeInteger(generation) || generation < 1 || !Array.isArray(conditions)) { - throw new Error('Gateway has no current readiness status; wait for the ALB controller and retry deployment'); - } - - // Both conditions must describe this spec, not the previous association subnet. - for (const type of ['Accepted', 'Deployment']) { - const condition = conditions.find(condition => condition?.type === type); - - if (condition?.status !== 'True' || condition.observedGeneration !== generation) { - throw new Error( - `Gateway ${type} is not ready for generation ${generation}; wait for the ALB controller and retry deployment` - ); - } - } -} - -function main() { - const [release, valuesFile, outputFile, ...extra] = process.argv.slice(2); - - if (!release || !valuesFile || !outputFile || extra.length > 0) { - throw new Error(USAGE); - } - - if (release.length > MAX_HELM_RELEASE_NAME_LENGTH || !HELM_RELEASE_NAME.test(release)) { - throw new Error('The Helm release name is invalid'); - } - - if (pathsReferToSameFile(valuesFile, outputFile)) { - throw new Error('The output file must be different from the input values file'); - } - - // A failed refresh must not leave an earlier trust file available for upgrade. - rmSync(outputFile, { force: true }); - - const knownNetworks = discoverKnownNetworks(release, valuesFile); - const values = { forwardedHeaders: { knownNetworks } }; - writeValuesFile(outputFile, values); - console.log(`Discovered ${knownNetworks.length} gateway subnet prefix(es)`); -} - -function pathsReferToSameFile(firstPath, secondPath) { - if (resolve(firstPath) === resolve(secondPath)) { - return true; - } - - try { - return realpathSync(firstPath) === realpathSync(secondPath); - } catch (error) { - if (error.code === 'ENOENT') { - return false; - } - - throw error; - } -} - -function readAssociationSubnetIds(gatewayName, gatewayNamespace) { - const gateway = runJson( - 'kubectl', - ['get', APPLICATION_LOAD_BALANCER_RESOURCE, gatewayName, '--namespace', gatewayNamespace, '--output', 'json'], - `ApplicationLoadBalancer ${gatewayNamespace}/${gatewayName}` - ); - - if (!gateway || typeof gateway !== 'object' || Array.isArray(gateway)) { - throw new Error(`ApplicationLoadBalancer ${gatewayNamespace}/${gatewayName} returned an invalid object`); - } - - ensureGatewayReady(gateway); - - const associations = gateway.spec?.associations; - - if (!Array.isArray(associations) || associations.length === 0) { - throw new Error('Gateway has no association subnet IDs; refusing to infer proxy trust'); - } - - const hasInvalidAssociation = associations.some( - association => typeof association !== 'string' || !ASSOCIATION_SUBNET_ID.test(association) - ); - - if (hasInvalidAssociation) { - throw new Error('Gateway contains an invalid Azure association subnet resource ID'); - } - - return [...new Set(associations)]; -} - -// Read the chart's rendered annotations so gateway selection cannot drift from ingress. -function readIngressAnnotation(manifest, annotationName) { - const annotationPattern = new RegExp( - `^\\s+alb\\.networking\\.azure\\.io/${annotationName}:\\s*([a-z0-9.-]+)\\s*$`, - 'gm' - ); - const values = [...new Set([...manifest.matchAll(annotationPattern)].map(match => match[1]))]; - - if (values.length === 0) { - throw new Error(`Rendered API ingress is missing the ${annotationName} annotation`); - } - - if (values.length > 1) { - throw new Error(`Rendered API ingresses use conflicting ${annotationName} annotations: ${values.join(', ')}`); - } - - return values[0]; -} - -function readSubnetPrefixes(subnetId) { - const subnet = runJson( - 'az', - [ - 'network', - 'vnet', - 'subnet', - 'show', - '--ids', - subnetId, - '--query', - '{id:id,addressPrefix:addressPrefix,addressPrefixes:addressPrefixes,delegations:delegations[].serviceName}', - '--output', - 'json', - '--only-show-errors' - ], - `Azure subnet ${subnetId}` - ); - - if (!subnet || typeof subnet !== 'object' || Array.isArray(subnet)) { - throw new Error(`Azure subnet ${subnetId} returned an invalid object`); - } - - if (typeof subnet.id !== 'string' || subnet.id.toLowerCase() !== subnetId.toLowerCase()) { - throw new Error(`Azure returned a different subnet for association ${subnetId}`); - } - - if (!Array.isArray(subnet.delegations) || !subnet.delegations.includes(TRAFFIC_CONTROLLER_DELEGATION)) { - throw new Error(`Association subnet ${subnetId} is not delegated to ${TRAFFIC_CONTROLLER_DELEGATION}`); - } - - const hasInvalidAddressPrefix = - subnet.addressPrefix !== null && subnet.addressPrefix !== undefined && typeof subnet.addressPrefix !== 'string'; - - if (hasInvalidAddressPrefix) { - throw new Error(`Association subnet ${subnetId} returned an invalid addressPrefix`); - } - - const hasInvalidAddressPrefixes = - subnet.addressPrefixes !== null && - subnet.addressPrefixes !== undefined && - (!Array.isArray(subnet.addressPrefixes) || - !subnet.addressPrefixes.every(prefix => typeof prefix === 'string')); - - if (hasInvalidAddressPrefixes) { - throw new Error(`Association subnet ${subnetId} returned invalid addressPrefixes`); - } - - const addressPrefixes = subnet.addressPrefixes ?? []; - const prefixes = [subnet.addressPrefix, ...addressPrefixes].filter(prefix => prefix !== null && prefix !== undefined); - - if (prefixes.length === 0) { - throw new Error(`Association subnet ${subnetId} has no address prefixes`); - } - - for (const prefix of prefixes) { - const [address, prefixLength, extra] = typeof prefix === 'string' ? prefix.split('/') : []; - const addressFamily = isIP(address ?? ''); - const maximumPrefixLength = addressFamily === 4 ? 32 : 128; - const hasValidPrefixLength = - /^[1-9][0-9]*$/.test(prefixLength ?? '') && Number(prefixLength) <= maximumPrefixLength; - - if (!addressFamily || extra !== undefined || !hasValidPrefixLength) { - throw new Error(`Association subnet ${subnetId} has an invalid or unrestricted prefix: ${prefix}`); - } - } - - return prefixes; -} - -function run(command, arguments_) { - const options = { - encoding: 'utf8', - timeout: 60_000, - windowsHide: true - }; - - // Azure CLI uses az.cmd on Windows, which requires the command interpreter. - // Reject shell syntax before quoting arguments, including the discovered subnet ID. - if (process.platform === 'win32' && command === 'az') { - // eslint-disable-next-line no-control-regex -- Control characters are unsafe in a command line. - if (arguments_.some(argument => /[\x00-\x1f"<>|&^%!\\]/.test(argument))) { - throw new Error('Azure CLI arguments contain unsupported Windows command-shell characters'); - } - - const commandLine = `"az.cmd ${arguments_.map(argument => `"${argument}"`).join(' ')}"`; - return execFileSync(process.env.ComSpec ?? 'cmd.exe', ['/d', '/v:off', '/s', '/c', commandLine], { - ...options, - windowsVerbatimArguments: true - }); - } - - return execFileSync(command, arguments_, options); -} - -function runJson(command, arguments_, description) { - const output = run(command, arguments_); - - try { - return JSON.parse(output); - } catch (error) { - throw new Error(`${description} returned invalid JSON`, { cause: error }); - } -} - -function writeValuesFile(outputFile, values) { - const temporaryFile = `${outputFile}.${randomUUID()}.tmp`; - - try { - writeFileSync(temporaryFile, `${JSON.stringify(values, null, 2)}\n`, { - encoding: 'utf8', - flag: 'wx', - mode: 0o600 - }); - renameSync(temporaryFile, outputFile); - } finally { - rmSync(temporaryFile, { force: true }); - } -} - -const isMainModule = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; - -if (isMainModule) { - try { - main(); - } catch (error) { - console.error(`Forwarded-header discovery failed: ${error.message}`); - process.exitCode = 1; - } -} diff --git a/tests/ci/forwarded-headers.test.js b/tests/ci/forwarded-headers.test.js deleted file mode 100644 index 110f7f428e..0000000000 --- a/tests/ci/forwarded-headers.test.js +++ /dev/null @@ -1,114 +0,0 @@ -import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import { fileURLToPath } from 'node:url'; - -const repositoryRoot = fileURLToPath(new URL('../../', import.meta.url)); -const discoveryScript = join(repositoryRoot, 'k8s/scripts/discover-forwarded-headers.js'); -const subnetId = '/subscriptions/subscription/resourceGroups/gateway-group/providers/Microsoft.Network/virtualNetworks/network/subnets/gateway'; - -function discover(directory, association = subnetId) { - const output = join(directory, 'forwarding.json'); - const azureFixture = join(directory, 'azure-fixture.mjs'); - writeFileSync(azureFixture, ` - import assert from 'node:assert/strict'; - assert.deepEqual(process.argv.slice(2), [ - 'network', 'vnet', 'subnet', 'show', '--ids', process.env.TEST_SUBNET, - '--query', '{id:id,addressPrefix:addressPrefix,addressPrefixes:addressPrefixes,delegations:delegations[].serviceName}', - '--output', 'json', '--only-show-errors' - ]); - console.log(JSON.stringify({ - id: process.env.TEST_SUBNET, - addressPrefix: '10.60.64.0/24', - delegations: ['Microsoft.ServiceNetworking/trafficControllers'] - })); - `); - // Exercise the real command interpreter and argument forwarding on Windows. - writeFileSync(join(directory, 'az.cmd'), `@"${process.execPath}" "${azureFixture}" %*\r\n`); - - const runner = join(directory, 'runner.mjs'); - writeFileSync(runner, ` - import assert from 'node:assert/strict'; - import childProcess from 'node:child_process'; - import { syncBuiltinESMExports } from 'node:module'; - import { pathToFileURL } from 'node:url'; - const execute = childProcess.execFileSync; - childProcess.execFileSync = (command, args, options) => { - if (command === 'helm') { - return ' alb.networking.azure.io/alb-name: public-alb\\n alb.networking.azure.io/alb-namespace: kube-system\\n'; - } - if (command === 'kubectl') { - assert.deepEqual(args, [ - 'get', 'applicationloadbalancer.alb.networking.azure.io', 'public-alb', - '--namespace', 'kube-system', '--output', 'json' - ]); - return JSON.stringify({ - metadata: { generation: 1 }, - spec: { associations: [process.env.TEST_SUBNET] }, - status: { conditions: ['Accepted', 'Deployment'].map(type => ({ type, status: 'True', observedGeneration: 1 })) } - }); - } - if (command === 'az') { - assert.notEqual(process.platform, 'win32', 'Windows must invoke az.cmd through its interpreter'); - return execute(process.execPath, [process.env.TEST_AZURE_FIXTURE, ...args], options); - } - return execute(command, args, options); - }; - syncBuiltinESMExports(); - process.argv = [process.execPath, process.env.TEST_DISCOVERY_SCRIPT, 'ex-test', 'values.yaml', process.env.TEST_OUTPUT]; - await import(pathToFileURL(process.env.TEST_DISCOVERY_SCRIPT)); - `); - - const result = spawnSync(process.execPath, [runner], { - cwd: directory, - encoding: 'utf8', - env: { - ...process.env, - PATH: `${directory}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH}`, - TEST_AZURE_FIXTURE: azureFixture, - TEST_DISCOVERY_SCRIPT: discoveryScript, - TEST_OUTPUT: output, - TEST_SUBNET: association - }, - timeout: 15_000, - windowsHide: true - }); - - assert.ifError(result.error); - return { ...result, output }; -} - -test('discovery uses the registered gateway resource and preserves Azure CLI arguments', () => { - const directory = mkdtempSync(join(tmpdir(), 'forwarding test ')); - try { - const result = discover(directory); - assert.equal(result.status, 0, result.stderr); - assert.deepEqual(JSON.parse(readFileSync(result.output, 'utf8')), { - forwardedHeaders: { knownNetworks: ['10.60.64.0/24'] } - }); - } finally { - rmSync(directory, { force: true, recursive: true }); - } -}); - -test('subnet data cannot expand Windows shell syntax and failed discovery removes stale output', () => { - const directory = mkdtempSync(join(tmpdir(), 'forwarding test ')); - try { - const output = join(directory, 'forwarding.json'); - writeFileSync(output, 'stale forwarding values'); - const result = discover(directory, `${subnetId}%PATH%`); - if (process.platform === 'win32') { - assert.equal(result.status, 1); - assert.match(result.stderr, /unsupported Windows command-shell characters/); - assert.equal(existsSync(output), false); - } else { - // Direct execution must pass the percent signs literally, without shell expansion. - assert.equal(result.status, 0, result.stderr); - } - } finally { - rmSync(directory, { force: true, recursive: true }); - } -});