diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 751ff94325..64c0941bc9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -842,7 +842,7 @@ jobs: if: ${{ needs.version.outputs.is_prod_deploy == 'true' }} run: | sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml - helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml ex-prod --namespace ex-prod ./k8s/exceptionless + helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --wait --timeout 5m ex-prod --namespace ex-prod ./k8s/exceptionless - name: Summary if: ${{ always() }} diff --git a/docs/docs/self-hosting/docker.md b/docs/docs/self-hosting/docker.md index c701d41e45..2e277d27da 100644 --- a/docs/docs/self-hosting/docker.md +++ b/docs/docs/self-hosting/docker.md @@ -4,6 +4,8 @@ title: "Docker" # Docker +If you place a reverse proxy in front of the container, configure its [trusted forwarding peers](/docs/self-hosting/#reverse-proxies-and-https) before upgrading. The direct-port examples below do not require proxy trust settings. + If you would like to test Exceptionless locally, please follow this section. ## Requirements diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index 829ae550b0..e5c4a801b7 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -12,6 +12,46 @@ You can also use Kubernetes while self-hosting Exceptionless. We'll cover both t * [Kubernetes](/docs/self-hosting/kubernetes) * [Upgrading](/docs/self-hosting/upgrading-self-hosted-instance) +## Reverse proxies and HTTPS + +Direct HTTP/HTTPS hosting needs no forwarded-header configuration. The API accepts `X-Forwarded-For` and `X-Forwarded-Proto` only from loopback or explicitly trusted proxies. A known proxy is the **source IP of the connection reaching the API**, not its hostname, public frontend IP, Kubernetes Service IP, or proxy identifier. For Docker bridges or NAT, verify the address seen inside the API container. + +Before upgrading from unrestricted forwarding, configure every non-loopback proxy that must supply client addresses or HTTPS schemes. Otherwise requests use the proxy IP and backend scheme: unrelated users can share IP-based limits and HTTPS detection can be incorrect. Set the public website `EX_BaseURL` and API `EX_ApiUrl` separately; forwarded headers do not configure public origins. + +Use this section in the web application's `appsettings.Local.yml`, replacing the example address with the verified proxy source IP: + +```yaml +ForwardedHeaders: + Enabled: true + ForwardLimit: 1 + KnownProxies: + - 192.0.2.10 +``` + +The equivalent Docker Compose `environment` entries, or Helm values under the chart's existing `config` map, are: + +```yaml +EX_ForwardedHeaders__Enabled: "true" +EX_ForwardedHeaders__ForwardLimit: "1" +EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" +``` + +For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup. + +All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. Exceptionless reads and validates this section once at startup into `TrustedForwardedHeadersOptions`, then configures ASP.NET Core's `ForwardedHeadersOptions` from those typed values through [.NET's options registration](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/options?view=aspnetcore-10.0). ASP.NET Core does not automatically bind this section. The group and `Enabled` switch are Exceptionless settings. + +`ForwardLimit` and `KnownProxies` match [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). The `KnownNetworks` configuration key retains the older framework name; Exceptionless parses its CIDR strings into .NET 10's [`KnownIPNetworks` property](https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.builder.forwardedheadersoptions.knownipnetworks?view=aspnetcore-10.0). Both address lists are parsed and validated explicitly while preserving the framework's loopback defaults. Boolean settings use .NET's typed `GetValue` conversion; malformed booleans fail startup. + +`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero also disables processing for compatibility; negative or non-integer limits are invalid. + +Do not set the separate framework variable `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true`. [Microsoft documents that this shortcut clears the trusted-peer lists](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies). ASP.NET Core exposes it internally as [`ForwardedHeaders_Enabled`](https://github.com/dotnet/aspnetcore/blob/release/10.0/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs#L21); Exceptionless rejects the active shortcut at startup. This framework key is not part of the Exceptionless configuration group and cannot replace `ForwardedHeaders:Enabled`. + +The public edge proxy must derive the scheme from its connection and replace untrusted forwarding headers, or append the observed client address so the configured hop limit cannot reach user-supplied entries. Configure Nginx, Caddy, Traefik, or other proxies accordingly. Loopback proxies and IIS out-of-process integration normally need no additional trust entries. A remote IIS/ARR proxy needs the same explicit peer configuration as other remote proxies. + +Environment variables override YAML by key; command-line values override both. Use double underscores for nesting and numbered entries for lists. Overriding index `0` does not remove other entries inherited from YAML; remove obsolete trust entries from their source and restart after changes. Avoid defining the same key with both prefixed and unprefixed environment names. + +The API processes IP/scheme headers before redirect, authentication, and throttling middleware. It does not consume `X-Forwarded-Host` or `X-Forwarded-Prefix`; preserve the public Host and root paths. Verify HTTPS, client IPs, redirects, and IP-based limits through the actual proxy chain before rollout. Enable `EX_Serilog__MinimumLevel__Override__Microsoft.AspNetCore.HttpOverrides: "Debug"` temporarily to diagnose unknown peers. See [Microsoft's proxy guidance](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0). + --- [Next > Docker](/docs/self-hosting/docker) diff --git a/docs/docs/self-hosting/kubernetes.md b/docs/docs/self-hosting/kubernetes.md index 667ab9a4de..8c551e28e7 100644 --- a/docs/docs/self-hosting/kubernetes.md +++ b/docs/docs/self-hosting/kubernetes.md @@ -25,7 +25,74 @@ The following section will cover how to configure exceptionless inside of a Kube All configuration options and settings can be found in the various option classes located [here](https://github.com/exceptionless/Exceptionless/tree/master/src/Exceptionless.Core/Configuration). -_Please note that if you are specifying configuration via `docker-compose`, then you will need to drop the `EX_` and `EX_ConnectionStrings__` prefixes._ +The same `EX_` environment keys work in Docker Compose. Nested keys use double underscores. + +### Trusted ingress peers + +Before deploying behind ingress, configure [trusted forwarding peers](/docs/self-hosting/#reverse-proxies-and-https) through the Helm chart's existing `config` map. For example, after replacing the documentation address with a verified proxy source address: + +```yaml +config: + EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10" + EX_ForwardedHeaders__Enabled: "true" + EX_ForwardedHeaders__ForwardLimit: "1" +``` + +For a dedicated ingress subnet with a single proxy hop, configure the chart's `forwardedHeaders.knownNetworks` list instead. Replace this example with your verified proxy subnet: + +```yaml +forwardedHeaders: + knownNetworks: + - 192.0.2.0/24 +``` + +The chart converts this list into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys, enables forwarding, and sets a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining a nonempty list with manual `ForwardedHeaders` entries in `config` is rejected; remove conflicting entries from saved release values. The reusable chart's list is empty by default. For multiple proxy hops or other forwarding options, use explicit settings in `config` instead. + +The hosted development and production values files explicitly trust `10.60.64.0/24`, the dedicated `ex-net-v6/ex-agc` association subnet for `kube-system/public-alb`. Update both files with any gateway subnet change. Deployments use these reviewed values without querying Azure for proxy trust. Self-hosted installations must configure their own proxy addresses. + +The repository's Azure Application Gateway for Containers ingress terminates TLS and routes directly to application Services on port 80 and pods on port 8080. Verify the API-visible peer and any intermediate proxies before the first rollout and whenever the network changes. The ALB controller pod and Service address are not substitutes for the gateway's [association subnet](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/application-gateway-for-containers-components). + +### Rollout checks and configuration failures + +Before the first hosted rollout, verify the configured subnet against the gateway's association subnet and actual proxy path, and remove conflicting saved forwarding overrides. Repeat these checks when the network configuration changes. The separate framework `ForwardedHeaders_Enabled` shortcut must be absent or false. + +| Failure | Result and recovery | +| --- | --- | +| Invalid application settings, such as a malformed IP, boolean, or `/0` network | New application processes fail at startup. Correct the configuration and redeploy. Production Helm waits up to five minutes for readiness and reports failure; it does not automatically roll back. | +| A valid CIDR does not contain the actual proxy, or forwarding is disabled | The app can remain ready while it ignores forwarded headers. Users can share the proxy's IP limits, and the request scheme reflects the backend connection. Readiness checks do not detect this. | +| The trusted range includes other callers | Those callers can supply forged client IPs or schemes. Use only the dedicated gateway subnet or verified proxy IPs. | + +After the first authorized development rollout, verify client IPs, HTTPS detection, and rejection of spoofed forwarding headers before a production release. The existing version and readiness checks do not prove those behaviors. + +Use the existing global-admin-only `GET /api/v2/admin/echo` endpoint for these checks. Its `ip_address` and `scheme` fields report the values the application sees after forwarded-header processing. Do not share the full echo response: its `headers` field can contain credentials and cookies. + +| Step | Check | Pass condition | +| --- | --- | --- | +| Before merge | Required CI and configured gateway trust | CI passes; environment values match the verified dedicated gateway subnet and proxy path. Merging main starts the dev deployment. | +| Configuration | Saved Helm values, rendered ConfigMap, and new app/API pod environment | `EX_ForwardedHeaders__Enabled=true`, `EX_ForwardedHeaders__ForwardLimit=1`, and indexed `EX_ForwardedHeaders__KnownNetworks__*` match environment values. No conflicting forwarding overrides remain. Inspect only these keys, not unrelated secrets. | +| Rollout | Pod readiness, startup logs, dev migrations, and `/api/v2/about` | New app/API pods are ready, no configuration errors occur, migrations finish, and the expected version is served. | +| Baseline request | Call admin echo through HTTPS ingress with no forwarding or `X-Original-*` headers supplied by the client | `ip_address` equals the caller's known public egress IP and `scheme` is `https`. The baseline `X-Original-For` identifies a gateway peer within the configured subnet. Repeat on both app and API ingress. | +| Spoofed request | Repeat through the same ingress with `X-Forwarded-For: 203.0.113.123` and `X-Forwarded-Proto: http` | `ip_address` stays equal to the caller's actual egress IP and `scheme` stays `https`. Fail if either supplied value controls the result. | +| Direct untrusted request | In an authorized dev check, request the backend directly from a source outside trusted networks with the same fake headers | Forwarded values are ignored; `ip_address` is the direct peer and `scheme` is the backend transport scheme. Loopback or port-forwarding cannot prove this because loopback is trusted. | +| Production release | After dev passes, use the normal release-tag workflow and repeat baseline/spoof checks | Production serves the expected version with correct IP and scheme on both hosts. Stop promotion on any failed check; readiness alone is insufficient. | + +For each request, retain only the observed IP, scheme, host, and pass/fail result. A global administrator can make the requests with their normal authenticated client. An unauthenticated request must still return `401`, and a non-admin request must return `403`. + +For an authorized rollout, the following commands prompt for the administrator's password and print only the diagnostic fields. Set `BASE_URL` to the HTTPS ingress being checked and `ADMIN_EMAIL` to the administrator's email. Run the baseline first, then the spoof check; both must report the same real client IP and `scheme: "https"`. + +```sh +# Baseline: do not supply forwarding or X-Original-* headers. +curl --fail --silent --show-error --user "$ADMIN_EMAIL" \ + "$BASE_URL/api/v2/admin/echo" \ + | jq '{ip_address, scheme, gateway_peer: .headers["X-Original-For"]}' + +# Spoof check: these supplied values must not become the effective IP or scheme. +curl --fail --silent --show-error --user "$ADMIN_EMAIL" \ + --header 'X-Forwarded-For: 203.0.113.123' \ + --header 'X-Forwarded-Proto: http' \ + "$BASE_URL/api/v2/admin/echo" \ + | jq '{ip_address, scheme}' +``` ## ConnectionStrings diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index 20068a2efc..efa7d8eaaa 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -4,6 +4,10 @@ title: "Upgrading" # Upgrading +## Trusted proxy migration + +Upgrading from unrestricted forwarding requires [explicit trust for non-loopback reverse proxies](/docs/self-hosting/#reverse-proxies-and-https). Configure and verify the actual ingress peers before rollout, or client-IP limits and HTTPS detection can change. Direct hosting needs no new settings. + **Please ensure that you have created backups before upgrading!** **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.** diff --git a/k8s/ex-dev-values.yaml b/k8s/ex-dev-values.yaml index 0641e5e2ff..341471ab7d 100644 --- a/k8s/ex-dev-values.yaml +++ b/k8s/ex-dev-values.yaml @@ -1,5 +1,9 @@ appMode: Staging appScope: dev +# Dedicated public-alb association subnet (ex-net-v6/ex-agc); update with gateway network changes. +forwardedHeaders: + knownNetworks: + - 10.60.64.0/24 app: image: repository: exceptionless/app-ci diff --git a/k8s/ex-prod-values.yaml b/k8s/ex-prod-values.yaml index 363760edeb..c46a23cdc3 100644 --- a/k8s/ex-prod-values.yaml +++ b/k8s/ex-prod-values.yaml @@ -1,5 +1,9 @@ appMode: Production appScope: prod +# Dedicated public-alb association subnet (ex-net-v6/ex-agc); update with gateway network changes. +forwardedHeaders: + knownNetworks: + - 10.60.64.0/24 app: minReplicaCount: 2 maxReplicaCount: 5 diff --git a/k8s/exceptionless/templates/config.yaml b/k8s/exceptionless/templates/config.yaml index 6c53f373a8..2823ac70bc 100644 --- a/k8s/exceptionless/templates/config.yaml +++ b/k8s/exceptionless/templates/config.yaml @@ -17,6 +17,19 @@ data: {{- end }} EX_DisabledPlugins: GeoPlugin,LocationPlugin EX_EnableSsl: "true" + {{- $knownNetworks := dig "forwardedHeaders" "knownNetworks" (list) .Values.AsMap }} + {{- if $knownNetworks }} + {{- range $key, $value := .Values.config }} + {{- if hasPrefix "forwardedheaders:" (lower (replace "__" ":" (trimPrefix "EX_" $key))) }} + {{- fail "Do not combine forwardedHeaders.knownNetworks with ForwardedHeaders config overrides; remove the old overrides" }} + {{- end }} + {{- end }} + EX_ForwardedHeaders__Enabled: "true" + EX_ForwardedHeaders__ForwardLimit: "1" + {{- range $index, $network := $knownNetworks }} + EX_ForwardedHeaders__KnownNetworks__{{ $index }}: {{ $network | quote }} + {{- end }} + {{- end }} EX_ConnectionStrings__Elasticsearch: {{ default "server=http://elasticsearch:9200" .Values.elasticsearch.connectionString }} {{- if (empty .Values.redis.connectionString) }} EX_ConnectionStrings__Redis: {{ template "exceptionless.fullname" . }}-redis-master,abortConnect=false diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index b2439e84b2..dc37340412 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -22,6 +22,11 @@ jobs: config: {} # key: value +# Verified, dedicated ingress subnets; configure them in environment values. +# A replacement list prevents stale subnet entries surviving --reuse-values. +forwardedHeaders: + knownNetworks: [] + ingress: enabled: false clusterIssuer: cert-manager-webhook-dnsimple-production diff --git a/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs b/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs index 9bf11786b4..c758517674 100644 --- a/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/AdminHandler.cs @@ -248,7 +248,8 @@ public Task> Handle(GetAdminEcho message) return Task.FromResult>(new { httpContext.Request.Headers, - IpAddress = httpContext.Request.GetClientIpAddress() + IpAddress = httpContext.Request.GetClientIpAddress(), + httpContext.Request.Scheme }); } diff --git a/src/Exceptionless.Web/Extensions/HttpExtensions.cs b/src/Exceptionless.Web/Extensions/HttpExtensions.cs index 80b4fb3fd5..832573624d 100644 --- a/src/Exceptionless.Web/Extensions/HttpExtensions.cs +++ b/src/Exceptionless.Web/Extensions/HttpExtensions.cs @@ -138,7 +138,9 @@ public static ICollection GetAssociatedOrganizationIds(this HttpRequest { ArgumentNullException.ThrowIfNull(request); - return request.HttpContext.Connection.RemoteIpAddress?.ToString(); + var address = request.HttpContext.Connection.RemoteIpAddress; + + return address is { IsIPv4MappedToIPv6: true } ? address.MapToIPv4().ToString() : address?.ToString(); } public static string? GetClientKeyHash(this HttpRequest request) diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index 64bcb26728..77663e730d 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -123,6 +123,7 @@ public static async Task Main(string[] args) builder.Services.AddSingleton(configuration); builder.Services.AddSingleton(apmConfig); builder.Services.AddAppOptions(options); + builder.Services.AddSingleton(TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration)); builder.Services.AddHttpContextAccessor(); builder.Services.AddCsp(nonceByteAmount: 32); @@ -134,13 +135,8 @@ public static async Task Main(string[] args) .SetPreflightMaxAge(TimeSpan.FromMinutes(5)) .WithExposedHeaders("ETag", Headers.LegacyConfigurationVersion, Headers.ConfigurationVersion, HeaderNames.Link, Headers.RateLimit, Headers.RateLimitRemaining, Headers.ResultCount))); - builder.Services.Configure(o => - { - o.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; - o.RequireHeaderSymmetry = false; - o.KnownIPNetworks.Clear(); - o.KnownProxies.Clear(); - }); + builder.Services.AddOptions() + .Configure(ForwardedHeadersConfiguration.Configure); builder.Services.ConfigureHttpJsonOptions(o => { @@ -233,6 +229,7 @@ ApplicationException applicationException when applicationException.Message.Cont } }); app.UseStatusCodePages(WriteProblemDetailsStatusCodeResponseAsync); + app.UseForwardedHeaders(); app.UseHealthChecks("/health", new HealthCheckOptions { @@ -303,7 +300,6 @@ ApplicationException applicationException when applicationException.Message.Cont app.UseMiddleware(); app.UseCors("AllowAny"); app.UseHttpMethodOverride(); - app.UseForwardedHeaders(); app.UseAuthentication(); app.UseAuthorization(); diff --git a/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs new file mode 100644 index 0000000000..053dbfe1a2 --- /dev/null +++ b/src/Exceptionless.Web/Security/ForwardedHeadersConfiguration.cs @@ -0,0 +1,26 @@ +using Microsoft.AspNetCore.HttpOverrides; + +namespace Exceptionless.Web.Security; + +public static class ForwardedHeadersConfiguration +{ + public static void Configure(ForwardedHeadersOptions options, TrustedForwardedHeadersOptions settings) + { + options.ForwardedHeaders = settings.Enabled + ? ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto + : ForwardedHeaders.None; + options.RequireHeaderSymmetry = false; + options.ForwardLimit = settings.ForwardLimit; + + // Retain the framework's loopback defaults and add explicitly trusted ingress peers. + foreach (var address in settings.KnownProxies) + { + options.KnownProxies.Add(address); + } + + foreach (var network in settings.KnownNetworks) + { + options.KnownIPNetworks.Add(network); + } + } +} diff --git a/src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs b/src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs new file mode 100644 index 0000000000..c1176bec70 --- /dev/null +++ b/src/Exceptionless.Web/Security/TrustedForwardedHeadersOptions.cs @@ -0,0 +1,86 @@ +using System.Net; + +namespace Exceptionless.Web.Security; + +public sealed class TrustedForwardedHeadersOptions +{ + private const string FrameworkEnabledKey = "ForwardedHeaders_Enabled"; + + public bool Enabled { get; private init; } = true; + public int ForwardLimit { get; private init; } = 1; + public IReadOnlyList KnownProxies { get; private init; } = []; + public IReadOnlyList KnownNetworks { get; private init; } = []; + + public static TrustedForwardedHeadersOptions ReadFromConfiguration(IConfiguration configuration) + { + // ASP.NET Core's separate shortcut clears proxy trust; it is not our nested Enabled setting. + // https://github.com/dotnet/aspnetcore/blob/release/10.0/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs + if (configuration.GetValue(FrameworkEnabledKey, false)) + { + throw new InvalidOperationException($"The ASP.NET Core {FrameworkEnabledKey} shortcut must not be enabled. Use the ForwardedHeaders configuration section to enable forwarding and configure trusted peers."); + } + + var section = configuration.GetSection("ForwardedHeaders"); + int forwardLimit = section.GetValue(nameof(ForwardLimit), 1); + if (forwardLimit < 0) + { + throw new InvalidOperationException($"{section.Path}:{nameof(ForwardLimit)} must be zero or positive."); + } + + return new TrustedForwardedHeadersOptions + { + Enabled = section.GetValue(nameof(Enabled), true), + ForwardLimit = forwardLimit, + KnownProxies = ReadKnownProxies(section), + KnownNetworks = ReadKnownNetworks(section) + }; + } + + private static IReadOnlyList ReadKnownProxies(IConfigurationSection section) + { + var addresses = new List(); + foreach (var entry in GetEntries(section, nameof(KnownProxies))) + { + if (!IPAddress.TryParse(entry.Value, out var address)) + { + throw new InvalidOperationException($"{entry.Path} must be an IP address without a port."); + } + + addresses.Add(address); + } + + return addresses.AsReadOnly(); + } + + private static IReadOnlyList ReadKnownNetworks(IConfigurationSection section) + { + var networks = new List(); + foreach (var entry in GetEntries(section, nameof(KnownNetworks))) + { + if (!IPNetwork.TryParse(entry.Value, out var network)) + { + throw new InvalidOperationException($"{entry.Path} must be a valid IPv4 or IPv6 CIDR."); + } + + if (network.PrefixLength is 0) + { + throw new InvalidOperationException($"{entry.Path} must identify a trusted ingress network, not a /0 network."); + } + + networks.Add(network); + } + + return networks.AsReadOnly(); + } + + private static IEnumerable GetEntries(IConfigurationSection configuration, string name) + { + var section = configuration.GetSection(name); + if (section.Value is not null) + { + throw new InvalidOperationException($"{section.Path} must be a list. Use numbered environment keys such as EX_ForwardedHeaders__{name}__0."); + } + + return section.GetChildren(); + } +} diff --git a/src/Exceptionless.Web/appsettings.yml b/src/Exceptionless.Web/appsettings.yml index d4c5606fbb..45b903e9eb 100644 --- a/src/Exceptionless.Web/appsettings.yml +++ b/src/Exceptionless.Web/appsettings.yml @@ -33,6 +33,14 @@ Serilog: # Optional public API URL for OAuth discovery and resource validation; defaults to BaseURL. ApiUrl: +# Reverse proxy trust: see docs/docs/self-hosting/index.md#reverse-proxies-and-https. +# Configure only verified API-visible peers; loopback is trusted by default. +ForwardedHeaders: + Enabled: true + ForwardLimit: 1 + KnownProxies: [] + KnownNetworks: [] + Assistant: Endpoint: https://openrouter.ai/api/v1/chat/completions Model: "~deepseek/deepseek-v4-flash-latest" diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs index 65f4c29ed4..b5fdb347b0 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs @@ -1124,12 +1124,24 @@ public Task GetSettings_WithoutAuth_ReturnsUnauthorized() } [Fact] - public Task EchoRequest_AsGlobalAdmin_ReturnsIpAddress() + public async Task EchoRequest_AsGlobalAdmin_ReturnsForwardedIpAddressAndScheme() { - return SendRequestAsync(r => r + // Arrange + const string clientIpAddress = "203.0.113.42"; + const string scheme = "https"; + + // Act + var response = await SendRequestAsAsync(r => r .AsGlobalAdminUser() .AppendPaths("admin", "echo") + .Header("X-Forwarded-For", clientIpAddress) + .Header("X-Forwarded-Proto", scheme) .StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(clientIpAddress, response.GetProperty("ip_address").GetString()); + Assert.Equal(scheme, response.GetProperty("scheme").GetString()); + Assert.Equal(JsonValueKind.Object, response.GetProperty("headers").ValueKind); } [Fact] diff --git a/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs b/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs index 352e066be8..14275b27f8 100644 --- a/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs +++ b/tests/Exceptionless.Tests/Extensions/HttpExtensionsTests.cs @@ -1,11 +1,12 @@ using System.Text; using Exceptionless.Web.Extensions; +using Foundatio.Xunit; using Microsoft.AspNetCore.Http; using Xunit; namespace Exceptionless.Tests.Extensions; -public sealed class HttpExtensionsTests +public sealed class HttpExtensionsTests(ITestOutputHelper output) : TestWithLoggingBase(output) { [Theory] [InlineData(null)] @@ -33,6 +34,24 @@ public void GetBasicAuth_InvalidHeader_ReturnsNull(string? authorization) Assert.Null(credentials); } + [Theory] + [InlineData("192.0.2.1", "192.0.2.1")] + [InlineData("::ffff:192.0.2.1", "192.0.2.1")] + [InlineData("2001:db8::1", "2001:db8::1")] + [InlineData(null, null)] + public void GetClientIpAddress_Address_NormalizesMappedIpv4(string? value, string? expected) + { + // Arrange + var request = new DefaultHttpContext().Request; + request.HttpContext.Connection.RemoteIpAddress = value is null ? null : System.Net.IPAddress.Parse(value); + + // Act + string? address = request.GetClientIpAddress(); + + // Assert + Assert.Equal(expected, address); + } + [Theory] [InlineData("Basic", "user@example.com", "password")] [InlineData("bAsIc", "user@example.com", "password")] diff --git a/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs new file mode 100644 index 0000000000..066cd038bd --- /dev/null +++ b/tests/Exceptionless.Tests/Security/ForwardedHeadersConfigurationTests.cs @@ -0,0 +1,198 @@ +using System.Collections; +using System.Net; +using Exceptionless.Core.Configuration; +using Exceptionless.Web.Extensions; +using Exceptionless.Web.Security; +using Foundatio.Xunit; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Exceptionless.Tests.Security; + +public sealed class ForwardedHeadersConfigurationTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData("192.0.2.10", "198.51.100.20", "1", null, "198.51.100.20", "http")] + [InlineData("198.51.100.20", "198.51.100.20", "1", null, "203.0.113.40", "https")] + [InlineData("198.51.100.20", "198.51.100.20", "0", null, "198.51.100.20", "http")] + [InlineData("198.51.100.20", "::ffff:198.51.100.20", "1", null, "203.0.113.40", "https")] + [InlineData("2001:db8::20", "2001:db8::20", "1", null, "203.0.113.40", "https")] + [InlineData("198.51.100.20", "198.51.100.20", "1", "false", "198.51.100.20", "http")] + public async Task Invoke_ForwardedAddress_RespectsConfiguredPeerAndLimit(string trustedPeer, string remoteAddress, string limit, string? enabled, string expectedAddress, string expectedScheme) + { + // Arrange + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ForwardedHeaders:Enabled"] = enabled, + ["ForwardedHeaders:KnownProxies:0"] = trustedPeer, + ["ForwardedHeaders:ForwardLimit"] = limit + }).Build(); + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration); + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, settings); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); + context.Request.Scheme = "http"; + context.Request.Headers["X-Forwarded-For"] = "203.0.113.40"; + context.Request.Headers["X-Forwarded-Proto"] = "https"; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act + await middleware.Invoke(context); + + // Assert + Assert.Equal(expectedAddress, context.Request.GetClientIpAddress()); + Assert.Equal(expectedScheme, context.Request.Scheme); + } + + [Theory] + [InlineData("1", "192.0.2.10", "192.0.2.11", "http")] + [InlineData("2", "192.0.2.10", "203.0.113.40", "https")] + [InlineData("3", "192.0.2.10", "203.0.113.40", "https")] + [InlineData("2", "::ffff:192.0.2.10", "203.0.113.40", "https")] + public async Task Invoke_ForwardedChain_UsesConfiguredDepthAndStopsAtUnknownPeer(string limit, string remoteAddress, string expectedAddress, string expectedScheme) + { + // Arrange + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ForwardedHeaders:KnownNetworks:0"] = "192.0.2.0/24", + ["ForwardedHeaders:ForwardLimit"] = limit + }).Build(); + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration); + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, settings); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); + context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40, 192.0.2.11"; + context.Request.Headers["X-Forwarded-Proto"] = "http, https, http"; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act + await middleware.Invoke(context); + + // Assert + Assert.Equal(expectedAddress, context.Request.GetClientIpAddress()); + Assert.Equal(expectedScheme, context.Request.Scheme); + } + + [Theory] + [InlineData("http")] + [InlineData("https")] + public async Task Invoke_DirectRequest_PreservesConnectionAddressAndScheme(string scheme) + { + // Arrange + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(new ConfigurationBuilder().Build()); + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, settings); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse("203.0.113.40"); + context.Request.Scheme = scheme; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act + await middleware.Invoke(context); + + // Assert + Assert.Equal("203.0.113.40", context.Request.GetClientIpAddress()); + Assert.Equal(scheme, context.Request.Scheme); + } + + [Theory] + [InlineData("ForwardedHeaders:ForwardLimit", "-1")] + [InlineData("ForwardedHeaders_Enabled", "true")] + public void ReadFromConfiguration_UnsafeScalarConfiguration_ThrowsWithKey(string key, string value) + { + // Arrange + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + [key] = value + }).Build(); + + // Act + var exception = Record.Exception(() => TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration)); + + // Assert + Assert.IsType(exception); + Assert.Contains(key, exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("KnownNetworks:0", "0.0.0.0/0")] + [InlineData("KnownNetworks:0", "::/0")] + [InlineData("KnownNetworks:0", "192.0.999.0/24")] + [InlineData("KnownNetworks:0", "192.0.2.0/33")] + [InlineData("KnownProxies:0", "proxy.example.com")] + [InlineData("KnownProxies:0", "192.0.2.10:8080")] + [InlineData("KnownProxies:0", "")] + [InlineData("KnownProxies", "192.0.2.10,192.0.2.11")] + [InlineData("KnownNetworks", "192.0.2.0/24")] + public void ReadFromConfiguration_InvalidPeerConfiguration_ThrowsWithKey(string key, string value) + { + // Arrange + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + [$"ForwardedHeaders:{key}"] = value + }).Build(); + + // Act + var exception = Record.Exception(() => TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration)); + + // Assert + Assert.IsType(exception); + Assert.Contains($"ForwardedHeaders:{key}", exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("127.0.0.1")] + [InlineData("::1")] + [InlineData("::ffff:127.0.0.1")] + public async Task Invoke_Defaults_TrustsLoopbackAndProcessesOneHop(string remoteAddress) + { + // Arrange + var settings = TrustedForwardedHeadersOptions.ReadFromConfiguration(new ConfigurationBuilder().Build()); + var options = new ForwardedHeadersOptions(); + ForwardedHeadersConfiguration.Configure(options, settings); + var context = new DefaultHttpContext(); + context.Connection.RemoteIpAddress = IPAddress.Parse(remoteAddress); + context.Request.Headers["X-Forwarded-For"] = "198.51.100.1, 203.0.113.40"; + context.Request.Headers["X-Forwarded-Proto"] = "http, https"; + var middleware = new ForwardedHeadersMiddleware(_ => Task.CompletedTask, NullLoggerFactory.Instance, Options.Create(options)); + + // Act + await middleware.Invoke(context); + + // Assert + Assert.Equal("203.0.113.40", context.Request.GetClientIpAddress()); + Assert.Equal("https", context.Request.Scheme); + } + + [Theory] + [InlineData("true", true)] + [InlineData("false", false)] + public void ReadFromConfiguration_PrefixedEnvironmentLists_ReadsNestedSettings(string enabled, bool expectedEnabled) + { + // Arrange + var provider = new CustomEnvironmentVariablesConfigurationProvider(); + using var configuration = new ConfigurationRoot([provider]); + provider.Load(new Hashtable + { + ["EX_ForwardedHeaders__Enabled"] = enabled, + ["EX_ForwardedHeaders__KnownProxies__0"] = "192.0.2.10", + ["EX_ForwardedHeaders__KnownNetworks__0"] = "2001:db8::/64", + ["EX_ForwardedHeaders__ForwardLimit"] = "2" + }); + + // Act + var options = TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration); + + // Assert + Assert.Contains(IPAddress.Parse("192.0.2.10"), options.KnownProxies); + Assert.Contains(System.Net.IPNetwork.Parse("2001:db8::/64"), options.KnownNetworks); + Assert.Equal(2, options.ForwardLimit); + Assert.Equal(expectedEnabled, options.Enabled); + } +}