Configure trusted forwarded header peers

.github/workflows/build.yaml

@@ -842,7 +842,7 @@ jobs:
         if: ${{ needs.version.outputs.is_prod_deploy == 'true' }}
         run: |
           sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml
-          helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml ex-prod --namespace ex-prod ./k8s/exceptionless
+          helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-prod-values.yaml --wait --timeout 5m ex-prod --namespace ex-prod ./k8s/exceptionless
 
       - name: Summary
         if: ${{ always() }}

docs/docs/self-hosting/docker.md

@@ -4,6 +4,8 @@ title: "Docker"
 
 # Docker
 
+If you place a reverse proxy in front of the container, configure its [trusted forwarding peers](/docs/self-hosting/#reverse-proxies-and-https) before upgrading. The direct-port examples below do not require proxy trust settings.
+
 If you would like to test Exceptionless locally, please follow this section.
 
 ## Requirements

docs/docs/self-hosting/index.md

@@ -12,6 +12,46 @@ You can also use Kubernetes while self-hosting Exceptionless. We'll cover both t
 * [Kubernetes](/docs/self-hosting/kubernetes)
 * [Upgrading](/docs/self-hosting/upgrading-self-hosted-instance)
 
+## Reverse proxies and HTTPS
+
+Direct HTTP/HTTPS hosting needs no forwarded-header configuration. The API accepts `X-Forwarded-For` and `X-Forwarded-Proto` only from loopback or explicitly trusted proxies. A known proxy is the **source IP of the connection reaching the API**, not its hostname, public frontend IP, Kubernetes Service IP, or proxy identifier. For Docker bridges or NAT, verify the address seen inside the API container.
+
+Before upgrading from unrestricted forwarding, configure every non-loopback proxy that must supply client addresses or HTTPS schemes. Otherwise requests use the proxy IP and backend scheme: unrelated users can share IP-based limits and HTTPS detection can be incorrect. Set the public website `EX_BaseURL` and API `EX_ApiUrl` separately; forwarded headers do not configure public origins.
+
+Use this section in the web application's `appsettings.Local.yml`, replacing the example address with the verified proxy source IP:
+
+```yaml
+ForwardedHeaders:
+  Enabled: true
+  ForwardLimit: 1
+  KnownProxies:
+    - 192.0.2.10
+```
+
+The equivalent Docker Compose `environment` entries, or Helm values under the chart's existing `config` map, are:
+
+```yaml
+EX_ForwardedHeaders__Enabled: "true"
+EX_ForwardedHeaders__ForwardLimit: "1"
+EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10"
+```
+
+For changing proxy addresses, use `KnownNetworks` **instead of** `KnownProxies` with a verified, dedicated proxy CIDR, for example `EX_ForwardedHeaders__KnownNetworks__0: "192.0.2.0/24"`. Do not trust the whole cluster, Docker bridge, or private address space when other workloads can connect from it. Use network access controls to restrict backend access. IPv4 and IPv6 addresses/CIDRs are supported; IPv4 entries also match IPv4-mapped IPv6 peers. Use canonical network CIDRs; hostnames, ports, comma-separated scalar lists, malformed entries, and `/0` networks are rejected at startup.
+
+All supported Exceptionless settings belong under `ForwardedHeaders`. `Enabled` defaults to `true`; set `ForwardedHeaders:Enabled` to `false` (environment variable `EX_ForwardedHeaders__Enabled=false`) to leave client addresses and schemes unchanged. Exceptionless reads and validates this section once at startup into `TrustedForwardedHeadersOptions`, then configures ASP.NET Core's `ForwardedHeadersOptions` from those typed values through [.NET's options registration](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/options?view=aspnetcore-10.0). ASP.NET Core does not automatically bind this section. The group and `Enabled` switch are Exceptionless settings.
+
+`ForwardLimit` and `KnownProxies` match [ASP.NET Core's forwarded-header options](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0#forwarded-headers-middleware-options). The `KnownNetworks` configuration key retains the older framework name; Exceptionless parses its CIDR strings into .NET 10's [`KnownIPNetworks` property](https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.builder.forwardedheadersoptions.knownipnetworks?view=aspnetcore-10.0). Both address lists are parsed and validated explicitly while preserving the framework's loopback defaults. Boolean settings use .NET's typed `GetValue` conversion; malformed booleans fail startup.
+
+`ForwardLimit` defaults to one entry, processed from right to left. Multiple proxies require a finite limit matching the intended chain and trust for each traversed peer. Processing stops at an unknown peer even with unused hop budget. Zero also disables processing for compatibility; negative or non-integer limits are invalid.
+
+Do not set the separate framework variable `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true`. [Microsoft documents that this shortcut clears the trusted-peer lists](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/8.0/forwarded-headers-unknown-proxies). ASP.NET Core exposes it internally as [`ForwardedHeaders_Enabled`](https://github.com/dotnet/aspnetcore/blob/release/10.0/src/DefaultBuilder/src/ForwardedHeadersOptionsSetup.cs#L21); Exceptionless rejects the active shortcut at startup. This framework key is not part of the Exceptionless configuration group and cannot replace `ForwardedHeaders:Enabled`.
+
+The public edge proxy must derive the scheme from its connection and replace untrusted forwarding headers, or append the observed client address so the configured hop limit cannot reach user-supplied entries. Configure Nginx, Caddy, Traefik, or other proxies accordingly. Loopback proxies and IIS out-of-process integration normally need no additional trust entries. A remote IIS/ARR proxy needs the same explicit peer configuration as other remote proxies.
+
+Environment variables override YAML by key; command-line values override both. Use double underscores for nesting and numbered entries for lists. Overriding index `0` does not remove other entries inherited from YAML; remove obsolete trust entries from their source and restart after changes. Avoid defining the same key with both prefixed and unprefixed environment names.
+
+The API processes IP/scheme headers before redirect, authentication, and throttling middleware. It does not consume `X-Forwarded-Host` or `X-Forwarded-Prefix`; preserve the public Host and root paths. Verify HTTPS, client IPs, redirects, and IP-based limits through the actual proxy chain before rollout. Enable `EX_Serilog__MinimumLevel__Override__Microsoft.AspNetCore.HttpOverrides: "Debug"` temporarily to diagnose unknown peers. See [Microsoft's proxy guidance](https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/proxy-load-balancer?view=aspnetcore-10.0).
+
 ---
 
 [Next > Docker](/docs/self-hosting/docker)

docs/docs/self-hosting/kubernetes.md

@@ -25,7 +25,74 @@ The following section will cover how to configure exceptionless inside of a Kube
 
 All configuration options and settings can be found in the various option classes located [here](https://github.com/exceptionless/Exceptionless/tree/master/src/Exceptionless.Core/Configuration).
 
-_Please note that if you are specifying configuration via `docker-compose`, then you will need to drop the `EX_` and `EX_ConnectionStrings__` prefixes._
+The same `EX_` environment keys work in Docker Compose. Nested keys use double underscores.
+
+### Trusted ingress peers
+
+Before deploying behind ingress, configure [trusted forwarding peers](/docs/self-hosting/#reverse-proxies-and-https) through the Helm chart's existing `config` map. For example, after replacing the documentation address with a verified proxy source address:
+
+```yaml
+config:
+  EX_ForwardedHeaders__KnownProxies__0: "192.0.2.10"
+  EX_ForwardedHeaders__Enabled: "true"
+  EX_ForwardedHeaders__ForwardLimit: "1"
+```
+
+For a dedicated ingress subnet with a single proxy hop, configure the chart's `forwardedHeaders.knownNetworks` list instead. Replace this example with your verified proxy subnet:
+
+```yaml
+forwardedHeaders:
+  knownNetworks:
+    - 192.0.2.0/24
+```
+
+The chart converts this list into indexed `EX_ForwardedHeaders__KnownNetworks__*` keys, enables forwarding, and sets a forward limit of one for both application hosts. Replacing the list removes obsolete subnet entries even with `--reuse-values`. Combining a nonempty list with manual `ForwardedHeaders` entries in `config` is rejected; remove conflicting entries from saved release values. The reusable chart's list is empty by default. For multiple proxy hops or other forwarding options, use explicit settings in `config` instead.
+
+The hosted development and production values files explicitly trust `10.60.64.0/24`, the dedicated `ex-net-v6/ex-agc` association subnet for `kube-system/public-alb`. Update both files with any gateway subnet change. Deployments use these reviewed values without querying Azure for proxy trust. Self-hosted installations must configure their own proxy addresses.
+
+The repository's Azure Application Gateway for Containers ingress terminates TLS and routes directly to application Services on port 80 and pods on port 8080. Verify the API-visible peer and any intermediate proxies before the first rollout and whenever the network changes. The ALB controller pod and Service address are not substitutes for the gateway's [association subnet](https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/application-gateway-for-containers-components).
+
+### Rollout checks and configuration failures
+
+Before the first hosted rollout, verify the configured subnet against the gateway's association subnet and actual proxy path, and remove conflicting saved forwarding overrides. Repeat these checks when the network configuration changes. The separate framework `ForwardedHeaders_Enabled` shortcut must be absent or false.
+
+| Failure | Result and recovery |
+| --- | --- |
+| Invalid application settings, such as a malformed IP, boolean, or `/0` network | New application processes fail at startup. Correct the configuration and redeploy. Production Helm waits up to five minutes for readiness and reports failure; it does not automatically roll back. |
+| A valid CIDR does not contain the actual proxy, or forwarding is disabled | The app can remain ready while it ignores forwarded headers. Users can share the proxy's IP limits, and the request scheme reflects the backend connection. Readiness checks do not detect this. |
+| The trusted range includes other callers | Those callers can supply forged client IPs or schemes. Use only the dedicated gateway subnet or verified proxy IPs. |
+
+After the first authorized development rollout, verify client IPs, HTTPS detection, and rejection of spoofed forwarding headers before a production release. The existing version and readiness checks do not prove those behaviors.
+
+Use the existing global-admin-only `GET /api/v2/admin/echo` endpoint for these checks. Its `ip_address` and `scheme` fields report the values the application sees after forwarded-header processing. Do not share the full echo response: its `headers` field can contain credentials and cookies.
+
+| Step | Check | Pass condition |
+| --- | --- | --- |
+| Before merge | Required CI and configured gateway trust | CI passes; environment values match the verified dedicated gateway subnet and proxy path. Merging main starts the dev deployment. |
+| Configuration | Saved Helm values, rendered ConfigMap, and new app/API pod environment | `EX_ForwardedHeaders__Enabled=true`, `EX_ForwardedHeaders__ForwardLimit=1`, and indexed `EX_ForwardedHeaders__KnownNetworks__*` match environment values. No conflicting forwarding overrides remain. Inspect only these keys, not unrelated secrets. |
+| Rollout | Pod readiness, startup logs, dev migrations, and `/api/v2/about` | New app/API pods are ready, no configuration errors occur, migrations finish, and the expected version is served. |
+| Baseline request | Call admin echo through HTTPS ingress with no forwarding or `X-Original-*` headers supplied by the client | `ip_address` equals the caller's known public egress IP and `scheme` is `https`. The baseline `X-Original-For` identifies a gateway peer within the configured subnet. Repeat on both app and API ingress. |
+| Spoofed request | Repeat through the same ingress with `X-Forwarded-For: 203.0.113.123` and `X-Forwarded-Proto: http` | `ip_address` stays equal to the caller's actual egress IP and `scheme` stays `https`. Fail if either supplied value controls the result. |
+| Direct untrusted request | In an authorized dev check, request the backend directly from a source outside trusted networks with the same fake headers | Forwarded values are ignored; `ip_address` is the direct peer and `scheme` is the backend transport scheme. Loopback or port-forwarding cannot prove this because loopback is trusted. |
+| Production release | After dev passes, use the normal release-tag workflow and repeat baseline/spoof checks | Production serves the expected version with correct IP and scheme on both hosts. Stop promotion on any failed check; readiness alone is insufficient. |
+
+For each request, retain only the observed IP, scheme, host, and pass/fail result. A global administrator can make the requests with their normal authenticated client. An unauthenticated request must still return `401`, and a non-admin request must return `403`.
+
+For an authorized rollout, the following commands prompt for the administrator's password and print only the diagnostic fields. Set `BASE_URL` to the HTTPS ingress being checked and `ADMIN_EMAIL` to the administrator's email. Run the baseline first, then the spoof check; both must report the same real client IP and `scheme: "https"`.
+
+```sh
+# Baseline: do not supply forwarding or X-Original-* headers.
+curl --fail --silent --show-error --user "$ADMIN_EMAIL" \
+  "$BASE_URL/api/v2/admin/echo" \
+  | jq '{ip_address, scheme, gateway_peer: .headers["X-Original-For"]}'
+
+# Spoof check: these supplied values must not become the effective IP or scheme.
+curl --fail --silent --show-error --user "$ADMIN_EMAIL" \
+  --header 'X-Forwarded-For: 203.0.113.123' \
+  --header 'X-Forwarded-Proto: http' \
+  "$BASE_URL/api/v2/admin/echo" \
+  | jq '{ip_address, scheme}'
+```
 
 ## ConnectionStrings
 

docs/docs/self-hosting/upgrading-self-hosted-instance.md

@@ -4,6 +4,10 @@ title: "Upgrading"
 
 # Upgrading
 
+## Trusted proxy migration
+
+Upgrading from unrestricted forwarding requires [explicit trust for non-loopback reverse proxies](/docs/self-hosting/#reverse-proxies-and-https). Configure and verify the actual ingress peers before rollout, or client-IP limits and HTTPS detection can change. Direct hosting needs no new settings.
+
 **Please ensure that you have created backups before upgrading!**
 
 **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.**

k8s/ex-dev-values.yaml

@@ -1,5 +1,9 @@
 appMode: Staging
 appScope: dev
+# Dedicated public-alb association subnet (ex-net-v6/ex-agc); update with gateway network changes.
+forwardedHeaders:
+  knownNetworks:
+    - 10.60.64.0/24
 app:
   image:
     repository: exceptionless/app-ci

k8s/ex-prod-values.yaml

@@ -1,5 +1,9 @@
 appMode: Production
 appScope: prod
+# Dedicated public-alb association subnet (ex-net-v6/ex-agc); update with gateway network changes.
+forwardedHeaders:
+  knownNetworks:
+    - 10.60.64.0/24
 app:
   minReplicaCount: 2
   maxReplicaCount: 5

k8s/exceptionless/templates/config.yaml

@@ -17,6 +17,19 @@ data:
     {{- end }}
     EX_DisabledPlugins: GeoPlugin,LocationPlugin
     EX_EnableSsl: "true"
+    {{- $knownNetworks := dig "forwardedHeaders" "knownNetworks" (list) .Values.AsMap }}
+    {{- if $knownNetworks }}
+    {{- range $key, $value := .Values.config }}
+    {{- if hasPrefix "forwardedheaders:" (lower (replace "__" ":" (trimPrefix "EX_" $key))) }}
+    {{- fail "Do not combine forwardedHeaders.knownNetworks with ForwardedHeaders config overrides; remove the old overrides" }}
+    {{- end }}
+    {{- end }}
+    EX_ForwardedHeaders__Enabled: "true"
+    EX_ForwardedHeaders__ForwardLimit: "1"
+    {{- range $index, $network := $knownNetworks }}
+    EX_ForwardedHeaders__KnownNetworks__{{ $index }}: {{ $network | quote }}
+    {{- end }}
+    {{- end }}
     EX_ConnectionStrings__Elasticsearch: {{ default "server=http://elasticsearch:9200" .Values.elasticsearch.connectionString }}
     {{- if (empty .Values.redis.connectionString) }}
     EX_ConnectionStrings__Redis: {{ template "exceptionless.fullname" . }}-redis-master,abortConnect=false

k8s/exceptionless/values.yaml

@@ -22,6 +22,11 @@ jobs:
 config: {}
 #  key: value
 
+# Verified, dedicated ingress subnets; configure them in environment values.
+# A replacement list prevents stale subnet entries surviving --reuse-values.
+forwardedHeaders:
+  knownNetworks: []
+
 ingress:
   enabled: false
   clusterIssuer: cert-manager-webhook-dnsimple-production

src/Exceptionless.Web/Api/Handlers/AdminHandler.cs

@@ -248,7 +248,8 @@ public Task<Result<object>> Handle(GetAdminEcho message)
         return Task.FromResult<Result<object>>(new
         {
             httpContext.Request.Headers,
-            IpAddress = httpContext.Request.GetClientIpAddress()
+            IpAddress = httpContext.Request.GetClientIpAddress(),
+            httpContext.Request.Scheme
         });
     }
 

src/Exceptionless.Web/Extensions/HttpExtensions.cs

@@ -138,7 +138,9 @@ public static ICollection<string> GetAssociatedOrganizationIds(this HttpRequest
     {
         ArgumentNullException.ThrowIfNull(request);
 
-        return request.HttpContext.Connection.RemoteIpAddress?.ToString();
+        var address = request.HttpContext.Connection.RemoteIpAddress;
+
+        return address is { IsIPv4MappedToIPv6: true } ? address.MapToIPv4().ToString() : address?.ToString();
     }
 
     public static string? GetClientKeyHash(this HttpRequest request)

src/Exceptionless.Web/Program.cs

@@ -123,6 +123,7 @@ public static async Task<int> Main(string[] args)
             builder.Services.AddSingleton(configuration);
             builder.Services.AddSingleton(apmConfig);
             builder.Services.AddAppOptions(options);
+            builder.Services.AddSingleton(TrustedForwardedHeadersOptions.ReadFromConfiguration(configuration));
             builder.Services.AddHttpContextAccessor();
             builder.Services.AddCsp(nonceByteAmount: 32);
 
@@ -134,13 +135,8 @@ public static async Task<int> Main(string[] args)
                 .SetPreflightMaxAge(TimeSpan.FromMinutes(5))
                 .WithExposedHeaders("ETag", Headers.LegacyConfigurationVersion, Headers.ConfigurationVersion, HeaderNames.Link, Headers.RateLimit, Headers.RateLimitRemaining, Headers.ResultCount)));
 
-            builder.Services.Configure<ForwardedHeadersOptions>(o =>
-            {
-                o.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
-                o.RequireHeaderSymmetry = false;
-                o.KnownIPNetworks.Clear();
-                o.KnownProxies.Clear();
-            });
+            builder.Services.AddOptions<ForwardedHeadersOptions>()
+                .Configure<TrustedForwardedHeadersOptions>(ForwardedHeadersConfiguration.Configure);
 
             builder.Services.ConfigureHttpJsonOptions(o =>
             {
@@ -233,6 +229,7 @@ ApplicationException applicationException when applicationException.Message.Cont
                 }
             });
             app.UseStatusCodePages(WriteProblemDetailsStatusCodeResponseAsync);
+            app.UseForwardedHeaders();
 
             app.UseHealthChecks("/health", new HealthCheckOptions
             {
@@ -303,7 +300,6 @@ ApplicationException applicationException when applicationException.Message.Cont
             app.UseMiddleware<McpOriginValidationMiddleware>();
             app.UseCors("AllowAny");
             app.UseHttpMethodOverride();
-            app.UseForwardedHeaders();
 
             app.UseAuthentication();
             app.UseAuthorization();