diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml index eb6979280a..c065bfd8de 100644 --- a/k8s/exceptionless/templates/api.yaml +++ b/k8s/exceptionless/templates/api.yaml @@ -28,6 +28,11 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} spec: + # SSE connections are long-lived; give the pod enough time to drain before SIGTERM. + # The preStop sleep lets the ALB/ingress controller deregister the pod before traffic stops, + # then the remaining window allows ASP.NET Core to cancel RequestAborted tokens and clean up. + # When push is eventually enabled behind a Gateway API RoutePolicy, revisit this value. + terminationGracePeriodSeconds: 60 topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname @@ -39,6 +44,13 @@ spec: - name: {{ template "exceptionless.name" . }}-api image: "{{ .Values.api.image.repository }}:{{ .Values.version }}" imagePullPolicy: {{ .Values.api.image.pullPolicy }} + lifecycle: + preStop: + # Give the ALB ~15s to deregister this pod before SIGTERM fires. + # The total graceful window is terminationGracePeriodSeconds (60s) minus + # this sleep, leaving 45s. ASP.NET Core uses 40s and keeps 5s of safety margin. + exec: + command: ["sleep", "15"] livenessProbe: httpGet: path: /health @@ -82,7 +94,10 @@ spec: {{- include "exceptionless.otel-env" . | indent 12 }} - name: RunJobsInProcess value: 'false' - - name: EnableWebSockets + # SSE rollout prerequisite: Azure Application Gateway for Containers Ingress API + # does not support the routeTimeout=0s override required for long-lived SSE streams. + # Keep push disabled here until this route moves to Gateway API + RoutePolicy. + - name: EnablePush value: 'false' {{- if (empty .Values.storage.connectionString) }} volumeMounts: @@ -163,6 +178,8 @@ metadata: alb.networking.azure.io/alb-namespace: {{ .Values.ingress.albNamespace }} alb.networking.azure.io/alb-frontend: {{ template "exceptionless.fullname" . }}-fe cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} + # SSE is not safe to enable behind the current AGC Ingress API path. + # Migrate to Gateway API and attach a RoutePolicy with routeTimeout: 0s before enabling push. spec: ingressClassName: azure-alb-external tls: diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 9cfb19de39..086f68352c 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -112,6 +112,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.TryAddEnumerable(ServiceDescriptor.Singleton, WorkItemDuplicateDetectionQueueBehavior>()); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddStartupAction(); services.AddSingleton(s => new InMemoryMessageBus(new InMemoryMessageBusOptions @@ -242,8 +243,8 @@ public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions if (String.IsNullOrEmpty(appOptions.StorageOptions.Provider)) logger.LogWarning("Distributed storage is NOT enabled on {MachineName}", Environment.MachineName); - if (!appOptions.EnableWebSockets) - logger.LogWarning("Web Sockets is NOT enabled on {MachineName}", Environment.MachineName); + if (!appOptions.EnablePush) + logger.LogWarning("Real-time push (SSE) is NOT enabled on {MachineName}", Environment.MachineName); if (String.IsNullOrEmpty(appOptions.EmailOptions.SmtpHost)) logger.LogWarning("Emails will NOT be sent until the SmtpHost is configured on {MachineName}", Environment.MachineName); diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs index ff6217829a..634f0d1f03 100644 --- a/src/Exceptionless.Core/Configuration/AppOptions.cs +++ b/src/Exceptionless.Core/Configuration/AppOptions.cs @@ -55,7 +55,18 @@ public class AppOptions public bool EnableRepositoryNotifications { get; internal set; } - public bool EnableWebSockets { get; internal set; } + /// + /// Controls whether real-time push (SSE) is enabled. Reads from either 'EnablePush' + /// or legacy 'EnableWebSockets' config key for backward compatibility. + /// + public bool EnablePush { get; internal set; } + + [Obsolete("Use EnablePush instead. This alias remains for source compatibility during the push transport rollout.")] + public bool EnableWebSockets + { + get => EnablePush; + internal set => EnablePush = value; + } public string? Version { get; internal set; } @@ -112,7 +123,8 @@ public static AppOptions ReadFromConfiguration(IConfiguration config) options.BulkBatchSize = config.GetValue(nameof(options.BulkBatchSize), 1000); options.EnableRepositoryNotifications = config.GetValue(nameof(options.EnableRepositoryNotifications), true); - options.EnableWebSockets = config.GetValue(nameof(options.EnableWebSockets), true); + // Support both new 'EnablePush' and legacy 'EnableWebSockets' config keys + options.EnablePush = config.GetValue(nameof(options.EnablePush), config.GetValue("EnableWebSockets", true)); try { diff --git a/src/Exceptionless.Core/Services/MessageService.cs b/src/Exceptionless.Core/Services/MessageService.cs index 0fe3e988a9..81b7b581b0 100644 --- a/src/Exceptionless.Core/Services/MessageService.cs +++ b/src/Exceptionless.Core/Services/MessageService.cs @@ -6,7 +6,6 @@ using Foundatio.Repositories.Elasticsearch; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; namespace Exceptionless.Core.Services; @@ -19,9 +18,7 @@ public class MessageService : IDisposable, IStartupAction private readonly IEventRepository _eventRepository; private readonly ITokenRepository _tokenRepository; private readonly IWebHookRepository _webHookRepository; - private readonly IConnectionMapping _connectionMapping; private readonly AppOptions _options; - private readonly ILogger _logger; private readonly List _disposeActions = []; public MessageService( @@ -43,9 +40,12 @@ public MessageService( _eventRepository = eventRepository; _tokenRepository = tokenRepository; _webHookRepository = webHookRepository; - _connectionMapping = connectionMapping; _options = options; - _logger = loggerFactory.CreateLogger() ?? NullLogger.Instance; + + // Preserve the public constructor shape for existing composition roots while push + // publication no longer depends on distributed connection state or trace logging. + _ = connectionMapping; + _ = loggerFactory; } public Task RunAsync(CancellationToken shutdownToken = default) @@ -74,22 +74,13 @@ public Task RunAsync(CancellationToken shutdownToken = default) _disposeActions.Add(() => repo.BeforePublishEntityChanged.RemoveHandler(handler)); } - private async Task OnBeforePublishEntityChangedAsync(object sender, BeforePublishEntityChangedEventArgs args) + private Task OnBeforePublishEntityChangedAsync(object sender, BeforePublishEntityChangedEventArgs args) where T : class, IIdentity, new() { - int listenerCount = await GetNumberOfListeners(args.Message); - args.Cancel = listenerCount == 0; - if (args.Cancel) - _logger.LogTrace("Cancelled {EntityType} Entity Changed Message: {@Message}", typeof(T).Name, args.Message); - } - - private Task GetNumberOfListeners(EntityChanged message) - { - var entityChanged = ExtendedEntityChanged.Create(message, false); - if (String.IsNullOrEmpty(entityChanged.OrganizationId)) - return Task.FromResult(1); // Return 1 as we have no idea if people are listening. - - return _connectionMapping.GetGroupConnectionCountAsync(entityChanged.OrganizationId); + // Push routing is replica-local. Publishing must not depend on immortal distributed + // connection indexes because another replica may own the interested client. + args.Cancel = false; + return Task.CompletedTask; } public void Dispose() diff --git a/src/Exceptionless.Core/Utility/AppDiagnostics.cs b/src/Exceptionless.Core/Utility/AppDiagnostics.cs index c081925b6c..9545fe4c74 100644 --- a/src/Exceptionless.Core/Utility/AppDiagnostics.cs +++ b/src/Exceptionless.Core/Utility/AppDiagnostics.cs @@ -125,6 +125,11 @@ public GaugeInfo(Meter meter, string name) internal static readonly Counter SavedViewsSize = Meter.CreateCounter("ex.savedviews.size", description: "Size of user saved views"); internal static readonly Counter SavedViewsViewTypeSize = Meter.CreateCounter("ex.savedviews.viewtype.size", description: "Size of user saved views by view type"); + + internal static readonly Counter PushSseConnectionsOpened = Meter.CreateCounter("ex.push.connections.sse.opened", description: "SSE push connections opened"); + internal static readonly Counter PushSseConnectionsClosed = Meter.CreateCounter("ex.push.connections.sse.closed", description: "SSE push connections closed"); + internal static readonly Counter PushWebSocketConnectionsOpened = Meter.CreateCounter("ex.push.connections.websocket.opened", description: "WebSocket push connections opened"); + internal static readonly Counter PushWebSocketConnectionsClosed = Meter.CreateCounter("ex.push.connections.websocket.closed", description: "WebSocket push connections closed"); } public static class MetricsClientExtensions diff --git a/src/Exceptionless.Core/Utility/IConnectionLeaseStore.cs b/src/Exceptionless.Core/Utility/IConnectionLeaseStore.cs new file mode 100644 index 0000000000..c274b3ffa0 --- /dev/null +++ b/src/Exceptionless.Core/Utility/IConnectionLeaseStore.cs @@ -0,0 +1,77 @@ +namespace Exceptionless.Core.Utility; + +public interface IConnectionLeaseStore +{ + Task TryAcquireAsync(string userId, string connectionId, int maxConnections, TimeSpan leaseDuration); + Task RenewAsync(string userId, string connectionId, TimeSpan leaseDuration); + Task ReleaseAsync(string userId, string connectionId); +} + +public sealed class ConnectionLeaseStore(TimeProvider timeProvider) : IConnectionLeaseStore +{ + private readonly Dictionary> _leases = []; + private readonly object _lock = new(); + + public Task TryAcquireAsync(string userId, string connectionId, int maxConnections, TimeSpan leaseDuration) + { + if (maxConnections <= 0) + return Task.FromResult(false); + + lock (_lock) + { + var now = timeProvider.GetUtcNow(); + if (!_leases.TryGetValue(userId, out var userLeases)) + { + userLeases = []; + _leases[userId] = userLeases; + } + + RemoveExpired(userLeases, now); + if (!userLeases.ContainsKey(connectionId) && userLeases.Count >= maxConnections) + return Task.FromResult(false); + + userLeases[connectionId] = now + leaseDuration; + return Task.FromResult(true); + } + } + + public Task RenewAsync(string userId, string connectionId, TimeSpan leaseDuration) + { + lock (_lock) + { + var now = timeProvider.GetUtcNow(); + if (!_leases.TryGetValue(userId, out var userLeases)) + return Task.FromResult(false); + + RemoveExpired(userLeases, now); + if (!userLeases.ContainsKey(connectionId)) + return Task.FromResult(false); + + userLeases[connectionId] = now + leaseDuration; + return Task.FromResult(true); + } + } + + public Task ReleaseAsync(string userId, string connectionId) + { + lock (_lock) + { + if (_leases.TryGetValue(userId, out var userLeases)) + { + userLeases.Remove(connectionId); + if (userLeases.Count is 0) + _leases.Remove(userId); + } + } + + return Task.CompletedTask; + } + + private static void RemoveExpired(Dictionary leases, DateTimeOffset now) + { + foreach (string connectionId in leases.Where(pair => pair.Value <= now).Select(pair => pair.Key).ToArray()) + leases.Remove(connectionId); + } +} + +public sealed class ConnectionLeaseStoreException(string message, Exception innerException) : Exception(message, innerException); diff --git a/src/Exceptionless.Insulation/Bootstrapper.cs b/src/Exceptionless.Insulation/Bootstrapper.cs index 5ce4f4f7b3..f7b5640212 100644 --- a/src/Exceptionless.Insulation/Bootstrapper.cs +++ b/src/Exceptionless.Insulation/Bootstrapper.cs @@ -112,6 +112,7 @@ private static void RegisterCache(IServiceCollection container, CacheOptions opt container.ReplaceSingleton(CreateRedisCacheClient); container.ReplaceSingleton(); + container.ReplaceSingleton(); } } diff --git a/src/Exceptionless.Insulation/Redis/RedisConnectionLeaseStore.cs b/src/Exceptionless.Insulation/Redis/RedisConnectionLeaseStore.cs new file mode 100644 index 0000000000..06f4949f80 --- /dev/null +++ b/src/Exceptionless.Insulation/Redis/RedisConnectionLeaseStore.cs @@ -0,0 +1,84 @@ +using Exceptionless.Core; +using Exceptionless.Core.Utility; +using StackExchange.Redis; + +namespace Exceptionless.Insulation.Redis; + +public sealed class RedisConnectionLeaseStore : IConnectionLeaseStore +{ + private const string AcquireScript = """ + local now = redis.call('TIME') + local now_ms = (now[1] * 1000) + math.floor(now[2] / 1000) + redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms) + if redis.call('ZSCORE', KEYS[1], ARGV[1]) then + redis.call('ZADD', KEYS[1], now_ms + ARGV[3], ARGV[1]) + redis.call('PEXPIRE', KEYS[1], ARGV[3] * 2) + return 1 + end + if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[2]) then return 0 end + redis.call('ZADD', KEYS[1], now_ms + ARGV[3], ARGV[1]) + redis.call('PEXPIRE', KEYS[1], ARGV[3] * 2) + return 1 + """; + private const string RenewScript = """ + local now = redis.call('TIME') + local now_ms = (now[1] * 1000) + math.floor(now[2] / 1000) + local expires = redis.call('ZSCORE', KEYS[1], ARGV[1]) + if not expires or tonumber(expires) <= now_ms then + redis.call('ZREM', KEYS[1], ARGV[1]) + return 0 + end + redis.call('ZADD', KEYS[1], now_ms + ARGV[2], ARGV[1]) + redis.call('PEXPIRE', KEYS[1], ARGV[2] * 2) + return 1 + """; + private readonly IConnectionMultiplexer _multiplexer; + private readonly string _keyPrefix; + + public RedisConnectionLeaseStore(IConnectionMultiplexer multiplexer, AppOptions options) + { + _multiplexer = multiplexer; + _keyPrefix = $"PushLease:{options.AppScope}:"; + } + + public async Task TryAcquireAsync(string userId, string connectionId, int maxConnections, TimeSpan leaseDuration) + { + try + { + var result = await Database.ScriptEvaluateAsync(AcquireScript, [GetKey(userId)], [connectionId, maxConnections, (long)leaseDuration.TotalMilliseconds]); + return (long)result == 1; + } + catch (RedisException ex) + { + throw new ConnectionLeaseStoreException("Unable to acquire a push connection lease.", ex); + } + } + + public async Task RenewAsync(string userId, string connectionId, TimeSpan leaseDuration) + { + try + { + var result = await Database.ScriptEvaluateAsync(RenewScript, [GetKey(userId)], [connectionId, (long)leaseDuration.TotalMilliseconds]); + return (long)result == 1; + } + catch (RedisException ex) + { + throw new ConnectionLeaseStoreException("Unable to renew a push connection lease.", ex); + } + } + + public async Task ReleaseAsync(string userId, string connectionId) + { + try + { + await Database.SortedSetRemoveAsync(GetKey(userId), connectionId); + } + catch (RedisException ex) + { + throw new ConnectionLeaseStoreException("Unable to release a push connection lease.", ex); + } + } + + private IDatabase Database => _multiplexer.GetDatabase(); + private RedisKey GetKey(string userId) => _keyPrefix + userId; +} diff --git a/src/Exceptionless.Web/Bootstrapper.cs b/src/Exceptionless.Web/Bootstrapper.cs index 7eb404ae13..5e052697c6 100644 --- a/src/Exceptionless.Web/Bootstrapper.cs +++ b/src/Exceptionless.Web/Bootstrapper.cs @@ -13,7 +13,9 @@ public class Bootstrapper { public static void RegisterServices(IServiceCollection services, AppOptions appOptions, ILoggerFactory loggerFactory) { + services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js index 0cee86b9a6..77fc55f16c 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js @@ -1,6 +1,7 @@ (function () { "use strict"; + // Deprecated: keep the legacy Angular client on WebSocket during the SSE rollout. angular .module("exceptionless.websocket", ["app.config", "exceptionless", "exceptionless.auth"]) .factory("websocketService", function ($ExceptionlessClient, $rootScope, $timeout, authService, BASE_URL) { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts new file mode 100644 index 0000000000..7f852f1319 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts @@ -0,0 +1,331 @@ +import { DocumentVisibility } from '$shared/document-visibility.svelte'; + +import { accessToken } from '../auth/index.svelte'; + +export interface SseClientOptions { + /** + * Base URL for SSE connection (e.g., 'http://localhost:5200') + * If not provided, constructs from window.location + */ + baseUrl?: string; + /** + * Connection timeout in milliseconds + * Default: 10000ms (10 seconds) + */ + connectionTimeout?: number; + /** + * Custom reconnection delay calculator + * Default uses exponential backoff: 1s, 2s, 4s, 8s, 16s, max 30s + * For testing, can return 0 to reconnect immediately + */ + reconnectDelay?: (attempt: number) => number; + /** Delay before probing again when push is disabled or a rolling-deploy replica returns 404. */ + unavailableRetryDelay?: number; +} + +// SSE connection state constants (same values as EventSource.*) +export const SSE_CONNECTING = 0; +export const SSE_OPEN = 1; +export const SSE_CLOSED = 2; + +// EventSource does not support custom Authorization headers, so the app uses fetch + +// ReadableStream to keep bearer tokens out of the query string. +export class SseClient { + public readyState = $state(SSE_CLOSED); + + /** + * Lazy getter for SSE URL. + */ + public get url(): string { + if (this._url === null) { + if (this._options.baseUrl) { + this._url = `${this._options.baseUrl}${this._path}`; + } else { + const { host, protocol } = window.location; + this._url = `${protocol}//${host}${this._path}`; + } + } + + return this._url; + } + + private _options: SseClientOptions; + private _path: string; + private _url: null | string = null; + private abortController: AbortController | null = null; + private accessToken: null | string = null; + private authFailed: boolean = false; + private connectionTimeoutId: null | ReturnType = null; + private forcedClose: boolean = false; + private hasConnectedBefore: boolean = false; + private pausedForVisibility: boolean = false; + private reconnectAttempts: number = 0; + private reconnectTimeoutId: null | ReturnType = null; + + private streamGeneration: number = 0; + + /** + * @param path - SSE endpoint path (default: '/api/v2/push') + * @param options - Optional configuration + */ + constructor(path: string = '/api/v2/push', options: SseClientOptions = {}) { + this._path = path; + this._options = options; + + const visibility = new DocumentVisibility(); + + $effect(() => { + if (this.accessToken !== accessToken.current) { + this.accessToken = accessToken.current; + this.reconnectAttempts = 0; + this.authFailed = false; + this.pausedForVisibility = false; + this.close(false); + } else if (!visibility.visible) { + this.pausedForVisibility = true; + this.close(false); + } else { + this.pausedForVisibility = false; + } + + if ( + this.accessToken && + visibility.visible && + this.readyState === SSE_CLOSED && + this.reconnectTimeoutId === null && + !this.authFailed && + !this.forcedClose + ) { + this.connect(); + } + }); + } + + public close(isManual: boolean = true): boolean { + clearTimeout(this.reconnectTimeoutId!); + this.reconnectTimeoutId = null; + clearTimeout(this.connectionTimeoutId!); + this.connectionTimeoutId = null; + this.forcedClose = isManual; + + if (this.abortController) { + this.streamGeneration++; + this.abortController.abort(); + this.abortController = null; + this.readyState = SSE_CLOSED; + return true; + } + + return false; + } + + public connect() { + const isReconnect: boolean = this.hasConnectedBefore; + const generation = ++this.streamGeneration; + + this.readyState = SSE_CONNECTING; + this.forcedClose = false; + + this.abortController = new AbortController(); + const { signal } = this.abortController; + + this.onConnecting(isReconnect); + + // Connection timeout + clearTimeout(this.connectionTimeoutId!); + const timeout = this._options.connectionTimeout ?? 10000; + this.connectionTimeoutId = setTimeout(() => { + this.connectionTimeoutId = null; + if (this.readyState === SSE_CONNECTING) { + console.warn(`[SseClient] Connection timeout after ${timeout}ms`); + this.abortController?.abort(); + } + }, timeout); + + this.startStream(signal, isReconnect, generation); + } + + public onClose: () => void = () => {}; + public onConnecting: (isReconnect: boolean) => void = () => {}; + public onError: (error: unknown) => void = () => {}; + public onMessage: (ev: MessageEvent) => void = () => {}; + public onOpen: (isReconnect: boolean) => void = () => {}; + + /** + * Calculate reconnection delay using exponential backoff + */ + private getReconnectDelay(attempt: number): number { + if (this._options.reconnectDelay) { + return this._options.reconnectDelay(attempt); + } + + // Default: exponential backoff 1s, 2s, 4s, 8s, 16s, max 30s + return Math.min(1000 * Math.pow(2, attempt - 1), 30000); + } + + private scheduleReconnect(delayOverride?: number) { + if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || this.pausedForVisibility || !(this.accessToken ?? accessToken.current)) { + this.readyState = SSE_CLOSED; + return; + } + + this.reconnectAttempts++; + const delay = delayOverride ?? this.getReconnectDelay(this.reconnectAttempts); + + this.readyState = SSE_CONNECTING; + this.onConnecting(true); + this.onClose(); + + clearTimeout(this.reconnectTimeoutId!); + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = null; + this.connect(); + }, delay); + } + + private async startStream(signal: AbortSignal, isReconnect: boolean, generation: number) { + try { + const token = this.accessToken ?? accessToken.current; + const response = await fetch(this.url, { + headers: { + Accept: 'text/event-stream', + Authorization: `Bearer ${token}` + }, + signal + }); + + clearTimeout(this.connectionTimeoutId!); + this.connectionTimeoutId = null; + + if (!response.ok) { + // Auth failures - don't reconnect + if (response.status === 401 || response.status === 403) { + console.warn('[SseClient] Auth failure, not reconnecting', { status: response.status }); + this.authFailed = true; + if (response.status === 401 && accessToken.current) { + accessToken.current = ''; + } + + this.readyState = SSE_CLOSED; + this.onClose(); + return; + } + + if (response.status === 404) { + console.info('[SseClient] Push endpoint unavailable, probing again later'); + this.scheduleReconnect(this._options.unavailableRetryDelay ?? 300000); + return; + } + + // Rate limited + if (response.status === 429) { + console.warn('[SseClient] Rate limited, will retry'); + this.scheduleReconnect(); + return; + } + + throw new Error(`SSE connection failed: ${response.status}`); + } + + if (!response.body) { + throw new Error('SSE response has no body'); + } + + const contentType = response.headers.get('content-type')?.split(';', 1).at(0)?.trim().toLowerCase(); + if (contentType !== 'text/event-stream') { + throw new Error(`SSE response has invalid content type: ${contentType ?? 'missing'}`); + } + + if (generation !== this.streamGeneration) { + this.readyState = SSE_CLOSED; + return; + } + + this.readyState = SSE_OPEN; + this.reconnectAttempts = 0; + this.hasConnectedBefore = true; + this.onOpen(isReconnect); + + // Read the stream + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + if (generation !== this.streamGeneration) { + this.readyState = SSE_CLOSED; + return; + } + + buffer += decoder.decode(value, { stream: true }); + + // Process complete SSE messages (separated by double newline) + const messages = buffer.split(/\r?\n\r?\n/); + buffer = messages.pop() ?? ''; + + for (const message of messages) { + if (!message.trim()) { + continue; + } + + // Parse SSE format + const lines = message.split(/\r?\n/); + const dataLines: string[] = []; + + for (const line of lines) { + if (line.startsWith('data: ')) { + dataLines.push(line.slice(6)); + } else if (line.startsWith('data:')) { + dataLines.push(line.slice(5)); + } else if (line.startsWith(':')) { + // Comment (keep-alive), ignore + continue; + } + } + + if (dataLines.length > 0) { + const data = dataLines.join('\n'); + // Create a MessageEvent-like object for compatibility + const event = new MessageEvent('message', { data }); + this.onMessage(event); + } + } + } + } catch (error: unknown) { + clearTimeout(this.connectionTimeoutId!); + this.connectionTimeoutId = null; + + if (generation !== this.streamGeneration) { + this.readyState = SSE_CLOSED; + return; + } + + if (signal.aborted && (this.forcedClose || this.pausedForVisibility)) { + // Intentional close - don't reconnect + this.readyState = SSE_CLOSED; + this.onClose(); + return; + } + + if (signal.aborted) { + // Timeout or other abort - try reconnect + this.scheduleReconnect(); + return; + } + + console.error('[SseClient] Stream error', error); + this.onError(error); + } + + // Stream ended (server closed connection) - reconnect + if (generation === this.streamGeneration && !this.forcedClose && !this.pausedForVisibility) { + this.scheduleReconnect(); + } + } +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts new file mode 100644 index 0000000000..7fea19cb01 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts @@ -0,0 +1,404 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { accessToken } from '../auth/index.svelte'; +import { SSE_CLOSED, SSE_CONNECTING, SSE_OPEN, SseClient, type SseClientOptions } from './sse-client.svelte'; + +// Mock the auth module +vi.mock('../auth/index.svelte', () => ({ + accessToken: { + current: 'test-token-123' + } +})); + +// Mock DocumentVisibility to always return visible +vi.mock('$shared/document-visibility.svelte', () => { + return { + DocumentVisibility: class { + visible = true; + } + }; +}); + +function createClient(options?: SseClientOptions): SseClient { + return new SseClient('/api/v2/push', { + baseUrl: 'http://localhost:5200', + reconnectDelay: () => 50, + ...options + }); +} + +// Creates a response whose stream stays open indefinitely (for testing open connections) +function createOpenSseResponse(initialEvents: string[] = []) { + return new Response( + new ReadableStream({ + start(controller) { + for (const event of initialEvents) { + controller.enqueue(new TextEncoder().encode(event)); + } + // intentionally never close + } + }), + { + headers: { 'Content-Type': 'text/event-stream' }, + status: 200 + } + ); +} + +// Helper to create a mock fetch response that streams SSE data +function createSseResponse(events: string[] = [], options: { delay?: number; status?: number } = {}) { + const { delay = 0, status = 200 } = options; + + return new Response( + new ReadableStream({ + async start(controller) { + for (const event of events) { + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + + controller.enqueue(new TextEncoder().encode(event)); + } + + controller.close(); + } + }), + { + headers: { 'Content-Type': 'text/event-stream' }, + status + } + ); +} + +describe('SseClient', () => { + let fetchMock: ReturnType>; + let activeClients: SseClient[] = []; + + beforeEach(() => { + accessToken.current = 'test-token-123'; + fetchMock = vi.fn(); + global.fetch = fetchMock as typeof fetch; + }); + + afterEach(() => { + for (const client of activeClients) { + client.close(); + } + + activeClients = []; + vi.restoreAllMocks(); + }); + + function trackedClient(options?: SseClientOptions): SseClient { + const client = createClient(options); + activeClients.push(client); + return client; + } + + describe('Connection Lifecycle', () => { + it('should connect successfully and call onOpen', async () => { + const onOpen = vi.fn(); + fetchMock.mockResolvedValue(createOpenSseResponse([': keepalive\n\n'])); + + const client = trackedClient(); + client.onOpen = onOpen; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(fetchMock).toHaveBeenCalledWith( + 'http://localhost:5200/api/v2/push', + expect.objectContaining({ + headers: expect.objectContaining({ + Accept: 'text/event-stream', + Authorization: 'Bearer test-token-123' + }) + }) + ); + expect(onOpen).toHaveBeenCalledWith(false); + + client.close(); + }); + + it('should set readyState to CONNECTING then OPEN', async () => { + fetchMock.mockResolvedValue(createOpenSseResponse([': keepalive\n\n'])); + + const client = trackedClient(); + client.connect(); + + expect(client.readyState).toBe(SSE_CONNECTING); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(client.readyState).toBe(SSE_OPEN); + + client.close(); + }); + + it('should call onConnecting with isReconnect=false on first connection', async () => { + const onConnecting = vi.fn(); + fetchMock.mockResolvedValue(createSseResponse([])); + + const client = trackedClient(); + client.onConnecting = onConnecting; + client.connect(); + + expect(onConnecting).toHaveBeenCalledWith(false); + client.close(); + }); + }); + + describe('Disconnection', () => { + it('should close when close() is called', async () => { + // Create a response that never closes + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + start() { + // Never close - simulate long-lived connection + } + }), + { headers: { 'Content-Type': 'text/event-stream' }, status: 200 } + ) + ); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + const result = client.close(); + + expect(result).toBe(true); + expect(client.readyState).toBe(SSE_CLOSED); + }); + + it('should return false when closing already closed connection', () => { + const client = trackedClient(); + const result = client.close(); + + expect(result).toBe(false); + }); + + it('should NOT reconnect after manual close', async () => { + // Use a stream that stays open (never closes) so we can test manual close + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + start() { + // intentionally never close - stream stays open + } + }), + { headers: { 'Content-Type': 'text/event-stream' }, status: 200 } + ) + ); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + client.close(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(client.readyState).toBe(SSE_CLOSED); + // fetch should only be called once (no reconnect) + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('should allow reconnect after internal close', async () => { + fetchMock.mockImplementation(() => Promise.resolve(createOpenSseResponse([': keepalive\n\n']))); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(client.close(false)).toBe(true); + + client.connect(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(client.readyState).toBe(SSE_OPEN); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + }); + + describe('Auth Failure Handling', () => { + it('should NOT reconnect on 401', async () => { + fetchMock.mockImplementation(() => Promise.resolve(new Response(null, { status: 401 }))); + + const onClose = vi.fn(); + const client = trackedClient(); + client.onClose = onClose; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onClose).toHaveBeenCalledTimes(1); + expect(client.readyState).toBe(SSE_CLOSED); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(accessToken.current).toBe(''); + }); + + it('should NOT reconnect on 403', async () => { + fetchMock.mockImplementation(() => Promise.resolve(new Response(null, { status: 403 }))); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(client.readyState).toBe(SSE_CLOSED); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('should slowly probe again when push endpoint is unavailable', async () => { + const onClose = vi.fn(); + fetchMock.mockImplementation(() => Promise.resolve(new Response(null, { status: 404 }))); + + const client = trackedClient({ unavailableRetryDelay: 25 }); + client.onClose = onClose; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onClose).toHaveBeenCalled(); + expect(fetchMock.mock.calls.length).toBeGreaterThan(1); + }); + }); + + describe('Reconnection Logic', () => { + it('should reconnect when stream ends normally', async () => { + let callCount = 0; + fetchMock.mockImplementation(() => { + callCount++; + return Promise.resolve(createSseResponse([': keepalive\n\n'])); + }); + + const client = trackedClient({ baseUrl: 'http://localhost:5200', reconnectDelay: () => 10 }); + client.connect(); + + // Wait for initial connection + stream end + reconnect + await new Promise((resolve) => setTimeout(resolve, 200)); + + expect(callCount).toBeGreaterThan(1); + client.close(); + }); + + it('should use custom reconnectDelay', async () => { + const reconnectDelay = vi.fn(() => 50); + fetchMock.mockResolvedValue(createSseResponse([])); + + const client = trackedClient({ baseUrl: 'http://localhost:5200', reconnectDelay }); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 150)); + + expect(reconnectDelay).toHaveBeenCalled(); + client.close(); + }); + }); + + describe('Message Handling', () => { + it('should parse SSE data messages and call onMessage', async () => { + const onMessage = vi.fn(); + const sseData = 'data: {"type":"StackChanged","message":{"id":"123"}}\n\n'; + fetchMock.mockResolvedValue(createSseResponse([sseData])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onMessage).toHaveBeenCalledWith( + expect.objectContaining({ + data: '{"type":"StackChanged","message":{"id":"123"}}' + }) + ); + client.close(); + }); + + it('should ignore keep-alive comments', async () => { + const onMessage = vi.fn(); + const sseData = ': keepalive\n\ndata: {"type":"test","message":{}}\n\n'; + fetchMock.mockResolvedValue(createSseResponse([sseData])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Should only get the data message, not the keepalive + expect(onMessage).toHaveBeenCalledTimes(1); + expect(onMessage).toHaveBeenCalledWith( + expect.objectContaining({ + data: '{"type":"test","message":{}}' + }) + ); + client.close(); + }); + + it('should handle multiple messages in one chunk', async () => { + const onMessage = vi.fn(); + const sseData = 'data: {"type":"A","message":{}}\n\ndata: {"type":"B","message":{}}\n\n'; + fetchMock.mockResolvedValue(createSseResponse([sseData])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onMessage).toHaveBeenCalledTimes(2); + client.close(); + }); + + it('should handle messages split across chunks', async () => { + const onMessage = vi.fn(); + fetchMock.mockResolvedValue(createSseResponse(['data: {"type":"Sp', 'lit","message":{}}\n\n'])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onMessage).toHaveBeenCalledWith( + expect.objectContaining({ + data: '{"type":"Split","message":{}}' + }) + ); + client.close(); + }); + + it('should parse CRLF-delimited and multiline SSE data', async () => { + const onMessage = vi.fn(); + fetchMock.mockResolvedValue(createSseResponse(['data: first\r\ndata: second\r\n\r\n'])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(onMessage).toHaveBeenCalledWith(expect.objectContaining({ data: 'first\nsecond' })); + }); + + it('should reject a successful response with a non-SSE content type', async () => { + const onError = vi.fn(); + fetchMock.mockResolvedValue(new Response('', { headers: { 'Content-Type': 'text/html' }, status: 200 })); + + const client = trackedClient({ reconnectDelay: () => 1000 }); + client.onError = onError; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining('invalid content type') })); + }); + }); + + describe('URL Construction', () => { + it('should construct correct SSE URL with base URL', () => { + const client = trackedClient({ baseUrl: 'http://localhost:5200' }); + expect(client.url).toBe('http://localhost:5200/api/v2/push'); + }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts deleted file mode 100644 index c816c5bcb5..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { DocumentVisibility } from '$shared/document-visibility.svelte'; - -import { accessToken } from '../auth/index.svelte'; - -export interface WebSocketClientOptions { - /** - * Base URL for WebSocket connection (e.g., 'ws://localhost:1234') - * If not provided, constructs from window.location - */ - baseUrl?: string; - /** - * Connection timeout in milliseconds - * Default: 10000ms (10 seconds) - */ - connectionTimeout?: number; - /** - * Custom reconnection delay calculator - * Default uses exponential backoff: 1s, 2s, 4s, 8s, 16s, max 30s - * For testing, can return 0 to reconnect immediately - */ - reconnectDelay?: (attempt: number) => number; -} - -export class WebSocketClient { - public readyState = $state(WebSocket.CLOSED); - - /** - * Lazy getter for WebSocket URL. - * Constructed on first access. Uses baseUrl from options if provided, otherwise constructs from window.location. - */ - public get url(): string { - if (this._url === null) { - if (this._options.baseUrl) { - this._url = `${this._options.baseUrl}${this._path}`; - } else { - const { host, protocol } = window.location; - const wsProtocol = protocol === 'https:' ? 'wss://' : 'ws://'; - this._url = `${wsProtocol}${host}${this._path}`; - } - } - - return this._url; - } - - private _options: WebSocketClientOptions; - private _path: string; - private _url: null | string = null; - private accessToken: null | string = null; - private connectionTimeoutId: null | ReturnType = null; - private forcedClose: boolean = false; - private hasConnectedBefore: boolean = false; - private reconnectAttempts: number = 0; - private reconnectTimeoutId: null | ReturnType = null; - - private ws: null | WebSocket = null; - - /** - * @param path - WebSocket path (default: '/api/v2/push') - * @param options - Optional configuration - */ - constructor(path: string = '/api/v2/push', options: WebSocketClientOptions = {}) { - this._path = path; - this._options = options; - - const visibility = new DocumentVisibility(); - - $effect(() => { - if (this.accessToken !== accessToken.current) { - this.accessToken = accessToken.current; - this.reconnectAttempts = 0; // Reset backoff on token change - this.close(); - } else if (!visibility.visible) { - this.close(); - } - - // Only auto-connect if we're fully closed and don't have a pending reconnect attempt - // Don't try to connect if we're CONNECTING, OPEN, or CLOSING - if (this.accessToken && visibility.visible && this.readyState === WebSocket.CLOSED && this.reconnectTimeoutId === null) { - this.connect(); - } - }); - } - - public close(): boolean { - clearTimeout(this.reconnectTimeoutId!); - this.reconnectTimeoutId = null; - clearTimeout(this.connectionTimeoutId!); - this.connectionTimeoutId = null; - - if (this.ws) { - this.forcedClose = true; - this.ws.close(); - return true; - } - - return false; - } - - public connect() { - // isReconnect means: have we successfully connected before? - const isReconnect: boolean = this.hasConnectedBefore; - - // Reset state - this.readyState = WebSocket.CONNECTING; - this.forcedClose = false; - - try { - this.ws = new WebSocket(`${this.url}?access_token=${this.accessToken}`); - this.onConnecting(isReconnect); - } catch (error) { - console.error('[WebSocketClient] Failed to create WebSocket', error); - throw error; - } - - // Connection timeout: if we don't connect within configured timeout, force close - clearTimeout(this.connectionTimeoutId!); - const timeout = this._options.connectionTimeout ?? 10000; - this.connectionTimeoutId = setTimeout(() => { - this.connectionTimeoutId = null; - if (this.ws && this.readyState === WebSocket.CONNECTING) { - console.warn(`[WebSocketClient] Connection timeout after ${timeout}ms`); - this.ws.close(); - } - }, timeout); - - this.ws.onopen = (event: Event) => { - clearTimeout(this.connectionTimeoutId!); - this.connectionTimeoutId = null; - this.readyState = WebSocket.OPEN; - this.reconnectAttempts = 0; // Reset backoff on successful connection - this.hasConnectedBefore = true; // Mark that we've connected successfully - this.onOpen(event, isReconnect); - }; - - this.ws.onclose = (event: CloseEvent) => { - clearTimeout(this.connectionTimeoutId!); - this.connectionTimeoutId = null; - this.ws = null; - - if (this.forcedClose) { - this.readyState = WebSocket.CLOSED; - this.onClose(event); - return; - } - - // Don't retry on authentication/authorization failures - // Code 1008 (Policy Violation) is explicit auth failure - // Code 1006 (Abnormal Closure) during handshake could be 401/403 - // Codes 4xxx are custom application codes (e.g., 4401=401, 4403=403) - const isAuthFailure = event.code === 1008 || (event.code === 1006 && event.wasClean === false) || (event.code >= 4400 && event.code < 4500); - if (isAuthFailure) { - console.warn('[WebSocketClient] Auth failure detected, not reconnecting', { - code: event.code, - reason: event.reason - }); - this.readyState = WebSocket.CLOSED; - this.onClose(event); - return; // Let the auth system handle redirect to login - } - - // Calculate reconnection delay with exponential backoff - this.reconnectAttempts++; - const delay = this.getReconnectDelay(this.reconnectAttempts); - - this.onConnecting(true); // Always true when reconnecting after close - this.onClose(event); - - // Schedule reconnect - clear any existing timeout first - clearTimeout(this.reconnectTimeoutId!); - this.reconnectTimeoutId = setTimeout(() => { - this.reconnectTimeoutId = null; - this.connect(); - }, delay); - }; - - this.ws.onmessage = (event) => { - this.onMessage(event); - }; - - this.ws.onerror = (event) => { - console.error('[WebSocketClient] onerror triggered', { - event, - readyState: this.readyState, - reconnectAttempts: this.reconnectAttempts - }); - this.onError(event); - }; - } - - public onClose: (ev: CloseEvent) => void = () => {}; - - public onConnecting: (isReconnect: boolean) => void = () => {}; - public onError: (ev: Event) => void = () => {}; - public onMessage: (ev: MessageEvent) => void = () => {}; - - public onOpen: (ev: Event, isReconnect: boolean) => void = () => {}; - - public send(data: Parameters[0]) { - if (this.ws) { - return this.ws.send(data); - } else { - throw new Error('INVALID_STATE_ERR : Pausing to reconnect websocket'); - } - } - - /** - * Calculate reconnection delay using exponential backoff - * Can be overridden via options for testing - */ - private getReconnectDelay(attempt: number): number { - if (this._options.reconnectDelay) { - return this._options.reconnectDelay(attempt); - } - - // Default: exponential backoff 1s, 2s, 4s, 8s, 16s, max 30s - return Math.min(1000 * Math.pow(2, attempt - 1), 30000); - } -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts deleted file mode 100644 index c7138d1206..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts +++ /dev/null @@ -1,494 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import WS from 'vitest-websocket-mock'; - -import { WebSocketClient, type WebSocketClientOptions } from './web-socket-client.svelte'; - -// Mock the auth module -vi.mock('../auth/index.svelte', () => ({ - accessToken: { - current: 'test-token-123' - } -})); - -// Mock DocumentVisibility to always return visible -vi.mock('$shared/document-visibility.svelte', () => { - return { - DocumentVisibility: class { - visible = true; - } - }; -}); - -let server: WS; - -beforeEach(() => { - server = new WS('ws://localhost:1234/api/v2/push'); -}); - -afterEach(() => { - WS.clean(); -}); - -function createClient(path?: string, options?: WebSocketClientOptions): WebSocketClient { - return new WebSocketClient(path, { - baseUrl: 'ws://localhost:1234', - reconnectDelay: () => 0, - ...options - }); -} - -describe('WebSocketClient', () => { - describe('Connection Lifecycle', () => { - it('should connect successfully', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - expect(client.readyState).toBe(WebSocket.OPEN); - client.close(); - }); - - it('should set readyState to CONNECTING then OPEN', async () => { - const client = createClient(); - client.connect(); - - expect(client.readyState).toBe(WebSocket.CONNECTING); - await server.connected; - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - }); - - it('should call onConnecting with isReconnect=false on first connection', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - expect(onConnecting).toHaveBeenCalledWith(false); - await server.connected; - - client.close(); - }); - - it('should call onOpen with isReconnect=false on first connection', async () => { - const onOpen = vi.fn(); - const client = createClient(); - client.onOpen = onOpen; - - client.connect(); - await server.connected; - - expect(onOpen).toHaveBeenCalledWith(expect.anything(), false); - client.close(); - }); - - it('should handle multiple connect calls gracefully', async () => { - const client = createClient(); - - client.connect(); - client.connect(); - client.connect(); - - await server.connected; - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - }); - - it('should use custom connectionTimeout option', async () => { - const onConnecting = vi.fn(); - const client = new WebSocketClient('/api/v2/push', { - baseUrl: 'ws://localhost:9999', - connectionTimeout: 75, // Very short timeout - reconnectDelay: () => 1000 // Prevent immediate reconnect - }); - client.onConnecting = onConnecting; - - client.connect(); - expect(client.readyState).toBe(WebSocket.CONNECTING); - - // Wait for custom timeout to expire and close to be triggered - await new Promise((resolve) => setTimeout(resolve, 150)); - - // onConnecting was called with isReconnect=false for initial connect - expect(onConnecting).toHaveBeenCalledWith(false); - }); - }); - - describe('Disconnection', () => { - it('should close WebSocket when close() is called', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - const result = client.close(); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(result).toBe(true); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should return false when closing already closed connection', () => { - const client = createClient(); - client.close(); - const result = client.close(); - - expect(result).toBe(false); - }); - - it('should call onClose callback', async () => { - const onClose = vi.fn(); - const client = createClient(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 1000, reason: 'Test', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 1000, - reason: 'Test', - wasClean: true - }) - ); - }); - - it('should NOT reconnect after manual close', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - client.close(); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - }); - - describe('Reconnection Logic', () => { - it('should NOT reconnect on policy violation (code 1008) - auth failure', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 1008, reason: 'Policy Violation', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 1008, - reason: 'Policy Violation', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should NOT reconnect on abnormal closure (code 1006, wasClean=false) - connection lost unexpectedly', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 1006, reason: 'Abnormal Closure', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 1006, - reason: 'Abnormal Closure', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should NOT reconnect on unauthorized (code 4401) - 401 HTTP equivalent', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 4401, reason: 'Unauthorized', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 4401, - reason: 'Unauthorized', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should NOT reconnect on forbidden (code 4403) - 403 HTTP equivalent', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 4403, reason: 'Forbidden', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 4403, - reason: 'Forbidden', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should reconnect on normal closure (code 1000) - server initiated graceful close', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - onConnecting.mockClear(); - - server.close({ code: 1000, reason: 'Normal Closure', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - await server.connected; - - expect(onConnecting).toHaveBeenCalledWith(true); - client.close(); - }); - - it('should reconnect on going away (code 1001) - server restart', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - onConnecting.mockClear(); - - server.close({ code: 1001, reason: 'Going Away', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - await server.connected; - - expect(onConnecting).toHaveBeenCalledWith(true); - client.close(); - }); - - it('should call onConnecting with isReconnect=true on reconnection', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - expect(onConnecting).toHaveBeenCalledWith(false); - onConnecting.mockClear(); - - server.close({ code: 1000, reason: 'Test', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onConnecting).toHaveBeenCalledWith(true); - await server.connected; - client.close(); - }); - }); - - describe('Message Handling', () => { - it('should send messages when connected', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - client.send('test message'); - await expect(server).toReceiveMessage('test message'); - - client.close(); - }); - - it('should throw error when sending while disconnected', () => { - const client = createClient(); - - expect(() => client.send('test')).toThrow('INVALID_STATE_ERR'); - }); - - it('should call onMessage callback when receiving messages', async () => { - const onMessage = vi.fn(); - const client = createClient(); - client.onMessage = onMessage; - - client.connect(); - await server.connected; - - server.send('test data'); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onMessage).toHaveBeenCalledWith( - expect.objectContaining({ - data: 'test data' - }) - ); - - client.close(); - }); - - it('should receive JSON messages', async () => { - const onMessage = vi.fn(); - const client = createClient(); - client.onMessage = onMessage; - - client.connect(); - await server.connected; - - const message = JSON.stringify({ data: 'hello', type: 'test' }); - server.send(message); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onMessage).toHaveBeenCalledWith( - expect.objectContaining({ - data: message - }) - ); - - client.close(); - }); - }); - - describe('Error Handling', () => { - it('should call onError callback', async () => { - const onError = vi.fn(); - const client = createClient(); - client.onError = onError; - - client.connect(); - await server.connected; - - server.error(); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onError).toHaveBeenCalled(); - client.close(); - }); - }); - - describe('URL Construction', () => { - it('should construct correct WebSocket URL', () => { - const client = createClient('/api/v2/push'); - - expect(client.url).toBe('ws://localhost:1234/api/v2/push'); - }); - - it('should use custom base URL', async () => { - const customClient = new WebSocketClient('/api/v2/push', { - baseUrl: 'ws://custom-host:5000', - reconnectDelay: () => 0 - }); - - const customServer = new WS('ws://custom-host:5000/api/v2/push'); - customClient.connect(); - await customServer.connected; - - expect(customClient.readyState).toBe(WebSocket.OPEN); - - customClient.close(); - customServer.close(); - }); - - it('should handle custom path', async () => { - const client = createClient('/custom/path'); - const customServer = new WS('ws://localhost:1234/custom/path'); - - client.connect(); - await customServer.connected; - - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - customServer.close(); - }); - }); - - describe('Options - getReconnectDelay', () => { - it('should use custom getReconnectDelay from options', async () => { - const getReconnectDelay = vi.fn(() => 100); - const client = new WebSocketClient('/api/v2/push', { - baseUrl: 'ws://localhost:1234', - reconnectDelay: getReconnectDelay - }); - - client.connect(); - await server.connected; - - server.close({ code: 1000, reason: 'Test', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - await server.connected; - - expect(getReconnectDelay).toHaveBeenCalled(); - client.close(); - }); - - it('should use immediate reconnection with getReconnectDelay: () => 0', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - onConnecting.mockClear(); - - const start = Date.now(); - server.close({ code: 1000, reason: 'Test', wasClean: true }); - - // Wait for reconnection attempt - await new Promise((resolve) => setTimeout(resolve, 50)); - - // Verify reconnection happened quickly (within 50ms) - const elapsed = Date.now() - start; - expect(onConnecting).toHaveBeenCalledWith(true); - expect(elapsed).toBeLessThan(100); - - client.close(); - }); - }); - - describe('Edge Cases', () => { - it('should handle rapid connect/disconnect cycles', async () => { - const client = createClient(); - - client.connect(); - client.close(); - client.connect(); - await server.connected; - - expect(client.readyState).toBe(WebSocket.OPEN); - client.close(); - }); - - it('should maintain connection state correctly', async () => { - const client = createClient(); - - expect(client.readyState).toBe(WebSocket.CLOSED); - - client.connect(); - await server.connected; - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - await new Promise((resolve) => setTimeout(resolve, 10)); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 9deb9cb990..de420ebd2b 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -28,8 +28,8 @@ import { getMeQuery, invalidateUserQueries } from '$features/users/api.svelte'; import { getGravatarFromCurrentUser } from '$features/users/gravatar.svelte'; import { invalidateWebhookQueries } from '$features/webhooks/api.svelte'; - import { isEntityChangedType, type WebSocketMessageType } from '$features/websockets/models'; - import { WebSocketClient } from '$features/websockets/web-socket-client.svelte'; + import { type EntityChanged, isEntityChangedType, type UserMembershipChanged, type WebSocketMessageType } from '$features/websockets/models'; + import { SseClient } from '$features/websockets/sse-client.svelte'; import { Telemetry } from '$lib/telemetry'; import { useMiddleware } from '@exceptionless/fetchclient'; import { useQueryClient } from '@tanstack/svelte-query'; @@ -159,11 +159,31 @@ } } - // This event is fired when a user is added or removed from an organization. - // if (data.type === "UserMembershipChanged" && data.message?.organization_id) { - // $rootScope.$emit("OrganizationChanged", data.message); - // $rootScope.$emit("ProjectChanged", data.message); - // } + // When a user is added or removed from an organization, invalidate org/project caches + // so the UI reflects the membership change without a manual reload. + if (data.type === 'UserMembershipChanged') { + const membershipMessage = data.message as UserMembershipChanged; + if (membershipMessage.organization_id) { + const organizationChangedMessage: EntityChanged = { + change_type: membershipMessage.change_type, + data: {}, + id: membershipMessage.organization_id, + organization_id: membershipMessage.organization_id, + type: 'Organization' + }; + const projectChangedMessage: EntityChanged = { + change_type: membershipMessage.change_type, + data: {}, + organization_id: membershipMessage.organization_id, + type: 'Project' + }; + + await Promise.all([ + invalidateOrganizationQueries(queryClient, organizationChangedMessage), + invalidateProjectQueries(queryClient, projectChangedMessage) + ]); + } + } } // Close Sidebar on page change on mobile @@ -191,7 +211,7 @@ } }); - // WebSocket + keyboard shortcuts — only depends on token, not navigation + // SSE + keyboard shortcuts — only depends on token, not navigation $effect(() => { const currentToken = accessToken.current; @@ -252,15 +272,15 @@ document.addEventListener('keydown', handleKeydown, { capture: true }); - const ws = new WebSocketClient(); - ws.onMessage = onMessage; - ws.onOpen = (_, isReconnect) => { + const sse = new SseClient(); + sse.onMessage = onMessage; + sse.onOpen = (isReconnect) => { if (isReconnect) { queryClient.invalidateQueries(); document.dispatchEvent( new CustomEvent('refresh', { bubbles: true, - detail: 'WebSocket Connected' + detail: 'SSE Connected' }) ); } @@ -268,7 +288,7 @@ return () => { document.removeEventListener('keydown', handleKeydown, { capture: true }); - ws?.close(); + sse?.close(); }; }); diff --git a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs index fe059935db..7a8578c6ee 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs @@ -1,7 +1,6 @@ using Exceptionless.Core; using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; -using Exceptionless.Core.Utility; using Foundatio.Extensions.Hosting.Startup; using Foundatio.Messaging; using Foundatio.Repositories.Models; @@ -13,16 +12,18 @@ public sealed class MessageBusBroker : IStartupAction { private static readonly string TokenTypeName = nameof(Token); private static readonly string UserTypeName = nameof(User); - private readonly WebSocketConnectionManager _connectionManager; - private readonly IConnectionMapping _connectionMapping; + private readonly SseConnectionManager _sseConnectionManager; + private readonly WebSocketConnectionManager _webSocketConnectionManager; + private readonly PushConnectionRegistry _connectionRegistry; private readonly IMessageSubscriber _subscriber; private readonly AppOptions _options; private readonly ILogger _logger; - public MessageBusBroker(WebSocketConnectionManager connectionManager, IConnectionMapping connectionMapping, IMessageSubscriber subscriber, AppOptions options, ILogger logger) + public MessageBusBroker(SseConnectionManager sseConnectionManager, WebSocketConnectionManager webSocketConnectionManager, PushConnectionRegistry connectionRegistry, IMessageSubscriber subscriber, AppOptions options, ILogger logger) { - _connectionManager = connectionManager; - _connectionMapping = connectionMapping; + _sseConnectionManager = sseConnectionManager; + _webSocketConnectionManager = webSocketConnectionManager; + _connectionRegistry = connectionRegistry; _subscriber = subscriber; _options = options; _logger = logger; @@ -30,7 +31,7 @@ public MessageBusBroker(WebSocketConnectionManager connectionManager, IConnectio public async Task RunAsync(CancellationToken shutdownToken = default) { - if (!_options.EnableWebSockets) + if (!_options.EnablePush) return; _logger.LogDebug("Subscribing to message bus notifications"); @@ -45,7 +46,7 @@ await Task.WhenAll( _logger.LogDebug("Subscribed to message bus notifications"); } - private async Task OnUserMembershipChangedAsync(UserMembershipChanged userMembershipChanged, CancellationToken cancellationToken = default) + internal async Task OnUserMembershipChangedAsync(UserMembershipChanged userMembershipChanged, CancellationToken cancellationToken = default) { if (String.IsNullOrEmpty(userMembershipChanged?.OrganizationId)) { @@ -54,14 +55,21 @@ private async Task OnUserMembershipChangedAsync(UserMembershipChanged userMember } // manage user organization group membership - var userConnectionIds = await _connectionMapping.GetUserIdConnectionsAsync(userMembershipChanged.UserId); + var userConnectionIds = _connectionRegistry.GetUserConnections(userMembershipChanged.UserId); _logger.LogTrace("Attempting to update user {User} active groups for {UserConnectionCount} connections", userMembershipChanged.UserId, userConnectionIds.Count); + if (userMembershipChanged.ChangeType is ChangeType.Removed && userConnectionIds.Count > 0) + TypedSend(userConnectionIds, userMembershipChanged); + foreach (string connectionId in userConnectionIds) { if (userMembershipChanged.ChangeType is ChangeType.Added) - await _connectionMapping.GroupAddAsync(userMembershipChanged.OrganizationId, connectionId); + { + _connectionRegistry.AddGroup(connectionId, userMembershipChanged.OrganizationId); + } else if (userMembershipChanged.ChangeType is ChangeType.Removed) - await _connectionMapping.GroupRemoveAsync(userMembershipChanged.OrganizationId, connectionId); + { + _connectionRegistry.RemoveGroup(connectionId, userMembershipChanged.OrganizationId); + } } await GroupSendAsync(userMembershipChanged.OrganizationId, userMembershipChanged); @@ -88,10 +96,10 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can return; } - var userConnectionIds = await _connectionMapping.GetUserIdConnectionsAsync(entityChanged.Id); + var userConnectionIds = _connectionRegistry.GetUserConnections(entityChanged.Id); _logger.LogTrace("Sending {UserTypeName} message to user: {UserId} (to {UserConnectionCount} connections)", UserTypeName, entityChanged.Id, userConnectionIds.Count); foreach (string connectionId in userConnectionIds) - await TypedSendAsync(connectionId, entityChanged); + TypedSend(connectionId, entityChanged); return; } @@ -104,21 +112,20 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can if (userId is not null) { - var userConnectionIds = await _connectionMapping.GetUserIdConnectionsAsync(userId); + var userConnectionIds = _connectionRegistry.GetUserConnections(userId); - // Auth token removed = logout. Close sockets immediately without sending; + // Auth token removed = logout. Close connections immediately without sending; // there is no point delivering a message to a connection we are about to tear down. if (isAuthToken && entityChanged.ChangeType is ChangeType.Removed) { - _logger.LogTrace("Auth token removed for user {UserId}; closing {ConnectionCount} WebSocket connection(s)", userId, userConnectionIds.Count); - string? organizationId = entityChanged.OrganizationId; - foreach (string connectionId in userConnectionIds) + string? tokenId = entityChanged.Id; + var localConnectionIds = tokenId is null ? userConnectionIds : _connectionRegistry.RevokeToken(tokenId); + _logger.LogTrace("Auth token removed for user {UserId}; closing {ConnectionCount} local push connection(s)", userId, localConnectionIds.Count); + foreach (string connectionId in localConnectionIds) { - if (organizationId is { Length: > 0 }) - await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); - - await _connectionMapping.UserIdRemoveAsync(userId, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); + await _sseConnectionManager.RemoveConnectionAsync(connectionId); + await _webSocketConnectionManager.RemoveConnectionAsync(connectionId); + _connectionRegistry.Unregister(connectionId); } return; @@ -126,7 +133,7 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can _logger.LogTrace("Sending {TokenTypeName} message for user: {UserId} (to {UserConnectionCount} connections)", TokenTypeName, userId, userConnectionIds.Count); foreach (string connectionId in userConnectionIds) - await TypedSendAsync(connectionId, entityChanged); + TypedSend(connectionId, entityChanged); return; } @@ -172,40 +179,52 @@ private Task OnPlanChangedAsync(PlanChanged planChanged, CancellationToken cance private Task OnReleaseNotificationAsync(ReleaseNotification notification, CancellationToken cancellationToken = default) { _logger.LogTrace("Sending release notification message: {Message}", notification.Message); - return TypedBroadcastAsync(notification); + TypedBroadcast(notification); + return Task.CompletedTask; } private Task OnSystemNotificationAsync(SystemNotification notification, CancellationToken cancellationToken = default) { _logger.LogTrace("Sending system notification message: {Message}", notification.Message); - return TypedBroadcastAsync(notification); + TypedBroadcast(notification); + return Task.CompletedTask; } - private async Task GroupSendAsync(string group, object value) + private Task GroupSendAsync(string group, object value) { - var connectionIds = await _connectionMapping.GetGroupConnectionsAsync(group); + var connectionIds = _connectionRegistry.GetGroupConnections(group); if (connectionIds.Count is 0) { _logger.LogTrace("Ignoring group message to {Group}: No Connections", group); - return; + return Task.CompletedTask; } - await TypedSendAsync(connectionIds.ToList(), value); + TypedSend(connectionIds, value); + return Task.CompletedTask; } - public Task TypedSendAsync(string connectionId, object value) + public void TypedSend(string connectionId, object value) { - return _connectionManager.SendMessageAsync(connectionId, new TypedMessage { Type = GetMessageType(value), Message = value }); + var message = new TypedMessage { Type = GetMessageType(value), Message = value }; + bool canDrop = CanDrop(value); + _sseConnectionManager.SendMessage(connectionId, message, canDrop); + _webSocketConnectionManager.SendMessage(connectionId, message); } - public Task TypedSendAsync(IList connectionIds, object value) + public void TypedSend(IEnumerable connectionIds, object value) { - return _connectionManager.SendMessageAsync(connectionIds, new TypedMessage { Type = GetMessageType(value), Message = value }); + var message = new TypedMessage { Type = GetMessageType(value), Message = value }; + bool canDrop = CanDrop(value); + _sseConnectionManager.SendMessage(connectionIds, message, canDrop); + _webSocketConnectionManager.SendMessage(connectionIds, message); } - public Task TypedBroadcastAsync(object value) + public void TypedBroadcast(object value) { - return _connectionManager.SendMessageToAllAsync(new TypedMessage { Type = GetMessageType(value), Message = value }); + var message = new TypedMessage { Type = GetMessageType(value), Message = value }; + bool canDrop = CanDrop(value); + _sseConnectionManager.SendMessageToAll(message, canDrop); + _webSocketConnectionManager.SendMessageToAll(message); } private static string GetMessageType(object value) @@ -215,6 +234,11 @@ private static string GetMessageType(object value) return value.GetType().Name; } + + internal static bool CanDrop(object value) + { + return value is not (PlanOverage or ReleaseNotification or SystemNotification or UserMembershipChanged); + } } public record TypedMessage diff --git a/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs b/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs deleted file mode 100644 index 227a5372ae..0000000000 --- a/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs +++ /dev/null @@ -1,154 +0,0 @@ -using System.Net.WebSockets; -using System.Text; -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Utility; - -namespace Exceptionless.Web.Hubs; - -public class MessageBusBrokerMiddleware -{ - private readonly ILogger _logger; - private readonly WebSocketConnectionManager _connectionManager; - private readonly IConnectionMapping _connectionMapping; - private readonly RequestDelegate _next; - - public MessageBusBrokerMiddleware(RequestDelegate next, WebSocketConnectionManager connectionManager, IConnectionMapping connectionMapping, ILogger logger) - { - _next = next; - _connectionManager = connectionManager; - _connectionMapping = connectionMapping; - _logger = logger; - } - - public async Task Invoke(HttpContext context) - { - if (!context.WebSockets.IsWebSocketRequest || !context.User.IsAuthenticated()) - { - await _next(context); - return; - } - - using var socket = await context.WebSockets.AcceptWebSocketAsync(); - string connectionId = _connectionManager.AddWebSocket(socket); - await OnConnected(context, socket, connectionId); - bool disconnected = false; - - try - { - await ReceiveAsync(socket, async (result, data) => - { - switch (result.MessageType) - { - case WebSocketMessageType.Text: - _logger.LogTrace("WebSocket got message {ConnectionId}", connectionId); - // ignore incoming messages - return; - case WebSocketMessageType.Close: - await OnDisconnected(context, socket, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); - disconnected = true; - return; - } - }); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) { } - - // This will be hit when the connection is lost. - if (!disconnected) - { - await OnDisconnected(context, socket, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); - } - } - - private async Task OnConnected(HttpContext context, WebSocket socket, string connectionId) - { - _logger.LogTrace("WebSocket connected {ConnectionId} ({State})", connectionId, socket.State); - - try - { - foreach (string organizationId in context.User.GetOrganizationIds()) - await _connectionMapping.GroupAddAsync(organizationId, connectionId); - - string? userId = context.User.GetUserId(); - if (!String.IsNullOrEmpty(userId)) - await _connectionMapping.UserIdAddAsync(userId, connectionId); - } - catch (Exception ex) - { - _logger.LogError(ex, "OnConnected Error: {Message}", ex.Message); - throw; - } - } - - private async Task OnDisconnected(HttpContext context, WebSocket socket, string connectionId) - { - _logger.LogTrace("WebSocket disconnected {ConnectionId} ({State})", connectionId, socket.State); - - try - { - foreach (string organizationId in context.User.GetOrganizationIds()) - await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); - - string? userId = context.User.GetUserId(); - if (!String.IsNullOrEmpty(userId)) - await _connectionMapping.UserIdRemoveAsync(userId, connectionId); - } - catch (Exception ex) - { - _logger.LogError(ex, "OnDisconnected Error: {Message}", ex.Message); - throw; - } - } - - private async Task ReceiveAsync(WebSocket socket, Func handleMessage) - { - var buffer = new ArraySegment(new byte[1024 * 4]); - var result = await socket.ReceiveAsync(buffer, CancellationToken.None); - LogFrame(result, buffer.Array); - - while (!result.CloseStatus.HasValue) - { - string data; - - using (var ms = new MemoryStream()) - { - do - { - result = await socket.ReceiveAsync(buffer, CancellationToken.None); - LogFrame(result, buffer.Array); - - if (buffer.Array is not null) - await ms.WriteAsync(buffer.Array, buffer.Offset, result.Count); - } while (!result.EndOfMessage); - - ms.Seek(0, SeekOrigin.Begin); - - using (var reader = new StreamReader(ms, Encoding.UTF8)) - data = await reader.ReadToEndAsync(); - } - - await handleMessage(result, data); - } - } - - private void LogFrame(WebSocketReceiveResult frame, byte[]? buffer) - { - if (!_logger.IsEnabled(LogLevel.Debug)) - return; - - if (frame.CloseStatus.HasValue) - { - _logger.LogDebug("Close: {CloseStatus} {CloseStatusDescription}", frame.CloseStatus.Value, frame.CloseStatusDescription); - } - else - { - string? content = "<>"; - if (frame.MessageType == WebSocketMessageType.Text) - content = buffer is not null ? Encoding.UTF8.GetString(buffer, 0, frame.Count) : null; - - _logger.LogDebug("Received Frame {MessageType}: length={FrameCount}, end={FrameEndOfMessage}: {Content}", frame.MessageType, frame.Count, frame.EndOfMessage, content); - } - - } -} diff --git a/src/Exceptionless.Web/Hubs/PushConnectionLease.cs b/src/Exceptionless.Web/Hubs/PushConnectionLease.cs new file mode 100644 index 0000000000..112fb75a0b --- /dev/null +++ b/src/Exceptionless.Web/Hubs/PushConnectionLease.cs @@ -0,0 +1,97 @@ +using Exceptionless.Core.Utility; + +namespace Exceptionless.Web.Hubs; + +public sealed class PushConnectionLease : IAsyncDisposable +{ + internal static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(1); + internal static readonly TimeSpan RenewalInterval = TimeSpan.FromSeconds(15); + private readonly CancellationTokenSource _leaseLost = new(); + private readonly IConnectionLeaseStore _store; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _renewalTask; + private readonly string _userId; + private readonly string _connectionId; + private int _disposeState; + + public CancellationToken LeaseLost => _leaseLost.Token; + + private PushConnectionLease(IConnectionLeaseStore store, TimeProvider timeProvider, ILogger logger, string userId, string connectionId) + { + _store = store; + _timeProvider = timeProvider; + _logger = logger; + _userId = userId; + _connectionId = connectionId; + _renewalTask = RenewAsync(); + } + + public static async Task TryAcquireAsync(IConnectionLeaseStore store, TimeProvider timeProvider, ILogger logger, string userId, string connectionId, int maxConnections) + { + return await store.TryAcquireAsync(userId, connectionId, maxConnections, LeaseDuration).ConfigureAwait(false) + ? new PushConnectionLease(store, timeProvider, logger, userId, connectionId) + : null; + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + return; + + await _stop.CancelAsync().ConfigureAwait(false); + await _renewalTask.ConfigureAwait(false); + try + { + await _store.ReleaseAsync(_userId, _connectionId).ConfigureAwait(false); + } + catch (ConnectionLeaseStoreException ex) + { + // The lease expires without cleanup; a release failure must not mask request teardown. + _logger.LogWarning(ex, "Unable to release push lease {ConnectionId}", _connectionId); + } + finally + { + _stop.Dispose(); + _leaseLost.Dispose(); + } + } + + private async Task RenewAsync() + { + using var timer = new PeriodicTimer(RenewalInterval, _timeProvider); + DateTimeOffset lastSuccessfulRenewal = _timeProvider.GetUtcNow(); + + try + { + while (await timer.WaitForNextTickAsync(_stop.Token).ConfigureAwait(false)) + { + try + { + if (await _store.RenewAsync(_userId, _connectionId, LeaseDuration).ConfigureAwait(false)) + { + lastSuccessfulRenewal = _timeProvider.GetUtcNow(); + continue; + } + + _logger.LogWarning("Push lease was lost for {ConnectionId}", _connectionId); + await _leaseLost.CancelAsync().ConfigureAwait(false); + return; + } + catch (ConnectionLeaseStoreException ex) + { + _logger.LogWarning(ex, "Unable to renew push lease {ConnectionId}", _connectionId); + if (_timeProvider.GetUtcNow() - lastSuccessfulRenewal >= LeaseDuration) + { + await _leaseLost.CancelAsync().ConfigureAwait(false); + return; + } + } + } + } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) + { + } + } +} diff --git a/src/Exceptionless.Web/Hubs/PushConnectionRegistry.cs b/src/Exceptionless.Web/Hubs/PushConnectionRegistry.cs new file mode 100644 index 0000000000..5d8b626d89 --- /dev/null +++ b/src/Exceptionless.Web/Hubs/PushConnectionRegistry.cs @@ -0,0 +1,135 @@ +namespace Exceptionless.Web.Hubs; + +/// +/// Process-local ownership index. Redis pub/sub delivers every push message to every API +/// replica, so each replica only routes to and cleans up connections it actually owns. +/// +public sealed class PushConnectionRegistry(TimeProvider timeProvider) +{ + private static readonly TimeSpan RevocationTombstoneDuration = TimeSpan.FromMinutes(5); + private readonly Dictionary _connections = []; + private readonly Dictionary> _groupConnections = []; + private readonly Dictionary _revokedTokens = []; + private readonly Dictionary> _tokenConnections = []; + private readonly Dictionary> _userConnections = []; + private readonly object _lock = new(); + + public bool TryRegister(string connectionId, string userId, string? tokenId, IEnumerable organizationIds) + { + lock (_lock) + { + RemoveExpiredRevocations(); + if (tokenId is not null && _revokedTokens.ContainsKey(tokenId)) + return false; + + var registration = new Registration(userId, tokenId, organizationIds); + _connections.Add(connectionId, registration); + AddToIndex(_userConnections, userId, connectionId); + if (tokenId is not null) + AddToIndex(_tokenConnections, tokenId, connectionId); + foreach (string organizationId in registration.OrganizationIds) + AddToIndex(_groupConnections, organizationId, connectionId); + + return true; + } + } + + public IReadOnlyCollection RevokeToken(string tokenId) + { + lock (_lock) + { + RemoveExpiredRevocations(); + _revokedTokens[tokenId] = timeProvider.GetUtcNow() + RevocationTombstoneDuration; + return GetIndexedConnections(_tokenConnections, tokenId); + } + } + + public IReadOnlyCollection GetUserConnections(string userId) + { + lock (_lock) + return GetIndexedConnections(_userConnections, userId); + } + + public IReadOnlyCollection GetGroupConnections(string organizationId) + { + lock (_lock) + return GetIndexedConnections(_groupConnections, organizationId); + } + + public IReadOnlyCollection GetGroups(string connectionId) + { + lock (_lock) + return _connections.TryGetValue(connectionId, out var registration) ? registration.OrganizationIds.ToArray() : []; + } + + public void AddGroup(string connectionId, string organizationId) + { + lock (_lock) + { + if (_connections.TryGetValue(connectionId, out var registration) && registration.OrganizationIds.Add(organizationId)) + AddToIndex(_groupConnections, organizationId, connectionId); + } + } + + public void RemoveGroup(string connectionId, string organizationId) + { + lock (_lock) + { + if (_connections.TryGetValue(connectionId, out var registration) && registration.OrganizationIds.Remove(organizationId)) + RemoveFromIndex(_groupConnections, organizationId, connectionId); + } + } + + public void Unregister(string connectionId) + { + lock (_lock) + { + if (!_connections.Remove(connectionId, out var registration)) + return; + + RemoveFromIndex(_userConnections, registration.UserId, connectionId); + if (registration.TokenId is not null) + RemoveFromIndex(_tokenConnections, registration.TokenId, connectionId); + foreach (string organizationId in registration.OrganizationIds) + RemoveFromIndex(_groupConnections, organizationId, connectionId); + } + } + + private static void AddToIndex(Dictionary> index, string key, string connectionId) + { + if (!index.TryGetValue(key, out var connections)) + { + connections = []; + index[key] = connections; + } + + connections.Add(connectionId); + } + + private static string[] GetIndexedConnections(Dictionary> index, string key) + { + return index.TryGetValue(key, out var connections) ? connections.ToArray() : []; + } + + private static void RemoveFromIndex(Dictionary> index, string key, string connectionId) + { + if (!index.TryGetValue(key, out var connections)) + return; + + connections.Remove(connectionId); + if (connections.Count is 0) + index.Remove(key); + } + + private void RemoveExpiredRevocations() + { + var now = timeProvider.GetUtcNow(); + foreach (string tokenId in _revokedTokens.Where(pair => pair.Value <= now).Select(pair => pair.Key).ToArray()) + _revokedTokens.Remove(tokenId); + } + + private sealed record Registration(string UserId, string? TokenId, IEnumerable InitialOrganizationIds) + { + public HashSet OrganizationIds { get; } = [.. InitialOrganizationIds]; + } +} diff --git a/src/Exceptionless.Web/Hubs/SseConnection.cs b/src/Exceptionless.Web/Hubs/SseConnection.cs new file mode 100644 index 0000000000..8cf1d60ac1 --- /dev/null +++ b/src/Exceptionless.Web/Hubs/SseConnection.cs @@ -0,0 +1,269 @@ +using Foundatio.Serializer; + +namespace Exceptionless.Web.Hubs; + +/// +/// Represents a single SSE client connection. Owns a write loop that serializes +/// all sends through a bounded dedup queue, preventing concurrent writes to the +/// underlying HttpResponse stream. +/// +/// Design: delivery is best-effort. If a slow client fills the bounded queue, the +/// connection is aborted so reconnect handling performs a full cache resynchronization. +/// +/// Deduplication: messages with the same serialized payload are coalesced — if an +/// identical message is already queued, the newer duplicate is skipped. This reduces +/// redundant client refreshes during burst scenarios (e.g., rapid entity updates). +/// +public sealed class SseConnection : IAsyncDisposable +{ + private static readonly byte[] KeepAliveBytes = ": keepalive\n\n"u8.ToArray(); + private readonly HttpResponse _response; + private readonly ITextSerializer _serializer; + private readonly DedupQueue _queue; + private readonly CancellationTokenSource _cts; + private readonly CancellationToken _connectionAborted; + private readonly Task _writeLoop; + private readonly ILogger _logger; + private long _droppedMessages; + private long _dedupedMessages; + private int _disposeState; + + public string ConnectionId { get; } + public CancellationToken ConnectionAborted => _connectionAborted; + + /// Number of messages dropped due to backpressure (queue full). + public long DroppedMessages => Interlocked.Read(ref _droppedMessages); + + /// Number of messages skipped due to deduplication. + public long DedupedMessages => Interlocked.Read(ref _dedupedMessages); + + public SseConnection(string connectionId, HttpResponse response, ITextSerializer serializer, CancellationToken requestAborted, ILogger logger, int capacity = 64) + { + ConnectionId = connectionId; + _response = response; + _serializer = serializer; + _logger = logger; + _queue = new DedupQueue(capacity); + + _cts = CancellationTokenSource.CreateLinkedTokenSource(requestAborted); + _connectionAborted = _cts.Token; + _writeLoop = Task.Run(() => WriteLoopAsync(_cts.Token)); + } + + /// + /// Enqueue a message to be written. Returns false if the connection is closed. + /// If an identical message (same serialized payload) is already queued, the new + /// one is skipped (deduped) and this returns true. + /// + public bool TryWrite(object message, bool canDrop = true) + { + if (_cts.IsCancellationRequested) + return false; + + string data = _serializer.SerializeToString(message); + var result = _queue.TryEnqueue(new SseEvent { Data = data, DedupeKey = canDrop ? data : null }); + + if (result == EnqueueResult.Deduped) + { + Interlocked.Increment(ref _dedupedMessages); + return true; + } + + if (result is EnqueueResult.Full) + { + Interlocked.Increment(ref _droppedMessages); + Abort(); + return false; + } + + if (result is EnqueueResult.Closed) + return false; + + return true; + } + + /// + /// Send a keep-alive comment to prevent proxy/LB timeouts. + /// Keep-alives bypass dedup and are skipped when the data queue is full. + /// + public bool TryWriteKeepAlive() + { + if (_cts.IsCancellationRequested) + return false; + + return _queue.TryEnqueue(SseEvent.KeepAlive) is not EnqueueResult.Closed; + } + + /// + /// Abort the connection. The write loop will complete and the middleware will clean up. + /// + public void Abort() + { + try { _cts.Cancel(); } + catch (ObjectDisposedException ex) + { + _logger.LogDebug(ex, "SSE cancellation token source was already disposed for {ConnectionId}", ConnectionId); + } + + _queue.Complete(); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + return; + Abort(); + using (_queue) + using (_cts) + { + try + { + await _writeLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) { } + } + } + + private async Task WriteLoopAsync(CancellationToken ct) + { + try + { + while (!ct.IsCancellationRequested) + { + var evt = await _queue.DequeueAsync(ct); + if (evt is null) + break; // Queue completed + + var bytes = evt.Value.IsKeepAlive + ? KeepAliveBytes + : System.Text.Encoding.UTF8.GetBytes($"data: {evt.Value.Data}\n\n"); + + await _response.Body.WriteAsync(bytes, ct); + await _response.Body.FlushAsync(ct); + } + } + catch (OperationCanceledException) { } + catch (ObjectDisposedException) { } + catch (IOException) { } + finally + { + // Always signal ConnectionAborted so the middleware's Task.Delay unblocks + // and process-local registration and lease cleanup happens reliably. + _queue.Complete(); + if (!_cts.IsCancellationRequested) + { + try + { + _cts.Cancel(); + } + catch (ObjectDisposedException ex) + { + _logger.LogDebug(ex, "SSE cancellation token source was already disposed for {ConnectionId}", ConnectionId); + } + } + } + } + + internal readonly record struct SseEvent + { + public string? Data { get; init; } + + /// + /// Key used for deduplication. If null, no dedup is applied (e.g., keep-alive). + /// For data messages, this is the serialized payload — identical payloads trigger + /// the same client-side cache invalidation, so coalescing is safe. + /// + public string? DedupeKey { get; init; } + public bool IsKeepAlive { get; init; } + public static SseEvent KeepAlive => new() { IsKeepAlive = true }; + } + + internal enum EnqueueResult + { + Enqueued, + Deduped, + Full, + Closed + } + + /// + /// Bounded FIFO queue with deduplication. Thread-safe for multiple writers and a single reader. + /// When full, rejects new items so the owner can abort and force a reconnect resynchronization. + /// If an item with the same DedupeKey is already queued, the new item is skipped. + /// + internal sealed class DedupQueue : IDisposable + { + private readonly object _lock = new(); + private readonly LinkedList _list = new(); + private readonly Dictionary> _index = new(); + private readonly SemaphoreSlim _signal = new(0); + private readonly int _capacity; + private bool _completed; + + public DedupQueue(int capacity) + { + _capacity = capacity; + } + + public EnqueueResult TryEnqueue(SseEvent evt) + { + lock (_lock) + { + if (_completed) + return EnqueueResult.Closed; + + // Dedup check: if same key is already queued, skip + if (evt.DedupeKey is not null && _index.ContainsKey(evt.DedupeKey)) + return EnqueueResult.Deduped; + + if (_list.Count >= _capacity) + return EnqueueResult.Full; + + var node = _list.AddLast(evt); + if (evt.DedupeKey is not null) + _index[evt.DedupeKey] = node; + + _signal.Release(); + return EnqueueResult.Enqueued; + } + } + + public async Task DequeueAsync(CancellationToken ct) + { + await _signal.WaitAsync(ct); + + lock (_lock) + { + if (_list.Count == 0) + return null; // Completed + + var node = _list.First!; + RemoveNode(node); + return node.Value; + } + } + + public void Complete() + { + lock (_lock) + { + if (_completed) + return; + _completed = true; + _signal.Release(); // Wake up the reader so it sees null + } + } + + public void Dispose() + { + _signal.Dispose(); + } + + private void RemoveNode(LinkedListNode node) + { + _list.Remove(node); + if (node.Value.DedupeKey is not null) + _index.Remove(node.Value.DedupeKey); + } + } +} diff --git a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs new file mode 100644 index 0000000000..32b7072e6f --- /dev/null +++ b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs @@ -0,0 +1,211 @@ +using System.Collections.Concurrent; +using Exceptionless.Core; +using Foundatio.Serializer; + +namespace Exceptionless.Web.Hubs; + +/// +/// Manages active SSE connections. Replaces WebSocketConnectionManager. +/// Sends keep-alive comments every 15 seconds to prevent proxy/LB disconnects. +/// Proactively prunes dead connections during keep-alive sweeps. +/// +public sealed class SseConnectionManager : IDisposable, IAsyncDisposable +{ + private readonly ConcurrentDictionary _connections = new(); + private readonly ConcurrentDictionary> _pendingDisposals = new(); + private readonly Timer? _timer; + private readonly ITextSerializer _serializer; + private readonly ILogger _logger; + + /// + /// Maximum number of concurrent connections per user to prevent resource exhaustion. + /// + public int MaxConnectionsPerUser { get; init; } = 10; + + public SseConnectionManager(AppOptions options, ITextSerializer serializer, ILoggerFactory loggerFactory) + { + _serializer = serializer; + _logger = loggerFactory.CreateLogger(); + + if (!options.EnablePush) + return; + + _timer = new Timer(SendKeepAlive, null, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(15)); + } + + private void SendKeepAlive(object? state) + { + if (_connections.IsEmpty) + return; + + int sent = 0; + int pruned = 0; + + foreach (var (connectionId, connection) in _connections) + { + if (connection.ConnectionAborted.IsCancellationRequested) + { + TryRemove(connectionId); + pruned++; + continue; + } + + if (!connection.TryWriteKeepAlive()) + { + // Write failed — connection is dead, prune it + TryRemove(connectionId); + pruned++; + } + else + { + sent++; + } + } + + if (_logger.IsEnabled(LogLevel.Trace)) + _logger.LogTrace("SSE keep-alive: sent={SentCount}, pruned={PrunedCount}, active={ActiveCount}", sent, pruned, _connections.Count); + } + + public SseConnection? GetConnectionById(string connectionId) + { + return _connections.TryGetValue(connectionId, out var connection) ? connection : null; + } + + public ICollection GetAll() + { + return _connections.Values; + } + + public int ConnectionCount => _connections.Count; + + public SseConnection AddConnection(string connectionId, HttpResponse response, CancellationToken requestAborted) + { + var connection = new SseConnection(connectionId, response, _serializer, requestAborted, _logger); + if (!_connections.TryAdd(connectionId, connection)) + { + connection.DisposeAsync().AsTask().GetAwaiter().GetResult(); + throw new InvalidOperationException($"An SSE connection with id '{connectionId}' is already registered."); + } + AppDiagnostics.PushSseConnectionsOpened.Add(1); + AppDiagnostics.Gauge("push.connections.sse.active", _connections.Count); + return connection; + } + + public async Task RemoveConnectionAsync(string connectionId) + { + if (_connections.TryRemove(connectionId, out var connection)) + { + await DisposeConnectionAsync(connectionId, connection).ConfigureAwait(false); + return; + } + + if (_pendingDisposals.TryGetValue(connectionId, out var pendingDisposal)) + await pendingDisposal.Value.ConfigureAwait(false); + } + + private void TryRemove(string connectionId) + { + if (_connections.TryRemove(connectionId, out var connection)) + _ = ObserveDisposeAsync(connectionId, DisposeConnectionAsync(connectionId, connection)); + } + + public bool SendMessage(string connectionId, object message, bool canDrop = true) + { + if (!_connections.TryGetValue(connectionId, out var connection)) + return false; + + if (connection.ConnectionAborted.IsCancellationRequested) + { + TryRemove(connectionId); + return false; + } + + if (connection.TryWrite(message, canDrop)) + return true; + + TryRemove(connectionId); + return false; + } + + public void SendMessage(IEnumerable connectionIds, object message, bool canDrop = true) + { + foreach (string connectionId in connectionIds) + SendMessage(connectionId, message, canDrop); + } + + public void SendMessageToAll(object message, bool canDrop = true) + { + foreach (var (connectionId, connection) in _connections) + { + if (connection.ConnectionAborted.IsCancellationRequested) + { + TryRemove(connectionId); + continue; + } + + if (!connection.TryWrite(message, canDrop)) + TryRemove(connectionId); + } + } + + public void Dispose() + { + // Synchronous disposal: used by test hosts and non-async disposal paths. + // For production host shutdown, the DI container will prefer DisposeAsync(). + DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + + public async ValueTask DisposeAsync() + { + _timer?.Dispose(); + + var disposeTasks = new List(); + + foreach (var (connectionId, connection) in _connections) + { + if (_connections.TryRemove(connectionId, out var activeConnection)) + disposeTasks.Add(DisposeConnectionAsync(connectionId, activeConnection)); + } + + foreach (var pendingDisposal in _pendingDisposals.Values) + disposeTasks.Add(pendingDisposal.Value); + + if (disposeTasks.Count > 0) + await Task.WhenAll(disposeTasks).ConfigureAwait(false); + } + + private Task DisposeConnectionAsync(string connectionId, SseConnection connection) + { + var pendingDisposal = _pendingDisposals.GetOrAdd(connectionId, _ => new Lazy(() => DisposeConnectionCoreAsync(connectionId, connection))); + return pendingDisposal.Value; + } + + private async Task DisposeConnectionCoreAsync(string connectionId, SseConnection connection) + { + try + { + connection.Abort(); + await connection.DisposeAsync().ConfigureAwait(false); + } + finally + { + _pendingDisposals.TryRemove(connectionId, out _); + AppDiagnostics.PushSseConnectionsClosed.Add(1); + AppDiagnostics.Gauge("push.connections.sse.active", _connections.Count); + } + } + + private async Task ObserveDisposeAsync(string connectionId, Task disposeTask) + { + try + { + await disposeTask.ConfigureAwait(false); + } + catch (OperationCanceledException) { } + catch (ObjectDisposedException) { } + catch (InvalidOperationException ex) + { + _logger.LogDebug(ex, "SSE connection cleanup failed for {ConnectionId}", connectionId); + } + } +} diff --git a/src/Exceptionless.Web/Hubs/SseMiddleware.cs b/src/Exceptionless.Web/Hubs/SseMiddleware.cs new file mode 100644 index 0000000000..4a9bf422b0 --- /dev/null +++ b/src/Exceptionless.Web/Hubs/SseMiddleware.cs @@ -0,0 +1,117 @@ +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Utility; + +namespace Exceptionless.Web.Hubs; + +/// +/// Handles SSE connections at /api/v2/push. Replaces MessageBusBrokerMiddleware (WebSocket). +/// Accepts authenticated GET requests, sets SSE response headers, registers the connection +/// in the process-local ownership registry, and holds the response open until disconnect. +/// +public class SseMiddleware +{ + private static readonly PathString _sseEndpoint = new("/api/v2/push"); + private readonly ILogger _logger; + private readonly SseConnectionManager _connectionManager; + private readonly IConnectionLeaseStore _leaseStore; + private readonly PushConnectionRegistry _connectionRegistry; + private readonly TimeProvider _timeProvider; + private readonly IHostApplicationLifetime _applicationLifetime; + private readonly RequestDelegate _next; + + public SseMiddleware(RequestDelegate next, SseConnectionManager connectionManager, IConnectionLeaseStore leaseStore, PushConnectionRegistry connectionRegistry, TimeProvider timeProvider, IHostApplicationLifetime applicationLifetime, ILogger logger) + { + _next = next; + _connectionManager = connectionManager; + _leaseStore = leaseStore; + _connectionRegistry = connectionRegistry; + _timeProvider = timeProvider; + _applicationLifetime = applicationLifetime; + _logger = logger; + } + + public async Task Invoke(HttpContext context) + { + if (!context.Request.Path.StartsWithSegments(_sseEndpoint, StringComparison.Ordinal) + || !HttpMethods.IsGet(context.Request.Method) + || context.WebSockets.IsWebSocketRequest) + { + await _next(context); + return; + } + + if (!context.User.IsAuthenticated()) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + string? userId = context.User.GetUserId(); + if (String.IsNullOrEmpty(userId)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + string connectionId = Guid.NewGuid().ToString("N"); + PushConnectionLease? lease; + try + { + lease = await PushConnectionLease.TryAcquireAsync(_leaseStore, _timeProvider, _logger, userId, connectionId, _connectionManager.MaxConnectionsPerUser).ConfigureAwait(false); + } + catch (ConnectionLeaseStoreException ex) + { + _logger.LogError(ex, "Push lease store is unavailable"); + context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; + return; + } + + if (lease is null) + { + _logger.LogWarning("User {UserId} exceeded max SSE connections ({Max})", userId, _connectionManager.MaxConnectionsPerUser); + context.Response.StatusCode = StatusCodes.Status429TooManyRequests; + return; + } + + await using (lease) + using (var connectionLifetime = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted, lease.LeaseLost, _applicationLifetime.ApplicationStopping)) + { + if (!_connectionRegistry.TryRegister(connectionId, userId, context.User.GetLoggedInUsersTokenId(), context.User.GetOrganizationIds())) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + SseConnection? connection = null; + + try + { + // Set SSE response headers + context.Response.Headers.ContentType = "text/event-stream"; + context.Response.Headers.CacheControl = "no-cache, no-store"; + context.Response.Headers["X-Accel-Buffering"] = "no"; // nginx + + // Disable response buffering + var bufferingFeature = context.Features.Get(); + bufferingFeature?.DisableBuffering(); + + connection = _connectionManager.AddConnection(connectionId, context.Response, connectionLifetime.Token); + _logger.LogTrace("SSE connected {ConnectionId}", connectionId); + + // Send initial connected event + connection.TryWrite(new { type = "Connected", message = new { connection_id = connectionId } }); + + // Hold the response open until the client disconnects or the connection is aborted + await Task.Delay(Timeout.Infinite, connection.ConnectionAborted).ConfigureAwait(false); + } + catch (OperationCanceledException) { } + finally + { + _logger.LogTrace("SSE disconnected {ConnectionId}", connectionId); + if (connection is not null) + await _connectionManager.RemoveConnectionAsync(connectionId).ConfigureAwait(false); + _connectionRegistry.Unregister(connectionId); + } + } + } +} diff --git a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs index dfdd8747de..1155b90915 100644 --- a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Net.WebSockets; using System.Text; using Exceptionless.Core; @@ -6,189 +6,261 @@ namespace Exceptionless.Web.Hubs; -public class WebSocketConnectionManager : IDisposable +/// +/// Temporary WebSocket compatibility layer for the Angular rollout. Remove once the +/// SSE rollout is complete and the websocket active-connection gauge remains at zero. +/// +public sealed class WebSocketConnectionManager : IDisposable { - private static readonly ArraySegment _keepAliveMessage = new(Encoding.ASCII.GetBytes("{}"), 0, 2); - private readonly ConcurrentDictionary _connections = new(); + private static readonly ArraySegment KeepAliveMessage = new(Encoding.ASCII.GetBytes("{}"), 0, 2); + private readonly ConcurrentDictionary _connections = new(); private readonly Timer? _timer; private readonly ITextSerializer _serializer; private readonly ILogger _logger; + public int MaxConnectionsPerUser { get; init; } = 10; + public int ConnectionCount => _connections.Count; + public WebSocketConnectionManager(AppOptions options, ITextSerializer serializer, ILoggerFactory loggerFactory) { _serializer = serializer; _logger = loggerFactory.CreateLogger(); - if (!options.EnableWebSockets) + if (!options.EnablePush) return; - _timer = new Timer(KeepAlive, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); + _timer = new Timer(SendKeepAlive, null, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(15)); } - private void KeepAlive(object? state) + private void SendKeepAlive(object? state) { - if (_connections is { IsEmpty: true, Count: 0 }) + if (_connections.IsEmpty) return; - Task.Factory.StartNew(async () => + foreach (var (connectionId, connection) in _connections) { - var sockets = GetAll(); - var openSockets = sockets.Where(s => s.State == WebSocketState.Open).ToArray(); - _logger.LogTrace("Sending web socket keep alive to {OpenSocketsCount} open connections of {SocketCount} total connections", openSockets.Length, sockets.Count); - - foreach (var socket in openSockets) + if (!CanSend(connection.Socket)) { - try - { - await socket.SendAsync(buffer: _keepAliveMessage, - messageType: WebSocketMessageType.Text, - endOfMessage: true, - cancellationToken: CancellationToken.None); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) - { - // NOTE: This will not remove it from the ConnectionMappings. - await RemoveWebSocketAsync(socket); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error sending keep alive socket message: {Message}", ex.Message); - } + _ = RemoveConnectionAsync(connectionId); + continue; } - }); + + _ = SendKeepAliveAsync(connectionId, connection); + } + } + + public WebSocket? GetConnectionById(string connectionId) + { + return _connections.TryGetValue(connectionId, out var connection) ? connection.Socket : null; } public WebSocket? GetWebSocketById(string connectionId) { - return _connections.TryGetValue(connectionId, out var socket) ? socket : null; + return GetConnectionById(connectionId); } public ICollection GetAll() { - return _connections.Values; + return _connections.Values.Select(connection => connection.Socket).ToArray(); } public string GetConnectionId(WebSocket socket) { - return _connections.FirstOrDefault(p => p.Value == socket).Key; + return _connections.FirstOrDefault(pair => pair.Value.Socket == socket).Key; } - public string AddWebSocket(WebSocket socket) + public string AddConnection(WebSocket socket) { string connectionId = Guid.NewGuid().ToString("N"); - _connections.TryAdd(connectionId, socket); - return connectionId; + return AddConnection(connectionId, socket); } - private Task RemoveWebSocketAsync(WebSocket socket) + public string AddConnection(string connectionId, WebSocket socket) { - string id = GetConnectionId(socket); - if (String.IsNullOrEmpty(id) || !_connections.TryRemove(id, out var _)) - return Task.CompletedTask; + if (!_connections.TryAdd(connectionId, new ManagedWebSocket(socket))) + throw new InvalidOperationException($"A websocket connection with id '{connectionId}' is already registered."); - return CloseWebSocketAsync(socket); + AppDiagnostics.PushWebSocketConnectionsOpened.Add(1); + AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); + return connectionId; } - public Task RemoveWebSocketAsync(string id) + public string AddWebSocket(WebSocket socket) { - if (!_connections.TryRemove(id, out var socket)) - return Task.CompletedTask; - - return CloseWebSocketAsync(socket); + return AddConnection(socket); } - private async Task CloseWebSocketAsync(WebSocket socket) + public async Task RemoveConnectionAsync(string connectionId) { - if (!CanSendWebSocketMessage(socket)) + if (!_connections.TryRemove(connectionId, out var connection)) return; - try + var socket = connection.Socket; + + if (!CanClose(socket)) { - await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closed by manager", CancellationToken.None); + AppDiagnostics.PushWebSocketConnectionsClosed.Add(1); + AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); + return; } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) + + await connection.SendLock.WaitAsync().ConfigureAwait(false); + try { - // Ignored + await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closed by manager", CancellationToken.None).ConfigureAwait(false); } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) { } + catch (ObjectDisposedException) { } catch (Exception ex) { - _logger.LogError(ex, "Error closing web socket: {Message}", ex.Message); + _logger.LogDebug(ex, "Error closing websocket {ConnectionId}", connectionId); } + finally + { + AppDiagnostics.PushWebSocketConnectionsClosed.Add(1); + AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); + connection.SendLock.Release(); + } + } + + public Task RemoveWebSocketAsync(string connectionId) + { + return RemoveConnectionAsync(connectionId); } - private Task SendMessageAsync(WebSocket socket, object message) + public bool SendMessage(string connectionId, object message) { - if (!CanSendWebSocketMessage(socket)) - return Task.CompletedTask; + if (!_connections.TryGetValue(connectionId, out var connection)) + return false; - string serializedMessage = _serializer.SerializeToString(message); - Task.Factory.StartNew(async () => + if (!CanSend(connection.Socket)) { - if (!CanSendWebSocketMessage(socket)) - return; + _ = RemoveConnectionAsync(connectionId); + return false; + } - try - { - byte[] bytes = Encoding.UTF8.GetBytes(serializedMessage); - await socket.SendAsync(buffer: new ArraySegment(bytes, 0, bytes.Length), - messageType: WebSocketMessageType.Text, - endOfMessage: true, - cancellationToken: CancellationToken.None); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) - { - // Ignored - } - catch (Exception ex) - { - _logger.LogError(ex, "Error sending socket message: {Message}", ex.Message); - } - }); + _ = SendMessageAsync(connectionId, connection, message); + return true; + } + public Task SendMessageAsync(string connectionId, object message) + { + SendMessage(connectionId, message); return Task.CompletedTask; } - public Task SendMessageAsync(string connectionId, object message) + public void SendMessage(IEnumerable connectionIds, object message) { - var socket = GetWebSocketById(connectionId); - return socket is not null ? SendMessageAsync(socket, message) : Task.CompletedTask; + foreach (var connectionId in connectionIds) + SendMessage(connectionId, message); } public Task SendMessageAsync(IEnumerable connectionIds, object message) { - return Task.WhenAll(connectionIds.Select(id => - { - var socket = GetWebSocketById(id); - return socket is not null ? SendMessageAsync(socket, message) : Task.CompletedTask; - })); + SendMessage(connectionIds, message); + return Task.CompletedTask; } - public async Task SendMessageToAllAsync(object message, bool throwOnError = true) + public void SendMessageToAll(object message) { - foreach (var socket in GetAll()) + foreach (var (connectionId, connection) in _connections) { - if (!CanSendWebSocketMessage(socket)) - continue; - - try + if (!CanSend(connection.Socket)) { - await SendMessageAsync(socket, message); - } - catch (Exception) - { - if (throwOnError) - throw; + _ = RemoveConnectionAsync(connectionId); + continue; } + + _ = SendMessageAsync(connectionId, connection, message); } } - private bool CanSendWebSocketMessage(WebSocket socket) + public Task SendMessageToAllAsync(object message, bool throwOnError = true) { - return socket.State != WebSocketState.Aborted && socket.State != WebSocketState.Closed && socket.State != WebSocketState.CloseSent; + SendMessageToAll(message); + return Task.CompletedTask; } public void Dispose() { _timer?.Dispose(); } + + private Task SendKeepAliveAsync(string connectionId, ManagedWebSocket connection) + { + return SendAsync(connectionId, connection, KeepAliveMessage, "keepalive"); + } + + private async Task SendAsync(string connectionId, ManagedWebSocket connection, ArraySegment bytes, string operation) + { + await connection.SendLock.WaitAsync().ConfigureAwait(false); + bool removeConnection = false; + try + { + if (CanSend(connection.Socket)) + await connection.Socket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None).ConfigureAwait(false); + } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) + { + removeConnection = true; + } + catch (ObjectDisposedException) + { + removeConnection = true; + } + catch (WebSocketException ex) + { + _logger.LogDebug(ex, "Error sending websocket {Operation} for {ConnectionId}", operation, connectionId); + } + finally + { + connection.SendLock.Release(); + } + + if (removeConnection) + await RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + + private Task SendMessageAsync(string connectionId, ManagedWebSocket connection, object message) + { + try + { + string serializedMessage = _serializer.SerializeToString(message); + byte[] bytes = Encoding.UTF8.GetBytes(serializedMessage); + return SendAsync(connectionId, connection, new ArraySegment(bytes), "message"); + } + catch (InvalidOperationException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (OperationCanceledException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (NotSupportedException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (EncoderFallbackException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + + return Task.CompletedTask; + } + + private static bool CanSend(WebSocket socket) + { + return socket.State is WebSocketState.Open; + } + + private static bool CanClose(WebSocket socket) + { + return socket.State is WebSocketState.Open or WebSocketState.CloseReceived; + } + + private sealed record ManagedWebSocket(WebSocket Socket) + { + public SemaphoreSlim SendLock { get; } = new(1, 1); + } } diff --git a/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs new file mode 100644 index 0000000000..69fcf8ab13 --- /dev/null +++ b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs @@ -0,0 +1,132 @@ +using System.Net.WebSockets; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Utility; + +namespace Exceptionless.Web.Hubs; + +/// +/// Temporary WebSocket endpoint compatibility for the Angular rollout. Keep this in place +/// until all clients are on SSE and websocket active connections stay at zero. +/// +public sealed class WebSocketPushMiddleware +{ + private static readonly PathString PushEndpoint = new("/api/v2/push"); + private readonly ILogger _logger; + private readonly WebSocketConnectionManager _connectionManager; + private readonly IConnectionLeaseStore _leaseStore; + private readonly PushConnectionRegistry _connectionRegistry; + private readonly TimeProvider _timeProvider; + private readonly IHostApplicationLifetime _applicationLifetime; + private readonly RequestDelegate _next; + + public WebSocketPushMiddleware(RequestDelegate next, WebSocketConnectionManager connectionManager, IConnectionLeaseStore leaseStore, PushConnectionRegistry connectionRegistry, TimeProvider timeProvider, IHostApplicationLifetime applicationLifetime, ILogger logger) + { + _next = next; + _connectionManager = connectionManager; + _leaseStore = leaseStore; + _connectionRegistry = connectionRegistry; + _timeProvider = timeProvider; + _applicationLifetime = applicationLifetime; + _logger = logger; + } + + public async Task Invoke(HttpContext context) + { + if (!context.Request.Path.StartsWithSegments(PushEndpoint, StringComparison.Ordinal) + || !context.WebSockets.IsWebSocketRequest) + { + await _next(context); + return; + } + + if (!context.User.IsAuthenticated()) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + string? userId = context.User.GetUserId(); + if (String.IsNullOrEmpty(userId)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + string connectionId = Guid.NewGuid().ToString("N"); + PushConnectionLease? lease; + try + { + lease = await PushConnectionLease.TryAcquireAsync(_leaseStore, _timeProvider, _logger, userId, connectionId, _connectionManager.MaxConnectionsPerUser).ConfigureAwait(false); + } + catch (ConnectionLeaseStoreException ex) + { + _logger.LogError(ex, "Push lease store is unavailable"); + context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; + return; + } + + if (lease is null) + { + _logger.LogWarning("User {UserId} exceeded max websocket push connections ({Max})", userId, _connectionManager.MaxConnectionsPerUser); + context.Response.StatusCode = StatusCodes.Status429TooManyRequests; + return; + } + + await using (lease) + using (var connectionLifetime = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted, lease.LeaseLost, _applicationLifetime.ApplicationStopping)) + { + if (!_connectionRegistry.TryRegister(connectionId, userId, context.User.GetLoggedInUsersTokenId(), context.User.GetOrganizationIds())) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + WebSocket? socket = null; + + try + { + socket = await context.WebSockets.AcceptWebSocketAsync(); + _connectionManager.AddConnection(connectionId, socket); + _logger.LogTrace("WebSocket push connected {ConnectionId}", connectionId); + await ReceiveUntilCloseAsync(socket, connectionLifetime.Token).ConfigureAwait(false); + } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) + { + _logger.LogDebug(ex, "WebSocket push closed prematurely for {ConnectionId}", connectionId); + } + catch (OperationCanceledException ex) + { + _logger.LogDebug(ex, "WebSocket push canceled for {ConnectionId}", connectionId); + } + finally + { + _logger.LogTrace("WebSocket push disconnected {ConnectionId}", connectionId); + try + { + await _connectionManager.RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + finally + { + socket?.Dispose(); + _connectionRegistry.Unregister(connectionId); + } + } + } + } + + private static async Task ReceiveUntilCloseAsync(WebSocket socket, CancellationToken cancellationToken) + { + var buffer = new byte[4096]; + + while (socket.State is WebSocketState.Open) + { + WebSocketReceiveResult result; + do + { + result = await socket.ReceiveAsync(new ArraySegment(buffer), cancellationToken).ConfigureAwait(false); + if (result.MessageType is WebSocketMessageType.Close) + return; + } while (!result.EndOfMessage); + } + } +} diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index 11a0fd0f57..955d9a3d91 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -74,6 +74,12 @@ public static IHostBuilder CreateHostBuilder(IConfigurationRoot config, string e var builder = Host.CreateDefaultBuilder() .UseEnvironment(environment) + .ConfigureHostOptions(o => + { + // Align with k8s terminationGracePeriodSeconds (60s) minus preStop sleep (15s). + // Keep a five-second safety margin for kubelet/runtime termination overhead. + o.ShutdownTimeout = TimeSpan.FromSeconds(40); + }) .ConfigureLogging(b => b.ClearProviders()) // clears .net providers since we are telling serilog to write to providers we only want it to be the otel provider .UseSerilog((ctx, sp, c) => { diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index e856f08b37..e923fa9e48 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -318,10 +318,11 @@ ApplicationException applicationException when applicationException.Message.Cont // Reject event posts in organizations over their max event limits. app.UseMiddleware(); - if (options.EnableWebSockets) + if (options.EnablePush) { app.UseWebSockets(); - app.UseMiddleware(); + app.UseMiddleware(); + app.UseMiddleware(); } app.UseEndpoints(endpoints => diff --git a/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs b/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs index ee7722b814..4c1d69ec1a 100644 --- a/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs +++ b/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs @@ -21,7 +21,7 @@ public class ThrottlingMiddleware private static readonly PathString _v1ProjectConfigPath = new("/api/v1/project/config"); private static readonly PathString _v2ProjectConfigPath = new("/api/v2/projects/config"); private static readonly PathString _heartbeatPath = new("/api/v2/events/session/heartbeat"); - private static readonly PathString _webSocketPath = new("/api/v2/push"); + private static readonly PathString _ssePath = new("/api/v2/push"); public ThrottlingMiddleware(RequestDelegate next, ICacheClient cacheClient, ThrottlingOptions options, TimeProvider timeProvider) @@ -111,7 +111,7 @@ private bool IsUnthrottledRoute(HttpContext context) return context.Request.Path.StartsWithSegments(_v2ProjectConfigPath, StringComparison.Ordinal) || context.Request.Path.StartsWithSegments(_heartbeatPath, StringComparison.Ordinal) - || context.Request.Path.StartsWithSegments(_webSocketPath, StringComparison.Ordinal) + || context.Request.Path.StartsWithSegments(_ssePath, StringComparison.Ordinal) || context.Request.Path.StartsWithSegments(_v1ProjectConfigPath, StringComparison.Ordinal); } } diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index 6ba9596517..1fa8cdac7f 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Net; +using System.Text.Json; using Aspire.Hosting; using Aspire.Hosting.Testing; using Exceptionless.Insulation.Configuration; @@ -13,6 +14,8 @@ namespace Exceptionless.Tests; public class AppWebHostFactory : WebApplicationFactory, IAsyncLifetime { private const string SharedElasticsearchUrl = "http://localhost:9200"; + private static readonly string[] s_indexPrefixes = ["events", "migrations", "organizations", "projects", "saved-views", "stacks", "tokens", "users", "webhooks"]; + private static readonly string s_runScope = $"test-{Guid.NewGuid().ToString("N")[..8]}"; private static int s_counter = -1; private static readonly Lazy> s_sharedAppHost = new(StartSharedAppHostAsync, LazyThreadSafetyMode.ExecutionAndPublication); private static readonly ConcurrentQueue s_pool = new(); @@ -24,17 +27,26 @@ public AppWebHostFactory() instanceId = Interlocked.Increment(ref s_counter); InstanceId = instanceId; - AppScope = instanceId == 0 ? "test" : $"test-{instanceId}"; + AppScope = instanceId == 0 ? s_runScope : $"{s_runScope}-{instanceId}"; } public string AppScope { get; } public int InstanceId { get; } public bool IndexesHaveBeenConfigured { get; set; } + public async Task GetRedisConnectionStringAsync(CancellationToken cancellationToken) + { + var app = await s_sharedAppHost.Value; + return await app.GetConnectionStringAsync("Redis", cancellationToken) + ?? throw new InvalidOperationException("Redis did not expose a connection string."); + } + public async ValueTask InitializeAsync() { _ = await s_sharedAppHost.Value; - await WaitForElasticsearchAsync(new Uri(SharedElasticsearchUrl)); + var elasticsearchUri = new Uri(SharedElasticsearchUrl); + await WaitForElasticsearchAsync(elasticsearchUri); + await CleanupElasticsearchSliceAsync(elasticsearchUri); } private static async Task StartSharedAppHostAsync() @@ -74,6 +86,40 @@ private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) throw new TimeoutException("Timed out waiting for the shared Elasticsearch container to be ready."); } + private async Task CleanupElasticsearchSliceAsync(Uri elasticsearchUri) + { + await WaitForElasticsearchAsync(elasticsearchUri); + + using var client = new HttpClient + { + BaseAddress = elasticsearchUri, + Timeout = TimeSpan.FromSeconds(10) + }; + + foreach (string pattern in s_indexPrefixes.Select(prefix => Uri.EscapeDataString($"{AppScope}-{prefix}*"))) + { + using var listResponse = await client.GetAsync($"/_cat/indices/{pattern}?h=index&format=json&expand_wildcards=all"); + if (listResponse.StatusCode == HttpStatusCode.NotFound) + continue; + + listResponse.EnsureSuccessStatusCode(); + + string payloadJson = await listResponse.Content.ReadAsStringAsync(); + var payload = JsonSerializer.Deserialize>(payloadJson, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }) + ?? []; + + foreach (string indexName in payload.Select(record => record.Index).Where(name => !String.IsNullOrEmpty(name)).Distinct()) + { + using var deleteResponse = await client.DeleteAsync($"/{Uri.EscapeDataString(indexName)}?ignore_unavailable=true"); + if (deleteResponse.StatusCode != HttpStatusCode.NotFound) + deleteResponse.EnsureSuccessStatusCode(); + } + } + } + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseSolutionRelativeContentRoot("src/Exceptionless.Web", "*.slnx"); @@ -94,14 +140,24 @@ protected override IHostBuilder CreateHostBuilder() return Web.Program.CreateHostBuilder(config, Environments.Development); } - public override ValueTask DisposeAsync() + public override async ValueTask DisposeAsync() { - if (!_sliceReleased) + try { - s_pool.Enqueue(InstanceId); - _sliceReleased = true; + await base.DisposeAsync(); } + finally + { + if (!_sliceReleased) + { + s_pool.Enqueue(InstanceId); + _sliceReleased = true; + } + } + } - return base.DisposeAsync(); + private sealed class CatIndexRecord + { + public string Index { get; set; } = String.Empty; } } diff --git a/tests/Exceptionless.Tests/Configuration/AppOptionsPushTests.cs b/tests/Exceptionless.Tests/Configuration/AppOptionsPushTests.cs new file mode 100644 index 0000000000..28ec251a56 --- /dev/null +++ b/tests/Exceptionless.Tests/Configuration/AppOptionsPushTests.cs @@ -0,0 +1,39 @@ +using Exceptionless.Core; +using Xunit; + +namespace Exceptionless.Tests.Configuration; + +public sealed class AppOptionsPushTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ReadFromConfiguration_LegacyEnableWebSockets_ControlsPush(bool enabled) + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["BaseURL"] = "http://localhost", + ["EnableWebSockets"] = enabled.ToString() + }).Build(); + + var options = AppOptions.ReadFromConfiguration(configuration); + + Assert.Equal(enabled, options.EnablePush); +#pragma warning disable CS0618 + Assert.Equal(enabled, options.EnableWebSockets); +#pragma warning restore CS0618 + } + + [Fact] + public void ReadFromConfiguration_EnablePush_TakesPrecedenceOverLegacySetting() + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["BaseURL"] = "http://localhost", + ["EnablePush"] = "false", + ["EnableWebSockets"] = "true" + }).Build(); + + Assert.False(AppOptions.ReadFromConfiguration(configuration).EnablePush); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs b/tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs new file mode 100644 index 0000000000..3a48a8c642 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs @@ -0,0 +1,45 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; + +namespace Exceptionless.Tests.Hubs; + +/// +/// A minimal fake HttpResponse for testing SSE connections. +/// Captures written data in a MemoryStream. +/// The WriteAsync extension on HttpResponse writes to Body directly, +/// so the MemoryStream captures all output. +/// +internal sealed class FakeHttpResponse : HttpResponse, IDisposable +{ + private readonly MemoryStream _body = new(); + private readonly HeaderDictionary _headers = new(); + + public override HttpContext HttpContext => null!; + public override int StatusCode { get; set; } + public override IHeaderDictionary Headers => _headers; + public override Stream Body + { + get => _body; + set { } + } + public override long? ContentLength { get; set; } + public override string? ContentType { get; set; } + public override IResponseCookies Cookies => null!; + public override bool HasStarted => true; + + /// + /// Get all data written to this response as a string. + /// + public string WrittenData => Encoding.UTF8.GetString(_body.ToArray()); + + public override void OnCompleted(Func callback, object state) { } + public override void OnStarting(Func callback, object state) { } + public override void Redirect(string location, bool permanent) { } + + public void Dispose() + { + _body.Dispose(); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/PushConnectionRegistryTests.cs b/tests/Exceptionless.Tests/Hubs/PushConnectionRegistryTests.cs new file mode 100644 index 0000000000..b77ccc366c --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/PushConnectionRegistryTests.cs @@ -0,0 +1,47 @@ +using Exceptionless.Web.Hubs; +using Exceptionless.Tests.Utility; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +public sealed class PushConnectionRegistryTests +{ + [Fact] + public void TryRegister_RevocationArrivesFirst_RejectsInFlightConnection() + { + var registry = new PushConnectionRegistry(new ProxyTimeProvider()); + + Assert.Empty(registry.RevokeToken("token")); + Assert.False(registry.TryRegister("connection", "user", "token", ["organization"])); + } + + [Fact] + public void RevokeToken_RegisteredConnection_ReturnsOnlyLocalTokenConnections() + { + var registry = new PushConnectionRegistry(new ProxyTimeProvider()); + Assert.True(registry.TryRegister("target", "user", "token", ["organization"])); + Assert.True(registry.TryRegister("other", "user", "other-token", ["organization"])); + + Assert.Equal(["target"], registry.RevokeToken("token")); + } + + [Fact] + public void MembershipAndUnregister_KeepRoutingIndexesConsistent() + { + var registry = new PushConnectionRegistry(new ProxyTimeProvider()); + Assert.True(registry.TryRegister("connection", "user", "token", ["first"])); + + registry.AddGroup("connection", "second"); + registry.RemoveGroup("connection", "first"); + + Assert.Equal(["connection"], registry.GetUserConnections("user")); + Assert.Empty(registry.GetGroupConnections("first")); + Assert.Equal(["connection"], registry.GetGroupConnections("second")); + + registry.Unregister("connection"); + + Assert.Empty(registry.GetUserConnections("user")); + Assert.Empty(registry.GetGroupConnections("second")); + Assert.Empty(registry.RevokeToken("token")); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs b/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs new file mode 100644 index 0000000000..55f73f9bdc --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs @@ -0,0 +1,214 @@ +using System.Net; +using System.Text; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Hubs; +using Exceptionless.Web.Models; +using Foundatio.Messaging; +using Foundatio.Repositories.Models; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +/// +/// Integration tests for the SSE endpoint (/api/v2/push). +/// These test the full HTTP pipeline including auth, middleware, and message delivery. +/// +public sealed class SseIntegrationTests : IntegrationTestsBase +{ + private readonly IMessagePublisher _messagePublisher; + + public SseIntegrationTests(ITestOutputHelper output, AppWebHostFactory factory) + : base(output, factory) + { + _messagePublisher = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task ConnectWithValidToken_ReturnsEventStream() + { + var token = await CreateTokenAsync(); + + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", $"Bearer {token}"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/event-stream", response.Content.Headers.ContentType?.MediaType); + } + + [Fact] + public async Task ConnectWithoutAuth_Returns401() + { + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, cts.Token); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ConnectWithInvalidToken_Returns401() + { + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", "Bearer invalid-token-xyz"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, cts.Token); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ConnectWithAccessTokenQueryParam_Succeeds() + { + var token = await CreateTokenAsync(); + + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v2/push?access_token={token}"); + request.Headers.Add("Accept", "text/event-stream"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ConnectedClient_ReceivesEntityChangedMessage() + { + var token = await CreateTokenAsync(); + var orgId = SampleDataService.TEST_ORG_ID; + + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", $"Bearer {token}"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var stream = await response.Content.ReadAsStreamAsync(cts.Token); + using var reader = new StreamReader(stream, Encoding.UTF8); + + // Read the initial "Connected" event + string? connectedEvent = await ReadSseEventAsync(reader, cts.Token); + Assert.NotNull(connectedEvent); + Assert.Contains("Connected", connectedEvent); + + // Publish an EntityChanged message to the organization + var entityChanged = new EntityChanged + { + Id = "stack-123", + Type = "Stack", + ChangeType = ChangeType.Saved + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = orgId; +#pragma warning disable xUnit1051 + await _messagePublisher.PublishAsync(entityChanged); +#pragma warning restore xUnit1051 + + // Wait for and read the message + string? receivedEvent = await ReadSseEventAsync(reader, cts.Token); + Assert.NotNull(receivedEvent); + Assert.Contains("StackChanged", receivedEvent); + Assert.Contains("stack-123", receivedEvent); + } + + [Fact] + public async Task SseEndpoint_IsExemptFromThrottling() + { + var token = await CreateTokenAsync(); + + using var client = _server.CreateClient(); + + // Make multiple SSE connection attempts - should not be throttled + for (int i = 0; i < 5; i++) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", $"Bearer {token}"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(3)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + // Should never be 429 + Assert.NotEqual(HttpStatusCode.TooManyRequests, response.StatusCode); + } + } + + /// + /// Read a single SSE event (terminated by double newline) from the stream. + /// + private static async Task ReadSseEventAsync(StreamReader reader, CancellationToken ct) + { + var sb = new StringBuilder(); + int emptyLineCount = 0; + + while (!ct.IsCancellationRequested) + { + string? line = await reader.ReadLineAsync(ct); + if (line is null) + return sb.Length > 0 ? sb.ToString() : null; + + if (line.Length == 0) + { + emptyLineCount++; + if (emptyLineCount >= 1 && sb.Length > 0) + return sb.ToString(); + continue; + } + + // Skip comments (keep-alive) + if (line.StartsWith(':')) + continue; + + emptyLineCount = 0; + sb.AppendLine(line); + } + + return sb.Length > 0 ? sb.ToString() : null; + } + + private async Task CreateTokenAsync() + { + var result = await SendRequestAsAsync(r => r + .Post() + .AppendPath("auth/login") + .Content(new Login + { + Email = SampleDataService.TEST_USER_EMAIL, + Password = SampleDataService.TEST_USER_PASSWORD + }) + .StatusCodeShouldBeOk() + ); + + return result?.Token ?? throw new InvalidOperationException("Login did not return a token."); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/SseTests.cs b/tests/Exceptionless.Tests/Hubs/SseTests.cs new file mode 100644 index 0000000000..e30003281a --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/SseTests.cs @@ -0,0 +1,684 @@ +using Exceptionless.Core; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Web.Hubs; +using Foundatio.Repositories.Models; +using Foundatio.Serializer; +using System.Security.Claims; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +public sealed class SseConnectionManagerTests : TestWithServices +{ + public SseConnectionManagerTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void AddConnection_NewConnection_CanLookupAndEnumerate() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-1"; + var connection = manager.AddConnection(connectionId, response, cts.Token); + + Assert.NotNull(connection); + Assert.Same(connection, manager.GetConnectionById(connectionId)); + Assert.Equal(1, manager.ConnectionCount); + Assert.Contains(connection, manager.GetAll()); + } + + [Fact] + public async Task RemoveConnectionAsync_ExistingConnection_RemovesAndAborts() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-2"; + var connection = manager.AddConnection(connectionId, response, cts.Token); + + await manager.RemoveConnectionAsync(connectionId); + + Assert.Null(manager.GetConnectionById(connectionId)); + Assert.Equal(0, manager.ConnectionCount); + Assert.True(connection.ConnectionAborted.IsCancellationRequested); + } + + [Fact] + public async Task RemoveConnectionAsync_UnknownConnection_DoesNothing() + { + using var manager = CreateManager(); + + await manager.RemoveConnectionAsync("nonexistent"); + + Assert.Equal(0, manager.ConnectionCount); + } + + [Fact] + public void SendMessage_ValidConnection_EnqueuesMessage() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-3"; + manager.AddConnection(connectionId, response, cts.Token); + + bool sent = manager.SendMessage(connectionId, new { type = "test", message = "hello" }); + + Assert.True(sent); + } + + [Fact] + public void SendMessage_UnknownConnection_ReturnsFalse() + { + using var manager = CreateManager(); + + bool sent = manager.SendMessage("missing", new { type = "test" }); + + Assert.False(sent); + } + + [Fact] + public async Task SendMessage_AbortedConnection_ReturnsFalseAndRemoves() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-4"; + manager.AddConnection(connectionId, response, cts.Token); + + await cts.CancelAsync(); + + bool sent = manager.SendMessage(connectionId, new { type = "test" }); + + Assert.False(sent); + // Connection should be cleaned up + Assert.Null(manager.GetConnectionById(connectionId)); + } + + [Fact] + public void SendMessageToAll_MultipleConnections_SendsToAll() + { + using var manager = CreateManager(); + using var response1 = new FakeHttpResponse(); + using var response2 = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + manager.AddConnection("conn-1", response1, cts1.Token); + manager.AddConnection("conn-2", response2, cts2.Token); + + manager.SendMessageToAll(new { type = "broadcast" }); + + // Both connections should have received the message (enqueued) + Assert.Equal(2, manager.ConnectionCount); + } + + private SseConnectionManager CreateManager() + { + var options = new AppOptions { EnablePush = true }; + return new SseConnectionManager(options, GetService(), Log); + } +} + +/// +/// Tests for MessageBusBroker using SSE connections. +/// +public sealed class SseBrokerTests : TestWithServices +{ + private readonly MessageBusBroker _broker; + private readonly SseConnectionManager _connectionManager; + private readonly PushConnectionRegistry _connectionRegistry; + + public SseBrokerTests(ITestOutputHelper output) : base(output) + { + _broker = GetService(); + _connectionManager = GetService(); + _connectionRegistry = GetService(); + } + + [Fact] + public void CanDrop_UserMembershipChanged_ReturnsFalse() + { + // Arrange + var message = new UserMembershipChanged + { + UserId = "membership-user", + OrganizationId = "membership-organization", + ChangeType = ChangeType.Removed + }; + + // Act + bool canDrop = MessageBusBroker.CanDrop(message); + + // Assert + Assert.False(canDrop); + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesConnectionsAndClearsMapping() + { + const string userId = "test-user-id"; + const string organizationId = "test-org-id"; + using var response1 = new FakeHttpResponse(); + using var response2 = new FakeHttpResponse(); + using var unrelatedResponse = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + using var ctsu = new CancellationTokenSource(); + + string connId1 = "conn-auth-1"; + string connId2 = "conn-auth-2"; + string unrelatedConnId = "conn-unrelated"; + + _connectionManager.AddConnection(connId1, response1, cts1.Token); + _connectionManager.AddConnection(connId2, response2, cts2.Token); + _connectionManager.AddConnection(unrelatedConnId, unrelatedResponse, ctsu.Token); + Assert.True(_connectionRegistry.TryRegister(connId1, userId, "test-token-id", [organizationId])); + Assert.True(_connectionRegistry.TryRegister(connId2, userId, "test-token-id", [organizationId])); + Assert.True(_connectionRegistry.TryRegister(unrelatedConnId, "unrelated-user", "unrelated-token", [organizationId])); + + try + { + var entityChanged = new EntityChanged + { + Id = "test-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + // Connections should be removed + Assert.Null(_connectionManager.GetConnectionById(connId1)); + Assert.Null(_connectionManager.GetConnectionById(connId2)); + Assert.NotNull(_connectionManager.GetConnectionById(unrelatedConnId)); + + Assert.Empty(_connectionRegistry.GetUserConnections(userId)); + } + finally + { + await _connectionManager.RemoveConnectionAsync(unrelatedConnId); + _connectionRegistry.Unregister(connId1); + _connectionRegistry.Unregister(connId2); + _connectionRegistry.Unregister(unrelatedConnId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_NonAuthTokenRemoved_DoesNotCloseConnections() + { + const string userId = "test-user-id-2"; + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "conn-nonauth"; + _connectionManager.AddConnection(connectionId, response, cts.Token); + Assert.True(_connectionRegistry.TryRegister(connectionId, userId, "authentication-token", [])); + + try + { + var entityChanged = new EntityChanged + { + Id = "test-api-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + // IsAuthenticationToken intentionally omitted (defaults false) + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + // Connection should NOT be closed + Assert.NotNull(_connectionManager.GetConnectionById(connectionId)); + } + finally + { + await _connectionManager.RemoveConnectionAsync(connectionId); + _connectionRegistry.Unregister(connectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_OrganizationMessage_SentToGroupOnly() + { + const string orgId = "org-1"; + const string otherOrgId = "org-2"; + using var responseInOrg = new FakeHttpResponse(); + using var responseOutOrg = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + string inOrgConn = "conn-in-org"; + string outOrgConn = "conn-out-org"; + + _connectionManager.AddConnection(inOrgConn, responseInOrg, cts1.Token); + _connectionManager.AddConnection(outOrgConn, responseOutOrg, cts2.Token); + Assert.True(_connectionRegistry.TryRegister(inOrgConn, "user-in", "token-in", [orgId])); + Assert.True(_connectionRegistry.TryRegister(outOrgConn, "user-out", "token-out", [otherOrgId])); + + try + { + var entityChanged = new EntityChanged + { + Id = "stack-123", + Type = "Stack", + ChangeType = ChangeType.Saved + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = orgId; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + // Give write loop a moment to process + await Task.Delay(200, TestContext.Current.CancellationToken); + + // In-org connection should receive message, out-org should not + Assert.True(responseInOrg.WrittenData.Length > 0, "In-org connection should receive message"); + Assert.Equal(0, responseOutOrg.WrittenData.Length); + } + finally + { + await _connectionManager.RemoveConnectionAsync(inOrgConn); + await _connectionManager.RemoveConnectionAsync(outOrgConn); + _connectionRegistry.Unregister(inOrgConn); + _connectionRegistry.Unregister(outOrgConn); + } + } + + [Fact] + public async Task OnEntityChangedAsync_UserMessage_SentToUserOnly() + { + const string userId = "user-target"; + const string otherUserId = "user-other"; + using var responseTarget = new FakeHttpResponse(); + using var responseOther = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + string targetConn = "conn-target-user"; + string otherConn = "conn-other-user"; + + _connectionManager.AddConnection(targetConn, responseTarget, cts1.Token); + _connectionManager.AddConnection(otherConn, responseOther, cts2.Token); + Assert.True(_connectionRegistry.TryRegister(targetConn, userId, "target-token", [])); + Assert.True(_connectionRegistry.TryRegister(otherConn, otherUserId, "other-token", [])); + + try + { + var entityChanged = new EntityChanged + { + Id = userId, + Type = nameof(User), + ChangeType = ChangeType.Saved + }; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.True(responseTarget.WrittenData.Length > 0, "Target user should receive message"); + Assert.Equal(0, responseOther.WrittenData.Length); + } + finally + { + await _connectionManager.RemoveConnectionAsync(targetConn); + await _connectionManager.RemoveConnectionAsync(otherConn); + _connectionRegistry.Unregister(targetConn); + _connectionRegistry.Unregister(otherConn); + } + } + + [Fact] + public async Task OnUserMembershipChangedAsync_AddedAndRemoved_UpdatesForwardAndReverseMappings() + { + const string userId = "membership-user"; + const string organizationId = "membership-org"; + using var response1 = new FakeHttpResponse(); + using var response2 = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + string connectionId1 = "membership-conn-1"; + string connectionId2 = "membership-conn-2"; + + _connectionManager.AddConnection(connectionId1, response1, cts1.Token); + _connectionManager.AddConnection(connectionId2, response2, cts2.Token); + Assert.True(_connectionRegistry.TryRegister(connectionId1, userId, "membership-token", [])); + Assert.True(_connectionRegistry.TryRegister(connectionId2, userId, "membership-token", [])); + + try + { + var addMessage = new UserMembershipChanged { + UserId = userId, + OrganizationId = organizationId, + ChangeType = ChangeType.Added + }; + + await _broker.OnUserMembershipChangedAsync(addMessage, TestContext.Current.CancellationToken); + + var organizationConnections = _connectionRegistry.GetGroupConnections(organizationId); + Assert.Contains(connectionId1, organizationConnections); + Assert.Contains(connectionId2, organizationConnections); + Assert.Contains(organizationId, _connectionRegistry.GetGroups(connectionId1)); + Assert.Contains(organizationId, _connectionRegistry.GetGroups(connectionId2)); + + var removeMessage = addMessage with { ChangeType = ChangeType.Removed }; + await _broker.OnUserMembershipChangedAsync(removeMessage, TestContext.Current.CancellationToken); + + Assert.Empty(_connectionRegistry.GetGroupConnections(organizationId)); + Assert.Empty(_connectionRegistry.GetGroups(connectionId1)); + Assert.Empty(_connectionRegistry.GetGroups(connectionId2)); + } + finally + { + await _connectionManager.RemoveConnectionAsync(connectionId1); + await _connectionManager.RemoveConnectionAsync(connectionId2); + _connectionRegistry.Unregister(connectionId1); + _connectionRegistry.Unregister(connectionId2); + } + } + + [Fact] + public async Task OnUserMembershipChangedAsync_Removed_SendsRefreshToRemovedUserAndRemainingOrganizationMembers() + { + const string removedUserId = "removed-user"; + const string remainingUserId = "remaining-user"; + const string organizationId = "shared-org"; + using var removedResponse = new FakeHttpResponse(); + using var remainingResponse = new FakeHttpResponse(); + using var removedCts = new CancellationTokenSource(); + using var remainingCts = new CancellationTokenSource(); + + string removedConnectionId = "removed-conn"; + string remainingConnectionId = "remaining-conn"; + + _connectionManager.AddConnection(removedConnectionId, removedResponse, removedCts.Token); + _connectionManager.AddConnection(remainingConnectionId, remainingResponse, remainingCts.Token); + Assert.True(_connectionRegistry.TryRegister(removedConnectionId, removedUserId, "removed-token", [organizationId])); + Assert.True(_connectionRegistry.TryRegister(remainingConnectionId, remainingUserId, "remaining-token", [organizationId])); + + try + { + var message = new UserMembershipChanged { + UserId = removedUserId, + OrganizationId = organizationId, + ChangeType = ChangeType.Removed + }; + + await _broker.OnUserMembershipChangedAsync(message, TestContext.Current.CancellationToken); + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.Contains(nameof(UserMembershipChanged), removedResponse.WrittenData); + Assert.Contains(nameof(UserMembershipChanged), remainingResponse.WrittenData); + Assert.DoesNotContain(removedConnectionId, _connectionRegistry.GetGroupConnections(organizationId)); + Assert.Empty(_connectionRegistry.GetGroups(removedConnectionId)); + } + finally + { + await _connectionManager.RemoveConnectionAsync(removedConnectionId); + await _connectionManager.RemoveConnectionAsync(remainingConnectionId); + _connectionRegistry.Unregister(removedConnectionId); + _connectionRegistry.Unregister(remainingConnectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_ClearsAllLocalOrganizationRegistrations() + { + const string userId = "tracked-user"; + const string firstOrganizationId = "tracked-org-1"; + const string secondOrganizationId = "tracked-org-2"; + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "tracked-conn"; + _connectionManager.AddConnection(connectionId, response, cts.Token); + Assert.True(_connectionRegistry.TryRegister(connectionId, userId, "tracked-token-id", [firstOrganizationId, secondOrganizationId])); + + try + { + var entityChanged = new EntityChanged + { + Id = "tracked-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = firstOrganizationId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + Assert.Null(_connectionManager.GetConnectionById(connectionId)); + Assert.Empty(_connectionRegistry.GetUserConnections(userId)); + } + finally + { + _connectionRegistry.Unregister(connectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_NonOwnerCannotConsumeOwnerRevocation() + { + const string userId = "replicated-user"; + const string tokenId = "replicated-token"; + const string connectionId = "owner-connection"; + using var ownerResponse = new FakeHttpResponse(); + using var ownerCancellation = new CancellationTokenSource(); + using var ownerSse = new SseConnectionManager(new AppOptions { EnablePush = true }, GetService(), Log); + using var ownerWebSocket = new WebSocketConnectionManager(new AppOptions { EnablePush = true }, GetService(), Log); + using var nonOwnerSse = new SseConnectionManager(new AppOptions { EnablePush = true }, GetService(), Log); + using var nonOwnerWebSocket = new WebSocketConnectionManager(new AppOptions { EnablePush = true }, GetService(), Log); + var ownerRegistry = new PushConnectionRegistry(TimeProvider); + var nonOwnerRegistry = new PushConnectionRegistry(TimeProvider); + var options = new AppOptions { EnablePush = true }; + var subscriber = GetService(); + var ownerBroker = new MessageBusBroker(ownerSse, ownerWebSocket, ownerRegistry, subscriber, options, Log.CreateLogger()); + var nonOwnerBroker = new MessageBusBroker(nonOwnerSse, nonOwnerWebSocket, nonOwnerRegistry, subscriber, options, Log.CreateLogger()); + + ownerSse.AddConnection(connectionId, ownerResponse, ownerCancellation.Token); + Assert.True(ownerRegistry.TryRegister(connectionId, userId, tokenId, ["organization"])); + var message = new EntityChanged { Id = tokenId, Type = nameof(Token), ChangeType = ChangeType.Removed }; + message.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + message.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await nonOwnerBroker.OnEntityChangedAsync(message, TestContext.Current.CancellationToken); + Assert.NotNull(ownerSse.GetConnectionById(connectionId)); + + await ownerBroker.OnEntityChangedAsync(message, TestContext.Current.CancellationToken); + Assert.Null(ownerSse.GetConnectionById(connectionId)); + } + +} + +/// +/// Tests for the deduplication behavior of SseConnection. +/// Validates that identical messages queued in quick succession are coalesced. +/// +public sealed class SseDeduplicationTests : TestWithServices +{ + public SseDeduplicationTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public async Task DuplicateMessages_AreDeduped_OnlyOneQueued() + { + var queue = new SseConnection.DedupQueue(8); + var evt = new SseConnection.SseEvent { Data = "{\"type\":\"StackChanged\",\"id\":\"stack-123\",\"change_type\":1}", DedupeKey = "stack-123" }; + int dedupedCount = 0; + + for (int i = 0; i < 5; i++) + { + if (queue.TryEnqueue(evt) == SseConnection.EnqueueResult.Deduped) + dedupedCount++; + } + + using var cts = new CancellationTokenSource(); + var queued = await queue.DequeueAsync(cts.Token); + + Assert.NotNull(queued); + Assert.Equal(evt.Data, queued!.Value.Data); + Assert.Equal(4, dedupedCount); + } + + [Fact] + public async Task DifferentMessages_AreNotDeduped() + { + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var serializer = GetService(); + + await using var connection = new SseConnection("dedup-test-2", response, serializer, cts.Token, Log.CreateLogger()); + + // Send 3 different messages + connection.TryWrite(new { type = "StackChanged", id = "stack-1" }); + connection.TryWrite(new { type = "StackChanged", id = "stack-2" }); + connection.TryWrite(new { type = "ProjectChanged", id = "proj-1" }); + + await Task.Delay(200, TestContext.Current.CancellationToken); + connection.Abort(); + await Task.Delay(50, TestContext.Current.CancellationToken); + + string output = response.WrittenData; + int dataLineCount = output.Split("data: ").Length - 1; + Assert.Equal(3, dataLineCount); + Assert.Equal(0, connection.DedupedMessages); + } + + [Fact] + public async Task SameMessage_AfterFirstIsConsumed_IsNotDeduped() + { + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var serializer = GetService(); + + await using var connection = new SseConnection("dedup-test-3", response, serializer, cts.Token, Log.CreateLogger()); + + var message = new { type = "StackChanged", id = "stack-repeat" }; + + // Send first message and wait for it to be consumed + connection.TryWrite(message); + await Task.Delay(200, TestContext.Current.CancellationToken); + + // Send same message again — should NOT be deduped because first was already consumed + connection.TryWrite(message); + await Task.Delay(200, TestContext.Current.CancellationToken); + + connection.Abort(); + await Task.Delay(50, TestContext.Current.CancellationToken); + + string output = response.WrittenData; + int dataLineCount = output.Split("data: ").Length - 1; + Assert.Equal(2, dataLineCount); + Assert.Equal(0, connection.DedupedMessages); + } + + [Fact] + public async Task KeepAlive_IsNeverDeduped() + { + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var serializer = GetService(); + + await using var connection = new SseConnection("dedup-test-4", response, serializer, cts.Token, Log.CreateLogger()); + + // Send multiple keep-alives — none should be deduped + connection.TryWriteKeepAlive(); + connection.TryWriteKeepAlive(); + connection.TryWriteKeepAlive(); + + await Task.Delay(200, TestContext.Current.CancellationToken); + connection.Abort(); + await Task.Delay(50, TestContext.Current.CancellationToken); + + string output = response.WrittenData; + int keepAliveCount = output.Split(": keepalive").Length - 1; + Assert.Equal(3, keepAliveCount); + } + + [Fact] + public async Task TryEnqueue_CapacityExceeded_DropsOldestWithoutPhantomSignal() + { + // Arrange + using var queue = new SseConnection.DedupQueue(3); + + // Act + for (int i = 0; i < 3; i++) + queue.TryEnqueue(new SseConnection.SseEvent { Data = $"msg-{i}", DedupeKey = $"key-{i}" }); + var overflow = queue.TryEnqueue(new SseConnection.SseEvent { Data = "overflow", DedupeKey = "overflow" }); + + var item1 = await queue.DequeueAsync(TestContext.Current.CancellationToken); + var item2 = await queue.DequeueAsync(TestContext.Current.CancellationToken); + var item3 = await queue.DequeueAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(SseConnection.EnqueueResult.Full, overflow); + Assert.Equal("msg-0", item1!.Value.Data); + Assert.Equal("msg-1", item2!.Value.Data); + Assert.Equal("msg-2", item3!.Value.Data); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromMilliseconds(50)); + await Assert.ThrowsAnyAsync(() => queue.DequeueAsync(cts.Token)); + } + + [Fact] + public async Task CriticalMessage_WhenQueueFull_ReturnsFullWithoutEvictingQueuedMessages() + { + var queue = new SseConnection.DedupQueue(2); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "lossy-1", DedupeKey = "lossy-1" }); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-1" }); + + var result = queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-2" }); + + using var cts = new CancellationTokenSource(); + var item1 = await queue.DequeueAsync(cts.Token); + var item2 = await queue.DequeueAsync(cts.Token); + + Assert.Equal(SseConnection.EnqueueResult.Full, result); + Assert.Equal("lossy-1", item1!.Value.Data); + Assert.Equal("critical-1", item2!.Value.Data); + } + + [Fact] + public async Task DroppableMessage_WhenQueueFullOfCriticalMessages_DoesNotEvictCriticalMessage() + { + // Arrange + using var queue = new SseConnection.DedupQueue(1); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-1" }); + + // Act + var result = queue.TryEnqueue(new SseConnection.SseEvent { Data = "lossy-1", DedupeKey = "lossy-1" }); + var item = await queue.DequeueAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(SseConnection.EnqueueResult.Full, result); + Assert.True(item.HasValue); + Assert.Equal("critical-1", item.GetValueOrDefault().Data); + } + + [Fact] + public async Task KeepAlive_WhenQueueFull_DoesNotEvictCriticalMessage() + { + var queue = new SseConnection.DedupQueue(1); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-1" }); + + var result = queue.TryEnqueue(SseConnection.SseEvent.KeepAlive); + + using var cts = new CancellationTokenSource(); + var item = await queue.DequeueAsync(cts.Token); + + Assert.Equal(SseConnection.EnqueueResult.Full, result); + Assert.True(item.HasValue); + var dequeued = item.GetValueOrDefault(); + Assert.Equal("critical-1", dequeued.Data); + Assert.False(dequeued.IsKeepAlive); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs b/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs index c8343c7b3a..54aabbe351 100644 --- a/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs +++ b/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs @@ -6,6 +6,7 @@ namespace Exceptionless.Tests.Hubs; internal sealed class TestWebSocket : WebSocket { private WebSocketState _state; + private int _closeCount; public TestWebSocket(WebSocketState state = WebSocketState.Open) { @@ -13,7 +14,6 @@ public TestWebSocket(WebSocketState state = WebSocketState.Open) } public int CloseCount => _closeCount; - private int _closeCount; public List SentMessages { get; } = []; public override WebSocketCloseStatus? CloseStatus { get; } = WebSocketCloseStatus.NormalClosure; public override string? CloseStatusDescription { get; } = "Closed"; @@ -47,7 +47,7 @@ public override Task ReceiveAsync(ArraySegment buf public override Task SendAsync(ArraySegment buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken) { - SentMessages.Add(Encoding.ASCII.GetString(buffer.Array!, buffer.Offset, buffer.Count)); + SentMessages.Add(Encoding.UTF8.GetString(buffer.Array!, buffer.Offset, buffer.Count)); return Task.CompletedTask; } } diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs new file mode 100644 index 0000000000..e85647cca8 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs @@ -0,0 +1,151 @@ +using System.Net.WebSockets; +using Exceptionless.Core; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Web.Hubs; +using Foundatio.Repositories.Models; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +public sealed class WebSocketConnectionCompatibilityTests : TestWithServices +{ + public WebSocketConnectionCompatibilityTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void AddConnection_NewSocket_CanLookupAndEnumerateConnection() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(); + + string connectionId = manager.AddConnection(socket); + + Assert.False(String.IsNullOrEmpty(connectionId)); + Assert.Same(socket, manager.GetConnectionById(connectionId)); + Assert.Same(socket, Assert.Single(manager.GetAll())); + } + + [Fact] + public async Task RemoveConnectionAsync_ExistingConnection_RemovesAndClosesSocket() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(); + string connectionId = manager.AddConnection(socket); + + await manager.RemoveConnectionAsync(connectionId); + + Assert.Null(manager.GetConnectionById(connectionId)); + Assert.Empty(manager.GetAll()); + Assert.Equal(1, socket.CloseCount); + Assert.Equal(WebSocketState.Closed, socket.State); + } + + [Fact] + public void SendMessage_ClosedSocket_ReturnsFalseAndRemovesConnection() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(WebSocketState.Closed); + string connectionId = manager.AddConnection(socket); + + bool sent = manager.SendMessage(connectionId, new { type = "test" }); + + Assert.False(sent); + Assert.Null(manager.GetConnectionById(connectionId)); + } + + private WebSocketConnectionManager CreateManager() + { + var options = new AppOptions { EnablePush = true }; + return new WebSocketConnectionManager(options, GetService(), Log); + } +} + +public sealed class PushCompatibilityBrokerTests : TestWithServices +{ + private readonly MessageBusBroker _broker; + private readonly SseConnectionManager _sseConnectionManager; + private readonly WebSocketConnectionManager _webSocketConnectionManager; + private readonly PushConnectionRegistry _connectionRegistry; + + public PushCompatibilityBrokerTests(ITestOutputHelper output) : base(output) + { + _broker = GetService(); + _sseConnectionManager = GetService(); + _webSocketConnectionManager = GetService(); + _connectionRegistry = GetService(); + } + + [Fact] + public async Task OnEntityChangedAsync_FansOutToSseAndWebSocketConnections() + { + const string organizationId = "compat-org"; + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var socket = new TestWebSocket(); + + string sseConnectionId = "compat-sse"; + string webSocketConnectionId = _webSocketConnectionManager.AddConnection(socket); + _sseConnectionManager.AddConnection(sseConnectionId, response, cts.Token); + Assert.True(_connectionRegistry.TryRegister(sseConnectionId, "sse-user", "sse-token", [organizationId])); + Assert.True(_connectionRegistry.TryRegister(webSocketConnectionId, "websocket-user", "websocket-token", [organizationId])); + + try + { + var entityChanged = new EntityChanged + { + Id = "stack-compat", + Type = "Stack", + ChangeType = ChangeType.Saved + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.Contains("StackChanged", response.WrittenData); + Assert.Single(socket.SentMessages); + Assert.Contains("StackChanged", socket.SentMessages[0]); + } + finally + { + await _sseConnectionManager.RemoveConnectionAsync(sseConnectionId); + await _webSocketConnectionManager.RemoveConnectionAsync(webSocketConnectionId); + _connectionRegistry.Unregister(sseConnectionId); + _connectionRegistry.Unregister(webSocketConnectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketConnectionsAndClearsLocalRegistration() + { + const string userId = "compat-user"; + const string organizationId = "compat-org"; + var socket = new TestWebSocket(); + string connectionId = _webSocketConnectionManager.AddConnection(socket); + Assert.True(_connectionRegistry.TryRegister(connectionId, userId, "compat-token", [organizationId])); + + try + { + var entityChanged = new EntityChanged + { + Id = "compat-token", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + Assert.Null(_webSocketConnectionManager.GetConnectionById(connectionId)); + Assert.Equal(1, socket.CloseCount); + Assert.Empty(_connectionRegistry.GetUserConnections(userId)); + } + finally + { + _connectionRegistry.Unregister(connectionId); + } + } +} diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs index 9018916fdc..0af21c2074 100644 --- a/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs +++ b/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs @@ -1,5 +1,4 @@ using System.Net.WebSockets; -using System.Text; using Exceptionless.Core; using Exceptionless.Web.Hubs; using Foundatio.Serializer; @@ -14,14 +13,11 @@ public WebSocketConnectionManagerTests(ITestOutputHelper output) : base(output) [Fact] public void AddWebSocket_NewSocket_CanLookupAndEnumerateConnection() { - // Arrange using var manager = CreateManager(); var socket = new TestWebSocket(); - // Act string connectionId = manager.AddWebSocket(socket); - // Assert Assert.False(String.IsNullOrEmpty(connectionId)); Assert.Same(socket, manager.GetWebSocketById(connectionId)); Assert.Equal(connectionId, manager.GetConnectionId(socket)); @@ -31,15 +27,12 @@ public void AddWebSocket_NewSocket_CanLookupAndEnumerateConnection() [Fact] public async Task RemoveWebSocketAsync_ExistingConnection_RemovesAndClosesSocket() { - // Arrange using var manager = CreateManager(); var socket = new TestWebSocket(); string connectionId = manager.AddWebSocket(socket); - // Act await manager.RemoveWebSocketAsync(connectionId); - // Assert Assert.Null(manager.GetWebSocketById(connectionId)); Assert.Empty(manager.GetAll()); Assert.Equal(1, socket.CloseCount); @@ -49,15 +42,12 @@ public async Task RemoveWebSocketAsync_ExistingConnection_RemovesAndClosesSocket [Fact] public async Task RemoveWebSocketAsync_ClosedSocket_RemovesWithoutClosingAgain() { - // Arrange using var manager = CreateManager(); var socket = new TestWebSocket(WebSocketState.Closed); string connectionId = manager.AddWebSocket(socket); - // Act await manager.RemoveWebSocketAsync(connectionId); - // Assert Assert.Null(manager.GetWebSocketById(connectionId)); Assert.Empty(manager.GetAll()); Assert.Equal(0, socket.CloseCount); @@ -66,34 +56,28 @@ public async Task RemoveWebSocketAsync_ClosedSocket_RemovesWithoutClosingAgain() [Fact] public async Task RemoveWebSocketAsync_UnknownConnection_DoesNothing() { - // Arrange using var manager = CreateManager(); - // Act await manager.RemoveWebSocketAsync("missing"); - // Assert Assert.Empty(manager.GetAll()); } [Fact] public async Task SendMessageToAllAsync_ClosedSockets_DoesNotSend() { - // Arrange using var manager = CreateManager(); var socket = new TestWebSocket(WebSocketState.Closed); manager.AddWebSocket(socket); - // Act await manager.SendMessageToAllAsync(new { type = "test" }); - // Assert Assert.Empty(socket.SentMessages); } private WebSocketConnectionManager CreateManager() { - var options = new AppOptions { EnableWebSockets = false }; + var options = new AppOptions { EnablePush = false }; return new WebSocketConnectionManager(options, GetService(), Log); } } diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs index a34a33ddc7..46b1d484e6 100644 --- a/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs +++ b/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs @@ -1,6 +1,5 @@ using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; -using Exceptionless.Core.Utility; using Exceptionless.Web.Hubs; using Foundatio.Repositories.Models; using Xunit; @@ -8,27 +7,26 @@ namespace Exceptionless.Tests.Hubs; /// -/// Tests for WebSocket behavior. Calls +/// Tests for WebSocket behavior. Calls /// directly so they do not depend on -/// message bus wiring or EnableWebSockets in test host configuration. +/// message bus wiring or EnablePush in test host configuration. /// public sealed class WebSocketTests : TestWithServices { private readonly MessageBusBroker _broker; - private readonly IConnectionMapping _connectionMapping; private readonly WebSocketConnectionManager _connectionManager; + private readonly PushConnectionRegistry _connectionRegistry; public WebSocketTests(ITestOutputHelper output) : base(output) { _broker = GetService(); - _connectionMapping = GetService(); _connectionManager = GetService(); + _connectionRegistry = GetService(); } [Fact] public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketsAndClearsUserMapping() { - // Arrange const string userId = "test-user-id"; const string organizationId = "test-organization-id"; var socket1 = new TestWebSocket(); @@ -38,15 +36,12 @@ public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketsAndClear string connectionId1 = _connectionManager.AddWebSocket(socket1); string connectionId2 = _connectionManager.AddWebSocket(socket2); string unrelatedConnectionId = _connectionManager.AddWebSocket(unrelatedSocket); + Assert.True(_connectionRegistry.TryRegister(connectionId1, userId, "test-token-id", [organizationId])); + Assert.True(_connectionRegistry.TryRegister(connectionId2, userId, "test-token-id", [organizationId])); + Assert.True(_connectionRegistry.TryRegister(unrelatedConnectionId, "unrelated-user", "unrelated-token-id", [organizationId])); try { - await _connectionMapping.UserIdAddAsync(userId, connectionId1); - await _connectionMapping.UserIdAddAsync(userId, connectionId2); - await _connectionMapping.GroupAddAsync(organizationId, connectionId1); - await _connectionMapping.GroupAddAsync(organizationId, connectionId2); - await _connectionMapping.GroupAddAsync(organizationId, unrelatedConnectionId); - var entityChanged = new EntityChanged { Id = "test-token-id", @@ -57,10 +52,8 @@ public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketsAndClear entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; - // Act — call the broker directly; no message bus or EnableWebSockets dependency await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); - // Assert – sockets closed and removed from manager Assert.Null(_connectionManager.GetWebSocketById(connectionId1)); Assert.Null(_connectionManager.GetWebSocketById(connectionId2)); Assert.Same(unrelatedSocket, _connectionManager.GetWebSocketById(unrelatedConnectionId)); @@ -69,33 +62,31 @@ public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketsAndClear Assert.Equal(1, socket2.CloseCount); Assert.Equal(0, unrelatedSocket.CloseCount); - // Assert – user-id mapping removed by broker - var remaining = await _connectionMapping.GetUserIdConnectionsAsync(userId); - Assert.Empty(remaining); - var organizationConnections = await _connectionMapping.GetGroupConnectionsAsync(organizationId); + Assert.Empty(_connectionRegistry.GetUserConnections(userId)); + var organizationConnections = _connectionRegistry.GetGroupConnections(organizationId); Assert.DoesNotContain(connectionId1, organizationConnections); Assert.DoesNotContain(connectionId2, organizationConnections); Assert.Contains(unrelatedConnectionId, organizationConnections); } finally { - await _connectionMapping.GroupRemoveAsync(organizationId, unrelatedConnectionId); await _connectionManager.RemoveWebSocketAsync(unrelatedConnectionId); + _connectionRegistry.Unregister(connectionId1); + _connectionRegistry.Unregister(connectionId2); + _connectionRegistry.Unregister(unrelatedConnectionId); } } [Fact] public async Task OnEntityChangedAsync_NonAuthTokenRemoved_DoesNotCloseWebSockets() { - // Arrange const string userId = "test-user-id-2"; var socket = new TestWebSocket(); string connectionId = _connectionManager.AddWebSocket(socket); + Assert.True(_connectionRegistry.TryRegister(connectionId, userId, "authentication-token", [])); try { - await _connectionMapping.UserIdAddAsync(userId, connectionId); - var entityChanged = new EntityChanged { Id = "test-api-token-id", @@ -103,19 +94,16 @@ public async Task OnEntityChangedAsync_NonAuthTokenRemoved_DoesNotCloseWebSocket ChangeType = ChangeType.Removed }; entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; - // IsAuthenticationToken intentionally omitted (defaults false) - // Act await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); - // Assert – socket should NOT be closed for a non-auth token removal Assert.Equal(0, socket.CloseCount); Assert.Same(socket, _connectionManager.GetWebSocketById(connectionId)); } finally { - await _connectionMapping.UserIdRemoveAsync(userId, connectionId); await _connectionManager.RemoveWebSocketAsync(connectionId); + _connectionRegistry.Unregister(connectionId); } } } diff --git a/tests/Exceptionless.Tests/Utility/ConnectionLeaseStoreTests.cs b/tests/Exceptionless.Tests/Utility/ConnectionLeaseStoreTests.cs new file mode 100644 index 0000000000..be85a44697 --- /dev/null +++ b/tests/Exceptionless.Tests/Utility/ConnectionLeaseStoreTests.cs @@ -0,0 +1,35 @@ +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Utility; +using Xunit; + +namespace Exceptionless.Tests.Utility; + +public sealed class ConnectionLeaseStoreTests +{ + [Fact] + public async Task TryAcquireAsync_ConcurrentConnections_AcquiresExactlyTheLimit() + { + var timeProvider = new ProxyTimeProvider(); + var store = new ConnectionLeaseStore(timeProvider); + + bool[] acquired = await Task.WhenAll(Enumerable.Range(0, 100) + .Select(index => store.TryAcquireAsync("user", $"connection-{index}", 10, TimeSpan.FromMinutes(1)))); + + Assert.Equal(10, acquired.Count(value => value)); + } + + [Fact] + public async Task TryAcquireAsync_ExpiredLease_ReclaimsCapacityAfterReplicaLoss() + { + var timeProvider = new ProxyTimeProvider(); + var store = new ConnectionLeaseStore(timeProvider); + + Assert.True(await store.TryAcquireAsync("user", "lost-connection", 1, TimeSpan.FromMinutes(1))); + Assert.False(await store.TryAcquireAsync("user", "blocked-connection", 1, TimeSpan.FromMinutes(1))); + + timeProvider.Advance(TimeSpan.FromMinutes(1)); + + Assert.True(await store.TryAcquireAsync("user", "replacement-connection", 1, TimeSpan.FromMinutes(1))); + Assert.False(await store.RenewAsync("user", "lost-connection", TimeSpan.FromMinutes(1))); + } +} diff --git a/tests/Exceptionless.Tests/Utility/RedisConnectionLeaseStoreTests.cs b/tests/Exceptionless.Tests/Utility/RedisConnectionLeaseStoreTests.cs new file mode 100644 index 0000000000..a75bd017d4 --- /dev/null +++ b/tests/Exceptionless.Tests/Utility/RedisConnectionLeaseStoreTests.cs @@ -0,0 +1,36 @@ +using Exceptionless.Core; +using Exceptionless.Insulation.Redis; +using StackExchange.Redis; +using Xunit; + +namespace Exceptionless.Tests.Utility; + +public sealed class RedisConnectionLeaseStoreTests : IClassFixture +{ + private readonly AppWebHostFactory _factory; + + public RedisConnectionLeaseStoreTests(AppWebHostFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task TryAcquireAsync_TwoProvidersShareAtomicLimitAndRecoverExpiredLeases() + { + string connectionString = await _factory.GetRedisConnectionStringAsync(TestContext.Current.CancellationToken); + await using var multiplexer = await ConnectionMultiplexer.ConnectAsync(connectionString); + string scope = $"lease-test-{Guid.NewGuid():N}"; + var first = new RedisConnectionLeaseStore(multiplexer, new AppOptions { AppScope = scope }); + var second = new RedisConnectionLeaseStore(multiplexer, new AppOptions { AppScope = scope }); + string[] connectionIds = Enumerable.Range(0, 40).Select(index => $"connection-{index}").ToArray(); + + bool[] acquired = await Task.WhenAll(connectionIds.Select((connectionId, index) => + (index % 2 is 0 ? first : second).TryAcquireAsync("user", connectionId, 10, TimeSpan.FromMilliseconds(500)))); + + Assert.Equal(10, acquired.Count(value => value)); + await Task.Delay(TimeSpan.FromMilliseconds(750), TestContext.Current.CancellationToken); + Assert.True(await second.TryAcquireAsync("user", "replacement", 10, TimeSpan.FromSeconds(5))); + Assert.False(await first.RenewAsync("user", connectionIds[Array.FindIndex(acquired, value => value)], TimeSpan.FromSeconds(5))); + await second.ReleaseAsync("user", "replacement"); + } +} diff --git a/tests/Exceptionless.Tests/appsettings.yml b/tests/Exceptionless.Tests/appsettings.yml index 5914619e15..c7f422bb67 100644 --- a/tests/Exceptionless.Tests/appsettings.yml +++ b/tests/Exceptionless.Tests/appsettings.yml @@ -21,7 +21,7 @@ ContactEmailAddress: "contact@exceptionless.test" # Runs the jobs in the current website process RunJobsInProcess: false -EnableWebSockets: false +EnablePush: true Serilog: MinimumLevel: Warning diff --git a/tests/http/push.http b/tests/http/push.http new file mode 100644 index 0000000000..db874cbf25 --- /dev/null +++ b/tests/http/push.http @@ -0,0 +1,29 @@ +@url = http://localhost:7110 +@apiUrl = {{url}}/api/v2 +@email = admin@exceptionless.test +@password = tester + +### login to test account +# @name login +POST {{apiUrl}}/auth/login +Content-Type: application/json + +{ + "email": "{{email}}", + "password": "{{password}}" +} + +### + +@token = {{login.response.body.$.token}} + +### SSE push via bearer token +# This request intentionally stays open. Cancel it manually after verifying headers/events. +GET {{apiUrl}}/push +Accept: text/event-stream +Authorization: Bearer {{token}} + +### SSE push via query-string token +# This request intentionally stays open. Cancel it manually after verifying headers/events. +GET {{apiUrl}}/push?access_token={{token}} +Accept: text/event-stream