feat: add Deleted counter to usage metrics

src/Exceptionless.Core/Jobs/CleanupDataJob.cs

@@ -27,6 +27,7 @@ public class CleanupDataJob : JobWithLockBase, IHealthCheck
     private readonly ITokenRepository _tokenRepository;
     private readonly IWebHookRepository _webHookRepository;
     private readonly BillingManager _billingManager;
+    private readonly UsageService _usageService;
     private readonly AppOptions _appOptions;
     private readonly ILockProvider _lockProvider;
     private readonly ICacheClient _cacheClient;
@@ -43,6 +44,7 @@ public CleanupDataJob(
         ILockProvider lockProvider,
         ICacheClient cacheClient,
         BillingManager billingManager,
+        UsageService usageService,
         AppOptions appOptions,
         TimeProvider timeProvider,
         IResiliencePolicyProvider resiliencePolicyProvider,
@@ -57,6 +59,7 @@ ILoggerFactory loggerFactory
         _tokenRepository = tokenRepository;
         _webHookRepository = webHookRepository;
         _billingManager = billingManager;
+        _usageService = usageService;
         _appOptions = appOptions;
         _lockProvider = lockProvider;
         _cacheClient = cacheClient;
@@ -164,7 +167,7 @@ private async Task CleanupSoftDeletedStacksAsync(JobContext context)
         {
             try
             {
-                await RemoveStacksAsync(stackResults.Documents, context);
+                await RemoveStacksAsync(stackResults.Documents, context, trackDeletedUsage: true);
             }
             catch (Exception ex)
             {
@@ -206,24 +209,37 @@ private async Task RemoveProjectsAsync(Project project, JobContext context)
         await RenewLockAsync(context);
         long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id);
 
+        if (removedEvents > 0)
+            await _usageService.IncrementDeletedAsync(project.OrganizationId, project.Id, removedEvents);
+
         await RenewLockAsync(context);
         long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id);
 
         await _projectRepository.RemoveAsync(project);
         _logger.RemoveProjectComplete(project.Name, project.Id, removedStacks, removedEvents);
     }
 
-    private async Task RemoveStacksAsync(IReadOnlyCollection<Stack> stacks, JobContext context)
+    private async Task RemoveStacksAsync(IReadOnlyCollection<Stack> stacks, JobContext context, bool trackDeletedUsage = false)
     {
         await RenewLockAsync(context);
 
-        string[] stackIds = stacks.Select(s => s.Id).ToArray();
-        long removedEvents = await _eventRepository.RemoveAllByStackIdsAsync(stackIds);
-        await _stackRepository.RemoveAsync(stacks);
-        foreach (var orgGroup in stacks.GroupBy(s => (s.OrganizationId, s.ProjectId)))
-            await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(orgGroup.Key.OrganizationId, orgGroup.Key.ProjectId));
+        var groups = stacks.GroupBy(s => (s.OrganizationId, s.ProjectId)).ToList();
+        foreach (var group in groups)
+            await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(group.Key.OrganizationId, group.Key.ProjectId));
+
+        long totalRemovedEvents = 0;
+        foreach (var group in groups)
+        {
+            string[] groupStackIds = group.Select(s => s.Id).ToArray();
+            long groupRemovedEvents = await _eventRepository.RemoveAllByStackIdsAsync(groupStackIds);
+            totalRemovedEvents += groupRemovedEvents;
+
+            if (trackDeletedUsage && groupRemovedEvents > 0)
+                await _usageService.IncrementDeletedAsync(group.Key.OrganizationId, group.Key.ProjectId, groupRemovedEvents);
+        }
 
-        _logger.RemoveStacksComplete(stackIds.Length, removedEvents);
+        await _stackRepository.RemoveAsync(stacks);
+        _logger.RemoveStacksComplete(stacks.Count, totalRemovedEvents);
     }
 
     private async Task EnforceRetentionAsync(JobContext context)

src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs

@@ -1,6 +1,7 @@
 using Exceptionless.Core.Models.WorkItems;
 using Exceptionless.Core.Repositories;
 using Exceptionless.Core.Repositories.Queries;
+using Exceptionless.Core.Services;
 using Foundatio.Caching;
 using Foundatio.Jobs;
 using Foundatio.Lock;
@@ -14,13 +15,15 @@ public class ResetProjectDataWorkItemHandler : WorkItemHandlerBase
     private readonly IStackRepository _stackRepository;
     private readonly ICacheClient _cacheClient;
     private readonly ILockProvider _lockProvider;
+    private readonly UsageService _usageService;
 
-    public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+    public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, UsageService usageService, ILoggerFactory loggerFactory) : base(loggerFactory)
     {
         _eventRepository = eventRepository;
         _stackRepository = stackRepository;
         _cacheClient = cacheClient;
         _lockProvider = lockProvider;
+        _usageService = usageService;
     }
 
     public override Task<ILock?> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
@@ -41,6 +44,9 @@ public override async Task HandleItemAsync(WorkItemContext context)
             long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId);
             await context.ReportProgressAsync(50, $"Events removed: {removedEvents}");
 
+            if (removedEvents > 0)
+                await _usageService.IncrementDeletedAsync(workItem.OrganizationId, workItem.ProjectId, removedEvents);
+
             long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId);
             await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(workItem.OrganizationId, workItem.ProjectId));
 

src/Exceptionless.Core/Models/UsageInfo.cs

@@ -9,6 +9,7 @@ public record UsageInfo
     public int Blocked { get; set; }
     public int Discarded { get; set; }
     public int TooBig { get; set; }
+    public long Deleted { get; set; }
 }
 
 public record UsageHourInfo
@@ -18,6 +19,7 @@ public record UsageHourInfo
     public int Blocked { get; set; }
     public int Discarded { get; set; }
     public int TooBig { get; set; }
+    public long Deleted { get; set; }
 }
 
 public record UsageInfoResponse

src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs

@@ -60,13 +60,15 @@ public static PropertiesDescriptor<Organization> AddUsageMappings(this Propertie
                 .Number(fu => fu.Name(i => i.Blocked))
                 .Number(fu => fu.Name(i => i.Discarded))
                 .Number(fu => fu.Name(i => i.Limit))
-                .Number(fu => fu.Name(i => i.TooBig))))
+                .Number(fu => fu.Name(i => i.TooBig))
+                .Number(fu => fu.Name(i => i.Deleted))))
             .Object<UsageInfo>(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p
                 .Date(fu => fu.Name(i => i.Date))
                 .Number(fu => fu.Name(i => i.Total))
                 .Number(fu => fu.Name(i => i.Blocked))
                 .Number(fu => fu.Name(i => i.Discarded))
                 .Number(fu => fu.Name(i => i.Limit))
-                .Number(fu => fu.Name(i => i.TooBig))));
+                .Number(fu => fu.Name(i => i.TooBig))
+                .Number(fu => fu.Name(i => i.Deleted))));
     }
 }

src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs

@@ -51,13 +51,15 @@ public static PropertiesDescriptor<Project> AddUsageMappings(this PropertiesDesc
                 .Number(fu => fu.Name(i => i.Blocked))
                 .Number(fu => fu.Name(i => i.Discarded))
                 .Number(fu => fu.Name(i => i.Limit))
-                .Number(fu => fu.Name(i => i.TooBig))))
+                .Number(fu => fu.Name(i => i.TooBig))
+                .Number(fu => fu.Name(i => i.Deleted))))
             .Object<UsageInfo>(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p
                 .Date(fu => fu.Name(i => i.Date))
                 .Number(fu => fu.Name(i => i.Total))
                 .Number(fu => fu.Name(i => i.Blocked))
                 .Number(fu => fu.Name(i => i.Discarded))
                 .Number(fu => fu.Name(i => i.Limit))
-                .Number(fu => fu.Name(i => i.TooBig))));
+                .Number(fu => fu.Name(i => i.TooBig))
+                .Number(fu => fu.Name(i => i.Deleted))));
     }
 }

src/Exceptionless.Core/Services/UsageService.cs

@@ -81,6 +81,7 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow)
                     var bucketBlocked = await _cache.GetAsync<int>(GetBucketBlockedCacheKey(bucketUtc, organizationId));
                     var bucketDiscarded = await _cache.GetAsync<int>(GetBucketDiscardedCacheKey(bucketUtc, organizationId));
                     var bucketTooBig = await _cache.GetAsync<int>(GetBucketTooBigCacheKey(bucketUtc, organizationId));
+                    var bucketDeleted = await _cache.GetAsync<long>(GetBucketDeletedCacheKey(bucketUtc, organizationId));
 
                     organization.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime;
 
@@ -90,12 +91,14 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow)
                     usage.Blocked += bucketBlocked?.Value ?? 0;
                     usage.Discarded += bucketDiscarded?.Value ?? 0;
                     usage.TooBig += bucketTooBig?.Value ?? 0;
+                    usage.Deleted += bucketDeleted?.Value ?? 0;
 
                     var hourlyUsage = organization.GetHourlyUsage(bucketUtc);
                     hourlyUsage.Total += bucketTotal?.Value ?? 0;
                     hourlyUsage.Blocked += bucketBlocked?.Value ?? 0;
                     hourlyUsage.Discarded += bucketDiscarded?.Value ?? 0;
                     hourlyUsage.TooBig += bucketTooBig?.Value ?? 0;
+                    hourlyUsage.Deleted += bucketDeleted?.Value ?? 0;
 
                     organization.TrimUsage(_timeProvider);
 
@@ -104,6 +107,7 @@ await _cache.RemoveAllAsync(new[] {
                         GetBucketBlockedCacheKey(bucketUtc, organizationId),
                         GetBucketDiscardedCacheKey(bucketUtc, organizationId),
                         GetBucketTooBigCacheKey(bucketUtc, organizationId),
+                        GetBucketDeletedCacheKey(bucketUtc, organizationId),
                         GetThrottledKey(bucketUtc, organizationId)
                     });
 
@@ -159,6 +163,7 @@ private async Task SavePendingProjectUsageAsync(DateTime utcNow)
                     var bucketBlocked = await _cache.GetAsync<int>(GetBucketBlockedCacheKey(bucketUtc, project.OrganizationId, projectId));
                     var bucketDiscarded = await _cache.GetAsync<int>(GetBucketDiscardedCacheKey(bucketUtc, project.OrganizationId, projectId));
                     var bucketTooBig = await _cache.GetAsync<int>(GetBucketTooBigCacheKey(bucketUtc, project.OrganizationId, projectId));
+                    var bucketDeleted = await _cache.GetAsync<long>(GetBucketDeletedCacheKey(bucketUtc, project.OrganizationId, projectId));
 
                     project.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime;
 
@@ -171,20 +176,23 @@ private async Task SavePendingProjectUsageAsync(DateTime utcNow)
                     usage.Blocked += bucketBlocked?.Value ?? 0;
                     usage.Discarded += bucketDiscarded?.Value ?? 0;
                     usage.TooBig += bucketTooBig?.Value ?? 0;
+                    usage.Deleted += bucketDeleted?.Value ?? 0;
 
                     var hourlyUsage = project.GetHourlyUsage(bucketUtc);
                     hourlyUsage.Total += bucketTotal?.Value ?? 0;
                     hourlyUsage.Blocked += bucketBlocked?.Value ?? 0;
                     hourlyUsage.Discarded += bucketDiscarded?.Value ?? 0;
                     hourlyUsage.TooBig += bucketTooBig?.Value ?? 0;
+                    hourlyUsage.Deleted += bucketDeleted?.Value ?? 0;
 
                     project.TrimUsage(_timeProvider);
 
                     await _cache.RemoveAllAsync(new[] {
                         GetBucketTotalCacheKey(bucketUtc, project.OrganizationId, projectId),
                         GetBucketDiscardedCacheKey(bucketUtc, project.OrganizationId, projectId),
                         GetBucketBlockedCacheKey(bucketUtc, project.OrganizationId, projectId),
-                        GetBucketTooBigCacheKey(bucketUtc, project.OrganizationId, projectId)
+                        GetBucketTooBigCacheKey(bucketUtc, project.OrganizationId, projectId),
+                        GetBucketDeletedCacheKey(bucketUtc, project.OrganizationId, projectId)
                     });
 
                     await _cache.SetAsync(GetTotalCacheKey(utcNow, project.OrganizationId, projectId), usage.Total, TimeSpan.FromHours(8));
@@ -332,6 +340,10 @@ public async Task<UsageInfoResponse> GetUsageAsync(string organizationId, string
             usage.CurrentUsage.TooBig += bucketTooBig?.Value ?? 0;
             usage.CurrentHourUsage.TooBig += bucketTooBig?.Value ?? 0;
 
+            var bucketDeleted = await _cache.GetAsync<long>(GetBucketDeletedCacheKey(bucketUtc, organizationId, projectId));
+            usage.CurrentUsage.Deleted += bucketDeleted?.Value ?? 0;
+            usage.CurrentHourUsage.Deleted += bucketDeleted?.Value ?? 0;
+
             bucketUtc = bucketUtc.Add(_bucketSize);
         }
 
@@ -473,6 +485,29 @@ public async Task IncrementTooBigAsync(string organizationId, string? projectId)
         AppDiagnostics.PostTooBig.Add(1);
     }
 
+    public async Task IncrementDeletedAsync(string organizationId, string? projectId, long eventCount = 1)
+    {
+        if (eventCount <= 0)
+            return;
+
+        var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
+
+        var tasks = new List<Task>(4)
+        {
+            _cache.IncrementAsync(GetBucketDeletedCacheKey(utcNow, organizationId), eventCount, TimeSpan.FromHours(8)),
+            _cache.ListAddAsync(GetOrganizationSetKey(utcNow), organizationId, TimeSpan.FromHours(8))
+        };
+
+        if (!String.IsNullOrEmpty(projectId))
+        {
+            tasks.Add(_cache.IncrementAsync(GetBucketDeletedCacheKey(utcNow, organizationId, projectId), eventCount, TimeSpan.FromHours(8)));
+            tasks.Add(_cache.ListAddAsync(GetProjectSetKey(utcNow), projectId, TimeSpan.FromHours(8)));
+        }
+
+        await Task.WhenAll(tasks);
+        AppDiagnostics.EventsDeleted.Add(eventCount);
+    }
+
     private int GetBucketEventLimit(int maxEventsPerMonth)
     {
         if (maxEventsPerMonth < 5000)
@@ -539,6 +574,16 @@ private string GetBucketTooBigCacheKey(DateTime utcTime, string organizationId,
         return $"usage:{bucket}:{organizationId}:{projectId}:toobig";
     }
 
+    private string GetBucketDeletedCacheKey(DateTime utcTime, string organizationId, string? projectId = null)
+    {
+        int bucket = GetCurrentBucket(utcTime);
+
+        if (String.IsNullOrEmpty(projectId))
+            return $"usage:{bucket}:{organizationId}:deleted";
+
+        return $"usage:{bucket}:{organizationId}:{projectId}:deleted";
+    }
+
     private string GetOrganizationSetKey(DateTime utcTime)
     {
         int bucket = GetCurrentBucket(utcTime);

src/Exceptionless.Core/Utility/AppDiagnostics.cs

@@ -97,6 +97,7 @@ public GaugeInfo(Meter meter, string name)
     internal static readonly Counter<int> EventsDiscarded = Meter.CreateCounter<int>("ex.events.discarded", description: "Events that were discarded");
     internal static readonly Counter<int> EventsBlocked = Meter.CreateCounter<int>("ex.events.blocked", description: "Events that were blocked");
     internal static readonly Counter<int> EventsProcessCancelled = Meter.CreateCounter<int>("ex.events.processing.cancelled", description: "Events that started processing and were cancelled");
+    internal static readonly Counter<long> EventsDeleted = Meter.CreateCounter<long>("ex.events.deleted", description: "Events that were deleted");
     internal static readonly Counter<int> EventsRetryCount = Meter.CreateCounter<int>("ex.events.retry.count", description: "Events where processing was retried");
     internal static readonly Counter<int> EventsRetryErrors = Meter.CreateCounter<int>("ex.events.retry.errors", description: "Events where retry processing got an error");
     internal static readonly Histogram<double> EventsFieldCount = Meter.CreateHistogram<double>("ex.events.field.count", description: "Number of fields per event");

src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage-controller.js

@@ -146,6 +146,10 @@
                         });
 
                         vm.chart.options.series[4].data = vm.organization.usage.map(function (item) {
+                            return { x: moment.utc(item.date).unix(), y: item.deleted || 0, data: item };
+                        });
+
+                        vm.chart.options.series[5].data = vm.organization.usage.map(function (item) {
                             return { x: moment.utc(item.date).unix(), y: item.limit, data: item };
                         });
 
@@ -287,6 +291,11 @@
                                     color: "#ccc",
                                     renderer: "stack",
                                 },
+                                {
+                                    name: translateService.T("Deleted"),
+                                    color: "#f0ad4e",
+                                    renderer: "stack",
+                                },
                                 {
                                     name: translateService.T("Limit"),
                                     color: "#a94442",

src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage-controller.js

@@ -261,7 +261,11 @@
                             return { x: moment.utc(item.date).unix(), y: item.too_big, data: item };
                         });
 
-                        vm.chart.options.series[5].data = vm.organization.usage.map(function (item) {
+                        vm.chart.options.series[5].data = vm.project.usage.map(function (item) {
+                            return { x: moment.utc(item.date).unix(), y: item.deleted || 0, data: item };
+                        });
+
+                        vm.chart.options.series[6].data = vm.organization.usage.map(function (item) {
                             return { x: moment.utc(item.date).unix(), y: item.limit, data: item };
                         });
 
@@ -788,6 +792,11 @@
                                     color: "#ccc",
                                     renderer: "stack",
                                 },
+                                {
+                                    name: translateService.T("Deleted"),
+                                    color: "#f0ad4e",
+                                    renderer: "stack",
+                                },
                                 {
                                     name: translateService.T("Limit"),
                                     color: "#a94442",

src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts

@@ -407,6 +407,8 @@ export interface UsageHourInfo {
   discarded: number;
   /** @format int32 */
   too_big: number;
+  /** @format int32 */
+  deleted: number;
 }
 
 export interface UsageInfo {
@@ -422,6 +424,8 @@ export interface UsageInfo {
   discarded: number;
   /** @format int32 */
   too_big: number;
+  /** @format int32 */
+  deleted: number;
 }
 
 export interface User {

src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts

@@ -460,6 +460,7 @@ export const UsageHourInfoSchema = object({
   blocked: int32(),
   discarded: int32(),
   too_big: int32(),
+  deleted: number(),
 });
 export type UsageHourInfoFormData = Infer<typeof UsageHourInfoSchema>;
 
@@ -470,6 +471,7 @@ export const UsageInfoSchema = object({
   blocked: int32(),
   discarded: int32(),
   too_big: int32(),
+  deleted: number(),
 });
 export type UsageInfoFormData = Infer<typeof UsageInfoSchema>;