Skip to content

Commit 3a73522

Browse files
rian-berian-be
authored
Feat: introduce diagnostics and concurrency benchmark suites (#71)
2 parents ade5baf + 039200f commit 3a73522

22 files changed

Lines changed: 1039 additions & 5 deletions

‎.github/workflows/publish-artifacts.yml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ on:
66
signing_enabled:
77
description: "Whether package signing was enabled for this run."
88
value: ${{ jobs.package.outputs.signing_enabled }}
9+
signature_verification_enabled:
10+
description: "Whether signed package verification was enabled for this run."
11+
value: ${{ jobs.package.outputs.signature_verification_enabled }}
912
inputs:
1013
package_version:
1114
description: "Optional package version, usually the release tag without the leading v."
@@ -47,6 +50,8 @@ on:
4750
required: false
4851
NUGET_SIGN_TIMESTAMP_URL:
4952
required: false
53+
NUGET_SIGN_SKIP_VERIFICATION:
54+
required: false
5055

5156
permissions:
5257
contents: read
@@ -57,6 +62,7 @@ jobs:
5762
runs-on: ubuntu-latest
5863
outputs:
5964
signing_enabled: ${{ steps.signing.outputs.enabled }}
65+
signature_verification_enabled: ${{ steps.signing.outputs.verify_enabled }}
6066

6167
steps:
6268
- name: Checkout
@@ -150,6 +156,7 @@ jobs:
150156
NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }}
151157
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
152158
NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }}
159+
NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }}
153160
run: |
154161
if [ -n "$NUGET_SIGN_CERTIFICATE_BASE64" ] \
155162
&& [ -n "$NUGET_SIGN_CERTIFICATE_PASSWORD" ] \
@@ -162,6 +169,14 @@ jobs:
162169
echo "Package signing disabled. Continuing with unsigned packages."
163170
fi
164171
172+
if [ "${NUGET_SIGN_SKIP_VERIFICATION,,}" = "true" ]; then
173+
echo "verify_enabled=false" >> "$GITHUB_OUTPUT"
174+
echo "Signed package verification disabled by NUGET_SIGN_SKIP_VERIFICATION."
175+
else
176+
echo "verify_enabled=true" >> "$GITHUB_OUTPUT"
177+
echo "Signed package verification enabled."
178+
fi
179+
165180
- name: Import signing certificate
166181
if: ${{ steps.signing.outputs.enabled == 'true' }}
167182
env:
@@ -186,7 +201,7 @@ jobs:
186201
done
187202
188203
- name: Verify signed packages
189-
if: ${{ steps.signing.outputs.enabled == 'true' }}
204+
if: ${{ steps.signing.outputs.enabled == 'true' && steps.signing.outputs.verify_enabled == 'true' }}
190205
env:
191206
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
192207
run: |

‎.github/workflows/publish-attested.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ on:
2222
required: false
2323
NUGET_SIGN_TIMESTAMP_URL:
2424
required: false
25+
NUGET_SIGN_SKIP_VERIFICATION:
26+
required: false
2527

2628
permissions:
2729
contents: write
@@ -39,6 +41,7 @@ jobs:
3941
NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }}
4042
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
4143
NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }}
44+
NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }}
4245

4346
release:
4447
name: Upload artifacts to draft release

‎.github/workflows/publish-packages.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ jobs:
106106
NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }}
107107
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
108108
NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }}
109+
NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }}
109110

110111
publish-nuget:
111112
name: Publish to NuGet.org
@@ -130,7 +131,7 @@ jobs:
130131
merge-multiple: true
131132

132133
- name: Verify signed packages
133-
if: ${{ needs.package.outputs.signing_enabled == 'true' }}
134+
if: ${{ needs.package.outputs.signing_enabled == 'true' && needs.package.outputs.signature_verification_enabled == 'true' }}
134135
env:
135136
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
136137
run: |
@@ -180,7 +181,7 @@ jobs:
180181
merge-multiple: true
181182

182183
- name: Verify signed packages
183-
if: ${{ needs.package.outputs.signing_enabled == 'true' }}
184+
if: ${{ needs.package.outputs.signing_enabled == 'true' && needs.package.outputs.signature_verification_enabled == 'true' }}
184185
env:
185186
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
186187
run: |

‎.github/workflows/release-drafter.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,4 @@ jobs:
4343
NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }}
4444
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
4545
NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }}
46+
NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }}

‎Benchmarks/Concurrency/BatchSchedulingBenchmarks.cs‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
using BenchmarkDotNet.Attributes;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using ModularityKit.Mutator.Abstractions.Audit;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
using ModularityKit.Mutator.Abstractions.History;
6+
using ModularityKit.Mutator.Benchmarks.Concurrency.Support;
7+
using ModularityKit.Mutator.Benchmarks.Diagnostics.Support;
8+
9+
namespace ModularityKit.Mutator.Benchmarks.Concurrency;
10+
11+
/// <summary>
12+
/// Benchmarks concurrent batch executions competing for limited runtime availability.
13+
/// </summary>
14+
[BenchmarkCategory("Concurrency")]
15+
[MemoryDiagnoser]
16+
[InProcess]
17+
public class BatchSchedulingBenchmarks
18+
{
19+
private const int RuntimeSlots = 2;
20+
21+
private IMutationEngine _engine = null!;
22+
private BatchScenario[] _scenarios = null!;
23+
24+
/// <summary>
25+
/// Controls how many batch executions compete for runtime slots during a single benchmark iteration.
26+
/// </summary>
27+
[Params(2, 4)]
28+
public int ConcurrentBatches { get; set; }
29+
30+
/// <summary>
31+
/// Controls how many mutations each competing batch executes.
32+
/// </summary>
33+
[Params(4, 16)]
34+
public int BatchSize { get; set; }
35+
36+
/// <summary>
37+
/// Prepares the engine and precomputed batch scenarios for the selected parameters.
38+
/// </summary>
39+
[GlobalSetup]
40+
public void Setup()
41+
{
42+
_engine = ConcurrencyBenchmarkScenario.BuildEngine(
43+
RuntimeSlots,
44+
services =>
45+
{
46+
services.AddSingleton<IMutationAuditor, NoOpAuditor>();
47+
services.AddSingleton<IMutationHistoryStore, NoOpHistoryStore>();
48+
});
49+
50+
_scenarios = Enumerable
51+
.Range(0, ConcurrentBatches)
52+
.Select(CreateBatchScenario)
53+
.ToArray();
54+
}
55+
56+
/// <summary>
57+
/// Measures scheduler pressure when several ordered batches compete for a limited number of engine slots.
58+
/// </summary>
59+
[Benchmark]
60+
public async Task ConcurrentBatches_LimitedRuntimeAvailability()
61+
{
62+
var tasks = new Task[ConcurrentBatches];
63+
64+
for (var index = 0; index < ConcurrentBatches; index++)
65+
{
66+
var scenario = _scenarios[index];
67+
tasks[index] = _engine.ExecuteBatchAsync(scenario.Mutations, scenario.State);
68+
}
69+
70+
await Task.WhenAll(tasks);
71+
GC.KeepAlive(tasks);
72+
}
73+
74+
private BatchScenario CreateBatchScenario(int batchIndex)
75+
{
76+
var stateId = $"batch-state-{batchIndex}";
77+
var mutations = Enumerable
78+
.Range(0, BatchSize)
79+
.Select(step => (IMutation<ConcurrencyState>)ConcurrencyBenchmarkScenario.CreateCommitMutation(stateId, $"batch-{batchIndex}-{step}"))
80+
.ToArray();
81+
82+
return new BatchScenario(new ConcurrencyState(batchIndex, 0), mutations);
83+
}
84+
85+
private sealed record BatchScenario(
86+
ConcurrencyState State,
87+
IReadOnlyList<IMutation<ConcurrencyState>> Mutations);
88+
}

‎Benchmarks/Concurrency/GateContentionBenchmarks.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
using BenchmarkDotNet.Attributes;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using ModularityKit.Mutator.Abstractions.Audit;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
using ModularityKit.Mutator.Abstractions.History;
6+
using ModularityKit.Mutator.Benchmarks.Concurrency.Support;
7+
using ModularityKit.Mutator.Benchmarks.Diagnostics.Support;
8+
9+
namespace ModularityKit.Mutator.Benchmarks.Concurrency;
10+
11+
/// <summary>
12+
/// Benchmarks state-level gate contention in the core mutation runtime.
13+
/// </summary>
14+
[BenchmarkCategory("Concurrency")]
15+
[MemoryDiagnoser]
16+
[InProcess]
17+
public class GateContentionBenchmarks
18+
{
19+
private const string SharedStateId = "shared-concurrency-state";
20+
21+
private IMutationEngine _engine = null!;
22+
private ConcurrencyState _state = null!;
23+
24+
/// <summary>
25+
/// Prepares an engine with a concurrency limit high enough to isolate state-gate contention.
26+
/// </summary>
27+
[GlobalSetup]
28+
public void Setup()
29+
{
30+
_engine = ConcurrencyBenchmarkScenario.BuildEngine(
31+
maxConcurrentMutations: 4,
32+
configureServices: services =>
33+
{
34+
services.AddSingleton<IMutationAuditor, NoOpAuditor>();
35+
services.AddSingleton<IMutationHistoryStore, NoOpHistoryStore>();
36+
});
37+
38+
_state = new ConcurrencyState(42, 0);
39+
}
40+
41+
/// <summary>
42+
/// Measures two concurrent executions targeting the same state identifier while one execution blocks the gate.
43+
/// </summary>
44+
[Benchmark]
45+
public async Task SharedStateGate_TwoConcurrentExecutions()
46+
{
47+
using var gate = new BlockingMutationGate();
48+
var firstMutation = ConcurrencyBenchmarkScenario.CreateBlockingMutation(gate, SharedStateId, "first");
49+
var secondMutation = ConcurrencyBenchmarkScenario.CreateCommitMutation(SharedStateId, "second");
50+
51+
var firstTask = _engine.ExecuteAsync(firstMutation, _state);
52+
53+
if (!gate.WaitForEntries(expectedEntries: 1, timeout: TimeSpan.FromSeconds(5)))
54+
throw new InvalidOperationException("Blocking benchmark mutation did not enter the gate in time.");
55+
56+
var secondTask = _engine.ExecuteAsync(secondMutation, _state);
57+
58+
Thread.SpinWait(100_000);
59+
gate.Release();
60+
61+
var results = await Task.WhenAll(firstTask, secondTask);
62+
GC.KeepAlive(results);
63+
}
64+
}

‎Benchmarks/Concurrency/ParallelExecutionBenchmarks.cs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using BenchmarkDotNet.Attributes;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using ModularityKit.Mutator.Abstractions.Audit;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
using ModularityKit.Mutator.Abstractions.History;
6+
using ModularityKit.Mutator.Benchmarks.Concurrency.Support;
7+
using ModularityKit.Mutator.Benchmarks.Diagnostics.Support;
8+
9+
namespace ModularityKit.Mutator.Benchmarks.Concurrency;
10+
11+
/// <summary>
12+
/// Benchmarks parallel execution throughput across distinct runtime state identifiers.
13+
/// </summary>
14+
[BenchmarkCategory("Concurrency")]
15+
[MemoryDiagnoser]
16+
[InProcess]
17+
public class ParallelExecutionBenchmarks
18+
{
19+
private IMutationEngine _engine = null!;
20+
private ConcurrencyState[] _states = null!;
21+
private IncrementConcurrencyMutation[] _mutations = null!;
22+
23+
/// <summary>
24+
/// Controls how many distinct mutation executions run in parallel during a single benchmark iteration.
25+
/// </summary>
26+
[Params(2, 8)]
27+
public int Parallelism { get; set; }
28+
29+
/// <summary>
30+
/// Prepares the engine, state snapshots, and mutation list for the selected parallelism level.
31+
/// </summary>
32+
[GlobalSetup]
33+
public void Setup()
34+
{
35+
_engine = ConcurrencyBenchmarkScenario.BuildEngine(
36+
Parallelism,
37+
services =>
38+
{
39+
services.AddSingleton<IMutationAuditor, NoOpAuditor>();
40+
services.AddSingleton<IMutationHistoryStore, NoOpHistoryStore>();
41+
});
42+
43+
_states = Enumerable
44+
.Range(0, Parallelism)
45+
.Select(index => new ConcurrencyState(index, 0))
46+
.ToArray();
47+
48+
_mutations = Enumerable
49+
.Range(0, Parallelism)
50+
.Select(index => ConcurrencyBenchmarkScenario.CreateCommitMutation($"parallel-state-{index}", $"parallel-{index}"))
51+
.ToArray();
52+
}
53+
54+
/// <summary>
55+
/// Measures concurrent execution across distinct state identifiers without diagnostics storage noise.
56+
/// </summary>
57+
[Benchmark(Baseline = true)]
58+
public async Task ParallelDistinctStates_ExecuteAsync()
59+
{
60+
var tasks = new Task[Parallelism];
61+
62+
for (var index = 0; index < Parallelism; index++)
63+
tasks[index] = _engine.ExecuteAsync(_mutations[index], _states[index]);
64+
65+
await Task.WhenAll(tasks);
66+
GC.KeepAlive(tasks);
67+
}
68+
}

‎Benchmarks/Concurrency/Support/BlockingGateMutation.cs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using ModularityKit.Mutator.Abstractions;
2+
using ModularityKit.Mutator.Abstractions.Changes;
3+
using ModularityKit.Mutator.Abstractions.Context;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
using ModularityKit.Mutator.Abstractions.Intent;
6+
using ModularityKit.Mutator.Abstractions.Results;
7+
8+
namespace ModularityKit.Mutator.Benchmarks.Concurrency.Support;
9+
10+
/// <summary>
11+
/// Minimal commit mutation that blocks inside the execution pipeline to expose gate contention.
12+
/// </summary>
13+
internal sealed class BlockingGateMutation(
14+
MutationContext context,
15+
BlockingMutationGate gate) : IMutation<ConcurrencyState>
16+
{
17+
/// <summary>
18+
/// Gets the benchmark mutation intent metadata.
19+
/// </summary>
20+
public MutationIntent Intent { get; } = new()
21+
{
22+
OperationName = "BlockingGateMutation",
23+
Category = "Benchmark",
24+
Description = "Block inside mutation execution to expose core runtime gate contention.",
25+
RiskLevel = MutationRiskLevel.Low,
26+
IsReversible = true
27+
};
28+
29+
/// <summary>
30+
/// Gets the execution context bound to the benchmark mutation instance.
31+
/// </summary>
32+
public MutationContext Context { get; } = context;
33+
34+
/// <summary>
35+
/// Applies the benchmark mutation after waiting on the shared gate.
36+
/// </summary>
37+
/// <param name="state">The input state.</param>
38+
/// <returns>The successful mutation result containing the updated state and changes.</returns>
39+
public MutationResult<ConcurrencyState> Apply(ConcurrencyState state)
40+
{
41+
gate.Enter();
42+
43+
var nextState = state with
44+
{
45+
Counter = state.Counter + 1,
46+
Revision = state.Revision + 1
47+
};
48+
49+
return MutationResult<ConcurrencyState>.Success(
50+
nextState,
51+
ChangeSet.Single(
52+
StateChange.Modified(nameof(ConcurrencyState.Counter), state.Counter, nextState.Counter)));
53+
}
54+
55+
/// <summary>
56+
/// Validates the provided state before mutation execution.
57+
/// </summary>
58+
/// <param name="state">The input state.</param>
59+
/// <returns>A successful validation result.</returns>
60+
public ValidationResult Validate(ConcurrencyState state) => ValidationResult.Success();
61+
62+
/// <summary>
63+
/// Simulates the benchmark mutation using the same state transition as commit execution.
64+
/// </summary>
65+
/// <param name="state">The input state.</param>
66+
/// <returns>The simulated mutation result.</returns>
67+
public MutationResult<ConcurrencyState> Simulate(ConcurrencyState state) => Apply(state);
68+
}

0 commit comments

Comments
 (0)