Skip to content

Commit 97d5cc7

Browse files
rian-berian-be
committed
benchmarks: add policy evaluation suite for #57 
1 parent 575bca2 commit 97d5cc7

4 files changed

Lines changed: 240 additions & 0 deletions

File tree

‎Benchmarks/Policy/PolicyBenchmarkSupport.cs‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Changes;
4+
using ModularityKit.Mutator.Abstractions.Context;
5+
using ModularityKit.Mutator.Abstractions.Engine;
6+
using ModularityKit.Mutator.Abstractions.Intent;
7+
using ModularityKit.Mutator.Abstractions.Policies;
8+
using ModularityKit.Mutator.Abstractions.Results;
9+
using ModularityKit.Mutator.Runtime;
10+
11+
namespace ModularityKit.Mutator.Benchmarks.Policy;
12+
13+
/// <summary>
14+
/// Shared support types for policy evaluation benchmark scenarios.
15+
/// </summary>
16+
internal static class PolicyBenchmarkSupport
17+
{
18+
public const string StateId = "policy-benchmark-state";
19+
20+
public static IMutationEngine BuildEngine(Action<IMutationEngine>? configure = null)
21+
{
22+
var services = new ServiceCollection();
23+
services.AddMutators(MutationEngineOptions.Performance);
24+
25+
var engine = services
26+
.BuildServiceProvider()
27+
.GetRequiredService<IMutationEngine>();
28+
29+
configure?.Invoke(engine);
30+
return engine;
31+
}
32+
33+
public static MinimalPolicyMutation CreateMutation()
34+
{
35+
var context = MutationContext.System("policy-benchmark")
36+
with
37+
{
38+
StateId = StateId,
39+
Mode = MutationMode.Commit,
40+
CorrelationId = $"{StateId}:policy-pass"
41+
};
42+
43+
return new MinimalPolicyMutation(context);
44+
}
45+
46+
/// <summary>
47+
/// Minimal state used by policy evaluation benchmarks.
48+
/// </summary>
49+
/// <param name="Name">The logical state name.</param>
50+
/// <param name="Counter">The mutable numeric field exercised by the benchmark mutation.</param>
51+
public sealed record PolicyBenchmarkState(string Name, int Counter);
52+
53+
/// <summary>
54+
/// Minimal mutation used to isolate policy pipeline overhead from unrelated runtime work.
55+
/// </summary>
56+
public sealed class MinimalPolicyMutation(MutationContext context) : IMutation<PolicyBenchmarkState>
57+
{
58+
public MutationIntent Intent { get; } = new()
59+
{
60+
OperationName = "MinimalPolicyMutation",
61+
Category = "Benchmark",
62+
Description = "Minimal mutation used to isolate policy evaluation overhead.",
63+
RiskLevel = MutationRiskLevel.Low,
64+
IsReversible = true
65+
};
66+
67+
public MutationContext Context { get; } = context;
68+
69+
public MutationResult<PolicyBenchmarkState> Apply(PolicyBenchmarkState state)
70+
{
71+
var nextState = state with { Counter = state.Counter + 1 };
72+
73+
return MutationResult<PolicyBenchmarkState>.Success(
74+
nextState,
75+
ChangeSet.Single(StateChange.Modified(nameof(PolicyBenchmarkState.Counter), state.Counter, nextState.Counter)));
76+
}
77+
78+
public ValidationResult Validate(PolicyBenchmarkState state) => ValidationResult.Success();
79+
80+
public MutationResult<PolicyBenchmarkState> Simulate(PolicyBenchmarkState state) => Apply(state);
81+
}
82+
83+
/// <summary>
84+
/// Synchronous allow policy used in benchmark scenarios.
85+
/// </summary>
86+
public sealed class SyncAllowBenchmarkPolicy : IMutationPolicy<PolicyBenchmarkState>
87+
{
88+
public SyncAllowBenchmarkPolicy(int priority) => Priority = priority;
89+
90+
public string Name => $"{nameof(SyncAllowBenchmarkPolicy)}_{Priority}";
91+
92+
public int Priority { get; }
93+
94+
public string? Description => "Synchronous allow policy for benchmark measurements.";
95+
96+
public PolicyDecision Evaluate(IMutation<PolicyBenchmarkState> mutation, PolicyBenchmarkState state)
97+
=> PolicyDecision.Allow(Name, "Synchronous benchmark policy allowed the mutation.");
98+
}
99+
100+
/// <summary>
101+
/// Asynchronous allow policy used in benchmark scenarios.
102+
/// </summary>
103+
public sealed class AsyncAllowBenchmarkPolicy : IMutationPolicy<PolicyBenchmarkState>
104+
{
105+
public AsyncAllowBenchmarkPolicy(int priority) => Priority = priority;
106+
107+
public string Name => $"{nameof(AsyncAllowBenchmarkPolicy)}_{Priority}";
108+
109+
public int Priority { get; }
110+
111+
public string? Description => "Asynchronous allow policy for benchmark measurements.";
112+
113+
public async Task<PolicyDecision> EvaluateAsync(
114+
IMutation<PolicyBenchmarkState> mutation,
115+
PolicyBenchmarkState state,
116+
CancellationToken cancellationToken = default)
117+
{
118+
await Task.CompletedTask;
119+
cancellationToken.ThrowIfCancellationRequested();
120+
121+
return PolicyDecision.Allow(Name, "Asynchronous benchmark policy allowed the mutation.");
122+
}
123+
}
124+
}

‎Benchmarks/Policy/PolicyEvaluationMultiBenchmarks.cs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions.Engine;
3+
4+
namespace ModularityKit.Mutator.Benchmarks.Policy;
5+
6+
/// <summary>
7+
/// Benchmarks aggregate overhead for multiple policies evaluated in a single runtime pass.
8+
/// </summary>
9+
[MemoryDiagnoser]
10+
[InProcess]
11+
public class PolicyEvaluationMultiBenchmarks
12+
{
13+
private IMutationEngine _multiPolicyEngine = null!;
14+
private PolicyBenchmarkSupport.PolicyBenchmarkState _state = null!;
15+
private PolicyBenchmarkSupport.MinimalPolicyMutation _mutation = null!;
16+
17+
/// <summary>
18+
/// Prepares an engine with mixed synchronous and asynchronous allow policies.
19+
/// </summary>
20+
[GlobalSetup]
21+
public void Setup()
22+
{
23+
_multiPolicyEngine = PolicyBenchmarkSupport.BuildEngine(
24+
engine =>
25+
{
26+
engine.RegisterPolicy(new PolicyBenchmarkSupport.SyncAllowBenchmarkPolicy(priority: 300));
27+
engine.RegisterPolicy(new PolicyBenchmarkSupport.AsyncAllowBenchmarkPolicy(priority: 200));
28+
engine.RegisterPolicy(new PolicyBenchmarkSupport.SyncAllowBenchmarkPolicy(priority: 100));
29+
engine.RegisterPolicy(new PolicyBenchmarkSupport.AsyncAllowBenchmarkPolicy(priority: 0));
30+
});
31+
32+
_state = new PolicyBenchmarkSupport.PolicyBenchmarkState("alpha", 42);
33+
_mutation = PolicyBenchmarkSupport.CreateMutation();
34+
}
35+
36+
/// <summary>
37+
/// Measures the overhead of evaluating several allow policies in priority order.
38+
/// </summary>
39+
[Benchmark]
40+
public async Task MultipleMixedPolicies_Allow()
41+
{
42+
var result = await _multiPolicyEngine.ExecuteAsync(_mutation, _state);
43+
GC.KeepAlive(result);
44+
}
45+
}

‎Benchmarks/Policy/PolicyEvaluationSingleBenchmarks.cs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions.Engine;
3+
4+
namespace ModularityKit.Mutator.Benchmarks.Policy;
5+
6+
/// <summary>
7+
/// Benchmarks single-policy evaluation overhead against a no-policy baseline.
8+
/// </summary>
9+
[MemoryDiagnoser]
10+
[InProcess]
11+
public class PolicyEvaluationSingleBenchmarks
12+
{
13+
private IMutationEngine _baselineEngine = null!;
14+
private IMutationEngine _syncPolicyEngine = null!;
15+
private IMutationEngine _asyncPolicyEngine = null!;
16+
private PolicyBenchmarkSupport.PolicyBenchmarkState _state = null!;
17+
private PolicyBenchmarkSupport.MinimalPolicyMutation _mutation = null!;
18+
19+
/// <summary>
20+
/// Prepares the baseline, synchronous-policy, and asynchronous-policy engines.
21+
/// </summary>
22+
[GlobalSetup]
23+
public void Setup()
24+
{
25+
_baselineEngine = PolicyBenchmarkSupport.BuildEngine();
26+
_syncPolicyEngine = PolicyBenchmarkSupport.BuildEngine(
27+
engine => engine.RegisterPolicy(new PolicyBenchmarkSupport.SyncAllowBenchmarkPolicy(priority: 100)));
28+
_asyncPolicyEngine = PolicyBenchmarkSupport.BuildEngine(
29+
engine => engine.RegisterPolicy(new PolicyBenchmarkSupport.AsyncAllowBenchmarkPolicy(priority: 100)));
30+
_state = new PolicyBenchmarkSupport.PolicyBenchmarkState("alpha", 42);
31+
_mutation = PolicyBenchmarkSupport.CreateMutation();
32+
}
33+
34+
/// <summary>
35+
/// Measures the execution baseline without any registered policies.
36+
/// </summary>
37+
[Benchmark(Baseline = true)]
38+
public async Task NoPolicy_Baseline()
39+
{
40+
var result = await _baselineEngine.ExecuteAsync(_mutation, _state);
41+
GC.KeepAlive(result);
42+
}
43+
44+
/// <summary>
45+
/// Measures the overhead of one synchronous allow policy.
46+
/// </summary>
47+
[Benchmark]
48+
public async Task SingleSyncPolicy_Allow()
49+
{
50+
var result = await _syncPolicyEngine.ExecuteAsync(_mutation, _state);
51+
GC.KeepAlive(result);
52+
}
53+
54+
/// <summary>
55+
/// Measures the overhead of one asynchronous allow policy.
56+
/// </summary>
57+
[Benchmark]
58+
public async Task SingleAsyncPolicy_Allow()
59+
{
60+
var result = await _asyncPolicyEngine.ExecuteAsync(_mutation, _state);
61+
GC.KeepAlive(result);
62+
}
63+
}

‎Benchmarks/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ This folder contains BenchmarkDotNet measurements for `ModularityKit.Mutator`.
1010
- batch execution overhead
1111
- throughput for single mutation execution across multiple state sizes
1212
- throughput for batch mutation execution across multiple state sizes and batch sizes
13+
- policy evaluation overhead for no-policy, synchronous policy, asynchronous policy, and mixed multi-policy runs
1314

1415
The throughput benchmarks use a cloned array-backed state so state-size effects remain visible in the
1516
actual mutation path rather than being hidden behind an artificial inner loop.
@@ -34,6 +35,13 @@ Run the throughput-focused suite:
3435
dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*MutationEngineThroughputBenchmarks*'
3536
```
3637

38+
Run the policy evaluation suite:
39+
40+
```bash
41+
dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*PolicyEvaluationSingleBenchmarks*'
42+
dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*PolicyEvaluationMultiBenchmarks*'
43+
```
44+
3745
Key parameters reported by BenchmarkDotNet:
3846

3947
- `StateSize` controls the size of the cloned mutation state

0 commit comments

Comments
 (0)