Skip to content

Commit c78a464

Browse files
rian-berian-be
authored
Feat: add results benchmark suite (#72)
2 parents 3a73522 + f6c4399 commit c78a464

5 files changed

Lines changed: 302 additions & 1 deletion

File tree

‎.github/workflows/publish-docs.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
name: Publish Docs
22

3+
env:
4+
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
5+
36
on:
47
push:
58
branches: [ main ]
@@ -49,9 +52,15 @@ jobs:
4952
needs: build
5053
environment:
5154
name: github-pages
52-
url: ${{ steps.deployment.outputs.page_url }}
55+
url: ${{ steps.deployment_retry.outputs.page_url || steps.deployment.outputs.page_url }}
5356

5457
steps:
5558
- name: Deploy to GitHub Pages
5659
id: deployment
5760
uses: actions/deploy-pages@v4
61+
continue-on-error: true
62+
63+
- name: Retry GitHub Pages deployment
64+
id: deployment_retry
65+
if: steps.deployment.outcome == 'failure'
66+
uses: actions/deploy-pages@v4

‎Benchmarks/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ This folder contains BenchmarkDotNet measurements for `ModularityKit.Mutator`.
1313
- policy evaluation overhead for no policy, synchronous policy, asynchronous policy, and mixed multi policy runs
1414
- interception, audit/history, and logging diagnostics overhead in the core runtime
1515
- parallel execution, state gate contention, and concurrent batch scheduling pressure in the core runtime
16+
- mutation result creation and history/audit output materialization in the core runtime
1617

1718
The throughput benchmarks use cloned array backed state so state size effects remain visible in the
1819
actual mutation path rather than being hidden behind an artificial inner loop.
@@ -56,6 +57,12 @@ Run the concurrency suite:
5657
dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --anyCategories Concurrency
5758
```
5859

60+
Run the results suite:
61+
62+
```bash
63+
dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --anyCategories Results
64+
```
65+
5966
Key parameters reported by BenchmarkDotNet:
6067

6168
- `StateSize` controls the size of the cloned mutation state

‎Benchmarks/Results/MutationOutputMaterializationBenchmarks.cs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions.Audit;
3+
using ModularityKit.Mutator.Abstractions.History;
4+
using ModularityKit.Mutator.Abstractions.Results;
5+
using ModularityKit.Mutator.Benchmarks.Results.Support;
6+
7+
namespace ModularityKit.Mutator.Benchmarks.Results;
8+
9+
/// <summary>
10+
/// Benchmarks materialization of history and audit output from an executed mutation result.
11+
/// </summary>
12+
[BenchmarkCategory("Results")]
13+
[MemoryDiagnoser]
14+
[InProcess]
15+
public class MutationOutputMaterializationBenchmarks
16+
{
17+
private MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState> _result = null!;
18+
private string _executionId = string.Empty;
19+
private TimeSpan _duration;
20+
21+
/// <summary>
22+
/// Prepares a representative executed mutation result for output materialization benchmarks.
23+
/// </summary>
24+
[GlobalSetup]
25+
public void Setup()
26+
{
27+
_result = ResultsBenchmarkSupport.CreateExecutedResult(sideEffectCount: 3, changeCount: 4);
28+
_executionId = "results-benchmark-execution";
29+
_duration = TimeSpan.FromMilliseconds(2);
30+
}
31+
32+
/// <summary>
33+
/// Measures materialization of the mutation history entry, including change and side effect copying.
34+
/// </summary>
35+
[Benchmark(Baseline = true)]
36+
public MutationHistoryEntry HistoryEntry_Materialization()
37+
{
38+
return new MutationHistoryEntry
39+
{
40+
ExecutionId = _executionId,
41+
StateId = ResultsBenchmarkSupport.StateId,
42+
Intent = ResultsBenchmarkSupport.CreateIntent(
43+
"ResultHistoryMaterialization",
44+
"Materialize history output for benchmark results."),
45+
Context = ResultsBenchmarkSupport.CreateContext("history"),
46+
Changes = _result.Changes,
47+
SideEffects = _result.SideEffects.ToList(),
48+
Timestamp = DateTimeOffset.UtcNow,
49+
ExecutionTime = _duration
50+
};
51+
}
52+
53+
/// <summary>
54+
/// Measures materialization of the audit entry produced from the same executed mutation result.
55+
/// </summary>
56+
[Benchmark]
57+
public MutationAuditEntry AuditEntry_Materialization()
58+
{
59+
return new MutationAuditEntry
60+
{
61+
ExecutionId = _executionId,
62+
StateId = ResultsBenchmarkSupport.StateId,
63+
StateType = nameof(ResultsBenchmarkSupport.ResultBenchmarkState),
64+
MutationIntent = ResultsBenchmarkSupport.CreateIntent(
65+
"ResultAuditMaterialization",
66+
"Materialize audit output for benchmark results."),
67+
Context = ResultsBenchmarkSupport.CreateContext("audit"),
68+
Changes = _result.Changes,
69+
IsSuccess = _result.IsSuccess,
70+
ErrorMessage = null,
71+
PolicyDecisions = _result.PolicyDecisions,
72+
SideEffects = _result.SideEffects,
73+
Timestamp = DateTimeOffset.UtcNow,
74+
Duration = _duration
75+
};
76+
}
77+
}

‎Benchmarks/Results/MutationResultCreationBenchmarks.cs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions.Changes;
3+
using ModularityKit.Mutator.Abstractions.Effects;
4+
using ModularityKit.Mutator.Abstractions.Results;
5+
using ModularityKit.Mutator.Benchmarks.Results.Support;
6+
7+
namespace ModularityKit.Mutator.Benchmarks.Results;
8+
9+
/// <summary>
10+
/// Benchmarks the cost of creating mutation results with and without side effects.
11+
/// </summary>
12+
[BenchmarkCategory("Results")]
13+
[MemoryDiagnoser]
14+
[InProcess]
15+
public class MutationResultCreationBenchmarks
16+
{
17+
private ResultsBenchmarkSupport.ResultBenchmarkState _state = null!;
18+
private ChangeSet _changes = null!;
19+
20+
/// <summary>
21+
/// Prepares the shared state and change set used by the result creation cases.
22+
/// </summary>
23+
[GlobalSetup]
24+
public void Setup()
25+
{
26+
_state = new ResultsBenchmarkSupport.ResultBenchmarkState(0, 42);
27+
_changes = ResultsBenchmarkSupport.CreateChangeSet(_state.Revision, 2);
28+
}
29+
30+
/// <summary>
31+
/// Measures creation of a successful mutation result with no side effects.
32+
/// </summary>
33+
[Benchmark(Baseline = true)]
34+
public MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState> Success_NoSideEffects()
35+
=> MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState>.Success(
36+
_state with
37+
{
38+
Revision = _state.Revision + 1,
39+
Value = _state.Value + 1
40+
},
41+
_changes);
42+
43+
/// <summary>
44+
/// Measures creation of a successful mutation result with one side effect.
45+
/// </summary>
46+
[Benchmark]
47+
public MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState> Success_SingleSideEffect()
48+
{
49+
var sideEffect = SideEffect.Create(
50+
"ResultMaterialization",
51+
"Single side effect",
52+
new ResultsBenchmarkSupport.SideEffectPayload(1, "single"),
53+
SideEffectSeverity.Info);
54+
55+
return MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState>.Success(
56+
_state with
57+
{
58+
Revision = _state.Revision + 1,
59+
Value = _state.Value + 1
60+
},
61+
_changes,
62+
[sideEffect]);
63+
}
64+
65+
/// <summary>
66+
/// Measures creation of a successful mutation result with several side effects.
67+
/// </summary>
68+
[Benchmark]
69+
public MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState> Success_MultipleSideEffects()
70+
=> MutationResult<ResultsBenchmarkSupport.ResultBenchmarkState>.Success(
71+
_state with
72+
{
73+
Revision = _state.Revision + 1,
74+
Value = _state.Value + 1
75+
},
76+
_changes,
77+
ResultsBenchmarkSupport.CreateSideEffects(4));
78+
}

‎Benchmarks/Results/Support/ResultsBenchmarkSupport.cs‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
using ModularityKit.Mutator.Abstractions.Audit;
2+
using ModularityKit.Mutator.Abstractions.Changes;
3+
using ModularityKit.Mutator.Abstractions.Context;
4+
using ModularityKit.Mutator.Abstractions.Effects;
5+
using ModularityKit.Mutator.Abstractions.Engine;
6+
using ModularityKit.Mutator.Abstractions.Intent;
7+
using ModularityKit.Mutator.Abstractions.Results;
8+
9+
namespace ModularityKit.Mutator.Benchmarks.Results.Support;
10+
11+
/// <summary>
12+
/// Shared support types for result materialization benchmark scenarios.
13+
/// </summary>
14+
public static class ResultsBenchmarkSupport
15+
{
16+
/// <summary>
17+
/// Gets the shared state identifier used by result benchmark cases.
18+
/// </summary>
19+
public const string StateId = "results-benchmark-state";
20+
21+
/// <summary>
22+
/// Creates a reusable mutation context for result benchmarks.
23+
/// </summary>
24+
/// <param name="operationSuffix">The suffix used to distinguish benchmark cases.</param>
25+
/// <returns>A system mutation context bound to the shared benchmark state.</returns>
26+
public static MutationContext CreateContext(string operationSuffix)
27+
{
28+
return MutationContext.System("results-benchmark")
29+
with
30+
{
31+
StateId = StateId,
32+
Mode = MutationMode.Commit,
33+
CorrelationId = $"{StateId}:{operationSuffix}"
34+
};
35+
}
36+
37+
/// <summary>
38+
/// Creates a reusable mutation intent for result benchmarks.
39+
/// </summary>
40+
/// <param name="operationName">The operation name reported by the mutation.</param>
41+
/// <param name="description">The human-readable description.</param>
42+
/// <returns>A benchmark mutation intent.</returns>
43+
public static MutationIntent CreateIntent(string operationName, string description)
44+
=> new()
45+
{
46+
OperationName = operationName,
47+
Category = "Benchmark",
48+
Description = description,
49+
RiskLevel = MutationRiskLevel.Low,
50+
IsReversible = true
51+
};
52+
53+
/// <summary>
54+
/// Creates a change set with a stable revision marker and a configurable number of appended slot updates.
55+
/// </summary>
56+
/// <param name="revision">The revision number before mutation.</param>
57+
/// <param name="updates">The number of slot updates to include.</param>
58+
/// <returns>A populated change set.</returns>
59+
public static ChangeSet CreateChangeSet(int revision, int updates)
60+
{
61+
var changes = new List<StateChange>(updates + 1)
62+
{
63+
StateChange.Modified(nameof(ResultBenchmarkState.Revision), revision, revision + 1)
64+
};
65+
66+
for (var index = 0; index < updates; index++)
67+
{
68+
changes.Add(StateChange.Modified(
69+
$"Slots[{index}]",
70+
index,
71+
index + 1));
72+
}
73+
74+
return ChangeSet.FromChanges([.. changes]);
75+
}
76+
77+
/// <summary>
78+
/// Creates a fixed list of side effects with predictable payload shape.
79+
/// </summary>
80+
/// <param name="count">The number of side effects to create.</param>
81+
/// <returns>A read-only side effect list.</returns>
82+
public static IReadOnlyList<SideEffect> CreateSideEffects(int count)
83+
{
84+
var sideEffects = new List<SideEffect>(count);
85+
86+
for (var index = 0; index < count; index++)
87+
{
88+
sideEffects.Add(SideEffect.Create(
89+
"ResultMaterialization",
90+
$"Side effect #{index}",
91+
new SideEffectPayload(index, $"payload-{index}"),
92+
SideEffectSeverity.Info));
93+
}
94+
95+
return sideEffects;
96+
}
97+
98+
/// <summary>
99+
/// Creates a reusable mutation result used as the input for output materialization benchmarks.
100+
/// </summary>
101+
/// <param name="sideEffectCount">The number of side effects to attach to the result.</param>
102+
/// <param name="changeCount">The number of slot changes to include in the result.</param>
103+
/// <returns>A mutation result prepopulated with changes and side effects.</returns>
104+
public static MutationResult<ResultBenchmarkState> CreateExecutedResult(
105+
int sideEffectCount,
106+
int changeCount)
107+
{
108+
var state = new ResultBenchmarkState(42, 0);
109+
var nextState = state with { Revision = state.Revision + 1 };
110+
111+
return MutationResult<ResultBenchmarkState>.Success(
112+
nextState,
113+
CreateChangeSet(state.Revision, changeCount),
114+
CreateSideEffects(sideEffectCount));
115+
}
116+
117+
/// <summary>
118+
/// Minimal state used by result benchmark scenarios.
119+
/// </summary>
120+
/// <param name="Revision">The revision counter advanced on each benchmark mutation.</param>
121+
/// <param name="Value">The mutable numeric value exercised by the benchmark mutation.</param>
122+
public sealed record ResultBenchmarkState(int Revision, int Value);
123+
124+
/// <summary>
125+
/// Typed payload used to give side effects realistic materialization shape.
126+
/// </summary>
127+
/// <param name="Index">The ordinal of the side effect.</param>
128+
/// <param name="Token">A stable payload token.</param>
129+
public sealed record SideEffectPayload(int Index, string Token);
130+
}

0 commit comments

Comments
 (0)