Skip to content

Commit ade5baf

Browse files
rian-berian-be
authored
Feat: add engine throughput and policy evaluation suites (#70)
2 parents a5d6618 + 97d5cc7 commit ade5baf

15 files changed

Lines changed: 742 additions & 212 deletions

‎.github/workflows/publish-artifacts.yml‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ name: Publish Artifacts
22

33
on:
44
workflow_call:
5+
outputs:
6+
signing_enabled:
7+
description: "Whether package signing was enabled for this run."
8+
value: ${{ jobs.package.outputs.signing_enabled }}
59
inputs:
610
package_version:
711
description: "Optional package version, usually the release tag without the leading v."
@@ -36,13 +40,13 @@ on:
3640
default: true
3741
secrets:
3842
NUGET_SIGN_CERTIFICATE_BASE64:
39-
required: true
43+
required: false
4044
NUGET_SIGN_CERTIFICATE_PASSWORD:
41-
required: true
45+
required: false
4246
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT:
43-
required: true
47+
required: false
4448
NUGET_SIGN_TIMESTAMP_URL:
45-
required: true
49+
required: false
4650

4751
permissions:
4852
contents: read
@@ -51,6 +55,8 @@ jobs:
5155
package:
5256
name: Pack packages
5357
runs-on: ubuntu-latest
58+
outputs:
59+
signing_enabled: ${{ steps.signing.outputs.enabled }}
5460

5561
steps:
5662
- name: Checkout
@@ -137,32 +143,38 @@ jobs:
137143
-o nupkg
138144
-p:PackageVersion=${{ steps.version.outputs.redis_version }}
139145
140-
- name: Import signing certificate
146+
- name: Resolve signing mode
147+
id: signing
141148
env:
142149
NUGET_SIGN_CERTIFICATE_BASE64: ${{ secrets.NUGET_SIGN_CERTIFICATE_BASE64 }}
150+
NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }}
151+
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
152+
NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }}
143153
run: |
144-
if [ -z "$NUGET_SIGN_CERTIFICATE_BASE64" ]; then
145-
echo "NUGET_SIGN_CERTIFICATE_BASE64 secret is required for package signing." >&2
146-
exit 1
154+
if [ -n "$NUGET_SIGN_CERTIFICATE_BASE64" ] \
155+
&& [ -n "$NUGET_SIGN_CERTIFICATE_PASSWORD" ] \
156+
&& [ -n "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ] \
157+
&& [ -n "$NUGET_SIGN_TIMESTAMP_URL" ]; then
158+
echo "enabled=true" >> "$GITHUB_OUTPUT"
159+
echo "Package signing enabled."
160+
else
161+
echo "enabled=false" >> "$GITHUB_OUTPUT"
162+
echo "Package signing disabled. Continuing with unsigned packages."
147163
fi
148164
165+
- name: Import signing certificate
166+
if: ${{ steps.signing.outputs.enabled == 'true' }}
167+
env:
168+
NUGET_SIGN_CERTIFICATE_BASE64: ${{ secrets.NUGET_SIGN_CERTIFICATE_BASE64 }}
169+
run: |
149170
printf '%s' "$NUGET_SIGN_CERTIFICATE_BASE64" | base64 --decode > signing-cert.pfx
150171
151172
- name: Sign packages
173+
if: ${{ steps.signing.outputs.enabled == 'true' }}
152174
env:
153175
NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }}
154176
NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }}
155177
run: |
156-
if [ -z "$NUGET_SIGN_CERTIFICATE_PASSWORD" ]; then
157-
echo "NUGET_SIGN_CERTIFICATE_PASSWORD secret is required for package signing." >&2
158-
exit 1
159-
fi
160-
161-
if [ -z "$NUGET_SIGN_TIMESTAMP_URL" ]; then
162-
echo "NUGET_SIGN_TIMESTAMP_URL secret is required for package signing." >&2
163-
exit 1
164-
fi
165-
166178
for package in nupkg/*.nupkg; do
167179
dotnet nuget sign "$package" \
168180
--certificate-path signing-cert.pfx \
@@ -174,14 +186,10 @@ jobs:
174186
done
175187
176188
- name: Verify signed packages
189+
if: ${{ steps.signing.outputs.enabled == 'true' }}
177190
env:
178191
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
179192
run: |
180-
if [ -z "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ]; then
181-
echo "NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT secret is required for package verification." >&2
182-
exit 1
183-
fi
184-
185193
for package in nupkg/*.nupkg; do
186194
dotnet nuget verify "$package" \
187195
--all \

‎.github/workflows/publish-attested.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@ on:
1515
type: string
1616
secrets:
1717
NUGET_SIGN_CERTIFICATE_BASE64:
18-
required: true
18+
required: false
1919
NUGET_SIGN_CERTIFICATE_PASSWORD:
20-
required: true
20+
required: false
2121
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT:
22-
required: true
22+
required: false
2323
NUGET_SIGN_TIMESTAMP_URL:
24-
required: true
24+
required: false
2525

2626
permissions:
2727
contents: write
@@ -56,6 +56,11 @@ jobs:
5656
path: dist
5757
merge-multiple: true
5858

59+
- name: Attest package artifacts
60+
uses: actions/attest-build-provenance@v2
61+
with:
62+
subject-path: dist/*.nupkg
63+
5964
- name: Create or update draft release
6065
env:
6166
GITHUB_TOKEN: ${{ github.token }}

‎.github/workflows/publish-packages.yml‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,10 @@ jobs:
130130
merge-multiple: true
131131

132132
- name: Verify signed packages
133+
if: ${{ needs.package.outputs.signing_enabled == 'true' }}
133134
env:
134135
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
135136
run: |
136-
if [ -z "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ]; then
137-
echo "NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT secret is required for package verification." >&2
138-
exit 1
139-
fi
140-
141137
for package in dist/*.nupkg; do
142138
dotnet nuget verify "$package" \
143139
--all \
@@ -184,14 +180,10 @@ jobs:
184180
merge-multiple: true
185181

186182
- name: Verify signed packages
183+
if: ${{ needs.package.outputs.signing_enabled == 'true' }}
187184
env:
188185
NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }}
189186
run: |
190-
if [ -z "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ]; then
191-
echo "NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT secret is required for package verification." >&2
192-
exit 1
193-
fi
194-
195187
for package in dist/*.nupkg; do
196188
dotnet nuget verify "$package" \
197189
--all \

‎Benchmarks/Engine/MutationEngineBatchBenchmarks.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Context;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
6+
namespace ModularityKit.Mutator.Benchmarks.Engine;
7+
8+
/// <summary>
9+
/// Benchmarks batch commit execution overhead for the performance-oriented engine path.
10+
/// </summary>
11+
[MemoryDiagnoser]
12+
[InProcess]
13+
public class MutationEngineBatchBenchmarks
14+
{
15+
private IMutationEngine _performanceEngine = null!;
16+
private MutationEngineBenchmarkSupport.CounterState _state = null!;
17+
private IReadOnlyList<IMutation<MutationEngineBenchmarkSupport.CounterState>> _batchMutations = null!;
18+
19+
/// <summary>
20+
/// Controls how many commit mutations are executed in a single batch benchmark iteration.
21+
/// </summary>
22+
[Params(10, 100)]
23+
public int BatchSize { get; set; }
24+
25+
/// <summary>
26+
/// Prepares the engine, base state, and batch mutation list for the selected batch size.
27+
/// </summary>
28+
[GlobalSetup]
29+
public void Setup()
30+
{
31+
_performanceEngine = MutationEngineBenchmarkSupport.BuildEngine(MutationEngineOptions.Performance);
32+
_state = new MutationEngineBenchmarkSupport.CounterState(42);
33+
_batchMutations = [.. Enumerable.Range(0, BatchSize)
34+
.Select(i => MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Commit, $"batch-{i}"))];
35+
}
36+
37+
/// <summary>
38+
/// Measures sequential batch commit execution without policy pressure.
39+
/// </summary>
40+
[Benchmark]
41+
public async Task Batch_Commit_Performance_NoPolicy()
42+
{
43+
var result = await _performanceEngine.ExecuteBatchAsync(_batchMutations, _state);
44+
GC.KeepAlive(result);
45+
}
46+
}

‎Benchmarks/Engine/MutationEngineBenchmarkSupport.cs‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Changes;
4+
using ModularityKit.Mutator.Abstractions.Context;
5+
using ModularityKit.Mutator.Abstractions.Engine;
6+
using ModularityKit.Mutator.Abstractions.Intent;
7+
using ModularityKit.Mutator.Abstractions.Policies;
8+
using ModularityKit.Mutator.Abstractions.Results;
9+
using ModularityKit.Mutator.Runtime;
10+
11+
namespace ModularityKit.Mutator.Benchmarks.Engine;
12+
13+
/// <summary>
14+
/// Shared support types for core engine benchmark scenarios.
15+
/// </summary>
16+
internal static class MutationEngineBenchmarkSupport
17+
{
18+
public const string CounterStateId = "benchmark-counter";
19+
20+
public static IMutationEngine BuildEngine(
21+
MutationEngineOptions options,
22+
Action<IMutationEngine>? configure = null)
23+
{
24+
var services = new ServiceCollection();
25+
services.AddMutators(options);
26+
27+
var engine = services
28+
.BuildServiceProvider()
29+
.GetRequiredService<IMutationEngine>();
30+
31+
configure?.Invoke(engine);
32+
return engine;
33+
}
34+
35+
public static IncrementCounterMutation CreateCounterMutation(MutationMode mode, string operationSuffix)
36+
{
37+
var context = MutationContext.System("benchmark")
38+
with
39+
{
40+
StateId = CounterStateId,
41+
Mode = mode,
42+
CorrelationId = $"{CounterStateId}:{operationSuffix}"
43+
};
44+
45+
return new IncrementCounterMutation(context);
46+
}
47+
48+
/// <summary>
49+
/// Minimal counter state used by engine benchmark scenarios.
50+
/// </summary>
51+
/// <param name="Value">The current counter value.</param>
52+
public sealed record CounterState(int Value);
53+
54+
/// <summary>
55+
/// Minimal counter mutation shared by core engine benchmark scenarios.
56+
/// </summary>
57+
public sealed class IncrementCounterMutation(MutationContext context) : IMutation<CounterState>
58+
{
59+
public MutationIntent Intent { get; } = new()
60+
{
61+
OperationName = "IncrementCounter",
62+
Category = "Benchmark",
63+
Description = "Increment the benchmark counter by one",
64+
RiskLevel = MutationRiskLevel.Low,
65+
IsReversible = true
66+
};
67+
68+
public MutationContext Context { get; } = context;
69+
70+
public MutationResult<CounterState> Apply(CounterState state)
71+
{
72+
var next = state with { Value = state.Value + 1 };
73+
74+
return MutationResult<CounterState>.Success(
75+
next,
76+
ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value)));
77+
}
78+
79+
public ValidationResult Validate(CounterState state)
80+
{
81+
var result = ValidationResult.Success();
82+
83+
if (state.Value < 0)
84+
result.AddError(nameof(CounterState.Value), "Counter value must be non-negative.");
85+
86+
return result;
87+
}
88+
89+
public MutationResult<CounterState> Simulate(CounterState state)
90+
{
91+
var next = state with { Value = state.Value + 1 };
92+
93+
return MutationResult<CounterState>.Success(
94+
next,
95+
ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value)));
96+
}
97+
}
98+
99+
/// <summary>
100+
/// Trivial allow policy used to measure policy-aware engine paths.
101+
/// </summary>
102+
public sealed class AllowAllCounterPolicy : IMutationPolicy<CounterState>
103+
{
104+
public string Name => nameof(AllowAllCounterPolicy);
105+
106+
public int Priority => 0;
107+
108+
public string? Description => "Always allows the benchmark counter mutation.";
109+
110+
public PolicyDecision Evaluate(IMutation<CounterState> mutation, CounterState state)
111+
=> PolicyDecision.Allow(Name, "Benchmark policy allows all mutations.");
112+
}
113+
}

‎Benchmarks/Engine/MutationEngineCommitBenchmarks.cs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Engine;
4+
5+
namespace ModularityKit.Mutator.Benchmarks.Engine;
6+
7+
/// <summary>
8+
/// Benchmarks single commit execution for the core mutation engine with and without policy evaluation.
9+
/// </summary>
10+
[MemoryDiagnoser]
11+
[InProcess]
12+
public class MutationEngineCommitBenchmarks
13+
{
14+
private IMutationEngine _performanceEngine = null!;
15+
private IMutationEngine _strictEngine = null!;
16+
private MutationEngineBenchmarkSupport.CounterState _state = null!;
17+
private MutationEngineBenchmarkSupport.IncrementCounterMutation _commitMutation = null!;
18+
19+
/// <summary>
20+
/// Prepares the benchmark engines, state snapshot, and commit mutation instance.
21+
/// </summary>
22+
[GlobalSetup]
23+
public void Setup()
24+
{
25+
_performanceEngine = MutationEngineBenchmarkSupport.BuildEngine(MutationEngineOptions.Performance);
26+
_strictEngine = MutationEngineBenchmarkSupport.BuildEngine(
27+
MutationEngineOptions.Strict,
28+
engine => engine.RegisterPolicy(new MutationEngineBenchmarkSupport.AllowAllCounterPolicy()));
29+
30+
_state = new MutationEngineBenchmarkSupport.CounterState(42);
31+
_commitMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(Abstractions.Context.MutationMode.Commit, "commit-one");
32+
}
33+
34+
/// <summary>
35+
/// Measures a commit execution through the performance-oriented runtime path without policies.
36+
/// </summary>
37+
[Benchmark(Baseline = true)]
38+
public async Task Commit_Performance_NoPolicy()
39+
{
40+
var result = await _performanceEngine.ExecuteAsync(_commitMutation, _state);
41+
GC.KeepAlive(result);
42+
}
43+
44+
/// <summary>
45+
/// Measures a commit execution through the strict runtime path with one allow policy.
46+
/// </summary>
47+
[Benchmark]
48+
public async Task Commit_Strict_WithPolicy()
49+
{
50+
var result = await _strictEngine.ExecuteAsync(_commitMutation, _state);
51+
GC.KeepAlive(result);
52+
}
53+
}

0 commit comments

Comments
 (0)