diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml index 725b5e0..84cb681 100644 --- a/.github/workflows/publish-artifacts.yml +++ b/.github/workflows/publish-artifacts.yml @@ -2,6 +2,10 @@ name: Publish Artifacts on: workflow_call: + outputs: + signing_enabled: + description: "Whether package signing was enabled for this run." + value: ${{ jobs.package.outputs.signing_enabled }} inputs: package_version: description: "Optional package version, usually the release tag without the leading v." @@ -36,13 +40,13 @@ on: default: true secrets: NUGET_SIGN_CERTIFICATE_BASE64: - required: true + required: false NUGET_SIGN_CERTIFICATE_PASSWORD: - required: true + required: false NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: - required: true + required: false NUGET_SIGN_TIMESTAMP_URL: - required: true + required: false permissions: contents: read @@ -51,6 +55,8 @@ jobs: package: name: Pack packages runs-on: ubuntu-latest + outputs: + signing_enabled: ${{ steps.signing.outputs.enabled }} steps: - name: Checkout @@ -137,32 +143,38 @@ jobs: -o nupkg -p:PackageVersion=${{ steps.version.outputs.redis_version }} - - name: Import signing certificate + - name: Resolve signing mode + id: signing env: NUGET_SIGN_CERTIFICATE_BASE64: ${{ secrets.NUGET_SIGN_CERTIFICATE_BASE64 }} + NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }} + NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} + NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }} run: | - if [ -z "$NUGET_SIGN_CERTIFICATE_BASE64" ]; then - echo "NUGET_SIGN_CERTIFICATE_BASE64 secret is required for package signing." >&2 - exit 1 + if [ -n "$NUGET_SIGN_CERTIFICATE_BASE64" ] \ + && [ -n "$NUGET_SIGN_CERTIFICATE_PASSWORD" ] \ + && [ -n "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ] \ + && [ -n "$NUGET_SIGN_TIMESTAMP_URL" ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + echo "Package signing enabled." + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "Package signing disabled. Continuing with unsigned packages." fi + - name: Import signing certificate + if: ${{ steps.signing.outputs.enabled == 'true' }} + env: + NUGET_SIGN_CERTIFICATE_BASE64: ${{ secrets.NUGET_SIGN_CERTIFICATE_BASE64 }} + run: | printf '%s' "$NUGET_SIGN_CERTIFICATE_BASE64" | base64 --decode > signing-cert.pfx - name: Sign packages + if: ${{ steps.signing.outputs.enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }} NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }} run: | - if [ -z "$NUGET_SIGN_CERTIFICATE_PASSWORD" ]; then - echo "NUGET_SIGN_CERTIFICATE_PASSWORD secret is required for package signing." >&2 - exit 1 - fi - - if [ -z "$NUGET_SIGN_TIMESTAMP_URL" ]; then - echo "NUGET_SIGN_TIMESTAMP_URL secret is required for package signing." >&2 - exit 1 - fi - for package in nupkg/*.nupkg; do dotnet nuget sign "$package" \ --certificate-path signing-cert.pfx \ @@ -174,14 +186,10 @@ jobs: done - name: Verify signed packages + if: ${{ steps.signing.outputs.enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} run: | - if [ -z "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ]; then - echo "NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT secret is required for package verification." >&2 - exit 1 - fi - for package in nupkg/*.nupkg; do dotnet nuget verify "$package" \ --all \ diff --git a/.github/workflows/publish-attested.yml b/.github/workflows/publish-attested.yml index e050fc9..1c012b3 100644 --- a/.github/workflows/publish-attested.yml +++ b/.github/workflows/publish-attested.yml @@ -15,13 +15,13 @@ on: type: string secrets: NUGET_SIGN_CERTIFICATE_BASE64: - required: true + required: false NUGET_SIGN_CERTIFICATE_PASSWORD: - required: true + required: false NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: - required: true + required: false NUGET_SIGN_TIMESTAMP_URL: - required: true + required: false permissions: contents: write @@ -56,6 +56,11 @@ jobs: path: dist merge-multiple: true + - name: Attest package artifacts + uses: actions/attest-build-provenance@v2 + with: + subject-path: dist/*.nupkg + - name: Create or update draft release env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index eadc238..3185d94 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -130,14 +130,10 @@ jobs: merge-multiple: true - name: Verify signed packages + if: ${{ needs.package.outputs.signing_enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} run: | - if [ -z "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ]; then - echo "NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT secret is required for package verification." >&2 - exit 1 - fi - for package in dist/*.nupkg; do dotnet nuget verify "$package" \ --all \ @@ -184,14 +180,10 @@ jobs: merge-multiple: true - name: Verify signed packages + if: ${{ needs.package.outputs.signing_enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} run: | - if [ -z "$NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT" ]; then - echo "NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT secret is required for package verification." >&2 - exit 1 - fi - for package in dist/*.nupkg; do dotnet nuget verify "$package" \ --all \ diff --git a/Benchmarks/Engine/MutationEngineBatchBenchmarks.cs b/Benchmarks/Engine/MutationEngineBatchBenchmarks.cs new file mode 100644 index 0000000..1a5c4f1 --- /dev/null +++ b/Benchmarks/Engine/MutationEngineBatchBenchmarks.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; + +namespace ModularityKit.Mutator.Benchmarks.Engine; + +/// +/// Benchmarks batch commit execution overhead for the performance-oriented engine path. +/// +[MemoryDiagnoser] +[InProcess] +public class MutationEngineBatchBenchmarks +{ + private IMutationEngine _performanceEngine = null!; + private MutationEngineBenchmarkSupport.CounterState _state = null!; + private IReadOnlyList> _batchMutations = null!; + + /// + /// Controls how many commit mutations are executed in a single batch benchmark iteration. + /// + [Params(10, 100)] + public int BatchSize { get; set; } + + /// + /// Prepares the engine, base state, and batch mutation list for the selected batch size. + /// + [GlobalSetup] + public void Setup() + { + _performanceEngine = MutationEngineBenchmarkSupport.BuildEngine(MutationEngineOptions.Performance); + _state = new MutationEngineBenchmarkSupport.CounterState(42); + _batchMutations = [.. Enumerable.Range(0, BatchSize) + .Select(i => MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Commit, $"batch-{i}"))]; + } + + /// + /// Measures sequential batch commit execution without policy pressure. + /// + [Benchmark] + public async Task Batch_Commit_Performance_NoPolicy() + { + var result = await _performanceEngine.ExecuteBatchAsync(_batchMutations, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/Engine/MutationEngineBenchmarkSupport.cs b/Benchmarks/Engine/MutationEngineBenchmarkSupport.cs new file mode 100644 index 0000000..792c282 --- /dev/null +++ b/Benchmarks/Engine/MutationEngineBenchmarkSupport.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Abstractions.Policies; +using ModularityKit.Mutator.Abstractions.Results; +using ModularityKit.Mutator.Runtime; + +namespace ModularityKit.Mutator.Benchmarks.Engine; + +/// +/// Shared support types for core engine benchmark scenarios. +/// +internal static class MutationEngineBenchmarkSupport +{ + public const string CounterStateId = "benchmark-counter"; + + public static IMutationEngine BuildEngine( + MutationEngineOptions options, + Action? configure = null) + { + var services = new ServiceCollection(); + services.AddMutators(options); + + var engine = services + .BuildServiceProvider() + .GetRequiredService(); + + configure?.Invoke(engine); + return engine; + } + + public static IncrementCounterMutation CreateCounterMutation(MutationMode mode, string operationSuffix) + { + var context = MutationContext.System("benchmark") + with + { + StateId = CounterStateId, + Mode = mode, + CorrelationId = $"{CounterStateId}:{operationSuffix}" + }; + + return new IncrementCounterMutation(context); + } + + /// + /// Minimal counter state used by engine benchmark scenarios. + /// + /// The current counter value. + public sealed record CounterState(int Value); + + /// + /// Minimal counter mutation shared by core engine benchmark scenarios. + /// + public sealed class IncrementCounterMutation(MutationContext context) : IMutation + { + public MutationIntent Intent { get; } = new() + { + OperationName = "IncrementCounter", + Category = "Benchmark", + Description = "Increment the benchmark counter by one", + RiskLevel = MutationRiskLevel.Low, + IsReversible = true + }; + + public MutationContext Context { get; } = context; + + public MutationResult Apply(CounterState state) + { + var next = state with { Value = state.Value + 1 }; + + return MutationResult.Success( + next, + ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value))); + } + + public ValidationResult Validate(CounterState state) + { + var result = ValidationResult.Success(); + + if (state.Value < 0) + result.AddError(nameof(CounterState.Value), "Counter value must be non-negative."); + + return result; + } + + public MutationResult Simulate(CounterState state) + { + var next = state with { Value = state.Value + 1 }; + + return MutationResult.Success( + next, + ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value))); + } + } + + /// + /// Trivial allow policy used to measure policy-aware engine paths. + /// + public sealed class AllowAllCounterPolicy : IMutationPolicy + { + public string Name => nameof(AllowAllCounterPolicy); + + public int Priority => 0; + + public string? Description => "Always allows the benchmark counter mutation."; + + public PolicyDecision Evaluate(IMutation mutation, CounterState state) + => PolicyDecision.Allow(Name, "Benchmark policy allows all mutations."); + } +} diff --git a/Benchmarks/Engine/MutationEngineCommitBenchmarks.cs b/Benchmarks/Engine/MutationEngineCommitBenchmarks.cs new file mode 100644 index 0000000..0a59ceb --- /dev/null +++ b/Benchmarks/Engine/MutationEngineCommitBenchmarks.cs @@ -0,0 +1,53 @@ +using BenchmarkDotNet.Attributes; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Engine; + +namespace ModularityKit.Mutator.Benchmarks.Engine; + +/// +/// Benchmarks single commit execution for the core mutation engine with and without policy evaluation. +/// +[MemoryDiagnoser] +[InProcess] +public class MutationEngineCommitBenchmarks +{ + private IMutationEngine _performanceEngine = null!; + private IMutationEngine _strictEngine = null!; + private MutationEngineBenchmarkSupport.CounterState _state = null!; + private MutationEngineBenchmarkSupport.IncrementCounterMutation _commitMutation = null!; + + /// + /// Prepares the benchmark engines, state snapshot, and commit mutation instance. + /// + [GlobalSetup] + public void Setup() + { + _performanceEngine = MutationEngineBenchmarkSupport.BuildEngine(MutationEngineOptions.Performance); + _strictEngine = MutationEngineBenchmarkSupport.BuildEngine( + MutationEngineOptions.Strict, + engine => engine.RegisterPolicy(new MutationEngineBenchmarkSupport.AllowAllCounterPolicy())); + + _state = new MutationEngineBenchmarkSupport.CounterState(42); + _commitMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(Abstractions.Context.MutationMode.Commit, "commit-one"); + } + + /// + /// Measures a commit execution through the performance-oriented runtime path without policies. + /// + [Benchmark(Baseline = true)] + public async Task Commit_Performance_NoPolicy() + { + var result = await _performanceEngine.ExecuteAsync(_commitMutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures a commit execution through the strict runtime path with one allow policy. + /// + [Benchmark] + public async Task Commit_Strict_WithPolicy() + { + var result = await _strictEngine.ExecuteAsync(_commitMutation, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/Engine/MutationEngineModeBenchmarks.cs b/Benchmarks/Engine/MutationEngineModeBenchmarks.cs new file mode 100644 index 0000000..d216e7e --- /dev/null +++ b/Benchmarks/Engine/MutationEngineModeBenchmarks.cs @@ -0,0 +1,54 @@ +using BenchmarkDotNet.Attributes; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; + +namespace ModularityKit.Mutator.Benchmarks.Engine; + +/// +/// Benchmarks non-commit engine modes to isolate simulate and validate-only execution overhead. +/// +[MemoryDiagnoser] +[InProcess] +public class MutationEngineModeBenchmarks +{ + private IMutationEngine _strictEngine = null!; + private MutationEngineBenchmarkSupport.CounterState _state = null!; + private MutationEngineBenchmarkSupport.IncrementCounterMutation _simulateMutation = null!; + private MutationEngineBenchmarkSupport.IncrementCounterMutation _validateMutation = null!; + + /// + /// Prepares the strict engine and mode-specific mutations used in this suite. + /// + [GlobalSetup] + public void Setup() + { + _strictEngine = MutationEngineBenchmarkSupport.BuildEngine( + MutationEngineOptions.Strict, + engine => engine.RegisterPolicy(new MutationEngineBenchmarkSupport.AllowAllCounterPolicy())); + + _state = new MutationEngineBenchmarkSupport.CounterState(42); + _simulateMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Simulate, "simulate-one"); + _validateMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Validate, "validate-one"); + } + + /// + /// Measures the simulate path with strict engine behavior and one allow policy. + /// + [Benchmark(Baseline = true)] + public async Task Simulate_Strict_WithPolicy() + { + var result = await _strictEngine.ExecuteAsync(_simulateMutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures the validate-only path with strict engine behavior and one allow policy. + /// + [Benchmark] + public async Task ValidateOnly_Strict_WithPolicy() + { + var result = await _strictEngine.ExecuteAsync(_validateMutation, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/Engine/MutationEngineThroughputBenchmarks.cs b/Benchmarks/Engine/MutationEngineThroughputBenchmarks.cs new file mode 100644 index 0000000..3cc0c46 --- /dev/null +++ b/Benchmarks/Engine/MutationEngineThroughputBenchmarks.cs @@ -0,0 +1,155 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Abstractions.Results; +using ModularityKit.Mutator.Runtime; + +namespace ModularityKit.Mutator.Benchmarks.Engine; + +/// +/// Benchmarks mutation engine throughput across varying state sizes and batch sizes. +/// +[MemoryDiagnoser] +[InProcess] +public class MutationEngineThroughputBenchmarks +{ + private const string StateId = "throughput-state"; + + private IMutationEngine _engine = null!; + private ThroughputState _singleState = null!; + private ThroughputState _batchState = null!; + private ReplaceSlotMutation _singleMutation = null!; + private IReadOnlyList> _batchMutations = null!; + + /// + /// Controls the number of slots cloned and updated by each benchmarked mutation. + /// + [Params(32, 1024, 16384)] + public int StateSize { get; set; } + + /// + /// Controls how many mutations are executed in the batch throughput scenario. + /// + [Params(8, 64)] + public int BatchSize { get; set; } + + /// + /// Prepares the engine, state snapshots, and mutation lists for the selected benchmark parameters. + /// + [GlobalSetup] + public void Setup() + { + _engine = BuildEngine(); + _singleState = CreateState(StateSize); + _batchState = CreateState(StateSize); + _singleMutation = CreateMutation(StateSize / 2, 1, "single"); + _batchMutations = [.. Enumerable.Range(0, BatchSize).Select(i => CreateMutation(i % StateSize, (i % 4) + 1, $"batch-{i}"))]; + } + + /// + /// Measures single-mutation throughput for the configured state size. + /// + [Benchmark(Baseline = true)] + public async Task SingleMutation_Commit_Throughput() + { + var result = await _engine.ExecuteAsync(_singleMutation, _singleState); + GC.KeepAlive(result); + } + + /// + /// Measures batch-mutation throughput for the configured state size and batch size. + /// + [Benchmark] + public async Task BatchMutation_Commit_Throughput() + { + var result = await _engine.ExecuteBatchAsync(_batchMutations, _batchState); + GC.KeepAlive(result); + } + + private static IMutationEngine BuildEngine() + { + var services = new ServiceCollection(); + services.AddMutators(MutationEngineOptions.Performance); + + return services + .BuildServiceProvider() + .GetRequiredService(); + } + + private static ThroughputState CreateState(int size) + { + var values = new int[size]; + + for (var index = 0; index < values.Length; index++) + values[index] = index; + + return new ThroughputState(values, 0); + } + + private static ReplaceSlotMutation CreateMutation(int slot, int delta, string operationSuffix) + { + var context = MutationContext.System("benchmark-throughput") + with + { + StateId = StateId, + Mode = MutationMode.Commit, + CorrelationId = $"{StateId}:{operationSuffix}" + }; + + return new ReplaceSlotMutation(context, slot, delta); + } + + private sealed record ThroughputState(int[] Slots, int Revision); + + private sealed class ReplaceSlotMutation( + MutationContext context, + int slot, + int delta) + : IMutation + { + public MutationIntent Intent { get; } = new() + { + OperationName = "ReplaceSlot", + Category = "Benchmark", + Description = "Clone state and update a single slot to measure throughput-sensitive execution paths.", + RiskLevel = MutationRiskLevel.Low, + IsReversible = true + }; + + public MutationContext Context { get; } = context; + + public MutationResult Apply(ThroughputState state) + { + var nextSlots = (int[])state.Slots.Clone(); + var before = nextSlots[slot]; + nextSlots[slot] = before + delta; + + var nextState = state with + { + Slots = nextSlots, + Revision = state.Revision + 1 + }; + + return MutationResult.Success( + nextState, + ChangeSet.Single(StateChange.Modified($"Slots[{slot}]", before, nextSlots[slot]))); + } + + public ValidationResult Validate(ThroughputState state) + { + var result = ValidationResult.Success(); + + if (slot < 0 || slot >= state.Slots.Length) + result.AddError(nameof(state.Slots), $"Slot index {slot} is outside the benchmark state."); + + return result; + } + + public MutationResult Simulate(ThroughputState state) + => Apply(state); + } +} diff --git a/Benchmarks/MutationEngineBenchmarks.cs b/Benchmarks/MutationEngineBenchmarks.cs deleted file mode 100644 index e7cf79a..0000000 --- a/Benchmarks/MutationEngineBenchmarks.cs +++ /dev/null @@ -1,161 +0,0 @@ -using BenchmarkDotNet.Attributes; -using Microsoft.Extensions.DependencyInjection; -using ModularityKit.Mutator.Abstractions; -using ModularityKit.Mutator.Abstractions.Changes; -using ModularityKit.Mutator.Abstractions.Context; -using ModularityKit.Mutator.Abstractions.Engine; -using ModularityKit.Mutator.Abstractions.Intent; -using ModularityKit.Mutator.Abstractions.Policies; -using ModularityKit.Mutator.Abstractions.Results; -using ModularityKit.Mutator.Runtime; - -namespace ModularityKit.Mutator.Benchmarks; - -[MemoryDiagnoser] -[InProcess] -public class MutationEngineBenchmarks -{ - private const string StateId = "benchmark-counter"; - - private IMutationEngine _performanceEngine = null!; - private IMutationEngine _strictEngine = null!; - private CounterState _state = null!; - private IncrementCounterMutation _commitMutation = null!; - private IncrementCounterMutation _simulateMutation = null!; - private IncrementCounterMutation _validateMutation = null!; - private IReadOnlyList> _batchMutations = null!; - - [GlobalSetup] - public void Setup() - { - _performanceEngine = BuildEngine(MutationEngineOptions.Performance, addAllowPolicy: false); - _strictEngine = BuildEngine(MutationEngineOptions.Strict, addAllowPolicy: true); - - _state = new CounterState(42); - _commitMutation = CreateMutation(MutationMode.Commit, "commit-one"); - _simulateMutation = CreateMutation(MutationMode.Simulate, "simulate-one"); - _validateMutation = CreateMutation(MutationMode.Validate, "validate-one"); - _batchMutations = Enumerable.Range(0, 10) - .Select(i => CreateMutation(MutationMode.Commit, $"batch-{i}")) - .ToArray(); - } - - [Benchmark(Baseline = true)] - public async Task Commit_Performance_NoPolicy() - { - var result = await _performanceEngine.ExecuteAsync(_commitMutation, _state); - GC.KeepAlive(result); - } - - [Benchmark] - public async Task Commit_Strict_WithPolicy() - { - var result = await _strictEngine.ExecuteAsync(_commitMutation, _state); - GC.KeepAlive(result); - } - - [Benchmark] - public async Task Simulate_Strict_WithPolicy() - { - var result = await _strictEngine.ExecuteAsync(_simulateMutation, _state); - GC.KeepAlive(result); - } - - [Benchmark] - public async Task ValidateOnly_Strict_WithPolicy() - { - var result = await _strictEngine.ExecuteAsync(_validateMutation, _state); - GC.KeepAlive(result); - } - - [Benchmark] - public async Task Batch_Commit_Performance_NoPolicy() - { - var result = await _performanceEngine.ExecuteBatchAsync(_batchMutations, _state); - GC.KeepAlive(result); - } - - private static IMutationEngine BuildEngine( - MutationEngineOptions options, - bool addAllowPolicy) - { - var services = new ServiceCollection(); - services.AddMutators(options); - var provider = services.BuildServiceProvider(); - var engine = provider.GetRequiredService(); - - if (addAllowPolicy) - engine.RegisterPolicy(new AllowAllCounterPolicy()); - - return engine; - } - - private static IncrementCounterMutation CreateMutation(MutationMode mode, string operationSuffix) - { - var context = MutationContext.System("benchmark") - with - { - StateId = StateId, - Mode = mode, - CorrelationId = $"{StateId}:{operationSuffix}" - }; - - return new IncrementCounterMutation(context); - } - - private sealed record CounterState(int Value); - - private sealed class IncrementCounterMutation(MutationContext context) : IMutation - { - public MutationIntent Intent { get; } = new() - { - OperationName = "IncrementCounter", - Category = "Benchmark", - Description = "Increment the benchmark counter by one", - RiskLevel = MutationRiskLevel.Low, - IsReversible = true - }; - - public MutationContext Context { get; } = context; - - public MutationResult Apply(CounterState state) - { - var next = state with { Value = state.Value + 1 }; - - return MutationResult.Success( - next, - ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value))); - } - - public ValidationResult Validate(CounterState state) - { - var result = ValidationResult.Success(); - - if (state.Value < 0) - result.AddError(nameof(CounterState.Value), "Counter value must be non-negative."); - - return result; - } - - public MutationResult Simulate(CounterState state) - { - var next = state with { Value = state.Value + 1 }; - - return MutationResult.Success( - next, - ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value))); - } - } - - private sealed class AllowAllCounterPolicy : IMutationPolicy - { - public string Name => nameof(AllowAllCounterPolicy); - - public int Priority => 0; - - public string? Description => "Always allows the benchmark counter mutation."; - - public PolicyDecision Evaluate(IMutation mutation, CounterState state) - => PolicyDecision.Allow(Name, "Benchmark policy allows all mutations."); - } -} diff --git a/Benchmarks/Policy/PolicyBenchmarkSupport.cs b/Benchmarks/Policy/PolicyBenchmarkSupport.cs new file mode 100644 index 0000000..4269b0d --- /dev/null +++ b/Benchmarks/Policy/PolicyBenchmarkSupport.cs @@ -0,0 +1,124 @@ +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Abstractions.Policies; +using ModularityKit.Mutator.Abstractions.Results; +using ModularityKit.Mutator.Runtime; + +namespace ModularityKit.Mutator.Benchmarks.Policy; + +/// +/// Shared support types for policy evaluation benchmark scenarios. +/// +internal static class PolicyBenchmarkSupport +{ + public const string StateId = "policy-benchmark-state"; + + public static IMutationEngine BuildEngine(Action? configure = null) + { + var services = new ServiceCollection(); + services.AddMutators(MutationEngineOptions.Performance); + + var engine = services + .BuildServiceProvider() + .GetRequiredService(); + + configure?.Invoke(engine); + return engine; + } + + public static MinimalPolicyMutation CreateMutation() + { + var context = MutationContext.System("policy-benchmark") + with + { + StateId = StateId, + Mode = MutationMode.Commit, + CorrelationId = $"{StateId}:policy-pass" + }; + + return new MinimalPolicyMutation(context); + } + + /// + /// Minimal state used by policy evaluation benchmarks. + /// + /// The logical state name. + /// The mutable numeric field exercised by the benchmark mutation. + public sealed record PolicyBenchmarkState(string Name, int Counter); + + /// + /// Minimal mutation used to isolate policy pipeline overhead from unrelated runtime work. + /// + public sealed class MinimalPolicyMutation(MutationContext context) : IMutation + { + public MutationIntent Intent { get; } = new() + { + OperationName = "MinimalPolicyMutation", + Category = "Benchmark", + Description = "Minimal mutation used to isolate policy evaluation overhead.", + RiskLevel = MutationRiskLevel.Low, + IsReversible = true + }; + + public MutationContext Context { get; } = context; + + public MutationResult Apply(PolicyBenchmarkState state) + { + var nextState = state with { Counter = state.Counter + 1 }; + + return MutationResult.Success( + nextState, + ChangeSet.Single(StateChange.Modified(nameof(PolicyBenchmarkState.Counter), state.Counter, nextState.Counter))); + } + + public ValidationResult Validate(PolicyBenchmarkState state) => ValidationResult.Success(); + + public MutationResult Simulate(PolicyBenchmarkState state) => Apply(state); + } + + /// + /// Synchronous allow policy used in benchmark scenarios. + /// + public sealed class SyncAllowBenchmarkPolicy : IMutationPolicy + { + public SyncAllowBenchmarkPolicy(int priority) => Priority = priority; + + public string Name => $"{nameof(SyncAllowBenchmarkPolicy)}_{Priority}"; + + public int Priority { get; } + + public string? Description => "Synchronous allow policy for benchmark measurements."; + + public PolicyDecision Evaluate(IMutation mutation, PolicyBenchmarkState state) + => PolicyDecision.Allow(Name, "Synchronous benchmark policy allowed the mutation."); + } + + /// + /// Asynchronous allow policy used in benchmark scenarios. + /// + public sealed class AsyncAllowBenchmarkPolicy : IMutationPolicy + { + public AsyncAllowBenchmarkPolicy(int priority) => Priority = priority; + + public string Name => $"{nameof(AsyncAllowBenchmarkPolicy)}_{Priority}"; + + public int Priority { get; } + + public string? Description => "Asynchronous allow policy for benchmark measurements."; + + public async Task EvaluateAsync( + IMutation mutation, + PolicyBenchmarkState state, + CancellationToken cancellationToken = default) + { + await Task.CompletedTask; + cancellationToken.ThrowIfCancellationRequested(); + + return PolicyDecision.Allow(Name, "Asynchronous benchmark policy allowed the mutation."); + } + } +} diff --git a/Benchmarks/Policy/PolicyEvaluationMultiBenchmarks.cs b/Benchmarks/Policy/PolicyEvaluationMultiBenchmarks.cs new file mode 100644 index 0000000..bca8165 --- /dev/null +++ b/Benchmarks/Policy/PolicyEvaluationMultiBenchmarks.cs @@ -0,0 +1,45 @@ +using BenchmarkDotNet.Attributes; +using ModularityKit.Mutator.Abstractions.Engine; + +namespace ModularityKit.Mutator.Benchmarks.Policy; + +/// +/// Benchmarks aggregate overhead for multiple policies evaluated in a single runtime pass. +/// +[MemoryDiagnoser] +[InProcess] +public class PolicyEvaluationMultiBenchmarks +{ + private IMutationEngine _multiPolicyEngine = null!; + private PolicyBenchmarkSupport.PolicyBenchmarkState _state = null!; + private PolicyBenchmarkSupport.MinimalPolicyMutation _mutation = null!; + + /// + /// Prepares an engine with mixed synchronous and asynchronous allow policies. + /// + [GlobalSetup] + public void Setup() + { + _multiPolicyEngine = PolicyBenchmarkSupport.BuildEngine( + engine => + { + engine.RegisterPolicy(new PolicyBenchmarkSupport.SyncAllowBenchmarkPolicy(priority: 300)); + engine.RegisterPolicy(new PolicyBenchmarkSupport.AsyncAllowBenchmarkPolicy(priority: 200)); + engine.RegisterPolicy(new PolicyBenchmarkSupport.SyncAllowBenchmarkPolicy(priority: 100)); + engine.RegisterPolicy(new PolicyBenchmarkSupport.AsyncAllowBenchmarkPolicy(priority: 0)); + }); + + _state = new PolicyBenchmarkSupport.PolicyBenchmarkState("alpha", 42); + _mutation = PolicyBenchmarkSupport.CreateMutation(); + } + + /// + /// Measures the overhead of evaluating several allow policies in priority order. + /// + [Benchmark] + public async Task MultipleMixedPolicies_Allow() + { + var result = await _multiPolicyEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/Policy/PolicyEvaluationSingleBenchmarks.cs b/Benchmarks/Policy/PolicyEvaluationSingleBenchmarks.cs new file mode 100644 index 0000000..c8407d7 --- /dev/null +++ b/Benchmarks/Policy/PolicyEvaluationSingleBenchmarks.cs @@ -0,0 +1,63 @@ +using BenchmarkDotNet.Attributes; +using ModularityKit.Mutator.Abstractions.Engine; + +namespace ModularityKit.Mutator.Benchmarks.Policy; + +/// +/// Benchmarks single-policy evaluation overhead against a no-policy baseline. +/// +[MemoryDiagnoser] +[InProcess] +public class PolicyEvaluationSingleBenchmarks +{ + private IMutationEngine _baselineEngine = null!; + private IMutationEngine _syncPolicyEngine = null!; + private IMutationEngine _asyncPolicyEngine = null!; + private PolicyBenchmarkSupport.PolicyBenchmarkState _state = null!; + private PolicyBenchmarkSupport.MinimalPolicyMutation _mutation = null!; + + /// + /// Prepares the baseline, synchronous-policy, and asynchronous-policy engines. + /// + [GlobalSetup] + public void Setup() + { + _baselineEngine = PolicyBenchmarkSupport.BuildEngine(); + _syncPolicyEngine = PolicyBenchmarkSupport.BuildEngine( + engine => engine.RegisterPolicy(new PolicyBenchmarkSupport.SyncAllowBenchmarkPolicy(priority: 100))); + _asyncPolicyEngine = PolicyBenchmarkSupport.BuildEngine( + engine => engine.RegisterPolicy(new PolicyBenchmarkSupport.AsyncAllowBenchmarkPolicy(priority: 100))); + _state = new PolicyBenchmarkSupport.PolicyBenchmarkState("alpha", 42); + _mutation = PolicyBenchmarkSupport.CreateMutation(); + } + + /// + /// Measures the execution baseline without any registered policies. + /// + [Benchmark(Baseline = true)] + public async Task NoPolicy_Baseline() + { + var result = await _baselineEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures the overhead of one synchronous allow policy. + /// + [Benchmark] + public async Task SingleSyncPolicy_Allow() + { + var result = await _syncPolicyEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures the overhead of one asynchronous allow policy. + /// + [Benchmark] + public async Task SingleAsyncPolicy_Allow() + { + var result = await _asyncPolicyEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/README.md b/Benchmarks/README.md index d7862d6..2f079b2 100644 --- a/Benchmarks/README.md +++ b/Benchmarks/README.md @@ -8,6 +8,12 @@ This folder contains BenchmarkDotNet measurements for `ModularityKit.Mutator`. - strict engine execution with policy checks - simulate and validate only paths - batch execution overhead +- throughput for single mutation execution across multiple state sizes +- throughput for batch mutation execution across multiple state sizes and batch sizes +- policy evaluation overhead for no-policy, synchronous policy, asynchronous policy, and mixed multi-policy runs + +The throughput benchmarks use a cloned array-backed state so state-size effects remain visible in the +actual mutation path rather than being hidden behind an artificial inner loop. ## Run @@ -20,9 +26,27 @@ dotnet build Benchmarks/ModularityKit.Mutator.Benchmarks.csproj -c Release Run a specific benchmark: ```bash -dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*MutationEngineBenchmarks.Commit_Performance_NoPolicy*' +dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*MutationEngineCommitBenchmarks*' +``` + +Run the throughput-focused suite: + +```bash +dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*MutationEngineThroughputBenchmarks*' ``` +Run the policy evaluation suite: + +```bash +dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*PolicyEvaluationSingleBenchmarks*' +dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*PolicyEvaluationMultiBenchmarks*' +``` + +Key parameters reported by BenchmarkDotNet: + +- `StateSize` controls the size of the cloned mutation state +- `BatchSize` controls the number of mutations executed in the batch benchmark + ## Notes - The benchmark harness is configured for the current environment. diff --git a/Docs/Package-Signing.md b/Docs/Package-Signing.md index cade2f0..512c963 100644 --- a/Docs/Package-Signing.md +++ b/Docs/Package-Signing.md @@ -1,9 +1,10 @@ # Package Signing -`ModularityKit.Mutator` release packages are signed as part of the standard package publish path. +`ModularityKit.Mutator` can sign release packages as part of the standard package publish path. -The repository signs generated `.nupkg` artifacts before they are uploaded or pushed to package feeds. -Unsigned release packages are treated as a validation failure. +When signing material is configured, the repository signs generated `.nupkg` artifacts before they +are uploaded or pushed to package feeds. When signing material is not configured, the release path +continues with unsigned packages and relies on GitHub artifact attestations for provenance. ## Signing approach @@ -21,21 +22,28 @@ This keeps signing explicit, auditable, and integrated with the existing `pack` The standard package release path is: 1. pack packages in `.github/workflows/publish-artifacts.yml` -2. sign every `.nupkg` -3. verify every signed `.nupkg` -4. upload signed artifacts -5. download and verify signed artifacts again before pushing to NuGet.org or GitHub Packages +2. sign every `.nupkg` when signing secrets are configured +3. verify every signed `.nupkg` when signing is enabled +4. upload artifacts +5. generate GitHub artifact attestations in `.github/workflows/publish-attested.yml` +6. download and verify signed artifacts again before pushing to NuGet.org or GitHub Packages when signing is enabled The signing step changes package contents only by adding signature metadata. -## Required GitHub secrets +## GitHub secrets -Configure these repository secrets before using the package publish workflows: +Package signing is optional. Configure these repository secrets to enable signed package output: - `NUGET_SIGN_CERTIFICATE_BASE64` - `NUGET_SIGN_CERTIFICATE_PASSWORD` - `NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT` - `NUGET_SIGN_TIMESTAMP_URL` + +Without those secrets, the workflows still pack packages and the attested release path still emits +GitHub provenance attestations. + +NuGet.org publishing still requires: + - `NUGET_USERNAME` Notes: @@ -55,7 +63,7 @@ Contributors can still: - pack projects locally - run tests and smoke tests -Signing is enforced in the repository release workflows, not for ordinary local development. +Signing is optional in the repository release workflows and not required for ordinary local development. If you have access to the signing certificate locally, you can validate package signatures with: diff --git a/README.md b/README.md index ef4b75f..49f98d0 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ https://modularitykit.github.io/ModularityKit.Mutator/ ## Package signing -Release packages are signed and verified in the standard package publish workflow. See -[`Docs/Package-Signing.md`](Docs/Package-Signing.md) for the signing approach, required secrets, and -local verification expectations. +Release packages can be signed and verified in the standard package publish workflow when signing +secrets are configured. Without signing secrets, the attested release path still emits GitHub +artifact attestations. See [`Docs/Package-Signing.md`](Docs/Package-Signing.md) for the signing +approach, optional secrets, and local verification expectations.