Skip to content

Commit 866695f

Browse files
rian-berian-be
committed
Add example smoke test suite
1 parent e58e5a4 commit 866695f

12 files changed

Lines changed: 489 additions & 4 deletions

‎.gitignore‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,20 @@
1-
.idea/
1+
# DocFX generated HTML site
2+
_site/
3+
4+
# DocFX metadata output
5+
obj/api/
6+
7+
# .NET build artifacts
28
bin/
39
obj/
4-
_site/
5-
__pycache__/
6-
*.pyc
10+
*.dll
11+
*.exe
12+
*.pdb
13+
*.deps.json
14+
*.runtimeconfig.json
15+
16+
# Dumps
17+
*.dmp
18+
*.dump
19+
*.mdmp
20+
*.core

‎ModularityKit.Mutator.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
<Project Path="Examples/Governance/VersionedResolution/VersionedResolution.csproj" />
1919
</Folder>
2020
<Folder Name="/Tests/">
21+
<Project Path="Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj" />
2122
<Project Path="Tests/ModularityKit.Mutator.Tests/ModularityKit.Mutator.Tests.csproj" />
2223
<Project Path="Tests/ModularityKit.Mutator.Governance.Tests/ModularityKit.Mutator.Governance.Tests.csproj" />
2324
<Project Path="Tests/ModularityKit.Mutator.Governance.Redis.Tests/ModularityKit.Mutator.Governance.Redis.Tests.csproj" />

‎Tests/.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
.idea/
2+
bin/
3+
obj/
4+
_site/
5+
__pycache__/
6+
*.pyc

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/AssemblyInfo.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
using Xunit;
2+
3+
[assembly: CollectionBehavior(DisableTestParallelization = true)]

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Core/CoreExamplesSmokeTests.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using ModularityKit.Mutator.Examples.SmokeTests.Support;
2+
3+
namespace ModularityKit.Mutator.Examples.SmokeTests.Examples.Core;
4+
5+
/// <summary>
6+
/// Smoke coverage for the executable samples shipped under <c>Examples/Core</c>.
7+
/// </summary>
8+
public sealed class CoreExamplesSmokeTests
9+
{
10+
[Fact]
11+
public Task BillingQuotas_runs_successfully()
12+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("BillingQuotas", "Examples/Core/BillingQuotas/BillingQuotas.csproj"));
13+
14+
[Fact]
15+
public Task FeatureFlags_runs_successfully()
16+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("FeatureFlags", "Examples/Core/FeatureFlags/FeatureFlags.csproj"));
17+
18+
[Fact]
19+
public Task IamRoles_runs_successfully()
20+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("IamRoles", "Examples/Core/IamRoles/IamRoles.csproj"));
21+
22+
[Fact]
23+
public Task WorkflowApprovals_runs_successfully()
24+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("WorkflowApprovals", "Examples/Core/WorkflowApprovals/WorkflowApprovals.csproj"));
25+
26+
private static ExampleSmokeCase Create(string name, string projectPath)
27+
=> new(
28+
name,
29+
projectPath,
30+
result =>
31+
{
32+
if (result.TimedOut)
33+
return "process timed out";
34+
35+
if (result.ExitCode != 0)
36+
return $"expected exit code 0 but got {result.ExitCode}";
37+
38+
if (string.IsNullOrWhiteSpace(result.StandardOutput))
39+
return "example did not produce any stdout";
40+
41+
if (!result.StandardOutput.Contains("METRICS & STATISTICS", StringComparison.Ordinal))
42+
return "expected metrics section in stdout";
43+
44+
if (result.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase))
45+
return "stderr contains unhandled exception output";
46+
47+
return null;
48+
});
49+
}

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Governance/GovernanceExamplesSmokeTests.cs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
using ModularityKit.Mutator.Examples.SmokeTests.Support;
2+
3+
namespace ModularityKit.Mutator.Examples.SmokeTests.Examples.Governance;
4+
5+
/// <summary>
6+
/// Smoke coverage for the executable samples shipped under <c>Examples/Governance</c>.
7+
/// </summary>
8+
public sealed class GovernanceExamplesSmokeTests
9+
{
10+
[Fact]
11+
public Task ApprovalWorkflow_runs_successfully()
12+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("ApprovalWorkflow", "Examples/Governance/ApprovalWorkflow/ApprovalWorkflow.csproj"));
13+
14+
[Fact]
15+
public Task GovernedExecution_runs_successfully()
16+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("GovernedExecution", "Examples/Governance/GovernedExecution/GovernedExecution.csproj"));
17+
18+
[Fact]
19+
public Task DecisionTaxonomy_runs_successfully()
20+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("DecisionTaxonomy", "Examples/Governance/DecisionTaxonomy/DecisionTaxonomy.csproj"));
21+
22+
[Fact]
23+
public Task Queries_runs_successfully()
24+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("Queries", "Examples/Governance/Queries/Queries.csproj"));
25+
26+
[Fact]
27+
public Task RedisQueries_runs_successfully()
28+
=> ExampleSmokeRunner.RunAndAssertAsync(CreateRedis());
29+
30+
[Fact]
31+
public Task RequestLifecycle_runs_successfully()
32+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("RequestLifecycle", "Examples/Governance/RequestLifecycle/RequestLifecycle.csproj"));
33+
34+
[Fact]
35+
public Task VersionedResolution_runs_successfully()
36+
=> ExampleSmokeRunner.RunAndAssertAsync(Create("VersionedResolution", "Examples/Governance/VersionedResolution/VersionedResolution.csproj"));
37+
38+
private static ExampleSmokeCase Create(string name, string projectPath)
39+
=> new(
40+
name,
41+
projectPath,
42+
result =>
43+
{
44+
if (result.TimedOut)
45+
return "process timed out";
46+
47+
if (result.ExitCode != 0)
48+
return $"expected exit code 0 but got {result.ExitCode}";
49+
50+
if (string.IsNullOrWhiteSpace(result.StandardOutput) && string.IsNullOrWhiteSpace(result.StandardError))
51+
return "example did not produce any output";
52+
53+
if (result.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase))
54+
return "stderr contains unhandled exception output";
55+
56+
return null;
57+
});
58+
59+
private static ExampleSmokeCase CreateRedis()
60+
=> new(
61+
"RedisQueries",
62+
"Examples/Governance/RedisQueries/RedisQueries.csproj",
63+
result =>
64+
{
65+
if (result.TimedOut)
66+
return "process timed out";
67+
68+
if (result.ExitCode != 0)
69+
return $"expected exit code 0 but got {result.ExitCode}";
70+
71+
var hasRedisOutput = result.StandardOutput.Contains("Pending Approval Queue", StringComparison.Ordinal)
72+
&& result.StandardOutput.Contains("Recent Execution Outcomes", StringComparison.Ordinal);
73+
74+
var hasExpectedDependencyWarning =
75+
result.StandardError.Contains("Could not connect to Redis", StringComparison.Ordinal)
76+
&& result.StandardError.Contains("Start Redis locally or set MODULARITYKIT_REDIS to a reachable endpoint.", StringComparison.Ordinal);
77+
78+
if (!hasRedisOutput && !hasExpectedDependencyWarning)
79+
return "expected either Redis query output or the documented Redis prerequisite warning";
80+
81+
if (result.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase))
82+
return "stderr contains unhandled exception output";
83+
84+
return null;
85+
},
86+
new Dictionary<string, string?>
87+
{
88+
["MODULARITYKIT_REDIS"] = "localhost:6379,connectTimeout=1000,abortConnect=false,syncTimeout=1000"
89+
});
90+
}

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/GlobalUsings.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
global using Xunit;

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net10.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<IsPackable>false</IsPackable>
8+
<GenerateDocumentationFile>true</GenerateDocumentationFile>
9+
<NoWarn>$(NoWarn);1591</NoWarn>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
14+
<PackageReference Include="xunit" Version="2.9.3" />
15+
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4">
16+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
17+
<PrivateAssets>all</PrivateAssets>
18+
</PackageReference>
19+
</ItemGroup>
20+
21+
</Project>

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/README.md‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Example Smoke Tests
2+
3+
This project provides lightweight smoke coverage for the executable samples under:
4+
5+
- `Examples/Core`
6+
- `Examples/Governance`
7+
8+
The goal is to catch sample drift against the current public API and runtime behavior without turning the examples into a second full test suite.
9+
10+
## Structure
11+
12+
- `Examples/Core` contains smoke tests for core runtime samples.
13+
- `Examples/Governance` contains smoke tests for governance and Redis-backed samples.
14+
- `Support` contains the shared runner and result models used by the smoke layer.
15+
16+
## How it works
17+
18+
Each smoke test:
19+
20+
1. builds the target example project in `Release`
21+
2. executes the built assembly with `dotnet exec`
22+
3. captures `stdout` and `stderr`
23+
4. validates a small set of expected signals for that sample
24+
25+
The runner keeps the checks focused on sample viability:
26+
27+
- non-zero exit codes fail the test
28+
- hung processes are terminated by timeout
29+
- failures include captured output for fast diagnosis
30+
31+
## Redis example behavior
32+
33+
`RedisQueries` accepts either:
34+
35+
- normal Redis-backed query output, or
36+
- the documented "could not connect to Redis" message
37+
38+
That keeps the sample smoke testable even when Redis is not running locally.
39+
40+
## Run
41+
42+
Build:
43+
44+
```bash
45+
dotnet build Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj -c Debug
46+
```
47+
48+
Run:
49+
50+
```bash
51+
dotnet test Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj -c Debug --no-build
52+
```
53+
54+
Run a single smoke test:
55+
56+
```bash
57+
dotnet test Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj -c Debug --no-build --filter FullyQualifiedName=ModularityKit.Mutator.Examples.SmokeTests.GovernanceExamplesSmokeTests.RedisQueries_runs_successfully
58+
```
59+
60+
## Environment note
61+
62+
`vstest` uses a local socket. In restricted sandboxes that block local bind operations, the test host may need to run outside the sandbox even though the examples themselves are local processes.

‎Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleRunResult.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
namespace ModularityKit.Mutator.Examples.SmokeTests.Support;
2+
3+
/// <summary>
4+
/// Captures the observable outcome of running an example process.
5+
/// </summary>
6+
public sealed record ExampleRunResult(
7+
int ExitCode,
8+
string StandardOutput,
9+
string StandardError,
10+
bool TimedOut);

0 commit comments

Comments
 (0)