From ddf4e4c68faac954dffef1f8c29849a2b2cfd335 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 30 Jan 2026 20:29:31 +0000 Subject: [PATCH 1/2] Refactor toolkit into independent components The aim is to make it easier to add and maintain individual features. Common options have been refactored into separate classes for reuse. --- Harp.Toolkit/ListCommand.cs | 17 ++++++++ Harp.Toolkit/PortNameOption.cs | 13 ++++++ Harp.Toolkit/PortTimeoutOption.cs | 12 ++++++ Harp.Toolkit/Program.cs | 58 +++------------------------ Harp.Toolkit/UpdateFirmwareCommand.cs | 43 ++++++++++++++++++++ 5 files changed, 90 insertions(+), 53 deletions(-) create mode 100644 Harp.Toolkit/ListCommand.cs create mode 100644 Harp.Toolkit/PortNameOption.cs create mode 100644 Harp.Toolkit/PortTimeoutOption.cs create mode 100644 Harp.Toolkit/UpdateFirmwareCommand.cs diff --git a/Harp.Toolkit/ListCommand.cs b/Harp.Toolkit/ListCommand.cs new file mode 100644 index 0000000..12abf22 --- /dev/null +++ b/Harp.Toolkit/ListCommand.cs @@ -0,0 +1,17 @@ +using System.CommandLine; +using System.IO.Ports; + +namespace Harp.Toolkit; + +public class ListCommand : Command +{ + public ListCommand() + : base("list", "Lists all available system serial ports.") + { + SetAction(parseResult => + { + var portNames = SerialPort.GetPortNames(); + Console.WriteLine($"PortNames: [{string.Join(", ", portNames)}]"); + }); + } +} diff --git a/Harp.Toolkit/PortNameOption.cs b/Harp.Toolkit/PortNameOption.cs new file mode 100644 index 0000000..b9168df --- /dev/null +++ b/Harp.Toolkit/PortNameOption.cs @@ -0,0 +1,13 @@ +using System.CommandLine; + +namespace Harp.Toolkit; + +public class PortNameOption : Option +{ + public PortNameOption() + : base("--port") + { + Description = "Specifies the name of the serial port used to communicate with the device."; + Required = true; + } +} diff --git a/Harp.Toolkit/PortTimeoutOption.cs b/Harp.Toolkit/PortTimeoutOption.cs new file mode 100644 index 0000000..c0d5ee3 --- /dev/null +++ b/Harp.Toolkit/PortTimeoutOption.cs @@ -0,0 +1,12 @@ +using System.CommandLine; + +namespace Harp.Toolkit; + +public class PortTimeoutOption : Option +{ + public PortTimeoutOption() + : base("--timeout") + { + Description = "Specifies an optional timeout, in milliseconds, to receive a response from the device."; + } +} diff --git a/Harp.Toolkit/Program.cs b/Harp.Toolkit/Program.cs index 502a73a..dbc7efe 100644 --- a/Harp.Toolkit/Program.cs +++ b/Harp.Toolkit/Program.cs @@ -8,61 +8,13 @@ internal class Program { static async Task Main(string[] args) { - Option portNameOption = new("--port") - { - Description = "Specifies the name of the serial port used to communicate with the device.", - Required = true - }; - - Option portTimeoutOption = new("--timeout") - { - Description = "Specifies an optional timeout, in milliseconds, to receive a response from the device." - }; - - Option firmwarePathOption = new("--path") - { - Description = "Specifies the path of the firmware file to write to the device.", - Required = true - }; - - Option forceUpdateOption = new("--force") - { - Description = "Indicates whether to force a firmware update on the device regardless of compatibility." - }; - - var listCommand = new Command("list", description: "Lists all available system serial ports."); - listCommand.SetAction(parseResult => - { - var portNames = SerialPort.GetPortNames(); - Console.WriteLine($"PortNames: [{string.Join(", ", portNames)}]"); - }); - - var updateCommand = new Command("update", description: "Update the device firmware from a local HEX file."); - updateCommand.Options.Add(portNameOption); - updateCommand.Options.Add(firmwarePathOption); - updateCommand.Options.Add(forceUpdateOption); - updateCommand.SetAction(async parseResult => - { - var firmwarePath = parseResult.GetRequiredValue(firmwarePathOption); - var portName = parseResult.GetRequiredValue(portNameOption); - var forceUpdate = parseResult.GetValue(forceUpdateOption); - - var firmware = DeviceFirmware.FromFile(firmwarePath.FullName); - Console.WriteLine($"{firmware.Metadata}"); - ProgressBar.Write(0); - try - { - var progress = new Progress(ProgressBar.Update); - await Bootloader.UpdateFirmwareAsync(portName, firmware, forceUpdate, progress); - } - finally { Console.WriteLine(); } - }); - - var rootCommand = new RootCommand("Tool for inspecting, updating and interfacing with Harp devices."); + RootCommand rootCommand = new("Tool for inspecting, updating and interfacing with Harp devices."); + PortNameOption portNameOption = new(); + PortTimeoutOption portTimeoutOption = new(); rootCommand.Options.Add(portNameOption); rootCommand.Options.Add(portTimeoutOption); - rootCommand.Subcommands.Add(listCommand); - rootCommand.Subcommands.Add(updateCommand); + rootCommand.Subcommands.Add(new ListCommand()); + rootCommand.Subcommands.Add(new UpdateFirmwareCommand()); rootCommand.SetAction(async parseResult => { var portName = parseResult.GetRequiredValue(portNameOption); diff --git a/Harp.Toolkit/UpdateFirmwareCommand.cs b/Harp.Toolkit/UpdateFirmwareCommand.cs new file mode 100644 index 0000000..c346735 --- /dev/null +++ b/Harp.Toolkit/UpdateFirmwareCommand.cs @@ -0,0 +1,43 @@ +using System.CommandLine; +using Bonsai.Harp; + +namespace Harp.Toolkit; + +public class UpdateFirmwareCommand : Command +{ + public UpdateFirmwareCommand() + : base("update", "Update the device firmware from a local HEX file.") + { + PortNameOption portNameOption = new(); + Option firmwarePathOption = new("--path") + { + Description = "Specifies the path of the firmware file to write to the device.", + Required = true + }; + + Option forceUpdateOption = new("--force") + { + Description = "Indicates whether to force a firmware update on the device regardless of compatibility." + }; + + Options.Add(portNameOption); + Options.Add(firmwarePathOption); + Options.Add(forceUpdateOption); + SetAction(async parseResult => + { + var firmwarePath = parseResult.GetRequiredValue(firmwarePathOption); + var portName = parseResult.GetRequiredValue(portNameOption); + var forceUpdate = parseResult.GetValue(forceUpdateOption); + + var firmware = DeviceFirmware.FromFile(firmwarePath.FullName); + Console.WriteLine($"{firmware.Metadata}"); + ProgressBar.Write(0); + try + { + var progress = new Progress(ProgressBar.Update); + await Bootloader.UpdateFirmwareAsync(portName, firmware, forceUpdate, progress); + } + finally { Console.WriteLine(); } + }); + } +} From f988c4bc03ea80e0690f5bf8de36ad95f65881f2 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 1 Feb 2026 13:11:16 -0800 Subject: [PATCH 2/2] Add draft for benchmarking tool --- Harp.Toolkit/Benchmark/BenchmarkCommand.cs | 152 +++++++++++++++++ Harp.Toolkit/Benchmark/HarpTestAttribute.cs | 7 + Harp.Toolkit/Benchmark/HtmlReportGenerator.cs | 21 +++ Harp.Toolkit/Benchmark/Report.cs | 8 + Harp.Toolkit/Benchmark/ReportTemplate.cshtml | 116 +++++++++++++ Harp.Toolkit/Benchmark/Result.cs | 161 ++++++++++++++++++ Harp.Toolkit/Benchmark/Runner.cs | 53 ++++++ Harp.Toolkit/Benchmark/Suite.cs | 71 ++++++++ .../Benchmark/Suites/RoundTripTestSuite.cs | 42 +++++ .../Benchmark/Suites/TimestampSecondSuite.cs | 24 +++ Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs | 21 +++ Harp.Toolkit/Harp.Toolkit.csproj | 9 + Harp.Toolkit/Program.cs | 1 + 13 files changed, 686 insertions(+) create mode 100644 Harp.Toolkit/Benchmark/BenchmarkCommand.cs create mode 100644 Harp.Toolkit/Benchmark/HarpTestAttribute.cs create mode 100644 Harp.Toolkit/Benchmark/HtmlReportGenerator.cs create mode 100644 Harp.Toolkit/Benchmark/Report.cs create mode 100644 Harp.Toolkit/Benchmark/ReportTemplate.cshtml create mode 100644 Harp.Toolkit/Benchmark/Result.cs create mode 100644 Harp.Toolkit/Benchmark/Runner.cs create mode 100644 Harp.Toolkit/Benchmark/Suite.cs create mode 100644 Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs create mode 100644 Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs create mode 100644 Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs diff --git a/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/Harp.Toolkit/Benchmark/BenchmarkCommand.cs new file mode 100644 index 0000000..f6754ef --- /dev/null +++ b/Harp.Toolkit/Benchmark/BenchmarkCommand.cs @@ -0,0 +1,152 @@ +using System.CommandLine; +using Spectre.Console; + +namespace Harp.Toolkit; +public class BenchmarkCommand : Command +{ + public BenchmarkCommand() + : base("benchmark", "Run benchmark tests on the device.") + { + PortNameOption portNameOption = new(); + Option fileOption = new("--report") + { + Description = "Path to the HTML report generated after running tests.", + Required = false, + }; + + Option verboseOption = new("--verbose") + { + Description = "Show detailed results for each test.", + Required = false, + }; + Options.Add(portNameOption); + Options.Add(fileOption); + Options.Add(verboseOption); + SetAction(parsedResult => + { + string portName = parsedResult.GetRequiredValue(portNameOption); + FileInfo? reportFile = parsedResult.GetValue(fileOption); + bool verbose = parsedResult.GetValue(verboseOption); + return RunBenchmarks(portName, reportFile, verbose, CancellationToken.None); + }); + } + + static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, CancellationToken cancellationToken) + { + AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); + + var runner = new CoreRunner(); + var report = new Report + { + DeviceName = $"Harp Device ({portName})", + RunDate = DateTime.Now + }; + + await AnsiConsole.Progress() + .StartAsync(async ctx => + { + var task = ctx.AddTask("[green]Running tests...[/]", true, runner.TestCount); + + await foreach (var (suite, result) in runner.RunAllAsync(portName, cancellationToken)) + { + task.Increment(1); + AnsiConsole.MarkupLine($"[grey]{suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}"); + + var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name); + if (suiteResult == null) + { + suiteResult = new SuiteResult + { + Name = suite.GetType().Name, + Description = suite.Description + }; + report.Suites.Add(suiteResult); + } + suiteResult.Results.Add(result); + } + }); + + if (verbose) + { + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Rule("[yellow]Detailed Results[/]")); + foreach (var suite in report.Suites) + { + AnsiConsole.MarkupLine($"[bold underline]{suite.Name}[/]"); + AnsiConsole.MarkupLine($"[dim]{suite.Description}[/]"); + + var table = new Table(); + table.AddColumn("Test Case"); + table.AddColumn("Status"); + table.AddColumn("Details"); + table.AddColumn("Message"); + + foreach (var test in suite.Results) + { + string details = ""; + string message = test.Result.Message ?? ""; + + if (test.Result is NumericBenchmarkResult bsr) + { + details = $"Mean: {bsr.Summary.Mean:F4}\nMedian: {bsr.Summary.Median:F4}\nStdDev: {bsr.Summary.StdDev:F4}\nMin: {bsr.Summary.Min:F4}\nMax: {bsr.Summary.Max:F4}\nPercentiles: 99th={bsr.Summary.Percentile99:F4}, 01th={bsr.Summary.Percentile01:F4}"; + } + else if (test.Result is ErrorResult er) + { + details = $"{er.Exception.GetType().Name}"; + } + else + { + var valProp = test.Result?.GetType().GetProperty("Value"); + if (valProp != null) + { + var val = valProp.GetValue(test.Result); + details = val?.ToString() ?? ""; + } + } + + table.AddRow( + new Markup($"[bold]{test.Name}[/]\n[dim]{test.Description}[/]"), + new Markup(GetResultMarkup(test.Result)), + new Markup(details), + new Markup(message) + ); + } + AnsiConsole.Write(table); + AnsiConsole.WriteLine(); + } + } + + if (reportFile != null) + { + AnsiConsole.Markup("Generating HTML report..."); + string html = await HtmlReportGenerator.GenerateAsync(report); + string fileName = reportFile?.FullName ?? $"TestReport_{DateTime.Now:yyyyMMdd_HHmmss}.html"; + await File.WriteAllTextAsync(fileName, html, cancellationToken); + AnsiConsole.MarkupLine($"[green]Done![/] Report generated: [link]{fileName}[/]"); + } + } + + static string GetResultMarkup(IResult result) + { + return result.Status switch + { + Status.Passed => "[green]Passed[/]", + Status.Failed => "[red]Failed[/]", + Status.Error => "[red]Error[/]", + Status.Skipped => "[yellow]Skipped[/]", + _ => $"[white]{result.Status}[/]" + }; + } + + class CoreRunner : Runner + { + public CoreRunner() : base() + { + AddSuite(new WhoAmISuite()); + AddSuite(new RoundTripTestSuite()); + AddSuite(new TimestampSecondsSuite()); + } + } +} + + diff --git a/Harp.Toolkit/Benchmark/HarpTestAttribute.cs b/Harp.Toolkit/Benchmark/HarpTestAttribute.cs new file mode 100644 index 0000000..f0bf7cb --- /dev/null +++ b/Harp.Toolkit/Benchmark/HarpTestAttribute.cs @@ -0,0 +1,7 @@ +namespace Harp.Toolkit; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class HarpTestAttribute : Attribute +{ + public string? Description { get; set; } +} diff --git a/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs b/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs new file mode 100644 index 0000000..ede8e25 --- /dev/null +++ b/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs @@ -0,0 +1,21 @@ +using System.Reflection; +using RazorLight; + +namespace Harp.Toolkit; + +public static class HtmlReportGenerator +{ + public static async Task GenerateAsync(Report report) + { + var engine = new RazorLightEngineBuilder() + .UseFileSystemProject(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)) + .UseMemoryCachingProvider() + .Build(); + + // The template is copied to the output directory under Reporting/ReportTemplate.cshtml + // RazorLight expects the path relative to the project root (which we set to the assembly location) + string templatePath = Path.Combine("Benchmark", "ReportTemplate.cshtml"); + + return await engine.CompileRenderAsync(templatePath, report); + } +} diff --git a/Harp.Toolkit/Benchmark/Report.cs b/Harp.Toolkit/Benchmark/Report.cs new file mode 100644 index 0000000..cc716f2 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Report.cs @@ -0,0 +1,8 @@ +namespace Harp.Toolkit; + +public class Report +{ + public string DeviceName { get; set; } = "Unknown Device"; + public DateTime RunDate { get; set; } = DateTime.Now; + public List Suites { get; set; } = new(); +} diff --git a/Harp.Toolkit/Benchmark/ReportTemplate.cshtml b/Harp.Toolkit/Benchmark/ReportTemplate.cshtml new file mode 100644 index 0000000..9cdfbac --- /dev/null +++ b/Harp.Toolkit/Benchmark/ReportTemplate.cshtml @@ -0,0 +1,116 @@ +@using Harp.Toolkit +@model Harp.Toolkit.Report + + + + + + + Test Report - @Model.DeviceName + + + + +
+
+
+

@Model.DeviceName

+

Test Execution Report • @Model.RunDate.ToString("MMMM dd, yyyy HH:mm:ss")

+
+
+ Harp Toolkit Test +
+
+ + @foreach (var suite in Model.Suites) + { +
+
+

@suite.Name

+

@suite.Description

+
+
+
+ + + + + + + + + + + @foreach (var test in suite.Results) + { + + + + + + + } + +
Test CaseStatusResult DetailsMessage
+
@test.Name
+
@test.Description
+ 		+ + @(test.Result?.Status.ToString().ToUpper() ?? "SKIPPED") + + + @if (test.Result is NumericBenchmarkResult bsr) + { +
+
+
Mean: @bsr.Summary.Mean.ToString("F4")
+
Median: @bsr.Summary.Median.ToString("F4")
+
StdDev: @bsr.Summary.StdDev.ToString("F4")
+
+
+
Min: @bsr.Summary.Min.ToString("F4")
+
Max: @bsr.Summary.Max.ToString("F4")
+
+
+ } + else + { + var valProp = test.Result?.GetType().GetProperty("Value"); + object? val = null; + if (valProp != null) + { + val = valProp.GetValue(test.Result); + } + + if (val != null) + { + @val + } + } + + @if (test.Result is ErrorResult er && er.Exception != null) + { +
+ 
@er.Exception.ToString()
+
+ } + 		+ @test.Result?.Message +
+
+
+
+ } +
+ + diff --git a/Harp.Toolkit/Benchmark/Result.cs b/Harp.Toolkit/Benchmark/Result.cs new file mode 100644 index 0000000..312f97a --- /dev/null +++ b/Harp.Toolkit/Benchmark/Result.cs @@ -0,0 +1,161 @@ + +namespace Harp.Toolkit; + + +public enum Status +{ + Passed, + Failed, + Skipped, + Error +} + +public interface IResult +{ + string? Message { get; } + + Status Status { get; } +} + +public class ErrorResult(Exception exception) : IResult +{ + public string? Message { get; } = exception.Message; + public Status Status { get; } = Status.Error; + public Exception Exception { get; } = exception; +} + +public class Result : IResult +{ + public Result(T value, Status status, string message = "") + { + Status = status; + Value = value; + Message = message; + } + + public Result(T value, Func predicate, Func? messageFactory = null) + { + bool evaluation = predicate(value); + Status = evaluation ? Status.Passed : Status.Failed; + Value = value; + Message = messageFactory?.Invoke(value, evaluation) ?? string.Empty; + } + + + public string Message { get; } + public Status Status { get; } + public T Value { get; } + + public override string? ToString() + { + return $"Result(Status={Status}, Value={Value}, Message={Message})"; + } +} + + +public class AssertionResult : Result +{ + public AssertionResult(bool value, string message = "") + : base(value, value ? Status.Passed : Status.Failed, message) + { + } + + public AssertionResult(bool value, Func? messageFactory = null) + : base( + value, + v => v, + messageFactory is null ? null : ((value, evaluation) => messageFactory(value))) + { + + } +} + + +public class NumericBenchmarkResult : Result +{ + + public NumericBenchmarkResult(double[] values, Status status, string message = "") + : base(values, status, message) + { + Summary = new BenchmarkSummary(values); + } + + public NumericBenchmarkResult(BenchmarkSummary summary, Status status, string message = "") + : base(summary.Values, status, message) + { + Summary = summary; + } + + public NumericBenchmarkResult(double[] values, Func predicate, Func? messageFactory = null) + : base(values, predicate, messageFactory) + { + Summary = new BenchmarkSummary(values); + } + + public BenchmarkSummary Summary { get; } +} + + +public class BenchmarkSummary +{ + public readonly double[] Values; + + + public BenchmarkSummary(double[] values) + { + Values = values ?? Array.Empty(); + // TODO consider copying here since we are mutating + Array.Sort(Values); + } + + public double Mean => Values.Length == 0 ? double.NaN : Values.Average(); + + public double StdDev + { + get + { + if (Values.Length == 0) return double.NaN; + var mean = Mean; + var sumOfSquares = Values.Sum(v => (v - mean) * (v - mean)); + return Math.Sqrt(sumOfSquares / Values.Length); + } + } + + public double Median + { + get + { + if (Values.Length == 0) return double.NaN; + int mid = Values.Length / 2; + if (Values.Length % 2 == 0) + return (Values[mid - 1] + Values[mid]) / 2.0; + else + return Values[mid]; + } + } + + public double Max => Values.Length == 0 ? double.NaN : Values[Values.Length - 1]; + + public double Min => Values.Length == 0 ? double.NaN : Values[0]; + + public double Percentile99 => Percentile(0.99); + public double Percentile01 => Percentile(0.01); + + public double Percentile(double percentile) + { + if (Values.Length == 0) return double.NaN; + if (percentile < 0f || percentile > 1.0f) + { + throw new ArgumentOutOfRangeException(nameof(percentile), "Percentile must be between 0 and 1."); + } + + double rank = percentile * (Values.Length - 1); + int lower = (int)Math.Floor(rank); + int upper = (int)Math.Ceiling(rank); + if (lower == upper) return Values[lower]; + // Apparently this is how you solve rounding with percentiles + // https://en.wikipedia.org/wiki/Percentile + double weight = rank - lower; + return Values[lower] * (1 - weight) + Values[upper] * weight; + } +} diff --git a/Harp.Toolkit/Benchmark/Runner.cs b/Harp.Toolkit/Benchmark/Runner.cs new file mode 100644 index 0000000..ea062ba --- /dev/null +++ b/Harp.Toolkit/Benchmark/Runner.cs @@ -0,0 +1,53 @@ +using System.Runtime.CompilerServices; + +namespace Harp.Toolkit; + +public class Runner +{ + private readonly List suites = new(); + + public Runner() + { + } + + public int TestCount => suites.Sum(s => s.TestCount); + + public IEnumerable CollectSuites() + { + return suites.AsReadOnly(); + } + + public async IAsyncEnumerable<(Suite Suite, MethodResult Result)> RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var suite in suites) + { + await foreach (var result in suite.RunAllAsync(portName, cancellationToken)) + { + yield return (suite, result); + } + } + } + + public void AddSuite(Suite suite) + { + if (suite == null) + { + throw new ArgumentNullException(nameof(suite)); + } + suites.Add(suite); + } + + public void ClearSuites() + { + suites.Clear(); + } + + public bool RemoveSuite(Suite suite) + { + if (suite == null) + { + throw new ArgumentNullException(nameof(suite)); + } + return suites.Remove(suite); + } +} diff --git a/Harp.Toolkit/Benchmark/Suite.cs b/Harp.Toolkit/Benchmark/Suite.cs new file mode 100644 index 0000000..5f27464 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suite.cs @@ -0,0 +1,71 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Bonsai.Harp; + +namespace Harp.Toolkit; + + +public abstract class Suite +{ + public abstract string Description { get; } + + public int TestCount => CollectTests().Count(); + + private IEnumerable<(MethodInfo Method, HarpTestAttribute Attribute)> CollectTests() + { + return GetType() + .GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + .Select(m => (Method: m, Attribute: m.GetCustomAttribute()!)) + .Where(x => x.Attribute != null); + } + + public async IAsyncEnumerable RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var (method, attr) in CollectTests()) + { + cancellationToken.ThrowIfCancellationRequested(); + + IResult testResult; + try + { + object? resultObj = method.Invoke(this, new object[] { portName }); + if (resultObj is Task task) + { + testResult = await task; + } + else if (resultObj is IResult syncResult) + { + testResult = syncResult; + } + else + { + throw new InvalidOperationException($"Test method '{method.Name}' must return IResult or Task."); + } + } + catch (Exception ex) + { + testResult = new ErrorResult(ex.InnerException ?? ex); + } + yield return new MethodResult + { + Result = testResult, + Name = method.Name, + Description = attr.Description ?? string.Empty + }; + } + } +} + +public class SuiteResult +{ + public required string Name { get; set; } + public string Description { get; set; } = string.Empty; + public List Results { get; set; } = new(); +} + +public class MethodResult +{ + public required string Name { get; set; } + public string Description { get; set; } = string.Empty; + public required IResult Result { get; set; } +} diff --git a/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs b/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs new file mode 100644 index 0000000..52031f7 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs @@ -0,0 +1,42 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit; + +public class RoundTripTestSuite : Suite +{ + private double maxRoundTripDelayMs; + public RoundTripTestSuite(double maxRoundTripDelayMs = 4.0) + { + this.maxRoundTripDelayMs = maxRoundTripDelayMs; + } + + public override string Description => "A bunch of tests to benchmark round trip read/writes."; + + [HarpTest(Description = "Benchmarks the round trip time for a WhoAmI read command.")] + public async Task BenchmarkRoundTrip(string portName) + { + const int n = 1000; + double[] timestamps = new double[n]; + HarpMessage probe = WhoAmI.FromPayload(MessageType.Read, default); + using (var device = new AsyncDevice(portName)) + { + for (int i = 0; i < n; i++) + { + var reply = await device.CommandAsync(probe); + timestamps[i] = reply.GetTimestamp(); + } + } + var derivatives = timestamps + .Zip(timestamps.Skip(1), (previous, current) => (current - previous) * 1e3) + .ToArray(); + var benchmark = new BenchmarkSummary(derivatives); + if (benchmark.Max > maxRoundTripDelayMs) + { + return new NumericBenchmarkResult(benchmark, Status.Failed, $"Round trip WhoAmI read benchmark exceeded maximum allowed delay of {maxRoundTripDelayMs} ms."); + } + else + { + return new NumericBenchmarkResult(benchmark, Status.Passed, "Round trip WhoAmI read benchmark."); + } + } +} diff --git a/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs b/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs new file mode 100644 index 0000000..75f9266 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs @@ -0,0 +1,24 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit; + +public class TimestampSecondsSuite : Suite +{ + public override string Description => "Timestamp Seconds Register Tests"; + + [HarpTest(Description = "Validates that the Timestamp Seconds register is writable.")] + public async Task IsWritable(string portName) + { + const uint setSeconds = 42; + using (var device = new AsyncDevice(portName)) + { + await device.WriteTimestampSecondsAsync(setSeconds); + await Task.Delay(1); + HarpMessage response = await device.CommandAsync(TimestampSeconds.FromPayload(MessageType.Read, default)); + double readSeconds = response.GetTimestamp(); + return new AssertionResult( + readSeconds - setSeconds < 1.0, + (success) => success ? $"`TimestampSeconds` register is writable and updates as expected." : $"`TimestampSeconds` register is not writable, Expected value: {setSeconds}, read value: {readSeconds}."); + } + } +} diff --git a/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs b/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs new file mode 100644 index 0000000..9954bc1 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs @@ -0,0 +1,21 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit; + +public class WhoAmISuite : Suite +{ + public override string Description => "WhoAmI Register Tests"; + + [HarpTest(Description = "Validates that the WhoAmI register exists and contains a value.")] + public async Task CheckWhoAmI(string portName) + { + using (var device = new AsyncDevice(portName)) + { + int value = await device.ReadWhoAmIAsync(); + return new Result( + value, + (v) => v > 0 && v < 9999, + (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); + } + } +} diff --git a/Harp.Toolkit/Harp.Toolkit.csproj b/Harp.Toolkit/Harp.Toolkit.csproj index 076ea54..291dffc 100644 --- a/Harp.Toolkit/Harp.Toolkit.csproj +++ b/Harp.Toolkit/Harp.Toolkit.csproj @@ -6,11 +6,20 @@ enable 0.1.0 enable + true + + + + + + + PreserveNewest + diff --git a/Harp.Toolkit/Program.cs b/Harp.Toolkit/Program.cs index dbc7efe..6ce4961 100644 --- a/Harp.Toolkit/Program.cs +++ b/Harp.Toolkit/Program.cs @@ -15,6 +15,7 @@ static async Task Main(string[] args) rootCommand.Options.Add(portTimeoutOption); rootCommand.Subcommands.Add(new ListCommand()); rootCommand.Subcommands.Add(new UpdateFirmwareCommand()); + rootCommand.Subcommands.Add(new BenchmarkCommand()); rootCommand.SetAction(async parseResult => { var portName = parseResult.GetRequiredValue(portNameOption);