From 9ea2cb2103c81503dcda63871f42ac574ff89a72 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:01:26 +0100 Subject: [PATCH 1/2] fix(generator): preserve gcloud resource synopsis metadata Land the reviewed resource parsing prerequisite before latest-main regeneration. Refs #5112, #4850. --- .../Fixtures/Gcloud/kms-keyrings-delete.txt | 53 ++++ .../metastore-services-migrations-delete.txt | 79 ++++++ ...metastore-services-migrations-describe.txt | 73 +++++ .../Cli/GcloudGroupDescriptionTests.cs | 7 +- .../Cli/GcloudResourceArgumentTests.cs | 258 ++++++++++++++++++ .../Scrapers/UsageSynopsisParserTests.cs | 104 ++++++- .../Models/CliArgumentGroup.cs | 7 +- .../PublicAPI.Unshipped.txt | 5 + .../Scrapers/Cli/CliScraperBase.cs | 13 +- .../Scrapers/Cli/GcloudCliScraper.cs | 207 ++++++++------ .../Scrapers/Cli/UsageSynopsisParser.cs | 116 +++++--- 11 files changed, 794 insertions(+), 128 deletions(-) create mode 100644 tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/kms-keyrings-delete.txt create mode 100644 tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-delete.txt create mode 100644 tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-describe.txt create mode 100644 tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudResourceArgumentTests.cs diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/kms-keyrings-delete.txt b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/kms-keyrings-delete.txt new file mode 100644 index 00000000000..f44116148c0 --- /dev/null +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/kms-keyrings-delete.txt @@ -0,0 +1,53 @@ +NAME + gcloud kms keyrings delete - delete a keyring + +SYNOPSIS + gcloud kms keyrings delete (KEYRING : --location=LOCATION) + [GCLOUD_WIDE_FLAG ...] + +DESCRIPTION + Deletes a keyring within the given location. + +EXAMPLES + The following command deletes the keyring fellowship within location + us-east1: + + $ gcloud kms keyrings delete fellowship --location=us-east1 + +POSITIONAL ARGUMENTS + Keyring resource - The KMS keyring resource. The arguments in this group + can be used to specify the attributes of this resource. (NOTE) Some + attributes are not given arguments in this group but can be set in other + ways. + + To set the project attribute: + * provide the argument keyring on the command line with a fully + specified name; + * set the property core/project. + + This must be specified. + + KEYRING + ID of the keyring or fully qualified identifier for the keyring. + + To set the keyring attribute: + + provide the argument keyring on the command line. + + This positional argument must be specified if any of the other + arguments in this group are specified. + + --location=LOCATION + The Google Cloud location for the keyring. + + To set the location attribute: + + provide the argument keyring on the command line with a fully + specified name; + + provide the argument --location on the command line. + +GCLOUD WIDE FLAGS + These flags are available to all commands: --access-token-file, --account, + --billing-project, --configuration, --flags-file, --flatten, --format, + --help, --impersonate-service-account, --log-http, --project, --quiet, + --trace-token, --user-output-enabled, --verbosity. + + Run $ gcloud help for details. diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-delete.txt b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-delete.txt new file mode 100644 index 00000000000..11c77109455 --- /dev/null +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-delete.txt @@ -0,0 +1,79 @@ +NAME + gcloud metastore services migrations delete - delete a migration + +SYNOPSIS + gcloud metastore services migrations delete + (MIGRATION : --location=LOCATION --service=SERVICE) [--async] + [GCLOUD_WIDE_FLAG ...] + +DESCRIPTION + Delete a migration. + + If run asynchronously with --async, exits after printing an operation name + that can be used to poll the status of the deletion via: + + gcloud metastore operations describe + +EXAMPLES + To delete a migration named my-migration for the service my-service, run: + + $ gcloud metastore services migrations delete my-migration \ + --service=my-service + +POSITIONAL ARGUMENTS + Migration resource - Migration to delete. The arguments in this group can + be used to specify the attributes of this resource. (NOTE) Some attributes + are not given arguments in this group but can be set in other ways. + + To set the project attribute: + * provide the argument migration on the command line with a fully + specified name; + * provide the argument --project on the command line; + * set the property core/project. + + This must be specified. + + MIGRATION + ID of the migration or fully qualified identifier for the migration. + + To set the migration attribute: + + provide the argument migration on the command line. + + This positional argument must be specified if any of the other + arguments in this group are specified. + + --location=LOCATION + The location of the Dataproc Metastore service. + + If not specified, will use default metastore/location. + + To set the location attribute: + + provide the argument migration on the command line with a fully + specified name; + + provide the argument --location on the command line; + + set the property metastore/location. + + --service=SERVICE + The identifier of the Dataproc Metastore service + + To set the service attribute: + + provide the argument migration on the command line with a fully + specified name; + + provide the argument --service on the command line. + +FLAGS + --async + Return immediately, without waiting for the operation in progress to + complete. + +GCLOUD WIDE FLAGS + These flags are available to all commands: --access-token-file, --account, + --billing-project, --configuration, --flags-file, --flatten, --format, + --help, --impersonate-service-account, --log-http, --project, --quiet, + --trace-token, --user-output-enabled, --verbosity. + + Run $ gcloud help for details. + +API REFERENCE + This command uses the metastore/v1 API. The full documentation for this API + can be found at: https://cloud.google.com/dataproc-metastore/docs diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-describe.txt b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-describe.txt new file mode 100644 index 00000000000..694564f9baa --- /dev/null +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Fixtures/Gcloud/metastore-services-migrations-describe.txt @@ -0,0 +1,73 @@ +NAME + gcloud metastore services migrations describe - describe a migration + +SYNOPSIS + gcloud metastore services migrations describe + (MIGRATION : --location=LOCATION --service=SERVICE) + [GCLOUD_WIDE_FLAG ...] + +DESCRIPTION + Describe a migration. + + Displays all details of a migration given a migration ID. + +EXAMPLES + To describe a migration with the ID my-migration under service my-service, + run: + + $ gcloud metastore services migrations describe my-migration \ + --service=my-service + +POSITIONAL ARGUMENTS + Migration resource - Arguments and flags that specify the migration you + want to describe. The arguments in this group can be used to specify the + attributes of this resource. (NOTE) Some attributes are not given + arguments in this group but can be set in other ways. + + To set the project attribute: + * provide the argument migration on the command line with a fully + specified name; + * provide the argument --project on the command line; + * set the property core/project. + + This must be specified. + + MIGRATION + ID of the migration or fully qualified identifier for the migration. + + To set the migration attribute: + + provide the argument migration on the command line. + + This positional argument must be specified if any of the other + arguments in this group are specified. + + --location=LOCATION + The location of the Dataproc Metastore service. + + If not specified, will use default metastore/location. + + To set the location attribute: + + provide the argument migration on the command line with a fully + specified name; + + provide the argument --location on the command line; + + set the property metastore/location. + + --service=SERVICE + The identifier of the Dataproc Metastore service + + To set the service attribute: + + provide the argument migration on the command line with a fully + specified name; + + provide the argument --service on the command line. + +GCLOUD WIDE FLAGS + These flags are available to all commands: --access-token-file, --account, + --billing-project, --configuration, --flags-file, --flatten, --format, + --help, --impersonate-service-account, --log-http, --project, --quiet, + --trace-token, --user-output-enabled, --verbosity. + + Run $ gcloud help for details. + +API REFERENCE + This command uses the metastore/v1 API. The full documentation for this API + can be found at: https://cloud.google.com/dataproc-metastore/docs diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudGroupDescriptionTests.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudGroupDescriptionTests.cs index 61df9ab3403..90743b6a5c5 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudGroupDescriptionTests.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudGroupDescriptionTests.cs @@ -42,7 +42,12 @@ public async Task Gcloud_Scopes_Same_Indentation_Groups_From_Authoritative_Help( "Fixtures", "Gcloud", "metastore-services-migrations-start.txt")); var command = (await CreateGcloudScraper().Parse( ["gcloud", "metastore", "services", "migrations", "start"], help))!; - var root = command.ArgumentGroups.Single(); + var root = command.ArgumentGroups.Single(group => + group.FlattenArguments().Any(argument => argument.SwitchName == "--async")); + + await Assert.That(command.PositionalArguments.Single().PropertyName).IsEqualTo("Service"); + await Assert.That(command.Options.Single(option => option.SwitchName == "--location").Description) + .StartsWith("Service resource").And.DoesNotContain("Hive").And.DoesNotContain("Iceberg"); await Assert.That(root.Description).IsNull(); await Assert.That(root.Groups).Count().IsEqualTo(2); diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudResourceArgumentTests.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudResourceArgumentTests.cs new file mode 100644 index 00000000000..4359477c4e4 --- /dev/null +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/Cli/GcloudResourceArgumentTests.cs @@ -0,0 +1,258 @@ +using Microsoft.Extensions.Logging.Abstractions; +using ModularPipelines.OptionsGenerator.Generators; +using ModularPipelines.OptionsGenerator.Models; +using ModularPipelines.OptionsGenerator.Scrapers.Cli; +using ModularPipelines.OptionsGenerator.TypeDetection; + +namespace ModularPipelines.OptionsGenerator.Tests.Scrapers.Cli; + +public class GcloudResourceArgumentTests +{ + [Test] + [Arguments("B")] + [Arguments("[B]")] + public async Task Incomplete_Synopsis_Does_Not_Guess_Operand_Order_Or_Requiredness(string middle) + { + var help = $$""" + NAME + gcloud example copy - copy resources + SYNOPSIS + gcloud example copy A C + POSITIONAL ARGUMENTS + A + The first operand. + {{middle}} + The middle operand. + C + The last operand. + """; + var exception = await Assert.That(async () => + { await new TestScraper().Parse(["gcloud", "example", "copy"], help); }) + .Throws(); + await Assert.That(exception!.Message).Contains("synopsis omits declared positional operand 'B'"); + } + + [Test] + public async Task Without_Synopsis_Explicit_Operand_Declarations_Retain_Order_And_Optionality() + { + const string help = """ + NAME + gcloud example copy - copy resources + POSITIONAL ARGUMENTS + A + The first operand. + [B] + The optional middle operand. + C + The last operand. + """; + var command = (await new TestScraper().Parse(["gcloud", "example", "copy"], help))!; + await Assert.That(command.PositionalArguments.Select(argument => argument.PropertyName)) + .IsEquivalentTo(["A", "B", "C"], TUnit.Assertions.Enums.CollectionOrdering.Matching); + await Assert.That(command.PositionalArguments.Select(argument => argument.IsRequired)) + .IsEquivalentTo([true, false, true], TUnit.Assertions.Enums.CollectionOrdering.Matching); + } + + [Test] + [Arguments("kms-keyrings-delete", "kms keyrings delete", "Keyring", "--location")] + [Arguments("metastore-services-migrations-describe", "metastore services migrations describe", "Migration", "--location,--service")] + [Arguments("metastore-services-migrations-delete", "metastore services migrations delete", "Migration", "--location,--service,--async")] + public async Task Resource_Groups_Preserve_Required_Operands_And_Optional_Selectors( + string fixture, string path, string operandName, string switches) + { + var help = await File.ReadAllTextAsync(Path.Combine(AppContext.BaseDirectory, "Fixtures", "Gcloud", fixture + ".txt")); + var command = (await ScrapeFixture(path, help)).Single(); + + var operand = command.PositionalArguments.Single(); + await Assert.That(command.HasOperandTakingUsage).IsTrue(); + await Assert.That(command.UsageSynopsis).Contains($"gcloud {path}"); + await Assert.That(command.UsagePositionalArguments.Single().PropertyName).IsEqualTo(operandName); + await Assert.That(operand.PropertyName).IsEqualTo(operandName); + await Assert.That(operand.IsRequired).IsTrue(); + await Assert.That(operand.PositionIndex).IsEqualTo(0); + await Assert.That(operand.Description).Contains("fully qualified identifier"); + await Assert.That(command.Options.Select(option => option.SwitchName)).IsEquivalentTo(switches.Split(',')); + await Assert.That(command.Options.All(option => !option.IsRequired)).IsTrue(); + await Assert.That(command.RequiredAlternativeGroups).IsEmpty(); + + var tool = new CliToolDefinition + { + ToolName = "gcloud", + NamespacePrefix = "Gcloud", + TargetNamespace = "ModularPipelines.Google", + OutputDirectory = "src/ModularPipelines.Google", + Commands = [command], + }; + var generated = (await new OptionsClassGenerator().GenerateAsync(tool)).Single().Content; + await Assert.That(generated).Contains($"string {operandName}"); + await Assert.That(generated).Contains("public string? Location"); + await Assert.That(generated).Contains("[property: CliArgument(0, Phase = CommandLinePhase.EarlyOperand, Required = true)]"); + var services = await new SubDomainClassGenerator().GenerateAsync(tool); + var service = string.Join(Environment.NewLine, services.Select(file => file.Content)); + await Assert.That(service).Contains($"{command.ClassName} options,"); + await Assert.That(service).DoesNotContain($"{command.ClassName}? options = null"); + } + + [Test] + public async Task Resource_Operands_Retain_Synopsis_Order_And_Optional_Repetition() + { + const string help = """ + NAME + gcloud example move - move resources + SYNOPSIS + gcloud example move SOURCE DESTINATION [EXTRA ...] [GCLOUD_WIDE_FLAG ...] + POSITIONAL ARGUMENTS + Source resource - The source to move. + SOURCE + The source identifier. + EXAMPLE + This uppercase example is documentation, not an operand. + --source-location=LOCATION + The source location or a fully qualified source name. + Destination resource - The destination to use. + DESTINATION + The destination identifier. + --destination-location=LOCATION + The destination location or a fully qualified destination name. + [EXTRA ...] + Additional optional identifiers. + GCLOUD WIDE FLAGS + --project=PROJECT + """; + var command = (await new TestScraper().Parse(["gcloud", "example", "move"], help))!; + + await Assert.That(string.Join(",", command.PositionalArguments.Select(argument => argument.PropertyName))) + .IsEqualTo("Source,Destination,Extra"); + await Assert.That(string.Join(",", command.PositionalArguments.Select(argument => argument.PositionIndex))) + .IsEqualTo("0,1,2"); + await Assert.That(command.PositionalArguments.Take(2).All(argument => argument.IsRequired)).IsTrue(); + var extra = command.PositionalArguments[2]; + await Assert.That(extra.IsRequired).IsFalse(); + await Assert.That(extra.IsVariadic).IsTrue(); + await Assert.That(extra.CSharpType).IsEqualTo("IEnumerable?"); + await Assert.That(command.Options.Select(option => option.SwitchName)) + .IsEquivalentTo(["--source-location", "--destination-location"]); + } + + [Test] + public async Task Optional_Resource_Can_Use_Configuration_Without_An_Operand() + { + const string help = """ + NAME + gcloud example describe - describe a resource + SYNOPSIS + gcloud example describe [RESOURCE : --location=LOCATION] [GCLOUD_WIDE_FLAG ...] + POSITIONAL ARGUMENTS + Resource resource - The resource to describe. + RESOURCE + The resource identifier; otherwise use the configured default. + --location=LOCATION + The location; otherwise use the configured default or a fully qualified name. + """; + var command = (await new TestScraper().Parse(["gcloud", "example", "describe"], help))!; + + await Assert.That(command.PositionalArguments.Single().IsRequired).IsFalse(); + await Assert.That(command.PositionalArguments.Single().CSharpType).IsEqualTo("string?"); + await Assert.That(command.Options.Single().IsRequired).IsFalse(); + await Assert.That(command.RequiredAlternativeGroups).IsEmpty(); + } + + [Test] + public async Task Shared_Synopsis_Preserves_Required_Option_Alternatives() + { + const string help = """ + SYNOPSIS + gcloud example export (--destination=DESTINATION | --stdout) [GCLOUD_WIDE_FLAG ...] + FLAGS + --destination=DESTINATION + Where to write the export. + --stdout + Write the export to standard output. + """; + var command = (await ScrapeFixture("example export", help)).Single(); + + await Assert.That(command.RequiredAlternativeGroups.Single().PropertyNames) + .IsEquivalentTo(["Destination", "Stdout"]); + } + + [Test] + public async Task Shared_Synopsis_Excludes_The_Gcloud_Wide_Flag_Placeholder() + { + const string help = """ + SYNOPSIS + gcloud example list [GCLOUD_WIDE_FLAG ...] + FLAGS + --filter=FILTER + Filter the returned resources. + """; + var command = (await ScrapeFixture("example list", help)).Single(); + + await Assert.That(command.HasOperandTakingUsage).IsFalse(); + await Assert.That(command.UsagePositionalArguments).IsEmpty(); + await Assert.That(command.Options.Single().PropertyName).IsEqualTo("Filter"); + } + + [Test] + public async Task Shared_Synopsis_Rejects_Commands_With_Unrepresented_Operands() + { + const string help = """ + SYNOPSIS + gcloud example show RESOURCE [GCLOUD_WIDE_FLAG ...] + FLAGS + --format=FORMAT + Format the resource details. + """; + + await Assert.That(await ScrapeFixture("example show", help)).IsEmpty(); + } + + private static async Task> ScrapeFixture(string path, string help) + { + var scraper = new GcloudCliScraper(new FixtureExecutor(path.Split(' '), help), + new HelpTextCache(NullLogger.Instance), NullLogger.Instance); + var commands = new List(); + await foreach (var command in scraper.ScrapeAsync()) + { + commands.Add(command); + } + return commands; + } + + private sealed class TestScraper() : GcloudCliScraper( + new UnusedExecutor(), + new HelpTextCache(NullLogger.Instance), + NullLogger.Instance) + { + public Task Parse(string[] path, string help) => + ParseCommandAsync(path, help, CancellationToken.None); + } + + private sealed class FixtureExecutor(string[] path, string help) : UnusedExecutor + { + public override Task ExecuteAsync(string command, string arguments, + CancellationToken cancellationToken = default, string? workingDirectory = null) + { + var prefix = arguments.Split(' ').TakeWhile(part => part != "--help").ToArray(); + if (!prefix.SequenceEqual(path.Take(prefix.Length))) + { + throw new InvalidOperationException($"Unexpected command: {arguments}"); + } + return Task.FromResult(new CliCommandResult + { + ExitCode = 0, + StandardOutput = prefix.Length == path.Length ? help : $"COMMANDS\n {path[prefix.Length]}\n", + StandardError = string.Empty, + }); + } + } + + private class UnusedExecutor : ICliCommandExecutor + { + public virtual Task ExecuteAsync(string command, string arguments, + CancellationToken cancellationToken = default, string? workingDirectory = null) => + throw new InvalidOperationException("Execution was not expected."); + + public Task IsAvailableAsync(string command, CancellationToken cancellationToken = default) => + Task.FromResult(true); + } +} diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/UsageSynopsisParserTests.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/UsageSynopsisParserTests.cs index 046f2f4548b..eac16006201 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/UsageSynopsisParserTests.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator.Tests/Scrapers/UsageSynopsisParserTests.cs @@ -9,6 +9,108 @@ namespace ModularPipelines.OptionsGenerator.Tests.Scrapers; public class UsageSynopsisParserTests { + [Test] + [Arguments("(RESOURCE|ALIAS : --location=LOCATION)")] + [Arguments("[RESOURCE | ALIAS : --location=LOCATION]")] + [Arguments("(SOURCE : DESTINATION | ALTERNATIVE)")] + [Arguments("(RESOURCE : --location=LOCATION | --global)")] + public async Task Rejects_Ambiguous_Alternatives_Across_Colon_Groups(string group) + { + await Assert.That(() => UsageSynopsisParser.Parse( + $"Usage: tool show {group}", ["tool", "show"])) + .Throws() + .And.HasMessageContaining("ambiguous alternatives in colon group"); + } + + [Test] + [Arguments("((RESOURCE|ALIAS) : --location=LOCATION)")] + [Arguments("(RESOURCE : --location=(REGION|ZONE))")] + public async Task Colon_Groups_Preserve_Explicitly_Nested_Alternatives(string group) + { + var result = UsageSynopsisParser.Parse($"Usage: tool show {group}", ["tool", "show"]); + + var operand = result.PositionalArguments.Single(); + await Assert.That(operand.PropertyName).IsEqualTo("Resource"); + await Assert.That(operand.IsRequired).IsTrue(); + } + + [Test] + [Arguments("(RESOURCE : --location=LOCATION)", true)] + [Arguments("[RESOURCE : --location=LOCATION]", false)] + [Arguments("(RESOURCE\n : --location=LOCATION --service=SERVICE)", true)] + public async Task Joins_Indented_Synopsis_Continuation_Lines(string resourceGroup, bool required) + { + var result = UsageSynopsisParser.Parse( + $"SYNOPSIS\n gcloud example show\n {resourceGroup}\n [--async]\nDESCRIPTION\n Show a resource.", + ["gcloud", "example", "show"], acceptedHeadings: ["SYNOPSIS"]); + + var operand = result.PositionalArguments.Single(); + await Assert.That(operand.PropertyName).IsEqualTo("Resource"); + await Assert.That(operand.IsRequired).IsEqualTo(required); + await Assert.That(result.Synopsis).Contains("[--async]").And.DoesNotContain("DESCRIPTION"); + } + + [Test] + public async Task Indented_Synopsis_Alternatives_Remain_Separate() + { + const string help = """ + SYNOPSIS + tool show + RESOURCE + tool show + --all + DESCRIPTION + Show one or all resources. + """; + var result = UsageSynopsisParser.Parse(help, ["tool", "show"], acceptedHeadings: ["SYNOPSIS"]); + + await Assert.That(result.MatchedSynopsisCount).IsEqualTo(2); + await Assert.That(result.PositionalArguments.Single().PropertyName).IsEqualTo("Resource"); + await Assert.That(result.PositionalArguments.Single().IsRequired).IsFalse(); + } + + [Test] + [Arguments("(RESOURCE : --location=LOCATION)", true, false)] + [Arguments("[RESOURCE : --location=LOCATION]", false, false)] + [Arguments("(RESOURCE [RESOURCE ...] : --location=LOCATION)", true, true)] + [Arguments("[RESOURCE ... : --location=LOCATION]", false, true)] + [Arguments("(RESOURCE : --location=LOCATION --service=SERVICE)", true, false)] + [Arguments("(RESOURCE : [--location=LOCATION])", true, false)] + public async Task Resource_Groups_Keep_Only_Their_Positional_Inputs( + string resourceGroup, bool required, bool variadic) + { + var result = UsageSynopsisParser.Parse( + $"SYNOPSIS\n gcloud example show {resourceGroup}", + ["gcloud", "example", "show"], acceptedHeadings: ["SYNOPSIS"]); + var operand = result.PositionalArguments.Single(); + + await Assert.That(operand.PropertyName).IsEqualTo("Resource"); + await Assert.That(operand.IsRequired).IsEqualTo(required); + await Assert.That(operand.IsVariadic).IsEqualTo(variadic); + await Assert.That(operand.AssociatedOptionSwitch).IsNull(); + } + + [Test] + [Arguments("(SOURCE : DESTINATION)", true, true, CommandLinePhase.EarlyOperand, 1)] + [Arguments("[SOURCE : DESTINATION]", false, false, CommandLinePhase.EarlyOperand, 1)] + [Arguments("(SOURCE : [DESTINATION])", true, false, CommandLinePhase.EarlyOperand, 1)] + [Arguments("(SOURCE : --location=LOCATION DESTINATION)", true, true, CommandLinePhase.Passthrough, 0)] + public async Task Colon_Groups_Preserve_Operands_After_The_Separator( + string group, bool sourceRequired, bool destinationRequired, + CommandLinePhase destinationPhase, int destinationIndex) + { + var result = UsageSynopsisParser.Parse($"Usage: tool copy {group}", ["tool", "copy"]); + + await Assert.That(result.PositionalArguments.Select(argument => argument.PropertyName)) + .IsEquivalentTo(["Source", "Destination"]); + await Assert.That(result.PositionalArguments[0].PositionIndex).IsEqualTo(0); + await Assert.That(result.PositionalArguments[1].PositionIndex).IsEqualTo(destinationIndex); + await Assert.That(result.PositionalArguments[1].Phase).IsEqualTo(destinationPhase); + await Assert.That(result.PositionalArguments[0].IsRequired).IsEqualTo(sourceRequired); + await Assert.That(result.PositionalArguments[1].IsRequired).IsEqualTo(destinationRequired); + await Assert.That(result.PositionalArguments[1].AssociatedOptionSwitch).IsNull(); + } + [Test] public async Task Ignores_Azure_Usage_Examples() { @@ -1477,7 +1579,7 @@ public TestNewmanCliScraper() { } - public IReadOnlyList Extract(string helpText) => ExtractSubcommands(helpText).ToList(); + public IReadOnlyList Extract(string helpText) => [.. ExtractSubcommands(helpText)]; } private sealed class CountingUsageScraper : CliScraperBase diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Models/CliArgumentGroup.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Models/CliArgumentGroup.cs index e13175580da..cadf5bef9f6 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Models/CliArgumentGroup.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Models/CliArgumentGroup.cs @@ -62,10 +62,15 @@ private IEnumerable FlattenArguments(IReadOnlyList - /// Normalized switch name, including its leading dashes. + /// Normalized argument name, including leading dashes for a named option. /// public required string SwitchName { get; init; } + /// + /// Whether this declaration is a positional operand rather than a named option. + /// + public bool IsPositional { get; init; } + /// /// Value placeholder shown by help output. /// diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/PublicAPI.Unshipped.txt b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/PublicAPI.Unshipped.txt index 2033d8c6b1b..93cad8308cd 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/PublicAPI.Unshipped.txt +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/PublicAPI.Unshipped.txt @@ -1,8 +1,13 @@ #nullable enable *REMOVED*override ModularPipelines.OptionsGenerator.Scrapers.Cli.AwsCliScraper.MaxParallelism.get -> int override ModularPipelines.OptionsGenerator.Scrapers.Cli.GhCliScraper.CreateToolDefinition() -> ModularPipelines.OptionsGenerator.Models.CliToolDefinition! +override ModularPipelines.OptionsGenerator.Scrapers.Cli.GcloudCliScraper.UsageSynopsisHeadings.get -> System.Collections.Generic.IReadOnlyList! +override ModularPipelines.OptionsGenerator.Scrapers.Cli.GcloudCliScraper.ParseCommandAsync(string![]! commandPath, string! helpText, ModularPipelines.OptionsGenerator.Scrapers.Cli.UsageSynopsisParseResult! usage, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override ModularPipelines.OptionsGenerator.Scrapers.Cli.GcloudCliScraper.NormalizeUsageSynopsis(ModularPipelines.OptionsGenerator.Models.CliCommandDefinition! command, ModularPipelines.OptionsGenerator.Scrapers.Cli.UsageSynopsisParseResult! usage) -> ModularPipelines.OptionsGenerator.Scrapers.Cli.UsageSynopsisParseResult! ModularPipelines.OptionsGenerator.TypeDetection.CliCommandResult.HasProcessExitCode.get -> bool ModularPipelines.OptionsGenerator.TypeDetection.CliCommandResult.HasProcessExitCode.init -> void +ModularPipelines.OptionsGenerator.Models.CliArgumentDefinition.IsPositional.get -> bool +ModularPipelines.OptionsGenerator.Models.CliArgumentDefinition.IsPositional.init -> void ModularPipelines.OptionsGenerator.Models.CliCommandCoveragePolicy.ConditionallyAvailableCommands.get -> System.Collections.Generic.IReadOnlyList! ModularPipelines.OptionsGenerator.Models.CliCommandCoveragePolicy.ConditionallyAvailableCommands.init -> void ModularPipelines.OptionsGenerator.Models.CliConditionallyAvailableCommand diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/CliScraperBase.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/CliScraperBase.cs index 540a5897fc7..14959da3391 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/CliScraperBase.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/CliScraperBase.cs @@ -1815,18 +1815,23 @@ private static void ValidateArgumentGroups(CliCommandDefinition command) var emittedSwitches = command.Options .Select(option => option.SwitchName) .ToHashSet(StringComparer.OrdinalIgnoreCase); - var missingSwitches = command.ArgumentGroups + var emittedOperands = command.PositionalArguments + .Select(argument => argument.PropertyName) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var missingArguments = command.ArgumentGroups .SelectMany(group => group.FlattenArguments()) + .Where(argument => argument.IsPositional + ? !emittedOperands.Contains(NormalizePropertyName(argument.SwitchName) ?? argument.SwitchName) + : !emittedSwitches.Contains(argument.SwitchName)) .Select(argument => argument.SwitchName) - .Where(switchName => !emittedSwitches.Contains(switchName)) .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); - if (missingSwitches.Length != 0) + if (missingArguments.Length != 0) { throw new InvalidOperationException( $"{command.FullCommand} declares grouped arguments that were swallowed or omitted: " - + string.Join(", ", missingSwitches)); + + string.Join(", ", missingArguments)); } } diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/GcloudCliScraper.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/GcloudCliScraper.cs index 1e243ea19d4..9558b046064 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/GcloudCliScraper.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/GcloudCliScraper.cs @@ -13,6 +13,8 @@ namespace ModularPipelines.OptionsGenerator.Scrapers.Cli; /// public partial class GcloudCliScraper : CliScraperBase { + private static readonly string[] GcloudUsageSynopsisHeadings = ["SYNOPSIS"]; + private static readonly string[] StructuredExampleEndMarkers = ["JSON Example:", "YAML Example:", "File Example:"]; @@ -32,6 +34,8 @@ public partial class GcloudCliScraper : CliScraperBase /// protected override string ExecutablePath { get; } + protected override IReadOnlyList UsageSynopsisHeadings => GcloudUsageSynopsisHeadings; + #endregion public GcloudCliScraper(ICliCommandExecutor executor, IHelpTextCache helpCache, ILogger logger) @@ -95,6 +99,13 @@ protected override IEnumerable ExtractSubcommands(string helpText) protected override Task ParseCommandAsync( string[] commandPath, string helpText, + CancellationToken cancellationToken) => + ParseCommandAsync(commandPath, helpText, ParseUsageSynopsis(commandPath, helpText), cancellationToken); + + protected override Task ParseCommandAsync( + string[] commandPath, + string helpText, + UsageSynopsisParseResult usage, CancellationToken cancellationToken) { var commandParts = commandPath.Skip(1).ToArray(); @@ -108,7 +119,7 @@ protected override IEnumerable ExtractSubcommands(string helpText) var description = ExtractDescription(helpText); var parsedOptions = ParseOptions(helpText, commandParts); var options = parsedOptions.Options; - var positionalArgs = ParsePositionalArguments(helpText); + var positionalArgs = ParsePositionalArguments(usage, commandPath, parsedOptions.ArgumentGroups); var enums = options .Where(o => o.EnumDefinition is not null) @@ -129,6 +140,7 @@ protected override IEnumerable ExtractSubcommands(string helpText) Options = options, ArgumentGroups = parsedOptions.ArgumentGroups, PositionalArguments = positionalArgs, + UsageSynopsis = usage.Synopsis, SubDomainGroup = subDomain, Enums = enums }; @@ -140,12 +152,28 @@ protected override IEnumerable ExtractSubcommands(string helpText) #region Virtual Method Overrides + protected override UsageSynopsisParseResult NormalizeUsageSynopsis( + CliCommandDefinition command, UsageSynopsisParseResult usage) + { + // This synopsis placeholder denotes inherited flags, not a positional operand. + var operands = usage.PositionalArguments + .Where(argument => argument.PropertyName != "GcloudWideFlag") + .ToList(); + return usage with + { + PositionalArguments = operands, + HasOperandTokens = operands.Count > 0 || usage.UnparsedOperandTokens.Count > 0, + }; + } + /// /// gcloud uses "FLAGS" section instead of "Flags:" or "Options:". /// protected override bool HasOptions(string helpText) { - return helpText.Contains("\nFLAGS\n") || + return helpText.Contains("\nPOSITIONAL ARGUMENTS\n") || + helpText.Contains("\nPOSITIONAL ARGUMENTS\r\n") || + helpText.Contains("\nFLAGS\n") || helpText.Contains("\nFLAGS\r\n") || base.HasOptions(helpText); } @@ -174,30 +202,17 @@ private static List ExtractFromSection(string helpText, string sectionNa { var subcommands = new List(); - // Find the section - var sectionMatch = Regex.Match(helpText, $@"^{sectionName}\s*$", RegexOptions.Multiline); - if (!sectionMatch.Success) - { - return subcommands; - } - - var sectionStart = sectionMatch.Index + sectionMatch.Length; - - // Find where section ends (next uppercase section header) - var nextMatch = SectionHeaderPattern().Match(helpText[sectionStart..]); - var sectionEnd = nextMatch.Success ? sectionStart + nextMatch.Index : helpText.Length; - - var section = helpText[sectionStart..sectionEnd]; - - // gcloud format: command names are indented with 5+ spaces at line start - // Example: " compute" - var matches = SubcommandPattern().Matches(section); - foreach (Match match in matches) + foreach (var (_, content) in ExtractSections(helpText, sectionName)) { - var name = match.Groups[1].Value.Trim(); - if (!string.IsNullOrEmpty(name)) + // gcloud command names are indented with five spaces at line start. + var matches = SubcommandPattern().Matches(content); + foreach (Match match in matches) { - subcommands.Add(name); + var name = match.Groups[1].Value.Trim(); + if (!string.IsNullOrEmpty(name)) + { + subcommands.Add(name); + } } } @@ -222,38 +237,50 @@ private static List ExtractFromSection(string helpText, string sectionNa var options = new List(); var seenOptions = new HashSet(StringComparer.OrdinalIgnoreCase); - // Find FLAGS section - var flagsMatch = FlagsSectionPattern().Match(helpText); - if (!flagsMatch.Success) + var argumentGroups = new List(); + foreach (var (name, content) in ExtractSections(helpText, "FLAGS", "POSITIONAL ARGUMENTS")) { - return (options, []); - } - - var sectionStart = flagsMatch.Index + flagsMatch.Length; + var group = ParseArgumentGroups(content, + name == "POSITIONAL ARGUMENTS" ? ParseGcloudResourceArgument : ParseGcloudArgument); + argumentGroups.Add(group); + foreach (var argument in group.FlattenArguments().Where(argument => !argument.IsPositional)) + { + foreach (var option in CreateOptions(argument, commandParts)) + { + if (!seenOptions.Add(option.SwitchName)) + { + continue; + } - // Find end of FLAGS section - var nextSectionMatch = SectionHeaderPattern().Match(helpText[sectionStart..]); - var sectionEnd = nextSectionMatch.Success ? sectionStart + nextSectionMatch.Index : helpText.Length; + options.Add(NormalizeRepeatability(option, helpText, commandParts)); + } + } + } - var flagsSection = helpText[sectionStart..sectionEnd]; - var argumentGroup = ParseArgumentGroups(flagsSection, ParseGcloudArgument); + return (options, argumentGroups); + } - foreach (var argument in argumentGroup.FlattenArguments()) + private static IEnumerable<(string Name, string Content)> ExtractSections(string helpText, params string[] sectionNames) + { + var headings = SectionHeaderPattern().Matches(helpText); + for (var index = 0; index < headings.Count; index++) { - foreach (var option in CreateOptions(argument, commandParts)) + var heading = headings[index]; + var name = heading.Value.Trim(); + if (!sectionNames.Contains(name, StringComparer.Ordinal)) { - if (!seenOptions.Add(option.SwitchName)) - { - continue; - } - - options.Add(NormalizeRepeatability(option, helpText, commandParts)); + continue; } - } - return (options, [argumentGroup]); + var start = heading.Index + heading.Length; + var end = index + 1 < headings.Count ? headings[index + 1].Index : helpText.Length; + yield return (name, helpText[start..end]); + } } + [GeneratedRegex(@"^[A-Z][A-Z_ ]*[ \t]*\r?$", RegexOptions.Multiline)] + private static partial Regex SectionHeaderPattern(); + private CliOptionDefinition NormalizeRepeatability( CliOptionDefinition option, string helpText, @@ -428,44 +455,63 @@ private static bool DescriptionMentionsSwitch(string? description, string switch }; } - private static List ParsePositionalArguments(string helpText) + private static CliArgumentDefinition? ParseGcloudResourceArgument(string line) { - var args = new List(); - - var sectionMatch = PositionalSectionPattern().Match(helpText); - if (!sectionMatch.Success) + if (ParseGcloudArgument(line) is { } option) { - return args; + return option; } - var sectionStart = sectionMatch.Index + sectionMatch.Length; - var nextMatch = SectionHeaderPattern().Match(helpText[sectionStart..]); - var sectionEnd = nextMatch.Success ? sectionStart + nextMatch.Index : helpText.Length; - - var section = helpText[sectionStart..sectionEnd]; + var match = ResourceOperandPattern().Match(line); + if (!match.Success) + { + return null; + } - // Match: " ARG_NAME [ARG_NAME ...]" - var argMatch = PositionalArgumentPattern().Match(section); - if (argMatch.Success) + return new CliArgumentDefinition { - var argName = argMatch.Groups[1].Value; - var isMultiple = argMatch.Value.Contains("..."); + SwitchName = match.Groups["name"].Value, + IsPositional = true, + ValueHint = line.Trim(), + Indentation = GetIndentation(line), + }; + } + + [GeneratedRegex(@"^[ \t]+\[?(?[A-Z][A-Z0-9_]*)(?:[ \t]+\[?\k)?(?:[ \t]*\.\.\.)?\]*[ \t]*$")] + private static partial Regex ResourceOperandPattern(); - var propertyName = NormalizePropertyName(argName); - if (propertyName is not null) + private static IReadOnlyList ParsePositionalArguments( + UsageSynopsisParseResult usage, string[] commandPath, IReadOnlyList groups) + { + var usageArguments = GetPositionalArguments(usage); + var arguments = groups.SelectMany(group => group.FlattenArguments()) + .Where(argument => argument.IsPositional) + .Select((argument, index) => { - args.Add(new CliPositionalArgument + var propertyName = NormalizePropertyName(argument.SwitchName)!; + var synopsisArgument = usageArguments.FirstOrDefault(candidate => + candidate.PropertyName.Equals(propertyName, StringComparison.OrdinalIgnoreCase)); + if (synopsisArgument is null && usage.HasExtractedSynopses) { - PropertyName = propertyName, - CSharpType = isMultiple ? "IEnumerable" : "string", - IsRequired = true, - PositionIndex = 0, - Description = null - }); - } - } + throw new InvalidOperationException( + $"{string.Join(" ", commandPath)} synopsis omits declared positional operand '{argument.SwitchName}'; " + + "its order and requiredness cannot be inferred safely."); + } - return args; + var variadic = argument.ValueHint?.Contains("...", StringComparison.Ordinal) == true; + var required = argument.ValueHint?.StartsWith('[') != true; + return (synopsisArgument ?? new CliPositionalArgument + { + PropertyName = propertyName, + CSharpType = (variadic ? "IEnumerable" : "string") + (required ? "" : "?"), + PositionIndex = index, + IsRequired = required, + IsVariadic = variadic, + }) with + { Description = argument.Documentation }; + }) + .OrderBy(argument => argument.PositionIndex); + return CliPositionalArgument.MergeDuplicates(arguments); } private static bool IsNumericHint(string hint) @@ -643,25 +689,12 @@ private static string DetermineCSharpType( @"\bmust be one of:\s*[A-Za-z][A-Za-z0-9_-]*\b", RegexOptions.IgnoreCase)] private static partial Regex TextualCategoriesPattern(); - - [GeneratedRegex(@"^[A-Z][A-Z_\s]+$", RegexOptions.Multiline)] - private static partial Regex SectionHeaderPattern(); - [GeneratedRegex(@"^\s{5}(\w[\w-]*)\s*$", RegexOptions.Multiline)] private static partial Regex SubcommandPattern(); [GeneratedRegex(@"^NAME\s*\n\s+gcloud[^\n]+-\s*(.+?)(?=\n\n|\nSYNOPSIS)", RegexOptions.Singleline)] private static partial Regex CommandDescriptionPattern(); - [GeneratedRegex(@"^FLAGS\s*$", RegexOptions.Multiline)] - private static partial Regex FlagsSectionPattern(); - - [GeneratedRegex(@"^POSITIONAL ARGUMENTS\s*$", RegexOptions.Multiline)] - private static partial Regex PositionalSectionPattern(); - - [GeneratedRegex(@"^\s{5}([A-Z][A-Z_]+)(?:\s+\[[A-Z][A-Z_]+\s*\.\.\.\])?", RegexOptions.Multiline)] - private static partial Regex PositionalArgumentPattern(); - [GeneratedRegex(@"must be (?:one of:?\s*)([a-zA-Z][a-zA-Z0-9_-]*(?:,\s*[a-zA-Z][a-zA-Z0-9_-]*)+)", RegexOptions.IgnoreCase)] private static partial Regex RequiredEnumValuesPattern(); diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs index d9d2f8d329c..c234770b2f3 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs @@ -11,8 +11,9 @@ public static class UsageSynopsisParser { private static readonly string[] DefaultUsageHeadings = ["usage"]; - private static readonly HashSet ControlTokens = new(StringComparer.OrdinalIgnoreCase) - { + private static readonly HashSet ControlTokens = + [ + with(StringComparer.OrdinalIgnoreCase), "arg", "args", "argument", @@ -28,25 +29,27 @@ public static class UsageSynopsisParser "options", "subcommand", "subcommands", - }; + ]; - private static readonly HashSet OptionControlTokens = new(StringComparer.OrdinalIgnoreCase) - { + private static readonly HashSet OptionControlTokens = + [ + with(StringComparer.OrdinalIgnoreCase), "flag", "flags", "global option", "global options", "option", "options", - }; + ]; - private static readonly HashSet CommandGroupPlaceholderNames = new(StringComparer.OrdinalIgnoreCase) - { + private static readonly HashSet CommandGroupPlaceholderNames = + [ + with(StringComparer.OrdinalIgnoreCase), "Command", "Commands", "Subcommand", "Subcommands", - }; + ]; /// /// Parses the best matching synopsis for a command. @@ -213,7 +216,7 @@ private static UsageSynopsisParseResult ParseSynopsis( }; } - private static IReadOnlyList ParseInlineRequiredAlternativeGroups( + private static List ParseInlineRequiredAlternativeGroups( IEnumerable operandTokens, CommandLinePhase phase) { @@ -272,7 +275,7 @@ private static IReadOnlySet GetRequiredAlternativeMemberKeys( .ToHashSet(StringComparer.Ordinal); private static IReadOnlyList GetCrossSynopsisRequiredAlternativeGroups( - IReadOnlyList candidates, + List candidates, IReadOnlyList selectedArguments) { if (candidates.Count <= 1 @@ -363,9 +366,7 @@ private static IReadOnlyList GetRequiredAlternat private static IReadOnlyList DistinctAlternativeMembers( IEnumerable members) => - members - .DistinctBy(GetAlternativeMemberKey, StringComparer.Ordinal) - .ToArray(); + [.. members.DistinctBy(GetAlternativeMemberKey, StringComparer.Ordinal)]; private static IReadOnlyList CollapseOptionAliases( IReadOnlyList members, @@ -450,7 +451,7 @@ private static IReadOnlyList SplitTopLevelAlternatives(string content) } alternatives.Add(content[start..].Trim()); - return alternatives.Where(static alternative => alternative.Length > 0).ToArray(); + return [.. alternatives.Where(static alternative => alternative.Length > 0)]; } private static ParsedOperands ParseOperandTokens( @@ -541,9 +542,7 @@ private static IReadOnlyList PreserveOptionTerminatorOnNe return arguments; } - return arguments - .Select(static argument => argument with { PrependOptionTerminator = true }) - .ToArray(); + return [.. arguments.Select(static argument => argument with { PrependOptionTerminator = true })]; } private static void ClearAssociatedOptionSwitch( @@ -677,7 +676,7 @@ private static IReadOnlyList RelaxArgumentsMissingFromAlt return selectedArguments; } - return selectedArguments + return [.. selectedArguments .Select(argument => alternatives.All(alternative => IsRequiredInAlternative( argument, @@ -688,8 +687,7 @@ private static IReadOnlyList RelaxArgumentsMissingFromAlt { CSharpType = $"{argument.CSharpType.TrimEnd('?')}?", IsRequired = false, - }) - .ToList(); + })]; } private static bool IsRequiredInAlternative( @@ -713,7 +711,7 @@ private static bool IsRequiredInAlternative( && candidate.Phase == selectedArgument.Phase); } - private static IReadOnlyList ExtractSynopses( + private static List ExtractSynopses( string helpText, IReadOnlyList acceptedHeadings) { @@ -848,6 +846,7 @@ private static bool TryReadSynopsisHeading( private static int ReadIndentedSynopses(string[] lines, int startIndex, List synopses) { var initialSynopsisCount = synopses.Count; + var synopsisIndentation = int.MaxValue; var index = startIndex; for (; index < lines.Length; index++) { @@ -863,12 +862,20 @@ private static int ReadIndentedSynopses(string[] lines, int startIndex, List synopsisIndentation) + { + synopses[^1] += " " + trimmed; + continue; + } + if (LooksLikeSectionHeading(trimmed)) { break; } synopses.Add(trimmed); + synopsisIndentation = indentation; } return index; @@ -880,7 +887,7 @@ private static bool LooksLikeSectionHeading(string line) => && line.Any(char.IsLetter) && line.Where(char.IsLetter).All(char.IsUpper)); - private static IReadOnlyList Tokenize(string synopsis) + private static List Tokenize(string synopsis) { var tokens = new List(); var index = 0; @@ -926,7 +933,7 @@ private static IReadOnlyList Tokenize(string synopsis) } private static CommandMatch? FindCommand( - IReadOnlyList tokens, + List tokens, IReadOnlyList commandPath) { for (var pathStart = 0; pathStart < commandPath.Count; pathStart++) @@ -1048,12 +1055,24 @@ private static bool TryParseNestedOperandGroup( } var content = TrimWrapper(normalizedToken).Trim(); + var nestedTokens = Tokenize(content); + if (nestedTokens.Contains(":") && SplitTopLevelAlternatives(content).Count > 1) + { + throw new InvalidOperationException( + $"Usage synopsis has ambiguous alternatives in colon group '{normalizedToken}'."); + } + + if (TryParseColonSeparatedOperands( + nestedTokens, IsRequiredUsageToken(normalizedToken), positionIndex, phase, out arguments)) + { + return true; + } + if (!content.Contains('[') || content.Contains('|')) { return false; } - var nestedTokens = Tokenize(content); if (nestedTokens.Count <= 1) { return false; @@ -1101,6 +1120,38 @@ private static bool TryParseNestedOperandGroup( return true; } + private static bool TryParseColonSeparatedOperands( + List tokens, + bool groupRequired, + int positionIndex, + CommandLinePhase phase, + out IReadOnlyList arguments) + { + arguments = []; + var separator = tokens.IndexOf(":"); + if (separator <= 0) + { + return false; + } + + // A selector-only suffix can come from a fully qualified resource or configuration. + // Other colon groups retain operands on both sides of the separator. + var hasOnlySelectors = separator < tokens.Count - 1 + && tokens.Skip(separator + 1).All(static token => + IsOptionControlToken(token) && GetOptionSwitches(token).Count > 0); + var operandTokens = hasOnlySelectors + ? tokens.Take(separator) + : tokens.Where(static token => token != ":"); + var groupArguments = ParseOperandTokens(operandTokens, phase).Arguments; + arguments = [.. groupArguments.Select((argument, index) => argument with + { + PositionIndex = positionIndex + index, + IsRequired = argument.IsRequired && groupRequired, + CSharpType = GetCSharpType(argument.IsRequired && groupRequired, argument.IsVariadic), + })]; + return arguments.Count > 0; + } + private static string TrimTrailingOperandPunctuation(string token) => token.Trim().TrimEnd(',', ';', ':'); @@ -1190,7 +1241,7 @@ private static string SelectCanonicalAlternative(string content) .Replace("\\", " Or ", StringComparison.Ordinal); } - private static IReadOnlyList CollapseAlternatives(IEnumerable sourceTokens) + private static List CollapseAlternatives(IEnumerable sourceTokens) { var tokens = sourceTokens.ToList(); var collapsed = new List(); @@ -1209,7 +1260,7 @@ private static IReadOnlyList CollapseAlternatives(IEnumerable so return collapsed; } - private static IReadOnlyList TrimTrailingUsageExplanation(IEnumerable sourceTokens) + private static List TrimTrailingUsageExplanation(IEnumerable sourceTokens) { var tokens = sourceTokens.ToList(); var punctuationBoundaryIndex = tokens.FindIndex(static token => @@ -1234,14 +1285,12 @@ private static IReadOnlyList TrimTrailingUsageExplanation(IEnumerable char.IsLetterOrDigit(character) ? character : ' ') - .ToArray()); + var cleaned = new string([.. content.Select(character => char.IsLetterOrDigit(character) ? character : ' ')]); var words = cleaned.Split(' ', StringSplitOptions.RemoveEmptyEntries); return words.Length == 0 ? null @@ -1373,10 +1422,9 @@ private static IReadOnlyList GetOptionSwitches(string token) return []; } - return alternatives + return [.. alternatives .SelectMany(GetOptionSwitchesFromAlternative) - .Distinct(StringComparer.Ordinal) - .ToArray(); + .Distinct(StringComparer.Ordinal)]; } private static IReadOnlyList GetOptionSwitchesFromAlternative(string alternative) From 8ea531d386ebf6707c1223a4260a0f376f49640a Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:37:48 +0100 Subject: [PATCH 2/2] refactor(generator): separate optional nested operand parsing (#5112) --- .../Scrapers/Cli/UsageSynopsisParser.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs index c234770b2f3..500141a409b 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/Scrapers/Cli/UsageSynopsisParser.cs @@ -1078,6 +1078,16 @@ private static bool TryParseNestedOperandGroup( return false; } + return TryParseOptionalNestedOperands(nestedTokens, positionIndex, phase, out arguments); + } + + private static bool TryParseOptionalNestedOperands( + List nestedTokens, + int positionIndex, + CommandLinePhase phase, + out IReadOnlyList arguments) + { + arguments = []; var parsedArguments = new List(); string? associatedOptionSwitch = null; foreach (var nestedToken in nestedTokens)