diff --git a/benchmarks/serde-benchmarks/build.gradle.kts b/benchmarks/serde-benchmarks/build.gradle.kts index 3c378ff7d..e3e46fcca 100644 --- a/benchmarks/serde-benchmarks/build.gradle.kts +++ b/benchmarks/serde-benchmarks/build.gradle.kts @@ -62,6 +62,11 @@ dependencies { // Protocol test document for converting test case params into typed shapes. jmh(project(":protocol-test-harness")) + + // Dynamic-client path: build the ApiOperation + input from the runtime + // model instead of codegen, selected via -Dsmithy-java.benchmark.client=dynamic. + jmh(project(":client:dynamic-client")) + jmh(project(":dynamic-schemas")) } // Smithy benchmark model files (tagged @httpRequestTests / @httpResponseTests diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0DeserializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0DeserializeBenchmark.java index e9155bb3a..4ea094413 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0DeserializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0DeserializeBenchmark.java @@ -51,7 +51,8 @@ public class AwsJson1_0DeserializeBenchmark { @Setup public void setup() { protocol = new AwsJson1Protocol(SERVICE_ID); - state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_JSON_BODY, CONTENT_TYPE, false); + state = DeserializeState + .forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_JSON_BODY, CONTENT_TYPE, false); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0SerializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0SerializeBenchmark.java index 00f96bb87..275458e99 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0SerializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0SerializeBenchmark.java @@ -55,7 +55,7 @@ public class AwsJson1_0SerializeBenchmark { @Setup public void setup() { protocol = new AwsJson1Protocol(SERVICE_ID); - state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE); + state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0VirtualThreadDeserializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0VirtualThreadDeserializeBenchmark.java new file mode 100644 index 000000000..c0f51eaee --- /dev/null +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0VirtualThreadDeserializeBenchmark.java @@ -0,0 +1,68 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.benchmarks.serde; + +import java.nio.charset.StandardCharsets; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; +import software.amazon.smithy.java.aws.client.awsjson.AwsJson1Protocol; +import software.amazon.smithy.java.core.schema.ApiOperation; +import software.amazon.smithy.java.core.schema.SerializableStruct; +import software.amazon.smithy.model.shapes.ShapeId; + +/** + * AWS JSON 1.0 deserialization measured on virtual carrier threads. + * + *

Companion to {@link AwsJson1_0DeserializeBenchmark} (platform-threaded). The benchmark + * body is a single {@code deserializeResponse} call; {@code @Fork(jvmArgsAppend = + * "-Djmh.executor=VIRTUAL")} runs JMH's measurement threads as virtual threads. This + * isolates how the shared serializer / string-cache pools behave under virtual-thread + * carriers, keeping virtual-thread creation cost out of the per-op measurement. + * + *

Run with {@code -prof gc} to compare per-op allocation against the platform-threaded run. + */ +@State(Scope.Benchmark) +@Fork(jvmArgsAppend = "-Djmh.executor=VIRTUAL") +public class AwsJson1_0VirtualThreadDeserializeBenchmark { + + private static final String GENERATED_PACKAGE = + "software.amazon.smithy.java.benchmarks.serde.generated.awsjson10.model"; + private static final ShapeId SERVICE_ID = + ShapeId.from("com.amazonaws.sdk.benchmark#AwsJsonRpc10DataPlane"); + private static final byte[] EMPTY_JSON_BODY = "{}".getBytes(StandardCharsets.UTF_8); + private static final String CONTENT_TYPE = "application/x-amz-json-1.0"; + + @Param({ + "awsJson1_0_GetItemOutput_S", + "awsJson1_0_GetItemOutput_M", + "awsJson1_0_GetItemOutput_L", + }) + public String testCaseId; + + private AwsJson1Protocol protocol; + private DeserializeState state; + + @Setup + public void setup() { + protocol = new AwsJson1Protocol(SERVICE_ID); + state = DeserializeState + .forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_JSON_BODY, CONTENT_TYPE, false); + } + + @Benchmark + public void deserialize(Blackhole bh) { + @SuppressWarnings({"unchecked", "rawtypes"}) + ApiOperation op = + (ApiOperation) state.operation; + bh.consume( + protocol.deserializeResponse(op, state.context, state.typeRegistry, state.request, state.response)); + } +} diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQueryDeserializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQueryDeserializeBenchmark.java index fee6ae843..a3e25cc5b 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQueryDeserializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQueryDeserializeBenchmark.java @@ -53,7 +53,8 @@ public class AwsQueryDeserializeBenchmark { @Setup public void setup() { protocol = new AwsQueryClientProtocol(SERVICE_ID, VERSION); - state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_XML_BODY, CONTENT_TYPE, false); + state = DeserializeState + .forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_XML_BODY, CONTENT_TYPE, false); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQuerySerializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQuerySerializeBenchmark.java index 77960230b..4cde93d5e 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQuerySerializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQuerySerializeBenchmark.java @@ -47,7 +47,7 @@ public class AwsQuerySerializeBenchmark { @Setup public void setup() { protocol = new AwsQueryClientProtocol(SERVICE_ID, VERSION); - state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE); + state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkOperations.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkOperations.java new file mode 100644 index 000000000..4bb612f8d --- /dev/null +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkOperations.java @@ -0,0 +1,87 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.benchmarks.serde; + +import software.amazon.smithy.java.core.schema.ApiOperation; +import software.amazon.smithy.java.core.schema.SerializableStruct; +import software.amazon.smithy.java.core.serde.TypeRegistry; +import software.amazon.smithy.java.dynamicclient.DynamicOperation; +import software.amazon.smithy.java.dynamicschemas.SchemaConverter; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; + +/** + * Resolves the {@link ApiOperation} a serde benchmark drives, in one of two + * modes selected by the {@code smithy-java.benchmark.client} system property: + * + *

+ * + *

Both modes return an {@code ApiOperation} whose {@code inputBuilder()} / + * {@code outputBuilder()} accept the same schema-guided document deserialization + * the benchmarks already use, so the only difference is operation construction — + * the {@code protocol.createRequest} / {@code deserializeResponse} calls in the + * benchmark loop are identical. + */ +final class BenchmarkOperations { + + /** System property selecting codegen (default) vs dynamic operation build. */ + static final String CLIENT_MODE_PROPERTY = "smithy-java.benchmark.client"; + + private BenchmarkOperations() {} + + static boolean isDynamic() { + return "dynamic".equalsIgnoreCase(System.getProperty(CLIENT_MODE_PROPERTY, "codegen")); + } + + /** A short label for the active mode, for result metadata / logging. */ + static String modeLabel() { + return isDynamic() ? "dynamic" : "codegen"; + } + + /** + * Resolve the operation for a benchmark test case. + * + * @param opShape the operation shape from the benchmark model + * @param generatedPackage codegen package for the protocol projection + * @param serviceId the service the operation is bound to (needed by + * the dynamic path to convert the service schema) + */ + static ApiOperation resolve( + OperationShape opShape, + String generatedPackage, + ShapeId serviceId + ) { + if (isDynamic()) { + return dynamicOperation(opShape, serviceId); + } + return SerializeState.resolveOperation(generatedPackage, opShape.getId().getName()); + } + + private static ApiOperation dynamicOperation( + OperationShape opShape, + ShapeId serviceId + ) { + var model = BenchmarkTestCases.model(); + ServiceShape service = model.expectShape(serviceId, ServiceShape.class); + var converter = new SchemaConverter(model); + return DynamicOperation.create( + opShape, + converter, + model, + service, + TypeRegistry.empty(), + (id, builder) -> {}); + } +} diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkTestCases.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkTestCases.java index e10e4d2e9..c41bbfebe 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkTestCases.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkTestCases.java @@ -38,6 +38,11 @@ final class BenchmarkTestCases { private BenchmarkTestCases() {} + /** The assembled benchmark model, shared by the dynamic-client path. */ + static Model model() { + return MODEL; + } + static RequestEntry request(String testCaseId) { RequestEntry entry = REQUESTS.get(testCaseId); if (entry == null) { diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/DeserializeState.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/DeserializeState.java index 42ffe3ce2..094335c04 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/DeserializeState.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/DeserializeState.java @@ -21,6 +21,7 @@ import software.amazon.smithy.java.io.datastream.DataStream; import software.amazon.smithy.java.io.uri.SmithyUri; import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ShapeId; /** * Reusable per-trial state for a deserialization benchmark. @@ -84,6 +85,7 @@ private DeserializeState( static DeserializeState forTestCase( String testCaseId, String generatedPackage, + ShapeId serviceId, byte[] emptyBody, String contentType, boolean base64DecodeBody @@ -92,7 +94,7 @@ static DeserializeState forTestCase( OperationShape opShape = entry.operation(); ApiOperation operation = - SerializeState.resolveOperation(generatedPackage, opShape.getId().getName()); + BenchmarkOperations.resolve(opShape, generatedPackage, serviceId); byte[] resolvedEmpty = emptyBody; if (resolvedEmpty == null) { diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1DeserializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1DeserializeBenchmark.java index 6af7dfab0..ca3c85da1 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1DeserializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1DeserializeBenchmark.java @@ -48,7 +48,8 @@ public class RestJson1DeserializeBenchmark { @Setup public void setup() { protocol = new RestJsonClientProtocol(SERVICE_ID); - state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_JSON_BODY, CONTENT_TYPE, false); + state = DeserializeState + .forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_JSON_BODY, CONTENT_TYPE, false); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1SerializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1SerializeBenchmark.java index c41c52664..be12c79f4 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1SerializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1SerializeBenchmark.java @@ -44,7 +44,7 @@ public class RestJson1SerializeBenchmark { @Setup public void setup() { protocol = new RestJsonClientProtocol(SERVICE_ID); - state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE); + state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlDeserializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlDeserializeBenchmark.java index c476e2486..86b68ea58 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlDeserializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlDeserializeBenchmark.java @@ -51,7 +51,8 @@ public class RestXmlDeserializeBenchmark { @Setup public void setup() { protocol = new RestXmlClientProtocol(SERVICE_ID); - state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_XML_BODY, CONTENT_TYPE, false); + state = DeserializeState + .forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_XML_BODY, CONTENT_TYPE, false); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlSerializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlSerializeBenchmark.java index f2db6736d..c1848e9b8 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlSerializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestXmlSerializeBenchmark.java @@ -44,7 +44,7 @@ public class RestXmlSerializeBenchmark { @Setup public void setup() { protocol = new RestXmlClientProtocol(SERVICE_ID); - state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE); + state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborDeserializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborDeserializeBenchmark.java index 5161ed658..36e5e2ac6 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborDeserializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborDeserializeBenchmark.java @@ -54,7 +54,8 @@ public class RpcV2CborDeserializeBenchmark { @Setup public void setup() { protocol = new RpcV2CborProtocol(SERVICE_ID); - state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_CBOR_BODY, CONTENT_TYPE, true); + state = DeserializeState + .forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_CBOR_BODY, CONTENT_TYPE, true); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborSerializeBenchmark.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborSerializeBenchmark.java index f6f94f68e..889487663 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborSerializeBenchmark.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RpcV2CborSerializeBenchmark.java @@ -50,7 +50,7 @@ public class RpcV2CborSerializeBenchmark { @Setup public void setup() { protocol = new RpcV2CborProtocol(SERVICE_ID); - state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE); + state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID); } @Benchmark diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/SerializeState.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/SerializeState.java index 3d0a16143..1aefa9674 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/SerializeState.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/SerializeState.java @@ -14,6 +14,7 @@ import software.amazon.smithy.java.protocoltests.harness.ProtocolTestDocument; import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ShapeId; /** * Reusable per-trial state for a serialization benchmark. @@ -56,11 +57,11 @@ private SerializeState( * projection (e.g., * {@code "software.amazon.smithy.java.benchmarks.serde.generated.awsjson10.model"}) */ - static SerializeState forTestCase(String testCaseId, String generatedPackage) { + static SerializeState forTestCase(String testCaseId, String generatedPackage, ShapeId serviceId) { RequestEntry entry = BenchmarkTestCases.request(testCaseId); OperationShape opShape = entry.operation(); - var operation = resolveOperation(generatedPackage, opShape.getId().getName()); + var operation = BenchmarkOperations.resolve(opShape, generatedPackage, serviceId); Node params = entry.testCase().getParams(); var inputBuilder = operation.inputBuilder(); @@ -87,6 +88,7 @@ static SerializeState forTestCase(String testCaseId, String generatedPackage) { static void main() { SerializeState.forTestCase("rpcv2Cbor_PutItemRequest_BinaryData_M", - "software.amazon.smithy.java.benchmarks.serde.generated.rpcv2cbor.model"); + "software.amazon.smithy.java.benchmarks.serde.generated.rpcv2cbor.model", + ShapeId.from("com.amazonaws.sdk.benchmark#SmithyRpcV2CborDataPlane")); } }