From 76357fc5cb94cdacb8cd9de6505b39962973b32b Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 15:55:25 -0700 Subject: [PATCH 01/12] fix(telemetry): align error names and log lines with JS - error.name is the simple type name of the user's error, the same live and after replay (last segment of the checkpointed errorType). - includeErrors(false) drops only per-operation errors and keeps the execution-level error, as in the JS exporter. - LambdaLogExporter prints the Lambda JSON log envelope {timestamp, level, message} so lambda-log-exporter readers that filter on level and parse message find Java records. --- telemetry-exporter/README.md | 17 +- .../durable/telemetry/ContentConfig.java | 6 +- .../telemetry/DurableExecutionTelemetry.java | 62 ++++- .../durable/telemetry/TelemetryPlugin.java | 7 +- .../exporters/LambdaLogExporter.java | 29 ++- .../durable/telemetry/ErrorNameTest.java | 217 ++++++++++++++++++ .../telemetry/ErrorPrivacyGateTest.java | 27 ++- .../exporters/LambdaLogExporterTest.java | 99 ++++++++ 8 files changed, 434 insertions(+), 30 deletions(-) create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index 17b661904..002a8a4d1 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -190,8 +190,21 @@ one event per record to `{logStreamPrefix}YYYY/MM/DD` (IAM: `logs:CreateLogStrea it is deliberately *not* a type-preserving clone. Mutating the argument is safe (it cannot corrupt the cached input snapshot or any later emission), and a transform that throws omits the field and logs the failure rather than silently dropping it or failing the execution. -- **`includeErrors`** gates **both** the execution-level error and each operation-level error; with - `includeErrors(false)` neither is emitted, so a sensitive failure message never reaches a record. +- **`includeErrors`** gates only the per-operation errors (`operations[].error` and + `operationsByName..error`), as in the JS exporter. With `includeErrors(false)` the + execution-level `error` is still emitted, so a FAILED record keeps its failure reason. +- **`error.name`** is the simple type name of the error the user's code threw (for example + `PaymentError`): no package, no enclosing-class `$` prefix. The same rule applies to the + execution `error`, `operations[].error` and `operationsByName..error`, and a failure gets + the same name whether it happens live or is replayed from a checkpoint (the last segment of the + checkpointed `errorType`). When no user error is involved, the SDK exception's simple name is + used. `error.message` is unchanged. +- **`LambdaLogExporter` line shape.** Each record is printed to stdout as one Lambda JSON log line, + `{"timestamp":"Z","level":"INFO","message":""}`, + because the JS readers of this destination filter on `level` and parse `message`. The Node + runtime adds this envelope around `console.log`; Java `System.out` is not wrapped, and Lambda + does not re-wrap a line that is already JSON. `maxRecordSizeBytes` is measured on the record + (`render`), not on the envelope, which adds escaping plus about 70 bytes. - **Non-fatal plugin failures are isolated.** Record construction, snapshotting, transforms, truncation and exporter render/export/flush contain and log ordinary failures, including `AssertionError` and optional-dependency `NoClassDefFoundError`. `VirtualMachineError` and diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java index ae3e3f9af..c92327163 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java @@ -12,9 +12,9 @@ * *

Mirrors the JS {@code ContentConfig}. {@code input}/{@code output} may be included as-is, excluded, or * transformed; all three are honored — the plugin reads execution input from {@code InvocationInfo.executionInput()} - * and output from {@code InvocationEndInfo.executionResult()}. {@code includeErrors} is honored for both the - * execution-level error and the per-operation error: with {@code includeErrors(false)} neither is emitted. - * Per-operation result opt-in is honored via {@link OperationOverride#withResult} (see + * and output from {@code InvocationEndInfo.executionResult()}. {@code includeErrors} gates only the per-operation + * errors, as in JS: with {@code includeErrors(false)} operation errors are omitted and the execution-level error is + * kept. Per-operation result opt-in is honored via {@link OperationOverride#withResult} (see * {@code OperationChangeItemInfo.result()}). * *

Transform value shape. The {@code inputTransform} and {@code outputTransform} functions receive a diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java index ff5b0f2f0..c830ac057 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java @@ -160,16 +160,62 @@ static ErrorInfo toErrorInfo(Throwable t) { // Operation and execution snapshot errors are exposed wrapped: operation failures as DurableOperationException // and unrecoverable execution failures as UnrecoverableDurableExecutionException. The wrapper's own // class/message would lose the original checkpointed failure identity. When the checkpointed ErrorObject is - // present (from either wrapper), derive name/message from its errorType/errorMessage, falling back to the - // throwable's own fields for any value the ErrorObject leaves null. + // present (from either wrapper), derive the message from its errorMessage, falling back to the throwable's + // own message when the ErrorObject leaves it null. The name follows errorName. ErrorObject error = extractErrorObject(t); - if (error != null) { - String name = - error.errorType() != null ? error.errorType() : t.getClass().getSimpleName(); - String message = error.errorMessage() != null ? error.errorMessage() : t.getMessage(); - return new ErrorInfo(name, message); + String message = error != null && error.errorMessage() != null ? error.errorMessage() : t.getMessage(); + return new ErrorInfo(errorName(t, error), message); + } + + /** + * Returns the cross-SDK {@code error.name}: the simple type name of the error the user's code threw (for example + * {@code PaymentError}), with no package and no enclosing-class {@code $} prefix. An SDK wrapper with a live cause + * reports the cause's simple name; otherwise a wrapper reports the last segment of its checkpointed + * {@code errorType} (which the core SDK writes as {@code Class.getName()}), so a replayed failure gets the same + * name as the live one. Any other throwable — including an SDK exception with no user error behind it — reports its + * own simple name. + */ + static String errorName(Throwable t, ErrorObject error) { + return errorName(t, error, 0); + } + + private static final int MAX_CAUSE_DEPTH = 16; + + private static String errorName(Throwable t, ErrorObject error, int depth) { + if (t instanceof DurableOperationException || t instanceof UnrecoverableDurableExecutionException) { + Throwable cause = t.getCause(); + if (cause != null && cause != t) { + // A wrapped wrapper resolves through its own cause/errorType; the depth bound guards a cyclic chain. + return depth < MAX_CAUSE_DEPTH + ? errorName(cause, extractErrorObject(cause), depth + 1) + : simpleName(cause.getClass()); + } + if (error != null && error.errorType() != null && !error.errorType().isBlank()) { + return simpleTypeName(error.errorType()); + } + } + return simpleName(t.getClass()); + } + + /** + * {@link Class#getSimpleName()}, except for anonymous and local classes, whose simple name is empty or differs from + * the checkpointed type name; those use the last segment of {@link Class#getName()} so live and replayed failures + * agree. + */ + static String simpleName(Class type) { + String simple = type.getSimpleName(); + if (simple.isEmpty() || type.isAnonymousClass() || type.isLocalClass()) { + return simpleTypeName(type.getName()); } - return new ErrorInfo(t.getClass().getSimpleName(), t.getMessage()); + return simple; + } + + /** Returns the part of a type name after the last {@code '.'} and the last {@code '$'}. */ + static String simpleTypeName(String typeName) { + String trimmed = typeName.trim(); + int cut = Math.max(trimmed.lastIndexOf('.'), trimmed.lastIndexOf('$')); + String last = trimmed.substring(cut + 1); + return last.isEmpty() ? trimmed : last; } /** diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java index db791a654..90d155783 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java @@ -381,10 +381,9 @@ private DurableExecutionTelemetryRecord buildRecord( output, content == null || content.includeOutput(), content == null ? null : content.outputTransform()); - // Honor ContentConfig.includeErrors for the execution-level error exactly as for operation-level errors - // below: with includeErrors(false) no execution error is emitted, so a sensitive failure message never - // reaches a record. Without this gate the execution error leaked even when errors were disabled. - if (settings.includeErrors && error != null) { + // The execution-level error is always kept, matching the JS exporter: ContentConfig.includeErrors gates only + // the per-operation errors below, so a FAILED record still says why it failed. + if (error != null) { record.error = DurableExecutionTelemetry.toErrorInfo(error); } record.operations = buildOperationRecords(operations); diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java index b44e84a8e..b64ba2e6b 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.telemetry.exporters; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.LinkedHashMap; +import java.util.Map; import software.amazon.lambda.durable.annotations.Experimental; import software.amazon.lambda.durable.telemetry.DurableExecutionTelemetryRecord; import software.amazon.lambda.durable.telemetry.Json; @@ -10,11 +15,20 @@ /** * Exports workflow telemetry records to CloudWatch Logs via {@code System.out.println} (Lambda captures stdout to the * function's own log group, so this needs no extra IAM). Emits the {@code operationsByName} map as ONE single-line JSON - * record, mirroring the JS {@code LambdaLogExporter} (which uses {@code console.log}). The conformance CloudWatch sink - * decodes both raw top-level JSON lines and the Lambda structured-logging envelope. + * record, mirroring the JS {@code LambdaLogExporter} (which uses {@code console.log}). + * + *

The line is the Lambda JSON log envelope {@code {"timestamp", "level": "INFO", "message"}} whose {@code message} + * is the compact record JSON. Readers of this destination filter on {@code level} and parse {@code message}. The Node + * runtime adds that envelope around {@code console.log}, but Java {@code System.out} is not wrapped, so this exporter + * writes it itself; Lambda does not re-wrap a line that is already JSON. {@link #maxRecordSizeBytes()} is measured on + * the record ({@link #render}), not on the envelope. */ @Experimental public final class LambdaLogExporter implements TelemetryExporter { + /** ISO-8601 UTC with exactly three fraction digits, matching JS {@code Date.toISOString()}. */ + private static final DateTimeFormatter TIMESTAMP = + DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC); + /** CloudWatch Logs caps a single log event at 256 KB. */ private final Integer maxRecordSizeBytes; @@ -38,6 +52,15 @@ public Object render(DurableExecutionTelemetryRecord record) { @Override public void export(DurableExecutionTelemetryRecord record) { - System.out.println(Json.stringify(render(record))); + System.out.println(logLine(Json.stringify(render(record)), Instant.now())); + } + + /** Builds the one-line Lambda JSON log envelope; keys in the shared cross-SDK order. */ + static String logLine(String recordJson, Instant now) { + Map line = new LinkedHashMap<>(); + line.put("timestamp", TIMESTAMP.format(now)); + line.put("level", "INFO"); + line.put("message", recordJson); + return Json.stringify(line); } } diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java new file mode 100644 index 000000000..815278acf --- /dev/null +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java @@ -0,0 +1,217 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.exception.DurableOperationException; +import software.amazon.lambda.durable.exception.IllegalDurableOperationException; +import software.amazon.lambda.durable.exception.StepInterruptedException; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.retry.RetryStrategies; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +/** + * {@code error.name} is the simple type name of the error the user's code threw — no package, no enclosing-class + * {@code $} prefix — for the execution {@code error}, {@code operations[].error} and + * {@code operationsByName..error}, and is the same whether the failure is live or replayed from a checkpoint. + */ +class ErrorNameTest { + + /** A nested user error, so its binary name is {@code ...ErrorNameTest$PaymentError}. */ + public static class PaymentError extends RuntimeException { + public PaymentError(String message) { + super(message); + } + } + + private static final class CapturingExporter implements TelemetryExporter { + final List records = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void export(DurableExecutionTelemetryRecord record) { + records.add(record); + } + } + + private static final StepConfig NO_RETRY = + StepConfig.builder().retryStrategy(RetryStrategies.Presets.NO_RETRY).build(); + + private static DurableConfig configWith(CapturingExporter exporter, DurableExecutionTelemetryConfig.Builder cfg) { + return DurableConfig.builder() + .withPlugins(DurableExecutionTelemetry.durableExecutionTelemetry( + cfg.addExporter(exporter).build())) + .build(); + } + + private static OperationRecord op(DurableExecutionTelemetryRecord rec, String name) { + return rec.operations().stream() + .filter(o -> name.equals(o.name())) + .findFirst() + .orElseThrow(); + } + + @SuppressWarnings("unchecked") + private static Map byNameError(DurableExecutionTelemetryRecord rec, String name) { + var byName = (Map) rec.toByNameWireMap().get("operationsByName"); + return (Map) ((Map) byName.get(name)).get("error"); + } + + @Test + void liveFailureUsesSimpleNameForExecutionOperationAndByName() { + var exporter = new CapturingExporter(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step( + "charge", + String.class, + sc -> { + throw new PaymentError("card declined"); + }, + NO_RETRY), + configWith(exporter, DurableExecutionTelemetryConfig.builder())); + + assertEquals(ExecutionStatus.FAILED, runner.run("in").getStatus()); + assertEquals(1, exporter.records.size()); + var rec = exporter.records.get(0); + + assertEquals("PaymentError", rec.error.name(), "execution error"); + assertEquals("card declined", rec.error.message()); + assertEquals("PaymentError", op(rec, "charge").error().name(), "operations[].error"); + assertEquals("card declined", op(rec, "charge").error().message()); + assertEquals("PaymentError", byNameError(rec, "charge").get("name"), "operationsByName..error"); + } + + @Test + void replayedFailureInLaterInvocationKeepsTheSameName() { + var exporter = new CapturingExporter(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + try { + context.step( + "charge", + String.class, + sc -> { + throw new PaymentError("card declined"); + }, + NO_RETRY); + } catch (RuntimeException ignored) { + // Swallowed so the failed step is replayed from its checkpoint after the wait. + } + context.wait("cooldown", Duration.ofSeconds(1)); + return context.step( + "charge-again", + String.class, + sc -> { + throw new PaymentError("still declined"); + }, + NO_RETRY); + }, + configWith( + exporter, + DurableExecutionTelemetryConfig.builder() + .emitMode(DurableExecutionTelemetryConfig.EmitMode.ON_CHANGE))); + + assertEquals(ExecutionStatus.FAILED, runner.runUntilComplete("in").getStatus()); + var records = List.copyOf(exporter.records); + var last = records.get(records.size() - 1); + assertEquals("FAILED", last.status()); + assertEquals("PaymentError", last.error.name()); + assertEquals("still declined", last.error.message()); + assertEquals("PaymentError", op(last, "charge-again").error().name()); + + // "charge" failed in the first invocation and is replayed from its checkpoint in the second; every record that + // carries it must report the same simple name. + long withCharge = 0; + for (var rec : records) { + var charge = rec.operations().stream() + .filter(o -> "charge".equals(o.name()) && o.error() != null) + .findFirst(); + if (charge.isPresent()) { + withCharge++; + assertEquals("PaymentError", charge.get().error().name()); + assertEquals("card declined", charge.get().error().message()); + } + } + assertTrue(withCharge >= 2, "charge error seen live and after replay, got " + withCharge); + assertEquals("PaymentError", byNameError(last, "charge").get("name")); + } + + @Test + void checkpointedErrorTypeAndLiveThrowableGiveTheSameName() { + // Live: the user's nested class. Replay: the checkpoint's errorType, which core writes as Class.getName(). + Throwable live = new PaymentError("x"); + assertEquals( + "software.amazon.lambda.durable.telemetry.ErrorNameTest$PaymentError", PaymentError.class.getName()); + assertEquals("PaymentError", DurableExecutionTelemetry.toErrorInfo(live).name()); + assertEquals( + "PaymentError", + DurableExecutionTelemetry.toErrorInfo(wrapped(PaymentError.class.getName(), "x")) + .name()); + + // Anonymous and local classes: getSimpleName is empty / differs from the binary name, so both paths use the + // last segment of getName. + class LocalError extends RuntimeException {} + Throwable anonymous = new RuntimeException() {}; + for (Throwable t : List.of(anonymous, new LocalError())) { + String fromLive = DurableExecutionTelemetry.toErrorInfo(t).name(); + String fromCheckpoint = DurableExecutionTelemetry.toErrorInfo( + wrapped(t.getClass().getName(), null)) + .name(); + assertEquals(fromCheckpoint, fromLive, t.getClass().getName()); + assertTrue(!fromLive.isEmpty() && !fromLive.contains("$") && !fromLive.contains("."), fromLive); + } + + // Plain type strings from other SDKs or service error codes. + assertEquals("PaymentError", DurableExecutionTelemetry.simpleTypeName("PaymentError")); + assertEquals("PaymentError", DurableExecutionTelemetry.simpleTypeName("a.b.Outer$Inner$PaymentError")); + assertEquals("PaymentError", DurableExecutionTelemetry.simpleTypeName("__main__.PaymentError")); + } + + @Test + void sdkWrapperWithLiveUserCauseUsesTheCause() { + var wrapper = new UnrecoverableDurableExecutionException( + ErrorObject.builder() + .errorType("ignored.Type") + .errorMessage("wrapped") + .build(), + true, + new PaymentError("card declined")); + var info = DurableExecutionTelemetry.toErrorInfo(wrapper); + assertEquals("PaymentError", info.name()); + assertEquals("wrapped", info.message(), "message unchanged"); + } + + @Test + void sdkOnlyFailureUsesTheSdkExceptionSimpleName() { + // Live SDK exception with no user error behind it. + assertEquals( + "IllegalDurableOperationException", + DurableExecutionTelemetry.toErrorInfo(new IllegalDurableOperationException("bad")) + .name()); + // Checkpointed SDK error type. + var interrupted = DurableExecutionTelemetry.toErrorInfo(wrapped(StepInterruptedException.class.getName(), "i")); + assertEquals("StepInterruptedException", interrupted.name()); + assertNotNull(interrupted.message()); + } + + private static DurableOperationException wrapped(String errorType, String message) { + return new DurableOperationException( + Operation.builder().id("op-1").build(), + ErrorObject.builder().errorType(errorType).errorMessage(message).build()); + } +} diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java index 872d93141..60b3b5923 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Instant; import java.util.ArrayList; @@ -20,10 +21,9 @@ import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; /** - * Fix 1 — error privacy gate. {@code ContentConfig.includeErrors(false)} must suppress both the - * execution-level error and every operation-level error, so a sensitive failure message never reaches an emitted record - * — in the canonical {@code operations} array rendering or the {@code operationsByName} rendering. Before the fix the - * operation error was gated but the execution error leaked unconditionally. + * Error gate. {@code ContentConfig.includeErrors(false)} suppresses every operation-level error — in the canonical + * {@code operations} array rendering and the {@code operationsByName} rendering — and keeps the execution-level + * {@code error}, matching the JS exporter. */ class ErrorPrivacyGateTest { @@ -88,20 +88,27 @@ private DurableExecutionTelemetryRecord runFailedExecution(boolean includeErrors } @Test - void includeErrorsFalseSuppressesBothErrorsAndKeepsSecretsOutOfBothRenderings() { + void includeErrorsFalseDropsOperationErrorsAndKeepsExecutionError() { var exporter = new CapturingExporter(); var rec = runFailedExecution(false, exporter); assertEquals("FAILED", rec.status()); - assertNull(rec.error, "execution-level error suppressed by includeErrors:false"); + assertNotNull(rec.error, "execution-level error kept with includeErrors:false (as in JS)"); + assertEquals("RuntimeException", rec.error.name()); + assertEquals(EXEC_SECRET, rec.error.message()); assertNull(rec.operations().get(0).error(), "operation-level error suppressed by includeErrors:false"); String arrayJson = Json.stringify(rec.toWireMap()); String byNameJson = Json.stringify(rec.toByNameWireMap()); - assertFalse(arrayJson.contains(EXEC_SECRET), "no execution secret in the operations-array JSON"); - assertFalse(arrayJson.contains(OP_SECRET), "no operation secret in the operations-array JSON"); - assertFalse(byNameJson.contains(EXEC_SECRET), "no execution secret in the operationsByName JSON"); - assertFalse(byNameJson.contains(OP_SECRET), "no operation secret in the operationsByName JSON"); + assertTrue(arrayJson.contains(EXEC_SECRET), "execution error in the operations-array JSON"); + assertFalse(arrayJson.contains(OP_SECRET), "no operation error in the operations-array JSON"); + assertTrue(byNameJson.contains(EXEC_SECRET), "execution error in the operationsByName JSON"); + assertFalse(byNameJson.contains(OP_SECRET), "no operation error in the operationsByName JSON"); + @SuppressWarnings("unchecked") + var byName = (Map) rec.toByNameWireMap().get("operationsByName"); + @SuppressWarnings("unchecked") + var entry = (Map) byName.get("failing-step"); + assertFalse(entry.containsKey("error"), "no operationsByName..error with includeErrors:false"); } @Test diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java new file mode 100644 index 000000000..ce673a55c --- /dev/null +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java @@ -0,0 +1,99 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.telemetry.exporters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.telemetry.DurableExecutionTelemetryRecord; +import software.amazon.lambda.durable.telemetry.Json; +import software.amazon.lambda.durable.telemetry.RecordFactory; +import software.amazon.lambda.durable.telemetry.Truncation; + +/** + * The {@code LambdaLogExporter} line is the Lambda JSON log envelope that the {@code lambda-log-exporter} readers query + * with {@code filter level = "INFO" and message like /DurableExecutionTelemetry/} and then parse {@code message}. + */ +class LambdaLogExporterTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Pattern TIMESTAMP = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z"); + /** The reader's Logs Insights {@code message like /DurableExecutionTelemetry/} filter. */ + private static final Pattern READER_MESSAGE_FILTER = Pattern.compile("DurableExecutionTelemetry"); + + private static String exportAndCapture(DurableExecutionTelemetryRecord record) { + PrintStream original = System.out; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (PrintStream capture = new PrintStream(buffer, true, StandardCharsets.UTF_8)) { + System.setOut(capture); + new LambdaLogExporter().export(record); + } finally { + System.setOut(original); + } + return buffer.toString(StandardCharsets.UTF_8); + } + + @Test + @SuppressWarnings("unchecked") + void exportPrintsOneEnvelopeLineWhoseMessageIsTheRecord() throws Exception { + var record = RecordFactory.sample(); + String out = exportAndCapture(record); + + List lines = out.lines().toList(); + assertEquals(1, lines.size(), "exactly one line: " + out); + String line = lines.get(0); + + // The line parses, with keys in the shared cross-SDK order and no requestId (not available to an exporter). + Map envelope = MAPPER.readValue(line, Map.class); + assertEquals(List.of("timestamp", "level", "message"), List.copyOf(envelope.keySet())); + assertTrue(TIMESTAMP.matcher((String) envelope.get("timestamp")).matches(), line); + assertEquals("INFO", envelope.get("level")); + + // message is the compact record JSON string, and parses back to the record's render shape. + String message = (String) envelope.get("message"); + assertEquals(Json.stringify(new LambdaLogExporter().render(record)), message); + Map parsed = MAPPER.readValue(message, Map.class); + assertEquals("DurableExecutionTelemetry", parsed.get("recordType")); + assertEquals(record.executionArn(), parsed.get("executionArn")); + assertTrue(parsed.containsKey("operationsByName")); + assertFalse(parsed.containsKey("operations")); + + // The reader's filter: level = "INFO" and message like /DurableExecutionTelemetry/. + assertTrue("INFO".equals(envelope.get("level")) + && READER_MESSAGE_FILTER.matcher(message).find()); + } + + @Test + void timestampAlwaysHasThreeFractionDigitsInUtc() { + assertEquals( + "{\"timestamp\":\"2026-10-21T08:00:00.000Z\",\"level\":\"INFO\",\"message\":\"{}\"}", + LambdaLogExporter.logLine("{}", Instant.parse("2026-10-21T08:00:00Z"))); + assertTrue(LambdaLogExporter.logLine("{}", Instant.parse("2026-10-21T08:00:00.123456789Z")) + .startsWith("{\"timestamp\":\"2026-10-21T08:00:00.123Z\"")); + } + + @Test + void truncationIsMeasuredOnTheRecordNotTheEnvelope() { + var record = RecordFactory.sample(); + var exporter = new LambdaLogExporter(); + int recordBytes = Json.byteSize(exporter.render(record)); + + // A limit equal to the rendered record keeps it whole, even though the printed envelope line is larger. + var shaped = Truncation.truncateRecord(record, recordBytes, exporter::render); + assertSame(record, shaped); + assertFalse(shaped.toWireMap().containsKey("truncated")); + String line = exportAndCapture(shaped).strip(); + assertTrue(line.getBytes(StandardCharsets.UTF_8).length > recordBytes, line); + } +} From 5b0ce8703dd3e36c802221c55b078119a3d357a6 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 21:41:46 -0700 Subject: [PATCH 02/12] fix(telemetry)!: apply the shared cross-SDK error.name rule - Reduce type strings exactly as Table A of the shared rule: last segment after '.' or '$', kept as is when that segment is empty, no trimming, and "Error" for a missing, empty or blank type (no wrapper-class fallback). - Derive live names from Class.getName() through the same reduction, so a class named Payment$Error is "Error" live and replayed, and anonymous and local classes follow Table A (pkg.Foo$1Local -> "1Local"). The anonymous/local special case is removed. - Treat a throwable as an SDK wrapper only when its class is declared in software.amazon.lambda.durable.exception. A user subclass of an SDK exception reports its own name. CallbackTimeoutException is never unwrapped. - When an operation's checkpoint has no error type, name it after the SDK exception the core throws for it (for example CallbackFailedException), so the operation and the execution agree. A timed-out callback with no checkpointed error now gets an operation error named CallbackTimeoutException with a null message. - Load error-name-vectors.json (shared with the JS and Python exporters) into a parameterized test, and run every Table B scenario that applies to Java through LocalDurableTestRunner, live and replayed. BREAKING CHANGE: error.name values change. A blank or missing checkpointed error type reports "Error" instead of the wrapper class name, names are no longer trimmed, a user subclass of an SDK exception reports its own name instead of its cause's, and a callback failed without an error type reports CallbackFailedException on the operation as well as the execution. --- .../telemetry/DurableExecutionTelemetry.java | 195 +++++- .../durable/telemetry/TelemetryPlugin.java | 18 +- .../exception/CyclicWrapperForTest.java | 26 + .../telemetry/ErrorNameScenarioTest.java | 599 ++++++++++++++++++ .../durable/telemetry/ErrorNameTest.java | 191 +++++- .../telemetry/ErrorNameVectorsTest.java | 86 +++ .../telemetry/OperationErrorIdentityTest.java | 5 +- .../durable/telemetry/Payment$Error.java | 13 + .../UnrecoverableErrorUnwrapTest.java | 5 +- .../test/resources/error-name-vectors.json | 21 + 10 files changed, 1111 insertions(+), 48 deletions(-) create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/exception/CyclicWrapperForTest.java create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/Payment$Error.java create mode 100644 telemetry-exporter/src/test/resources/error-name-vectors.json diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java index c830ac057..06ffb0a0c 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java @@ -3,13 +3,30 @@ package software.amazon.lambda.durable.telemetry; import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.Set; import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.annotations.Experimental; +import software.amazon.lambda.durable.exception.CallbackException; +import software.amazon.lambda.durable.exception.CallbackFailedException; +import software.amazon.lambda.durable.exception.CallbackTimeoutException; +import software.amazon.lambda.durable.exception.ChildContextFailedException; import software.amazon.lambda.durable.exception.DurableOperationException; +import software.amazon.lambda.durable.exception.InvokeException; +import software.amazon.lambda.durable.exception.InvokeFailedException; +import software.amazon.lambda.durable.exception.InvokeStoppedException; +import software.amazon.lambda.durable.exception.InvokeTimedOutException; +import software.amazon.lambda.durable.exception.MapIterationFailedException; +import software.amazon.lambda.durable.exception.ParallelBranchFailedException; +import software.amazon.lambda.durable.exception.StepException; +import software.amazon.lambda.durable.exception.StepFailedException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.exception.WaitForConditionFailedException; +import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; @@ -164,58 +181,166 @@ static ErrorInfo toErrorInfo(Throwable t) { // own message when the ErrorObject leaves it null. The name follows errorName. ErrorObject error = extractErrorObject(t); String message = error != null && error.errorMessage() != null ? error.errorMessage() : t.getMessage(); - return new ErrorInfo(errorName(t, error), message); + return new ErrorInfo(errorName(t), message); } + /** Package that declares the SDK's own exception types; only classes declared here can be SDK wrappers. */ + private static final String SDK_EXCEPTION_PACKAGE = DurableOperationException.class.getPackageName(); + + /** SDK base classes that never name a failure themselves: with nothing to unwrap they report {@code Error}. */ + private static final Set> SDK_BASE_WRAPPERS = Set.of( + DurableOperationException.class, + UnrecoverableDurableExecutionException.class, + StepException.class, + CallbackException.class, + InvokeException.class); + + /** Reported name when a type string is missing or blank (Table A, A12). */ + static final String DEFAULT_ERROR_NAME = "Error"; + + private static final int MAX_WRAPPER_DEPTH = 16; + /** - * Returns the cross-SDK {@code error.name}: the simple type name of the error the user's code threw (for example - * {@code PaymentError}), with no package and no enclosing-class {@code $} prefix. An SDK wrapper with a live cause - * reports the cause's simple name; otherwise a wrapper reports the last segment of its checkpointed - * {@code errorType} (which the core SDK writes as {@code Class.getName()}), so a replayed failure gets the same - * name as the live one. Any other throwable — including an SDK exception with no user error behind it — reports its - * own simple name. + * Returns the cross-SDK {@code error.name} of a throwable. + * + *

    + *
  1. A throwable that is not an SDK wrapper (see {@link #isSdkWrapper}) reports its own class name, reduced by + * {@link #simpleTypeName}. This includes user errors that have a cause and user subclasses of SDK exceptions. + *
  2. An SDK wrapper with a live cause is unwrapped, following the chain of SDK wrappers up to a depth of + * {@value #MAX_WRAPPER_DEPTH}; the depth bound also stops a cyclic chain. + *
  3. An SDK wrapper with no cause reports its checkpointed {@code errorType}, reduced. The core writes that type + * as {@link Class#getName()}, so a replayed failure gets the same name as the live one. + *
  4. An SDK wrapper whose {@code errorType} is blank reports the SDK failure class the core throws for its + * operation (for example {@code CallbackFailedException}), so the operation and the execution agree. + *
  5. A timed-out callback is never unwrapped: it reports {@code CallbackTimeoutException}. + *
*/ - static String errorName(Throwable t, ErrorObject error) { - return errorName(t, error, 0); + static String errorName(Throwable t) { + Throwable current = t; + for (int depth = 0; ; depth++) { + if (!isSdkWrapper(current)) { + return simpleTypeName(current.getClass().getName()); + } + String timeout = callbackTimeoutName(current); + if (timeout != null) { + return timeout; + } + Throwable cause = current.getCause(); + if (cause != null && cause != current) { + if (depth >= MAX_WRAPPER_DEPTH) { + return simpleTypeName(cause.getClass().getName()); + } + current = cause; + continue; + } + return wrapperName(current); + } } - private static final int MAX_CAUSE_DEPTH = 16; + /** + * {@code true} when {@code t} is one of the SDK's own wrapper types: an instance of + * {@link DurableOperationException} or {@link UnrecoverableDurableExecutionException} whose runtime class is + * declared in the SDK's exception package. A user subclass of an SDK exception is not a wrapper. A callback timeout + * is an SDK error, not a wrapper. + */ + static boolean isSdkWrapper(Throwable t) { + return (t instanceof DurableOperationException || t instanceof UnrecoverableDurableExecutionException) + && !(t instanceof CallbackTimeoutException) + && SDK_EXCEPTION_PACKAGE.equals(t.getClass().getPackageName()); + } - private static String errorName(Throwable t, ErrorObject error, int depth) { - if (t instanceof DurableOperationException || t instanceof UnrecoverableDurableExecutionException) { - Throwable cause = t.getCause(); - if (cause != null && cause != t) { - // A wrapped wrapper resolves through its own cause/errorType; the depth bound guards a cyclic chain. - return depth < MAX_CAUSE_DEPTH - ? errorName(cause, extractErrorObject(cause), depth + 1) - : simpleName(cause.getClass()); - } - if (error != null && error.errorType() != null && !error.errorType().isBlank()) { - return simpleTypeName(error.errorType()); + /** Name of an SDK wrapper that has no cause to follow. */ + private static String wrapperName(Throwable wrapper) { + ErrorObject error = extractErrorObject(wrapper); + String type = error == null ? null : error.errorType(); + if (type != null && !type.isBlank()) { + return simpleTypeName(type); + } + if (wrapper instanceof DurableOperationException doe && doe.getOperation() != null) { + var op = doe.getOperation(); + String failure = sdkFailureName(op.typeAsString(), op.subType(), op.status()); + if (failure != null) { + return failure; } } - return simpleName(t.getClass()); + if (!SDK_BASE_WRAPPERS.contains(wrapper.getClass())) { + return simpleTypeName(wrapper.getClass().getName()); + } + return DEFAULT_ERROR_NAME; + } + + /** {@code CallbackTimeoutException} for a wrapper whose operation is a timed-out callback, else {@code null}. */ + private static String callbackTimeoutName(Throwable wrapper) { + if (wrapper instanceof DurableOperationException doe + && doe.getOperation() != null + && doe.getOperation().type() == OperationType.CALLBACK + && doe.getOperation().status() == OperationStatus.TIMED_OUT) { + return CallbackTimeoutException.class.getSimpleName(); + } + return null; } /** - * {@link Class#getSimpleName()}, except for anonymous and local classes, whose simple name is empty or differs from - * the checkpointed type name; those use the last segment of {@link Class#getName()} so live and replayed failures - * agree. + * Returns the simple name of the SDK exception the core throws when an operation of this type, sub-type and status + * fails with no reconstructable user error, or {@code null} when there is none. Used when the checkpoint carries no + * error type, so an operation record and the execution error it propagates to get the same name. */ - static String simpleName(Class type) { - String simple = type.getSimpleName(); - if (simple.isEmpty() || type.isAnonymousClass() || type.isLocalClass()) { - return simpleTypeName(type.getName()); + static String sdkFailureName(String type, String subType, OperationStatus status) { + if (type == null || status == null) { + return null; + } + Class failure = switch (type) { + case "STEP" -> + status != OperationStatus.FAILED + ? null + : OperationSubType.WAIT_FOR_CONDITION.getValue().equals(subType) + ? WaitForConditionFailedException.class + : StepFailedException.class; + case "CALLBACK" -> + switch (status) { + case FAILED -> CallbackFailedException.class; + case TIMED_OUT -> CallbackTimeoutException.class; + default -> null; + }; + case "CHAINED_INVOKE" -> + switch (status) { + case FAILED -> InvokeFailedException.class; + case TIMED_OUT -> InvokeTimedOutException.class; + case STOPPED -> InvokeStoppedException.class; + default -> null; + }; + case "CONTEXT" -> status != OperationStatus.FAILED ? null : contextFailure(subType); + default -> null; + }; + return failure == null ? null : failure.getSimpleName(); + } + + private static Class contextFailure(String subType) { + if (OperationSubType.MAP_ITERATION.getValue().equals(subType)) { + return MapIterationFailedException.class; + } + if (OperationSubType.PARALLEL_BRANCH.getValue().equals(subType)) { + return ParallelBranchFailedException.class; } - return simple; + if (OperationSubType.RUN_IN_CHILD_CONTEXT.getValue().equals(subType) + || OperationSubType.WITH_RETRY.getValue().equals(subType)) { + return ChildContextFailedException.class; + } + return null; } - /** Returns the part of a type name after the last {@code '.'} and the last {@code '$'}. */ + /** + * Reduces a type string to the reported {@code error.name} (Table A of the cross-SDK rule): the part after the last + * {@code '.'} or {@code '$'}. If that part is empty the string is kept as it is. A {@code null}, empty or blank + * string gives {@code "Error"}. The string is not trimmed. + */ static String simpleTypeName(String typeName) { - String trimmed = typeName.trim(); - int cut = Math.max(trimmed.lastIndexOf('.'), trimmed.lastIndexOf('$')); - String last = trimmed.substring(cut + 1); - return last.isEmpty() ? trimmed : last; + if (typeName == null || typeName.isBlank()) { + return DEFAULT_ERROR_NAME; + } + int cut = Math.max(typeName.lastIndexOf('.'), typeName.lastIndexOf('$')); + String last = typeName.substring(cut + 1); + return last.isEmpty() ? typeName : last; } /** diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java index 90d155783..ee4baa766 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/TelemetryPlugin.java @@ -382,7 +382,8 @@ private DurableExecutionTelemetryRecord buildRecord( content == null || content.includeOutput(), content == null ? null : content.outputTransform()); // The execution-level error is always kept, matching the JS exporter: ContentConfig.includeErrors gates only - // the per-operation errors below, so a FAILED record still says why it failed. + // the per-operation errors below, so a FAILED record still says why it failed. This is intentional and means + // includeErrors(false) does not keep the execution error's message out of a record; see ContentConfig. if (error != null) { record.error = DurableExecutionTelemetry.toErrorInfo(error); } @@ -438,8 +439,19 @@ private List buildOperationRecords(Map records = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void export(DurableExecutionTelemetryRecord record) { + records.add(record); + } + } + + /** Runs {@code failing} directly (live) or catches its error, waits for a new invocation and rethrows (replay). */ + private static T failure(Mode mode, DurableContext ctx, Function failing) { + if (mode == Mode.LIVE) { + return failing.apply(ctx); + } + RuntimeException caught = null; + try { + failing.apply(ctx); + } catch (RuntimeException e) { + caught = e; + } + if (caught == null) { + throw new IllegalStateException("scenario did not fail"); + } + ctx.wait("pause", Duration.ofSeconds(1)); + throw caught; + } + + private static List run( + BiFunction handler, + Consumer> drive) { + var exporter = new CapturingExporter(); + var config = DurableConfig.builder() + .withPlugins( + DurableExecutionTelemetry.durableExecutionTelemetry(DurableExecutionTelemetryConfig.builder() + .operationDetail(DurableExecutionTelemetryConfig.OperationDetail.FULL_TREE) + // ON_CHANGE: the invocation that ran a nested operation emits a record carrying it. + .emitMode(DurableExecutionTelemetryConfig.EmitMode.ON_CHANGE) + .addExporter(exporter) + .build())) + .build(); + var runner = LocalDurableTestRunner.create(String.class, handler, config); + if (drive == null) { + assertEquals(ExecutionStatus.FAILED, runner.runUntilComplete("in").getStatus()); + } else { + drive.accept(runner); + } + var records = List.copyOf(exporter.records); + assertTrue(!records.isEmpty(), "at least one record"); + assertEquals("FAILED", records.get(records.size() - 1).status(), "final record status"); + return records; + } + + /** What a scenario reports: the execution error and the error of each named operation. */ + private record Observed(ErrorInfo execution, Map operations) {} + + /** + * The execution error comes from the final record. Each operation's error comes from the latest record that carries + * the operation: a completed child context can replay without its nested operations, so in a replayed run a nested + * operation is only in the record of the invocation that ran it. + */ + @SuppressWarnings("unchecked") + private static Observed observe(List records, List opNames) { + var last = records.get(records.size() - 1); + Map ops = new LinkedHashMap<>(); + for (String name : opNames) { + DurableExecutionTelemetryRecord rec = null; + OperationRecord op = null; + for (int i = records.size() - 1; i >= 0 && op == null; i--) { + rec = records.get(i); + op = rec.operations().stream() + .filter(o -> name.equals(o.name()) && o.error() != null) + .reduce((a, b) -> b) + .orElse(null); + } + assertNotNull(op, "operations[" + name + "].error in some record; records: " + describe(records)); + var byName = (Map) rec.toByNameWireMap().get("operationsByName"); + var entry = (Map) byName.get(name); + assertNotNull(entry, "operationsByName." + name); + var byNameError = (Map) entry.get("error"); + assertNotNull(byNameError, "operationsByName." + name + ".error"); + assertEquals(op.error().name(), byNameError.get("name"), "operationsByName == operations[] for " + name); + assertEquals(op.error().message(), byNameError.get("message"), "byName message for " + name); + ops.put(name, op.error()); + } + return new Observed(last.error, ops); + } + + private static String describe(List records) { + StringBuilder sb = new StringBuilder(); + for (var r : records) { + sb.append("\n ").append(r.status()).append(" error=").append(r.error == null ? null : r.error.name()); + for (var o : r.operations()) { + sb.append(" | ") + .append(o.name()) + .append('/') + .append(o.type()) + .append('/') + .append(o.status()); + sb.append('/').append(o.error() == null ? "-" : o.error().name()); + } + } + return sb.toString(); + } + + /** + * Runs a row live and replayed and checks it. {@code expectedOps} maps operation name to the expected name; + * {@code expectedOpMessage} is the expected message of every listed operation. A {@code null} expected message is + * checked by a dedicated test instead (SDK-generated messages). + */ + private static void check( + String row, + Function> handler, + Consumer> drive, + String expectedExecName, + String expectedExecMessage, + Map expectedOps, + String expectedOpMessage) { + Map seen = new LinkedHashMap<>(); + for (Mode mode : Mode.values()) { + var obs = observe(run(handler.apply(mode), drive), List.copyOf(expectedOps.keySet())); + String where = row + " " + mode; + assertNotNull(obs.execution(), where + " execution error"); + assertEquals(expectedExecName, obs.execution().name(), where + " execution error.name"); + if (expectedExecMessage != null) { + assertEquals(expectedExecMessage, obs.execution().message(), where + " execution error.message"); + } + expectedOps.forEach((name, expected) -> { + assertEquals(expected, obs.operations().get(name).name(), where + " " + name + " error.name"); + if (expectedOpMessage != null) { + assertEquals( + expectedOpMessage, + obs.operations().get(name).message(), + where + " " + name + " error.message"); + } + }); + seen.put(mode, obs); + } + var live = seen.get(Mode.LIVE); + var replayed = seen.get(Mode.REPLAYED); + assertEquals(live.execution().name(), replayed.execution().name(), row + " live == replayed (execution)"); + expectedOps + .keySet() + .forEach(name -> assertEquals( + live.operations().get(name).name(), + replayed.operations().get(name).name(), + row + " live == replayed (" + name + ")")); + } + + enum Row { + B1, + B2, + B3, + B4, + B5, + B6, + B7, + B8, + B9, + B10, + B11, + B12, + B13, + B14, + B15, + B16, + B17, + B18 + } + + @ParameterizedTest + @EnumSource(Row.class) + void tableB(Row row) { + switch (row) { + case B1 -> // A step throws PaymentError. + check( + "B1", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.step( + "charge", + String.class, + sc -> { + throw new PaymentError(MSG); + }, + NO_RETRY)), + null, + "PaymentError", + MSG, + Map.of("charge", "PaymentError"), + MSG); + case B2 -> { // A step fails once with TypeError, then on retry with PaymentError. + var retryOnce = StepConfig.builder() + .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1))) + .build(); + check( + "B2", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.step( + "charge", + String.class, + sc -> { + if (sc.getAttempt() == 1) { + throw new TypeError("first attempt"); + } + throw new PaymentError(MSG); + }, + retryOnce)), + null, + "PaymentError", + MSG, + Map.of("charge", "PaymentError"), + MSG); + } + case B3 -> // A child context throws PaymentError. + check( + "B3", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.runInChildContext("child", String.class, cc -> { + throw new PaymentError(MSG); + })), + null, + "PaymentError", + MSG, + Map.of("child", "PaymentError"), + MSG); + case B4 -> // A child context contains a step that throws PaymentError. + check( + "B4", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.runInChildContext( + "child", + String.class, + cc -> cc.step( + "charge", + String.class, + sc -> { + throw new PaymentError(MSG); + }, + NO_RETRY))), + null, + "PaymentError", + MSG, + Map.of("charge", "PaymentError", "child", "PaymentError"), + MSG); + case B5 -> // A map item throws PaymentError. + check( + "B5", + m -> (in, ctx) -> failure(m, ctx, c -> mapAndRethrow(c, false)), + null, + "PaymentError", + MSG, + Map.of("fanout-iteration-0", "PaymentError"), + MSG); + case B6 -> // A map item's step throws PaymentError. + check( + "B6", + m -> (in, ctx) -> failure(m, ctx, c -> mapAndRethrow(c, true)), + null, + "PaymentError", + MSG, + Map.of("charge", "PaymentError", "fanout-iteration-0", "PaymentError"), + MSG); + case B7 -> // A parallel branch throws PaymentError. + check( + "B7", + m -> (in, ctx) -> failure(m, ctx, c -> { + try (var p = c.parallel("par")) { + var bad = p.branch("bad", String.class, bc -> { + throw new PaymentError(MSG); + }); + p.get(); + return bad.get(); + } + }), + null, + "PaymentError", + MSG, + Map.of("bad", "PaymentError"), + MSG); + case B8 -> // A waitForCondition check throws PaymentError. + check( + "B8", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.waitForCondition("poll", String.class, (String state, StepContext sc) -> { + throw new PaymentError(MSG); + })), + null, + "PaymentError", + MSG, + Map.of("poll", "PaymentError"), + MSG); + case B9 -> // An invoked function throws PaymentError. + check( + "B9", + m -> (in, ctx) -> failure(m, ctx, c -> c.invoke("callee", "fn", "x", String.class)), + r -> { + r.run("in"); + r.failChainedInvoke( + "callee", + ErrorObject.builder() + .errorType(PaymentError.class.getName()) + .errorMessage(MSG) + .build()); + assertEquals( + ExecutionStatus.FAILED, + r.runUntilComplete("in").getStatus()); + }, + "PaymentError", + MSG, + Map.of("callee", "PaymentError"), + MSG); + case B10 -> // The waitForCallback submitter throws PaymentError. + check( + "B10", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.waitForCallback("wfc", String.class, (id, sc) -> { + throw new PaymentError(MSG); + })), + null, + "PaymentError", + MSG, + Map.of("wfc-submitter", "PaymentError", "wfc", "PaymentError"), + MSG); + case B11 -> // A callback times out. The checkpoint carries no error, so the operation has no message. + check( + "B11", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.createCallback("cb", String.class).get()), + r -> { + r.run("in"); + r.timeoutCallback(r.getCallbackId("cb")); + assertEquals( + ExecutionStatus.FAILED, + r.runUntilComplete("in").getStatus()); + }, + CallbackTimeoutException.class.getSimpleName(), + null, // see callbackTimeoutMessages + Map.of("cb", CallbackTimeoutException.class.getSimpleName()), + null); + case B12 -> // A callback is failed externally with type PaymentError. + check( + "B12", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.createCallback("cb", String.class).get()), + failCallback("com.acme.PaymentError"), + "PaymentError", + MSG, + Map.of("cb", "PaymentError"), + MSG); + case B13 -> { // A callback is failed externally with no type. + for (String blank : new String[] {null, "", " "}) { + check( + "B13[" + blank + "]", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.createCallback("cb", String.class).get()), + failCallback(blank), + CallbackFailedException.class.getSimpleName(), + MSG, + Map.of("cb", CallbackFailedException.class.getSimpleName()), + MSG); + } + } + case B14 -> // The handler throws PaymentError, which has its own cause TypeError. + check( + "B14", + m -> (in, ctx) -> failure(m, ctx, c -> { + throw new PaymentError(MSG, new TypeError("cause")); + }), + null, + "PaymentError", + MSG, + Map.of(), + null); + case B15 -> // The handler throws a user subclass of an SDK exception that has a cause. + check( + "B15", + m -> (in, ctx) -> failure(m, ctx, c -> { + throw new UserOperationError(MSG, new IOException("disk")); + }), + null, + "UserOperationError", + MSG, + Map.of(), + null); + case B16 -> // Java analogue: a handled step error, then a child context fails with an SDK error. + check( + "B16", + m -> (in, ctx) -> failure(m, ctx, c -> { + try { + c.step( + "validate", + String.class, + sc -> { + throw new IllegalArgumentException("x"); + }, + NO_RETRY); + } catch (IllegalArgumentException handled) { + // handled + } + return c.runInChildContext("child", String.class, cc -> { + throw new SerDesException("x"); + }); + }), + null, + "SerDesException", + "x", + Map.of("child", "SerDesException"), + "x"); + case B17 -> // An SDK-only failure with no user error: the step result cannot be serialized. + check( + "B17", + m -> (in, ctx) -> failure(m, ctx, c -> { + c.step("encode", Unserializable.class, sc -> new Unserializable(), NO_RETRY); + return "unreachable"; + }), + null, + "SerDesException", + null, // SDK-generated serialization message + Map.of("encode", "SerDesException"), + null); + case B18 -> // A class named Payment$Error is thrown from a step. + check( + "B18", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.step( + "charge", + String.class, + sc -> { + throw new Payment$Error(MSG); + }, + NO_RETRY)), + null, + "Error", + MSG, + Map.of("charge", "Error"), + MSG); + default -> fail("unhandled row " + row); + } + } + + /** + * Java {@code map} returns item failures as data ({@code MapResult.MapError}) rather than throwing, so the handler + * rethrows the failed item's user error type with its message; the execution error then propagates from the item. + */ + private static String mapAndRethrow(DurableContext ctx, boolean viaStep) { + var result = ctx.map("fanout", List.of("a"), String.class, (item, i, mc) -> { + if (viaStep) { + return mc.step( + "charge", + String.class, + sc -> { + throw new PaymentError(MSG); + }, + NO_RETRY); + } + throw new PaymentError(MSG); + }); + var error = result.getError(0); + assertNotNull(error, "map item failed"); + assertEquals(PaymentError.class.getName(), error.errorType()); + throw new PaymentError(error.errorMessage()); + } + + private static Consumer> failCallback(String errorType) { + return r -> { + r.run("in"); + r.failCallback( + r.getCallbackId("cb"), + ErrorObject.builder().errorType(errorType).errorMessage(MSG).build()); + assertEquals(ExecutionStatus.FAILED, r.runUntilComplete("in").getStatus()); + }; + } + + @ParameterizedTest + @EnumSource(Mode.class) + void callbackTimeoutMessages(Mode mode) { + var records = run( + (in, ctx) -> failure( + mode, ctx, c -> c.createCallback("cb", String.class).get()), + r -> { + r.run("in"); + r.timeoutCallback(r.getCallbackId("cb")); + r.runUntilComplete("in"); + }); + var rec = records.get(records.size() - 1); + assertTrue(rec.error.message().startsWith("Callback timed out: "), rec.error.message()); + var op = rec.operations().stream() + .filter(o -> "cb".equals(o.name())) + .findFirst() + .orElseThrow(); + assertNull(op.error().message(), "no checkpointed error, so no operation message"); + } +} diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java index 815278acf..d97505901 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java @@ -3,19 +3,28 @@ package software.amazon.lambda.durable.telemetry; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.CallbackDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.exception.CallbackFailedException; +import software.amazon.lambda.durable.exception.CallbackTimeoutException; +import software.amazon.lambda.durable.exception.CyclicWrapperForTest; import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.StepInterruptedException; @@ -163,18 +172,25 @@ void checkpointedErrorTypeAndLiveThrowableGiveTheSameName() { DurableExecutionTelemetry.toErrorInfo(wrapped(PaymentError.class.getName(), "x")) .name()); - // Anonymous and local classes: getSimpleName is empty / differs from the binary name, so both paths use the - // last segment of getName. + // Anonymous and local classes: Table A applied to Class.getName(), live and from the checkpoint, with no extra + // digit stripping (javac's binary names are ...ErrorNameTest$1 and ...ErrorNameTest$1LocalError). class LocalError extends RuntimeException {} Throwable anonymous = new RuntimeException() {}; - for (Throwable t : List.of(anonymous, new LocalError())) { + assertEquals("1", DurableExecutionTelemetry.toErrorInfo(anonymous).name()); + assertEquals( + "1LocalError", + DurableExecutionTelemetry.toErrorInfo(new LocalError()).name()); + for (Throwable t : List.of(anonymous, new LocalError(), new Payment$Error("x"))) { String fromLive = DurableExecutionTelemetry.toErrorInfo(t).name(); String fromCheckpoint = DurableExecutionTelemetry.toErrorInfo( wrapped(t.getClass().getName(), null)) .name(); assertEquals(fromCheckpoint, fromLive, t.getClass().getName()); - assertTrue(!fromLive.isEmpty() && !fromLive.contains("$") && !fromLive.contains("."), fromLive); } + // B18: a top-level class named Payment$Error gives "Error" both ways. + assertEquals( + "Error", + DurableExecutionTelemetry.toErrorInfo(new Payment$Error("x")).name()); // Plain type strings from other SDKs or service error codes. assertEquals("PaymentError", DurableExecutionTelemetry.simpleTypeName("PaymentError")); @@ -209,6 +225,173 @@ void sdkOnlyFailureUsesTheSdkExceptionSimpleName() { assertNotNull(interrupted.message()); } + /** B15: a user exception that subclasses an SDK exception is not an SDK wrapper and reports its own name. */ + public static class UserOperationError extends DurableOperationException { + public UserOperationError(String message, Throwable cause) { + super(Operation.builder().id("u").build(), null, message, cause); + } + } + + @Test + void userSubclassOfSdkWrapperReportsItsOwnName() { + var info = DurableExecutionTelemetry.toErrorInfo(new UserOperationError("mine", new IOException("disk"))); + assertEquals("UserOperationError", info.name()); + assertEquals("mine", info.message()); + assertFalse(DurableExecutionTelemetry.isSdkWrapper(new UserOperationError("mine", null))); + assertTrue(DurableExecutionTelemetry.isSdkWrapper(wrapped("x", "y"))); + // A callback timeout is an SDK error, never a wrapper. + assertFalse(DurableExecutionTelemetry.isSdkWrapper(new CallbackTimeoutException(callbackOp( + OperationStatus.TIMED_OUT, + ErrorObject.builder().errorType("States.Timeout").build())))); + } + + @Test + void userErrorWithCauseIsNotUnwrapped() { + // B14: a user error is never unwrapped, even if it has a cause. + assertEquals( + "PaymentErrorWithCause", + DurableExecutionTelemetry.toErrorInfo(new PaymentErrorWithCause(new IllegalStateException("inner"))) + .name()); + } + + static final class PaymentErrorWithCause extends RuntimeException { + PaymentErrorWithCause(Throwable cause) { + super("outer", cause); + } + } + + @Test + void nestedSdkWrappersAreFollowedToTheUserError() { + Throwable chain = new PaymentError("deep"); + for (int i = 0; i < 5; i++) { + chain = new UnrecoverableDurableExecutionException( + ErrorObject.builder() + .errorType("w" + i) + .errorMessage("w" + i) + .build(), + false, + chain); + } + var info = DurableExecutionTelemetry.toErrorInfo(chain); + assertEquals("PaymentError", info.name()); + assertEquals("w4", info.message(), "message is the outer record's message, unchanged"); + } + + @Test + void wrapperChainDepthIsBounded() { + Throwable chain = new PaymentError("deep"); + for (int i = 0; i < 40; i++) { + chain = new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorType("w" + i).errorMessage("w").build(), false, chain); + } + // Terminates; the walk stops after 16 levels at an SDK class name. + assertEquals( + "UnrecoverableDurableExecutionException", + DurableExecutionTelemetry.toErrorInfo(chain).name()); + } + + @Test + void cyclicWrapperChainTerminates() { + var a = new CyclicWrapperForTest("x.A"); + var b = new CyclicWrapperForTest("x.B"); + a.setOther(b); + b.setOther(a); + assertTrue(DurableExecutionTelemetry.isSdkWrapper(a)); + assertEquals( + "CyclicWrapperForTest", DurableExecutionTelemetry.toErrorInfo(a).name()); + } + + @Test + void blankCheckpointedTypeUsesTheSdkFailureClassOfTheOperation() { + // B13: the plugin snapshot hands the exporter a bare DurableOperationException; with a blank errorType the + // name is the SDK class the core throws for that operation, the same as the execution error. + for (String blank : new String[] {null, "", " "}) { + var error = ErrorObject.builder() + .errorType(blank) + .errorMessage("declined") + .build(); + var op = callbackOp(OperationStatus.FAILED, error); + assertEquals( + "CallbackFailedException", + DurableExecutionTelemetry.toErrorInfo(new DurableOperationException(op, error)) + .name()); + assertEquals( + "CallbackFailedException", + DurableExecutionTelemetry.toErrorInfo(new CallbackFailedException(op)) + .name()); + } + // B11: a timed-out callback is CallbackTimeoutException whatever its checkpoint says. + var timeoutError = ErrorObject.builder().errorType("States.Timeout").build(); + var timedOut = callbackOp(OperationStatus.TIMED_OUT, timeoutError); + assertEquals( + "CallbackTimeoutException", + DurableExecutionTelemetry.toErrorInfo(new DurableOperationException(timedOut, timeoutError)) + .name()); + assertEquals( + "CallbackTimeoutException", + DurableExecutionTelemetry.toErrorInfo(new CallbackTimeoutException(timedOut)) + .name()); + // No operation to map from and a base wrapper class: Table A's "Error", never the wrapper's name. + assertEquals( + "Error", + DurableExecutionTelemetry.toErrorInfo(wrapped(" ", "m")).name()); + assertEquals( + "Error", + DurableExecutionTelemetry.toErrorInfo(new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorMessage("m").build())) + .name()); + } + + @Test + void sdkFailureNamesMatchTheCoreExceptions() { + assertEquals( + "StepFailedException", + DurableExecutionTelemetry.sdkFailureName("STEP", "Step", OperationStatus.FAILED)); + assertEquals( + "WaitForConditionFailedException", + DurableExecutionTelemetry.sdkFailureName("STEP", "WaitForCondition", OperationStatus.FAILED)); + assertEquals( + "CallbackFailedException", + DurableExecutionTelemetry.sdkFailureName("CALLBACK", "Callback", OperationStatus.FAILED)); + assertEquals( + "CallbackTimeoutException", + DurableExecutionTelemetry.sdkFailureName("CALLBACK", "Callback", OperationStatus.TIMED_OUT)); + assertEquals( + "InvokeFailedException", + DurableExecutionTelemetry.sdkFailureName("CHAINED_INVOKE", "ChainedInvoke", OperationStatus.FAILED)); + assertEquals( + "InvokeTimedOutException", + DurableExecutionTelemetry.sdkFailureName("CHAINED_INVOKE", "ChainedInvoke", OperationStatus.TIMED_OUT)); + assertEquals( + "InvokeStoppedException", + DurableExecutionTelemetry.sdkFailureName("CHAINED_INVOKE", "ChainedInvoke", OperationStatus.STOPPED)); + assertEquals( + "MapIterationFailedException", + DurableExecutionTelemetry.sdkFailureName("CONTEXT", "MapIteration", OperationStatus.FAILED)); + assertEquals( + "ParallelBranchFailedException", + DurableExecutionTelemetry.sdkFailureName("CONTEXT", "ParallelBranch", OperationStatus.FAILED)); + assertEquals( + "ChildContextFailedException", + DurableExecutionTelemetry.sdkFailureName("CONTEXT", "RunInChildContext", OperationStatus.FAILED)); + assertNull(DurableExecutionTelemetry.sdkFailureName("CONTEXT", "WaitForCallback", OperationStatus.FAILED)); + assertNull(DurableExecutionTelemetry.sdkFailureName("STEP", "Step", OperationStatus.SUCCEEDED)); + assertNull(DurableExecutionTelemetry.sdkFailureName("WAIT", "Wait", OperationStatus.FAILED)); + assertNull(DurableExecutionTelemetry.sdkFailureName(null, null, null)); + } + + private static Operation callbackOp(OperationStatus status, ErrorObject error) { + return Operation.builder() + .id("cb-1") + .type(OperationType.CALLBACK) + .status(status) + .callbackDetails(CallbackDetails.builder() + .callbackId("cb-id") + .error(error) + .build()) + .build(); + } + private static DurableOperationException wrapped(String errorType, String message) { return new DurableOperationException( Operation.builder().id("op-1").build(), diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java new file mode 100644 index 000000000..c6d5018c8 --- /dev/null +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java @@ -0,0 +1,86 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.exception.DurableOperationException; + +/** + * Table A of the cross-SDK {@code error.name} rule. The vectors are loaded from {@code error-name-vectors.json}, which + * has the same content in the JS, Python and Java repositories; edit it in all three or not at all. + */ +class ErrorNameVectorsTest { + + private static final String VECTORS = "/error-name-vectors.json"; + + static Stream reduceVectors() throws IOException { + List out = new ArrayList<>(); + for (JsonNode v : load().get("reduce")) { + JsonNode input = v.get("input"); + out.add(Arguments.of( + v.get("id").asText(), + input.isNull() ? null : input.asText(), + v.get("expected").asText())); + } + return out.stream(); + } + + private static JsonNode load() throws IOException { + try (InputStream in = ErrorNameVectorsTest.class.getResourceAsStream(VECTORS)) { + assertNotNull(in, VECTORS + " on the test classpath"); + return new ObjectMapper().readTree(in); + } + } + + @Test + void vectorFileHasTheSharedShape() throws IOException { + JsonNode root = load(); + assertEquals(1, root.get("version").asInt()); + assertTrue(root.get("reduce").size() >= 15, "all Table A rows present"); + } + + @ParameterizedTest(name = "{0}: [{1}] -> [{2}]") + @MethodSource("reduceVectors") + void reducesTypeString(String id, String input, String expected) { + assertEquals(expected, DurableExecutionTelemetry.simpleTypeName(input), id); + } + + /** + * The same vectors through the record path: a checkpointed operation error whose {@code errorType} is the input. + * The operation type is one with no SDK failure class of its own, so a blank type falls through to Table A. + */ + @ParameterizedTest(name = "{0}: checkpointed [{1}] -> [{2}]") + @MethodSource("reduceVectors") + void checkpointedErrorTypeUsesTheSameReduction(String id, String input, String expected) { + var op = Operation.builder() + .id("op-1") + .type(OperationType.WAIT) + .status(OperationStatus.FAILED) + .build(); + var error = ErrorObject.builder() + .errorType(input) + .errorMessage("card declined") + .build(); + var info = DurableExecutionTelemetry.toErrorInfo(new DurableOperationException(op, error)); + assertEquals(expected, info.name(), id); + assertEquals("card declined", info.message(), "message unchanged"); + } +} diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/OperationErrorIdentityTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/OperationErrorIdentityTest.java index 2cf759857..affd91360 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/OperationErrorIdentityTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/OperationErrorIdentityTest.java @@ -108,7 +108,8 @@ void fallsBackToThrowableFieldsWhenErrorObjectFieldsMissing() { ARN, START); - // ErrorObject present but errorType null: name falls back to the throwable's simple class name. + // ErrorObject present but errorType null, and an operation with no SDK failure class: the SDK wrapper is never + // the name (rule 1), so a missing type gives "Error" (Table A, A12). The message still falls back. ErrorObject partial = ErrorObject.builder().errorMessage("only a message").build(); Throwable opError = @@ -117,7 +118,7 @@ void fallsBackToThrowableFieldsWhenErrorObjectFieldsMissing() { "req", ARN, true, START, failedOps(opError), InvocationStatus.FAILED, opError, "in", null)); var rec = exporter.records.get(0); - assertEquals("DurableOperationException", rec.error.name(), "null errorType falls back to throwable class"); + assertEquals("Error", rec.error.name(), "null errorType gives Error, not the wrapper class"); assertEquals("only a message", rec.error.message()); } } diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/Payment$Error.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/Payment$Error.java new file mode 100644 index 000000000..2b7d38972 --- /dev/null +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/Payment$Error.java @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.telemetry; + +/** + * A top-level user error whose declared name contains {@code '$'}, which is legal Java. Its binary name is + * {@code software.amazon.lambda.durable.telemetry.Payment$Error}, so Table A reduces it to {@code Error} (row B18). + */ +public class Payment$Error extends RuntimeException { + public Payment$Error(String message) { + super(message); + } +} diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/UnrecoverableErrorUnwrapTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/UnrecoverableErrorUnwrapTest.java index 6972ef255..8af9530b2 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/UnrecoverableErrorUnwrapTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/UnrecoverableErrorUnwrapTest.java @@ -137,10 +137,7 @@ void fallsBackToThrowableFieldsWhenUnrecoverableErrorTypeMissing() { "req", ARN, true, START, ops(OperationStatus.FAILED), InvocationStatus.FAILED, execError, "in", null)); var rec = exporter.records.get(0); - assertEquals( - "UnrecoverableDurableExecutionException", - rec.error.name(), - "null errorType falls back to throwable class"); + assertEquals("Error", rec.error.name(), "null errorType gives Error, not the wrapper class (A12)"); assertEquals("only a message", rec.error.message()); } } diff --git a/telemetry-exporter/src/test/resources/error-name-vectors.json b/telemetry-exporter/src/test/resources/error-name-vectors.json new file mode 100644 index 000000000..1a0f2d8df --- /dev/null +++ b/telemetry-exporter/src/test/resources/error-name-vectors.json @@ -0,0 +1,21 @@ +{ + "description": "Durable Execution Telemetry error.name reduction vectors. Input is a type string; expected is the reported error.name. Shared across the JS, Python and Java exporters. Do not edit in one repo only.", + "version": 1, + "reduce": [ + { "id": "A1", "input": "PaymentError", "expected": "PaymentError" }, + { "id": "A2", "input": "com.acme.PaymentError", "expected": "PaymentError" }, + { "id": "A3", "input": "com.acme.Outer$PaymentError", "expected": "PaymentError" }, + { "id": "A4", "input": "a.b.Outer$Inner$PaymentError", "expected": "PaymentError" }, + { "id": "A5", "input": "__main__.PaymentError", "expected": "PaymentError" }, + { "id": "A6", "input": "States.Timeout", "expected": "Timeout" }, + { "id": "A7", "input": "trailing.", "expected": "trailing." }, + { "id": "A8", "input": "Outer$", "expected": "Outer$" }, + { "id": "A9a", "input": ".", "expected": "." }, + { "id": "A9b", "input": "$", "expected": "$" }, + { "id": "A10", "input": "pkg.Foo$1Local", "expected": "1Local" }, + { "id": "A11", "input": " PaymentError ", "expected": " PaymentError " }, + { "id": "A12a", "input": "", "expected": "Error" }, + { "id": "A12b", "input": " ", "expected": "Error" }, + { "id": "A12c", "input": null, "expected": "Error" } + ] +} From 5f281eefad0eed575b4399689925015e58ca3e61 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 21:41:46 -0700 Subject: [PATCH 03/12] docs(telemetry)!: state that includeErrors(false) keeps the execution error includeErrors(false) drops only per-operation errors, as in the JS exporter; the execution-level error and its message are always emitted. Document this in the ContentConfig javadoc, say that no setting hides the execution error message, and tell users to keep secrets out of exception messages. Rename the ErrorPrivacyGateTest constants and tests so they describe this intended behaviour instead of a "secret" that leaks. BREAKING CHANGE: in 2.3.0, includeErrors(false) also suppressed the execution-level error. It no longer does; FAILED records carry the execution error name and message whatever the setting. --- .../durable/telemetry/ContentConfig.java | 24 ++++++++++--- .../telemetry/ErrorPrivacyGateTest.java | 34 ++++++++++++------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java index c92327163..a3b3bd2c9 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/ContentConfig.java @@ -12,10 +12,18 @@ * *

Mirrors the JS {@code ContentConfig}. {@code input}/{@code output} may be included as-is, excluded, or * transformed; all three are honored — the plugin reads execution input from {@code InvocationInfo.executionInput()} - * and output from {@code InvocationEndInfo.executionResult()}. {@code includeErrors} gates only the per-operation - * errors, as in JS: with {@code includeErrors(false)} operation errors are omitted and the execution-level error is - * kept. Per-operation result opt-in is honored via {@link OperationOverride#withResult} (see - * {@code OperationChangeItemInfo.result()}). + * and output from {@code InvocationEndInfo.executionResult()}. Per-operation result opt-in is honored via + * {@link OperationOverride#withResult} (see {@code OperationChangeItemInfo.result()}). + * + *

{@code includeErrors} gates only the per-operation errors ({@code operations[].error} and + * {@code operationsByName..error}), as in the JS exporter. With {@code includeErrors(false)} the + * execution-level {@code error}, including its message, is still emitted on every FAILED record. In Java the + * execution error is usually the failed operation's own exception rethrown, so its message is usually the operation's + * message too. Earlier releases of this exporter also suppressed the execution error; that is no longer the case. No + * setting suppresses the execution error message, and the input/output transforms do not apply to errors. Do not put + * secrets or personal data in exception messages that can reach the handler's return path: catch such an exception and + * rethrow one with a safe message, or, for a custom {@link TelemetryExporter}, remove or redact {@code error} in its + * {@link TelemetryExporter#render render}. * *

Transform value shape. The {@code inputTransform} and {@code outputTransform} functions receive a * detached, JSON-compatible copy of the value, never the SDK's original Java object: a POJO is presented as a @@ -101,6 +109,14 @@ public Builder outputTransform(Function transform) { return this; } + /** + * Whether per-operation errors ({@code operations[].error}, {@code operationsByName..error}) are emitted. + * Default {@code true}. {@code false} does not hide the execution-level {@code error} or its message; + * see the class documentation. + * + * @param include {@code false} to omit per-operation errors + * @return this builder + */ public Builder includeErrors(boolean include) { this.includeErrors = include; return this; diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java index 60b3b5923..5299895b7 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorPrivacyGateTest.java @@ -24,14 +24,18 @@ * Error gate. {@code ContentConfig.includeErrors(false)} suppresses every operation-level error — in the canonical * {@code operations} array rendering and the {@code operationsByName} rendering — and keeps the execution-level * {@code error}, matching the JS exporter. + * + *

This is intended: {@code includeErrors(false)} is not a privacy control for the execution error. Its message is + * emitted whatever the setting, so it must not carry secrets (see the {@link ContentConfig} documentation). Earlier + * releases also suppressed the execution error. */ class ErrorPrivacyGateTest { private static final String ARN = "arn:aws:lambda:us-west-2:1:function:f:$LATEST/durable-execution/exec-1/invocation-1"; private static final Instant START = Instant.parse("2026-08-05T00:00:00Z"); - private static final String EXEC_SECRET = "exec-secret-ssn-123-45-6789"; - private static final String OP_SECRET = "op-secret-token-abcdef"; + private static final String EXEC_MESSAGE = "execution failed: card declined"; + private static final String OP_MESSAGE = "operation failed: token rejected"; private static final class CapturingExporter implements TelemetryExporter { final List records = new ArrayList<>(); @@ -57,7 +61,7 @@ private Map failingOp() { OperationStatus.FAILED, 1, false, - new RuntimeException(OP_SECRET), + new RuntimeException(OP_MESSAGE), null)); return m; } @@ -80,7 +84,7 @@ private DurableExecutionTelemetryRecord runFailedExecution(boolean includeErrors START, failingOp(), InvocationStatus.FAILED, - new RuntimeException(EXEC_SECRET), + new RuntimeException(EXEC_MESSAGE), "in", null)); assertEquals(1, exporter.records.size()); @@ -88,22 +92,26 @@ private DurableExecutionTelemetryRecord runFailedExecution(boolean includeErrors } @Test - void includeErrorsFalseDropsOperationErrorsAndKeepsExecutionError() { + void includeErrorsFalseDropsOnlyOperationErrorsAndStillEmitsTheExecutionErrorMessage() { var exporter = new CapturingExporter(); var rec = runFailedExecution(false, exporter); assertEquals("FAILED", rec.status()); assertNotNull(rec.error, "execution-level error kept with includeErrors:false (as in JS)"); assertEquals("RuntimeException", rec.error.name()); - assertEquals(EXEC_SECRET, rec.error.message()); + assertEquals(EXEC_MESSAGE, rec.error.message()); assertNull(rec.operations().get(0).error(), "operation-level error suppressed by includeErrors:false"); String arrayJson = Json.stringify(rec.toWireMap()); String byNameJson = Json.stringify(rec.toByNameWireMap()); - assertTrue(arrayJson.contains(EXEC_SECRET), "execution error in the operations-array JSON"); - assertFalse(arrayJson.contains(OP_SECRET), "no operation error in the operations-array JSON"); - assertTrue(byNameJson.contains(EXEC_SECRET), "execution error in the operationsByName JSON"); - assertFalse(byNameJson.contains(OP_SECRET), "no operation error in the operationsByName JSON"); + assertTrue( + arrayJson.contains(EXEC_MESSAGE), + "execution error message emitted in the operations-array JSON despite includeErrors:false"); + assertFalse(arrayJson.contains(OP_MESSAGE), "no operation error in the operations-array JSON"); + assertTrue( + byNameJson.contains(EXEC_MESSAGE), + "execution error message emitted in the operationsByName JSON despite includeErrors:false"); + assertFalse(byNameJson.contains(OP_MESSAGE), "no operation error in the operationsByName JSON"); @SuppressWarnings("unchecked") var byName = (Map) rec.toByNameWireMap().get("operationsByName"); @SuppressWarnings("unchecked") @@ -112,13 +120,13 @@ void includeErrorsFalseDropsOperationErrorsAndKeepsExecutionError() { } @Test - void includeErrorsTrueKeepsBothErrors() { + void includeErrorsTrueEmitsExecutionAndOperationErrors() { var exporter = new CapturingExporter(); var rec = runFailedExecution(true, exporter); assertNotNull(rec.error, "execution-level error present when includeErrors:true"); - assertEquals(EXEC_SECRET, rec.error.message()); + assertEquals(EXEC_MESSAGE, rec.error.message()); assertNotNull(rec.operations().get(0).error(), "operation-level error present when includeErrors:true"); - assertEquals(OP_SECRET, rec.operations().get(0).error().message()); + assertEquals(OP_MESSAGE, rec.operations().get(0).error().message()); } } From 30fff3b3c9b00305634fc0aad01eb4da7fb54361 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 21:41:46 -0700 Subject: [PATCH 04/12] docs(telemetry): document the error.name rule and LambdaLogExporter line size - README: add the shared error.name rule in the same words as the JS and Python exporters, with the Java specifics; document the includeErrors change. - LambdaLogExporter: the printed line is the record plus 68 bytes plus one byte per quote or backslash, so up to 2 x maxRecordSizeBytes + 68; this is within the 1 MB CloudWatch Logs event limit. Replace the stale 256 KB comment and the "about 70 bytes" claim. Note that the INFO lines are dropped at an application log level of WARN or above. - LambdaLogExporterTest: assert the exact line size instead of the always-true "line > record", and check an escape-heavy record near the default limit. --- telemetry-exporter/README.md | 69 ++++++++++++++++--- .../exporters/LambdaLogExporter.java | 17 ++++- .../durable/telemetry/RecordFactory.java | 7 ++ .../exporters/LambdaLogExporterTest.java | 40 ++++++++++- 4 files changed, 120 insertions(+), 13 deletions(-) diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index 002a8a4d1..ca8833a23 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -191,20 +191,27 @@ one event per record to `{logStreamPrefix}YYYY/MM/DD` (IAM: `logs:CreateLogStrea the cached input snapshot or any later emission), and a transform that throws omits the field and logs the failure rather than silently dropping it or failing the execution. - **`includeErrors`** gates only the per-operation errors (`operations[].error` and - `operationsByName..error`), as in the JS exporter. With `includeErrors(false)` the - execution-level `error` is still emitted, so a FAILED record keeps its failure reason. -- **`error.name`** is the simple type name of the error the user's code threw (for example - `PaymentError`): no package, no enclosing-class `$` prefix. The same rule applies to the - execution `error`, `operations[].error` and `operationsByName..error`, and a failure gets - the same name whether it happens live or is replayed from a checkpoint (the last segment of the - checkpointed `errorType`). When no user error is involved, the SDK exception's simple name is - used. `error.message` is unchanged. + `operationsByName..error`), as in the JS exporter. **`includeErrors(false)` does not hide + the execution-level `error`**: its `name` and `message` are emitted on every FAILED record. In + 2.3.0 and earlier, `includeErrors(false)` also suppressed the execution error. + In Java the execution error is usually the failed operation's own exception rethrown, so its + message is usually the operation's message too. No setting suppresses the execution error + message, and the input/output transforms do not apply to errors. Do not put secrets or personal + data in exception messages that can reach the handler's return path: catch such an exception and + rethrow one with a safe message, or, in a custom `TelemetryExporter`, remove or redact `error` in + `render`. +- **`error.name`** follows the cross-SDK rule in [`error.name`](#errorname) below. - **`LambdaLogExporter` line shape.** Each record is printed to stdout as one Lambda JSON log line, `{"timestamp":"Z","level":"INFO","message":""}`, because the JS readers of this destination filter on `level` and parse `message`. The Node runtime adds this envelope around `console.log`; Java `System.out` is not wrapped, and Lambda does not re-wrap a line that is already JSON. `maxRecordSizeBytes` is measured on the record - (`render`), not on the envelope, which adds escaping plus about 70 bytes. + (`render`), not on the printed line. The record is embedded in `message` as a JSON string, so + every `"` and `\` in it is escaped again: the line is the record plus 68 bytes plus one byte + per quote or backslash in the record, and can be up to about twice the record size + (`2 × maxRecordSizeBytes + 68` in the worst case, about 512 KB with the default 256,000). That + is within the CloudWatch Logs limit of 1 MB per log event. + `LambdaLogExporter` lines are INFO; an application log level of WARN or above drops them. - **Non-fatal plugin failures are isolated.** Record construction, snapshotting, transforms, truncation and exporter render/export/flush contain and log ordinary failures, including `AssertionError` and optional-dependency `NoClassDefFoundError`. `VirtualMachineError` and @@ -229,6 +236,50 @@ one event per record to `{logStreamPrefix}YYYY/MM/DD` (IAM: `logs:CreateLogStrea Concurrent flush requests may share one flush, and the final record is delivered before the invocation returns. +## `error.name` + +This rule is shared by the JS, Python and Java exporters. `error.message` is never changed by +this rule. The rule applies to the execution-level `error`, every `operations[].error`, and every +`operationsByName..error`. + +1. A record names the error that caused the failure. A wrapper error that the SDK added is never + the name. +2. The SDK unwraps an error only when the error's type is one of the SDK's own wrapper types. A + user error is never unwrapped, even if it has a cause. +3. Unwrapping follows the whole chain of SDK wrappers, up to a depth of 16. A cycle stops the walk. +4. If the chain ends at a user error, the name is that error's type name. +5. If the chain ends at an SDK error that is not a wrapper, the name is that SDK error's simple + class name. A callback timeout is an example. +6. A type name that is a qualified string is reduced to its last segment after any `.` or `$`. If + that segment is empty, the string is kept as it is. +7. A failure gets the same name whether the record is built live or from a replayed checkpoint. +8. When the execution fails because an operation's error propagated, the execution `error.name` + equals that operation's `error.name`. + +User error types have the same name in every SDK, because the user names them. SDK error types +keep each language's own class name. For example, a callback timeout is `CallbackTimeoutError` in +JS and Python and `CallbackTimeoutException` in Java. Every SDK name for a timeout contains +`Timeout`, so a `/Timeout/` query keeps working. + +How the Java exporter applies it: + +- Rule 6 is applied to `Class.getName()` for live errors and to the checkpointed `errorType` for + replayed ones, so both give the same name. `com.acme.Outer$PaymentError` gives `PaymentError`; + a top-level class named `Payment$Error` gives `Error`; a local class `LocalError` declared in + `pkg.Foo` (binary name `pkg.Foo$1LocalError`) gives `1LocalError`; an anonymous class gives its + last segment, for example `1`. The string is not trimmed. A missing, empty or blank type gives + `Error`. A string that ends with a separator (`trailing.`, `Outer$`) is kept as it is. +- SDK wrappers are the subclasses of `DurableOperationException` and + `UnrecoverableDurableExecutionException` declared in `software.amazon.lambda.durable.exception`. + A user exception that subclasses one of them is not a wrapper and reports its own name. + `CallbackTimeoutException` is never unwrapped. +- When an operation failed with no error type (for example a callback failed externally without an + `errorType`, or a callback timed out), the operation and the execution report the SDK exception + the core throws for it, such as `CallbackFailedException` or `CallbackTimeoutException`. If the + checkpoint has no error at all, the operation error has that name and a `null` message. +- Non-class type strings are reduced too: an external callback failure with `errorType` + `States.Timeout` reports `Timeout`. + ## Conformance Validated against the Durable Execution Telemetry conformance suite behaviors `telemetry-1 … telemetry-18` diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java index b64ba2e6b..6091f12cf 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java @@ -20,8 +20,16 @@ *

The line is the Lambda JSON log envelope {@code {"timestamp", "level": "INFO", "message"}} whose {@code message} * is the compact record JSON. Readers of this destination filter on {@code level} and parse {@code message}. The Node * runtime adds that envelope around {@code console.log}, but Java {@code System.out} is not wrapped, so this exporter - * writes it itself; Lambda does not re-wrap a line that is already JSON. {@link #maxRecordSizeBytes()} is measured on - * the record ({@link #render}), not on the envelope. + * writes it itself; Lambda does not re-wrap a line that is already JSON. + * + *

Line size. {@link #maxRecordSizeBytes()} is measured on the record ({@link #render}), not on the printed + * line. The record is embedded in {@code message} as a JSON string, so every {@code "} and {@code \} in it is escaped + * again: the line is the record plus 68 bytes of envelope plus one byte per quote or backslash in the record, so it can + * be up to about twice the record size ({@code 2 × maxRecordSizeBytes + 68} in the worst case). With the default + * 256,000-byte limit that is about 512 KB, within the CloudWatch Logs limit of 1 MB per log event. + * + *

Lines are written with {@code "level":"INFO"}. With the Lambda JSON log format and an application log level of + * {@code WARN} or above, Lambda drops them. */ @Experimental public final class LambdaLogExporter implements TelemetryExporter { @@ -29,7 +37,10 @@ public final class LambdaLogExporter implements TelemetryExporter { private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC); - /** CloudWatch Logs caps a single log event at 256 KB. */ + /** + * Budget for the rendered record, not the printed line (see the class documentation). The default 256,000 keeps the + * worst-case escaped line (about 512 KB) under the CloudWatch Logs limit of 1 MB per log event. + */ private final Integer maxRecordSizeBytes; public LambdaLogExporter() { diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/RecordFactory.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/RecordFactory.java index fc116c2c6..29ed96204 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/RecordFactory.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/RecordFactory.java @@ -18,4 +18,11 @@ public static DurableExecutionTelemetryRecord sample() { new OperationRecord().id("op-1").name("fetch-user").type("STEP").status("SUCCEEDED")); return r; } + + /** {@link #sample()} with the given execution {@code input}. */ + public static DurableExecutionTelemetryRecord sampleWithInput(Object input) { + DurableExecutionTelemetryRecord r = sample(); + r.input = input; + return r; + } } diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java index ce673a55c..dce0c4771 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java @@ -94,6 +94,44 @@ var record = RecordFactory.sample(); assertSame(record, shaped); assertFalse(shaped.toWireMap().containsKey("truncated")); String line = exportAndCapture(shaped).strip(); - assertTrue(line.getBytes(StandardCharsets.UTF_8).length > recordBytes, line); + String recordJson = Json.stringify(exporter.render(shaped)); + assertEquals( + recordBytes + ENVELOPE_BYTES + escapedChars(recordJson), + line.getBytes(StandardCharsets.UTF_8).length, + "line = record + 68-byte envelope + one byte per quote or backslash in the record"); + } + + /** Bytes the envelope adds around an empty {@code message}. */ + private static final int ENVELOPE_BYTES = 68; + + private static long escapedChars(String json) { + return json.chars().filter(c -> c == '"' || c == '\\').count(); + } + + @Test + void envelopeAddsSixtyEightBytes() { + String empty = LambdaLogExporter.logLine("", Instant.parse("2026-10-21T08:00:00Z")); + assertEquals(ENVELOPE_BYTES, empty.getBytes(StandardCharsets.UTF_8).length, empty); + } + + /** + * An escape-heavy record just under the default 256,000-byte budget is not truncated, and its printed line is about + * twice the record: at most {@code 2 × maxRecordSizeBytes + 68}, and under the 1 MB CloudWatch Logs event limit. + */ + @Test + void escapeHeavyRecordNearTheLimitDoublesButStaysUnderOneMegabyte() { + var exporter = new LambdaLogExporter(); + int limit = exporter.maxRecordSizeBytes(); + for (char ch : new char[] {'"', '\\'}) { + var record = RecordFactory.sampleWithInput(String.valueOf(ch).repeat(127_000)); + int recordBytes = Json.byteSize(exporter.render(record)); + assertTrue(recordBytes <= limit && recordBytes > limit - 2_000, "record near the limit: " + recordBytes); + var shaped = Truncation.truncateRecord(record, limit, exporter::render); + assertSame(record, shaped, "not truncated"); + int lineBytes = exportAndCapture(shaped).strip().getBytes(StandardCharsets.UTF_8).length; + assertTrue(lineBytes > 2 * recordBytes - 1_000, "line about twice the record: " + lineBytes); + assertTrue(lineBytes <= 2 * limit + ENVELOPE_BYTES, "line within 2 x limit + 68: " + lineBytes); + assertTrue(lineBytes < 1_048_576, "line under the 1 MB CloudWatch Logs event limit: " + lineBytes); + } } } From 25d0c4c2a55540058a0ed8984dd35222a2fd4bff Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 22:40:13 -0700 Subject: [PATCH 05/12] fix(telemetry): apply spec v2 blank, depth, cycle and wrapper rules - Rule 6: blank means only space, \t, \n, \r, \f and \u000B, instead of String.isBlank(). U+001C to U+001F and Unicode spaces are kept. - Rule 3: the depth counts unwrap steps. At most 16 unwraps happen; if the error reached after them is still a wrapper, that wrapper's name (its checkpointed errorType) is reported. 16 wrappers give the user error, 17 and 18 give wrapper 17. A cycle stops at the first repeat and reports the last error visited before it, for two-cycles and self-cycles alike. - Wrappers are an explicit list matched on the exact runtime class, not "anything in the SDK exception package". StepInterruptedException, IllegalDurableOperationException, NonDeterministicExecutionException, InvokeTimedOutException and InvokeStoppedException are SDK errors. A timed-out or stopped invoke reports InvokeTimedOutException or InvokeStoppedException on the operation and the execution, whatever error type the checkpoint carries. - A hidden class name is cut at the first '/' before the reduction, live and from the checkpoint, so the JVM address suffix never reaches error.name. - Tests: 1/15/16/17/18/40 wrappers, two-cycle, self-cycle and a cycle after a prefix with exact names; a test that lists every class in the SDK exception package from the SDK jar and fails on an unclassified one; a hidden-class test; the blank characters. The test-only CyclicWrapperForTest in the SDK package is removed. --- .../telemetry/DurableExecutionTelemetry.java | 180 ++++++++++---- .../exception/CyclicWrapperForTest.java | 26 -- .../durable/telemetry/ErrorNameTest.java | 230 ++++++++++++++++-- .../durable/telemetry/HiddenProbeError.java | 13 + 4 files changed, 364 insertions(+), 85 deletions(-) delete mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/exception/CyclicWrapperForTest.java create mode 100644 telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/HiddenProbeError.java diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java index 06ffb0a0c..8ab8d9d0a 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java @@ -3,8 +3,11 @@ package software.amazon.lambda.durable.telemetry; import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.Set; import java.util.function.Function; +import java.util.function.UnaryOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.lambda.model.ErrorObject; @@ -13,17 +16,23 @@ import software.amazon.lambda.durable.annotations.Experimental; import software.amazon.lambda.durable.exception.CallbackException; import software.amazon.lambda.durable.exception.CallbackFailedException; +import software.amazon.lambda.durable.exception.CallbackSubmitterException; import software.amazon.lambda.durable.exception.CallbackTimeoutException; import software.amazon.lambda.durable.exception.ChildContextFailedException; +import software.amazon.lambda.durable.exception.DurableExecutionException; import software.amazon.lambda.durable.exception.DurableOperationException; +import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.InvokeException; import software.amazon.lambda.durable.exception.InvokeFailedException; import software.amazon.lambda.durable.exception.InvokeStoppedException; import software.amazon.lambda.durable.exception.InvokeTimedOutException; import software.amazon.lambda.durable.exception.MapIterationFailedException; +import software.amazon.lambda.durable.exception.NonDeterministicExecutionException; import software.amazon.lambda.durable.exception.ParallelBranchFailedException; +import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.exception.StepException; import software.amazon.lambda.durable.exception.StepFailedException; +import software.amazon.lambda.durable.exception.StepInterruptedException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; import software.amazon.lambda.durable.exception.WaitForConditionFailedException; import software.amazon.lambda.durable.model.OperationSubType; @@ -184,8 +193,39 @@ static ErrorInfo toErrorInfo(Throwable t) { return new ErrorInfo(errorName(t), message); } - /** Package that declares the SDK's own exception types; only classes declared here can be SDK wrappers. */ - private static final String SDK_EXCEPTION_PACKAGE = DurableOperationException.class.getPackageName(); + /** + * The SDK's own wrapper types, matched on the exact runtime class. Only these are unwrapped (rule 2). A subclass, + * including a user subclass of one of them, is not a wrapper. Every class in the SDK exception package is either + * here or in {@link #SDK_ERRORS}; a test fails when a new class is in neither. + */ + static final Set> SDK_WRAPPERS = Set.of( + DurableOperationException.class, + UnrecoverableDurableExecutionException.class, + StepException.class, + CallbackException.class, + InvokeException.class, + StepFailedException.class, + CallbackFailedException.class, + CallbackSubmitterException.class, + ChildContextFailedException.class, + InvokeFailedException.class, + MapIterationFailedException.class, + ParallelBranchFailedException.class, + WaitForConditionFailedException.class); + + /** + * SDK exceptions that are not wrappers: they name a failure themselves and report their simple class name (rule 5). + * Kept for the classification test; the walk needs only {@link #SDK_WRAPPERS}. + */ + static final Set> SDK_ERRORS = Set.of( + DurableExecutionException.class, + SerDesException.class, + CallbackTimeoutException.class, + InvokeTimedOutException.class, + InvokeStoppedException.class, + StepInterruptedException.class, + IllegalDurableOperationException.class, + NonDeterministicExecutionException.class); /** SDK base classes that never name a failure themselves: with nothing to unwrap they report {@code Error}. */ private static final Set> SDK_BASE_WRAPPERS = Set.of( @@ -195,66 +235,74 @@ static ErrorInfo toErrorInfo(Throwable t) { CallbackException.class, InvokeException.class); - /** Reported name when a type string is missing or blank (Table A, A12). */ + /** Reported name when a type string is missing or blank (Table A, A12 and A13). */ static final String DEFAULT_ERROR_NAME = "Error"; - private static final int MAX_WRAPPER_DEPTH = 16; + /** Maximum number of unwrap steps (rule 3). */ + static final int MAX_UNWRAPS = 16; /** * Returns the cross-SDK {@code error.name} of a throwable. * *

    *
  1. A throwable that is not an SDK wrapper (see {@link #isSdkWrapper}) reports its own class name, reduced by - * {@link #simpleTypeName}. This includes user errors that have a cause and user subclasses of SDK exceptions. - *
  2. An SDK wrapper with a live cause is unwrapped, following the chain of SDK wrappers up to a depth of - * {@value #MAX_WRAPPER_DEPTH}; the depth bound also stops a cyclic chain. - *
  3. An SDK wrapper with no cause reports its checkpointed {@code errorType}, reduced. The core writes that type - * as {@link Class#getName()}, so a replayed failure gets the same name as the live one. - *
  4. An SDK wrapper whose {@code errorType} is blank reports the SDK failure class the core throws for its - * operation (for example {@code CallbackFailedException}), so the operation and the execution agree. - *
  5. A timed-out callback is never unwrapped: it reports {@code CallbackTimeoutException}. + * {@link #simpleTypeName}. This includes user errors that have a cause, user subclasses of SDK exceptions and + * the SDK errors that are not wrappers, such as {@code CallbackTimeoutException}. + *
  6. A wrapper whose operation timed out (callback or invoke) or was stopped (invoke) reports the SDK error the + * core throws for it: {@code CallbackTimeoutException}, {@code InvokeTimedOutException} or + * {@code InvokeStoppedException}. These are SDK errors, not wrappers, so live and replayed records agree. + *
  7. A wrapper with a live cause is unwrapped. At most {@value #MAX_UNWRAPS} unwraps happen; if the error + * reached after them is still a wrapper, that wrapper's name is reported. If the cause is an error already + * visited, the walk stops and the last error visited before the repeat is reported. + *
  8. A wrapper's name is its checkpointed {@code errorType}, reduced. The core writes that type as + * {@link Class#getName()}, so a replayed failure gets the same name as the live one. + *
  9. A wrapper whose {@code errorType} is blank reports the SDK failure class the core throws for its operation + * (for example {@code CallbackFailedException}), so the operation and the execution agree. *
*/ static String errorName(Throwable t) { + return errorName(t, Throwable::getCause); + } + + /** + * {@link #errorName(Throwable)} with the cause lookup supplied, so a test can give the walk a cycle: the JDK's + * {@link Throwable#getCause()} never returns the throwable itself, and the SDK wrappers' causes are fixed at + * construction. + */ + static String errorName(Throwable t, UnaryOperator causeOf) { + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); Throwable current = t; - for (int depth = 0; ; depth++) { + for (int unwraps = 0; ; unwraps++) { if (!isSdkWrapper(current)) { - return simpleTypeName(current.getClass().getName()); + return javaTypeName(current.getClass().getName()); } - String timeout = callbackTimeoutName(current); - if (timeout != null) { - return timeout; + String sdkError = sdkErrorOfOperation(current); + if (sdkError != null) { + return sdkError; } - Throwable cause = current.getCause(); - if (cause != null && cause != current) { - if (depth >= MAX_WRAPPER_DEPTH) { - return simpleTypeName(cause.getClass().getName()); - } - current = cause; - continue; + visited.add(current); + Throwable cause = causeOf.apply(current); + if (cause == null || unwraps == MAX_UNWRAPS || visited.contains(cause)) { + return wrapperName(current); } - return wrapperName(current); + current = cause; } } /** - * {@code true} when {@code t} is one of the SDK's own wrapper types: an instance of - * {@link DurableOperationException} or {@link UnrecoverableDurableExecutionException} whose runtime class is - * declared in the SDK's exception package. A user subclass of an SDK exception is not a wrapper. A callback timeout - * is an SDK error, not a wrapper. + * {@code true} when the runtime class of {@code t} is exactly one of {@link #SDK_WRAPPERS}. A user subclass of an + * SDK exception is not a wrapper, and neither is an SDK error such as a callback timeout. */ static boolean isSdkWrapper(Throwable t) { - return (t instanceof DurableOperationException || t instanceof UnrecoverableDurableExecutionException) - && !(t instanceof CallbackTimeoutException) - && SDK_EXCEPTION_PACKAGE.equals(t.getClass().getPackageName()); + return t != null && SDK_WRAPPERS.contains(t.getClass()); } - /** Name of an SDK wrapper that has no cause to follow. */ + /** Name of an SDK wrapper whose cause is not followed. */ private static String wrapperName(Throwable wrapper) { ErrorObject error = extractErrorObject(wrapper); String type = error == null ? null : error.errorType(); - if (type != null && !type.isBlank()) { - return simpleTypeName(type); + if (!isBlankType(type)) { + return javaTypeName(type); } if (wrapper instanceof DurableOperationException doe && doe.getOperation() != null) { var op = doe.getOperation(); @@ -269,14 +317,24 @@ private static String wrapperName(Throwable wrapper) { return DEFAULT_ERROR_NAME; } - /** {@code CallbackTimeoutException} for a wrapper whose operation is a timed-out callback, else {@code null}. */ - private static String callbackTimeoutName(Throwable wrapper) { - if (wrapper instanceof DurableOperationException doe - && doe.getOperation() != null - && doe.getOperation().type() == OperationType.CALLBACK - && doe.getOperation().status() == OperationStatus.TIMED_OUT) { + /** + * The SDK error (not a wrapper) the core throws for a wrapper's operation, whatever its checkpoint says: a + * timed-out callback, or a timed-out or stopped invoke. {@code null} for any other operation. + */ + private static String sdkErrorOfOperation(Throwable wrapper) { + if (!(wrapper instanceof DurableOperationException doe) || doe.getOperation() == null) { + return null; + } + var op = doe.getOperation(); + if (op.type() == OperationType.CALLBACK && op.status() == OperationStatus.TIMED_OUT) { return CallbackTimeoutException.class.getSimpleName(); } + if (op.type() == OperationType.CHAINED_INVOKE && op.status() == OperationStatus.TIMED_OUT) { + return InvokeTimedOutException.class.getSimpleName(); + } + if (op.type() == OperationType.CHAINED_INVOKE && op.status() == OperationStatus.STOPPED) { + return InvokeStoppedException.class.getSimpleName(); + } return null; } @@ -329,13 +387,27 @@ private static Class contextFailure(String subType) { return null; } + /** + * {@link #simpleTypeName} for a Java class name, live or checkpointed. The name of a hidden class has a + * {@code /0x…} suffix that changes on every JVM start, so the name is cut at the first {@code '/'} first and the + * same failure keeps the same name across cold starts. + */ + static String javaTypeName(String className) { + if (className == null) { + return DEFAULT_ERROR_NAME; + } + int slash = className.indexOf('/'); + return simpleTypeName(slash < 0 ? className : className.substring(0, slash)); + } + /** * Reduces a type string to the reported {@code error.name} (Table A of the cross-SDK rule): the part after the last - * {@code '.'} or {@code '$'}. If that part is empty the string is kept as it is. A {@code null}, empty or blank - * string gives {@code "Error"}. The string is not trimmed. + * {@code '.'} or {@code '$'}. If that part is empty the string is kept as it is. A {@code null} string, or one made + * only of the six ASCII whitespace characters (see {@link #isBlankType}), gives {@code "Error"}. The string is not + * trimmed. */ static String simpleTypeName(String typeName) { - if (typeName == null || typeName.isBlank()) { + if (isBlankType(typeName)) { return DEFAULT_ERROR_NAME; } int cut = Math.max(typeName.lastIndexOf('.'), typeName.lastIndexOf('$')); @@ -343,6 +415,26 @@ static String simpleTypeName(String typeName) { return last.isEmpty() ? typeName : last; } + /** + * {@code true} for {@code null}, the empty string, or a string made only of space, {@code \t}, {@code \n}, + * {@code \r}, {@code \f} and {@code \u000B} (rule 6). No other character is blank: unlike {@link String#isBlank()}, + * U+001C to U+001F and the Unicode spaces such as U+2003 are not, and U+00A0 and U+FEFF are not either. + */ + static boolean isBlankType(String type) { + if (type == null) { + return true; + } + for (int i = 0; i < type.length(); i++) { + switch (type.charAt(i)) { + case ' ', '\t', '\n', '\r', '\f', '\u000B' -> {} + default -> { + return false; + } + } + } + return true; + } + /** * Returns the checkpointed {@link ErrorObject} carried by the SDK's error wrappers, or {@code null} when the * throwable is neither wrapper or carries no {@code ErrorObject}. Both {@link DurableOperationException} (operation diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/exception/CyclicWrapperForTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/exception/CyclicWrapperForTest.java deleted file mode 100644 index 7743f6acd..000000000 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/exception/CyclicWrapperForTest.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.exception; - -import software.amazon.awssdk.services.lambda.model.ErrorObject; - -/** - * Test-only SDK-package wrapper whose cause can be set, so the exporter's wrapper walk can be given a cycle. It is - * declared in the SDK exception package so the exporter treats it as an SDK wrapper. - */ -public final class CyclicWrapperForTest extends UnrecoverableDurableExecutionException { - private Throwable other; - - public CyclicWrapperForTest(String errorType) { - super(ErrorObject.builder().errorType(errorType).errorMessage("m").build()); - } - - public void setOther(Throwable other) { - this.other = other; - } - - @Override - public synchronized Throwable getCause() { - return other; - } -} diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java index d97505901..ae9c86e53 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java @@ -9,13 +9,21 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.CallbackDetails; +import software.amazon.awssdk.services.lambda.model.ChainedInvokeDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; @@ -24,9 +32,12 @@ import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.exception.CallbackFailedException; import software.amazon.lambda.durable.exception.CallbackTimeoutException; -import software.amazon.lambda.durable.exception.CyclicWrapperForTest; import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; +import software.amazon.lambda.durable.exception.InvokeStoppedException; +import software.amazon.lambda.durable.exception.InvokeTimedOutException; +import software.amazon.lambda.durable.exception.NonDeterministicExecutionException; +import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.exception.StepInterruptedException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; import software.amazon.lambda.durable.model.ExecutionStatus; @@ -277,28 +288,217 @@ void nestedSdkWrappersAreFollowedToTheUserError() { assertEquals("w4", info.message(), "message is the outer record's message, unchanged"); } + /** + * Rule 3 depth, shared with the JS and Python tests: a chain of {@code n} SDK wrappers around {@code PaymentError}. + * Wrapper {@code k} (1 is the outermost) carries the checkpointed {@code errorType} {@code test.W}. The depth + * counts unwrap steps: 16 wrappers need 16 unwraps and reach {@code PaymentError}; with 17 or more, the error + * reached after 16 unwraps is wrapper 17, which is still a wrapper, so its name {@code W17} is reported. + */ @Test - void wrapperChainDepthIsBounded() { - Throwable chain = new PaymentError("deep"); - for (int i = 0; i < 40; i++) { + void wrapperChainDepthCountsUnwrapSteps() { + assertEquals( + "PaymentError", + DurableExecutionTelemetry.toErrorInfo(wrapperChain(1)).name()); + assertEquals( + "PaymentError", + DurableExecutionTelemetry.toErrorInfo(wrapperChain(15)).name()); + assertEquals( + "PaymentError", + DurableExecutionTelemetry.toErrorInfo(wrapperChain(16)).name()); + assertEquals( + "W17", DurableExecutionTelemetry.toErrorInfo(wrapperChain(17)).name()); + assertEquals( + "W17", DurableExecutionTelemetry.toErrorInfo(wrapperChain(18)).name()); + assertEquals( + "W17", DurableExecutionTelemetry.toErrorInfo(wrapperChain(40)).name()); + // The message is the outermost wrapper's checkpointed message, unchanged. + assertEquals( + "m1", DurableExecutionTelemetry.toErrorInfo(wrapperChain(17)).message()); + } + + /** {@code n} wrappers, wrapper 1 outermost, wrapper {@code k} with type {@code test.W}, around PaymentError. */ + private static Throwable wrapperChain(int n) { + Throwable chain = new PaymentError("card declined"); + for (int k = n; k >= 1; k--) { chain = new UnrecoverableDurableExecutionException( - ErrorObject.builder().errorType("w" + i).errorMessage("w").build(), false, chain); + ErrorObject.builder() + .errorType("test.W" + k) + .errorMessage("m" + k) + .build(), + false, + chain); + } + return chain; + } + + /** + * Rule 3 cycles, shared with the JS and Python tests: the walk stops at the first repeat and reports the last error + * visited before it. The JDK never returns a throwable as its own cause, and the SDK wrappers' causes are fixed at + * construction, so the cycle is given through the walk's cause lookup. + */ + @Test + void cyclicWrapperChainReportsTheLastErrorBeforeTheRepeat() { + var a = wrapped("test.A", "a"); + var b = wrapped("test.B", "b"); + var self = wrapped("test.Self", "s"); + // Two-cycle A -> B -> A: visited A, B; A repeats; B is reported. + Map twoCycle = new IdentityHashMap<>(); + twoCycle.put(a, b); + twoCycle.put(b, a); + assertEquals("B", DurableExecutionTelemetry.errorName(a, twoCycle::get)); + assertEquals("A", DurableExecutionTelemetry.errorName(b, twoCycle::get)); + // Self-cycle S -> S: visited S; S repeats; S is reported. + Map selfCycle = new IdentityHashMap<>(); + selfCycle.put(self, self); + assertEquals("Self", DurableExecutionTelemetry.errorName(self, selfCycle::get)); + // A cycle that is entered after a prefix: P -> A -> B -> A reports B. + var p = wrapped("test.P", "p"); + Map prefixed = new IdentityHashMap<>(twoCycle); + prefixed.put(p, a); + assertEquals("B", DurableExecutionTelemetry.errorName(p, prefixed::get)); + } + + /** + * R1: wrapper detection is an explicit list matched on the exact runtime class. Every class in the SDK exception + * package, read from the SDK jar (or its classes directory), must be classified as a wrapper or an SDK error, so a + * new SDK exception cannot become a wrapper without a decision. + */ + @Test + void everySdkExceptionClassIsClassified() throws Exception { + var pkg = DurableOperationException.class.getPackageName(); + var found = sdkExceptionClassNames(pkg); + assertTrue(found.size() >= 21, "SDK exception classes found: " + found); + List unclassified = new ArrayList<>(); + for (String name : found) { + Class c = Class.forName(name, false, DurableOperationException.class.getClassLoader()); + if (!DurableExecutionTelemetry.SDK_WRAPPERS.contains(c) + && !DurableExecutionTelemetry.SDK_ERRORS.contains(c)) { + unclassified.add(name); + } + } + assertEquals(List.of(), unclassified, "classify each new SDK exception as a wrapper or an SDK error"); + for (var c : DurableExecutionTelemetry.SDK_WRAPPERS) { + assertFalse(DurableExecutionTelemetry.SDK_ERRORS.contains(c), c + " is in both lists"); + assertEquals(pkg, c.getPackageName()); + } + for (var c : DurableExecutionTelemetry.SDK_ERRORS) { + assertEquals(pkg, c.getPackageName()); + } + // The decisions of the review: these name a failure themselves. + for (var c : List.of( + StepInterruptedException.class, + IllegalDurableOperationException.class, + NonDeterministicExecutionException.class, + InvokeTimedOutException.class, + InvokeStoppedException.class, + CallbackTimeoutException.class, + SerDesException.class)) { + assertTrue(DurableExecutionTelemetry.SDK_ERRORS.contains(c), c.getSimpleName()); + } + } + + private static List sdkExceptionClassNames(String pkg) throws Exception { + var dir = pkg.replace('.', '/') + "/"; + var location = Path.of(DurableOperationException.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()); + List names = new ArrayList<>(); + if (Files.isDirectory(location)) { + try (var files = Files.list(location.resolve(dir))) { + files.map(f -> f.getFileName().toString()) + .filter(f -> f.endsWith(".class")) + .forEach(f -> names.add(pkg + "." + f.substring(0, f.length() - ".class".length()))); + } + } else { + try (var jar = new JarFile(location.toFile())) { + jar.stream() + .map(JarEntry::getName) + .filter(e -> e.startsWith(dir) && e.endsWith(".class")) + .filter(e -> e.indexOf('/', dir.length()) < 0) + .forEach(e -> names.add( + e.substring(0, e.length() - ".class".length()).replace('/', '.'))); + } } - // Terminates; the walk stops after 16 levels at an SDK class name. + names.removeIf(n -> n.endsWith(".package-info")); + Collections.sort(names); + return names; + } + + @Test + void sdkErrorsThatAreNotWrappersReportTheirOwnName() { + // Invoke timeout and stop are SDK errors, like a callback timeout: named after the core's exception, whatever + // error type the checkpoint carries, live (the core's exception) and from the snapshot (a bare wrapper). + for (var status : List.of(OperationStatus.TIMED_OUT, OperationStatus.STOPPED)) { + String expected = + status == OperationStatus.TIMED_OUT ? "InvokeTimedOutException" : "InvokeStoppedException"; + for (String type : new String[] {"States.Stopped", "com.acme.PaymentError", null}) { + var error = + ErrorObject.builder().errorType(type).errorMessage("m").build(); + var op = Operation.builder() + .id("inv") + .type(OperationType.CHAINED_INVOKE) + .status(status) + .chainedInvokeDetails( + ChainedInvokeDetails.builder().error(error).build()) + .build(); + assertEquals( + expected, + DurableExecutionTelemetry.toErrorInfo(new DurableOperationException(op, error)) + .name()); + Throwable live = status == OperationStatus.TIMED_OUT + ? new InvokeTimedOutException(op) + : new InvokeStoppedException(op); + assertFalse(DurableExecutionTelemetry.isSdkWrapper(live)); + assertEquals( + expected, DurableExecutionTelemetry.toErrorInfo(live).name()); + } + } + assertFalse(DurableExecutionTelemetry.isSdkWrapper(new IllegalDurableOperationException("bad"))); + assertFalse(DurableExecutionTelemetry.isSdkWrapper(new NonDeterministicExecutionException("nd"))); assertEquals( - "UnrecoverableDurableExecutionException", - DurableExecutionTelemetry.toErrorInfo(chain).name()); + "NonDeterministicExecutionException", + DurableExecutionTelemetry.toErrorInfo(new NonDeterministicExecutionException("nd")) + .name()); } + /** + * R2 (Java only): a hidden class's name ends in {@code /0x…}, which changes on every JVM start. The name is cut at + * the first {@code '/'} before the reduction, live and from the checkpoint. + */ @Test - void cyclicWrapperChainTerminates() { - var a = new CyclicWrapperForTest("x.A"); - var b = new CyclicWrapperForTest("x.B"); - a.setOther(b); - b.setOther(a); - assertTrue(DurableExecutionTelemetry.isSdkWrapper(a)); + void hiddenClassNameDropsTheAddressSuffix() throws Throwable { + byte[] bytes; + try (InputStream in = HiddenProbeError.class.getResourceAsStream("HiddenProbeError.class")) { + assertNotNull(in); + bytes = in.readAllBytes(); + } + Class hidden = MethodHandles.lookup().defineHiddenClass(bytes, true).lookupClass(); + assertTrue(hidden.isHidden()); + assertTrue(hidden.getName().contains("/"), hidden.getName()); + var error = (Throwable) hidden.getDeclaredConstructor().newInstance(); + assertEquals( + "HiddenProbeError", DurableExecutionTelemetry.toErrorInfo(error).name()); assertEquals( - "CyclicWrapperForTest", DurableExecutionTelemetry.toErrorInfo(a).name()); + "HiddenProbeError", + DurableExecutionTelemetry.toErrorInfo(wrapped(hidden.getName(), "m")) + .name()); + assertEquals("Lambda", DurableExecutionTelemetry.javaTypeName("pkg.Foo$$Lambda/0x1")); + assertEquals("Error", DurableExecutionTelemetry.javaTypeName("/0x1")); + } + + /** Rule 6 blank: only the six ASCII whitespace characters; the Table A vectors cover each single character. */ + @Test + void onlyAsciiWhitespaceIsBlank() { + assertTrue(DurableExecutionTelemetry.isBlankType(null)); + assertTrue(DurableExecutionTelemetry.isBlankType("")); + assertTrue(DurableExecutionTelemetry.isBlankType(" \t\n\r\f\u000B")); + for (char c : + new char[] {'\u00A0', '\uFEFF', '\u001C', '\u001D', '\u001E', '\u001F', '\u2003', '\u3000', '\u0085'}) { + assertFalse(DurableExecutionTelemetry.isBlankType(" " + c + " "), Integer.toHexString(c)); + assertEquals(" " + c + " ", DurableExecutionTelemetry.simpleTypeName(" " + c + " ")); + } } @Test diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/HiddenProbeError.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/HiddenProbeError.java new file mode 100644 index 000000000..f0638a226 --- /dev/null +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/HiddenProbeError.java @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.telemetry; + +/** + * Template for the hidden-class test in {@link ErrorNameTest}: its bytes are defined again as a hidden class, whose + * name gets a {@code /0x…} suffix. + */ +public class HiddenProbeError extends RuntimeException { + public HiddenProbeError() { + super("hidden"); + } +} From 40e0f985b70eff8fe60db6e4d6ed05358e5d0756 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 22:40:13 -0700 Subject: [PATCH 06/12] test(telemetry): load the version 2 error-name vectors Replace error-name-vectors.json with the spec version 2 file, byte for byte. It adds A13 (the six ASCII whitespace characters give "Error") and A14 to A17 (U+00A0, U+FEFF, U+001C and U+2003 are not blank and are kept). The test now asserts version 2, and its javadoc states that A12 and A13 hold on the record path only for operation types without an SDK failure class. --- .../telemetry/ErrorNameVectorsTest.java | 9 +- .../test/resources/error-name-vectors.json | 119 +++++++++++++++--- 2 files changed, 109 insertions(+), 19 deletions(-) diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java index c6d5018c8..dee85ff68 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameVectorsTest.java @@ -53,8 +53,8 @@ private static JsonNode load() throws IOException { @Test void vectorFileHasTheSharedShape() throws IOException { JsonNode root = load(); - assertEquals(1, root.get("version").asInt()); - assertTrue(root.get("reduce").size() >= 15, "all Table A rows present"); + assertEquals(2, root.get("version").asInt(), "spec version 2 vectors"); + assertTrue(root.get("reduce").size() >= 20, "all Table A rows present, A1 to A17"); } @ParameterizedTest(name = "{0}: [{1}] -> [{2}]") @@ -66,6 +66,11 @@ void reducesTypeString(String id, String input, String expected) { /** * The same vectors through the record path: a checkpointed operation error whose {@code errorType} is the input. * The operation type is one with no SDK failure class of its own, so a blank type falls through to Table A. + * + *

A12 and A13 hold on this path only for operation types without a failure class. For a STEP, CALLBACK, + * CHAINED_INVOKE or CONTEXT operation, a blank type reports the SDK failure class the core throws for it instead, + * for example {@code StepFailedException} or {@code CallbackFailedException} (Table B, B13), so the operation and + * the execution agree. */ @ParameterizedTest(name = "{0}: checkpointed [{1}] -> [{2}]") @MethodSource("reduceVectors") diff --git a/telemetry-exporter/src/test/resources/error-name-vectors.json b/telemetry-exporter/src/test/resources/error-name-vectors.json index 1a0f2d8df..bf3b77001 100644 --- a/telemetry-exporter/src/test/resources/error-name-vectors.json +++ b/telemetry-exporter/src/test/resources/error-name-vectors.json @@ -1,21 +1,106 @@ { - "description": "Durable Execution Telemetry error.name reduction vectors. Input is a type string; expected is the reported error.name. Shared across the JS, Python and Java exporters. Do not edit in one repo only.", - "version": 1, + "description": "Durable Execution Telemetry error.name reduction vectors (spec version 2). Input is a type string; expected is the reported error.name. Blank means only the six ASCII whitespace characters (space, \\t, \\n, \\r, \\f, \\v); names are never trimmed. Shared byte-for-byte across the JS, Python and Java exporters. Do not edit in one repo only.", + "version": 2, "reduce": [ - { "id": "A1", "input": "PaymentError", "expected": "PaymentError" }, - { "id": "A2", "input": "com.acme.PaymentError", "expected": "PaymentError" }, - { "id": "A3", "input": "com.acme.Outer$PaymentError", "expected": "PaymentError" }, - { "id": "A4", "input": "a.b.Outer$Inner$PaymentError", "expected": "PaymentError" }, - { "id": "A5", "input": "__main__.PaymentError", "expected": "PaymentError" }, - { "id": "A6", "input": "States.Timeout", "expected": "Timeout" }, - { "id": "A7", "input": "trailing.", "expected": "trailing." }, - { "id": "A8", "input": "Outer$", "expected": "Outer$" }, - { "id": "A9a", "input": ".", "expected": "." }, - { "id": "A9b", "input": "$", "expected": "$" }, - { "id": "A10", "input": "pkg.Foo$1Local", "expected": "1Local" }, - { "id": "A11", "input": " PaymentError ", "expected": " PaymentError " }, - { "id": "A12a", "input": "", "expected": "Error" }, - { "id": "A12b", "input": " ", "expected": "Error" }, - { "id": "A12c", "input": null, "expected": "Error" } + { + "id": "A1", + "input": "PaymentError", + "expected": "PaymentError" + }, + { + "id": "A2", + "input": "com.acme.PaymentError", + "expected": "PaymentError" + }, + { + "id": "A3", + "input": "com.acme.Outer$PaymentError", + "expected": "PaymentError" + }, + { + "id": "A4", + "input": "a.b.Outer$Inner$PaymentError", + "expected": "PaymentError" + }, + { + "id": "A5", + "input": "__main__.PaymentError", + "expected": "PaymentError" + }, + { + "id": "A6", + "input": "States.Timeout", + "expected": "Timeout" + }, + { + "id": "A7", + "input": "trailing.", + "expected": "trailing." + }, + { + "id": "A8", + "input": "Outer$", + "expected": "Outer$" + }, + { + "id": "A9a", + "input": ".", + "expected": "." + }, + { + "id": "A9b", + "input": "$", + "expected": "$" + }, + { + "id": "A10", + "input": "pkg.Foo$1Local", + "expected": "1Local" + }, + { + "id": "A11", + "input": " PaymentError ", + "expected": " PaymentError " + }, + { + "id": "A12a", + "input": "", + "expected": "Error" + }, + { + "id": "A12b", + "input": " ", + "expected": "Error" + }, + { + "id": "A12c", + "input": null, + "expected": "Error" + }, + { + "id": "A13", + "input": " \t\n\r\f\u000b", + "expected": "Error" + }, + { + "id": "A14", + "input": "\u00a0", + "expected": "\u00a0" + }, + { + "id": "A15", + "input": "\ufeff", + "expected": "\ufeff" + }, + { + "id": "A16", + "input": "\u001c", + "expected": "\u001c" + }, + { + "id": "A17", + "input": "\u2003", + "expected": "\u2003" + } ] } From 4ff7fa70dae51f7461eac59f28ec56e8bb4848c2 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 22:40:13 -0700 Subject: [PATCH 07/12] test(telemetry): run Table B v2 and check the final replayed record - Add B13b (callback failed with errorType "Error" reports "Error" on the operation and the execution), B20 (two map items) and B21 (two parallel branches) with each item's and branch's own type, and B19 as not applicable (JS only). - Mark the execution cells of B5, B6, B20 and B21 as not applicable, with the reason in the test: Java map and parallel return failures as data, so no operation error propagates and the SDK never chooses between failures. The item, branch and step names are still asserted. - B17 asserts that the operation message equals the execution message. - Assert operation names on the final record, not only on earlier ON_CHANGE records. A replayed final record may omit an operation only when an ancestor context is in it as completed. - Invoke timed out and stopped report InvokeTimedOutException and InvokeStoppedException, live and replayed; a timed-out invoke has a null operation message. --- .../telemetry/ErrorNameScenarioTest.java | 213 +++++++++++++++++- 1 file changed, 201 insertions(+), 12 deletions(-) diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java index a593dc9b1..d7e8d7843 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java @@ -18,6 +18,7 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import software.amazon.awssdk.services.lambda.model.ErrorObject; @@ -56,6 +57,13 @@ public PaymentError(String message, Throwable cause) { } } + /** Second user error type of rows B20 and B21. */ + public static class FraudError extends RuntimeException { + public FraudError(String message) { + super(message); + } + } + /** Stand-in for the JS {@code TypeError} of rows B2 and B14. */ public static class TypeError extends RuntimeException { public TypeError(String message) { @@ -142,11 +150,14 @@ private record Observed(ErrorInfo execution, Map operations) /** * The execution error comes from the final record. Each operation's error comes from the latest record that carries - * the operation: a completed child context can replay without its nested operations, so in a replayed run a nested - * operation is only in the record of the invocation that ran it. + * the operation. The final record is checked too, because it is the one a reader of an {@code ON_COMPLETE} export + * sees: every operation it carries must have the same name and {@code operationsByName} entry. A live final record + * must carry every operation. A replayed final record may omit an operation only when it is nested in a context + * that the final record carries as completed: the core replays a completed context from its checkpoint without + * running its body, so the final invocation's snapshot has no children for it. */ @SuppressWarnings("unchecked") - private static Observed observe(List records, List opNames) { + private static Observed observe(List records, List opNames, Mode mode) { var last = records.get(records.size() - 1); Map ops = new LinkedHashMap<>(); for (String name : opNames) { @@ -167,11 +178,47 @@ private static Observed observe(List records, L assertNotNull(byNameError, "operationsByName." + name + ".error"); assertEquals(op.error().name(), byNameError.get("name"), "operationsByName == operations[] for " + name); assertEquals(op.error().message(), byNameError.get("message"), "byName message for " + name); + var inFinal = last.operations().stream() + .filter(o -> name.equals(o.name()) && o.error() != null) + .reduce((a, b) -> b) + .orElse(null); + if (inFinal == null) { + assertTrue( + mode == Mode.REPLAYED && hasCompletedAncestorInFinal(records, op), + mode + " final record carries operations[" + name + "].error; records: " + describe(records)); + ops.put(name, op.error()); + continue; + } + assertEquals(op.error().name(), inFinal.error().name(), "final record name for " + name); + var finalByName = (Map) ((Map) + ((Map) last.toByNameWireMap().get("operationsByName")).get(name)) + .get("error"); + assertEquals(op.error().name(), finalByName.get("name"), "final record operationsByName for " + name); ops.put(name, op.error()); } return new Observed(last.error, ops); } + /** {@code true} when an ancestor context of {@code op} is in the final record with a terminal status. */ + private static boolean hasCompletedAncestorInFinal( + List records, OperationRecord op) { + Map parents = new LinkedHashMap<>(); + for (var r : records) { + for (var o : r.operations()) { + parents.putIfAbsent(o.id(), o.parentId()); + } + } + var last = records.get(records.size() - 1); + var terminal = List.of("SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED", "CANCELLED"); + for (String id = op.parentId(); id != null; id = parents.get(id)) { + String ancestor = id; + if (last.operations().stream().anyMatch(o -> ancestor.equals(o.id()) && terminal.contains(o.status()))) { + return true; + } + } + return false; + } + private static String describe(List records) { StringBuilder sb = new StringBuilder(); for (var r : records) { @@ -204,16 +251,27 @@ private static void check( String expectedOpMessage) { Map seen = new LinkedHashMap<>(); for (Mode mode : Mode.values()) { - var obs = observe(run(handler.apply(mode), drive), List.copyOf(expectedOps.keySet())); + var obs = observe(run(handler.apply(mode), drive), List.copyOf(expectedOps.keySet()), mode); String where = row + " " + mode; assertNotNull(obs.execution(), where + " execution error"); - assertEquals(expectedExecName, obs.execution().name(), where + " execution error.name"); + if (expectedExecName == null) { + // Not applicable in Java; the reason is in EXECUTION_NOT_APPLICABLE. + assertTrue(!EXECUTION_NOT_APPLICABLE.getOrDefault(row, "").isEmpty(), row + " needs a reason"); + } else { + assertEquals(expectedExecName, obs.execution().name(), where + " execution error.name"); + } if (expectedExecMessage != null) { assertEquals(expectedExecMessage, obs.execution().message(), where + " execution error.message"); } expectedOps.forEach((name, expected) -> { assertEquals(expected, obs.operations().get(name).name(), where + " " + name + " error.name"); - if (expectedOpMessage != null) { + if (SAME_AS_EXECUTION.equals(expectedOpMessage)) { + assertNotNull(obs.execution().message(), where + " execution error.message"); + assertEquals( + obs.execution().message(), + obs.operations().get(name).message(), + where + " " + name + " error.message == execution error.message"); + } else if (expectedOpMessage != null) { assertEquals( expectedOpMessage, obs.operations().get(name).message(), @@ -224,7 +282,9 @@ private static void check( } var live = seen.get(Mode.LIVE); var replayed = seen.get(Mode.REPLAYED); - assertEquals(live.execution().name(), replayed.execution().name(), row + " live == replayed (execution)"); + if (expectedExecName != null) { + assertEquals(live.execution().name(), replayed.execution().name(), row + " live == replayed (execution)"); + } expectedOps .keySet() .forEach(name -> assertEquals( @@ -233,6 +293,27 @@ private static void check( row + " live == replayed (" + name + ")")); } + /** Expected operation message meaning "equal to the execution error message" (an SDK-generated message). */ + private static final String SAME_AS_EXECUTION = ""; + + /** + * Table B execution cells that cannot happen in Java, with the reason. The operation cells of these rows are still + * asserted. + */ + private static final Map EXECUTION_NOT_APPLICABLE = Map.of( + "B5", + "Java map returns item failures as data (MapResult.MapError) and does not throw them; no operation error" + + " propagates, the handler throws a new PaymentError, so rule 8 is not exercised", + "B6", + "same as B5: Java map does not throw item failures", + "B20", + "same as B5: Java map does not throw item failures, so the SDK never has to choose between the two items" + + " (rule 9)", + "B21", + "Java parallel returns a ParallelResult of branch statuses and does not throw; only the branch future the" + + " handler calls get() on throws, and it throws that branch's own error, so the SDK never has to" + + " choose between branches (rule 9)"); + enum Row { B1, B2, @@ -247,11 +328,15 @@ enum Row { B11, B12, B13, + B13b, B14, B15, B16, B17, - B18 + B18, + B19, + B20, + B21 } @ParameterizedTest @@ -341,7 +426,7 @@ void tableB(Row row) { "B5", m -> (in, ctx) -> failure(m, ctx, c -> mapAndRethrow(c, false)), null, - "PaymentError", + null, // execution: not applicable in Java, see EXECUTION_NOT_APPLICABLE MSG, Map.of("fanout-iteration-0", "PaymentError"), MSG); @@ -350,7 +435,7 @@ void tableB(Row row) { "B6", m -> (in, ctx) -> failure(m, ctx, c -> mapAndRethrow(c, true)), null, - "PaymentError", + null, // execution: not applicable in Java, see EXECUTION_NOT_APPLICABLE MSG, Map.of("charge", "PaymentError", "fanout-iteration-0", "PaymentError"), MSG); @@ -464,6 +549,18 @@ void tableB(Row row) { MSG); } } + case B13b -> // A callback is failed externally with the explicit type "Error". + check( + "B13b", + m -> (in, ctx) -> failure( + m, + ctx, + c -> c.createCallback("cb", String.class).get()), + failCallback("Error"), + "Error", + MSG, + Map.of("cb", "Error"), + MSG); case B14 -> // The handler throws PaymentError, which has its own cause TypeError. check( "B14", @@ -519,9 +616,9 @@ void tableB(Row row) { }), null, "SerDesException", - null, // SDK-generated serialization message + null, // SDK-generated serialization message, not hard-coded Map.of("encode", "SerDesException"), - null); + SAME_AS_EXECUTION); case B18 -> // A class named Payment$Error is thrown from a step. check( "B18", @@ -540,6 +637,51 @@ void tableB(Row row) { MSG, Map.of("charge", "Error"), MSG); + case B19 -> // promise.all over a failing step. + Assumptions.abort("B19 not applicable in Java: promise.all is a JS combinator with no Java equivalent"); + case B20 -> // Two map items fail with the same message: item 0 PaymentError, item 1 FraudError. + check( + "B20", + m -> (in, ctx) -> failure(m, ctx, c -> { + var result = c.map("fanout", List.of("a", "b"), String.class, (item, i, mc) -> { + if (i == 0) { + throw new PaymentError(MSG); + } + throw new FraudError(MSG); + }); + assertEquals( + PaymentError.class.getName(), + result.getError(0).errorType()); + assertEquals( + FraudError.class.getName(), + result.getError(1).errorType()); + throw new PaymentError(result.getError(0).errorMessage()); + }), + null, + null, // execution: not applicable in Java, see EXECUTION_NOT_APPLICABLE + MSG, + Map.of("fanout-iteration-0", "PaymentError", "fanout-iteration-1", "FraudError"), + MSG); + case B21 -> // Two parallel branches fail with the same message and different types. + check( + "B21", + m -> (in, ctx) -> failure(m, ctx, c -> { + try (var p = c.parallel("par")) { + var pay = p.branch("pay", String.class, bc -> { + throw new PaymentError(MSG); + }); + p.branch("fraud", String.class, bc -> { + throw new FraudError(MSG); + }); + p.get(); // does not throw: branch failures are statuses + return pay.get(); + } + }), + null, + null, // execution: not applicable in Java, see EXECUTION_NOT_APPLICABLE + MSG, + Map.of("pay", "PaymentError", "fraud", "FraudError"), + MSG); default -> fail("unhandled row " + row); } } @@ -596,4 +738,51 @@ void callbackTimeoutMessages(Mode mode) { .orElseThrow(); assertNull(op.error().message(), "no checkpointed error, so no operation message"); } + + /** + * Invoke timeout and stop are SDK errors, not wrappers (rule 5): the operation and the execution report + * {@code InvokeTimedOutException} or {@code InvokeStoppedException}, live and replayed, whatever error type a stop + * carries. A timed-out invoke has no checkpointed error, so its operation message is {@code null}; a stopped invoke + * keeps the stop error's message. + */ + @ParameterizedTest + @EnumSource(Mode.class) + void invokeTimeoutAndStopAreSdkErrors(Mode mode) { + for (String outcome : List.of("timeout", "stop-states", "stop-user")) { + var records = run((in, ctx) -> failure(mode, ctx, c -> c.invoke("callee", "fn", "x", String.class)), r -> { + r.run("in"); + switch (outcome) { + case "timeout" -> r.timeoutChainedInvoke("callee"); + case "stop-states" -> + r.stopChainedInvoke( + "callee", + ErrorObject.builder() + .errorType("States.Stopped") + .errorMessage(MSG) + .build()); + default -> + r.stopChainedInvoke( + "callee", + ErrorObject.builder() + .errorType("com.acme.PaymentError") + .errorMessage(MSG) + .build()); + } + r.runUntilComplete("in"); + }); + var rec = records.get(records.size() - 1); + String expected = "timeout".equals(outcome) ? "InvokeTimedOutException" : "InvokeStoppedException"; + var op = rec.operations().stream() + .filter(o -> "callee".equals(o.name())) + .findFirst() + .orElseThrow(); + assertEquals(expected, rec.error.name(), outcome + " execution"); + assertEquals(expected, op.error().name(), outcome + " operation"); + if ("timeout".equals(outcome)) { + assertNull(op.error().message(), "no checkpointed error, so no operation message"); + } else { + assertEquals(MSG, op.error().message(), outcome + " operation message"); + } + } + } } From 16c6f55ca86b5061e116c2cf4334ef2db6541a32 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 22:40:13 -0700 Subject: [PATCH 08/12] docs(telemetry): state the spec v2 error.name rule - Rules 1 to 9 in the words of spec version 2, including the explicit callback ErrorType "Error" case. - Narrow the Timeout sentence to callback timeouts: InvokeTimedOutException does not contain "Timeout". - List the wrapper classes and the SDK errors; say that invoke timeout and stop are SDK errors and how they are named; describe the hidden-class cut. - Document the null operation message for timed-out callbacks and timed-out or stopped invokes without a checkpointed error, and the execution message "Callback timed out: ". --- telemetry-exporter/README.md | 68 ++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index ca8833a23..d460bba0c 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -243,23 +243,37 @@ this rule. The rule applies to the execution-level `error`, every `operations[]. `operationsByName..error`. 1. A record names the error that caused the failure. A wrapper error that the SDK added is never - the name. + the name, except in the bounded cases of rules 3 and 9. 2. The SDK unwraps an error only when the error's type is one of the SDK's own wrapper types. A user error is never unwrapped, even if it has a cause. -3. Unwrapping follows the whole chain of SDK wrappers, up to a depth of 16. A cycle stops the walk. +3. Unwrapping follows the chain of SDK wrappers. The depth counts unwrap steps, one per wrapper + passed, not levels in the operation tree. At most 16 unwraps happen. If the error reached after + 16 unwraps is still a wrapper, its name is reported. If the chain reaches an error it has + already visited, the walk stops and the last error visited before the repeat is reported. 4. If the chain ends at a user error, the name is that error's type name. 5. If the chain ends at an SDK error that is not a wrapper, the name is that SDK error's simple class name. A callback timeout is an example. 6. A type name that is a qualified string is reduced to its last segment after any `.` or `$`. If - that segment is empty, the string is kept as it is. + that segment is empty, the string is kept as it is. A missing type, or a type made only of blank + characters, is `Error`. The blank characters are the six ASCII whitespace characters: space, + `\t`, `\n`, `\r`, `\f` and `\v`. No other character counts as blank, and names are never + trimmed. 7. A failure gets the same name whether the record is built live or from a replayed checkpoint. 8. When the execution fails because an operation's error propagated, the execution `error.name` equals that operation's `error.name`. +9. If the SDK cannot tell which of several failures caused a wrapper, it does not guess. It + reports the wrapper's name. Two map items or parallel branches that fail with the same message + but different types are an example. + +A callback that is failed externally with an explicit `ErrorType` is named after that type, after +the reduction of rule 6. This includes the type `"Error"`, which reports `Error`. Only a missing or +blank type reports the SDK callback-failure class. User error types have the same name in every SDK, because the user names them. SDK error types keep each language's own class name. For example, a callback timeout is `CallbackTimeoutError` in -JS and Python and `CallbackTimeoutException` in Java. Every SDK name for a timeout contains -`Timeout`, so a `/Timeout/` query keeps working. +JS and Python and `CallbackTimeoutException` in Java. Every SDK name for a callback timeout +contains `Timeout`. An invoke timeout does not always: Java names it `InvokeTimedOutException`, so +a `/Timeout/` query that should also match invoke timeouts needs `/Timeout|TimedOut/`. How the Java exporter applies it: @@ -267,16 +281,42 @@ How the Java exporter applies it: replayed ones, so both give the same name. `com.acme.Outer$PaymentError` gives `PaymentError`; a top-level class named `Payment$Error` gives `Error`; a local class `LocalError` declared in `pkg.Foo` (binary name `pkg.Foo$1LocalError`) gives `1LocalError`; an anonymous class gives its - last segment, for example `1`. The string is not trimmed. A missing, empty or blank type gives - `Error`. A string that ends with a separator (`trailing.`, `Outer$`) is kept as it is. -- SDK wrappers are the subclasses of `DurableOperationException` and - `UnrecoverableDurableExecutionException` declared in `software.amazon.lambda.durable.exception`. - A user exception that subclasses one of them is not a wrapper and reports its own name. - `CallbackTimeoutException` is never unwrapped. + last segment, for example `1`. A string that ends with a separator (`trailing.`, `Outer$`) is + kept as it is. U+00A0, U+FEFF, U+001C and U+2003 are not blank, so a type made of them is kept. +- A hidden class's name ends in `/0x…`, a suffix that changes on every JVM start. The name is cut + at the first `/` before rule 6, live and from the checkpoint, so `pkg.Gen$HiddenErr/0x…` gives + `HiddenErr`. +- The SDK wrappers are an explicit list, matched on the exact runtime class: + `DurableOperationException`, `UnrecoverableDurableExecutionException`, `StepException`, + `CallbackException`, `InvokeException`, `StepFailedException`, `CallbackFailedException`, + `CallbackSubmitterException`, `ChildContextFailedException`, `InvokeFailedException`, + `MapIterationFailedException`, `ParallelBranchFailedException` and + `WaitForConditionFailedException`. A subclass of one of them, such as a user exception, is not a + wrapper and reports its own name. Every other class in `software.amazon.lambda.durable.exception` + is an SDK error that reports its simple class name: `CallbackTimeoutException`, + `InvokeTimedOutException`, `InvokeStoppedException`, `StepInterruptedException`, + `IllegalDurableOperationException`, `NonDeterministicExecutionException`, `SerDesException` and + `DurableExecutionException`. +- A timed-out callback, and a timed-out or stopped invoke, are SDK errors like the callback + timeout. The operation and the execution report `CallbackTimeoutException`, + `InvokeTimedOutException` or `InvokeStoppedException`, whatever error type the checkpoint + carries. A stopped invoke is not named after its stop error: a stop with `errorType` + `States.Stopped` or `com.acme.PaymentError` reports `InvokeStoppedException`. +- After 16 unwraps, or at a cycle, the reported wrapper's name is its checkpointed `errorType`, + reduced; with a blank type it is the SDK failure class below. - When an operation failed with no error type (for example a callback failed externally without an - `errorType`, or a callback timed out), the operation and the execution report the SDK exception - the core throws for it, such as `CallbackFailedException` or `CallbackTimeoutException`. If the - checkpoint has no error at all, the operation error has that name and a `null` message. + `errorType`), the operation and the execution report the SDK exception the core throws for it, + such as `CallbackFailedException`. +- When the checkpoint has no error at all, the operation error has the SDK exception's name and a + `null` message, because there is no original message to keep. This is the case for a timed-out + callback and a timed-out invoke in the local test runner, and for a stopped invoke stopped + without an error. Before this change such operations had no `error` object; a reader that tests + `error != null` now sees one with `"message": null`. The execution error of a timed-out callback + has the message `Callback timed out: `, built by `CallbackTimeoutException`; it is not + checkpointed, and the JS and Python exporters report `Callback timed out` without the id. +- Java `map` and `parallel` return item and branch failures as data and do not throw them, so the + SDK never has to choose between several failures (rule 9). The execution error is whatever the + handler throws. - Non-class type strings are reduced too: an external callback failure with `errorType` `States.Timeout` reports `Timeout`. From 0d12f6c79146ecb92c501b952a32dae0592855e6 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 21 Oct 2026 22:40:25 -0700 Subject: [PATCH 09/12] docs(telemetry)!: mark the LambdaLogExporter envelope line as breaking The envelope change came in an earlier commit of this branch without a footer. Say in the README that the line shape changed and what a reader of the old shape must do, and add the footer so release notes list it. BREAKING CHANGE: LambdaLogExporter prints each record as a Lambda JSON log line {"timestamp","level":"INFO","message":""} instead of the bare record JSON printed by 2.3.0. Readers that parsed each stdout line as a record must parse the message field. --- telemetry-exporter/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index d460bba0c..f590a4f98 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -212,6 +212,8 @@ one event per record to `{logStreamPrefix}YYYY/MM/DD` (IAM: `logs:CreateLogStrea (`2 × maxRecordSizeBytes + 68` in the worst case, about 512 KB with the default 256,000). That is within the CloudWatch Logs limit of 1 MB per log event. `LambdaLogExporter` lines are INFO; an application log level of WARN or above drops them. + This line shape is a breaking change: 2.3.0 printed the bare record JSON, so a reader that + parsed each stdout line as a record must now parse `message`. - **Non-fatal plugin failures are isolated.** Record construction, snapshotting, transforms, truncation and exporter render/export/flush contain and log ordinary failures, including `AssertionError` and optional-dependency `NoClassDefFoundError`. `VirtualMachineError` and From 396f8a5fd3845eff68b03dbf39953d0a403c3a28 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Thu, 22 Oct 2026 09:15:45 -0700 Subject: [PATCH 10/12] fix(telemetry): cap the printed LambdaLogExporter line, escaping included maxRecordSizeBytes was measured on the rendered record, but the record is escaped again inside the envelope's message, so an escape-heavy record that fit the budget could print a line of about twice the limit (300,000 backslashes with new LambdaLogExporter(1_000_000) printed about 1.2 MB). export now checks the final line and, when it is over the limit, truncates the record again with the escaped envelope as the measured rendering, so the printed line is at most maxRecordSizeBytes. --- telemetry-exporter/README.md | 12 ++-- .../exporters/LambdaLogExporter.java | 39 ++++++++---- .../exporters/LambdaLogExporterTest.java | 59 +++++++++++-------- 3 files changed, 67 insertions(+), 43 deletions(-) diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index f590a4f98..13ab6ef3e 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -205,12 +205,12 @@ one event per record to `{logStreamPrefix}YYYY/MM/DD` (IAM: `logs:CreateLogStrea `{"timestamp":"Z","level":"INFO","message":""}`, because the JS readers of this destination filter on `level` and parse `message`. The Node runtime adds this envelope around `console.log`; Java `System.out` is not wrapped, and Lambda - does not re-wrap a line that is already JSON. `maxRecordSizeBytes` is measured on the record - (`render`), not on the printed line. The record is embedded in `message` as a JSON string, so - every `"` and `\` in it is escaped again: the line is the record plus 68 bytes plus one byte - per quote or backslash in the record, and can be up to about twice the record size - (`2 × maxRecordSizeBytes + 68` in the worst case, about 512 KB with the default 256,000). That - is within the CloudWatch Logs limit of 1 MB per log event. + does not re-wrap a line that is already JSON. `maxRecordSizeBytes` caps the printed line, + envelope included. The record is embedded in `message` as a JSON string, so every `"` and `\` + in it is escaped again and the line can be up to about twice the record. When the line is over + the limit, the exporter truncates the record again, measuring the escaped envelope, until the + line fits. With the default 256,000, or any limit up to 1 MB, every line is within the + CloudWatch Logs limit of 1 MB per log event. `LambdaLogExporter` lines are INFO; an application log level of WARN or above drops them. This line shape is a breaking change: 2.3.0 printed the bare record JSON, so a reader that parsed each stdout line as a record must now parse `message`. diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java index 6091f12cf..baf455864 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporter.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.telemetry.exporters; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -11,6 +12,7 @@ import software.amazon.lambda.durable.telemetry.DurableExecutionTelemetryRecord; import software.amazon.lambda.durable.telemetry.Json; import software.amazon.lambda.durable.telemetry.TelemetryExporter; +import software.amazon.lambda.durable.telemetry.Truncation; /** * Exports workflow telemetry records to CloudWatch Logs via {@code System.out.println} (Lambda captures stdout to the @@ -22,11 +24,11 @@ * runtime adds that envelope around {@code console.log}, but Java {@code System.out} is not wrapped, so this exporter * writes it itself; Lambda does not re-wrap a line that is already JSON. * - *

Line size. {@link #maxRecordSizeBytes()} is measured on the record ({@link #render}), not on the printed - * line. The record is embedded in {@code message} as a JSON string, so every {@code "} and {@code \} in it is escaped - * again: the line is the record plus 68 bytes of envelope plus one byte per quote or backslash in the record, so it can - * be up to about twice the record size ({@code 2 × maxRecordSizeBytes + 68} in the worst case). With the default - * 256,000-byte limit that is about 512 KB, within the CloudWatch Logs limit of 1 MB per log event. + *

Line size. {@link #maxRecordSizeBytes()} caps the printed line, envelope included. The record is embedded + * in {@code message} as a JSON string, so every {@code "} and {@code \} in it is escaped again and the line can be up + * to about twice the record. When the line is over the limit, {@link #export} truncates the record again, measuring the + * escaped envelope, until the line fits. With the default 256,000-byte limit every line stays well within the + * CloudWatch Logs limit of 1 MB per log event, and so does any limit up to 1 MB. * *

Lines are written with {@code "level":"INFO"}. With the Lambda JSON log format and an application log level of * {@code WARN} or above, Lambda drops them. @@ -37,10 +39,7 @@ public final class LambdaLogExporter implements TelemetryExporter { private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC); - /** - * Budget for the rendered record, not the printed line (see the class documentation). The default 256,000 keeps the - * worst-case escaped line (about 512 KB) under the CloudWatch Logs limit of 1 MB per log event. - */ + /** Budget for the printed line, envelope included (see the class documentation). */ private final Integer maxRecordSizeBytes; public LambdaLogExporter() { @@ -63,15 +62,33 @@ public Object render(DurableExecutionTelemetryRecord record) { @Override public void export(DurableExecutionTelemetryRecord record) { - System.out.println(logLine(Json.stringify(render(record)), Instant.now())); + System.out.println(fittedLine(record, maxRecordSizeBytes, Instant.now())); + } + + /** + * Returns the envelope line for {@code record}, truncated so that the line, escaping included, is at most + * {@code maxBytes} bytes. The record passed in was truncated against {@link #render}, which does not count the + * second escaping of {@code message}, so it is truncated again here with the envelope as the measured rendering. + */ + String fittedLine(DurableExecutionTelemetryRecord record, Integer maxBytes, Instant now) { + String line = logLine(Json.stringify(render(record)), now); + if (maxBytes == null || maxBytes <= 0 || line.getBytes(StandardCharsets.UTF_8).length <= maxBytes) { + return line; + } + var fitted = Truncation.truncateRecord(record, maxBytes, r -> envelope(Json.stringify(render(r)), now)); + return logLine(Json.stringify(render(fitted)), now); } /** Builds the one-line Lambda JSON log envelope; keys in the shared cross-SDK order. */ static String logLine(String recordJson, Instant now) { + return Json.stringify(envelope(recordJson, now)); + } + + private static Map envelope(String recordJson, Instant now) { Map line = new LinkedHashMap<>(); line.put("timestamp", TIMESTAMP.format(now)); line.put("level", "INFO"); line.put("message", recordJson); - return Json.stringify(line); + return line; } } diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java index dce0c4771..764493a8a 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/exporters/LambdaLogExporterTest.java @@ -33,11 +33,15 @@ class LambdaLogExporterTest { private static final Pattern READER_MESSAGE_FILTER = Pattern.compile("DurableExecutionTelemetry"); private static String exportAndCapture(DurableExecutionTelemetryRecord record) { + return exportAndCapture(new LambdaLogExporter(), record); + } + + private static String exportAndCapture(LambdaLogExporter exporter, DurableExecutionTelemetryRecord record) { PrintStream original = System.out; ByteArrayOutputStream buffer = new ByteArrayOutputStream(); try (PrintStream capture = new PrintStream(buffer, true, StandardCharsets.UTF_8)) { System.setOut(capture); - new LambdaLogExporter().export(record); + exporter.export(record); } finally { System.setOut(original); } @@ -84,21 +88,16 @@ void timestampAlwaysHasThreeFractionDigitsInUtc() { } @Test - void truncationIsMeasuredOnTheRecordNotTheEnvelope() { + void lineThatFitsIsPrintedWhole() { var record = RecordFactory.sample(); var exporter = new LambdaLogExporter(); - int recordBytes = Json.byteSize(exporter.render(record)); - - // A limit equal to the rendered record keeps it whole, even though the printed envelope line is larger. - var shaped = Truncation.truncateRecord(record, recordBytes, exporter::render); - assertSame(record, shaped); - assertFalse(shaped.toWireMap().containsKey("truncated")); - String line = exportAndCapture(shaped).strip(); - String recordJson = Json.stringify(exporter.render(shaped)); + String line = exportAndCapture(record).strip(); + String recordJson = Json.stringify(exporter.render(record)); assertEquals( - recordBytes + ENVELOPE_BYTES + escapedChars(recordJson), + Json.byteSize(exporter.render(record)) + ENVELOPE_BYTES + escapedChars(recordJson), line.getBytes(StandardCharsets.UTF_8).length, "line = record + 68-byte envelope + one byte per quote or backslash in the record"); + assertFalse(line.contains("\\\"truncated\\\""), "not truncated: " + line); } /** Bytes the envelope adds around an empty {@code message}. */ @@ -115,23 +114,31 @@ void envelopeAddsSixtyEightBytes() { } /** - * An escape-heavy record just under the default 256,000-byte budget is not truncated, and its printed line is about - * twice the record: at most {@code 2 × maxRecordSizeBytes + 68}, and under the 1 MB CloudWatch Logs event limit. + * The limit caps the printed line, escaping included. An escape-heavy record that fits the record budget but whose + * line does not (300,000 backslashes are about 600 KB of record and 1.2 MB of line) is truncated again until the + * line fits; its message still parses and says it was truncated. */ @Test - void escapeHeavyRecordNearTheLimitDoublesButStaysUnderOneMegabyte() { - var exporter = new LambdaLogExporter(); - int limit = exporter.maxRecordSizeBytes(); - for (char ch : new char[] {'"', '\\'}) { - var record = RecordFactory.sampleWithInput(String.valueOf(ch).repeat(127_000)); - int recordBytes = Json.byteSize(exporter.render(record)); - assertTrue(recordBytes <= limit && recordBytes > limit - 2_000, "record near the limit: " + recordBytes); - var shaped = Truncation.truncateRecord(record, limit, exporter::render); - assertSame(record, shaped, "not truncated"); - int lineBytes = exportAndCapture(shaped).strip().getBytes(StandardCharsets.UTF_8).length; - assertTrue(lineBytes > 2 * recordBytes - 1_000, "line about twice the record: " + lineBytes); - assertTrue(lineBytes <= 2 * limit + ENVELOPE_BYTES, "line within 2 x limit + 68: " + lineBytes); - assertTrue(lineBytes < 1_048_576, "line under the 1 MB CloudWatch Logs event limit: " + lineBytes); + @SuppressWarnings("unchecked") + void escapeHeavyRecordIsTruncatedUntilThePrintedLineFits() throws Exception { + for (int limit : new int[] {256_000, 1_000_000}) { + var exporter = new LambdaLogExporter(limit); + for (char ch : new char[] {'"', '\\'}) { + int count = limit == 1_000_000 ? 300_000 : 127_000; + var record = RecordFactory.sampleWithInput(String.valueOf(ch).repeat(count)); + int recordBytes = Json.byteSize(exporter.render(record)); + assertTrue(recordBytes <= limit, "record fits the record budget: " + recordBytes); + var shaped = Truncation.truncateRecord(record, limit, exporter::render); + assertSame(record, shaped, "record-level truncation keeps it whole"); + + String line = exportAndCapture(exporter, shaped).strip(); + int lineBytes = line.getBytes(StandardCharsets.UTF_8).length; + assertTrue(lineBytes <= limit, limit + ": printed line within the limit: " + lineBytes); + Map envelope = MAPPER.readValue(line, Map.class); + Map parsed = MAPPER.readValue((String) envelope.get("message"), Map.class); + assertEquals(Boolean.TRUE, parsed.get("truncated"), "record says it was truncated"); + assertEquals(record.executionArn(), parsed.get("executionArn")); + } } } } From 8af7e0480f02c20f3cf2a4e902944eb381cbfaa5 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Thu, 22 Oct 2026 09:16:44 -0700 Subject: [PATCH 11/12] fix(telemetry): strip only a hidden-class suffix from Java type names javaTypeName cut every type name at the first '/', and wrapperName passed callback and invoke error types through it, so an external type such as payments/v1.PaymentError was named 'payments' instead of PaymentError. Only a trailing /0x hidden-class suffix of Class.getName() (live or as the core checkpointed it) is removed now, and callback and invoke error types, which come from outside the JVM, are reduced as a whole by the Table A rule. --- telemetry-exporter/README.md | 7 +++-- .../telemetry/DurableExecutionTelemetry.java | 26 +++++++++++++---- .../telemetry/ErrorNameScenarioTest.java | 28 +++++++++++++++++++ .../durable/telemetry/ErrorNameTest.java | 5 ++++ 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index 13ab6ef3e..4a73b7037 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -285,9 +285,10 @@ How the Java exporter applies it: `pkg.Foo` (binary name `pkg.Foo$1LocalError`) gives `1LocalError`; an anonymous class gives its last segment, for example `1`. A string that ends with a separator (`trailing.`, `Outer$`) is kept as it is. U+00A0, U+FEFF, U+001C and U+2003 are not blank, so a type made of them is kept. -- A hidden class's name ends in `/0x…`, a suffix that changes on every JVM start. The name is cut - at the first `/` before rule 6, live and from the checkpoint, so `pkg.Gen$HiddenErr/0x…` gives - `HiddenErr`. +- A hidden class's name ends in `/0x…`, a suffix that changes on every JVM start. Only a trailing + suffix matching `/0x[0-9a-fA-F]+$` is removed before rule 6, live and from the checkpoint, so + `pkg.Gen$HiddenErr/0x…` gives `HiddenErr`. Callback and invoke error types come from outside the + JVM and are never cut: `payments/v1.PaymentError` gives `PaymentError`. - The SDK wrappers are an explicit list, matched on the exact runtime class: `DurableOperationException`, `UnrecoverableDurableExecutionException`, `StepException`, `CallbackException`, `InvokeException`, `StepFailedException`, `CallbackFailedException`, diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java index 8ab8d9d0a..112ab2e9f 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.IdentityHashMap; import java.util.Set; +import java.util.regex.Pattern; import java.util.function.Function; import java.util.function.UnaryOperator; import org.slf4j.Logger; @@ -302,7 +303,8 @@ private static String wrapperName(Throwable wrapper) { ErrorObject error = extractErrorObject(wrapper); String type = error == null ? null : error.errorType(); if (!isBlankType(type)) { - return javaTypeName(type); + // A callback or invoke error type is supplied from outside the JVM and is never a Java class name. + return hasExternalErrorType(wrapper) ? simpleTypeName(type) : javaTypeName(type); } if (wrapper instanceof DurableOperationException doe && doe.getOperation() != null) { var op = doe.getOperation(); @@ -317,6 +319,15 @@ private static String wrapperName(Throwable wrapper) { return DEFAULT_ERROR_NAME; } + /** {@code true} when the wrapper's checkpointed error type was written by a callback or invoke caller. */ + private static boolean hasExternalErrorType(Throwable wrapper) { + if (!(wrapper instanceof DurableOperationException doe) || doe.getOperation() == null) { + return false; + } + var type = doe.getOperation().type(); + return type == OperationType.CALLBACK || type == OperationType.CHAINED_INVOKE; + } + /** * The SDK error (not a wrapper) the core throws for a wrapper's operation, whatever its checkpoint says: a * timed-out callback, or a timed-out or stopped invoke. {@code null} for any other operation. @@ -387,17 +398,20 @@ private static Class contextFailure(String subType) { return null; } + /** The address suffix of a hidden class's {@link Class#getName()}, which changes on every JVM start. */ + private static final Pattern HIDDEN_CLASS_SUFFIX = Pattern.compile("/0x[0-9a-fA-F]+$"); + /** - * {@link #simpleTypeName} for a Java class name, live or checkpointed. The name of a hidden class has a - * {@code /0x…} suffix that changes on every JVM start, so the name is cut at the first {@code '/'} first and the - * same failure keeps the same name across cold starts. + * {@link #simpleTypeName} for a Java class name ({@link Class#getName()}), live or as the core checkpointed it. + * Only a hidden-class suffix matching {@code /0x[0-9a-fA-F]+$} is removed first, so the same failure keeps the same + * name across cold starts. No other {@code '/'} is touched, and type strings from outside the JVM (callback and + * invoke error types) never come here. */ static String javaTypeName(String className) { if (className == null) { return DEFAULT_ERROR_NAME; } - int slash = className.indexOf('/'); - return simpleTypeName(slash < 0 ? className : className.substring(0, slash)); + return simpleTypeName(HIDDEN_CLASS_SUFFIX.matcher(className).replaceFirst("")); } /** diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java index d7e8d7843..10308344f 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java @@ -785,4 +785,32 @@ void invokeTimeoutAndStopAreSdkErrors(Mode mode) { } } } + + /** + * Callback and invoke error types come from outside the JVM: a {@code '/'} in them is not a hidden-class suffix and + * the type is reduced as a whole (Table A). + */ + @ParameterizedTest + @EnumSource(Mode.class) + void externalTypeWithSlashIsReducedAsAWhole(Mode mode) { + String external = "payments/v1.PaymentError"; + var viaCallback = run( + (in, ctx) -> failure( + mode, ctx, c -> c.createCallback("cb", String.class).get()), + failCallback(external)); + var cb = observe(viaCallback, List.of("cb"), mode); + assertEquals("PaymentError", cb.execution().name(), "callback execution"); + assertEquals("PaymentError", cb.operations().get("cb").name(), "callback operation"); + + var viaInvoke = run((in, ctx) -> failure(mode, ctx, c -> c.invoke("callee", "fn", "x", String.class)), r -> { + r.run("in"); + r.failChainedInvoke( + "callee", + ErrorObject.builder().errorType(external).errorMessage(MSG).build()); + r.runUntilComplete("in"); + }); + var inv = observe(viaInvoke, List.of("callee"), mode); + assertEquals("PaymentError", inv.execution().name(), "invoke execution"); + assertEquals("PaymentError", inv.operations().get("callee").name(), "invoke operation"); + } } diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java index ae9c86e53..d5fce28fe 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java @@ -486,6 +486,11 @@ void hiddenClassNameDropsTheAddressSuffix() throws Throwable { .name()); assertEquals("Lambda", DurableExecutionTelemetry.javaTypeName("pkg.Foo$$Lambda/0x1")); assertEquals("Error", DurableExecutionTelemetry.javaTypeName("/0x1")); + assertEquals("Lambda", DurableExecutionTelemetry.javaTypeName("pkg.Foo$$Lambda/0x00007f3aBcD")); + // Only a trailing /0x is a hidden-class suffix; any other '/' is part of the name. + assertEquals("PaymentError", DurableExecutionTelemetry.javaTypeName("payments/v1.PaymentError")); + assertEquals("Foo/0xZZ", DurableExecutionTelemetry.javaTypeName("pkg.Foo/0xZZ")); + assertEquals("b/0x1", DurableExecutionTelemetry.javaTypeName("a.b/0x1/0x2")); } /** Rule 6 blank: only the six ASCII whitespace characters; the Table A vectors cover each single character. */ From f6350db27a02f6b404f9e546d74293ea900a568e Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Thu, 22 Oct 2026 09:30:35 -0700 Subject: [PATCH 12/12] refactor(telemetry): rely on the core's wrapped-error checkpoints (spec v3) The core now checkpoints the wrapped error of a failed context (fix/checkpoint-wrapped-error), so a failed step or context always has an error type. Drop the exporter fallback that named a typeless step or context after the SDK failure class the core would throw; keep it only for callbacks and invokes, whose callers can leave the type out and whose timeouts carry no error. A stopped or timed-out invoke inside a child context now has one name for the invoke, every enclosing context and the execution, live and replayed; add nested stop/timeout scenarios. Table B v3: B21 asserts the execution names the branch error that propagated; B5, B6 and B20 stay not applicable because Java map does not throw. README states rule 9 and the core requirement as in spec v3. --- telemetry-exporter/README.md | 33 +++++++--- .../telemetry/DurableExecutionTelemetry.java | 31 ++------- .../telemetry/ErrorNameScenarioTest.java | 66 ++++++++++++++++--- .../durable/telemetry/ErrorNameTest.java | 26 ++------ 4 files changed, 93 insertions(+), 63 deletions(-) diff --git a/telemetry-exporter/README.md b/telemetry-exporter/README.md index 4a73b7037..7d802ac49 100644 --- a/telemetry-exporter/README.md +++ b/telemetry-exporter/README.md @@ -263,9 +263,17 @@ this rule. The rule applies to the execution-level `error`, every `operations[]. 7. A failure gets the same name whether the record is built live or from a replayed checkpoint. 8. When the execution fails because an operation's error propagated, the execution `error.name` equals that operation's `error.name`. -9. If the SDK cannot tell which of several failures caused a wrapper, it does not guess. It - reports the wrapper's name. Two map items or parallel branches that fail with the same message - but different types are an example. +9. If a checkpoint holds only a wrapper type, which an older SDK can write, the name is that + wrapper's name. The exporter never guesses a cause from other operations. + +The core SDK checkpoints the wrapped error, not the wrapper. When a child context, map item or +parallel branch fails because an SDK wrapper error escaped it, the Java SDK writes the original +error's `ErrorType`, `ErrorMessage`, `ErrorData` and `StackTrace` into that context's checkpoint. +An SDK error that names the failure itself (a timed-out callback, a timed-out or stopped invoke) is +written under its own class name. A `waitForCallback` context keeps the callback's own error. So +the exporter names an operation by reducing its own `ErrorType` (rule 6) and names the execution +by unwrapping its error's cause chain (rules 2 to 5). It keeps no matching by message, timestamp or +tree position. A callback that is failed externally with an explicit `ErrorType` is named after that type, after the reduction of rule 6. This includes the type `"Error"`, which reports `Error`. Only a missing or @@ -304,12 +312,16 @@ How the Java exporter applies it: timeout. The operation and the execution report `CallbackTimeoutException`, `InvokeTimedOutException` or `InvokeStoppedException`, whatever error type the checkpoint carries. A stopped invoke is not named after its stop error: a stop with `errorType` - `States.Stopped` or `com.acme.PaymentError` reports `InvokeStoppedException`. + `States.Stopped` or `com.acme.PaymentError` reports `InvokeStoppedException`. Inside a child + context the core checkpoints that SDK error as the context's error, so the invoke, every + enclosing context and the execution report the same name, live and replayed. - After 16 unwraps, or at a cycle, the reported wrapper's name is its checkpointed `errorType`, reduced; with a blank type it is the SDK failure class below. -- When an operation failed with no error type (for example a callback failed externally without an - `errorType`), the operation and the execution report the SDK exception the core throws for it, - such as `CallbackFailedException`. +- When a callback or invoke failed with no error type (for example a callback failed externally + without an `errorType`), the operation and the execution report the SDK exception the core throws + for it, such as `CallbackFailedException`. A failed step or context always has a type in its + checkpoint. A context checkpoint written by an older SDK without one reports the wrapper's name + for the execution and no operation error (rule 9). - When the checkpoint has no error at all, the operation error has the SDK exception's name and a `null` message, because there is no original message to keep. This is the case for a timed-out callback and a timed-out invoke in the local test runner, and for a stopped invoke stopped @@ -317,9 +329,10 @@ How the Java exporter applies it: `error != null` now sees one with `"message": null`. The execution error of a timed-out callback has the message `Callback timed out: `, built by `CallbackTimeoutException`; it is not checkpointed, and the JS and Python exporters report `Callback timed out` without the id. -- Java `map` and `parallel` return item and branch failures as data and do not throw them, so the - SDK never has to choose between several failures (rule 9). The execution error is whatever the - handler throws. +- Java `map` and `parallel` return item and branch failures as data and do not throw them. Each + item and branch is named after its own type. For `parallel` the execution error is the error of + the branch whose future the handler calls `get()` on (Table B, B21); `map` has no `throwIfError`, + so B5, B6 and B20 have no propagated execution error in Java. - Non-class type strings are reduced too: an external callback failure with `errorType` `States.Timeout` reports `Timeout`. diff --git a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java index 112ab2e9f..6cfecfb5b 100644 --- a/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java +++ b/telemetry-exporter/src/main/java/software/amazon/lambda/durable/telemetry/DurableExecutionTelemetry.java @@ -6,9 +6,9 @@ import java.util.Collections; import java.util.IdentityHashMap; import java.util.Set; -import java.util.regex.Pattern; import java.util.function.Function; import java.util.function.UnaryOperator; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.lambda.model.ErrorObject; @@ -36,7 +36,6 @@ import software.amazon.lambda.durable.exception.StepInterruptedException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; import software.amazon.lambda.durable.exception.WaitForConditionFailedException; -import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; @@ -350,21 +349,16 @@ private static String sdkErrorOfOperation(Throwable wrapper) { } /** - * Returns the simple name of the SDK exception the core throws when an operation of this type, sub-type and status - * fails with no reconstructable user error, or {@code null} when there is none. Used when the checkpoint carries no - * error type, so an operation record and the execution error it propagates to get the same name. + * Returns the simple name of the SDK exception the core throws for a callback or invoke that failed, timed out or + * was stopped with no error type: a callback or invoke caller can leave the type out, and a timeout carries no + * error. {@code null} for any other operation. A failed step or context always checkpoints a type, because the core + * checkpoints the wrapped error of a failed context; an older SDK's checkpoint without one reports no name. */ static String sdkFailureName(String type, String subType, OperationStatus status) { if (type == null || status == null) { return null; } Class failure = switch (type) { - case "STEP" -> - status != OperationStatus.FAILED - ? null - : OperationSubType.WAIT_FOR_CONDITION.getValue().equals(subType) - ? WaitForConditionFailedException.class - : StepFailedException.class; case "CALLBACK" -> switch (status) { case FAILED -> CallbackFailedException.class; @@ -378,26 +372,11 @@ static String sdkFailureName(String type, String subType, OperationStatus status case STOPPED -> InvokeStoppedException.class; default -> null; }; - case "CONTEXT" -> status != OperationStatus.FAILED ? null : contextFailure(subType); default -> null; }; return failure == null ? null : failure.getSimpleName(); } - private static Class contextFailure(String subType) { - if (OperationSubType.MAP_ITERATION.getValue().equals(subType)) { - return MapIterationFailedException.class; - } - if (OperationSubType.PARALLEL_BRANCH.getValue().equals(subType)) { - return ParallelBranchFailedException.class; - } - if (OperationSubType.RUN_IN_CHILD_CONTEXT.getValue().equals(subType) - || OperationSubType.WITH_RETRY.getValue().equals(subType)) { - return ChildContextFailedException.class; - } - return null; - } - /** The address suffix of a hidden class's {@link Class#getName()}, which changes on every JVM start. */ private static final Pattern HIDDEN_CLASS_SUFFIX = Pattern.compile("/0x[0-9a-fA-F]+$"); diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java index 10308344f..5ff418664 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameScenarioTest.java @@ -307,12 +307,8 @@ private static void check( "B6", "same as B5: Java map does not throw item failures", "B20", - "same as B5: Java map does not throw item failures, so the SDK never has to choose between the two items" - + " (rule 9)", - "B21", - "Java parallel returns a ParallelResult of branch statuses and does not throw; only the branch future the" - + " handler calls get() on throws, and it throws that branch's own error, so the SDK never has to" - + " choose between branches (rule 9)"); + "same as B5: Java map does not throw item failures and has no throwIfError, so no item's error propagates;" + + " each item's own type is still asserted"); enum Row { B1, @@ -662,7 +658,7 @@ void tableB(Row row) { MSG, Map.of("fanout-iteration-0", "PaymentError", "fanout-iteration-1", "FraudError"), MSG); - case B21 -> // Two parallel branches fail with the same message and different types. + case B21 -> // Two parallel branches fail like B20; the execution names the branch error that propagated. check( "B21", m -> (in, ctx) -> failure(m, ctx, c -> { @@ -678,7 +674,9 @@ void tableB(Row row) { } }), null, - null, // execution: not applicable in Java, see EXECUTION_NOT_APPLICABLE + // Java parallel does not throw; the error that propagates is the one of the branch whose + // future the handler calls get() on. + "PaymentError", MSG, Map.of("pay", "PaymentError", "fraud", "FraudError"), MSG); @@ -786,6 +784,58 @@ void invokeTimeoutAndStopAreSdkErrors(Mode mode) { } } + /** + * An invoke that is stopped or times out inside a child context: the invoke, the enclosing context and the + * execution report the same SDK error name, live and replayed, whatever error type a stop carries. The core + * checkpoints the SDK error, not the stop error's type, as the context's error. Nested two contexts deep as well. + */ + @ParameterizedTest + @EnumSource(Mode.class) + void stoppedOrTimedOutInvokeInsideAChildHasOneName(Mode mode) { + for (boolean nested : new boolean[] {false, true}) { + for (String outcome : List.of("timeout", "stop-states", "stop-user")) { + String row = (nested ? "nested " : "") + outcome + " " + mode; + Function invokeInChild = c -> + c.runInChildContext("child", String.class, cc -> cc.invoke("callee", "fn", "x", String.class)); + var records = run( + (in, ctx) -> failure( + mode, + ctx, + nested + ? c -> c.runInChildContext("outer", String.class, invokeInChild::apply) + : invokeInChild), + r -> { + r.run("in"); + switch (outcome) { + case "timeout" -> r.timeoutChainedInvoke("callee"); + case "stop-states" -> + r.stopChainedInvoke( + "callee", + ErrorObject.builder() + .errorType("States.Stopped") + .errorMessage(MSG) + .build()); + default -> + r.stopChainedInvoke( + "callee", + ErrorObject.builder() + .errorType("com.acme.PaymentError") + .errorMessage(MSG) + .build()); + } + r.runUntilComplete("in"); + }); + String expected = "timeout".equals(outcome) ? "InvokeTimedOutException" : "InvokeStoppedException"; + var names = nested ? List.of("callee", "child", "outer") : List.of("callee", "child"); + var obs = observe(records, names, mode); + assertEquals(expected, obs.execution().name(), row + " execution"); + for (String name : names) { + assertEquals(expected, obs.operations().get(name).name(), row + " " + name); + } + } + } + } + /** * Callback and invoke error types come from outside the JVM: a {@code '/'} in them is not a hidden-class suffix and * the type is reduced as a whole (Table A). diff --git a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java index d5fce28fe..98a2b134e 100644 --- a/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java +++ b/telemetry-exporter/src/test/java/software/amazon/lambda/durable/telemetry/ErrorNameTest.java @@ -464,8 +464,8 @@ void sdkErrorsThatAreNotWrappersReportTheirOwnName() { } /** - * R2 (Java only): a hidden class's name ends in {@code /0x…}, which changes on every JVM start. The name is cut at - * the first {@code '/'} before the reduction, live and from the checkpoint. + * R2 (Java only): a hidden class's name ends in {@code /0x…}, which changes on every JVM start. Only that trailing + * {@code /0x} is removed before the reduction, live and from the checkpoint. */ @Test void hiddenClassNameDropsTheAddressSuffix() throws Throwable { @@ -549,12 +549,6 @@ void blankCheckpointedTypeUsesTheSdkFailureClassOfTheOperation() { @Test void sdkFailureNamesMatchTheCoreExceptions() { - assertEquals( - "StepFailedException", - DurableExecutionTelemetry.sdkFailureName("STEP", "Step", OperationStatus.FAILED)); - assertEquals( - "WaitForConditionFailedException", - DurableExecutionTelemetry.sdkFailureName("STEP", "WaitForCondition", OperationStatus.FAILED)); assertEquals( "CallbackFailedException", DurableExecutionTelemetry.sdkFailureName("CALLBACK", "Callback", OperationStatus.FAILED)); @@ -570,17 +564,11 @@ void sdkFailureNamesMatchTheCoreExceptions() { assertEquals( "InvokeStoppedException", DurableExecutionTelemetry.sdkFailureName("CHAINED_INVOKE", "ChainedInvoke", OperationStatus.STOPPED)); - assertEquals( - "MapIterationFailedException", - DurableExecutionTelemetry.sdkFailureName("CONTEXT", "MapIteration", OperationStatus.FAILED)); - assertEquals( - "ParallelBranchFailedException", - DurableExecutionTelemetry.sdkFailureName("CONTEXT", "ParallelBranch", OperationStatus.FAILED)); - assertEquals( - "ChildContextFailedException", - DurableExecutionTelemetry.sdkFailureName("CONTEXT", "RunInChildContext", OperationStatus.FAILED)); - assertNull(DurableExecutionTelemetry.sdkFailureName("CONTEXT", "WaitForCallback", OperationStatus.FAILED)); - assertNull(DurableExecutionTelemetry.sdkFailureName("STEP", "Step", OperationStatus.SUCCEEDED)); + // A failed step or context always checkpoints its wrapped error's type, so there is no fallback for them. + assertNull(DurableExecutionTelemetry.sdkFailureName("STEP", "Step", OperationStatus.FAILED)); + assertNull(DurableExecutionTelemetry.sdkFailureName("CONTEXT", "RunInChildContext", OperationStatus.FAILED)); + assertNull(DurableExecutionTelemetry.sdkFailureName("CONTEXT", "MapIteration", OperationStatus.FAILED)); + assertNull(DurableExecutionTelemetry.sdkFailureName("CALLBACK", "Callback", OperationStatus.SUCCEEDED)); assertNull(DurableExecutionTelemetry.sdkFailureName("WAIT", "Wait", OperationStatus.FAILED)); assertNull(DurableExecutionTelemetry.sdkFailureName(null, null, null)); }