From 73bb14ae5a282d9ef81822684ed77d10ae9e7fd6 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 31 Aug 2026 23:12:58 +0000 Subject: [PATCH 01/11] feat: add TLS-backed filesystem SerDes --- docs/adr/005-filesystem-serdes.md | 567 ++++-------------- docs/advanced/configuration.md | 38 ++ docs/design.md | 25 +- .../FileSystemSerDesIntegrationTest.java | 47 ++ .../testing/LocalDurableTestRunner.java | 2 + .../amazon/lambda/durable/DurableConfig.java | 48 ++ .../durable/execution/DurableExecutor.java | 31 +- .../durable/execution/ExecutionManager.java | 8 + .../operation/BaseDurableOperation.java | 12 + .../durable/operation/InvokeOperation.java | 2 +- .../SerializableDurableOperation.java | 15 +- .../durable/serde/FileSystemPathEncoding.java | 12 + .../durable/serde/FileSystemSerDes.java | 318 ++++++++++ .../durable/serde/FileSystemSerDesMode.java | 12 + .../lambda/durable/serde/SerDesContext.java | 47 ++ .../lambda/durable/serde/SerDesRunner.java | 159 +++++ .../lambda/durable/util/ExceptionHelper.java | 13 +- .../lambda/durable/DurableConfigTest.java | 51 ++ .../amazon/lambda/durable/TestUtils.java | 14 + .../operation/ChildContextOperationTest.java | 2 + .../operation/ConcurrencyOperationTest.java | 1 + .../operation/InvokeOperationTest.java | 2 + .../operation/ParallelOperationTest.java | 1 + .../SerializableDurableOperationTest.java | 3 + .../durable/operation/StepOperationTest.java | 2 + .../WaitForConditionOperationTest.java | 2 + .../durable/serde/FileSystemSerDesTest.java | 163 +++++ .../durable/serde/SerDesRunnerTest.java | 124 ++++ 28 files changed, 1246 insertions(+), 475 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDesMode.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index ef0a0a17e..e754a383e 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -1,35 +1,18 @@ -# ADR-005: Payload Offloading for Filesystem Storage +# ADR-005: Filesystem SerDes with Thread-Local Context -**Status:** Proposed -**Date:** 2026-07-02 +**Status:** Accepted +**Date:** 2026-08-31 ## Context -Issue [#463](https://github.com/aws/aws-durable-execution-sdk-java/issues/463) asks for Java parity with the JavaScript SDK's filesystem-backed SerDes. The JavaScript implementation receives a `SerdesContext` containing a stable durable execution ARN and an entity ID, then stores either inline JSON or a file pointer in the checkpoint payload. That context lets the implementation choose a collision-free path for each operation payload. +Issue [#463](https://github.com/aws/aws-durable-execution-sdk-java/issues/463) requests parity with the JavaScript +SDK's filesystem-backed SerDes. A filesystem SerDes needs two values that are not present in the existing Java +`SerDes` methods: -The Java SDK currently exposes a smaller `SerDes` contract: +- the durable execution ARN, which isolates files belonging to different executions; +- a stable entity ID, which identifies the execution or operation payload. -```java -String serialize(Object value); - T deserialize(String data, TypeToken typeToken); -``` - -That contract is enough for inline JSON but not enough for external payload storage because the implementation cannot tell which durable execution, operation, payload kind, or exception it is handling. This is also the blocker noted in [#509](https://github.com/aws/aws-durable-execution-sdk-java/issues/509). - -There are a few Java-specific constraints: - -- `SerDes` is called from different threads today. Serialization usually happens on an operation worker thread, while deserialization can happen on the operation caller's context thread when `DurableFuture.get()` is called. -- The same operation payload can be deserialized multiple times in one invocation because most operation results are not cached after deserialization. -- Java uses the configured `SerDes` for both operation results and user-defined exception objects stored in `ErrorObject.errorData`. -- `DurableInputOutputSerDes` is a hard-coded internal serializer for the Lambda Durable Functions request and response envelope. It is separate from the customer-facing `DurableConfig.getSerDes()`. -- Filesystem-backed storage is optional and storage-specific. It should not add filesystem-oriented public surface area to the core SDK artifact. -- Filesystem persistence is not automatically durable. Lambda `/tmp` is not valid for replay across environments. Mounted S3 Files may have delayed synchronization and can lose recent writes if the runtime crashes before the mount flushes. EFS or an explicitly accepted S3 Files durability tradeoff should be required for production use. - -## Approach A: Reuse SerDes for Offload - -### Summary - -Keep the existing `SerDes` contract unchanged and implement `FileSystemSerDes` as an optional extra package. The implementation uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. +The public Java interface is intentionally small and is already implemented by customers: ```java public interface SerDes { @@ -39,496 +22,172 @@ public interface SerDes { } ``` -`FileSystemSerDes` acts as both serializer and payload offloader. It serializes values through a delegate SerDes, writes payloads to the filesystem when configured to do so, and stores a small envelope in the checkpoint. - -Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local `SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the `SerDes` interface. - -```java -public record SerDesContext( - String durableExecutionArn, - String entityId, - SerDesPayloadKind payloadKind, - String operationId, - String operationName, - String parentId, - OperationType operationType, - OperationSubType operationSubType, - Integer attempt) { - public static SerDesContext getCurrentContext() { - return SerDesContextHolder.getCurrentContext(); - } -} -``` - -The SDK owns setting and clearing this thread-local value around SDK-managed SerDes calls. The setter should not be part of the public customer API; customers only read the current context. If SerDes is called directly by customer code outside the SDK, `getCurrentContext()` returns `null`. +Changing those methods would break existing implementations. Filesystem I/O must also avoid blocking the user-operation +executor or the SDK coordination executor, and repeated `DurableFuture.get()` calls must not repeatedly read and decode +the same file. -### Package +## Decision -| Concern | Decision | -|---------|----------| -| Maven module directory | `extra-filesystem-serdes` | -| Maven artifact ID | `aws-durable-execution-sdk-java-extra-filesystem-serdes` | -| Maven group ID | `software.amazon.lambda.durable` | -| Java package | `software.amazon.lambda.durable.extra.filesystem` | -| Core dependency direction | Extra module depends on `aws-durable-execution-sdk-java`; core does not depend on extras. | +### Preserve the SerDes interface -### Configuration +The existing `SerDes` interface remains unchanged. The SDK exposes the active payload identity through: ```java -import software.amazon.lambda.durable.extra.filesystem.FileSystemSerDes; - -var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) - .storageMode(FileSystemStorageMode.ALWAYS) - .pathEncoding(FileSystemPathEncoding.URI) - .delegate(new JacksonSerDes()) - .previewGenerator(optionalPreviewGenerator) - .build(); - -return DurableConfig.builder() - .withSerDes(serDes) - .build(); +public record SerDesContext(String durableExecutionArn, String entityId) { + public static SerDesContext getCurrentContext(); +} ``` -Storage modes: +`getCurrentContext()` returns `null` outside an SDK-managed SerDes call. The SDK owns setting and clearing the context; +there is no public setter. -| Mode | Behavior | -|------|----------| -| `ALWAYS` | Always write the delegate-serialized value to a file and store a file envelope in the checkpoint. | -| `OVERFLOW` | Store inline until the checkpoint envelope approaches the service payload limit, then write to a file. | +The implementation uses a plain `ThreadLocal`, not an `InheritableThreadLocal`. Every call restores the previous value +in `finally`, which supports nesting and prevents context from leaking when executor threads are reused. -Path encodings: +### Use an invocation-scoped SerDesRunner -| Encoding | Behavior | -|----------|----------| -| `URI` | Use readable, escaped path segments derived from durable execution ARN and entity ID. | -| `HASH` | Use SHA-256 names for fixed-length, filesystem-safe paths. | +Each `ExecutionManager` creates one `SerDesRunner` for the Lambda invocation. The runner: -Envelope format: +1. dispatches the SerDes call to the configured SerDes executor; +2. installs `SerDesContext` inside that executor task; +3. invokes the unchanged SerDes method; +4. clears or restores the thread-local context; +5. returns the result or rethrows the original failure. -```json -{"data":""} -{"file":""} -{"file":"","preview":{ "...": "..." }} -``` - -`FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include `durableExecutionArn` and `entityId`. - -### Runtime flow - -```java -SerDesContextHolder.set(context); -try { - var checkpointPayload = fileSystemSerDes.serialize(value); - sendCheckpoint(checkpointPayload); -} finally { - SerDesContextHolder.clear(); -} -``` - -On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it delegates directly to the inner SerDes. If the envelope contains `file`, it reads file contents and delegates to the inner SerDes. - -### Threading - -Add a separate executor to `DurableConfig`: +The configured executor is available through: ```java DurableConfig.builder() - .withSerDesExecutorService(customSerDesExecutor) + .withSerDesExecutorService(customExecutor) .build(); ``` -The default should be a cached daemon thread pool named `durable-sdk-serdes-*`. - -The core SDK should route user payload SerDes calls through a helper, tentatively `SerDesRunner`, that: - -- Builds the correct `SerDesContext`. -- Sets `SerDesContext` in TLS inside the SerDes executor task. -- Invokes the existing `SerDes.serialize` and `SerDes.deserialize` methods. -- Clears TLS after each SerDes call. -- Wraps failures in `SerDesException` with operation and payload kind metadata. - -Because TLS is bound to a single Java thread, `SerDesRunner` must set `SerDesContext` inside the SerDes executor task before calling the user SerDes. It must not rely on inheritable thread-local propagation from the operation thread because cached pool threads can be reused across operations and invocations. - -### Caching - -Add an invocation-scoped cache for successful deserialization results. The cache key should include: - -- Durable execution ARN. -- `entityId`. -- Payload kind. -- Target `TypeToken` type. -- A hash of the serialized checkpoint string. - -The serialized string hash prevents stale results when a `WAIT_FOR_CONDITION` or retried step updates the same operation payload across attempts. Cache entries live only for the current Lambda invocation and are discarded when `ExecutionManager` closes. - -With this approach, SDK caching can avoid repeated calls to `FileSystemSerDes.deserialize`. If a cache miss occurs, `FileSystemSerDes` may perform a file read internally. - -### Exceptions - -Keep the current `ErrorObject` shape: - -- `errorType`: the Java exception class name. -- `errorMessage`: the exception message. -- `errorData`: the SerDes payload or file pointer for the exception object. -- `stackTrace`: SDK-serialized stack trace entries. - -When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entity ID distinct from the operation result. When deserializing, continue to load `Class.forName(errorType)` and call SerDes with `TypeToken.get(exceptionClass.asSubclass(Throwable.class))`. - -`FileSystemSerDes` does not own exception type reconstruction. It only stores and loads the exception JSON or file pointer. Reconstruction remains in `SerializableDurableOperation.deserializeException` and `DurableExecutor.buildErrorObject`. - -### Input and output - -Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. +The default is a shared cached daemon pool named `durable-serdes-*`. The SerDes executor must not be the same object as +the user-operation executor because operation threads synchronously wait for SerDes work and a shared saturated pool +could deadlock. -### Implementation plan +### Cache successful deserializations per invocation -1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the `SerDes` interface unchanged. -2. Add `SerDesRunner` and a `SerDesExecutor` default pool. Add `DurableConfig.withSerDesExecutorService(...)` and validation. -3. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while leaving `DurableInputOutputSerDes` internal. -4. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. -5. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. -6. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -7. Add the `extra-filesystem-serdes` Maven module with artifact ID `aws-durable-execution-sdk-java-extra-filesystem-serdes`, depending on the core SDK. -8. Implement `FileSystemSerDes` in `software.amazon.lambda.durable.extra.filesystem` with `ALWAYS` and `OVERFLOW` modes, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where supported by the filesystem, and clear validation errors for missing context. -9. Add unit tests for context construction, unchanged `SerDes` compatibility, TLS scoping and clearing, thread-pool isolation, cache hits, cache invalidation when serialized data changes, exception reconstruction, malformed filesystem envelopes, and extra-module packaging. -10. Add integration tests with `LocalDurableTestRunner` for step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and custom exception types. -11. Update README and advanced configuration docs with FileSystemSerDes dependency coordinates, FileSystemSerDes examples, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. +`SerDesRunner` keeps up to 256 successful deserializations in a weak-reference LRU cache for the lifetime of one +`ExecutionManager`. The cache key contains: -### Pros +- SerDes instance identity; +- durable execution ARN; +- entity ID; +- target `TypeToken`; +- SHA-256 of the checkpoint string. -- Delivers the requested parity feature with the smallest new public API surface. -- Uses an extension point customers already understand and can configure per operation. -- Keeps the first implementation in an optional `aws-durable-execution-sdk-java-extra-*` module. -- Avoids committing the core SDK to a generalized offloading envelope before the storage use cases are proven. -- Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. +The serialized-data hash prevents stale results when the same entity is updated, such as a retried step or +`waitForCondition` state. Concurrent callers share one in-flight deserialization. Failed deserializations are removed +from the cache and can be retried. -### Cons +Repeated reads return the same object instance while the cached value remains reachable. A new invocation creates a new +runner and cache. -- Uses serialization as a storage hook, so the name `SerDes` no longer means only object-to-string conversion. -- Forces customers who already have a custom SerDes to wrap or compose it with FileSystemSerDes. -- May lead to one-off storage SerDes implementations if S3, DynamoDB, or other backends are added later. -- Makes it harder for the SDK to reason separately about serialized text size, offloaded payload references, and storage lifecycle. -- The SDK treats the checkpoint envelope as opaque serialized data, so lifecycle and preview behavior are owned by the SerDes implementation. +### Add FileSystemSerDes to the core SDK -## Approach B: Create a PayloadOffloader Interface - -### Summary - -Introduce a dedicated offloading abstraction in the core SDK. SerDes remains responsible only for object-to-string conversion. The offloader decides whether to keep serialized data inline or store it in third-party storage. +`FileSystemSerDes` is a normal `SerDes` in `software.amazon.lambda.durable.serde`. Keeping it in the existing SDK +artifact avoids a new module and allows it to be selected globally or through existing operation-level SerDes +configuration. ```java -public interface PayloadOffloader { - OffloadedPayload offload(String serializedPayload, PayloadOffloadContext context); - - String load(OffloadedPayload payload, PayloadOffloadContext context); -} -``` - -`OffloadedPayload` is an SDK-owned envelope model that can represent inline data, a storage reference, and optional preview data. - -```java -public record OffloadedPayload( - PayloadStorageMode mode, - String data, - String reference, - Map preview) {} -``` - -`PayloadOffloadContext` carries the stable payload identity directly as an explicit method parameter: - -```java -public record PayloadOffloadContext( - String durableExecutionArn, - String entityId, - SerDesPayloadKind payloadKind, - String operationId, - String operationName, - String parentId, - OperationType operationType, - OperationSubType operationSubType, - Integer attempt) {} -``` - -### Package - -The `PayloadOffloader` interface and SDK-owned envelope model belong in the core SDK because the core runtime must apply them uniformly to root input/output, operation results, invoke payloads, callback results, wait-for-condition state, and exception payloads. - -Filesystem-specific implementation remains an extra package: - -| Concern | Decision | -|---------|----------| -| Core API package | `software.amazon.lambda.durable.offload` | -| Extra Maven module directory | `extra-filesystem-offloader` | -| Extra Maven artifact ID | `aws-durable-execution-sdk-java-extra-filesystem-offloader` | -| Extra Java package | `software.amazon.lambda.durable.extra.filesystem` | -| Core dependency direction | Extra module depends on `aws-durable-execution-sdk-java`; core does not depend on extras. | - -### Configuration - -```java -import software.amazon.lambda.durable.extra.filesystem.FileSystemPayloadOffloader; - -var offloader = FileSystemPayloadOffloader.builder(Path.of("/mnt/efs/durable-payloads")) - .storageMode(PayloadOffloadMode.ALWAYS) - .pathEncoding(FileSystemPathEncoding.URI) - .previewGenerator(optionalPreviewGenerator) +var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .pathEncoding(FileSystemPathEncoding.HASH) + .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) .build(); return DurableConfig.builder() - .withSerDes(new JacksonSerDes()) - .withPayloadOffloader(offloader) + .withSerDes(serDes) .build(); ``` -Configuration needs a precedence model: - -| Level | Behavior | -|-------|----------| -| Global `DurableConfig.withPayloadOffloader(...)` | Applies to all user payloads unless operation config overrides it. | -| Operation config offloader | Overrides the global offloader for a step, invoke, callback, child context, map, or wait-for-condition operation. | -| Disabled offloader | Forces inline payload storage for payloads where external storage is not desired. | - -SerDes selection remains independent: - -- `SerDes` converts objects to and from serialized text. -- `PayloadOffloader` converts serialized text to and from checkpoint-safe inline data or storage references. - -### Runtime flow +Storage modes: -```java -var serialized = serDes.serialize(value); -var offloaded = payloadOffloader.offload(serialized, offloadContext); -var checkpointPayload = offloadEnvelopeSerDes.serialize(offloaded); -sendCheckpoint(checkpointPayload); -``` +| Mode | Behavior | +| --- | --- | +| `ALWAYS` | Store every SDK-managed non-null payload in a file. | +| `OVERFLOW` | Keep the versioned envelope inline until it exceeds 255 KiB, then store the payload in a file. | -On replay: +Path encodings: -```java -var offloaded = offloadEnvelopeSerDes.deserialize(checkpointPayload, OffloadedPayload.class); -var serialized = payloadOffloader.load(offloaded, offloadContext); -var value = serDes.deserialize(serialized, typeToken); -``` +| Encoding | Behavior | +| --- | --- | +| `URI` | Percent-encode readable execution and entity path segments. | +| `HASH` | Use fixed-length SHA-256 path segments. | -The SDK owns the checkpoint/offload envelope. Storage implementations own only the storage reference and the read/write mechanics. +The default delegate is `JacksonSerDes`; a custom delegate can be supplied through the builder. -### Threading +### Envelope and file publication -Use a separate executor for blocking payload I/O. This can be the same configured executor as SerDes work or a distinct executor if the team wants independent tuning: +Java writes versioned envelopes: -```java -DurableConfig.builder() - .withSerDesExecutorService(customSerDesExecutor) - .withPayloadOffloadExecutorService(customOffloadExecutor) - .build(); +```json +{"__durable_execution_filesystem_serdes":1,"data":""} +{"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":""} +{"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":"","preview":{"id":"123"}} ``` -If a single executor is preferred, name it according to the broader responsibility, for example `durable-sdk-payload-*`. - -Because filesystem/S3/DynamoDB offloading can block, offload work should not run on the user operation executor or the SDK internal executor. - -### Caching - -The SDK can cache at two layers: - -| Cache | Key | Value | -|-------|-----|-------| -| Offloaded payload cache | Durable execution ARN, entity ID, payload kind, checkpoint payload hash | Resolved serialized text | -| Deserialized object cache | Durable execution ARN, entity ID, payload kind, target type, serialized text hash | Deserialized object | - -This lets the SDK avoid repeated file reads and repeated object reconstruction independently. It is also easier for tests and diagnostics because the SDK can observe whether a checkpoint payload is inline or externally referenced. - -### Exceptions - -Exception handling becomes uniform. The SDK first serializes the exception object with SerDes, then offloads the resulting `errorData` just like any other payload. - -The `ErrorObject` shape can remain the same if `errorData` stores the SDK-owned offload envelope: - -- `errorType`: the Java exception class name. -- `errorMessage`: the exception message. -- `errorData`: inline serialized exception data or an SDK-owned offload envelope. -- `stackTrace`: SDK-serialized stack trace entries. - -Reconstruction remains in `SerializableDurableOperation.deserializeException` and `DurableExecutor.buildErrorObject`, but those paths must first resolve the offloaded `errorData` before calling SerDes. - -### Input and output - -Root user input and output payloads should use the same SerDes-plus-offloader pipeline. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. +The additional marker and digest are ignored by the JavaScript implementation, which reads the `data`, `file`, and +`preview` fields. Java also reads the unversioned JavaScript envelopes. -This approach gives the SDK one consistent policy for root payloads, operation results, invoke payloads, callbacks, wait-for-condition state, map results, and exception payloads. +File names include the entity ID and serialized-payload digest. Files are created with `CREATE_NEW`; an existing file is +accepted only when its contents match. This prevents a later retry from overwriting data referenced by an earlier +checkpoint. Deserialization rejects paths outside the configured base directory and verifies the digest when present. -### Implementation plan +### Initial invocation input -1. Add `SerDesPayloadKind` and a shared payload identity builder that can create `PayloadOffloadContext` for root input/output, operation results, invoke payloads, callback results, wait-for-condition state, map results, and exception payloads. -2. Add `PayloadOffloader`, `PayloadOffloadContext`, `OffloadedPayload`, and an SDK-owned offload envelope serializer in the core SDK. -3. Add `DurableConfig.withPayloadOffloader(...)` and optional operation-level offloader configuration. -4. Define precedence rules between global offloader, operation offloader, disabled offloader, result SerDes, payload SerDes, and callback deserializers. -5. Add a payload pipeline helper, tentatively `PayloadCodec`, that composes SerDes, offload, caching, executor routing, and exception wrapping. -6. Update root input/output handling in `DurableExecutor` to use the payload pipeline while leaving `DurableInputOutputSerDes` internal. -7. Update all operation result, invoke payload, callback result, wait-for-condition state, child context, map, and exception paths to use the payload pipeline. -8. Add offloaded payload caching and deserialized object caching. -9. Add the `extra-filesystem-offloader` Maven module with artifact ID `aws-durable-execution-sdk-java-extra-filesystem-offloader`, depending on the core SDK. -10. Implement `FileSystemPayloadOffloader` in `software.amazon.lambda.durable.extra.filesystem` with `ALWAYS` and `OVERFLOW` modes, `URI` and `HASH` path encodings, atomic file writes where supported by the filesystem, and clear validation errors for missing context. -11. Add unit tests for offload envelope compatibility, precedence rules, thread-pool isolation, cache hits, cache invalidation, exception reconstruction, malformed references, and extra-module packaging. -12. Add integration tests with `LocalDurableTestRunner` for step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from external references, and custom exception types. -13. Update README and advanced configuration docs with offloader dependency coordinates, filesystem offloader examples, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. +The durable execution ARN does not exist when a caller serializes the initial Lambda input. Therefore, +`FileSystemSerDes.serialize()` delegates directly when `SerDesContext.getCurrentContext()` is `null`. The initial input +remains ordinary delegate JSON. -### Pros - -- Cleaner separation between object encoding and payload storage. -- Storage offloading works with any SerDes implementation without replacing it. -- Gives the SDK one place to enforce checkpoint envelope format, thresholds, previews, caching, and validation. -- Scales naturally to more backends and policies if third-party payload storage becomes a first-class feature. -- Lets the SDK cache resolved serialized text separately from deserialized objects. -- Makes exception, callback, invoke, root input/output, and operation result offloading more uniform. - -### Cons - -- Requires a new core SDK extension point and configuration model. -- Needs careful interaction rules with operation-level SerDes, payload SerDes, callback deserializers, test helpers, and error serialization. -- Requires a migration story for existing custom SerDes implementations that already return external references. -- Slows direct FileSystemSerDes parity while the broader offloading API is designed and stabilized. -- Diverges from the JavaScript `createFileSystemSerdes` naming and shape, even if the behavior is similar. -- Adds more core SDK responsibility because the runtime now owns the offload envelope. - -## Approach Comparison - -| Dimension | Approach A: Reuse SerDes | Approach B: PayloadOffloader | -|-----------|--------------------------|------------------------------| -| Responsibility boundary | Combines value serialization and storage-reference creation in one implementation. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | -| User configuration | Users replace or wrap their SerDes with `FileSystemSerDes`. Operation-level SerDes selection already exists. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | -| Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Requires `SerDesContext` TLS because the existing SerDes contract has no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | -| Applicability | Only payloads using the filesystem SerDes are offloaded. Other SerDes implementations must implement their own offload behavior or be wrapped. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | -| Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | -| Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | -| Exception handling | Works if every exception serialization path is routed through SerDes with `SerDesPayloadKind.EXCEPTION`. | Works uniformly because exception `errorData` is another serialized payload that can be offloaded after SerDes. | -| Third-party storage | Filesystem-specific; S3/DynamoDB would likely become more SerDes wrappers or extra packages. | Natural home for multiple storage backends: filesystem, S3, DynamoDB, EFS, S3 Files, or custom customer storage. | -| Immediate delivery risk | Lower. Builds on existing customization point. | Higher. Requires new API and more runtime integration. | -| Long-term design risk | Higher. Blurs SerDes semantics and may accumulate storage behavior in serializers. | Lower if offloading grows into a first-class feature, but higher if this remains a one-off filesystem parity feature. | - -## AI Recommendation - -**AI recommendation:** Prefer **Approach B: Create a `PayloadOffloader` interface** if the team is willing to treat payload offloading as a first-class Java SDK capability rather than only a JavaScript parity item. - -Reasoning: - -- The problem being solved is payload storage, not serialization. A dedicated offloader keeps the domain boundary clean. -- The SDK already needs to touch every payload path for context, caching, threading, exceptions, and root input/output. Once that plumbing exists, composing SerDes plus offloader is a more durable shape than putting storage behavior inside SerDes. -- Java customers are more likely to have custom Jackson/ObjectMapper SerDes implementations. Approach B lets them keep those and add offloading independently. -- Both approaches use one optional extra package for filesystem-specific code; that is not a differentiator. The package would be either filesystem SerDes or filesystem offloader depending on the chosen approach. The differentiator is that Approach B gives future storage extras such as S3 or DynamoDB offload the same focused core offloader contract instead of encoding storage behavior as more SerDes implementations. -- SDK-owned envelopes and two-layer caching make replay behavior easier to test and reason about. - -The main reason to choose Approach A is schedule and parity: it is smaller and maps directly to the JavaScript feature request. If the team needs to satisfy #463 quickly with minimal public API design, Approach A is a reasonable incremental step, but it should be documented as payload offloading implemented through SerDes rather than as the long-term ideal boundary. - -## Other Alternatives Considered - -### Add FileSystemSerDes without SerDesContext - -Rejected. A filesystem-backed implementation needs stable operation identity. Without context, it cannot choose a safe file name, distinguish result and exception payloads for the same operation, or avoid collisions across durable executions. - -### Put filesystem-backed offloading in the core SDK artifact - -Rejected. Filesystem-backed storage is optional, storage-specific functionality. Keeping it in an `aws-durable-execution-sdk-java-extra-*` artifact preserves a small core SDK and creates a repeatable package shape for future optional features. - -### Add context-aware SerDes overloads - -Rejected for Approach A. Explicit overloads are more discoverable, but they expand the public `SerDes` interface and force context into every custom implementation's method surface. Approach A uses `SerDesContext` TLS only to keep the existing `SerDes` contract unchanged. Approach B does not need SerDes TLS because `PayloadOffloader` receives `PayloadOffloadContext` explicitly. - -### Make SerDes async - -Deferred. The TypeScript SDK uses async SerDes because file and service I/O are naturally async in Node.js. Java can isolate blocking work with dedicated executors while preserving synchronous user-facing interfaces. A future major version can revisit `CompletionStage` and `CompletionStage` if there is a stronger need. - -### Run payload storage on the user executor - -Rejected. Filesystem-backed storage can block on mounted storage. Running that work on the user executor can starve user operation threads and make unrelated steps appear stuck. - -### Run payload storage on the internal SDK executor - -Rejected. The internal executor is for checkpointing, polling, and coordination. Blocking storage work should not compete with progress-making SDK tasks. - -### Cache inside the filesystem implementation only - -Rejected. The repeated-deserialization problem exists for every payload implementation. SDK-level caching also lets the cache key include operation metadata, target type, and the serialized checkpoint string. - -### Make DurableInputOutputSerDes user-configurable - -Rejected. The backend request/response envelope is protocol data. User payload customization should happen at the user payload boundary, not at the protocol envelope boundary. +After the invocation starts, the SDK routes root input deserialization, operation payloads, exceptions, and root output +through `SerDesRunner`. Chained invokes that use filesystem SerDes require the caller and callee to use compatible +configuration and have access to the same durable filesystem. ## Consequences Positive: -- Both approaches enable filesystem-backed payload storage without changing the existing `SerDes` interface. -- Filesystem-specific functionality stays out of the core SDK artifact. -- The repository gets a repeatable `aws-durable-execution-sdk-java-extra-xxx` artifact pattern for optional packages. -- Custom payload implementations get enough context to use external storage safely. -- Blocking payload work is isolated from user operation and SDK coordination threads. -- Repeated file reads and repeated object reconstruction can be reduced within an invocation. -- User exception type reconstruction remains supported. +- Existing custom `SerDes` implementations remain source and binary compatible. +- Filesystem and custom external-storage SerDes implementations receive stable payload identity. +- Blocking SerDes work is isolated from user and SDK coordination executors. +- Repeated deserialization and file reads are avoided within an invocation. +- The implementation uses existing global and per-operation SerDes configuration. +- Java and JavaScript filesystem envelopes are mutually readable. Negative: -- Adds executor, context, and caching machinery that must stay deterministic. -- Adds at least one Maven module and published artifact to release and document. -- Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. -- Repeated `get()` calls may return the same object instance in one invocation. -- Filesystem-backed storage introduces operational durability requirements outside the SDK's control. -- Approach A risks overloading the meaning of SerDes. -- Approach B requires a larger core SDK design before delivering filesystem parity. - -Deferred: - -- Choosing whether payload offloading is a first-class SDK concept or a parity feature implemented through SerDes. -- A fully async Java SerDes or payload pipeline contract. -- A separate, explicitly dangerous protocol-envelope customization API. -- File cleanup, retention policies, and lifecycle management for offloaded payloads. - -## Shared Design Constraints - -These constraints apply to both approaches above. - -### Stable payload identity - -Both approaches need a stable payload identity that can be used to address external storage. The identity must include the durable execution ARN, operation identity, payload kind, and enough operation metadata to distinguish result, input, callback, wait-for-condition state, and exception payloads. - -`entityId` is the primary stable key for external storage. It must be unique within a durable execution and include the payload kind so one operation can safely store multiple values: +- SerDes context is implicit thread-local state. +- Every SDK-managed SerDes call crosses an executor boundary. +- Repeated deserialization returns the same object instance within an invocation. +- Filesystem retention and cleanup remain the application's responsibility. +- Chained invoke payloads and results require shared storage and compatible SerDes configuration. -| Payload | Example entity ID | -|---------|-------------------| -| Root input | `execution//input` | -| Root output | `execution//output` | -| Root exception | `execution//exception` | -| Step result | `operation//result` | -| Step exception | `operation//exception` | -| Invoke payload | `operation//invoke-payload` | -| Invoke result | `operation//result` | -| Callback result | `operation//result` | -| Child context result | `operation//result` | -| Map result | `operation//result` | -| WaitForCondition state | `operation//state` | +## Operational Requirements -Do not include the checkpoint token or raw user payload in the context. +- Do not use Lambda's ephemeral `/tmp` directory. Replay may run in another execution environment. +- Use a shared durable mount such as EFS. +- S3 Files users must accept its synchronization and crash-durability characteristics. +- Configure lifecycle cleanup separately; the SDK does not delete persisted payload files. -### Extra package pattern +## Alternatives Rejected -Payload offloading implementations should live outside the core SDK artifact when they target a specific storage mechanism. +### Change the SerDes method signatures -Use the `aws-durable-execution-sdk-java-extra-xxx` artifact pattern. The filesystem payload package name depends on which approach is chosen; the repository should not publish both a filesystem SerDes package and a filesystem offloader package for the same feature. +Rejected because adding context parameters would break every existing implementation. -| Feature | Artifact ID | Java package | -|---------|-------------|--------------| -| Filesystem payload storage, Approach A | `aws-durable-execution-sdk-java-extra-filesystem-serdes` | `software.amazon.lambda.durable.extra.filesystem` | -| Filesystem payload storage, Approach B | `aws-durable-execution-sdk-java-extra-filesystem-offloader` | `software.amazon.lambda.durable.extra.filesystem` | -| Event deserialization helpers | `aws-durable-execution-sdk-java-extra-event-deserialization` | `software.amazon.lambda.durable.extra.eventdeserialization` | -| Virtual thread executor helpers | `aws-durable-execution-sdk-java-extra-virtual-thread-pool` | `software.amazon.lambda.durable.extra.virtualthreads` | +### Add a PayloadOffloader abstraction -Extra modules should be independently documented, tested, and versioned with the repository release. They may depend on the core SDK and normal support libraries, but the core SDK should expose stable extension points without knowing about any specific extra package. For filesystem payload storage, create exactly one extra module after choosing Approach A or Approach B. +Rejected for this feature because it introduces a new extension point, envelope model, precedence rules, and operation +configuration. A dedicated offloader can be reconsidered if multiple storage backends require one common lifecycle. -### Protocol SerDes boundary +### Put filesystem support in a separate Maven module -Do not make `DurableInputOutputSerDes` customizable as part of this ADR. It serializes the Lambda Durable Functions backend protocol envelope, not user payloads. Routing that envelope through external payload storage would risk storing checkpoint tokens or protocol data externally and would require the backend to understand file pointers. +Rejected for the initial implementation. The filesystem implementation uses only JDK and existing Jackson APIs, so a +new artifact would add release and documentation overhead without isolating an additional dependency. -User input and output payloads should still use the configured user payload mechanism when they are extracted from or written to the execution operation. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope remains handled by `DurableInputOutputSerDes`. +### Run SerDes inline or omit caching -If protocol customization is needed later, introduce a separate `ProtocolSerDes` configuration surface with a clear warning that it must produce the exact backend wire format. Do not reuse the user payload `SerDes` for that purpose. +Rejected because mounted filesystem I/O can block user progress and repeated reads can repeat externally visible work +and cost. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index bc8bf89d0..93572343e 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -34,12 +34,50 @@ public class OrderProcessor extends DurableHandler { | `withLambdaClientBuilder()` | Custom AWS Lambda client | Auto-configured Lambda client | | `withSerDes()` | Serializer for step results | Jackson with default settings | | `withExecutorService()` | Thread pool for user-defined operations | Cached daemon thread pool | +| `withSerDesExecutorService()` | Thread pool for serialization and payload I/O | Cached daemon thread pool | | `withLoggerConfig()` | Logger behavior configuration | Suppress logs during replay | | `withPollingStrategy()` | Backend polling strategy | Exponential backoff: 1s base, 2x rate, FULL jitter, 10s max | | `withCheckpointDelay()` | How often the SDK checkpoints updates | `Duration.ofSeconds(0)` (as soon as possible) | The `withExecutorService()` option configures the thread pool used for running user-defined operations. Internal SDK coordination (checkpoint batching, polling) runs on an SDK-managed thread pool. +The SerDes executor must be different from the user-operation executor. SerDes calls are synchronous from the +operation's perspective, so using one saturated pool for both can deadlock. + +### Filesystem-backed SerDes + +`FileSystemSerDes` stores operation and execution payloads on a shared durable filesystem while leaving small values +inline when configured for overflow mode: + +```java +var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .pathEncoding(FileSystemPathEncoding.HASH) + .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) + .build(); + +return DurableConfig.builder() + .withSerDes(fileSystemSerDes) + .build(); +``` + +`ALWAYS` writes every SDK-managed payload to a file. `OVERFLOW` stores the payload inline until the complete checkpoint +envelope exceeds 255 KiB. `URI` path encoding keeps identifiers readable, while `HASH` avoids filesystem name-length +and character restrictions. + +The SDK supplies a `SerDesContext` through thread-local storage during managed calls: + +```java +var context = SerDesContext.getCurrentContext(); +``` + +Custom SerDes implementations can use its durable execution ARN and entity ID for external storage. Calls run on the +dedicated SerDes executor, and successful deserializations are cached for the current Lambda invocation. + +Do not use Lambda's `/tmp` directory: replay can run in another execution environment. Use a shared durable mount such +as EFS. S3 Files users must account for synchronization and crash-durability behavior. Chained invokes require both +functions to use compatible SerDes configuration and access the same mount. + ### Dynamic plugin loading Dynamic plugin loading is an opt-in alternative to registering plugins in application code. Put provider JARs on the application class path, then set `DURABLE_EXECUTION_PLUGINS` to an ordered, comma-separated list of provider names: diff --git a/docs/design.md b/docs/design.md index eaba54af0..b9ce833eb 100644 --- a/docs/design.md +++ b/docs/design.md @@ -347,9 +347,12 @@ software.amazon.lambda.durable │ └── WaitForConditionResult # Check function return type (value + isDone) │ ├── serde/ -│ ├── SerDes # Interface -│ ├── JacksonSerDes # Jackson impl -│ └── AwsSdkV2Module # SDK type support +│ ├── SerDes # Interface +│ ├── JacksonSerDes # Jackson impl +│ ├── FileSystemSerDes # Shared-filesystem payload storage +│ ├── SerDesContext # Thread-local durable payload identity +│ ├── SerDesRunner # Executor dispatch + invocation cache +│ └── AwsSdkV2Module # SDK type support │ └── exception/ ├── DurableExecutionException @@ -653,20 +656,28 @@ For testing, use `DurableConfig.builder().withDurableExecutionClient(localMemory ```java public interface SerDes { String serialize(Object value); - T deserialize(String data, Class type); T deserialize(String data, TypeToken typeToken); } ``` +SDK-managed calls go through an invocation-scoped `SerDesRunner`. The runner dispatches work to the dedicated SerDes +executor, installs a `SerDesContext` in plain thread-local storage for the duration of the call, and caches successful +deserializations in a bounded weak-reference LRU by SerDes identity, execution ARN, entity ID, target type, and +serialized-data hash. The thread-local value is always restored in `finally`. + +`FileSystemSerDes` uses that context to build collision-free paths on a shared durable filesystem. Calls made before a +durable execution ARN exists, such as initial invocation input serialization, fall back to the delegate SerDes without +filesystem storage. + **TypeToken and Type Erasure:** Java's type erasure removes generic type parameters at runtime (`List` becomes `List`). This is problematic for deserialization—Jackson needs the full type to reconstruct objects correctly. `TypeToken` solves this by capturing generic types at compile time. Creating `new TypeToken>() {}` produces an anonymous subclass whose superclass type parameter is preserved in bytecode and accessible via reflection (`getGenericSuperclass()`). -The `SerDes` interface provides both `Class` and `TypeToken` overloads: -- Use `Class` for simple types: `String.class`, `User.class` -- Use `TypeToken` for parameterized types: `new TypeToken>() {}` +Durable operation APIs provide both `Class` and `TypeToken` overloads: +- Use `Class` for simple operation result types: `String.class`, `User.class` +- Use `TypeToken` for parameterized operation result types: `new TypeToken>() {}` --- diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java new file mode 100644 index 000000000..2eaec17b9 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -0,0 +1,47 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.serde.FileSystemSerDes; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class FileSystemSerDesIntegrationTest { + @TempDir + Path tempDir; + + @Test + void storesStepAndExecutionResultsAcrossReplay() throws Exception { + var stepRuns = new AtomicInteger(); + var serDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var result = context.step("persist", String.class, stepContext -> { + stepRuns.incrementAndGet(); + return input + "-stored"; + }); + context.wait("replay", Duration.ofSeconds(1)); + return result; + }, + config); + + var result = runner.runUntilComplete("value"); + + assertEquals("value-stored", result.getResult(String.class)); + assertEquals("value-stored", result.getOperation("persist").getStepResult(String.class)); + assertEquals(1, stepRuns.get()); + try (var files = Files.walk(tempDir)) { + assertTrue(files.filter(Files::isRegularFile).count() >= 2); + } + } +} diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 06d59d5d3..0d25f1d68 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -65,9 +65,11 @@ private LocalDurableTestRunner( .withDurableExecutionClient(storage) .withSerDes(customerConfig.getSerDes()) .withExecutorService(customerConfig.getExecutorService()) + .withSerDesExecutorService(customerConfig.getSerDesExecutorService()) .withPollingStrategy(customerConfig.getPollingStrategy()) .withCheckpointDelay(customerConfig.getCheckpointDelay()) .withLoggerConfig(customerConfig.getLoggerConfig()) + .withDeserializeAfterSerialization(customerConfig.shouldDeserializeAfterSerialization()) // Temporary: remove along with the checkpointEmptyMap flag in a future major version. .withCheckpointEmptyMap(customerConfig.shouldCheckpointEmptyMap()) .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java index 5101b9fda..e251701d4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java @@ -92,9 +92,18 @@ public final class DurableConfig { return t; }); + /** Default executor for customer SerDes calls and blocking payload storage I/O. */ + private static final ExecutorService DEFAULT_SERDES_THREAD_POOL = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r); + t.setName("durable-serdes-" + t.getId()); + t.setDaemon(true); + return t; + }); + private final DurableExecutionClient durableExecutionClient; private final SerDes serDes; private final ExecutorService executorService; + private final ExecutorService serDesExecutorService; private final LoggerConfig loggerConfig; private final PollingStrategy pollingStrategy; private final Duration checkpointDelay; @@ -109,6 +118,8 @@ private DurableConfig(Builder builder) { this.serDes = Objects.requireNonNullElseGet(builder.serDes, JacksonSerDes::new); this.executorService = Objects.requireNonNullElseGet(builder.executorService, DurableConfig::createDefaultExecutor); + this.serDesExecutorService = Objects.requireNonNullElseGet( + builder.serDesExecutorService, DurableConfig::createDefaultSerDesExecutor); this.loggerConfig = Objects.requireNonNullElseGet(builder.loggerConfig, LoggerConfig::defaults); this.pollingStrategy = Objects.requireNonNullElse(builder.pollingStrategy, PollingStrategies.Presets.DEFAULT); this.checkpointDelay = Objects.requireNonNullElseGet(builder.checkpointDelay, () -> Duration.ofSeconds(0)); @@ -164,6 +175,15 @@ public ExecutorService getExecutorService() { return executorService; } + /** + * Gets the executor used for customer SerDes calls and blocking payload storage I/O. + * + * @return SerDes ExecutorService instance (never null) + */ + public ExecutorService getSerDesExecutorService() { + return serDesExecutorService; + } + /** * Gets the configured LoggerConfig. * @@ -235,6 +255,13 @@ public void validateConfiguration() { if (getExecutorService() == null) { throw new IllegalStateException("ExecutorService configuration failed"); } + if (getSerDesExecutorService() == null) { + throw new IllegalStateException("SerDes ExecutorService configuration failed"); + } + if (getSerDesExecutorService() == getExecutorService()) { + throw new IllegalStateException( + "SerDes ExecutorService must be different from the user operation ExecutorService"); + } } /** @@ -311,11 +338,17 @@ private static ExecutorService createDefaultExecutor() { return DEFAULT_USER_THREAD_POOL; } + private static ExecutorService createDefaultSerDesExecutor() { + logger.debug("Creating default SerDes ExecutorService"); + return DEFAULT_SERDES_THREAD_POOL; + } + /** Builder for DurableConfig. Provides fluent API for configuring SDK components. */ public static final class Builder { private DurableExecutionClient durableExecutionClient; private SerDes serDes; private ExecutorService executorService; + private ExecutorService serDesExecutorService; private LoggerConfig loggerConfig; private PollingStrategy pollingStrategy; private Duration checkpointDelay; @@ -396,6 +429,21 @@ public Builder withExecutorService(ExecutorService executorService) { return this; } + /** + * Sets a dedicated executor for SerDes calls and blocking payload storage I/O. + * + *

The SerDes executor must be different from the user operation executor to avoid deadlock when operation + * threads synchronously wait for serialization. + * + * @param executorService dedicated SerDes ExecutorService + * @return This builder + */ + public Builder withSerDesExecutorService(ExecutorService executorService) { + this.serDesExecutorService = + Objects.requireNonNull(executorService, "SerDes ExecutorService cannot be null"); + return this; + } + /** * Sets a custom LoggerConfig. If not set, defaults to suppressing replay logs. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index d8db91326..9449e26ce 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -32,6 +32,8 @@ import software.amazon.lambda.durable.plugin.PluginInfoConverter; import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -60,6 +62,9 @@ public static DurableExecutionOutput execute( var isFirstInvocation = !executionManager.isReplaying(); var requestId = lambdaContext != null ? lambdaContext.getAwsRequestId() : null; var executionArn = input.durableExecutionArn(); + var serDesContext = new SerDesContext( + executionArn, executionManager.getExecutionOperation().id()); + var serDesRunner = executionManager.getSerDesRunner(); executionManager.registerActiveThread(null); // Captured for onInvocationEnd, which runs outside the handler thread below. @@ -78,7 +83,11 @@ public static DurableExecutionOutput execute( Throwable inputFailure = null; try { userInput = extractUserInput( - executionManager.getExecutionOperation(), config.getSerDes(), inputType); + executionManager.getExecutionOperation(), + config.getSerDes(), + inputType, + serDesRunner, + serDesContext); } catch (Throwable t) { inputFailure = t; } @@ -171,11 +180,12 @@ public static DurableExecutionOutput execute( cause, pluginExecutionInput.get(), null); - return DurableExecutionOutput.failure(buildErrorObject(cause, config.getSerDes())); + return DurableExecutionOutput.failure( + buildErrorObject(cause, config.getSerDes(), serDesRunner, serDesContext)); } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = config.getSerDes().serialize(result); + var outputPayload = serDesRunner.serialize(config.getSerDes(), result, serDesContext); var output = DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); fireOnInvocationEnd( @@ -254,7 +264,8 @@ private static String handleLargePayload(ExecutionManager executionManager, Stri return outputPayload; } - private static ErrorObject buildErrorObject(Throwable e, SerDes serDes) { + private static ErrorObject buildErrorObject( + Throwable e, SerDes serDes, SerDesRunner serDesRunner, SerDesContext serDesContext) { // exceptions thrown from operations, e.g. Step if (e instanceof DurableOperationException durableOperationException) { return durableOperationException.getErrorObject(); @@ -263,16 +274,22 @@ private static ErrorObject buildErrorObject(Throwable e, SerDes serDes) { return unrecoverableDurableExecutionException.getErrorObject(); } // exceptions thrown from non-operation code - return ExceptionHelper.buildErrorObject(e, serDes); + var errorData = serDesRunner.serialize(serDes, e, serDesContext); + return ExceptionHelper.buildErrorObject(e, errorData); } - private static I extractUserInput(Operation executionOp, SerDes serDes, TypeToken inputType) { + private static I extractUserInput( + Operation executionOp, + SerDes serDes, + TypeToken inputType, + SerDesRunner serDesRunner, + SerDesContext serDesContext) { if (executionOp.executionDetails() == null) { throw new IllegalDurableOperationException("EXECUTION operation missing executionDetails"); } var inputPayload = executionOp.executionDetails().inputPayload(); - return serDes.deserialize(inputPayload, inputType); + return serDesRunner.deserialize(serDes, inputPayload, inputType, serDesContext); } /** diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 0e9d8426e..09d6e54c7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -30,6 +30,7 @@ import software.amazon.lambda.durable.model.SafeCloseable; import software.amazon.lambda.durable.operation.BaseDurableOperation; import software.amazon.lambda.durable.plugin.PluginInfoConverter; +import software.amazon.lambda.durable.serde.SerDesRunner; /** * Central manager for durable execution coordination. @@ -65,6 +66,7 @@ public class ExecutionManager implements SafeCloseable { private final DurableConfig durableConfig; private final Set updatedOperationIdsSinceLastInvocation; private final Set initialOperationIds; + private final SerDesRunner serDesRunner; // ===== Thread Coordination ===== private final Map registeredOperations = new ConcurrentHashMap<>(); @@ -81,6 +83,7 @@ public ExecutionManager(DurableExecutionInput input, DurableConfig config, Conte durableConfig = config; this.durableExecutionArn = input.durableExecutionArn(); this.lambdaContext = lambdaContext; + this.serDesRunner = new SerDesRunner(config.getSerDesExecutorService()); // Store the set of operation IDs updated since the last successful invocation this.updatedOperationIdsSinceLastInvocation = @@ -129,6 +132,11 @@ public String getDurableExecutionArn() { return durableExecutionArn; } + /** Returns the invocation-scoped SerDes runner. */ + public SerDesRunner getSerDesRunner() { + return serDesRunner; + } + /** Returns {@code true} if the execution is currently replaying completed operations. */ public boolean isReplaying() { return executionMode.get() == ExecutionMode.REPLAY; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index 5cd40820e..b8b4a1706 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -30,6 +30,8 @@ import software.amazon.lambda.durable.plugin.PluginInfoConverter; import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.plugin.UserFunctionOutcome; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -113,6 +115,16 @@ public String getName() { return operationIdentifier.name(); } + /** Returns the context used for SerDes calls belonging to this operation. */ + protected SerDesContext getSerDesContext() { + return new SerDesContext(executionManager.getDurableExecutionArn(), getOperationId()); + } + + /** Returns the invocation-scoped SerDes runner. */ + protected SerDesRunner getSerDesRunner() { + return executionManager.getSerDesRunner(); + } + /** Gets the parent context. */ protected DurableContextImpl getContext() { return durableContext; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index 9e2c54ace..421123303 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -70,7 +70,7 @@ private void startInvocation() { .functionName(functionName) .tenantId(invokeConfig.tenantId()) .build()) - .payload(payloadSerDes.serialize(this.payload)); + .payload(getSerDesRunner().serialize(payloadSerDes, this.payload, getSerDesContext())); sendOperationUpdate(update); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index 6457c996d..a8b7c821e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -86,7 +86,7 @@ protected SerializableDurableOperation( */ protected T deserializeResult(String result) { try { - return resultSerDes.deserialize(result, resultTypeToken); + return getSerDesRunner().deserialize(resultSerDes, result, resultTypeToken, getSerDesContext()); } catch (SerDesException e) { logger.warn( "Failed to deserialize {} result for operation name '{}'. Ensure the result is properly encoded.", @@ -106,7 +106,7 @@ protected T deserializeResult(String result) { * @return the serialized string and the deserialized result */ protected SerializedResult serializeAndDeserializeResult(T result) { - var serialized = resultSerDes.serialize(result); + var serialized = getSerDesRunner().serialize(resultSerDes, result, getSerDesContext()); var deserialized = shouldDeserializeAfterSerialization() ? deserializeResult(serialized) : result; return new SerializedResult<>(serialized, deserialized); } @@ -119,7 +119,8 @@ protected SerializedResult serializeAndDeserializeResult(T result) { */ @SuppressWarnings("ThrowableNotThrown") protected ErrorObject serializeException(Throwable throwable) { - var error = ExceptionHelper.buildErrorObject(throwable, resultSerDes); + var errorData = getSerDesRunner().serialize(resultSerDes, throwable, getSerDesContext()); + var error = ExceptionHelper.buildErrorObject(throwable, errorData); if (shouldDeserializeAfterSerialization()) { deserializeException(error); } @@ -153,8 +154,12 @@ protected Throwable deserializeException(ErrorObject errorObject) { Class exceptionClass = Class.forName(errorType); if (Throwable.class.isAssignableFrom(exceptionClass)) { - original = - resultSerDes.deserialize(errorData, TypeToken.get(exceptionClass.asSubclass(Throwable.class))); + original = getSerDesRunner() + .deserialize( + resultSerDes, + errorData, + TypeToken.get(exceptionClass.asSubclass(Throwable.class)), + getSerDesContext()); if (original != null) { original.setStackTrace(ExceptionHelper.deserializeStackTrace(errorObject.stackTrace())); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java new file mode 100644 index 000000000..8e7629a8f --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Controls how durable execution and entity identifiers are encoded into filesystem paths. */ +public enum FileSystemPathEncoding { + /** Percent-encode identifiers to keep paths human-readable. */ + URI, + + /** Replace identifiers with fixed-length SHA-256 hashes. */ + HASH +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java new file mode 100644 index 000000000..76208cf94 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -0,0 +1,318 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.regex.Pattern; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; + +/** + * A SerDes that stores checkpoint payloads on a durable shared filesystem. + * + *

Do not use Lambda's ephemeral {@code /tmp} storage. The base path must be available to every execution environment + * that can serialize or deserialize the payload, such as an EFS mount or an S3 Files mount whose synchronization + * tradeoffs are acceptable for the workload. + * + *

Initial invocation input is serialized normally when no {@link SerDesContext} is available, because the durable + * execution ARN does not exist until after invocation starts. SDK-managed operation and output payloads are processed + * using the configured storage mode. + */ +public final class FileSystemSerDes implements SerDes { + private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final int ENVELOPE_VERSION = 1; + private static final int OVERFLOW_THRESHOLD_BYTES = 256 * 1024 - 1024; + private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); + private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( + "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); + + private final Path basePath; + private final FileSystemSerDesMode storageMode; + private final FileSystemPathEncoding pathEncoding; + private final SerDes delegate; + private final Function> previewGenerator; + + private FileSystemSerDes(Builder builder) { + basePath = builder.basePath.toAbsolutePath().normalize(); + storageMode = builder.storageMode; + pathEncoding = builder.pathEncoding; + delegate = builder.delegate; + previewGenerator = builder.previewGenerator; + } + + /** Creates a builder rooted at the given durable shared filesystem path. */ + public static Builder builder(Path basePath) { + return new Builder(basePath); + } + + @Override + public String serialize(Object value) { + var serialized = delegate.serialize(value); + if (serialized == null) { + return null; + } + + var context = SerDesContext.getCurrentContext(); + if (context == null) { + return serialized; + } + + if (storageMode == FileSystemSerDesMode.OVERFLOW) { + var inlineEnvelope = inlineEnvelope(serialized); + if (inlineEnvelope.getBytes(StandardCharsets.UTF_8).length <= OVERFLOW_THRESHOLD_BYTES) { + return inlineEnvelope; + } + } + + return fileEnvelope(value, serialized, context); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + if (data == null) { + return null; + } + + var envelope = parseEnvelope(data); + if (envelope == null) { + return delegate.deserialize(data, typeToken); + } + if (envelope.hasNonNull("data")) { + return delegate.deserialize(envelope.get("data").textValue(), typeToken); + } + + var serialized = readPayload(envelope.get("file").textValue()); + if (envelope.hasNonNull("sha256")) { + var expected = envelope.get("sha256").textValue(); + if (!expected.equals(sha256(serialized))) { + throw new SerDesException("Filesystem SerDes payload digest does not match stored content"); + } + } + return delegate.deserialize(serialized, typeToken); + } + + private String inlineEnvelope(String serialized) { + var envelope = ENVELOPE_MAPPER.createObjectNode(); + envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); + envelope.put("data", serialized); + return writeEnvelope(envelope); + } + + private String fileEnvelope(Object value, String serialized, SerDesContext context) { + var digest = sha256(serialized); + var file = payloadPath(context, digest); + + var envelope = ENVELOPE_MAPPER.createObjectNode(); + envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); + envelope.put("file", file.toString()); + envelope.put("sha256", digest); + if (previewGenerator != null) { + var preview = previewGenerator.apply(value); + if (preview != null) { + envelope.set("preview", ENVELOPE_MAPPER.valueToTree(preview)); + } + } + var encoded = writeEnvelope(envelope); + if (encoded.getBytes(StandardCharsets.UTF_8).length > OVERFLOW_THRESHOLD_BYTES) { + throw new SerDesException("Filesystem SerDes file envelope exceeds the checkpoint payload limit"); + } + + writePayload(file, serialized); + return encoded; + } + + private JsonNode parseEnvelope(String data) { + final JsonNode node; + try { + node = ENVELOPE_MAPPER.readTree(data); + } catch (IOException e) { + return null; + } + if (node == null || !node.isObject()) { + return null; + } + + if (node.has(ENVELOPE_MARKER)) { + if (!node.get(ENVELOPE_MARKER).canConvertToInt() + || node.get(ENVELOPE_MARKER).intValue() != ENVELOPE_VERSION + || !isValidEnvelope(node)) { + throw new SerDesException("Malformed filesystem SerDes envelope"); + } + return node; + } + + // Read envelopes produced by the JavaScript SDK while leaving ordinary user JSON untouched. + var legacyFieldCount = node.has("preview") ? 2 : 1; + return node.size() == legacyFieldCount && isValidEnvelope(node) ? node : null; + } + + private static boolean isValidEnvelope(JsonNode node) { + var hasData = node.has("data") && node.get("data").isTextual(); + var hasFile = node.has("file") && node.get("file").isTextual(); + if (hasData == hasFile) { + return false; + } + if (node.has("preview") && !node.get("preview").isObject()) { + return false; + } + return !node.has("sha256") || node.get("sha256").isTextual(); + } + + private Path payloadPath(SerDesContext context, String digest) { + var directory = executionDirectory(context.durableExecutionArn()); + var fileName = encode(context.entityId()) + "-" + digest + ".json"; + var file = directory.resolve(fileName).toAbsolutePath().normalize(); + if (!file.startsWith(basePath)) { + throw new SerDesException("Filesystem SerDes path escapes the configured base path"); + } + return file; + } + + private Path executionDirectory(String durableExecutionArn) { + if (pathEncoding == FileSystemPathEncoding.URI) { + var match = DURABLE_EXECUTION_ARN_PATTERN.matcher(durableExecutionArn); + if (match.matches()) { + return basePath.resolve(encode(match.group(1))) + .resolve(encode(match.group(2))) + .resolve(encode(match.group(3))); + } + } + return basePath.resolve(encode(durableExecutionArn)); + } + + private void writePayload(Path file, String serialized) { + try { + Files.createDirectories(file.getParent()); + var realBase = basePath.toRealPath(); + var realParent = file.getParent().toRealPath(); + if (!realParent.startsWith(realBase)) { + throw new SerDesException("Filesystem SerDes path resolves outside the configured base path"); + } + try { + Files.writeString( + file, + serialized, + StandardCharsets.UTF_8, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE); + } catch (FileAlreadyExistsException e) { + var existing = readPayload(file.toString()); + if (!existing.equals(serialized)) { + throw new SerDesException("Filesystem SerDes payload file already exists with different content"); + } + } + } catch (IOException e) { + throw new SerDesException("Failed to store filesystem SerDes payload", e); + } + } + + private String readPayload(String fileValue) { + var file = Path.of(fileValue).toAbsolutePath().normalize(); + if (!file.startsWith(basePath)) { + throw new SerDesException("Filesystem SerDes file is outside the configured base path"); + } + try { + var realBase = basePath.toRealPath(); + var realFile = file.toRealPath(); + if (!realFile.startsWith(realBase)) { + throw new SerDesException("Filesystem SerDes file resolves outside the configured base path"); + } + return Files.readString(realFile, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new SerDesException("Failed to load filesystem SerDes payload", e); + } + } + + private String encode(String value) { + return pathEncoding == FileSystemPathEncoding.HASH ? sha256(value) : percentEncode(value); + } + + private static String percentEncode(String value) { + var bytes = value.getBytes(StandardCharsets.UTF_8); + var encoded = new StringBuilder(bytes.length); + for (byte raw : bytes) { + int valueByte = raw & 0xff; + if ((valueByte >= 'a' && valueByte <= 'z') + || (valueByte >= 'A' && valueByte <= 'Z') + || (valueByte >= '0' && valueByte <= '9') + || valueByte == '-' + || valueByte == '_' + || valueByte == '.' + || valueByte == '~') { + encoded.append((char) valueByte); + } else { + encoded.append('%'); + encoded.append(Character.toUpperCase(Character.forDigit((valueByte >>> 4) & 0xf, 16))); + encoded.append(Character.toUpperCase(Character.forDigit(valueByte & 0xf, 16))); + } + } + return encoded.toString(); + } + + private static String sha256(String value) { + try { + var digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private static String writeEnvelope(JsonNode envelope) { + try { + return ENVELOPE_MAPPER.writeValueAsString(envelope); + } catch (IOException e) { + throw new SerDesException("Failed to create filesystem SerDes envelope", e); + } + } + + /** Builder for {@link FileSystemSerDes}. */ + public static final class Builder { + private final Path basePath; + private FileSystemSerDesMode storageMode = FileSystemSerDesMode.ALWAYS; + private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; + private SerDes delegate = new JacksonSerDes(); + private Function> previewGenerator; + + private Builder(Path basePath) { + this.basePath = Objects.requireNonNull(basePath, "basePath cannot be null"); + } + + public Builder storageMode(FileSystemSerDesMode storageMode) { + this.storageMode = Objects.requireNonNull(storageMode, "storageMode cannot be null"); + return this; + } + + public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { + this.pathEncoding = Objects.requireNonNull(pathEncoding, "pathEncoding cannot be null"); + return this; + } + + public Builder delegate(SerDes delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + return this; + } + + public Builder previewGenerator(Function> previewGenerator) { + this.previewGenerator = Objects.requireNonNull(previewGenerator, "previewGenerator cannot be null"); + return this; + } + + public FileSystemSerDes build() { + return new FileSystemSerDes(this); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDesMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDesMode.java new file mode 100644 index 000000000..3f4ed7ae6 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDesMode.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Controls when {@link FileSystemSerDes} stores serialized data on the filesystem. */ +public enum FileSystemSerDesMode { + /** Store every SDK-managed payload on the filesystem. */ + ALWAYS, + + /** Keep small payloads inline and store only payloads that exceed the checkpoint threshold. */ + OVERFLOW +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java new file mode 100644 index 000000000..c68c7dfc0 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java @@ -0,0 +1,47 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Identifies the durable payload currently being processed by a {@link SerDes}. + * + *

The SDK exposes this context through thread-local storage so the existing {@link SerDes} interface remains + * backward compatible. The context is available only while the SDK is invoking a SerDes method. + * + * @param durableExecutionArn ARN of the durable execution + * @param entityId stable identifier of the execution or operation payload + */ +public record SerDesContext(String durableExecutionArn, String entityId) { + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + public SerDesContext { + Objects.requireNonNull(durableExecutionArn, "durableExecutionArn cannot be null"); + Objects.requireNonNull(entityId, "entityId cannot be null"); + } + + /** + * Returns the context for the current SDK-managed SerDes call. + * + * @return the current context, or {@code null} when called outside an SDK-managed SerDes call + */ + public static SerDesContext getCurrentContext() { + return CURRENT.get(); + } + + static T callWithContext(SerDesContext context, Supplier action) { + var previous = CURRENT.get(); + CURRENT.set(Objects.requireNonNull(context, "context cannot be null")); + try { + return action.get(); + } finally { + if (previous == null) { + CURRENT.remove(); + } else { + CURRENT.set(previous); + } + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java new file mode 100644 index 000000000..aab88398e --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -0,0 +1,159 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.lang.ref.WeakReference; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.util.ExceptionHelper; + +/** + * Executes SDK-managed SerDes calls on a dedicated executor with {@link SerDesContext} installed in thread-local + * storage. + * + *

Each runner is scoped to one Lambda invocation. Successful deserializations are cached for that invocation so + * repeated reads of the same checkpoint payload do not repeat filesystem or other external I/O. + */ +public final class SerDesRunner { + static final int MAX_COMPLETED_DESERIALIZATIONS = 256; + private static final Object NULL_VALUE = new Object(); + + private final ExecutorService executorService; + private final ConcurrentHashMap> inFlightDeserializations = + new ConcurrentHashMap<>(); + private final Map> completedDeserializations = + Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry> eldest) { + return size() > MAX_COMPLETED_DESERIALIZATIONS; + } + }); + + public SerDesRunner(ExecutorService executorService) { + this.executorService = Objects.requireNonNull(executorService, "executorService cannot be null"); + } + + /** Serializes a value with the supplied durable payload context. */ + public String serialize(SerDes serDes, Object value, SerDesContext context) { + Objects.requireNonNull(serDes, "serDes cannot be null"); + return join(submit(context, () -> serDes.serialize(value))); + } + + /** Deserializes and caches a value for the current invocation using the supplied durable payload context. */ + @SuppressWarnings("unchecked") + public T deserialize(SerDes serDes, String data, TypeToken typeToken, SerDesContext context) { + Objects.requireNonNull(serDes, "serDes cannot be null"); + Objects.requireNonNull(typeToken, "typeToken cannot be null"); + Objects.requireNonNull(context, "context cannot be null"); + + var key = new CacheKey(serDes, context.durableExecutionArn(), context.entityId(), typeToken, hash(data)); + var cached = getCompleted(key); + if (cached != null) { + return cached == NULL_VALUE ? null : (T) cached; + } + + var pending = new CompletableFuture(); + var existing = inFlightDeserializations.putIfAbsent(key, pending); + if (existing != null) { + var value = join(existing); + return value == NULL_VALUE ? null : (T) value; + } + + try { + cached = getCompleted(key); + if (cached != null) { + pending.complete(cached); + return cached == NULL_VALUE ? null : (T) cached; + } + + var value = join(submit(context, () -> serDes.deserialize(data, typeToken))); + var cacheValue = value == null ? NULL_VALUE : value; + putCompleted(key, cacheValue); + pending.complete(cacheValue); + return value; + } catch (Throwable failure) { + pending.completeExceptionally(failure); + throw failure; + } finally { + inFlightDeserializations.remove(key, pending); + } + } + + private Object getCompleted(CacheKey key) { + synchronized (completedDeserializations) { + var reference = completedDeserializations.get(key); + if (reference == null) { + return null; + } + var value = reference.get(); + if (value == null) { + completedDeserializations.remove(key); + } + return value; + } + } + + private void putCompleted(CacheKey key, Object value) { + completedDeserializations.put(key, new WeakReference<>(value)); + } + + private CompletableFuture submit(SerDesContext context, Supplier action) { + Objects.requireNonNull(context, "context cannot be null"); + Objects.requireNonNull(action, "action cannot be null"); + return CompletableFuture.supplyAsync(() -> SerDesContext.callWithContext(context, action), executorService); + } + + private static T join(CompletableFuture future) { + try { + return future.join(); + } catch (Throwable failure) { + ExceptionHelper.sneakyThrow(ExceptionHelper.unwrapCompletableFuture(failure)); + return null; + } + } + + private static String hash(String data) { + if (data == null) { + return "null"; + } + try { + var digest = MessageDigest.getInstance("SHA-256").digest(data.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private record CacheKey( + SerDes serDes, String durableExecutionArn, String entityId, TypeToken typeToken, String dataHash) { + @Override + public boolean equals(Object other) { + return other instanceof CacheKey that + && serDes == that.serDes + && Objects.equals(durableExecutionArn, that.durableExecutionArn) + && Objects.equals(entityId, that.entityId) + && Objects.equals(typeToken, that.typeToken) + && Objects.equals(dataHash, that.dataHash); + } + + @Override + public int hashCode() { + int result = System.identityHashCode(serDes); + result = 31 * result + Objects.hashCode(durableExecutionArn); + result = 31 * result + Objects.hashCode(entityId); + result = 31 * result + Objects.hashCode(typeToken); + return 31 * result + Objects.hashCode(dataHash); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java b/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java index 912f7d547..790e48bd6 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java @@ -44,10 +44,21 @@ public static Throwable unwrapCompletableFuture(Throwable throwable) { * @return the ErrorObject */ public static ErrorObject buildErrorObject(Throwable throwable, SerDes serDes) { + return buildErrorObject(throwable, serDes.serialize(throwable)); + } + + /** + * build an ErrorObject from a Throwable and pre-serialized error data + * + * @param throwable the Throwable from which to build the errorObject + * @param errorData the serialized Throwable payload + * @return the ErrorObject + */ + public static ErrorObject buildErrorObject(Throwable throwable, String errorData) { return ErrorObject.builder() .errorType(throwable.getClass().getName()) .errorMessage(throwable.getMessage()) - .errorData(serDes.serialize(throwable)) + .errorData(errorData) .stackTrace(serializeStackTrace(throwable.getStackTrace())) .build(); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java index 266e43a6e..9d06110c0 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java @@ -33,12 +33,14 @@ class DurableConfigTest { private DurableExecutionClient mockClient; private SerDes mockSerDes; private ExecutorService mockExecutor; + private ExecutorService mockSerDesExecutor; @BeforeEach void setUp() { mockClient = mock(DurableExecutionClient.class); mockSerDes = mock(SerDes.class); mockExecutor = mock(ExecutorService.class); + mockSerDesExecutor = mock(ExecutorService.class); } @Test @@ -52,6 +54,8 @@ void testDefaultConfig_CreatesWithDefaults() { assertInstanceOf(JacksonSerDes.class, config.getSerDes()); assertNotNull(config.getExecutorService()); assertInstanceOf(ExecutorService.class, config.getExecutorService()); + assertNotNull(config.getSerDesExecutorService()); + assertInstanceOf(ExecutorService.class, config.getSerDesExecutorService()); } @Test @@ -87,6 +91,15 @@ void testBuilder_WithCustomExecutorService() { assertNotNull(config.getSerDes()); } + @Test + void testBuilder_WithCustomSerDesExecutorService() { + var config = DurableConfig.builder() + .withSerDesExecutorService(mockSerDesExecutor) + .build(); + + assertEquals(mockSerDesExecutor, config.getSerDesExecutorService()); + } + @Test void testBuilder_DeserializeAfterSerializationDefaultsToTrue() { var config = @@ -131,12 +144,14 @@ void testBuilder_WithAllCustomComponents() { .withDurableExecutionClient(mockClient) .withSerDes(mockSerDes) .withExecutorService(mockExecutor) + .withSerDesExecutorService(mockSerDesExecutor) .build(); assertNotNull(config); assertEquals(mockClient, config.getDurableExecutionClient()); assertEquals(mockSerDes, config.getSerDes()); assertEquals(mockExecutor, config.getExecutorService()); + assertEquals(mockSerDesExecutor, config.getSerDesExecutorService()); } @Test @@ -169,6 +184,7 @@ void testBuilder_FluentAPI() { assertSame(builder, builder.withDurableExecutionClient(mockClient)); assertSame(builder, builder.withSerDes(mockSerDes)); assertSame(builder, builder.withExecutorService(mockExecutor)); + assertSame(builder, builder.withSerDesExecutorService(mockSerDesExecutor)); assertSame(builder, builder.withDeserializeAfterSerialization(false)); } @@ -221,6 +237,27 @@ void testDefaultExecutorService_IsNotNull() { assertFalse(executor.isShutdown()); } + @Test + void testDefaultSerDesExecutorService_IsNotNull() { + var config = + DurableConfig.builder().withDurableExecutionClient(mockClient).build(); + + var executor = config.getSerDesExecutorService(); + assertNotNull(executor); + assertFalse(executor.isShutdown()); + } + + @Test + void testBuilder_RejectsSharedUserAndSerDesExecutor() { + var builder = DurableConfig.builder().withExecutorService(mockExecutor).withSerDesExecutorService(mockExecutor); + + var exception = assertThrows(IllegalStateException.class, builder::build); + + assertEquals( + "SerDes ExecutorService must be different from the user operation ExecutorService", + exception.getMessage()); + } + @Test void testBuilder_MultipleBuilds_CreateIndependentInstances() { var builder = DurableConfig.builder().withDurableExecutionClient(mockClient); @@ -233,6 +270,7 @@ void testBuilder_MultipleBuilds_CreateIndependentInstances() { // ExecutorService should be different instances (each gets its own) assertSame(config1.getExecutorService(), config2.getExecutorService()); + assertSame(config1.getSerDesExecutorService(), config2.getSerDesExecutorService()); } @Test @@ -244,6 +282,7 @@ void testBuilder_NullExecutorService_AllowedAndUsesDefault() { .build(); assertNotNull(config.getExecutorService()); + assertNotNull(config.getSerDesExecutorService()); } @Test @@ -435,6 +474,7 @@ void validateConfiguration_PassesForValidConfig() { .withDurableExecutionClient(mockClient) .withSerDes(mockSerDes) .withExecutorService(mockExecutor) + .withSerDesExecutorService(mockSerDesExecutor) .build(); // Should not throw — all fields are set @@ -474,6 +514,17 @@ void validateConfiguration_ThrowsWhenExecutorServiceIsNull() throws Exception { assertEquals("ExecutorService configuration failed", ex.getMessage()); } + @Test + void validateConfiguration_ThrowsWhenSerDesExecutorServiceIsNull() throws Exception { + var config = + DurableConfig.builder().withDurableExecutionClient(mockClient).build(); + + setField(config, "serDesExecutorService", null); + + var ex = assertThrows(IllegalStateException.class, config::validateConfiguration); + assertEquals("SerDes ExecutorService configuration failed", ex.getMessage()); + } + @Test void validateConfiguration_ChecksClientBeforeSerDes() throws Exception { var config = diff --git a/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java b/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java index 1aaba5d4c..e7f63b05f 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java @@ -8,11 +8,20 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import software.amazon.awssdk.services.lambda.model.*; import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.execution.ExecutionManager; import software.amazon.lambda.durable.execution.OperationIdGenerator; +import software.amazon.lambda.durable.serde.SerDesRunner; public class TestUtils { + private static final ExecutorService TEST_SERDES_EXECUTOR = Executors.newCachedThreadPool(runnable -> { + var thread = new Thread(runnable, "test-serdes"); + thread.setDaemon(true); + return thread; + }); public static DurableExecutionClient createMockClient() { var client = mock(DurableExecutionClient.class); @@ -69,4 +78,9 @@ public static DurableExecutionClient createMockClient() { public static String hashOperationId(String rawId) { return OperationIdGenerator.hashOperationId(rawId); } + + public static void configureSerDesRunner(ExecutionManager executionManager) { + when(executionManager.getDurableExecutionArn()).thenReturn("arn:test"); + when(executionManager.getSerDesRunner()).thenReturn(new SerDesRunner(TEST_SERDES_EXECUTOR)); + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/ChildContextOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/ChildContextOperationTest.java index 99d994538..240e3c575 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/ChildContextOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/ChildContextOperationTest.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TestUtils; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.RunInChildContextConfig; import software.amazon.lambda.durable.context.DurableContextImpl; @@ -70,6 +71,7 @@ public T deserialize(String data, TypeToken typeToken) { void setUp() { durableContext = mock(DurableContextImpl.class); executionManager = mock(ExecutionManager.class); + TestUtils.configureSerDesRunner(executionManager); when(durableContext.getExecutionManager()).thenReturn(executionManager); when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext("Root", ThreadType.CONTEXT)); when(durableContext.getDurableConfig()).thenReturn(createConfig()); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/ConcurrencyOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/ConcurrencyOperationTest.java index b6488139f..33de32606 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/ConcurrencyOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/ConcurrencyOperationTest.java @@ -59,6 +59,7 @@ class ConcurrencyOperationTest { void setUp() { durableContext = mock(DurableContextImpl.class); executionManager = mock(ExecutionManager.class); + TestUtils.configureSerDesRunner(executionManager); var childContext = mock(DurableContextImpl.class); this.childContext = childContext; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 2c1d76c74..3ac6cd6df 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.lambda.durable.TestUtils; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.context.DurableContextImpl; @@ -39,6 +40,7 @@ class InvokeOperationTest { @BeforeEach void setUp() { executionManager = mock(ExecutionManager.class); + TestUtils.configureSerDesRunner(executionManager); durableContext = mock(DurableContextImpl.class); when(durableContext.getExecutionManager()).thenReturn(executionManager); when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext("root", ThreadType.CONTEXT)); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/ParallelOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/ParallelOperationTest.java index e02287cdc..36d480a29 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/ParallelOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/ParallelOperationTest.java @@ -58,6 +58,7 @@ class ParallelOperationTest { void setUp() { durableContext = mock(DurableContextImpl.class); executionManager = mock(ExecutionManager.class); + TestUtils.configureSerDesRunner(executionManager); operationStore = new ConcurrentHashMap<>(); parallelCheckpointLatch = new CountDownLatch(1); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index bc9e940b8..4302f8068 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -42,6 +42,7 @@ import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; class SerializableDurableOperationTest { @@ -104,6 +105,8 @@ void setUp() { executionManager = mock(ExecutionManager.class); durableContext = mock(DurableContextImpl.class); when(durableContext.getExecutionManager()).thenReturn(executionManager); + when(executionManager.getDurableExecutionArn()).thenReturn("arn:test"); + when(executionManager.getSerDesRunner()).thenReturn(new SerDesRunner(internalExecutor)); when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext(CONTEXT_ID, ThreadType.CONTEXT)); when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(OPERATION); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java index be4962d71..54264d8bf 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java @@ -14,6 +14,7 @@ import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.StepDetails; import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TestUtils; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.context.DurableContextImpl; @@ -39,6 +40,7 @@ class StepOperationTest { @BeforeEach void setUp() { executionManager = mock(ExecutionManager.class); + TestUtils.configureSerDesRunner(executionManager); durableContext = mock(DurableContextImpl.class); when(durableContext.getExecutionManager()).thenReturn(executionManager); when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext("handler", ThreadType.CONTEXT)); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java index 69502a3c3..64d354d03 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java @@ -17,6 +17,7 @@ import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.awssdk.services.lambda.model.StepDetails; import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TestUtils; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.WaitForConditionConfig; import software.amazon.lambda.durable.context.DurableContextImpl; @@ -44,6 +45,7 @@ class WaitForConditionOperationTest { @BeforeEach void setUp() { executionManager = mock(ExecutionManager.class); + TestUtils.configureSerDesRunner(executionManager); durableContext = mock(DurableContextImpl.class); when(durableContext.getExecutionManager()).thenReturn(executionManager); when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext("handler", ThreadType.CONTEXT)); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java new file mode 100644 index 000000000..7d3a17681 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -0,0 +1,163 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; + +class FileSystemSerDesTest { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path tempDir; + + private ExecutorService executor; + private SerDesRunner runner; + + @BeforeEach + void setUp() { + executor = Executors.newSingleThreadExecutor(); + runner = new SerDesRunner(executor); + } + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void serializesNormallyWhenNoDurableContextExists() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + + assertEquals("\"input\"", serDes.serialize("input")); + assertEquals("input", serDes.deserialize("\"input\"", TypeToken.get(String.class))); + try (var files = Files.walk(tempDir)) { + assertEquals(0, files.filter(Files::isRegularFile).count()); + } + } + + @Test + void alwaysModeStoresAndLoadsPayload() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var context = new SerDesContext(realisticArn(), "operation/1/result"); + + var envelope = runner.serialize(serDes, new Value("stored"), context); + var node = MAPPER.readTree(envelope); + + assertEquals(1, node.get("__durable_execution_filesystem_serdes").intValue()); + assertTrue(node.hasNonNull("file")); + assertTrue(node.hasNonNull("sha256")); + assertEquals(new Value("stored"), runner.deserialize(serDes, envelope, TypeToken.get(Value.class), context)); + assertTrue(Files.exists(Path.of(node.get("file").textValue()))); + } + + @Test + void overflowModeKeepsSmallPayloadInline() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .build(); + var context = new SerDesContext(realisticArn(), "1"); + + var envelope = runner.serialize(serDes, "small", context); + var node = MAPPER.readTree(envelope); + + assertTrue(node.hasNonNull("data")); + assertFalse(node.has("file")); + assertEquals("small", runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + } + + @Test + void overflowModeStoresLargePayload() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .build(); + var context = new SerDesContext(realisticArn(), "1"); + + var envelope = runner.serialize(serDes, "x".repeat(300_000), context); + + assertTrue(MAPPER.readTree(envelope).hasNonNull("file")); + } + + @Test + void supportsHashPathEncodingAndPreview() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir) + .pathEncoding(FileSystemPathEncoding.HASH) + .previewGenerator(value -> Map.of("summary", ((Value) value).value())) + .build(); + var context = new SerDesContext(realisticArn(), "../unsafe/entity"); + + var node = MAPPER.readTree(runner.serialize(serDes, new Value("preview"), context)); + var file = Path.of(node.get("file").textValue()); + + assertEquals(64, tempDir.relativize(file).getName(0).toString().length()); + assertTrue(file.getFileName().toString().matches("[0-9a-f]{64}-[0-9a-f]{64}\\.json")); + assertEquals("preview", node.get("preview").get("summary").textValue()); + } + + @Test + void rejectsPreviewThatMakesFileEnvelopeTooLarge() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir) + .previewGenerator(value -> Map.of("large", "x".repeat(300_000))) + .build(); + var context = new SerDesContext(realisticArn(), "1"); + + assertThrows(SerDesException.class, () -> runner.serialize(serDes, "value", context)); + try (var files = Files.walk(tempDir)) { + assertEquals(0, files.filter(Files::isRegularFile).count()); + } + } + + @Test + void readsJavaScriptFilesystemEnvelope() throws Exception { + var payloadFile = tempDir.resolve("js-payload.json"); + Files.writeString(payloadFile, "{\"value\":\"js\"}", StandardCharsets.UTF_8); + var envelope = MAPPER.writeValueAsString(Map.of("file", payloadFile.toString())); + var serDes = FileSystemSerDes.builder(tempDir).build(); + + assertEquals(new Value("js"), serDes.deserialize(envelope, TypeToken.get(Value.class))); + } + + @Test + void rejectsFileOutsideConfiguredBasePath() throws Exception { + var externalFile = Files.createTempFile("filesystem-serdes", ".json"); + Files.writeString(externalFile, "\"secret\"", StandardCharsets.UTF_8); + var envelope = MAPPER.writeValueAsString(Map.of("file", externalFile.toString())); + var serDes = FileSystemSerDes.builder(tempDir).build(); + + assertThrows(SerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); + } + + @Test + void rejectsMalformedRecognizedEnvelope() { + var serDes = FileSystemSerDes.builder(tempDir).build(); + + assertThrows( + SerDesException.class, + () -> serDes.deserialize( + "{\"__durable_execution_filesystem_serdes\":1,\"data\":\"x\",\"file\":\"y\"}", + TypeToken.get(String.class))); + } + + private static String realisticArn() { + return "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + + "/durable-execution/execution-name/invocation-id"; + } + + record Value(String value) {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java new file mode 100644 index 000000000..9bfac2054 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -0,0 +1,124 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.TypeToken; + +class SerDesRunnerTest { + private ExecutorService executor; + private SerDesRunner runner; + + @BeforeEach + void setUp() { + executor = Executors.newSingleThreadExecutor(runnable -> new Thread(runnable, "test-serdes")); + runner = new SerDesRunner(executor); + } + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void executesOnConfiguredExecutorWithThreadLocalContext() { + var observedThread = new AtomicReference(); + var observedContext = new AtomicReference(); + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + observedThread.set(Thread.currentThread().getName()); + observedContext.set(SerDesContext.getCurrentContext()); + return super.serialize(value); + } + }; + var context = new SerDesContext("arn:test", "entity"); + + assertEquals("\"value\"", runner.serialize(serDes, "value", context)); + assertEquals("test-serdes", observedThread.get()); + assertEquals(context, observedContext.get()); + assertNull(SerDesContext.getCurrentContext()); + } + + @Test + void cachesSuccessfulDeserializationForInvocation() { + var calls = new AtomicInteger(); + var serDes = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return super.deserialize(data, typeToken); + } + }; + var context = new SerDesContext("arn:test", "entity"); + + var first = runner.deserialize(serDes, "{\"value\":\"cached\"}", TypeToken.get(Value.class), context); + var second = runner.deserialize(serDes, "{\"value\":\"cached\"}", TypeToken.get(Value.class), context); + + assertSame(first, second); + assertEquals(1, calls.get()); + } + + @Test + void cacheKeyIncludesSerializedPayload() { + var calls = new AtomicInteger(); + var serDes = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return super.deserialize(data, typeToken); + } + }; + var context = new SerDesContext("arn:test", "entity"); + + runner.deserialize(serDes, "{\"value\":\"one\"}", TypeToken.get(Value.class), context); + runner.deserialize(serDes, "{\"value\":\"two\"}", TypeToken.get(Value.class), context); + + assertEquals(2, calls.get()); + } + + @Test + void failedDeserializationIsNotCachedAndContextIsCleared() throws Exception { + var calls = new AtomicInteger(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + assertEquals(new SerDesContext("arn:test", "entity"), SerDesContext.getCurrentContext()); + throw new IllegalStateException("failed"); + } + }; + var context = new SerDesContext("arn:test", "entity"); + + assertThrows( + IllegalStateException.class, + () -> runner.deserialize(serDes, "\"value\"", TypeToken.get(String.class), context)); + assertThrows( + IllegalStateException.class, + () -> runner.deserialize(serDes, "\"value\"", TypeToken.get(String.class), context)); + + assertEquals(2, calls.get()); + assertNull(executor.submit(SerDesContext::getCurrentContext).get()); + assertTrue(executor.submit(() -> Thread.currentThread().getName().startsWith("test-serdes")) + .get()); + } + + record Value(String value) {} +} From 63774560bbe19ce24572aef440359a0d966b4cca Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 31 Aug 2026 23:54:43 +0000 Subject: [PATCH 02/11] fix: align filesystem SerDes configuration --- docs/adr/005-filesystem-serdes.md | 47 +++-- docs/advanced/configuration.md | 50 ++++- docs/design.md | 10 +- .../FileSystemSerDesIntegrationTest.java | 23 ++ .../testing/LocalDurableTestRunner.java | 10 +- .../amazon/lambda/durable/DurableConfig.java | 23 +- .../lambda/durable/serde/FieldMatchMode.java | 12 ++ .../durable/serde/FileSystemSerDes.java | 83 +++++--- .../lambda/durable/serde/PreviewConfig.java | 113 ++++++++++ .../lambda/durable/serde/PreviewField.java | 36 ++++ .../lambda/durable/serde/PreviewMode.java | 12 ++ .../lambda/durable/serde/SerDesPreview.java | 191 +++++++++++++++++ .../lambda/durable/serde/SerDesRunner.java | 14 +- .../lambda/durable/DurableConfigTest.java | 26 +-- .../durable/serde/FileSystemSerDesTest.java | 122 ++++++++++- .../durable/serde/SerDesPreviewTest.java | 196 ++++++++++++++++++ .../durable/serde/SerDesRunnerTest.java | 20 ++ 17 files changed, 883 insertions(+), 105 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index e754a383e..38d40f35b 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -48,7 +48,7 @@ in `finally`, which supports nesting and prevents context from leaking when exec Each `ExecutionManager` creates one `SerDesRunner` for the Lambda invocation. The runner: -1. dispatches the SerDes call to the configured SerDes executor; +1. executes inline or dispatches the SerDes call to the configured SerDes executor; 2. installs `SerDesContext` inside that executor task; 3. invokes the unchanged SerDes method; 4. clears or restores the thread-local context; @@ -62,9 +62,9 @@ DurableConfig.builder() .build(); ``` -The default is a shared cached daemon pool named `durable-serdes-*`. The SerDes executor must not be the same object as -the user-operation executor because operation threads synchronously wait for SerDes work and a shared saturated pool -could deadlock. +SerDes calls execute inline by default. A dedicated executor is opt-in for implementations that perform blocking +storage or network I/O. When configured, it must not be the same object as the user-operation executor because operation +threads synchronously wait for SerDes work and a shared saturated pool could deadlock. ### Cache successful deserializations per invocation @@ -94,7 +94,11 @@ configuration. var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemSerDesMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) - .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) + .checkpointEnvelopeLimitBytes(512 * 1024) + .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id")) + .mask(PreviewField.anywhere("email")) + .build()) .build(); return DurableConfig.builder() @@ -107,7 +111,7 @@ Storage modes: | Mode | Behavior | | --- | --- | | `ALWAYS` | Store every SDK-managed non-null payload in a file. | -| `OVERFLOW` | Keep the versioned envelope inline until it exceeds 255 KiB, then store the payload in a file. | +| `OVERFLOW` | Keep the versioned envelope inline until it exceeds the configured byte limit, then store it in a file. | Path encodings: @@ -116,7 +120,15 @@ Path encodings: | `URI` | Percent-encode readable execution and entity path segments. | | `HASH` | Use fixed-length SHA-256 path segments. | -The default delegate is `JacksonSerDes`; a custom delegate can be supplied through the builder. +The default checkpoint-envelope limit is 255 KiB and can be changed with `checkpointEnvelopeLimitBytes(...)`. + +The default delegate is `JacksonSerDes`; `.delegate(...)` controls how values are encoded inside files. Existing +operation-level SerDes configuration controls which boundaries use filesystem storage. For example, +`InvokeConfig.payloadSerDes(new JacksonSerDes()).serDes(fileSystemSerDes)` sends ordinary JSON while decoding the result +with filesystem storage. + +Structured previews support include-all/exclude-all modes, include/exclude/mask selectors, anywhere or exact-path +matching, custom mask text, and a default 4 KiB preview budget. Custom preview callbacks remain available. ### Envelope and file publication @@ -128,8 +140,8 @@ Java writes versioned envelopes: {"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":"","preview":{"id":"123"}} ``` -The additional marker and digest are ignored by the JavaScript implementation, which reads the `data`, `file`, and -`preview` fields. Java also reads the unversioned JavaScript envelopes. +Only envelopes containing the reserved version marker are interpreted as filesystem payloads. Unmarked JSON, including +objects with `data` or `file` fields, is passed to the delegate SerDes unchanged. File names include the entity ID and serialized-payload digest. Files are created with `CREATE_NEW`; an existing file is accepted only when its contents match. This prevents a later retry from overwriting data referenced by an earlier @@ -142,8 +154,8 @@ The durable execution ARN does not exist when a caller serializes the initial La remains ordinary delegate JSON. After the invocation starts, the SDK routes root input deserialization, operation payloads, exceptions, and root output -through `SerDesRunner`. Chained invokes that use filesystem SerDes require the caller and callee to use compatible -configuration and have access to the same durable filesystem. +through `SerDesRunner`. A chained-invoke boundary requires compatible filesystem configuration and shared storage only +when that boundary explicitly selects `FileSystemSerDes`. ## Consequences @@ -154,15 +166,15 @@ Positive: - Blocking SerDes work is isolated from user and SDK coordination executors. - Repeated deserialization and file reads are avoided within an invocation. - The implementation uses existing global and per-operation SerDes configuration. -- Java and JavaScript filesystem envelopes are mutually readable. +- Ordinary user JSON cannot be confused with a filesystem envelope. Negative: - SerDes context is implicit thread-local state. -- Every SDK-managed SerDes call crosses an executor boundary. +- Configuring a SerDes executor adds an executor boundary to every SDK-managed SerDes call. - Repeated deserialization returns the same object instance within an invocation. - Filesystem retention and cleanup remain the application's responsibility. -- Chained invoke payloads and results require shared storage and compatible SerDes configuration. +- Chained invoke payloads and results that select filesystem storage require a shared mount and compatible paths. ## Operational Requirements @@ -187,7 +199,8 @@ configuration. A dedicated offloader can be reconsidered if multiple storage bac Rejected for the initial implementation. The filesystem implementation uses only JDK and existing Jackson APIs, so a new artifact would add release and documentation overhead without isolating an additional dependency. -### Run SerDes inline or omit caching +### Run blocking filesystem work on the user or internal executor, or omit caching -Rejected because mounted filesystem I/O can block user progress and repeated reads can repeat externally visible work -and cost. +Rejected. Inline execution remains the default, but applications can isolate blocking filesystem work with the +optional SerDes executor. The user-operation and internal coordination executors must not be used for that I/O. +Omitting caching would repeat file reads and object reconstruction within one invocation. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 93572343e..3e4467aa9 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -34,15 +34,16 @@ public class OrderProcessor extends DurableHandler { | `withLambdaClientBuilder()` | Custom AWS Lambda client | Auto-configured Lambda client | | `withSerDes()` | Serializer for step results | Jackson with default settings | | `withExecutorService()` | Thread pool for user-defined operations | Cached daemon thread pool | -| `withSerDesExecutorService()` | Thread pool for serialization and payload I/O | Cached daemon thread pool | +| `withSerDesExecutorService()` | Optional thread pool for serialization and payload I/O | Inline on the calling thread | | `withLoggerConfig()` | Logger behavior configuration | Suppress logs during replay | | `withPollingStrategy()` | Backend polling strategy | Exponential backoff: 1s base, 2x rate, FULL jitter, 10s max | | `withCheckpointDelay()` | How often the SDK checkpoints updates | `Duration.ofSeconds(0)` (as soon as possible) | The `withExecutorService()` option configures the thread pool used for running user-defined operations. Internal SDK coordination (checkpoint batching, polling) runs on an SDK-managed thread pool. -The SerDes executor must be different from the user-operation executor. SerDes calls are synchronous from the -operation's perspective, so using one saturated pool for both can deadlock. +By default, SerDes calls run inline. Configure `withSerDesExecutorService()` when a SerDes performs blocking storage or +network I/O. The SerDes executor must be different from the user-operation executor because SerDes calls are +synchronous from the operation's perspective and a shared saturated pool can deadlock. ### Filesystem-backed SerDes @@ -53,7 +54,11 @@ inline when configured for overflow mode: var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemSerDesMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) - .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) + .checkpointEnvelopeLimitBytes(512 * 1024) + .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("status")) + .mask(PreviewField.anywhere("email")) + .build()) .build(); return DurableConfig.builder() @@ -62,8 +67,33 @@ return DurableConfig.builder() ``` `ALWAYS` writes every SDK-managed payload to a file. `OVERFLOW` stores the payload inline until the complete checkpoint -envelope exceeds 255 KiB. `URI` path encoding keeps identifiers readable, while `HASH` avoids filesystem name-length -and character restrictions. +envelope exceeds the configured limit, which defaults to 255 KiB. `URI` path encoding keeps identifiers readable, +while `HASH` avoids filesystem name-length and character restrictions. + +`PreviewConfig` supports include-all/exclude-all modes, exact-path or anywhere field matching, masking, and a default +4 KiB preview budget. A custom `previewGenerator(...)` remains available for non-standard preview logic. + +The filesystem wrapper and the value codec are configured independently. Use `.delegate(...)` to control how values are +encoded inside files: + +```java +var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .delegate(new MyCustomSerDes()) + .build(); +``` + +Existing operation configuration controls where filesystem storage is used. For example, an invoke can send an +ordinary JSON payload while decoding its result through filesystem storage: + +```java +var config = InvokeConfig.builder() + .payloadSerDes(new JacksonSerDes()) + .serDes(fileSystemSerDes) + .build(); +``` + +The same pattern applies to `StepConfig.serDes(...)`, callback, child-context, map, parallel, and wait-for-condition +configuration. The SDK supplies a `SerDesContext` through thread-local storage during managed calls: @@ -71,12 +101,12 @@ The SDK supplies a `SerDesContext` through thread-local storage during managed c var context = SerDesContext.getCurrentContext(); ``` -Custom SerDes implementations can use its durable execution ARN and entity ID for external storage. Calls run on the -dedicated SerDes executor, and successful deserializations are cached for the current Lambda invocation. +Custom SerDes implementations can use its durable execution ARN and entity ID for external storage. Calls use the +configured SerDes executor when present, and successful deserializations are cached for the current Lambda invocation. Do not use Lambda's `/tmp` directory: replay can run in another execution environment. Use a shared durable mount such -as EFS. S3 Files users must account for synchronization and crash-durability behavior. Chained invokes require both -functions to use compatible SerDes configuration and access the same mount. +as EFS. S3 Files users must account for synchronization and crash-durability behavior. A chained-invoke boundary only +requires shared storage and compatible filesystem configuration when that boundary explicitly uses `FileSystemSerDes`. ### Dynamic plugin loading diff --git a/docs/design.md b/docs/design.md index b9ce833eb..c1b82ef39 100644 --- a/docs/design.md +++ b/docs/design.md @@ -350,6 +350,8 @@ software.amazon.lambda.durable │ ├── SerDes # Interface │ ├── JacksonSerDes # Jackson impl │ ├── FileSystemSerDes # Shared-filesystem payload storage +│ ├── PreviewConfig # Structured preview selection and byte budget +│ ├── SerDesPreview # Structured preview builder │ ├── SerDesContext # Thread-local durable payload identity │ ├── SerDesRunner # Executor dispatch + invocation cache │ └── AwsSdkV2Module # SDK type support @@ -660,10 +662,10 @@ public interface SerDes { } ``` -SDK-managed calls go through an invocation-scoped `SerDesRunner`. The runner dispatches work to the dedicated SerDes -executor, installs a `SerDesContext` in plain thread-local storage for the duration of the call, and caches successful -deserializations in a bounded weak-reference LRU by SerDes identity, execution ARN, entity ID, target type, and -serialized-data hash. The thread-local value is always restored in `finally`. +SDK-managed calls go through an invocation-scoped `SerDesRunner`. The runner executes inline by default or dispatches +work to the configured SerDes executor, installs a `SerDesContext` in plain thread-local storage for the duration of the +call, and caches successful deserializations in a bounded weak-reference LRU by SerDes identity, execution ARN, entity +ID, target type, and serialized-data hash. The thread-local value is always restored in `finally`. `FileSystemSerDes` uses that context to build collision-free paths on a shared durable filesystem. Calls made before a durable execution ARN exists, such as initial invocation input serialization, fall back to the delegate SerDes without diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 2eaec17b9..cbc50a890 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -11,7 +11,9 @@ import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.serde.FileSystemSerDes; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; class FileSystemSerDesIntegrationTest { @@ -44,4 +46,25 @@ void storesStepAndExecutionResultsAcrossReplay() throws Exception { assertTrue(files.filter(Files::isRegularFile).count() >= 2); } } + + @Test + void operationConfigControlsWhereFilesystemStorageIsUsed() throws Exception { + var fileSystemSerDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder().withSerDes(new JacksonSerDes()).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step( + "filesystem-step", + String.class, + stepContext -> input + "-stored", + StepConfig.builder().serDes(fileSystemSerDes).build()), + config); + + var result = runner.runUntilComplete("value"); + + assertEquals("value-stored", result.getResult(String.class)); + try (var files = Files.walk(tempDir)) { + assertEquals(1, files.filter(Files::isRegularFile).count()); + } + } } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 0d25f1d68..183234530 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -61,19 +61,21 @@ private LocalDurableTestRunner( // Create config that uses customer's configuration but overrides the client with in-memory storage if (customerConfig != null) { // Use customer's config but override the client with our in-memory implementation - this.customerConfig = DurableConfig.builder() + var configBuilder = DurableConfig.builder() .withDurableExecutionClient(storage) .withSerDes(customerConfig.getSerDes()) .withExecutorService(customerConfig.getExecutorService()) - .withSerDesExecutorService(customerConfig.getSerDesExecutorService()) .withPollingStrategy(customerConfig.getPollingStrategy()) .withCheckpointDelay(customerConfig.getCheckpointDelay()) .withLoggerConfig(customerConfig.getLoggerConfig()) .withDeserializeAfterSerialization(customerConfig.shouldDeserializeAfterSerialization()) // Temporary: remove along with the checkpointEmptyMap flag in a future major version. .withCheckpointEmptyMap(customerConfig.shouldCheckpointEmptyMap()) - .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])) - .build(); + .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])); + if (customerConfig.getSerDesExecutorService() != null) { + configBuilder.withSerDesExecutorService(customerConfig.getSerDesExecutorService()); + } + this.customerConfig = configBuilder.build(); } else { // Fallback to default config with in-memory client this.customerConfig = diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java index e251701d4..7ed5a58c3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java @@ -92,14 +92,6 @@ public final class DurableConfig { return t; }); - /** Default executor for customer SerDes calls and blocking payload storage I/O. */ - private static final ExecutorService DEFAULT_SERDES_THREAD_POOL = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r); - t.setName("durable-serdes-" + t.getId()); - t.setDaemon(true); - return t; - }); - private final DurableExecutionClient durableExecutionClient; private final SerDes serDes; private final ExecutorService executorService; @@ -118,8 +110,7 @@ private DurableConfig(Builder builder) { this.serDes = Objects.requireNonNullElseGet(builder.serDes, JacksonSerDes::new); this.executorService = Objects.requireNonNullElseGet(builder.executorService, DurableConfig::createDefaultExecutor); - this.serDesExecutorService = Objects.requireNonNullElseGet( - builder.serDesExecutorService, DurableConfig::createDefaultSerDesExecutor); + this.serDesExecutorService = builder.serDesExecutorService; this.loggerConfig = Objects.requireNonNullElseGet(builder.loggerConfig, LoggerConfig::defaults); this.pollingStrategy = Objects.requireNonNullElse(builder.pollingStrategy, PollingStrategies.Presets.DEFAULT); this.checkpointDelay = Objects.requireNonNullElseGet(builder.checkpointDelay, () -> Duration.ofSeconds(0)); @@ -178,7 +169,7 @@ public ExecutorService getExecutorService() { /** * Gets the executor used for customer SerDes calls and blocking payload storage I/O. * - * @return SerDes ExecutorService instance (never null) + * @return the configured executor, or {@code null} when SerDes calls execute inline */ public ExecutorService getSerDesExecutorService() { return serDesExecutorService; @@ -255,10 +246,7 @@ public void validateConfiguration() { if (getExecutorService() == null) { throw new IllegalStateException("ExecutorService configuration failed"); } - if (getSerDesExecutorService() == null) { - throw new IllegalStateException("SerDes ExecutorService configuration failed"); - } - if (getSerDesExecutorService() == getExecutorService()) { + if (getSerDesExecutorService() != null && getSerDesExecutorService() == getExecutorService()) { throw new IllegalStateException( "SerDes ExecutorService must be different from the user operation ExecutorService"); } @@ -338,11 +326,6 @@ private static ExecutorService createDefaultExecutor() { return DEFAULT_USER_THREAD_POOL; } - private static ExecutorService createDefaultSerDesExecutor() { - logger.debug("Creating default SerDes ExecutorService"); - return DEFAULT_SERDES_THREAD_POOL; - } - /** Builder for DurableConfig. Provides fluent API for configuring SDK components. */ public static final class Builder { private DurableExecutionClient durableExecutionClient; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java new file mode 100644 index 000000000..36cf1e726 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Controls how a {@link PreviewField} matches a field in a structured value. */ +public enum FieldMatchMode { + /** Matches the field name at any depth in the object tree. */ + ANYWHERE, + + /** Matches the exact dot-separated path from the root object. */ + PATH +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 76208cf94..fe97e6195 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -2,8 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; @@ -34,15 +37,21 @@ public final class FileSystemSerDes implements SerDes { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final int ENVELOPE_VERSION = 1; - private static final int OVERFLOW_THRESHOLD_BYTES = 256 * 1024 - 1024; + private static final int DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); + private static final ObjectReader ENVELOPE_READER = ENVELOPE_MAPPER + .reader() + .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .with(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY); private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); + private static final Pattern SHA_256_DIGEST_PATTERN = Pattern.compile("[0-9a-f]{64}"); private final Path basePath; private final FileSystemSerDesMode storageMode; private final FileSystemPathEncoding pathEncoding; private final SerDes delegate; + private final int checkpointEnvelopeLimitBytes; private final Function> previewGenerator; private FileSystemSerDes(Builder builder) { @@ -50,6 +59,7 @@ private FileSystemSerDes(Builder builder) { storageMode = builder.storageMode; pathEncoding = builder.pathEncoding; delegate = builder.delegate; + checkpointEnvelopeLimitBytes = builder.checkpointEnvelopeLimitBytes; previewGenerator = builder.previewGenerator; } @@ -72,7 +82,7 @@ public String serialize(Object value) { if (storageMode == FileSystemSerDesMode.OVERFLOW) { var inlineEnvelope = inlineEnvelope(serialized); - if (inlineEnvelope.getBytes(StandardCharsets.UTF_8).length <= OVERFLOW_THRESHOLD_BYTES) { + if (fitsCheckpoint(inlineEnvelope)) { return inlineEnvelope; } } @@ -95,11 +105,9 @@ public T deserialize(String data, TypeToken typeToken) { } var serialized = readPayload(envelope.get("file").textValue()); - if (envelope.hasNonNull("sha256")) { - var expected = envelope.get("sha256").textValue(); - if (!expected.equals(sha256(serialized))) { - throw new SerDesException("Filesystem SerDes payload digest does not match stored content"); - } + var expected = envelope.get("sha256").textValue(); + if (!expected.equals(sha256(serialized))) { + throw new SerDesException("Filesystem SerDes payload digest does not match stored content"); } return delegate.deserialize(serialized, typeToken); } @@ -126,7 +134,7 @@ private String fileEnvelope(Object value, String serialized, SerDesContext conte } } var encoded = writeEnvelope(envelope); - if (encoded.getBytes(StandardCharsets.UTF_8).length > OVERFLOW_THRESHOLD_BYTES) { + if (!fitsCheckpoint(encoded)) { throw new SerDesException("Filesystem SerDes file envelope exceeds the checkpoint payload limit"); } @@ -137,26 +145,24 @@ private String fileEnvelope(Object value, String serialized, SerDesContext conte private JsonNode parseEnvelope(String data) { final JsonNode node; try { - node = ENVELOPE_MAPPER.readTree(data); - } catch (IOException e) { + node = ENVELOPE_READER.readTree(data); + } catch (JsonProcessingException e) { + if (data.contains(ENVELOPE_MARKER)) { + throw new SerDesException("Malformed filesystem SerDes envelope", e); + } return null; } - if (node == null || !node.isObject()) { + if (node == null || !node.isObject() || !node.has(ENVELOPE_MARKER)) { return null; } - if (node.has(ENVELOPE_MARKER)) { - if (!node.get(ENVELOPE_MARKER).canConvertToInt() - || node.get(ENVELOPE_MARKER).intValue() != ENVELOPE_VERSION - || !isValidEnvelope(node)) { - throw new SerDesException("Malformed filesystem SerDes envelope"); - } - return node; + if (!node.get(ENVELOPE_MARKER).isIntegralNumber() + || !node.get(ENVELOPE_MARKER).canConvertToInt() + || node.get(ENVELOPE_MARKER).intValue() != ENVELOPE_VERSION + || !isValidEnvelope(node)) { + throw new SerDesException("Malformed filesystem SerDes envelope"); } - - // Read envelopes produced by the JavaScript SDK while leaving ordinary user JSON untouched. - var legacyFieldCount = node.has("preview") ? 2 : 1; - return node.size() == legacyFieldCount && isValidEnvelope(node) ? node : null; + return node; } private static boolean isValidEnvelope(JsonNode node) { @@ -165,10 +171,18 @@ private static boolean isValidEnvelope(JsonNode node) { if (hasData == hasFile) { return false; } - if (node.has("preview") && !node.get("preview").isObject()) { + if (hasData) { + return node.size() == 2; + } + if (!node.has("sha256") + || !node.get("sha256").isTextual() + || !SHA_256_DIGEST_PATTERN + .matcher(node.get("sha256").textValue()) + .matches()) { return false; } - return !node.has("sha256") || node.get("sha256").isTextual(); + var hasPreview = node.has("preview"); + return (!hasPreview || node.get("preview").isObject()) && node.size() == (hasPreview ? 4 : 3); } private Path payloadPath(SerDesContext context, String digest) { @@ -240,6 +254,10 @@ private String encode(String value) { return pathEncoding == FileSystemPathEncoding.HASH ? sha256(value) : percentEncode(value); } + private boolean fitsCheckpoint(String envelope) { + return envelope.getBytes(StandardCharsets.UTF_8).length <= checkpointEnvelopeLimitBytes; + } + private static String percentEncode(String value) { var bytes = value.getBytes(StandardCharsets.UTF_8); var encoded = new StringBuilder(bytes.length); @@ -285,6 +303,7 @@ public static final class Builder { private FileSystemSerDesMode storageMode = FileSystemSerDesMode.ALWAYS; private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; private SerDes delegate = new JacksonSerDes(); + private int checkpointEnvelopeLimitBytes = DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES; private Function> previewGenerator; private Builder(Path basePath) { @@ -306,11 +325,27 @@ public Builder delegate(SerDes delegate) { return this; } + /** Sets the maximum UTF-8 size of an inline or file checkpoint envelope. */ + public Builder checkpointEnvelopeLimitBytes(int checkpointEnvelopeLimitBytes) { + if (checkpointEnvelopeLimitBytes <= 0) { + throw new IllegalArgumentException("checkpointEnvelopeLimitBytes must be positive"); + } + this.checkpointEnvelopeLimitBytes = checkpointEnvelopeLimitBytes; + return this; + } + public Builder previewGenerator(Function> previewGenerator) { this.previewGenerator = Objects.requireNonNull(previewGenerator, "previewGenerator cannot be null"); return this; } + /** Configures structured preview generation from the original value. */ + public Builder previewConfig(PreviewConfig previewConfig) { + Objects.requireNonNull(previewConfig, "previewConfig cannot be null"); + this.previewGenerator = value -> SerDesPreview.buildPreview(value, previewConfig); + return this; + } + public FileSystemSerDes build() { return new FileSystemSerDes(this); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java new file mode 100644 index 000000000..7d2f8ebf4 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Configuration for {@link SerDesPreview#buildPreview(Object, PreviewConfig)}. + * + * @param mode whether fields are included or excluded by default + * @param include fields made visible in {@link PreviewMode#EXCLUDE_ALL} mode + * @param exclude fields hidden from the preview + * @param mask fields made visible with their values replaced by {@code maskString} + * @param maskString replacement for masked field values + * @param maxPreviewBytes maximum estimated UTF-8 size of accepted preview entries + */ +public record PreviewConfig( + PreviewMode mode, + List include, + List exclude, + List mask, + String maskString, + int maxPreviewBytes) { + public static final String DEFAULT_MASK_STRING = "***"; + public static final int DEFAULT_MAX_PREVIEW_BYTES = 4096; + + public PreviewConfig { + Objects.requireNonNull(mode, "mode cannot be null"); + include = immutableFields(include, "include"); + exclude = immutableFields(exclude, "exclude"); + mask = immutableFields(mask, "mask"); + Objects.requireNonNull(maskString, "maskString cannot be null"); + if (maxPreviewBytes < 0) { + throw new IllegalArgumentException("maxPreviewBytes cannot be negative"); + } + } + + /** Creates a preview configuration builder. */ + public static Builder builder(PreviewMode mode) { + return new Builder(mode); + } + + private static List immutableFields(List fields, String name) { + Objects.requireNonNull(fields, name + " cannot be null"); + if (fields.stream().anyMatch(Objects::isNull)) { + throw new NullPointerException(name + " cannot contain null"); + } + return List.copyOf(fields); + } + + /** Builder for {@link PreviewConfig}. */ + public static final class Builder { + private final PreviewMode mode; + private final List include = new ArrayList<>(); + private final List exclude = new ArrayList<>(); + private final List mask = new ArrayList<>(); + private String maskString = DEFAULT_MASK_STRING; + private int maxPreviewBytes = DEFAULT_MAX_PREVIEW_BYTES; + + private Builder(PreviewMode mode) { + this.mode = Objects.requireNonNull(mode, "mode cannot be null"); + } + + /** Adds fields that should be visible. */ + public Builder include(PreviewField... fields) { + include.addAll(validFields(fields, "include")); + return this; + } + + /** Adds fields that should be hidden. */ + public Builder exclude(PreviewField... fields) { + exclude.addAll(validFields(fields, "exclude")); + return this; + } + + /** Adds fields whose values should be masked. */ + public Builder mask(PreviewField... fields) { + mask.addAll(validFields(fields, "mask")); + return this; + } + + /** Sets the value used for masked fields. */ + public Builder maskString(String maskString) { + this.maskString = Objects.requireNonNull(maskString, "maskString cannot be null"); + return this; + } + + /** Sets the maximum estimated UTF-8 preview size. */ + public Builder maxPreviewBytes(int maxPreviewBytes) { + if (maxPreviewBytes < 0) { + throw new IllegalArgumentException("maxPreviewBytes cannot be negative"); + } + this.maxPreviewBytes = maxPreviewBytes; + return this; + } + + /** Returns the immutable preview configuration. */ + public PreviewConfig build() { + return new PreviewConfig(mode, include, exclude, mask, maskString, maxPreviewBytes); + } + + private static List validFields(PreviewField[] fields, String name) { + Objects.requireNonNull(fields, name + " cannot be null"); + if (Arrays.stream(fields).anyMatch(Objects::isNull)) { + throw new NullPointerException(name + " cannot contain null"); + } + return List.of(fields); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java new file mode 100644 index 000000000..49bd84304 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; + +/** + * A field selector used by {@link PreviewConfig}. + * + * @param name a field name for {@link FieldMatchMode#ANYWHERE}, or a dot-separated path for {@link FieldMatchMode#PATH} + * @param match how the selector is matched + */ +public record PreviewField(String name, FieldMatchMode match) { + public PreviewField { + Objects.requireNonNull(name, "name cannot be null"); + Objects.requireNonNull(match, "match cannot be null"); + if (name.isBlank()) { + throw new IllegalArgumentException("name cannot be blank"); + } + } + + /** Creates a selector that matches this field name at any depth. */ + public PreviewField(String name) { + this(name, FieldMatchMode.ANYWHERE); + } + + /** Creates a selector that matches this field name at any depth. */ + public static PreviewField anywhere(String name) { + return new PreviewField(name, FieldMatchMode.ANYWHERE); + } + + /** Creates a selector that matches an exact dot-separated path. */ + public static PreviewField path(String name) { + return new PreviewField(name, FieldMatchMode.PATH); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java new file mode 100644 index 000000000..8dd0f4851 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Controls which fields are visible by default in a structured payload preview. */ +public enum PreviewMode { + /** Includes every field unless an exclude rule removes it. */ + INCLUDE_ALL, + + /** Excludes every field unless an include or mask rule selects it. */ + EXCLUDE_ALL +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java new file mode 100644 index 000000000..caa025079 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java @@ -0,0 +1,191 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import software.amazon.lambda.durable.exception.SerDesException; + +/** Utilities for building compact structured previews for externally stored SerDes payloads. */ +public final class SerDesPreview { + private static final ObjectMapper MAPPER = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + + private SerDesPreview() {} + + /** + * Builds a preview from an object using include, exclude, mask, path-matching, and byte-budget rules. + * + *

Object fields are traversed in their Jackson serialization order. Object arrays are flattened into their + * containing path, while scalar arrays are preserved as field values. Fields whose names contain dots are skipped + * because they cannot be distinguished from dot-separated paths. + * + * @return a nested preview map, or {@code null} when no fields are visible + */ + public static Map buildPreview(Object value, PreviewConfig config) { + Objects.requireNonNull(config, "config cannot be null"); + final JsonNode root; + try { + root = MAPPER.valueToTree(value); + } catch (IllegalArgumentException e) { + throw new SerDesException("Failed to convert value for preview generation", e); + } + return buildPreview(root, config); + } + + /** + * Builds a preview from a JSON string. + * + * @return a nested preview map, or {@code null} when no fields are visible + */ + public static Map buildPreviewFromJson(String value, PreviewConfig config) { + Objects.requireNonNull(value, "value cannot be null"); + Objects.requireNonNull(config, "config cannot be null"); + try { + return buildPreview(MAPPER.readTree(value), config); + } catch (JsonProcessingException e) { + throw new SerDesException("Built-in preview generation requires JSON", e); + } + } + + private static Map buildPreview(JsonNode root, PreviewConfig config) { + if (root == null || !root.isObject()) { + return null; + } + + var pairs = new ArrayList(); + collect(root, "", config, pairs); + if (pairs.isEmpty()) { + return null; + } + + Map result = new LinkedHashMap<>(); + for (var pair : pairs) { + var candidate = copy(result); + insert(candidate, pair.path(), pair.value()); + if (serializedSize(candidate) > config.maxPreviewBytes()) { + break; + } + result = candidate; + } + return result.isEmpty() ? null : result; + } + + private static void collect(JsonNode node, String pathPrefix, PreviewConfig config, List pairs) { + if (node == null || node.isNull()) { + return; + } + if (node.isArray()) { + for (var item : node) { + collect(item, pathPrefix, config, pairs); + } + return; + } + if (!node.isObject()) { + return; + } + + for (var field : node.properties()) { + var name = field.getKey(); + if (name.contains(".")) { + continue; + } + var path = pathPrefix.isEmpty() ? name : pathPrefix + "." + name; + var masked = isMatched(path, config.mask()); + var excluded = isMatched(path, config.exclude()); + var visible = !excluded + && (masked || config.mode() == PreviewMode.INCLUDE_ALL || isMatched(path, config.include())); + + if (!visible) { + if (!excluded) { + collect(field.getValue(), path, config, pairs); + } + continue; + } + if (masked) { + pairs.add(new PreviewEntry(path, config.maskString())); + } else if (isScalarArray(field.getValue())) { + pairs.add(new PreviewEntry(path, MAPPER.convertValue(field.getValue(), Object.class))); + } else if (field.getValue().isContainerNode()) { + collect(field.getValue(), path, config, pairs); + } else { + pairs.add(new PreviewEntry(path, MAPPER.convertValue(field.getValue(), Object.class))); + } + } + } + + private static boolean isScalarArray(JsonNode node) { + if (!node.isArray()) { + return false; + } + for (var item : node) { + if (item.isContainerNode()) { + return false; + } + } + return true; + } + + private static boolean isMatched(String path, List fields) { + for (var field : fields) { + if (field.match() == FieldMatchMode.PATH) { + if (path.equals(field.name())) { + return true; + } + } else { + for (var segment : path.split("\\.")) { + if (segment.equals(field.name())) { + return true; + } + } + } + } + return false; + } + + private static int serializedSize(Map preview) { + try { + return MAPPER.writeValueAsBytes(preview).length; + } catch (JsonProcessingException e) { + throw new SerDesException("Failed to measure preview size", e); + } + } + + @SuppressWarnings("unchecked") + private static Map copy(Map source) { + var copy = new LinkedHashMap(); + for (var entry : source.entrySet()) { + var value = entry.getValue(); + copy.put(entry.getKey(), value instanceof Map nested ? copy((Map) nested) : value); + } + return copy; + } + + @SuppressWarnings("unchecked") + private static void insert(Map result, String path, Object value) { + var parts = path.split("\\."); + Map current = result; + for (int index = 0; index < parts.length - 1; index++) { + var existing = current.get(parts[index]); + if (!(existing instanceof Map)) { + existing = new LinkedHashMap(); + current.put(parts[index], existing); + } + current = (Map) existing; + } + current.put(parts[parts.length - 1], value); + } + + private record PreviewEntry(String path, Object value) {} +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index aab88398e..48282361c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -40,8 +40,13 @@ protected boolean removeEldestEntry(Map.Entry> e } }); + /** + * Creates an invocation-scoped runner. + * + * @param executorService executor for SerDes calls, or {@code null} to execute inline + */ public SerDesRunner(ExecutorService executorService) { - this.executorService = Objects.requireNonNull(executorService, "executorService cannot be null"); + this.executorService = executorService; } /** Serializes a value with the supplied durable payload context. */ @@ -111,6 +116,13 @@ private void putCompleted(CacheKey key, Object value) { private CompletableFuture submit(SerDesContext context, Supplier action) { Objects.requireNonNull(context, "context cannot be null"); Objects.requireNonNull(action, "action cannot be null"); + if (executorService == null) { + try { + return CompletableFuture.completedFuture(SerDesContext.callWithContext(context, action)); + } catch (Throwable failure) { + return CompletableFuture.failedFuture(failure); + } + } return CompletableFuture.supplyAsync(() -> SerDesContext.callWithContext(context, action), executorService); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java index 9d06110c0..539ac792d 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -54,8 +55,7 @@ void testDefaultConfig_CreatesWithDefaults() { assertInstanceOf(JacksonSerDes.class, config.getSerDes()); assertNotNull(config.getExecutorService()); assertInstanceOf(ExecutorService.class, config.getExecutorService()); - assertNotNull(config.getSerDesExecutorService()); - assertInstanceOf(ExecutorService.class, config.getSerDesExecutorService()); + assertNull(config.getSerDesExecutorService()); } @Test @@ -238,13 +238,11 @@ void testDefaultExecutorService_IsNotNull() { } @Test - void testDefaultSerDesExecutorService_IsNotNull() { + void testDefaultSerDesExecutorService_IsNull() { var config = DurableConfig.builder().withDurableExecutionClient(mockClient).build(); - var executor = config.getSerDesExecutorService(); - assertNotNull(executor); - assertFalse(executor.isShutdown()); + assertNull(config.getSerDesExecutorService()); } @Test @@ -270,7 +268,8 @@ void testBuilder_MultipleBuilds_CreateIndependentInstances() { // ExecutorService should be different instances (each gets its own) assertSame(config1.getExecutorService(), config2.getExecutorService()); - assertSame(config1.getSerDesExecutorService(), config2.getSerDesExecutorService()); + assertNull(config1.getSerDesExecutorService()); + assertNull(config2.getSerDesExecutorService()); } @Test @@ -282,7 +281,7 @@ void testBuilder_NullExecutorService_AllowedAndUsesDefault() { .build(); assertNotNull(config.getExecutorService()); - assertNotNull(config.getSerDesExecutorService()); + assertNull(config.getSerDesExecutorService()); } @Test @@ -514,17 +513,6 @@ void validateConfiguration_ThrowsWhenExecutorServiceIsNull() throws Exception { assertEquals("ExecutorService configuration failed", ex.getMessage()); } - @Test - void validateConfiguration_ThrowsWhenSerDesExecutorServiceIsNull() throws Exception { - var config = - DurableConfig.builder().withDurableExecutionClient(mockClient).build(); - - setField(config, "serDesExecutorService", null); - - var ex = assertThrows(IllegalStateException.class, config::validateConfiguration); - assertEquals("SerDes ExecutorService configuration failed", ex.getMessage()); - } - @Test void validateConfiguration_ChecksClientBeforeSerDes() throws Exception { var config = diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 7d3a17681..e1733162c 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -94,6 +94,47 @@ void overflowModeStoresLargePayload() throws Exception { assertTrue(MAPPER.readTree(envelope).hasNonNull("file")); } + @Test + void checkpointEnvelopeLimitCanBeIncreasedForLargerInlinePayloads() throws Exception { + var value = "x".repeat(300_000); + var context = new SerDesContext(realisticArn(), "1"); + var defaultSerDes = FileSystemSerDes.builder(tempDir) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .build(); + var largerEnvelopeSerDes = FileSystemSerDes.builder(tempDir) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .checkpointEnvelopeLimitBytes(512 * 1024) + .build(); + + assertTrue( + MAPPER.readTree(runner.serialize(defaultSerDes, value, context)).hasNonNull("file")); + assertTrue(MAPPER.readTree(runner.serialize(largerEnvelopeSerDes, value, context)) + .hasNonNull("data")); + } + + @Test + void checkpointEnvelopeLimitMustBePositive() { + var zeroFailure = assertThrows(IllegalArgumentException.class, () -> FileSystemSerDes.builder(tempDir) + .checkpointEnvelopeLimitBytes(0)); + var negativeFailure = assertThrows(IllegalArgumentException.class, () -> FileSystemSerDes.builder(tempDir) + .checkpointEnvelopeLimitBytes(-1)); + + assertEquals("checkpointEnvelopeLimitBytes must be positive", zeroFailure.getMessage()); + assertEquals("checkpointEnvelopeLimitBytes must be positive", negativeFailure.getMessage()); + } + + @Test + void checkpointEnvelopeLimitAlsoAppliesToFileEnvelopes() { + var serDes = FileSystemSerDes.builder(tempDir) + .checkpointEnvelopeLimitBytes(1) + .build(); + + var failure = assertThrows( + SerDesException.class, () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + + assertTrue(failure.getMessage().contains("checkpoint payload limit")); + } + @Test void supportsHashPathEncodingAndPreview() throws Exception { var serDes = FileSystemSerDes.builder(tempDir) @@ -124,20 +165,49 @@ void rejectsPreviewThatMakesFileEnvelopeTooLarge() throws Exception { } @Test - void readsJavaScriptFilesystemEnvelope() throws Exception { - var payloadFile = tempDir.resolve("js-payload.json"); - Files.writeString(payloadFile, "{\"value\":\"js\"}", StandardCharsets.UTF_8); - var envelope = MAPPER.writeValueAsString(Map.of("file", payloadFile.toString())); + void structuredPreviewSelectsAndMasksFields() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir) + .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) + .mask(PreviewField.anywhere("email")) + .build()) + .build(); + var value = Map.of( + "id", + "order-1", + "email", + "root@example.com", + "customer", + Map.of("status", "ready", "email", "customer@example.com", "secret", "hidden")); + + var preview = MAPPER.readTree(runner.serialize(serDes, value, new SerDesContext(realisticArn(), "1"))) + .get("preview"); + + assertEquals("order-1", preview.get("id").textValue()); + assertEquals("***", preview.get("email").textValue()); + assertEquals("ready", preview.get("customer").get("status").textValue()); + assertEquals("***", preview.get("customer").get("email").textValue()); + assertFalse(preview.get("customer").has("secret")); + } + + @Test + void unmarkedDataAndFileObjectsAreDelegatedNormally() { var serDes = FileSystemSerDes.builder(tempDir).build(); - assertEquals(new Value("js"), serDes.deserialize(envelope, TypeToken.get(Value.class))); + assertEquals( + Map.of("data", "value"), + serDes.deserialize("{\"data\":\"value\"}", new TypeToken>() {})); + assertEquals( + Map.of("file", "value"), + serDes.deserialize("{\"file\":\"value\"}", new TypeToken>() {})); } @Test void rejectsFileOutsideConfiguredBasePath() throws Exception { var externalFile = Files.createTempFile("filesystem-serdes", ".json"); Files.writeString(externalFile, "\"secret\"", StandardCharsets.UTF_8); - var envelope = MAPPER.writeValueAsString(Map.of("file", externalFile.toString())); + var envelope = MAPPER.writeValueAsString(Map.of( + "__durable_execution_filesystem_serdes", 1, "file", externalFile.toString(), "sha256", "0".repeat(64))); var serDes = FileSystemSerDes.builder(tempDir).build(); assertThrows(SerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); @@ -154,6 +224,46 @@ void rejectsMalformedRecognizedEnvelope() { TypeToken.get(String.class))); } + @Test + void rejectsTrailingAndDuplicateFieldsInMarkedEnvelope() { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var validDataEnvelope = "{\"__durable_execution_filesystem_serdes\":1,\"data\":\"\\\"value\\\"\"}"; + + assertThrows( + SerDesException.class, + () -> serDes.deserialize(validDataEnvelope + " true", TypeToken.get(String.class))); + assertThrows( + SerDesException.class, + () -> serDes.deserialize( + "{\"__durable_execution_filesystem_serdes\":1," + + "\"__durable_execution_filesystem_serdes\":2,\"data\":\"\\\"value\\\"\"}", + TypeToken.get(String.class))); + } + + @Test + void customDelegateControlsValueEncoding() throws Exception { + var jackson = new JacksonSerDes(); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + return "custom:" + jackson.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return jackson.deserialize(data.substring("custom:".length()), typeToken); + } + }; + var serDes = FileSystemSerDes.builder(tempDir).delegate(delegate).build(); + var context = new SerDesContext(realisticArn(), "1"); + + var envelope = runner.serialize(serDes, new Value("custom"), context); + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + + assertTrue(Files.readString(file).startsWith("custom:")); + assertEquals(new Value("custom"), runner.deserialize(serDes, envelope, TypeToken.get(Value.class), context)); + } + private static String realisticArn() { return "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + "/durable-execution/execution-name/invocation-id"; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java new file mode 100644 index 000000000..32936c123 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java @@ -0,0 +1,196 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.SerDesException; + +class SerDesPreviewTest { + + @Test + void includeAllAppliesExcludeAndMaskRules() { + var value = Map.of( + "id", + "123", + "email", + "alice@example.com", + "ssn", + "000-00-0000", + "user", + Map.of("name", "Alice", "role", "admin")); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .exclude(PreviewField.anywhere("role")) + .mask(PreviewField.anywhere("ssn")) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals("123", preview.get("id")); + assertEquals("***", preview.get("ssn")); + assertFalse(nested(preview, "user").containsKey("role")); + assertEquals("Alice", nested(preview, "user").get("name")); + } + + @Test + void excludeAllIncludesSelectedAndMaskedFields() { + var value = Map.of("id", "123", "email", "alice@example.com", "ssn", "000-00-0000"); + var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id")) + .mask(PreviewField.anywhere("ssn")) + .build(); + + assertEquals(Map.of("id", "123", "ssn", "***"), SerDesPreview.buildPreview(value, config)); + } + + @Test + void excludeWinsOverMask() { + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .exclude(PreviewField.anywhere("ssn")) + .mask(PreviewField.anywhere("ssn")) + .build(); + + assertEquals(Map.of("id", "123"), SerDesPreview.buildPreview(Map.of("id", "123", "ssn", "secret"), config)); + } + + @Test + void pathAndAnywhereMatchingHaveDifferentScopes() { + var value = Map.of("email", "root@example.com", "user", Map.of("email", "nested@example.com", "id", "user-1")); + var pathConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.path("email")) + .build(); + var anywhereConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("email")) + .build(); + + var pathPreview = SerDesPreview.buildPreview(value, pathConfig); + var anywherePreview = SerDesPreview.buildPreview(value, anywhereConfig); + + assertEquals(Map.of("email", "root@example.com"), pathPreview); + assertEquals("root@example.com", anywherePreview.get("email")); + assertEquals("nested@example.com", nested(anywherePreview, "user").get("email")); + } + + @Test + void arraysMergeFieldsAtTheirContainingPath() { + var value = Map.of("items", List.of(Map.of("id", "first"), Map.of("email", "second@example.com"))); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(Map.of("id", "first", "email", "second@example.com"), nested(preview, "items")); + } + + @Test + void preservesScalarArrays() { + var preview = SerDesPreview.buildPreviewFromJson( + "{\"tags\":[\"a\",\"b\"]}", + PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build()); + + assertEquals(Map.of("tags", List.of("a", "b")), preview); + } + + @Test + void customMaskStringAndByteBudgetAreApplied() { + var value = new LinkedHashMap(); + value.put("first", "one"); + value.put("second", "two"); + value.put("secret", "hidden"); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .mask(PreviewField.anywhere("secret")) + .maskString("[REDACTED]") + .maxPreviewBytes(18) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(1, preview.size()); + assertTrue(preview.containsKey("first")); + } + + @Test + void nestedPreviewUsesExactSerializedByteBudget() { + var value = Map.of("a", Map.of("b", "x")); + + var tooSmall = SerDesPreview.buildPreview( + value, + PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(14) + .build()); + var exactFit = SerDesPreview.buildPreview( + value, + PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(15) + .build()); + + assertNull(tooSmall); + assertEquals(value, exactFit); + } + + @Test + void returnsNullWhenNoFieldsAreVisibleOrValueIsNotAnObject() { + var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL).build(); + + assertNull(SerDesPreview.buildPreview(Map.of("id", "123"), config)); + assertNull(SerDesPreview.buildPreview("value", config)); + assertNull(SerDesPreview.buildPreview(List.of(Map.of("id", "123")), config)); + } + + @Test + void objectPreviewUsesJacksonSerDesTimeFormats() { + var instant = Instant.parse("2026-08-26T03:30:00Z"); + var duration = Duration.ofMinutes(5); + var localDateTime = LocalDateTime.parse("2026-08-26T03:30:00"); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); + + var preview = SerDesPreview.buildPreview(new TemporalPayload(instant, duration, localDateTime), config); + + assertEquals("2026-08-26T03:30:00Z", preview.get("instant")); + assertEquals(0, new BigDecimal("300").compareTo((BigDecimal) preview.get("duration"))); + assertEquals("2026-08-26T03:30:00", preview.get("localDateTime")); + } + + @Test + void jsonPreviewRejectsMalformedJsonAndSkipsDottedFieldNames() { + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); + + assertThrows(SerDesException.class, () -> SerDesPreview.buildPreviewFromJson("not-json", config)); + assertEquals( + Map.of("safe", "value"), + SerDesPreview.buildPreviewFromJson("{\"safe\":\"value\",\"not.addressable\":\"secret\"}", config)); + } + + @Test + void validatesConfiguration() { + assertThrows(NullPointerException.class, () -> PreviewConfig.builder(null)); + assertNull(SerDesPreview.buildPreview( + Map.of("id", "123"), + PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(0) + .build())); + assertThrows(IllegalArgumentException.class, () -> PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(-1)); + assertThrows(IllegalArgumentException.class, () -> new PreviewField(" ")); + assertThrows(NullPointerException.class, () -> PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .include((PreviewField) null)); + } + + @SuppressWarnings("unchecked") + private static Map nested(Map value, String field) { + return (Map) value.get(field); + } + + private record TemporalPayload(Instant instant, Duration duration, LocalDateTime localDateTime) {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index 9bfac2054..d79d85210 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -52,6 +52,26 @@ public String serialize(Object value) { assertNull(SerDesContext.getCurrentContext()); } + @Test + void executesInlineWhenNoExecutorIsConfigured() { + var inlineRunner = new SerDesRunner(null); + var callingThread = Thread.currentThread(); + var observedThread = new AtomicReference(); + var context = new SerDesContext("arn:test", "entity"); + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + observedThread.set(Thread.currentThread()); + assertEquals(context, SerDesContext.getCurrentContext()); + return super.serialize(value); + } + }; + + assertEquals("\"value\"", inlineRunner.serialize(serDes, "value", context)); + assertSame(callingThread, observedThread.get()); + assertNull(SerDesContext.getCurrentContext()); + } + @Test void cachesSuccessfulDeserializationForInvocation() { var calls = new AtomicInteger(); From 1e5c5d5fb9d834b7bf4123aa694d2dc54cb6aea6 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 00:14:43 +0000 Subject: [PATCH 03/11] feat: expand filesystem SerDes coverage and retries --- README.md | 1 + docs/adr/005-filesystem-serdes.md | 17 +- docs/advanced/configuration.md | 3 + docs/advanced/error-handling.md | 1 + docs/advanced/serdes.md | 174 ++++++++++++++++++ docs/design.md | 4 +- .../FileSystemSerDesIntegrationTest.java | 171 +++++++++++++++++ .../durable/testing/AsyncExecution.java | 4 +- .../testing/CloudDurableTestRunner.java | 3 +- .../testing/LocalDurableTestRunner.java | 19 +- .../lambda/durable/testing/TestOperation.java | 20 +- .../lambda/durable/testing/TestResult.java | 30 ++- .../testing/cloud/HistoryEventProcessor.java | 119 +++++++++++- .../local/LocalMemoryExecutionClient.java | 20 +- .../durable/testing/AsyncExecutionTest.java | 91 +++++++++ .../testing/LocalDurableTestRunnerTest.java | 28 +++ .../durable/testing/TestOperationTest.java | 54 ++++++ .../cloud/HistoryEventProcessorTest.java | 130 +++++++++++++ .../exception/RetryableSerDesException.java | 19 ++ .../durable/serde/FileSystemSerDes.java | 38 ++-- .../lambda/durable/serde/RetrySerDes.java | 44 +++++ .../durable/serde/SerDesRetryExecutor.java | 87 +++++++++ .../durable/serde/FileSystemSerDesTest.java | 30 ++- .../lambda/durable/serde/RetrySerDesTest.java | 102 ++++++++++ 24 files changed, 1167 insertions(+), 42 deletions(-) create mode 100644 docs/advanced/serdes.md create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/AsyncExecutionTest.java create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java diff --git a/README.md b/README.md index 766a71b02..296cbc74d 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ See [Deploy Lambda durable functions with Infrastructure as Code](https://docs.a **Advanced Topics** - [Configuration](docs/advanced/configuration.md) - Customize SDK behaviour +- [Serialization](docs/advanced/serdes.md) - Custom SerDes, filesystem storage, previews, retries, and caching - [Error Handling](docs/advanced/error-handling.md) - SDK exceptions for handling failures - [Logging](docs/advanced/logging.md) - How to use DurableLogger - [Migrating from 1.x to 2.x](docs/migration-1.x-to-2.x.md) - Upgrade guide for breaking changes since `v1.2.1` diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 38d40f35b..66884a4ea 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -130,6 +130,12 @@ with filesystem storage. Structured previews support include-all/exclude-all modes, include/exclude/mask selectors, anywhere or exact-path matching, custom mask text, and a default 4 KiB preview budget. Custom preview callbacks remain available. +### Retry transient failures + +Filesystem `IOException`s are surfaced as `RetryableSerDesException`. `RetrySerDes` decorates any `SerDes` and applies +an existing `RetryStrategy` only to retryable failures. Permanent `SerDesException` failures propagate immediately. +Retry delays run inline or on the configured SerDes executor. + ### Envelope and file publication Java writes versioned envelopes: @@ -143,9 +149,10 @@ Java writes versioned envelopes: Only envelopes containing the reserved version marker are interpreted as filesystem payloads. Unmarked JSON, including objects with `data` or `file` fields, is passed to the delegate SerDes unchanged. -File names include the entity ID and serialized-payload digest. Files are created with `CREATE_NEW`; an existing file is -accepted only when its contents match. This prevents a later retry from overwriting data referenced by an earlier -checkpoint. Deserialization rejects paths outside the configured base directory and verifies the digest when present. +File names include the entity ID, serialized-payload digest, and a unique suffix. Files are created with `CREATE_NEW`, +and failed writes are removed before the retryable failure is propagated. This prevents a later serialization from +overwriting data referenced by an earlier checkpoint. Deserialization rejects paths outside the configured base +directory and verifies the digest. ### Initial invocation input @@ -157,6 +164,10 @@ After the invocation starts, the SDK routes root input deserialization, operatio through `SerDesRunner`. A chained-invoke boundary requires compatible filesystem configuration and shared storage only when that boundary explicitly selects `FileSystemSerDes`. +Local and cloud test utilities propagate `SerDesRunner` and durable payload contexts through result deserialization, +operation inspection, history processing, and asynchronous snapshots. Local runner invocations and cloud history +snapshots receive separate caches. + ## Consequences Positive: diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 3e4467aa9..08a4f98bf 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -108,6 +108,9 @@ Do not use Lambda's `/tmp` directory: replay can run in another execution enviro as EFS. S3 Files users must account for synchronization and crash-durability behavior. A chained-invoke boundary only requires shared storage and compatible filesystem configuration when that boundary explicitly uses `FileSystemSerDes`. +See [Serialization and Filesystem Storage](serdes.md) for envelope details, structured previews, retry configuration, +testing behavior, and operational guidance. + ### Dynamic plugin loading Dynamic plugin loading is an opt-in alternative to registering plugins in application code. Put provider JARs on the application class path, then set `DURABLE_EXECUTION_PLUGINS` to an ordered, comma-separated list of provider names: diff --git a/docs/advanced/error-handling.md b/docs/advanced/error-handling.md index e81deeb61..ff2cc252b 100644 --- a/docs/advanced/error-handling.md +++ b/docs/advanced/error-handling.md @@ -12,6 +12,7 @@ Error RuntimeException └── DurableExecutionException - General durable exception ├── SerDesException - Serialization and deserialization exception. + │ └── RetryableSerDesException - Transient SerDes I/O failure; retried only by RetrySerDes. ├── UnrecoverableDurableExecutionException - Execution cannot be recovered. The durable execution will be immediately terminated. │ ├── NonDeterministicExecutionException - Code changed between original execution and replay. Fix code to maintain determinism; don't change step order/names. │ └── IllegalDurableOperationException - An illegal operation was detected. The execution will be immediately terminated. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md new file mode 100644 index 000000000..1d364e0a4 --- /dev/null +++ b/docs/advanced/serdes.md @@ -0,0 +1,174 @@ +# Serialization and Filesystem Storage + +The SDK uses `SerDes` for handler input/output, durable operation results and state, invoke payloads/results, callback +results, and serialized exceptions. + +## Custom value encoding + +Implement `SerDes` to control object-to-string encoding: + +```java +public interface SerDes { + String serialize(Object value); + + T deserialize(String data, TypeToken typeToken); +} +``` + +Configure a default: + +```java +return DurableConfig.builder() + .withSerDes(new MyCustomSerDes()) + .build(); +``` + +Operation configuration can override the default: + +```java +var invokeConfig = InvokeConfig.builder() + .payloadSerDes(new JacksonSerDes()) + .serDes(fileSystemSerDes) + .build(); +``` + +This sends an ordinary JSON invoke payload while decoding the invoke result through filesystem storage. The same +selection model applies to steps, callbacks, child contexts, map, parallel, and wait-for-condition. + +## SerDesContext + +SDK-managed calls expose durable identity through thread-local storage without changing the `SerDes` interface: + +```java +var context = SerDesContext.getCurrentContext(); +if (context != null) { + var executionArn = context.durableExecutionArn(); + var entityId = context.entityId(); +} +``` + +The context is installed only while the SDK invokes `serialize` or `deserialize` and is restored in `finally`. +Direct customer calls return `null`. + +## Execution and caching + +SerDes calls execute inline by default. Configure a dedicated executor when serialization performs blocking filesystem +or network I/O: + +```java +return DurableConfig.builder() + .withSerDes(fileSystemSerDes) + .withSerDesExecutorService(Executors.newFixedThreadPool(8)) + .build(); +``` + +Do not reuse the user-operation executor. Operations synchronously wait for SerDes calls, so sharing a saturated pool +can deadlock. + +Each Lambda invocation owns a `SerDesRunner`. It shares concurrent reads and keeps up to 256 successful +deserializations in a weak-reference LRU cache. Cache identity includes the SerDes instance, execution ARN, entity ID, +target type, and serialized payload hash. + +Local and cloud testing utilities propagate the same contexts and caching behavior through `TestResult`, +`TestOperation`, history processing, and asynchronous execution snapshots. + +## FileSystemSerDes + +`FileSystemSerDes` stores serialized values on a shared durable filesystem: + +```java +var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .delegate(new JacksonSerDes()) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .pathEncoding(FileSystemPathEncoding.HASH) + .checkpointEnvelopeLimitBytes(256 * 1024 - 1024) + .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("status")) + .mask(PreviewField.anywhere("email")) + .build()) + .build(); +``` + +### Storage modes + +| Mode | Behavior | +| --- | --- | +| `ALWAYS` | Writes every SDK-managed non-null payload to a file. | +| `OVERFLOW` | Keeps the complete envelope inline until it exceeds the configured limit. | + +The default checkpoint-envelope limit is 255 KiB and can be changed with +`checkpointEnvelopeLimitBytes(...)`. + +### Path encoding + +| Encoding | Behavior | +| --- | --- | +| `URI` | Percent-encodes readable execution and entity path segments. | +| `HASH` | Uses fixed-length SHA-256 segments for arbitrary or long identifiers. | + +Files include the serialized-payload digest and a unique suffix. They are published with `CREATE_NEW`; failed writes are +cleaned up. Each envelope includes a SHA-256 digest that is verified when the file is loaded. + +Only objects containing the reserved version marker are treated as filesystem envelopes. Ordinary JSON with `data` or +`file` fields is passed to the configured delegate. + +```json +{"__durable_execution_filesystem_serdes":1,"data":""} +{"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":""} +``` + +### Structured previews + +`PreviewConfig` supports: + +- `INCLUDE_ALL` or `EXCLUDE_ALL` defaults; +- field matching anywhere or by exact dotted path; +- include, exclude, and mask selectors; +- configurable mask text; +- a default 4 KiB preview budget. + +Object arrays are flattened at their containing path, while scalar arrays are retained. Field names containing dots are +skipped because they are ambiguous with path selectors. Use `previewGenerator(...)` for custom behavior. + +### Initial input + +The durable execution ARN does not exist before the initial Lambda invocation starts. A direct +`FileSystemSerDes.serialize()` call therefore delegates normally when no `SerDesContext` exists. Root input is ordinary +delegate JSON; SDK-managed output and operation payloads can use filesystem storage after the invocation begins. + +### Operational requirements + +- Do not use Lambda `/tmp`; replay may run in another execution environment. +- Use a shared durable mount such as EFS. +- If using S3 Files, account for its synchronization and crash-durability behavior. +- Configure retention and cleanup separately; the SDK does not delete completed payload files. +- Chained-invoke boundaries that use filesystem storage require compatible mount paths on both sides. + +## Retrying transient SerDes failures + +Filesystem read/write `IOException`s are reported as `RetryableSerDesException`. Wrap a SerDes with `RetrySerDes` to +apply any existing `RetryStrategy`: + +```java +var retryingSerDes = new RetrySerDes( + fileSystemSerDes, + RetryStrategies.exponentialBackoff( + 4, + Duration.ofSeconds(1), + Duration.ofSeconds(10), + 2.0, + JitterStrategy.FULL)); +``` + +Only `RetryableSerDesException` is retried. Permanent `SerDesException` failures—malformed envelopes, invalid digests, +or incompatible data—fail immediately. Retry delays block the calling thread or the configured SerDes executor thread. + +## Testing + +`LocalDurableTestRunner` preserves the configured SerDes executor and creates a fresh `SerDesRunner` for each simulated +Lambda invocation. `TestResult.getResult()` and `TestOperation.getStepResult()` deserialize using durable contexts and +the invocation cache. + +`CloudDurableTestRunner` and `AsyncExecution` reconstruct the execution and operation contexts from history events. +Each asynchronous history snapshot receives its own cache so updated payloads cannot reuse values from an earlier +snapshot. diff --git a/docs/design.md b/docs/design.md index c1b82ef39..474e0a1fc 100644 --- a/docs/design.md +++ b/docs/design.md @@ -351,6 +351,7 @@ software.amazon.lambda.durable │ ├── JacksonSerDes # Jackson impl │ ├── FileSystemSerDes # Shared-filesystem payload storage │ ├── PreviewConfig # Structured preview selection and byte budget +│ ├── RetrySerDes # RetryableSerDesException decorator │ ├── SerDesPreview # Structured preview builder │ ├── SerDesContext # Thread-local durable payload identity │ ├── SerDesRunner # Executor dispatch + invocation cache @@ -377,7 +378,8 @@ software.amazon.lambda.durable ├── ChildContextFailedException ├── MapIterationFailedException ├── ParallelBranchFailedException - └── SerDesException + ├── SerDesException + └── RetryableSerDesException ``` --- diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index cbc50a890..e9c777f00 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -3,20 +3,34 @@ package software.amazon.lambda.durable; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.retry.JitterStrategy; +import software.amazon.lambda.durable.retry.RetryStrategies; +import software.amazon.lambda.durable.retry.WaitStrategies; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; class FileSystemSerDesIntegrationTest { + private static final ObjectMapper MAPPER = new ObjectMapper(); + @TempDir Path tempDir; @@ -67,4 +81,161 @@ void operationConfigControlsWhereFilesystemStorageIsUsed() throws Exception { assertEquals(1, files.filter(Files::isRegularFile).count()); } } + + @Test + void replaysStepWaitForConditionChildAndMapPayloads() throws Exception { + var stepRuns = new AtomicInteger(); + var pollRuns = new AtomicInteger(); + var serDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var stepResult = context.step("load-order", String.class, stepContext -> { + stepRuns.incrementAndGet(); + return input + "-loaded"; + }); + var pollResult = context.waitForCondition( + "poll-order", + Integer.class, + (state, stepContext) -> { + pollRuns.incrementAndGet(); + var next = state == null ? 1 : state + 1; + return next == 2 + ? WaitForConditionResult.stopPolling(next) + : WaitForConditionResult.continuePolling(next); + }, + WaitForConditionConfig.builder() + .waitStrategy(WaitStrategies.exponentialBackoff( + 5, Duration.ofSeconds(1), Duration.ofSeconds(10), 1, JitterStrategy.NONE)) + .build()); + var childResult = + context.runInChildContext("format-order", String.class, child -> stepResult + "-child"); + var mapResult = + context.map("map-order", List.of(1, 2), Integer.class, (item, index, child) -> item * 2); + return childResult + "-" + pollResult + "-" + mapResult.results(); + }, + config); + + var result = runner.runUntilComplete("order"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("order-loaded-child-2-[2, 4]", result.getResult(String.class)); + assertEquals(1, stepRuns.get()); + assertEquals(2, pollRuns.get()); + assertEquals("order-loaded", result.getOperation("load-order").getStepResult(String.class)); + assertEnvelopePointsToFile( + result.getOperation("load-order").getStepDetails().result()); + assertEnvelopePointsToFile( + result.getOperation("map-order").getContextDetails().result()); + } + + @Test + void acceptsRawCallbackAndInvokeResultsWithBoundarySpecificSerDes() { + var fileSystemSerDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder().withSerDes(fileSystemSerDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var approval = + context.createCallback("approval", String.class).get(); + return context.invoke( + "notify", + "target-function", + approval, + String.class, + InvokeConfig.builder() + .payloadSerDes(new JacksonSerDes()) + .serDes(fileSystemSerDes) + .build()); + }, + config); + + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + runner.completeCallback(runner.getCallbackId("approval"), "\"approved\""); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + runner.completeChainedInvoke("notify", "\"notified\""); + + var completed = runner.runUntilComplete("input"); + + assertEquals(ExecutionStatus.SUCCEEDED, completed.getStatus()); + assertEquals("notified", completed.getResult(String.class)); + } + + @Test + void repeatedGetUsesInvocationDeserializationCache() { + var resultDeserializations = new AtomicInteger(); + var fileSystemSerDes = FileSystemSerDes.builder(tempDir).build(); + var countingSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return fileSystemSerDes.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + if (typeToken.equals(TypeToken.get(Payload.class)) && SerDesContext.getCurrentContext() != null) { + resultDeserializations.incrementAndGet(); + } + return fileSystemSerDes.deserialize(data, typeToken); + } + }; + var config = DurableConfig.builder().withSerDes(countingSerDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var future = context.stepAsync("cached-step", Payload.class, stepContext -> new Payload(input)); + var first = future.get(); + var second = future.get(); + assertSame(first, second); + return first.value(); + }, + config); + + var result = runner.runUntilComplete("cached"); + + assertEquals("cached", result.getResult(String.class)); + assertEquals(1, resultDeserializations.get()); + } + + @Test + void customExceptionPayloadRoundTripsThroughFilesystem() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step( + "fail-step", + String.class, + stepContext -> { + throw new CustomFailure("boom"); + }, + StepConfig.builder() + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .build()), + config); + + var result = runner.runUntilComplete("input"); + + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + var operationError = result.getOperation("fail-step").getError(); + assertEquals(CustomFailure.class.getName(), operationError.errorType()); + assertEnvelopePointsToFile(operationError.errorData()); + } + + private void assertEnvelopePointsToFile(String envelope) throws Exception { + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + assertTrue(Files.exists(file)); + assertTrue(file.startsWith(tempDir)); + } + + record Payload(String value) {} + + public static class CustomFailure extends RuntimeException { + public CustomFailure() {} + + public CustomFailure(String message) { + super(message); + } + } } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java index 57b6c6921..06987dcc5 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java @@ -16,6 +16,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.cloud.HistoryEventProcessor; /** @@ -195,7 +196,8 @@ private void refreshHistory() { .build(); var response = lambdaClient.getDurableExecutionHistory(request); this.currentHistory = response.events(); - this.currentResult = processor.processEvents(currentHistory, outputType, serDes); + this.currentResult = + processor.processEvents(currentHistory, outputType, serDes, new SerDesRunner(null), executionArn); } catch (ResourceNotFoundException e) { // Execution doesn't exist yet - this can happen immediately after async invoke // Leave currentHistory as null, pollUntil will retry diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index b06b0dfc4..5e2d50e42 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -12,6 +12,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.cloud.HistoryEventProcessor; import software.amazon.lambda.durable.testing.cloud.HistoryPoller; @@ -161,7 +162,7 @@ public TestResult run(I input) { // Process events into TestResult var processor = new HistoryEventProcessor(); - var result = processor.processEvents(events, outputType, serDes); + var result = processor.processEvents(events, outputType, serDes, new SerDesRunner(null), executionArn); this.lastResult = result; return result; } catch (Exception e) { diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 183234530..bc0803917 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -23,6 +23,7 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; import software.amazon.lambda.durable.testing.local.OperationResult; @@ -47,6 +48,9 @@ public class LocalDurableTestRunner { // operation ID stay stable across reinvocations, while only per-invocation values (the checkpoint token) change. private final String executionName = UUID.randomUUID().toString(); private final String executionOperationId = UUID.randomUUID().toString(); + private final String executionArn = String.format( + "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST/durable-execution/%s/%s", + executionName, executionOperationId); private LocalDurableTestRunner( TypeToken inputType, @@ -246,11 +250,12 @@ public static LocalDurableTestRunner create(TypeToken inputType, /** Run a single invocation (may return PENDING if waiting/retrying). */ public TestResult run(I input) { + var serDesRunner = new SerDesRunner(customerConfig.getSerDesExecutorService()); var durableInput = createDurableInput(input); var output = DurableExecutor.execute(durableInput, mockLambdaContext(), inputType, handler, customerConfig); - return storage.toTestResult(output, outputType, serDes); + return storage.toTestResult(output, outputType, serDes, serDesRunner, executionArn, executionOperationId); } /** @@ -289,7 +294,14 @@ public void simulateFireAndForgetCheckpointLoss(String stepName) { /** Returns the {@link TestOperation} for the given operation name, or null if not found. */ public TestOperation getOperation(String name) { var op = storage.getOperationByName(name); - return op != null ? new TestOperation(op, serDes) : null; + return op != null + ? new TestOperation( + op, + List.of(), + serDes, + new SerDesRunner(customerConfig.getSerDesExecutorService()), + executionArn) + : null; } /** Get callback ID for a named callback operation. */ @@ -340,9 +352,6 @@ public void stopChainedInvoke(String name, ErrorObject error) { private DurableExecutionInput createDurableInput(I input) { // The last ARN segment must equal the EXECUTION operation ID (ExecutionManager parses the ARN to find it), and // both are stable across reinvocations so the execution keeps one identity — and one derived trace ID. - var executionArn = String.format( - "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST/durable-execution/%s/%s", - executionName, executionOperationId); var inputJson = serDes.serialize(input); // The list must contain exactly one EXECUTION operation, matching the backend, which keeps a single EXECUTION diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java index 31a28b988..4d15cb215 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java @@ -19,21 +19,36 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.execution.ExecutionManager; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; /** Wrapper for AWS SDK Operation providing convenient access methods. */ public class TestOperation { private final Operation operation; private final List events; private final SerDes serDes; + private final SerDesRunner serDesRunner; + private final String durableExecutionArn; public TestOperation(Operation operation, SerDes serDes) { this(operation, List.of(), serDes); } public TestOperation(Operation operation, List events, SerDes serDes) { + this(operation, events, serDes, null, null); + } + + public TestOperation( + Operation operation, + List events, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn) { this.operation = operation; this.events = events; this.serDes = serDes; + this.serDesRunner = serDesRunner; + this.durableExecutionArn = durableExecutionArn; } /** Returns the raw history events associated with this operation. */ @@ -119,7 +134,10 @@ public T getStepResult(TypeToken type) { if (details == null || details.result() == null) { return null; } - return serDes.deserialize(details.result(), type); + return serDesRunner == null + ? serDes.deserialize(details.result(), type) + : serDesRunner.deserialize( + serDes, details.result(), type, new SerDesContext(durableExecutionArn, operation.id())); } /** Returns the step error, or null if the step succeeded or this is not a step operation. */ diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java index 7de85beef..195348334 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java @@ -15,6 +15,8 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; /** * Represents the result of a durable execution, providing access to the execution status, output, operations, and @@ -33,6 +35,8 @@ public class TestResult { private final List allEvents; private final SerDes serDes; private final TypeToken resultType; + private final SerDesRunner serDesRunner; + private final SerDesContext outputContext; public TestResult( ExecutionStatus status, @@ -42,6 +46,20 @@ public TestResult( List allEvents, TypeToken resultType, SerDes serDes) { + this(status, resultPayload, error, operations, allEvents, resultType, serDes, null, null, null); + } + + public TestResult( + ExecutionStatus status, + String resultPayload, + ErrorObject error, + List operations, + List allEvents, + TypeToken resultType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn, + String executionOperationId) { this.status = status; this.resultPayload = resultPayload; this.error = error; @@ -51,6 +69,8 @@ public TestResult( this.allEvents = List.copyOf(allEvents); this.serDes = serDes; this.resultType = resultType; + this.serDesRunner = serDesRunner; + this.outputContext = serDesRunner == null ? null : new SerDesContext(durableExecutionArn, executionOperationId); } /** Returns the execution status (SUCCEEDED, FAILED, or PENDING). */ @@ -75,12 +95,18 @@ public T getResult(TypeToken resultType) { if (resultPayload == null || resultPayload.isEmpty()) { var lastEvent = allEvents.get(allEvents.size() - 1); if (lastEvent.eventType() == EventType.EXECUTION_SUCCEEDED) { - return serDes.deserialize( + return deserialize( lastEvent.executionSucceededDetails().result().payload(), resultType); } return null; } - return serDes.deserialize(resultPayload, resultType); + return deserialize(resultPayload, resultType); + } + + private T deserialize(String payload, TypeToken type) { + return serDesRunner == null + ? serDes.deserialize(payload, type) + : serDesRunner.deserialize(serDes, payload, type, outputContext); } /** Deserializes and returns the execution output if the result type is known. */ diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java index 4a3b8f1b2..13663efe7 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Objects; import software.amazon.awssdk.services.lambda.model.CallbackDetails; import software.amazon.awssdk.services.lambda.model.ChainedInvokeDetails; import software.amazon.awssdk.services.lambda.model.ContextDetails; @@ -16,8 +17,10 @@ import software.amazon.awssdk.services.lambda.model.StepDetails; import software.amazon.awssdk.services.lambda.model.WaitDetails; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.execution.ExecutionManager; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.AsyncExecution; import software.amazon.lambda.durable.testing.CloudDurableTestRunner; import software.amazon.lambda.durable.testing.TestOperation; @@ -37,11 +40,27 @@ public class HistoryEventProcessor { * @return a TestResult containing the execution status, output, and operation details */ public TestResult processEvents(List events, TypeToken outputType, SerDes serDes) { + return processEvents(events, outputType, serDes, null, null); + } + + /** + * Processes history with SDK-managed SerDes context for result and operation inspection. + * + * @param serDesRunner runner used for deserialization, or {@code null} for direct SerDes calls + * @param durableExecutionArn ARN for the execution, required when {@code serDesRunner} is provided + */ + public TestResult processEvents( + List events, + TypeToken outputType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn) { var operations = new HashMap(); var operationEvents = new HashMap>(); var status = ExecutionStatus.PENDING; String result = null; ErrorObject error = null; + String executionOperationId = executionOperationId(durableExecutionArn); for (var event : events) { var eventType = event.eventType(); @@ -56,7 +75,9 @@ public TestResult processEvents(List events, TypeToken outputTy switch (eventType) { case EXECUTION_STARTED -> { - // Execution started - no action needed, just track the event + if (operationId != null) { + executionOperationId = operationId; + } } case INVOCATION_COMPLETED -> { var details = event.invocationCompletedDetails(); @@ -111,7 +132,14 @@ public TestResult processEvents(List events, TypeToken outputTy if (operationId != null) { operations.putIfAbsent( operationId, - createStepOperation(operationId, event.name(), null, OperationStatus.STARTED, 1)); + createStepOperation( + operationId, + event.name(), + event.parentId(), + event.subType(), + null, + OperationStatus.STARTED, + 1)); } } case STEP_SUCCEEDED -> { @@ -126,7 +154,13 @@ public TestResult processEvents(List events, TypeToken outputTy operations.put( operationId, createStepOperation( - operationId, event.name(), stepResult, OperationStatus.SUCCEEDED, attempt)); + operationId, + event.name(), + event.parentId(), + event.subType(), + stepResult, + OperationStatus.SUCCEEDED, + attempt)); } } case STEP_FAILED -> { @@ -137,7 +171,14 @@ public TestResult processEvents(List events, TypeToken outputTy : 1; operations.put( operationId, - createStepOperation(operationId, event.name(), null, OperationStatus.FAILED, attempt)); + createStepOperation( + operationId, + event.name(), + event.parentId(), + event.subType(), + null, + OperationStatus.FAILED, + attempt)); } } @@ -224,7 +265,12 @@ public TestResult processEvents(List events, TypeToken outputTy CHAINED_INVOKE_TIMED_OUT, CHAINED_INVOKE_STOPPED -> { if (operationId != null) { - operations.putIfAbsent(operationId, createInvokeOperation(operationId, event)); + if (eventType + == software.amazon.awssdk.services.lambda.model.EventType.CHAINED_INVOKE_STARTED) { + operations.putIfAbsent(operationId, createInvokeOperation(operationId, event)); + } else { + operations.put(operationId, createInvokeOperation(operationId, event)); + } } } @@ -236,14 +282,58 @@ public TestResult processEvents(List events, TypeToken outputTy var testOperations = new ArrayList(); for (var entry : operations.entrySet()) { var opEvents = operationEvents.getOrDefault(entry.getKey(), List.of()); - testOperations.add(new TestOperation(entry.getValue(), opEvents, serDes)); + var operation = withEventTimestamps(entry.getValue(), opEvents); + testOperations.add(new TestOperation(operation, opEvents, serDes, serDesRunner, durableExecutionArn)); } - return new TestResult<>(status, result, error, testOperations, events, outputType, serDes); + return new TestResult<>( + status, + result, + error, + testOperations, + events, + outputType, + serDes, + serDesRunner, + durableExecutionArn, + executionOperationId); + } + + private Operation withEventTimestamps(Operation operation, List events) { + var startTimestamp = events.stream() + .map(Event::eventTimestamp) + .filter(Objects::nonNull) + .min(java.time.Instant::compareTo) + .orElse(operation.startTimestamp()); + var endTimestamp = ExecutionManager.isTerminalStatus(operation.status()) + ? events.stream() + .map(Event::eventTimestamp) + .filter(Objects::nonNull) + .max(java.time.Instant::compareTo) + .orElse(operation.endTimestamp()) + : operation.endTimestamp(); + return operation.toBuilder() + .startTimestamp(startTimestamp) + .endTimestamp(endTimestamp) + .build(); + } + + private static String executionOperationId(String durableExecutionArn) { + if (durableExecutionArn == null) { + return null; + } + var separator = durableExecutionArn.lastIndexOf('/'); + return separator >= 0 ? durableExecutionArn.substring(separator + 1) : durableExecutionArn; } private Operation createStepOperation( - String id, String name, String stepResult, OperationStatus status, Integer attempt) { + String id, + String name, + String parentId, + String subType, + String stepResult, + OperationStatus status, + Integer attempt) { var stepDetails = StepDetails.builder() .result(stepResult) .attempt(attempt != null ? attempt : 1) @@ -252,8 +342,10 @@ private Operation createStepOperation( return Operation.builder() .id(id) .name(name) + .parentId(parentId) .status(status) .type(OperationType.STEP) + .subType(subType) .stepDetails(stepDetails) .build(); } @@ -267,8 +359,10 @@ private Operation createWaitOperation(String id, String name, OperationStatus st return Operation.builder() .id(id) .name(name) + .parentId(event.parentId()) .status(status) .type(OperationType.WAIT) + .subType(event.subType()) .waitDetails(builder.build()) .build(); } @@ -302,8 +396,10 @@ private Operation createCallbackOperation(String id, String name, OperationStatu return Operation.builder() .id(id) .name(name) + .parentId(event.parentId()) .status(status) .type(OperationType.CALLBACK) + .subType(event.subType()) .callbackDetails(builder.build()) .build(); } @@ -315,7 +411,7 @@ private Operation createInvokeOperation(String id, Event event) { switch (event.eventType()) { case CHAINED_INVOKE_STARTED -> OperationStatus.STARTED; case CHAINED_INVOKE_SUCCEEDED -> { - var details = event.callbackSucceededDetails(); + var details = event.chainedInvokeSucceededDetails(); if (details != null && details.result() != null && details.result().payload() != null) { @@ -324,7 +420,7 @@ private Operation createInvokeOperation(String id, Event event) { yield OperationStatus.SUCCEEDED; } case CHAINED_INVOKE_FAILED -> { - var details = event.callbackFailedDetails(); + var details = event.chainedInvokeFailedDetails(); if (details != null && details.error() != null && details.error().payload() != null) { @@ -359,8 +455,10 @@ private Operation createInvokeOperation(String id, Event event) { return Operation.builder() .id(id) .name(event.name()) + .parentId(event.parentId()) .status(status) .type(OperationType.CHAINED_INVOKE) + .subType(event.subType()) .chainedInvokeDetails(builder.build()) .build(); } @@ -383,6 +481,7 @@ private Operation createContextOperation(String id, String name, OperationStatus return Operation.builder() .id(id) .name(name) + .parentId(event.parentId()) .status(status) .type(OperationType.CONTEXT) .subType(event.subType()) diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java index 25f016cd9..0cdc85f43 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java @@ -24,6 +24,7 @@ import software.amazon.lambda.durable.client.DurableExecutionClient; import software.amazon.lambda.durable.model.DurableExecutionOutput; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.TestOperation; import software.amazon.lambda.durable.testing.TestResult; @@ -131,9 +132,21 @@ public List getUpdatedOperationIdsSinceLastInvocation() { /** Build TestResult from current state. */ public TestResult toTestResult(DurableExecutionOutput output, TypeToken resultType, SerDes serDes) { + return toTestResult(output, resultType, serDes, null, null, null); + } + + /** Build TestResult from current state with SDK-managed SerDes context. */ + public TestResult toTestResult( + DurableExecutionOutput output, + TypeToken resultType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn, + String executionOperationId) { var testOperations = existingOperations.values().stream() .filter(op -> op.type() != OperationType.EXECUTION) - .map(op -> new TestOperation(op, eventProcessor.getEventsForOperation(op.id()), serDes)) + .map(op -> new TestOperation( + op, eventProcessor.getEventsForOperation(op.id()), serDes, serDesRunner, durableExecutionArn)) .toList(); return new TestResult<>( output.status(), @@ -142,7 +155,10 @@ public TestResult toTestResult(DurableExecutionOutput output, TypeToken T deserialize(String data, TypeToken typeToken) { + deserializations.incrementAndGet(); + return (T) data; + } + }; + var execution = new AsyncExecution<>( + EXECUTION_ARN, lambdaClient, TypeToken.get(String.class), serDes, Duration.ZERO, Duration.ofSeconds(1)); + var snapshots = new AtomicInteger(); + + execution.pollUntil(current -> { + assertEquals("step-result", current.getOperation("step").getStepResult(String.class)); + assertEquals("step-result", current.getOperation("step").getStepResult(String.class)); + return snapshots.incrementAndGet() == 2; + }); + + assertEquals(2, deserializations.get()); + } + + private static List stepEvents() { + var startedAt = Instant.parse("2026-08-25T00:00:00Z"); + return List.of( + Event.builder() + .id("step-id") + .name("step") + .subType("Step") + .eventType(EventType.STEP_STARTED) + .eventTimestamp(startedAt) + .stepStartedDetails(StepStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("step") + .subType("Step") + .eventType(EventType.STEP_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(1)) + .stepSucceededDetails(StepSucceededDetails.builder() + .result(EventResult.builder() + .payload("step-result") + .build()) + .retryDetails( + RetryDetails.builder().currentAttempt(1).build()) + .build()) + .build()); + } +} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 36f1bbced..8f8c4f90b 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -10,12 +10,15 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDesContext; class LocalDurableTestRunnerTest { @@ -114,4 +117,29 @@ public void onInvocationStart(InvocationInfo info) { assertNotNull(executionStartTimes.get(0)); assertEquals(executionStartTimes.get(0), executionStartTimes.get(1)); } + + @Test + void resultAndOperationInspectionUseDurableSerDesContext() { + var contexts = new CopyOnWriteArrayList(); + var serDes = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + if (SerDesContext.getCurrentContext() != null) { + contexts.add(SerDesContext.getCurrentContext()); + } + return super.deserialize(data, typeToken); + } + }; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step("step", String.class, stepContext -> input), + DurableConfig.builder().withSerDes(serDes).build()); + + var result = runner.run("value"); + result.getResult(String.class); + result.getOperation("step").getStepResult(String.class); + + assertTrue(contexts.stream().allMatch(context -> context.durableExecutionArn() != null)); + assertTrue(contexts.stream().map(SerDesContext::entityId).distinct().count() >= 2); + } } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java new file mode 100644 index 000000000..b18417f76 --- /dev/null +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java @@ -0,0 +1,54 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.StepDetails; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; + +class TestOperationTest { + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + + "/durable-execution/execution-id/invocation-id"; + + @Test + void deserializesStepResultWithDurableContext() { + var observedContext = new AtomicReference(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + observedContext.set(SerDesContext.getCurrentContext()); + return (T) data; + } + }; + var operation = Operation.builder() + .id("step-id") + .name("step") + .type(OperationType.STEP) + .subType("Step") + .status(OperationStatus.SUCCEEDED) + .stepDetails( + StepDetails.builder().attempt(2).result("step-result").build()) + .build(); + var testOperation = new TestOperation(operation, List.of(), serDes, new SerDesRunner(null), EXECUTION_ARN); + + assertEquals("step-result", testOperation.getStepResult(String.class)); + assertEquals(EXECUTION_ARN, observedContext.get().durableExecutionArn()); + assertEquals("step-id", observedContext.get().entityId()); + } +} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java new file mode 100644 index 000000000..ba9082e3b --- /dev/null +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -0,0 +1,130 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing.cloud; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.ChainedInvokeStartedDetails; +import software.amazon.awssdk.services.lambda.model.ChainedInvokeSucceededDetails; +import software.amazon.awssdk.services.lambda.model.Event; +import software.amazon.awssdk.services.lambda.model.EventResult; +import software.amazon.awssdk.services.lambda.model.EventType; +import software.amazon.awssdk.services.lambda.model.ExecutionStartedDetails; +import software.amazon.awssdk.services.lambda.model.ExecutionSucceededDetails; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.RetryDetails; +import software.amazon.awssdk.services.lambda.model.StepStartedDetails; +import software.amazon.awssdk.services.lambda.model.StepSucceededDetails; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; + +class HistoryEventProcessorTest { + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + + "/durable-execution/execution-id/invocation-id"; + + @Test + void deserializesCloudResultsWithDurablePayloadContext() { + var observedContexts = new ArrayList(); + var serDes = recordingStringSerDes(observedContexts); + var startedAt = Instant.parse("2026-08-24T00:00:00Z"); + var events = List.of( + Event.builder() + .id("invocation-id") + .name("execution") + .eventType(EventType.EXECUTION_STARTED) + .eventTimestamp(startedAt) + .executionStartedDetails( + ExecutionStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("step") + .subType("Step") + .eventType(EventType.STEP_STARTED) + .eventTimestamp(startedAt.plusSeconds(1)) + .stepStartedDetails(StepStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("step") + .subType("Step") + .eventType(EventType.STEP_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(3)) + .stepSucceededDetails(StepSucceededDetails.builder() + .result(EventResult.builder() + .payload("step-result") + .build()) + .retryDetails( + RetryDetails.builder().currentAttempt(2).build()) + .build()) + .build(), + Event.builder() + .id("invoke-id") + .name("invoke") + .eventType(EventType.CHAINED_INVOKE_STARTED) + .eventTimestamp(startedAt.plusSeconds(4)) + .chainedInvokeStartedDetails(ChainedInvokeStartedDetails.builder() + .functionName("target") + .build()) + .build(), + Event.builder() + .id("invoke-id") + .name("invoke") + .eventType(EventType.CHAINED_INVOKE_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(5)) + .chainedInvokeSucceededDetails(ChainedInvokeSucceededDetails.builder() + .result(EventResult.builder() + .payload("invoke-result") + .build()) + .build()) + .build(), + Event.builder() + .id("invocation-id") + .name("execution") + .eventType(EventType.EXECUTION_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(6)) + .executionSucceededDetails(ExecutionSucceededDetails.builder() + .result(EventResult.builder() + .payload("execution-result") + .build()) + .build()) + .build()); + + var result = new HistoryEventProcessor() + .processEvents(events, TypeToken.get(String.class), serDes, new SerDesRunner(null), EXECUTION_ARN); + + assertEquals("execution-result", result.getResult()); + assertEquals("step-result", result.getOperation("step").getStepResult(String.class)); + assertEquals(Duration.ofSeconds(2), result.getOperation("step").getDuration()); + assertEquals(OperationStatus.SUCCEEDED, result.getOperation("invoke").getStatus()); + assertEquals( + "invoke-result", + result.getOperation("invoke").getChainedInvokeDetails().result()); + assertEquals( + List.of("invocation-id", "step-id"), + observedContexts.stream().map(SerDesContext::entityId).toList()); + } + + private static SerDes recordingStringSerDes(List observedContexts) { + return new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + observedContexts.add(SerDesContext.getCurrentContext()); + return (T) data; + } + }; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java new file mode 100644 index 000000000..be06a762f --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java @@ -0,0 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.exception; + +/** + * Indicates a transient serialization or deserialization failure that may succeed when retried. + * + *

{@link software.amazon.lambda.durable.serde.RetrySerDes} retries only this exception type. Other + * {@link SerDesException} instances are treated as permanent failures. + */ +public class RetryableSerDesException extends SerDesException { + public RetryableSerDesException(String message, Throwable cause) { + super(message, cause); + } + + public RetryableSerDesException(String message) { + super(message); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index fe97e6195..feac73b8a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -8,8 +8,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -18,9 +18,12 @@ import java.util.HexFormat; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.UUID; import java.util.function.Function; import java.util.regex.Pattern; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; /** @@ -187,7 +190,7 @@ private static boolean isValidEnvelope(JsonNode node) { private Path payloadPath(SerDesContext context, String digest) { var directory = executionDirectory(context.durableExecutionArn()); - var fileName = encode(context.entityId()) + "-" + digest + ".json"; + var fileName = encode(context.entityId()) + "-" + digest + "-" + UUID.randomUUID() + ".json"; var file = directory.resolve(fileName).toAbsolutePath().normalize(); if (!file.startsWith(basePath)) { throw new SerDesException("Filesystem SerDes path escapes the configured base path"); @@ -215,21 +218,26 @@ private void writePayload(Path file, String serialized) { if (!realParent.startsWith(realBase)) { throw new SerDesException("Filesystem SerDes path resolves outside the configured base path"); } - try { - Files.writeString( - file, - serialized, - StandardCharsets.UTF_8, - StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE); - } catch (FileAlreadyExistsException e) { - var existing = readPayload(file.toString()); - if (!existing.equals(serialized)) { - throw new SerDesException("Filesystem SerDes payload file already exists with different content"); + var created = false; + try (var channel = + Files.newByteChannel(file, Set.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))) { + created = true; + var buffer = ByteBuffer.wrap(serialized.getBytes(StandardCharsets.UTF_8)); + while (buffer.hasRemaining()) { + channel.write(buffer); } + } catch (IOException failure) { + if (created) { + try { + Files.deleteIfExists(file); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } + throw failure; } } catch (IOException e) { - throw new SerDesException("Failed to store filesystem SerDes payload", e); + throw new RetryableSerDesException("Failed to store filesystem SerDes payload", e); } } @@ -246,7 +254,7 @@ private String readPayload(String fileValue) { } return Files.readString(realFile, StandardCharsets.UTF_8); } catch (IOException e) { - throw new SerDesException("Failed to load filesystem SerDes payload", e); + throw new RetryableSerDesException("Failed to load filesystem SerDes payload", e); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java new file mode 100644 index 000000000..6b7cb1119 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.retry.RetryStrategy; + +/** + * A SerDes decorator that retries transient failures from another {@link SerDes}. + * + *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block + * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. + */ +public final class RetrySerDes implements SerDes { + private final SerDes delegate; + private final SerDesRetryExecutor retryExecutor; + + /** + * Creates a retrying SerDes decorator. + * + * @param delegate the SerDes to invoke + * @param retryStrategy strategy that controls attempts and delays + */ + public RetrySerDes(SerDes delegate, RetryStrategy retryStrategy) { + this(delegate, retryStrategy, SerDesRetryExecutor.DEFAULT_SLEEPER); + } + + RetrySerDes(SerDes delegate, RetryStrategy retryStrategy, SerDesRetryExecutor.Sleeper sleeper) { + this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + retryExecutor = new SerDesRetryExecutor(retryStrategy, sleeper); + } + + @Override + public String serialize(Object value) { + return retryExecutor.execute("serialization", () -> delegate.serialize(value)); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return retryExecutor.execute("deserialization", () -> delegate.deserialize(data, typeToken)); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java new file mode 100644 index 000000000..aaf014ff3 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java @@ -0,0 +1,87 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryDecision; +import software.amazon.lambda.durable.retry.RetryStrategy; + +final class SerDesRetryExecutor { + static final Sleeper DEFAULT_SLEEPER = delay -> { + if (delay.getSeconds() > 0) { + TimeUnit.SECONDS.sleep(delay.getSeconds()); + } + if (delay.getNano() > 0) { + TimeUnit.NANOSECONDS.sleep(delay.getNano()); + } + }; + + private final RetryStrategy retryStrategy; + private final Sleeper sleeper; + + SerDesRetryExecutor(RetryStrategy retryStrategy, Sleeper sleeper) { + this.retryStrategy = Objects.requireNonNull(retryStrategy, "retryStrategy cannot be null"); + this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); + } + + T execute(String action, Supplier operation) { + int attempt = 1; + while (true) { + try { + return operation.get(); + } catch (RetryableSerDesException failure) { + var decision = makeRetryDecision(action, failure, attempt); + if (!decision.shouldRetry()) { + throw failure; + } + waitForRetry(action, failure, attempt, decision.delay()); + attempt++; + } + } + } + + private RetryDecision makeRetryDecision(String action, RetryableSerDesException failure, int attempt) { + try { + var decision = retryStrategy.makeRetryDecision(failure, attempt); + if (decision == null) { + throw new SerDesException( + String.format("Retry strategy returned null for SerDes %s attempt %d", action, attempt)); + } + return decision; + } catch (SerDesException e) { + throw e; + } catch (RuntimeException e) { + throw new SerDesException( + String.format("Retry strategy failed for SerDes %s attempt %d", action, attempt), e); + } + } + + private void waitForRetry(String action, RetryableSerDesException failure, int attempt, Duration delay) { + if (delay == null || delay.isNegative()) { + throw new SerDesException(String.format( + "Retry strategy returned an invalid delay for SerDes %s attempt %d", action, attempt)); + } + if (delay.isZero()) { + return; + } + try { + sleeper.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + var interrupted = new SerDesException( + String.format("Interrupted while waiting to retry SerDes %s after attempt %d", action, attempt), e); + interrupted.addSuppressed(failure); + throw interrupted; + } + } + + @FunctionalInterface + interface Sleeper { + void sleep(Duration delay) throws InterruptedException; + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index e1733162c..1768753d7 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; class FileSystemSerDesTest { @@ -147,10 +148,27 @@ void supportsHashPathEncodingAndPreview() throws Exception { var file = Path.of(node.get("file").textValue()); assertEquals(64, tempDir.relativize(file).getName(0).toString().length()); - assertTrue(file.getFileName().toString().matches("[0-9a-f]{64}-[0-9a-f]{64}\\.json")); + assertTrue(file.getFileName().toString().matches("[0-9a-f]{64}-[0-9a-f]{64}-[0-9a-f-]{36}\\.json")); assertEquals("preview", node.get("preview").get("summary").textValue()); } + @Test + void repeatedPayloadsUseDistinctImmutableFiles() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var context = new SerDesContext(realisticArn(), "1"); + + var first = Path.of(MAPPER.readTree(runner.serialize(serDes, "value", context)) + .get("file") + .textValue()); + var second = Path.of(MAPPER.readTree(runner.serialize(serDes, "value", context)) + .get("file") + .textValue()); + + assertFalse(first.equals(second)); + assertEquals("\"value\"", Files.readString(first)); + assertEquals("\"value\"", Files.readString(second)); + } + @Test void rejectsPreviewThatMakesFileEnvelopeTooLarge() throws Exception { var serDes = FileSystemSerDes.builder(tempDir) @@ -224,6 +242,16 @@ void rejectsMalformedRecognizedEnvelope() { TypeToken.get(String.class))); } + @Test + void missingPayloadFileIsRetryable() throws Exception { + var missing = tempDir.resolve("missing.json"); + var envelope = MAPPER.writeValueAsString(Map.of( + "__durable_execution_filesystem_serdes", 1, "file", missing.toString(), "sha256", "0".repeat(64))); + var serDes = FileSystemSerDes.builder(tempDir).build(); + + assertThrows(RetryableSerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); + } + @Test void rejectsTrailingAndDuplicateFieldsInMarkedEnvelope() { var serDes = FileSystemSerDes.builder(tempDir).build(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java new file mode 100644 index 000000000..003884f75 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -0,0 +1,102 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryDecision; + +class RetrySerDesTest { + + @Test + void retriesRetryableSerializationFailure() { + var attempts = new AtomicInteger(); + var delegate = new JacksonSerDes() { + @Override + public String serialize(Object value) { + if (attempts.incrementAndGet() == 1) { + throw new RetryableSerDesException("temporary"); + } + return super.serialize(value); + } + }; + var serDes = new RetrySerDes( + delegate, + (failure, attempt) -> attempt == 1 ? RetryDecision.retry(Duration.ZERO) : RetryDecision.fail(), + delay -> {}); + + assertEquals("\"value\"", serDes.serialize("value")); + assertEquals(2, attempts.get()); + } + + @Test + void doesNotRetryPermanentFailure() { + var attempts = new AtomicInteger(); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + attempts.incrementAndGet(); + throw new SerDesException("permanent"); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var serDes = new RetrySerDes(delegate, (failure, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + + assertThrows(SerDesException.class, () -> serDes.serialize("value")); + assertEquals(1, attempts.get()); + } + + @Test + void propagatesRetryableFailureWhenStrategyStops() { + var attempts = new AtomicInteger(); + var delegate = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + attempts.incrementAndGet(); + throw new RetryableSerDesException("still unavailable"); + } + }; + var serDes = new RetrySerDes( + delegate, + (failure, attempt) -> attempt < 3 ? RetryDecision.retry(Duration.ZERO) : RetryDecision.fail(), + delay -> {}); + + assertThrows( + RetryableSerDesException.class, () -> serDes.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals(3, attempts.get()); + } + + @Test + void retriesKeepTheSameThreadLocalContext() { + var attempts = new AtomicInteger(); + var observed = new AtomicReference(); + var delegate = new JacksonSerDes() { + @Override + public String serialize(Object value) { + observed.set(SerDesContext.getCurrentContext()); + if (attempts.incrementAndGet() == 1) { + throw new RetryableSerDesException("temporary"); + } + return super.serialize(value); + } + }; + var serDes = new RetrySerDes(delegate, (failure, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + var context = new SerDesContext("arn:test", "entity"); + + assertEquals("\"value\"", new SerDesRunner(null).serialize(serDes, "value", context)); + assertSame(context, observed.get()); + } +} From 3d72acd7d2a28b441036e7109c6bd408f741325b Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 00:21:54 +0000 Subject: [PATCH 04/11] fix: harden filesystem SerDes traversal --- docs/adr/005-filesystem-serdes.md | 5 + docs/advanced/configuration.md | 1 + docs/advanced/serdes.md | 5 + .../durable/serde/FileSystemSerDes.java | 172 +++++++++++++++--- .../durable/serde/FileSystemSerDesTest.java | 86 +++++++++ 5 files changed, 242 insertions(+), 27 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 66884a4ea..4f793f603 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -154,6 +154,10 @@ and failed writes are removed before the retryable failure is propagated. This p overwriting data referenced by an earlier checkpoint. Deserialization rejects paths outside the configured base directory and verifies the digest. +The filesystem provider must support `SecureDirectoryStream`. Directory components are opened relative to held parent +handles with symbolic-link following disabled, and file reads/writes use `NOFOLLOW_LINKS`. Providers without secure +directory streams fail closed. + ### Initial invocation input The durable execution ARN does not exist when a caller serializes the initial Lambda input. Therefore, @@ -192,6 +196,7 @@ Negative: - Do not use Lambda's ephemeral `/tmp` directory. Replay may run in another execution environment. - Use a shared durable mount such as EFS. - S3 Files users must accept its synchronization and crash-durability characteristics. +- Verify that the mounted Java filesystem provider supports `SecureDirectoryStream`. - Configure lifecycle cleanup separately; the SDK does not delete persisted payload files. ## Alternatives Rejected diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 08a4f98bf..19ee05880 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -107,6 +107,7 @@ configured SerDes executor when present, and successful deserializations are cac Do not use Lambda's `/tmp` directory: replay can run in another execution environment. Use a shared durable mount such as EFS. S3 Files users must account for synchronization and crash-durability behavior. A chained-invoke boundary only requires shared storage and compatible filesystem configuration when that boundary explicitly uses `FileSystemSerDes`. +The mounted Java filesystem provider must support `SecureDirectoryStream`; unsupported providers fail closed. See [Serialization and Filesystem Storage](serdes.md) for envelope details, structured previews, retry configuration, testing behavior, and operational guidance. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index 1d364e0a4..e07cdf772 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -109,6 +109,10 @@ The default checkpoint-envelope limit is 255 KiB and can be changed with Files include the serialized-payload digest and a unique suffix. They are published with `CREATE_NEW`; failed writes are cleaned up. Each envelope includes a SHA-256 digest that is verified when the file is loaded. +The mounted filesystem provider must support `SecureDirectoryStream`. The SDK traverses every directory relative to an +already-open parent with symlink following disabled and performs file I/O with `NOFOLLOW_LINKS`. Providers without this +capability fail closed. + Only objects containing the reserved version marker are treated as filesystem envelopes. Ordinary JSON with `data` or `file` fields is passed to the configured delegate. @@ -141,6 +145,7 @@ delegate JSON; SDK-managed output and operation payloads can use filesystem stor - Do not use Lambda `/tmp`; replay may run in another execution environment. - Use a shared durable mount such as EFS. - If using S3 Files, account for its synchronization and crash-durability behavior. +- Verify that the Java filesystem provider for the mount supports `SecureDirectoryStream`. - Configure retention and cleanup separately; the SDK does not delete completed payload files. - Chained-invoke boundaries that use filesystem storage require compatible mount paths on both sides. diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index feac73b8a..5e74f7c3c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -9,13 +9,22 @@ import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.SecureDirectoryStream; import java.nio.file.StandardOpenOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.HexFormat; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -33,6 +42,9 @@ * that can serialize or deserialize the payload, such as an EFS mount or an S3 Files mount whose synchronization * tradeoffs are acceptable for the workload. * + *

The filesystem provider must support {@link SecureDirectoryStream}. Directory components are traversed relative to + * held parent handles with symbolic-link following disabled, and file I/O uses {@link LinkOption#NOFOLLOW_LINKS}. + * *

Initial invocation input is serialized normally when no {@link SerDesContext} is available, because the durable * execution ARN does not exist until after invocation starts. SDK-managed operation and output payloads are processed * using the configured storage mode. @@ -212,29 +224,33 @@ private Path executionDirectory(String durableExecutionArn) { private void writePayload(Path file, String serialized) { try { - Files.createDirectories(file.getParent()); - var realBase = basePath.toRealPath(); - var realParent = file.getParent().toRealPath(); - if (!realParent.startsWith(realBase)) { - throw new SerDesException("Filesystem SerDes path resolves outside the configured base path"); - } - var created = false; - try (var channel = - Files.newByteChannel(file, Set.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))) { - created = true; - var buffer = ByteBuffer.wrap(serialized.getBytes(StandardCharsets.UTF_8)); - while (buffer.hasRemaining()) { - channel.write(buffer); - } - } catch (IOException failure) { - if (created) { - try { - Files.deleteIfExists(file); - } catch (IOException cleanupFailure) { - failure.addSuppressed(cleanupFailure); + try (var secureDirectory = openSecureDirectory(file.getParent(), true)) { + var created = false; + try (var channel = secureDirectory + .directory() + .newByteChannel( + file.getFileName(), + Set.of( + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS))) { + created = true; + var buffer = ByteBuffer.wrap(serialized.getBytes(StandardCharsets.UTF_8)); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } catch (FileAlreadyExistsException failure) { + throw failure; + } catch (IOException failure) { + if (created) { + try { + secureDirectory.directory().deleteFile(file.getFileName()); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } } + throw failure; } - throw failure; } } catch (IOException e) { throw new RetryableSerDesException("Failed to store filesystem SerDes payload", e); @@ -242,22 +258,90 @@ private void writePayload(Path file, String serialized) { } private String readPayload(String fileValue) { - var file = Path.of(fileValue).toAbsolutePath().normalize(); + var file = basePath.getFileSystem().getPath(fileValue).toAbsolutePath().normalize(); if (!file.startsWith(basePath)) { throw new SerDesException("Filesystem SerDes file is outside the configured base path"); } try { - var realBase = basePath.toRealPath(); - var realFile = file.toRealPath(); - if (!realFile.startsWith(realBase)) { - throw new SerDesException("Filesystem SerDes file resolves outside the configured base path"); + byte[] storedData; + try (var secureDirectory = openSecureDirectory(file.getParent(), false); + var channel = secureDirectory + .directory() + .newByteChannel( + file.getFileName(), Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)); + var input = Channels.newInputStream(channel)) { + storedData = input.readAllBytes(); } - return Files.readString(realFile, StandardCharsets.UTF_8); + return StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(storedData)) + .toString(); } catch (IOException e) { throw new RetryableSerDesException("Failed to load filesystem SerDes payload", e); } } + private SecureDirectoryHandle openSecureDirectory(Path directory, boolean createMissing) throws IOException { + if (directory == null || !directory.startsWith(basePath)) { + throw new SerDesException("Filesystem SerDes directory is outside the configured base path"); + } + var root = basePath.getRoot(); + if (root == null) { + throw new SerDesException("Filesystem SerDes base path must be absolute"); + } + + var openedStreams = new ArrayList>(); + try { + var current = requireSecureDirectoryStream(Files.newDirectoryStream(root), openedStreams); + var currentPath = root; + for (var component : root.relativize(directory)) { + var nextPath = currentPath.resolve(component); + DirectoryStream next; + try { + next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); + } catch (NoSuchFileException missing) { + if (!createMissing) { + throw missing; + } + try { + Files.createDirectory(nextPath); + } catch (FileAlreadyExistsException ignored) { + // Validate and open the entry relative to the held parent directory below. + } + next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); + } + current = requireSecureDirectoryStream(next, openedStreams); + currentPath = nextPath; + } + return new SecureDirectoryHandle(current, openedStreams); + } catch (IOException | RuntimeException failure) { + closeDirectoryStreams(openedStreams, failure); + throw failure; + } + } + + @SuppressWarnings("unchecked") + private static SecureDirectoryStream requireSecureDirectoryStream( + DirectoryStream stream, List> openedStreams) { + openedStreams.add(stream); + if (stream instanceof SecureDirectoryStream secureStream) { + return (SecureDirectoryStream) secureStream; + } + throw new SerDesException("FileSystemSerDes requires a filesystem provider with SecureDirectoryStream support"); + } + + private static void closeDirectoryStreams(List> streams, Throwable failure) { + for (int index = streams.size() - 1; index >= 0; index--) { + try { + streams.get(index).close(); + } catch (IOException closeFailure) { + failure.addSuppressed(closeFailure); + } + } + } + private String encode(String value) { return pathEncoding == FileSystemPathEncoding.HASH ? sha256(value) : percentEncode(value); } @@ -305,6 +389,40 @@ private static String writeEnvelope(JsonNode envelope) { } } + private static final class SecureDirectoryHandle implements AutoCloseable { + private final SecureDirectoryStream directory; + private final List> openedStreams; + + private SecureDirectoryHandle( + SecureDirectoryStream directory, List> openedStreams) { + this.directory = directory; + this.openedStreams = List.copyOf(openedStreams); + } + + private SecureDirectoryStream directory() { + return directory; + } + + @Override + public void close() throws IOException { + IOException failure = null; + for (int index = openedStreams.size() - 1; index >= 0; index--) { + try { + openedStreams.get(index).close(); + } catch (IOException closeFailure) { + if (failure == null) { + failure = closeFailure; + } else { + failure.addSuppressed(closeFailure); + } + } + } + if (failure != null) { + throw failure; + } + } + } + /** Builder for {@link FileSystemSerDes}. */ public static final class Builder { private final Path basePath; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 1768753d7..bd2fc0ebf 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -8,7 +8,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -252,6 +254,90 @@ void missingPayloadFileIsRetryable() throws Exception { assertThrows(RetryableSerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); } + @Test + void failsClosedWhenProviderLacksSecureDirectoryStreams() throws Exception { + var archive = tempDir.resolve("payloads.zip"); + try (var fileSystem = + FileSystems.newFileSystem(URI.create("jar:" + archive.toUri()), Map.of("create", "true"))) { + var serDes = + FileSystemSerDes.builder(fileSystem.getPath("/payloads")).build(); + + var failure = assertThrows( + SerDesException.class, + () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + + assertTrue(failure.getMessage().contains("SecureDirectoryStream support")); + } + } + + @Test + void rejectsSymbolicLinkDirectoryWhenWriting() throws Exception { + var outside = Files.createTempDirectory(tempDir.getParent(), "outside-payloads-"); + Files.createSymbolicLink(tempDir.resolve("test"), outside); + var serDes = FileSystemSerDes.builder(tempDir).build(); + + assertThrows( + SerDesException.class, () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + try (var files = Files.list(outside)) { + assertEquals(0, files.count()); + } + } + + @Test + void rejectsSymbolicLinkDirectoryWhenReading() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var context = new SerDesContext(realisticArn(), "1"); + var envelope = runner.serialize(serDes, "value", context); + var executionDirectory = tempDir.resolve("test"); + var outside = Files.createTempDirectory(tempDir.getParent(), "outside-payloads-"); + var movedDirectory = outside.resolve("test"); + Files.move(executionDirectory, movedDirectory); + Files.createSymbolicLink(executionDirectory, movedDirectory); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + } + + @Test + void rejectsSymbolicLinkPayloadFile() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var context = new SerDesContext(realisticArn(), "1"); + var envelope = runner.serialize(serDes, "value", context); + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + var outside = Files.createTempFile(tempDir.getParent(), "outside-payload-", ".json"); + Files.writeString(outside, "\"value\""); + Files.delete(file); + Files.createSymbolicLink(file, outside); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + } + + @Test + void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { + var outsideRoot = Files.createTempDirectory(tempDir.getParent(), "outside-root-"); + var linkedRoot = tempDir.resolve("linked-root"); + Files.createSymbolicLink(linkedRoot, outsideRoot); + var rootSerDes = FileSystemSerDes.builder(linkedRoot).build(); + + assertThrows( + SerDesException.class, + () -> runner.serialize(rootSerDes, "value", new SerDesContext(realisticArn(), "1"))); + + var outsideAncestor = Files.createTempDirectory(tempDir.getParent(), "outside-ancestor-"); + var linkedAncestor = tempDir.resolve("linked-ancestor"); + Files.createSymbolicLink(linkedAncestor, outsideAncestor); + var nestedSerDes = + FileSystemSerDes.builder(linkedAncestor.resolve("payloads")).build(); + + assertThrows( + SerDesException.class, + () -> runner.serialize(nestedSerDes, "value", new SerDesContext(realisticArn(), "1"))); + assertFalse(Files.exists(outsideAncestor.resolve("payloads"))); + } + @Test void rejectsTrailingAndDuplicateFieldsInMarkedEnvelope() { var serDes = FileSystemSerDes.builder(tempDir).build(); From cfd469a90542aa3c3f80bb606fcd3287d054e663 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 00:44:29 +0000 Subject: [PATCH 05/11] test: align filesystem SerDes coverage with #648 --- .github/workflows/e2e-tests.yml | 55 +++- .gitignore | 1 + docs/adr/005-filesystem-serdes.md | 2 +- docs/advanced/serdes.md | 9 +- docs/core/invoke.md | 13 + docs/wire-formats/filesystem-serdes.md | 157 +++++++++++ examples/README.md | 35 +++ examples/generate-template.py | 244 ++++++++++++++++-- .../durable/examples/ExampleTemplate.java | 2 + .../general/FileSystemSerDesExample.java | 88 +++++++ .../examples/CloudBasedIntegrationTest.java | 78 ++++++ .../general/FileSystemSerDesExampleTest.java | 49 ++++ examples/test_generate_template.py | 54 ++++ .../FileSystemSerDesIntegrationTest.java | 171 ++++++++++++ .../testing/LocalDurableTestRunnerTest.java | 41 +++ .../durable/serde/FileSystemSerDes.java | 84 +++++- .../durable/serde/FileSystemSerDesTest.java | 168 ++++++++++++ .../lambda/durable/serde/RetrySerDesTest.java | 72 ++++++ .../durable/serde/SerDesRunnerTest.java | 125 +++++++++ 19 files changed, 1408 insertions(+), 40 deletions(-) create mode 100644 docs/wire-formats/filesystem-serdes.md create mode 100644 examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java create mode 100644 examples/src/test/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExampleTest.java create mode 100644 examples/test_generate_template.py diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index ae25b371a..d16297e40 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -32,10 +32,44 @@ permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout +env: + AWS_REGION: us-west-2 + FILESYSTEM_INFRASTRUCTURE_STACK_NAME: JavaSDKFileSystemSerDesE2EInfrastructureStack + jobs: + filesystem-infrastructure: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: "${{ secrets.TEST_ROLE_ARN }}" + role-session-name: java-language-sdk-test-infrastructure + aws-region: ${{ env.AWS_REGION }} + allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} + - name: Test SAM template generator + run: python3 -m unittest test_generate_template.py + working-directory: ./examples + - name: Generate persistent filesystem SerDes E2E infrastructure template + run: | + python3 generate-template.py \ + --file-system-infrastructure-only \ + --output filesystem-infrastructure-template.yaml + working-directory: ./examples + - name: Ensure persistent filesystem SerDes E2E infrastructure + run: | + aws cloudformation deploy \ + --template-file filesystem-infrastructure-template.yaml \ + --stack-name ${{ env.FILESYSTEM_INFRASTRUCTURE_STACK_NAME }} \ + --no-fail-on-empty-changeset \ + --tags Purpose=JavaSDKFileSystemSerDesE2E + working-directory: ./examples + e2e-tests: + needs: filesystem-infrastructure env: - AWS_REGION: us-west-2 E2E_TEST_PARALLELISM: 4 runs-on: ubuntu-latest strategy: @@ -71,6 +105,9 @@ jobs: - name: Generate SAM template run: python3 generate-template.py working-directory: ./examples + - name: Generate filesystem SerDes E2E SAM template + run: python3 generate-template.py --file-system-only --output filesystem-template.yaml + working-directory: ./examples - name: sam build env: MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true @@ -78,6 +115,14 @@ jobs: sam build --debug --parameter-overrides \ 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples + - name: sam build filesystem SerDes E2E stack + env: + MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true + run: | + sam build --debug --template-file filesystem-template.yaml --build-dir .aws-sam-filesystem \ + --parameter-overrides \ + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=${{ env.FILESYSTEM_INFRASTRUCTURE_STACK_NAME }}' + working-directory: ./examples - name: Clean up unmanaged Lambda log groups run: | # TODO: Remove this one-time migration cleanup after existing e2e stacks adopt managed log groups. @@ -89,12 +134,20 @@ jobs: --resolve-image-repos --resolve-s3 --parameter-overrides \ 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples + - name: sam deploy filesystem SerDes E2E stack + run: | + sam deploy --template-file .aws-sam-filesystem/template.yaml \ + --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack \ + --resolve-s3 --parameter-overrides \ + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=${{ env.FILESYSTEM_INFRASTRUCTURE_STACK_NAME }}' + working-directory: ./examples - name: Record E2E log start time run: echo "E2E_LOG_START_TIME_MS=$(date +%s%3N)" >> "$GITHUB_ENV" - name: Cloud Based Integration Tests run: | mvn clean test -B \ -Dtest.cloud.enabled=true \ + -Dtest.filesystem.enabled=true \ -Dtest.aws.account='${{ secrets.TEST_ACCOUNT_ID }}' \ -Dtest=CloudBasedIntegrationTest \ -Dtest.function.name.prefix='Java${{ matrix.java }}-' \ diff --git a/.gitignore b/.gitignore index 378d24717..1addbb95d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ __pycache__/ # SAM .aws-sam/ examples/template.yaml +examples/filesystem-template.yaml samconfig.toml samconfig.toml.bak diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 4f793f603..4649b4bec 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -141,7 +141,7 @@ Retry delays run inline or on the configured SerDes executor. Java writes versioned envelopes: ```json -{"__durable_execution_filesystem_serdes":1,"data":""} +{"__durable_execution_filesystem_serdes":1,"data":"","sha256":""} {"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":""} {"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":"","preview":{"id":"123"}} ``` diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index e07cdf772..de5a703c5 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -117,10 +117,13 @@ Only objects containing the reserved version marker are treated as filesystem en `file` fields is passed to the configured delegate. ```json -{"__durable_execution_filesystem_serdes":1,"data":""} +{"__durable_execution_filesystem_serdes":1,"data":"","sha256":""} {"__durable_execution_filesystem_serdes":1,"file":"/mnt/efs/...json","sha256":""} ``` +See [Filesystem SerDes wire format](../wire-formats/filesystem-serdes.md) for exact schemas, member validation, path +construction, security requirements, and versioning. + ### Structured previews `PreviewConfig` supports: @@ -177,3 +180,7 @@ the invocation cache. `CloudDurableTestRunner` and `AsyncExecution` reconstruct the execution and operation contexts from history events. Each asynchronous history snapshot receives its own cache so updated payloads cannot reuse values from an earlier snapshot. + +See the runnable +[FileSystemSerDesExample](../../examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java) +for filesystem configuration, structured previews, retries, replay, and checksum verification. diff --git a/docs/core/invoke.md b/docs/core/invoke.md index e79c99cf0..20d15bbdb 100644 --- a/docs/core/invoke.md +++ b/docs/core/invoke.md @@ -21,3 +21,16 @@ var result = ctx.invoke("invoke-function", ); ``` + +Payload and result serialization are selected independently. For example, a standard JSON payload can be sent while a +filesystem-backed result is decoded: + +```java +var config = InvokeConfig.builder() + .payloadSerDes(new JacksonSerDes()) + .serDes(fileSystemSerDes) + .build(); +``` + +If either boundary uses `FileSystemSerDes`, both functions must be able to access the referenced shared mount path. +See [Serialization and Filesystem Storage](../advanced/serdes.md). diff --git a/docs/wire-formats/filesystem-serdes.md b/docs/wire-formats/filesystem-serdes.md new file mode 100644 index 000000000..70868e467 --- /dev/null +++ b/docs/wire-formats/filesystem-serdes.md @@ -0,0 +1,157 @@ +# Filesystem SerDes wire format + +## Status and scope + +This document defines the persisted strings produced by Java `FileSystemSerDes`. The format is versioned durable state: +changes must preserve replay of existing checkpoints or introduce an explicit migration boundary. + +The internal Lambda durable execution request/response envelope is not covered. It remains encoded by +`DurableInputOutputSerDes`. + +## Recognition + +A value is a filesystem envelope only when its top-level JSON object contains: + +```json +"__durable_execution_filesystem_serdes": 1 +``` + +Unmarked input is passed unchanged to the configured delegate SerDes. Marked malformed input, duplicate fields, +trailing tokens, unsupported versions, and unknown members are rejected. + +Text containing the marker inside a JSON string is not recognized as an envelope. + +## Version 1 schemas + +### Inline + +```json +{ + "__durable_execution_filesystem_serdes": 1, + "data": "", + "sha256": "<64 lowercase hexadecimal characters>" +} +``` + +### File + +```json +{ + "__durable_execution_filesystem_serdes": 1, + "file": "/absolute/path/to/payload.json", + "sha256": "<64 lowercase hexadecimal characters>" +} +``` + +### File with preview + +```json +{ + "__durable_execution_filesystem_serdes": 1, + "file": "/absolute/path/to/payload.json", + "sha256": "<64 lowercase hexadecimal characters>", + "preview": { + "id": "order-123", + "email": "***" + } +} +``` + +Exactly one of `data` or `file` is required. `preview` is valid only with `file`. The complete encoded envelope must fit +the configured `checkpointEnvelopeLimitBytes`. + +## Payload and digest + +The payload is the UTF-8 byte sequence of the delegate SerDes string. `sha256` is the lowercase hexadecimal SHA-256 +digest of those bytes. + +Deserialization verifies both inline and file payloads before invoking the delegate. Missing, malformed, or mismatched +digests are permanent `SerDesException` failures. + +Malformed UTF-8 file content and filesystem read failures are `RetryableSerDesException` failures. + +## Path construction + +All paths are rooted under the configured absolute base path. + +### URI encoding + +For a durable execution ARN matching: + +```text +arn::lambda:::function::/durable-execution// +``` + +the execution directory is: + +```text +/// +``` + +Other ARNs are encoded into one directory segment. + +The file name is: + +```text +--.json +``` + +URI encoding preserves ASCII letters, digits, `-`, `_`, `.`, and `~`; all other UTF-8 bytes are percent encoded. + +### Hash encoding + +`HASH` replaces the execution ARN and entity ID with lowercase SHA-256 hexadecimal strings. The payload digest and +unique UUID suffix remain in the file name. + +## Publication + +Files are immutable and published with `CREATE_NEW`. A failed write attempts to delete the partially created file before +returning a retryable failure. Repeated serialization writes a distinct UUID-suffixed file, even for identical payloads. + +No hard-link or rename operation is required. + +## Filesystem security requirements + +The provider must support `SecureDirectoryStream`. + +Directory traversal starts at the filesystem root. Every path component is opened relative to an already-held parent +directory with `NOFOLLOW_LINKS`. File reads and writes also use `NOFOLLOW_LINKS`. + +The implementation fails closed when: + +- the provider lacks `SecureDirectoryStream`; +- the base path, an ancestor, execution directory, or payload file is a symbolic link; +- a path is outside the configured base path; +- the file is missing or unreadable. + +Security failures represented by invalid envelope or path metadata are permanent. Filesystem I/O failures are +retryable. + +## Cross-execution references + +The file path is carried in the persisted envelope. A caller and callee may exchange a filesystem-backed invoke payload +or result only when both can access the same mounted path and use compatible delegate SerDes configuration. + +Use `InvokeConfig.payloadSerDes(...)` and `InvokeConfig.serDes(...)` to select filesystem storage independently for the +payload and result boundaries. + +## Preview rules + +Preview data is informational and is never used for deserialization. + +Built-in structured previews support: + +- include-all and exclude-all modes; +- include, exclude, and mask selectors; +- field-name matching at any depth; +- exact dotted-path matching; +- a configurable mask string; +- a configurable UTF-8 byte budget, defaulting to 4 KiB. + +Object arrays are flattened at the containing path. Scalar arrays are retained. Fields containing dots are omitted +because they are ambiguous with path selectors. + +## Versioning + +Readers accept version `1` only. Future versions must use a different marker value and must not reinterpret version 1 +members. diff --git a/examples/README.md b/examples/README.md index c17750835..0151edb9a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -77,6 +77,25 @@ mvn test -Dtest=CloudBasedIntegrationTest \ -Dtest.aws.region=us-east-1 ``` +For manually run cloud tests, the filesystem SerDes test is disabled by default because it requires VPC and EFS +infrastructure. Create the persistent infrastructure stack once: + +```bash +python3 generate-template.py \ + --file-system-infrastructure-only \ + --output filesystem-infrastructure-template.yaml +aws cloudformation deploy \ + --template-file filesystem-infrastructure-template.yaml \ + --stack-name JavaSDKFileSystemSerDesE2EInfrastructureStack +``` + +Then generate, build, and deploy the filesystem Lambda stack with +`FileSystemInfrastructureStackName=JavaSDKFileSystemSerDesE2EInfrastructureStack`, and include +`-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. + +GitHub Actions maintains one persistent infrastructure stack shared by every Java version and one persistent +filesystem Lambda stack per Java version. Each E2E matrix job updates its Lambda stack in place and runs the test. + ## Examples | Example | Description | @@ -87,6 +106,7 @@ mvn test -Dtest=CloudBasedIntegrationTest \ | [ErrorHandlingExample](src/main/java/software/amazon/lambda/durable/examples/general/ErrorHandlingExample.java) | Handling `StepFailedException` and `StepInterruptedException` | | [GenericTypesExample](src/main/java/software/amazon/lambda/durable/examples/general/GenericTypesExample.java) | Working with `List` and `Map` | | [CustomConfigExample](src/main/java/software/amazon/lambda/durable/examples/general/CustomConfigExample.java) | Custom Lambda client and SerDes | +| [FileSystemSerDesExample](src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java) | Durable filesystem payload storage, previews, retries, and replay | | [WaitAtLeastExample](src/main/java/software/amazon/lambda/durable/examples/wait/WaitAtLeastExample.java) | Concurrent `stepAsync()` with `wait()` | | [WaitAsyncExample](src/main/java/software/amazon/lambda/durable/examples/wait/WaitAsyncExample.java) | Non-blocking `waitAsync()` with concurrent step | | [RetryInProcessExample](src/main/java/software/amazon/lambda/durable/examples/step/RetryInProcessExample.java) | In-process retry with concurrent operations | @@ -96,6 +116,21 @@ mvn test -Dtest=CloudBasedIntegrationTest \ | [CustomShouldCompleteMapExample](src/main/java/software/amazon/lambda/durable/examples/map/CustomShouldCompleteMapExample.java) | Custom map completion with `shouldComplete` decisions | | [WaitForConditionExample](src/main/java/software/amazon/lambda/durable/examples/wait/WaitForConditionExample.java) | Poll a condition until met with `waitForCondition()` | +### Filesystem SerDes example + +The filesystem example requires a shared durable mount and the `FILESYSTEM_SERDES_PATH` environment variable. Do not +use Lambda `/tmp`. + +For a deployed function, mount EFS or another compatible shared filesystem at the same path in every execution +environment: + +```text +FILESYSTEM_SERDES_PATH=/mnt/efs/durable-payloads +``` + +The mounted Java filesystem provider must support `SecureDirectoryStream`. The example uses structured previews, +filesystem I/O retries, a durable wait that forces replay, and checksum verification after the payload is loaded again. + ## Cleanup ```bash diff --git a/examples/generate-template.py b/examples/generate-template.py index 2ffcd5d70..9cb380568 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -21,6 +21,7 @@ class ExampleFunction: package_name: str suffix: str condition: str | None + file_system: bool @property def logical_id(self) -> str: @@ -56,21 +57,23 @@ def is_top_level_durable_handler(source: str, class_name: str) -> bool: return bool(match and "extends DurableHandler" in match.group("header")) -def read_template_condition(source: str, class_name: str) -> str | None: +def read_template_metadata(source: str, class_name: str) -> tuple[str | None, bool]: class_match = re.search(rf"public\s+(?:final\s+)?class\s+{class_name}\b", source) if not class_match: - return None + return None, False prefix = source[: class_match.start()] matches = list( re.finditer(rf"@(?:[A-Za-z_][\w.]*\.)?{TEMPLATE_ANNOTATION}\s*(?:\((?P.*?)\))?", prefix, re.DOTALL) ) if not matches: - return None + return None, False body = matches[-1].group("body") or "" condition_match = re.search(r'condition\s*=\s*"([^"]+)"', body) - return condition_match.group(1) if condition_match else None + condition = condition_match.group(1) if condition_match else None + file_system = bool(re.search(r"\bfileSystem\s*=\s*true\b", body)) + return condition, file_system def discover_examples() -> list[ExampleFunction]: @@ -81,7 +84,7 @@ def discover_examples() -> list[ExampleFunction]: if not is_top_level_durable_handler(source, class_name): continue - condition = read_template_condition(source, class_name) + condition, file_system = read_template_metadata(source, class_name) package_name = read_package(source, path) examples.append( ExampleFunction( @@ -89,6 +92,7 @@ def discover_examples() -> list[ExampleFunction]: package_name=package_name, suffix=kebab_case(class_name), condition=condition, + file_system=file_system, ) ) return examples @@ -103,7 +107,15 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: ) if example.condition: lines.append(f" Condition: {example.condition}") - lines.append(f" DependsOn: {example.log_group_logical_id}") + if example.file_system: + lines.extend( + [ + " DependsOn:", + f" - {example.log_group_logical_id}", + ] + ) + else: + lines.append(f" DependsOn: {example.log_group_logical_id}") lines.extend( [ " Properties:", @@ -112,6 +124,26 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " Role: !Ref RoleArn", ] ) + if example.file_system: + lines.extend( + [ + " VpcConfig:", + " SecurityGroupIds:", + " - Fn::ImportValue:", + ' Fn::Sub: "${FileSystemInfrastructureStackName}-LambdaSecurityGroupId"', + " SubnetIds:", + " - Fn::ImportValue:", + ' Fn::Sub: "${FileSystemInfrastructureStackName}-SubnetId"', + " FileSystemConfigs:", + " - Arn:", + " Fn::ImportValue:", + ' Fn::Sub: "${FileSystemInfrastructureStackName}-AccessPointArn"', + " LocalMountPath: /mnt/efs", + " Environment:", + " Variables:", + " FILESYSTEM_SERDES_PATH: /mnt/efs/durable-payloads", + ] + ) lines.append("") @@ -134,6 +166,133 @@ def emit_log_group(lines: list[str], example: ExampleFunction) -> None: ) +def emit_file_system_resources(lines: list[str]) -> None: + lines.extend( + [ + " FileSystemVpc:", + " Type: AWS::EC2::VPC", + " Properties:", + " CidrBlock: 10.0.0.0/24", + " EnableDnsHostnames: true", + " EnableDnsSupport: true", + "", + " FileSystemSubnet:", + " Type: AWS::EC2::Subnet", + " Properties:", + " CidrBlock: 10.0.0.0/26", + " VpcId: !Ref FileSystemVpc", + "", + " FileSystemLambdaSecurityGroup:", + " Type: AWS::EC2::SecurityGroup", + " Properties:", + " GroupDescription: Lambda access to EFS and the Lambda API endpoint", + " VpcId: !Ref FileSystemVpc", + "", + " FileSystemMountSecurityGroup:", + " Type: AWS::EC2::SecurityGroup", + " Properties:", + " GroupDescription: EFS mount access from Lambda", + " VpcId: !Ref FileSystemVpc", + " SecurityGroupIngress:", + " - IpProtocol: tcp", + " FromPort: 2049", + " ToPort: 2049", + " SourceSecurityGroupId: !Ref FileSystemLambdaSecurityGroup", + "", + " FileSystemEndpointSecurityGroup:", + " Type: AWS::EC2::SecurityGroup", + " Properties:", + " GroupDescription: Lambda API endpoint access from Lambda", + " VpcId: !Ref FileSystemVpc", + " SecurityGroupIngress:", + " - IpProtocol: tcp", + " FromPort: 443", + " ToPort: 443", + " SourceSecurityGroupId: !Ref FileSystemLambdaSecurityGroup", + "", + " FileSystemLambdaEndpoint:", + " Type: AWS::EC2::VPCEndpoint", + " Properties:", + " PrivateDnsEnabled: true", + " SecurityGroupIds:", + " - !Ref FileSystemEndpointSecurityGroup", + ' ServiceName: !Sub "com.amazonaws.${AWS::Region}.lambda"', + " SubnetIds:", + " - !Ref FileSystemSubnet", + " VpcEndpointType: Interface", + " VpcId: !Ref FileSystemVpc", + "", + " FileSystem:", + " Type: AWS::EFS::FileSystem", + " Properties:", + " Encrypted: true", + " PerformanceMode: generalPurpose", + " ThroughputMode: bursting", + "", + " FileSystemMountTarget:", + " Type: AWS::EFS::MountTarget", + " Properties:", + " FileSystemId: !Ref FileSystem", + " SecurityGroups:", + " - !Ref FileSystemMountSecurityGroup", + " SubnetId: !Ref FileSystemSubnet", + "", + " FileSystemAccessPoint:", + " Type: AWS::EFS::AccessPoint", + " Properties:", + " FileSystemId: !Ref FileSystem", + " PosixUser:", + ' Gid: "1000"', + ' Uid: "1000"', + " RootDirectory:", + " CreationInfo:", + ' OwnerGid: "1000"', + ' OwnerUid: "1000"', + ' Permissions: "0777"', + " Path: /durable-serdes", + "", + ] + ) + + +def emit_file_system_outputs(lines: list[str]) -> None: + lines.extend( + [ + "Outputs:", + " SubnetId:", + " Description: Subnet used by the filesystem SerDes E2E Lambda function", + " Value: !Ref FileSystemSubnet", + " Export:", + ' Name: !Sub "${AWS::StackName}-SubnetId"', + "", + " LambdaSecurityGroupId:", + " Description: Security group used by the filesystem SerDes E2E Lambda function", + " Value: !Ref FileSystemLambdaSecurityGroup", + " Export:", + ' Name: !Sub "${AWS::StackName}-LambdaSecurityGroupId"', + "", + " AccessPointArn:", + " Description: EFS access point mounted by the filesystem SerDes E2E Lambda function", + " Value: !GetAtt FileSystemAccessPoint.Arn", + " Export:", + ' Name: !Sub "${AWS::StackName}-AccessPointArn"', + ] + ) + + +def render_file_system_infrastructure_template() -> str: + lines = [ + "# This file is generated by examples/generate-template.py. Do not edit it by hand.", + 'AWSTemplateFormatVersion: "2010-09-09"', + "Description: Persistent shared EFS infrastructure for filesystem SerDes E2E tests", + "", + "Resources:", + ] + emit_file_system_resources(lines) + emit_file_system_outputs(lines) + return "\n".join(lines) + "\n" + + def render_template(examples: list[ExampleFunction]) -> str: lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", @@ -160,29 +319,41 @@ def render_template(examples: list[ExampleFunction]) -> str: " RoleArn:", " Type: String", " Description: IAM Role ARN for Lambda function execution", - "", - "Conditions:", - " IsJava21OrLater:", - " !Or", - " - !Equals [!Ref JavaVersion, 'java21']", - " - !Equals [!Ref JavaVersion, 'java25']", - "", - "Globals:", - " Function:", - " Timeout: 900", - " MemorySize: 512", - " Architectures:", - " - !Ref Architecture", - " DurableConfig:", - " ExecutionTimeout: 300", - " RetentionPeriodInDays: 7", - " Runtime: !Ref JavaVersion", - " Environment:", - " Variables:", - " FUNCTION_NAME_PREFIX: !Ref FunctionNamePrefix", - "", - "Resources:", ] + if any(example.file_system for example in examples): + lines.extend( + [ + " FileSystemInfrastructureStackName:", + " Type: String", + " Description: Name of the shared persistent filesystem SerDes E2E infrastructure stack", + ] + ) + lines.extend( + [ + "", + "Conditions:", + " IsJava21OrLater:", + " !Or", + " - !Equals [!Ref JavaVersion, 'java21']", + " - !Equals [!Ref JavaVersion, 'java25']", + "", + "Globals:", + " Function:", + " Timeout: 900", + " MemorySize: 512", + " Architectures:", + " - !Ref Architecture", + " DurableConfig:", + " ExecutionTimeout: 300", + " RetentionPeriodInDays: 7", + " Runtime: !Ref JavaVersion", + " Environment:", + " Variables:", + " FUNCTION_NAME_PREFIX: !Ref FunctionNamePrefix", + "", + "Resources:", + ] + ) for example in examples: emit_log_group(lines, example) @@ -208,9 +379,26 @@ def render_template(examples: list[ExampleFunction]) -> str: def main() -> None: parser = argparse.ArgumentParser(description="Generate the examples SAM template from Java example handlers.") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Path to write the generated template.") + template_selection = parser.add_mutually_exclusive_group() + template_selection.add_argument( + "--file-system-only", + action="store_true", + help="Generate the filesystem SerDes E2E Lambda stack.", + ) + template_selection.add_argument( + "--file-system-infrastructure-only", + action="store_true", + help="Generate the shared persistent EFS infrastructure stack used by filesystem SerDes E2E tests.", + ) args = parser.parse_args() + if args.file_system_infrastructure_only: + args.output.write_text(render_file_system_infrastructure_template(), encoding="utf-8") + print(f"Generated persistent filesystem infrastructure template at {args.output}.") + return + examples = discover_examples() + examples = [example for example in examples if example.file_system == args.file_system_only] if not examples: raise RuntimeError("No DurableHandler examples found") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java index de93e95ca..55eab7934 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java @@ -12,4 +12,6 @@ @Target(ElementType.TYPE) public @interface ExampleTemplate { String condition() default ""; + + boolean fileSystem() default false; } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java new file mode 100644 index 000000000..0f6d4f08c --- /dev/null +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java @@ -0,0 +1,88 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.general; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.HexFormat; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.examples.ExampleTemplate; +import software.amazon.lambda.durable.retry.JitterStrategy; +import software.amazon.lambda.durable.retry.RetryStrategies; +import software.amazon.lambda.durable.serde.FileSystemPathEncoding; +import software.amazon.lambda.durable.serde.FileSystemSerDes; +import software.amazon.lambda.durable.serde.FileSystemSerDesMode; +import software.amazon.lambda.durable.serde.PreviewConfig; +import software.amazon.lambda.durable.serde.PreviewField; +import software.amazon.lambda.durable.serde.PreviewMode; +import software.amazon.lambda.durable.serde.RetrySerDes; + +/** Stores durable payloads on a shared filesystem and verifies them after replay. */ +@ExampleTemplate(fileSystem = true) +public class FileSystemSerDesExample + extends DurableHandler { + static final String FILE_SYSTEM_PATH_PROPERTY = "filesystem.serdes.path"; + private static final String FILE_SYSTEM_PATH_ENV = "FILESYSTEM_SERDES_PATH"; + + @Override + protected DurableConfig createConfiguration() { + var path = System.getProperty(FILE_SYSTEM_PATH_PROPERTY); + if (path == null || path.isBlank()) { + path = System.getenv(FILE_SYSTEM_PATH_ENV); + } + if (path == null || path.isBlank()) { + throw new IllegalStateException(FILE_SYSTEM_PATH_ENV + " must identify the mounted durable filesystem"); + } + + var fileSystemSerDes = FileSystemSerDes.builder(Path.of(path)) + .storageMode(FileSystemSerDesMode.ALWAYS) + .pathEncoding(FileSystemPathEncoding.HASH) + .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include( + PreviewField.anywhere("id"), + PreviewField.anywhere("length"), + PreviewField.anywhere("checksum")) + .mask(PreviewField.anywhere("email")) + .build()) + .build(); + var retryingSerDes = new RetrySerDes( + fileSystemSerDes, + RetryStrategies.exponentialBackoff( + 4, Duration.ofSeconds(1), Duration.ofSeconds(10), 2.0, JitterStrategy.FULL)); + return DurableConfig.builder().withSerDes(retryingSerDes).build(); + } + + @Override + public Output handleRequest(Input input, DurableContext context) { + var stored = context.step( + "store-payload", + Payload.class, + stepContext -> new Payload( + input.id(), input.email(), input.value(), input.value().length())); + context.wait("force-filesystem-replay", Duration.ofSeconds(1)); + return context.step( + "verify-payload", + Output.class, + stepContext -> new Output(stored.id(), stored.length(), sha256(stored.value()))); + } + + private static String sha256(String value) { + try { + return HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + public record Input(String id, String email, String value) {} + + public record Payload(String id, String email, String value, int length) {} + + public record Output(String id, int length, String checksum) {} +} diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java index 7490961a5..23f0f4095 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java @@ -5,8 +5,14 @@ import static org.junit.jupiter.api.Assertions.*; import static software.amazon.lambda.durable.TypeToken.get; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.util.HashMap; +import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -14,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledForJreRange; import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.api.condition.JRE; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -21,9 +28,11 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.EventType; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.examples.general.FileSystemSerDesExample; import software.amazon.lambda.durable.examples.general.GenericTypesExample; import software.amazon.lambda.durable.examples.types.ApprovalRequest; import software.amazon.lambda.durable.examples.types.GreetingRequest; @@ -38,6 +47,8 @@ @EnabledIf("isEnabled") class CloudBasedIntegrationTest { private static final int PERFORMANCE_TEST_REPEAT = 3; + private static final String FILE_SYSTEM_ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final ObjectMapper MAPPER = new ObjectMapper(); private static String account; private static String region; @@ -301,6 +312,44 @@ void testGenericTypesExample() { assertNotNull(runner.getOperation("fetch-categories")); } + @Test + @EnabledIfSystemProperty(named = "test.filesystem.enabled", matches = "true") + void testFileSystemSerDesExample() throws Exception { + var value = "filesystem-e2e-".repeat(24 * 1024); + var input = new FileSystemSerDesExample.Input("payload-1", "user@example.com", value); + var expectedChecksum = sha256(value); + var runner = CloudDurableTestRunner.create( + arn("file-system-ser-des-example"), + FileSystemSerDesExample.Input.class, + FileSystemSerDesExample.Output.class, + lambdaClient); + + var result = runner.run(input); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertTrue(result.getHistoryEvents().stream() + .filter(event -> event.eventType() == EventType.INVOCATION_COMPLETED) + .count() + >= 2); + assertNotNull(result.getOperation("force-filesystem-replay")); + + var storedEnvelope = assertFileSystemEnvelope( + result.getOperation("store-payload").getStepDetails().result()); + assertPreview(storedEnvelope, input.id(), value.length(), null, "***"); + + var verifiedEnvelope = assertFileSystemEnvelope( + result.getOperation("verify-payload").getStepDetails().result()); + assertPreview(verifiedEnvelope, input.id(), value.length(), expectedChecksum, null); + + var outputPayload = result.getHistoryEvents().stream() + .filter(event -> event.eventType() == EventType.EXECUTION_SUCCEEDED) + .map(event -> event.executionSucceededDetails().result().payload()) + .findFirst() + .orElseThrow(); + var outputEnvelope = assertFileSystemEnvelope(outputPayload); + assertPreview(outputEnvelope, input.id(), value.length(), expectedChecksum, null); + } + @Test void testGenericInputOutputExample() { final TypeToken>>> resultType = new TypeToken<>() {}; @@ -853,4 +902,33 @@ void testPluginExample() { assertNotNull(runner.getOperation("create-greeting")); assertNotNull(runner.getOperation("transform")); } + + private static JsonNode assertFileSystemEnvelope(String value) throws Exception { + var envelope = MAPPER.readTree(value); + assertEquals(1, envelope.get(FILE_SYSTEM_ENVELOPE_MARKER).intValue()); + assertTrue(envelope.get("sha256").textValue().matches("[0-9a-f]{64}")); + assertTrue(envelope.get("file").textValue().startsWith("/mnt/efs/durable-payloads/")); + return envelope; + } + + private static void assertPreview(JsonNode envelope, String id, int length, String checksum, String maskedEmail) { + var preview = envelope.get("preview"); + assertEquals(id, preview.get("id").textValue()); + assertEquals(length, preview.get("length").intValue()); + if (checksum != null) { + assertEquals(checksum, preview.get("checksum").textValue()); + } + if (maskedEmail != null) { + assertEquals(maskedEmail, preview.get("email").textValue()); + } + } + + private static String sha256(String value) { + try { + return HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } } diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExampleTest.java new file mode 100644 index 000000000..7fef1976f --- /dev/null +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExampleTest.java @@ -0,0 +1,49 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.general; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.parallel.ResourceLock; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class FileSystemSerDesExampleTest { + @TempDir + Path tempDir; + + @Test + @ResourceLock(FileSystemSerDesExample.FILE_SYSTEM_PATH_PROPERTY) + void storesAndReplaysPayloadsFromFilesystem() throws Exception { + System.setProperty(FileSystemSerDesExample.FILE_SYSTEM_PATH_PROPERTY, tempDir.toString()); + try { + var handler = new FileSystemSerDesExample(); + var runner = LocalDurableTestRunner.create(FileSystemSerDesExample.Input.class, handler); + var value = "filesystem-value-".repeat(1024); + + var result = + runner.runUntilComplete(new FileSystemSerDesExample.Input("payload-1", "user@example.com", value)); + var output = result.getResult(FileSystemSerDesExample.Output.class); + + assertEquals("payload-1", output.id()); + assertEquals(value.length(), output.length()); + assertEquals( + HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))), + output.checksum()); + try (var files = Files.walk(tempDir)) { + assertTrue(files.filter(Files::isRegularFile).count() >= 3); + } + } finally { + System.clearProperty(FileSystemSerDesExample.FILE_SYSTEM_PATH_PROPERTY); + } + } +} diff --git a/examples/test_generate_template.py b/examples/test_generate_template.py new file mode 100644 index 000000000..67dd54fd4 --- /dev/null +++ b/examples/test_generate_template.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +GENERATOR_PATH = Path(__file__).with_name("generate-template.py") +SPEC = importlib.util.spec_from_file_location("generate_template", GENERATOR_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"Unable to load {GENERATOR_PATH}") +generate_template = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = generate_template +SPEC.loader.exec_module(generate_template) + + +class GenerateTemplateTest(unittest.TestCase): + def test_default_template_does_not_include_file_system_infrastructure(self) -> None: + examples = [example for example in generate_template.discover_examples() if not example.file_system] + + template = generate_template.render_template(examples) + + self.assertNotIn("FileSystemInfrastructureStackName", template) + self.assertNotIn("AWS::EFS::FileSystem", template) + + def test_file_system_lambda_template_imports_persistent_infrastructure(self) -> None: + examples = [example for example in generate_template.discover_examples() if example.file_system] + + template = generate_template.render_template(examples) + + self.assertIn("FileSystemInfrastructureStackName:", template) + self.assertIn("${FileSystemInfrastructureStackName}-SubnetId", template) + self.assertIn("${FileSystemInfrastructureStackName}-LambdaSecurityGroupId", template) + self.assertIn("${FileSystemInfrastructureStackName}-AccessPointArn", template) + self.assertIn("FILESYSTEM_SERDES_PATH: /mnt/efs/durable-payloads", template) + self.assertNotIn("AWS::EFS::FileSystem", template) + self.assertNotIn("FileSystemMountTarget", template) + + def test_file_system_infrastructure_template_exports_shared_resources(self) -> None: + template = generate_template.render_file_system_infrastructure_template() + + self.assertIn("AWS::EFS::FileSystem", template) + self.assertIn("AWS::EFS::MountTarget", template) + self.assertIn("${AWS::StackName}-SubnetId", template) + self.assertIn("${AWS::StackName}-LambdaSecurityGroupId", template) + self.assertIn("${AWS::StackName}-AccessPointArn", template) + self.assertNotIn("AWS::Serverless::Function", template) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index e9c777f00..a17056d70 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -10,13 +10,24 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.model.WaitForConditionResult; import software.amazon.lambda.durable.retry.JitterStrategy; @@ -27,6 +38,8 @@ import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; +import software.amazon.lambda.durable.testing.local.OperationResult; class FileSystemSerDesIntegrationTest { private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -82,6 +95,134 @@ void operationConfigControlsWhereFilesystemStorageIsUsed() throws Exception { } } + @Test + void durableExecutorAcceptsRawServiceInputBeforeFilesystemEnvelopeExists() { + var executionArn = + "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST/durable-execution/execution/raw-input"; + var executionOperation = + executionOperation("raw-input", "execution", "\"service-input\"", OperationStatus.STARTED); + var client = new LocalMemoryExecutionClient(); + var serDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withSerDes(serDes) + .build(); + + var output = DurableExecutor.execute( + durableInput(executionArn, executionOperation, List.of(), List.of()), + null, + TypeToken.get(String.class), + (value, context) -> value + "-output", + config); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertEquals("service-input-output", serDes.deserialize(output.result(), TypeToken.get(String.class))); + } + + @Test + void callerAndCalleeExchangeOffloadedInvokePayloadAndResult() throws Exception { + var callerArn = + "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; + var calleeArn = + "arn:aws:lambda:us-east-1:123456789012:function:callee:1/durable-execution/callee-execution/callee-invocation"; + var serDes = FileSystemSerDes.builder(tempDir).build(); + var callerClient = new LocalMemoryExecutionClient(); + var callerConfig = DurableConfig.builder() + .withDurableExecutionClient(callerClient) + .withSerDes(serDes) + .build(); + BiFunction callerHandler = (input, context) -> + context.invoke("call-callee", "callee", new CrossInvokeRequest(input), CrossInvokeResponse.class); + var callerExecution = + executionOperation("caller-invocation", "caller-execution", "\"request\"", OperationStatus.STARTED); + + var pending = DurableExecutor.execute( + durableInput(callerArn, callerExecution, List.of(), List.of()), + null, + TypeToken.get(String.class), + callerHandler, + callerConfig); + + assertEquals(ExecutionStatus.PENDING, pending.status()); + var invokePayload = callerClient.getOperationUpdates().stream() + .filter(update -> + update.type() == OperationType.CHAINED_INVOKE && update.action() == OperationAction.START) + .findFirst() + .orElseThrow() + .payload(); + assertEnvelopePointsToFile(invokePayload); + + var calleeClient = new LocalMemoryExecutionClient(); + var calleeConfig = DurableConfig.builder() + .withDurableExecutionClient(calleeClient) + .withSerDes(serDes) + .build(); + var calleeExecution = + executionOperation("callee-invocation", "callee-execution", invokePayload, OperationStatus.STARTED); + var calleeOutput = DurableExecutor.execute( + durableInput(calleeArn, calleeExecution, List.of(), List.of()), + null, + TypeToken.get(CrossInvokeRequest.class), + (request, context) -> new CrossInvokeResponse("reply:" + request.value()), + calleeConfig); + + assertEquals(ExecutionStatus.SUCCEEDED, calleeOutput.status()); + assertEnvelopePointsToFile(calleeOutput.result()); + + callerClient.completeChainedInvoke("call-callee", OperationResult.succeeded(calleeOutput.result())); + var resumed = DurableExecutor.execute( + durableInput( + callerArn, + callerExecution, + callerClient.getAllOperations(), + callerClient.getUpdatedOperationIdsSinceLastInvocation()), + null, + TypeToken.get(String.class), + callerHandler, + callerConfig); + + assertEquals(ExecutionStatus.SUCCEEDED, resumed.status()); + assertEquals( + new CrossInvokeResponse("reply:request"), + serDes.deserialize(resumed.result(), TypeToken.get(CrossInvokeResponse.class))); + } + + @Test + void invokePayloadOverridePreservesStandardJsonWireContract() { + var callerArn = + "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; + var callerClient = new LocalMemoryExecutionClient(); + var fileSystemSerDes = FileSystemSerDes.builder(tempDir).build(); + var callerConfig = DurableConfig.builder() + .withDurableExecutionClient(callerClient) + .withSerDes(fileSystemSerDes) + .build(); + BiFunction handler = (input, context) -> context.invoke( + "call-standard", + "standard", + new CrossInvokeRequest(input), + String.class, + InvokeConfig.builder().payloadSerDes(new JacksonSerDes()).build()); + var execution = + executionOperation("caller-invocation", "caller-execution", "\"request\"", OperationStatus.STARTED); + + var pending = DurableExecutor.execute( + durableInput(callerArn, execution, List.of(), List.of()), + null, + TypeToken.get(String.class), + handler, + callerConfig); + + assertEquals(ExecutionStatus.PENDING, pending.status()); + var invokePayload = callerClient.getOperationUpdates().stream() + .filter(update -> + update.type() == OperationType.CHAINED_INVOKE && update.action() == OperationAction.START) + .findFirst() + .orElseThrow() + .payload(); + assertEquals("{\"value\":\"request\"}", invokePayload); + } + @Test void replaysStepWaitForConditionChildAndMapPayloads() throws Exception { var stepRuns = new AtomicInteger(); @@ -229,8 +370,38 @@ private void assertEnvelopePointsToFile(String envelope) throws Exception { assertTrue(file.startsWith(tempDir)); } + private static DurableExecutionInput durableInput( + String executionArn, Operation executionOperation, List operations, List updatedIds) { + var allOperations = new ArrayList(); + allOperations.add(executionOperation); + allOperations.addAll(operations); + return new DurableExecutionInput( + executionArn, + "checkpoint-token", + CheckpointUpdatedExecutionState.builder() + .operations(allOperations) + .build(), + updatedIds); + } + + private static Operation executionOperation(String id, String name, String inputPayload, OperationStatus status) { + return Operation.builder() + .id(id) + .name(name) + .type(OperationType.EXECUTION) + .status(status) + .startTimestamp(Instant.now()) + .executionDetails( + ExecutionDetails.builder().inputPayload(inputPayload).build()) + .build(); + } + record Payload(String value) {} + record CrossInvokeRequest(String value) {} + + record CrossInvokeResponse(String value) {} + public static class CustomFailure extends RuntimeException { public CustomFailure() {} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 8f8c4f90b..a382bfac5 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -5,18 +5,22 @@ import static org.junit.jupiter.api.Assertions.*; import static software.amazon.lambda.durable.TypeToken.get; +import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDesContext; @@ -142,4 +146,41 @@ public T deserialize(String data, TypeToken typeToken) { assertTrue(contexts.stream().allMatch(context -> context.durableExecutionArn() != null)); assertTrue(contexts.stream().map(SerDesContext::entityId).distinct().count() >= 2); } + + @Test + void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { + var stepExecutions = new AtomicInteger(); + var largeResult = "x".repeat(7 * 1024 * 1024); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> { + context.step("once", Void.class, step -> { + stepExecutions.incrementAndGet(); + return null; + }); + return largeResult; + }) + .withOutputType(String.class); + + var firstResult = runner.run("test"); + var replayResult = runner.run("test"); + + assertEquals(ExecutionStatus.SUCCEEDED, firstResult.getStatus()); + assertEquals(largeResult, firstResult.getResult()); + assertEquals(ExecutionStatus.SUCCEEDED, replayResult.getStatus()); + assertEquals(largeResult, replayResult.getResult()); + assertEquals(1, stepExecutions.get()); + } + + @Test + void filesystemSerDesUsesRawDelegateEncodingForInitialInput(@TempDir Path basePath) { + var config = DurableConfig.builder() + .withSerDes(FileSystemSerDes.builder(basePath).build()) + .build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult()); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 5e74f7c3c..f14afec81 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -116,14 +116,13 @@ public T deserialize(String data, TypeToken typeToken) { return delegate.deserialize(data, typeToken); } if (envelope.hasNonNull("data")) { - return delegate.deserialize(envelope.get("data").textValue(), typeToken); + var serialized = envelope.get("data").textValue(); + verifyDigest(serialized, envelope.get("sha256").textValue()); + return delegate.deserialize(serialized, typeToken); } var serialized = readPayload(envelope.get("file").textValue()); - var expected = envelope.get("sha256").textValue(); - if (!expected.equals(sha256(serialized))) { - throw new SerDesException("Filesystem SerDes payload digest does not match stored content"); - } + verifyDigest(serialized, envelope.get("sha256").textValue()); return delegate.deserialize(serialized, typeToken); } @@ -131,6 +130,7 @@ private String inlineEnvelope(String serialized) { var envelope = ENVELOPE_MAPPER.createObjectNode(); envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); envelope.put("data", serialized); + envelope.put("sha256", sha256(serialized)); return writeEnvelope(envelope); } @@ -162,7 +162,7 @@ private JsonNode parseEnvelope(String data) { try { node = ENVELOPE_READER.readTree(data); } catch (JsonProcessingException e) { - if (data.contains(ENVELOPE_MARKER)) { + if (containsFilesystemMarkerField(data)) { throw new SerDesException("Malformed filesystem SerDes envelope", e); } return null; @@ -186,9 +186,6 @@ private static boolean isValidEnvelope(JsonNode node) { if (hasData == hasFile) { return false; } - if (hasData) { - return node.size() == 2; - } if (!node.has("sha256") || !node.get("sha256").isTextual() || !SHA_256_DIGEST_PATTERN @@ -196,10 +193,79 @@ private static boolean isValidEnvelope(JsonNode node) { .matches()) { return false; } + if (hasData) { + return node.size() == 3; + } var hasPreview = node.has("preview"); return (!hasPreview || node.get("preview").isObject()) && node.size() == (hasPreview ? 4 : 3); } + private static boolean containsFilesystemMarkerField(String data) { + var index = 0; + while (index < data.length() && Character.isWhitespace(data.charAt(index))) { + index++; + } + if (index == data.length() || data.charAt(index) != '{') { + return false; + } + + var containerDepth = 1; + for (index++; index < data.length() && containerDepth > 0; index++) { + var current = data.charAt(index); + if (current == '{' || current == '[') { + containerDepth++; + } else if (current == '}' || current == ']') { + containerDepth--; + } else if (current == '"') { + var literalStart = index; + var valueStart = ++index; + var escaped = false; + while (index < data.length()) { + var literal = data.charAt(index); + if (escaped) { + escaped = false; + } else if (literal == '\\') { + escaped = true; + } else if (literal == '"') { + break; + } + index++; + } + if (containerDepth == 1 + && index < data.length() + && isFilesystemMarkerLiteral(data, literalStart, valueStart, index)) { + var next = index + 1; + while (next < data.length() && Character.isWhitespace(data.charAt(next))) { + next++; + } + if (next < data.length() && data.charAt(next) == ':') { + return true; + } + } + } + } + return false; + } + + private static boolean isFilesystemMarkerLiteral(String data, int literalStart, int valueStart, int literalEnd) { + if (literalEnd - valueStart == ENVELOPE_MARKER.length() + && data.regionMatches(valueStart, ENVELOPE_MARKER, 0, ENVELOPE_MARKER.length())) { + return true; + } + try { + return ENVELOPE_MARKER.equals( + ENVELOPE_MAPPER.readValue(data.substring(literalStart, literalEnd + 1), String.class)); + } catch (JsonProcessingException ignored) { + return false; + } + } + + private static void verifyDigest(String serialized, String expected) { + if (!expected.equals(sha256(serialized))) { + throw new SerDesException("Filesystem SerDes payload digest does not match stored content"); + } + } + private Path payloadPath(SerDesContext context, String digest) { var directory = executionDirectory(context.durableExecutionArn()); var fileName = encode(context.entityId()) + "-" + digest + "-" + UUID.randomUUID() + ".json"; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index bd2fc0ebf..763c94338 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -8,14 +8,18 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -23,6 +27,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryDecision; class FileSystemSerDesTest { private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -81,6 +86,7 @@ void overflowModeKeepsSmallPayloadInline() throws Exception { var node = MAPPER.readTree(envelope); assertTrue(node.hasNonNull("data")); + assertTrue(node.get("sha256").textValue().matches("[0-9a-f]{64}")); assertFalse(node.has("file")); assertEquals("small", runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); } @@ -171,6 +177,58 @@ void repeatedPayloadsUseDistinctImmutableFiles() throws Exception { assertEquals("\"value\"", Files.readString(second)); } + @Test + void verifiesInlineAndFilePayloadDigests() throws Exception { + var context = new SerDesContext(realisticArn(), "1"); + var inlineSerDes = FileSystemSerDes.builder(tempDir) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .build(); + var inline = (ObjectNode) MAPPER.readTree(runner.serialize(inlineSerDes, "value", context)); + inline.put("data", "\"tampered\""); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(inlineSerDes, inline.toString(), TypeToken.get(String.class), context)); + + var fileSerDes = FileSystemSerDes.builder(tempDir).build(); + var fileEnvelope = runner.serialize(fileSerDes, "expected", context); + var fileNode = MAPPER.readTree(fileEnvelope); + var file = Path.of(fileNode.get("file").textValue()); + Files.writeString(file, "\"tampered\""); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(fileSerDes, fileEnvelope, TypeToken.get(String.class), context)); + } + + @Test + void filePathContainsTheEnvelopeDigest() throws Exception { + var envelope = MAPPER.readTree(runner.serialize( + FileSystemSerDes.builder(tempDir).build(), "value", new SerDesContext(realisticArn(), "1"))); + + assertTrue( + envelope.get("file").textValue().contains(envelope.get("sha256").textValue())); + } + + @Test + void rejectsMissingAndMalformedPayloadDigest() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir) + .storageMode(FileSystemSerDesMode.OVERFLOW) + .build(); + var context = new SerDesContext(realisticArn(), "1"); + var envelope = (ObjectNode) MAPPER.readTree(runner.serialize(serDes, "value", context)); + + envelope.remove("sha256"); + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope.toString(), TypeToken.get(String.class), context)); + + envelope.put("sha256", "not-a-digest"); + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope.toString(), TypeToken.get(String.class), context)); + } + @Test void rejectsPreviewThatMakesFileEnvelopeTooLarge() throws Exception { var serDes = FileSystemSerDes.builder(tempDir) @@ -184,6 +242,26 @@ void rejectsPreviewThatMakesFileEnvelopeTooLarge() throws Exception { } } + @Test + void retryablePreviewFailureCanBeRetried() throws Exception { + var attempts = new AtomicInteger(); + var fileSystemSerDes = FileSystemSerDes.builder(tempDir) + .previewGenerator(value -> { + if (attempts.incrementAndGet() == 1) { + throw new RetryableSerDesException("preview unavailable"); + } + return Map.of("summary", "value"); + }) + .build(); + var serDes = new RetrySerDes( + fileSystemSerDes, (failure, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + + var envelope = MAPPER.readTree(runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + + assertEquals(2, attempts.get()); + assertEquals("value", envelope.get("preview").get("summary").textValue()); + } + @Test void structuredPreviewSelectsAndMasksFields() throws Exception { var serDes = FileSystemSerDes.builder(tempDir) @@ -244,6 +322,96 @@ void rejectsMalformedRecognizedEnvelope() { TypeToken.get(String.class))); } + @Test + void recognizesMalformedMarkerRegardlessOfWhitespaceOrFieldOrder() { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var malformed = List.of( + "{ \n \"__durable_execution_filesystem_serdes\" : 1", + "{\"precedingField\":true,\n \"__durable_execution_filesystem_serdes\" : 1", + "{\"\\u005f_durable_execution_filesystem_serdes\" : 1"); + + for (var envelope : malformed) { + assertThrows(SerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); + } + } + + @Test + void markerTextInsideStringDoesNotClaimMalformedJson() { + var value = "{\"message\":\"__durable_execution_filesystem_serdes\"} trailing"; + var delegate = new SerDes() { + @Override + public String serialize(Object input) { + return input.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + }; + + assertEquals( + value, + FileSystemSerDes.builder(tempDir) + .delegate(delegate) + .build() + .deserialize(value, TypeToken.get(String.class))); + } + + @Test + void rejectsUnsupportedAndOutOfRangeEnvelopeVersions() { + var serDes = FileSystemSerDes.builder(tempDir).build(); + for (var version : List.of("2", "4294967297")) { + var envelope = "{\"__durable_execution_filesystem_serdes\":" + + version + + ",\"data\":\"\\\"value\\\"\",\"sha256\":\"" + + "0".repeat(64) + + "\"}"; + assertThrows(SerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); + } + } + + @Test + void rejectsMalformedUtf8FilePayload() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var context = new SerDesContext(realisticArn(), "1"); + var envelope = runner.serialize(serDes, "value", context); + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + Files.write(file, new byte[] {(byte) 0xC3, (byte) 0x28}); + + assertThrows( + RetryableSerDesException.class, + () -> runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + } + + @Test + void uriEncodingUsesReadableExecutionPathAndFlatUnsafeEntity() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var context = new SerDesContext(realisticArn(), "../unsafe/entity"); + + var file = Path.of(MAPPER.readTree(runner.serialize(serDes, "value", context)) + .get("file") + .textValue()); + + assertEquals(tempDir.resolve("test").resolve("execution-name").resolve("invocation-id"), file.getParent()); + assertFalse(file.getFileName().toString().contains("/")); + assertTrue(file.getFileName().toString().startsWith("..%2Funsafe%2Fentity-")); + } + + @Test + void malformedExecutionArnFallsBackToOneEncodedDirectory() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var arn = "local/test:execution"; + + var file = Path.of(MAPPER.readTree(runner.serialize(serDes, "value", new SerDesContext(arn, "1"))) + .get("file") + .textValue()); + + assertEquals(1, tempDir.relativize(file.getParent()).getNameCount()); + assertEquals("local%2Ftest%3Aexecution", file.getParent().getFileName().toString()); + } + @Test void missingPayloadFileIsRetryable() throws Exception { var missing = tempDir.resolve("missing.json"); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index 003884f75..b510f6a79 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -3,8 +3,10 @@ package software.amazon.lambda.durable.serde; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; @@ -99,4 +101,74 @@ public String serialize(Object value) { assertEquals("\"value\"", new SerDesRunner(null).serialize(serDes, "value", context)); assertSame(context, observed.get()); } + + @Test + void retriesDeserializationAndUsesStrategyDelay() { + var attempts = new AtomicInteger(); + var observedDelay = new AtomicReference(); + var delegate = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + if (attempts.incrementAndGet() == 1) { + throw new RetryableSerDesException("temporary"); + } + return super.deserialize(data, typeToken); + } + }; + var serDes = new RetrySerDes( + delegate, (failure, attempt) -> RetryDecision.retry(Duration.ofMillis(25)), observedDelay::set); + + assertEquals("value", serDes.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals(Duration.ofMillis(25), observedDelay.get()); + assertEquals(2, attempts.get()); + } + + @Test + void rejectsInvalidStrategyResults() { + var failure = new RetryableSerDesException("temporary"); + var delegate = failingSerDes(failure); + + var nullDecision = new RetrySerDes(delegate, (error, attempt) -> null, delay -> {}); + assertTrue(assertThrows(SerDesException.class, () -> nullDecision.serialize("value")) + .getMessage() + .contains("returned null")); + + var negativeDelay = + new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(-1)), delay -> {}); + assertTrue(assertThrows(SerDesException.class, () -> negativeDelay.serialize("value")) + .getMessage() + .contains("invalid delay")); + } + + @Test + void restoresInterruptStatusWhenBackoffIsInterrupted() { + var serDes = new RetrySerDes( + failingSerDes(new RetryableSerDesException("temporary")), + (failure, attempt) -> RetryDecision.retry(Duration.ofSeconds(1)), + delay -> { + throw new InterruptedException("stop"); + }); + + try { + assertThrows(SerDesException.class, () -> serDes.serialize("value")); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + assertTrue(Thread.interrupted()); + assertFalse(Thread.currentThread().isInterrupted()); + } + } + + private static SerDes failingSerDes(RuntimeException failure) { + return new SerDes() { + @Override + public String serialize(Object value) { + throw failure; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + throw failure; + } + }; + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index d79d85210..03dd21eb0 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -8,8 +8,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterEach; @@ -140,5 +142,128 @@ public T deserialize(String data, TypeToken typeToken) { .get()); } + @Test + void cacheKeyIncludesSerDesIdentity() { + var calls = new AtomicInteger(); + var first = countingSerDes(calls); + var second = countingSerDes(calls); + var context = new SerDesContext("arn:test", "entity"); + + runner.deserialize(first, "{\"value\":\"same\"}", TypeToken.get(Value.class), context); + runner.deserialize(second, "{\"value\":\"same\"}", TypeToken.get(Value.class), context); + + assertEquals(2, calls.get()); + } + + @Test + void completedCacheEvictsLeastRecentlyUsedEntries() { + var calls = new AtomicInteger(); + var serDes = countingSerDes(calls); + var values = new java.util.ArrayList(); + for (int index = 0; index <= SerDesRunner.MAX_COMPLETED_DESERIALIZATIONS; index++) { + values.add(runner.deserialize( + serDes, + "{\"value\":\"" + index + "\"}", + TypeToken.get(Value.class), + new SerDesContext("arn:test", "entity-" + index))); + } + + runner.deserialize( + serDes, "{\"value\":\"0\"}", TypeToken.get(Value.class), new SerDesContext("arn:test", "entity-0")); + + assertEquals(SerDesRunner.MAX_COMPLETED_DESERIALIZATIONS + 2, calls.get()); + assertEquals(SerDesRunner.MAX_COMPLETED_DESERIALIZATIONS + 1, values.size()); + } + + @Test + void concurrentCacheMissesDeserializeOnlyOnce() throws Exception { + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var calls = new AtomicInteger(); + var serDes = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + entered.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + return super.deserialize(data, typeToken); + } + }; + var context = new SerDesContext("arn:test", "entity"); + var callers = Executors.newFixedThreadPool(2); + try { + var first = callers.submit( + () -> runner.deserialize(serDes, "{\"value\":\"x\"}", TypeToken.get(Value.class), context)); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + var second = callers.submit( + () -> runner.deserialize(serDes, "{\"value\":\"x\"}", TypeToken.get(Value.class), context)); + release.countDown(); + + assertSame(first.get(5, TimeUnit.SECONDS), second.get(5, TimeUnit.SECONDS)); + assertEquals(1, calls.get()); + } finally { + release.countDown(); + callers.shutdownNow(); + } + } + + @Test + void cachesNullDeserializationResults() { + var calls = new AtomicInteger(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return null; + } + }; + var context = new SerDesContext("arn:test", "entity"); + + assertNull(runner.deserialize(serDes, null, TypeToken.get(String.class), context)); + assertNull(runner.deserialize(serDes, null, TypeToken.get(String.class), context)); + assertEquals(1, calls.get()); + } + + @Test + void preservesFatalErrorsWithAndWithoutExecutor() { + var fatal = new AssertionError("fatal"); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + throw fatal; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + throw fatal; + } + }; + var context = new SerDesContext("arn:test", "entity"); + + assertSame(fatal, assertThrows(AssertionError.class, () -> runner.serialize(serDes, "value", context))); + assertSame(fatal, assertThrows(AssertionError.class, () -> new SerDesRunner(null) + .deserialize(serDes, "\"value\"", TypeToken.get(String.class), context))); + } + + private static SerDes countingSerDes(AtomicInteger calls) { + return new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return super.deserialize(data, typeToken); + } + }; + } + record Value(String value) {} } From 1db8ed22d403f0ee5662e984a30fa857f16795fc Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 00:54:33 +0000 Subject: [PATCH 06/11] fix: address filesystem SerDes review feedback --- docs/adr/005-filesystem-serdes.md | 5 ++ docs/advanced/configuration.md | 2 + docs/advanced/serdes.md | 14 ++++ docs/design.md | 4 + docs/wire-formats/filesystem-serdes.md | 8 ++ .../FileSystemSerDesIntegrationTest.java | 77 +++++++++++++++++++ .../lambda/durable/testing/TestOperation.java | 5 +- .../lambda/durable/testing/TestResult.java | 3 +- .../testing/LocalDurableTestRunnerTest.java | 5 +- .../durable/testing/TestOperationTest.java | 2 +- .../cloud/HistoryEventProcessorTest.java | 2 +- .../durable/execution/DurableExecutor.java | 14 ++-- .../operation/BaseDurableOperation.java | 6 +- .../durable/operation/InvokeOperation.java | 2 +- .../SerializableDurableOperation.java | 8 +- .../lambda/durable/serde/SerDesPreview.java | 19 +++-- .../operation/InvokeOperationTest.java | 48 ++++++++++++ .../SerializableDurableOperationTest.java | 42 ++++++++++ .../durable/serde/SerDesPreviewTest.java | 41 ++++++++++ 19 files changed, 281 insertions(+), 26 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 4649b4bec..7069f2e6d 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -44,6 +44,11 @@ there is no public setter. The implementation uses a plain `ThreadLocal`, not an `InheritableThreadLocal`. Every call restores the previous value in `finally`, which supports nesting and prevents context from leaking when executor threads are reused. +Entity IDs include a stable payload-kind suffix. Root input, output, and exceptions use `/input`, `/output`, and +`/exception`; operation invoke payloads, results/state, and exceptions use `/invoke-payload`, `/result`, and +`/exception`. This prevents deterministic external-storage keys for different payloads on the same operation from +colliding. + ### Use an invocation-scoped SerDesRunner Each `ExecutionManager` creates one `SerDesRunner` for the Lambda invocation. The runner: diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 19ee05880..261ca1873 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -103,6 +103,8 @@ var context = SerDesContext.getCurrentContext(); Custom SerDes implementations can use its durable execution ARN and entity ID for external storage. Calls use the configured SerDes executor when present, and successful deserializations are cached for the current Lambda invocation. +Root input, output, and exception IDs end in `/input`, `/output`, and `/exception`; operation payload IDs end in +`/invoke-payload`, `/result`, or `/exception`. Do not use Lambda's `/tmp` directory: replay can run in another execution environment. Use a shared durable mount such as EFS. S3 Files users must account for synchronization and crash-durability behavior. A chained-invoke boundary only diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index de5a703c5..8bdcdba1e 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -50,6 +50,20 @@ if (context != null) { The context is installed only while the SDK invokes `serialize` or `deserialize` and is restored in `finally`. Direct customer calls return `null`. +Entity IDs distinguish every persisted payload owned by the same execution or operation: + +| Payload | Entity ID | +| --- | --- | +| Root input | `/input` | +| Root output | `/output` | +| Root exception | `/exception` | +| Invoke request | `/invoke-payload` | +| Operation result or state | `/result` | +| Operation exception | `/exception` | + +Custom external-storage SerDes implementations can therefore use `entityId` as part of a deterministic key without a +result or exception overwriting an invoke request or prior operation state. + ## Execution and caching SerDes calls execute inline by default. Configure a dedicated executor when serialization performs blocking filesystem diff --git a/docs/design.md b/docs/design.md index 474e0a1fc..c3ab7144e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -669,6 +669,10 @@ work to the configured SerDes executor, installs a `SerDesContext` in plain thre call, and caches successful deserializations in a bounded weak-reference LRU by SerDes identity, execution ARN, entity ID, target type, and serialized-data hash. The thread-local value is always restored in `finally`. +The entity ID combines the execution or operation ID with a payload-kind suffix: `/input`, `/output`, or `/exception` +for root execution payloads, and `/invoke-payload`, `/result`, or `/exception` for operation payloads. Distinct durable +payloads therefore remain distinct even for a custom SerDes that uses deterministic external-storage keys. + `FileSystemSerDes` uses that context to build collision-free paths on a shared durable filesystem. Calls made before a durable execution ARN exists, such as initial invocation input serialization, fall back to the delegate SerDes without filesystem storage. diff --git a/docs/wire-formats/filesystem-serdes.md b/docs/wire-formats/filesystem-serdes.md index 70868e467..975296831 100644 --- a/docs/wire-formats/filesystem-serdes.md +++ b/docs/wire-formats/filesystem-serdes.md @@ -74,6 +74,14 @@ Malformed UTF-8 file content and filesystem read failures are `RetryableSerDesEx All paths are rooted under the configured absolute base path. +The entity ID used for path construction identifies both the durable owner and payload kind: + +- root input, output, and exceptions use `/input`, `/output`, and `/exception`; +- invoke requests use `/invoke-payload`; +- operation results/state and exceptions use `/result` and `/exception`. + +These suffixes prevent different payloads belonging to one operation from colliding in deterministic external storage. + ### URI encoding For a durable execution ARN matching: diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index a17056d70..3fc37abb9 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -13,6 +13,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import org.junit.jupiter.api.Test; @@ -364,6 +366,47 @@ void customExceptionPayloadRoundTripsThroughFilesystem() throws Exception { assertEnvelopePointsToFile(operationError.errorData()); } + @Test + void payloadKindEntityIdsPreserveDeterministicExternalStateAcrossReplay() { + var attempts = new AtomicInteger(); + var serDes = new DeterministicExternalSerDes(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.waitForCondition( + "poll", + String.class, + (state, stepContext) -> { + if (attempts.incrementAndGet() == 1) { + return WaitForConditionResult.continuePolling("checkpoint-state"); + } + throw new IllegalStateException("poll failed"); + }, + WaitForConditionConfig.builder() + .waitStrategy(WaitStrategies.exponentialBackoff( + 3, + Duration.ofSeconds(1), + Duration.ofSeconds(10), + 1, + JitterStrategy.NONE)) + .build()), + config) + .withOutputType(String.class); + + var pending = runner.run("input"); + var stateReference = pending.getOperation("poll").getStepDetails().result(); + + assertEquals(ExecutionStatus.PENDING, pending.getStatus()); + runner.advanceTime(); + + var failed = runner.run("input"); + + assertEquals(ExecutionStatus.FAILED, failed.getStatus()); + assertEquals("checkpoint-state", serDes.deserialize(stateReference, TypeToken.get(String.class))); + assertTrue(serDes.keys().stream().anyMatch(key -> key.endsWith("/result"))); + assertTrue(serDes.keys().stream().anyMatch(key -> key.endsWith("/exception"))); + } + private void assertEnvelopePointsToFile(String envelope) throws Exception { var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); assertTrue(Files.exists(file)); @@ -402,6 +445,40 @@ record CrossInvokeRequest(String value) {} record CrossInvokeResponse(String value) {} + private static final class DeterministicExternalSerDes implements SerDes { + private static final String REFERENCE_PREFIX = "external:"; + private final JacksonSerDes delegate = new JacksonSerDes(); + private final ConcurrentHashMap storage = new ConcurrentHashMap<>(); + + @Override + public String serialize(Object value) { + var context = SerDesContext.getCurrentContext(); + if (context == null) { + return delegate.serialize(value); + } + var key = context.durableExecutionArn() + "#" + context.entityId(); + storage.put(key, delegate.serialize(value)); + return REFERENCE_PREFIX + key; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + var serialized = data; + if (data != null && data.startsWith(REFERENCE_PREFIX)) { + var key = data.substring(REFERENCE_PREFIX.length()); + serialized = storage.get(key); + if (serialized == null) { + throw new IllegalStateException("Missing external value: " + key); + } + } + return delegate.deserialize(serialized, typeToken); + } + + Set keys() { + return Set.copyOf(storage.keySet()); + } + } + public static class CustomFailure extends RuntimeException { public CustomFailure() {} diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java index 4d15cb215..24675a4b7 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java @@ -137,7 +137,10 @@ public T getStepResult(TypeToken type) { return serDesRunner == null ? serDes.deserialize(details.result(), type) : serDesRunner.deserialize( - serDes, details.result(), type, new SerDesContext(durableExecutionArn, operation.id())); + serDes, + details.result(), + type, + new SerDesContext(durableExecutionArn, operation.id() + "/result")); } /** Returns the step error, or null if the step succeeded or this is not a step operation. */ diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java index 195348334..429ef1e6d 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java @@ -70,7 +70,8 @@ public TestResult( this.serDes = serDes; this.resultType = resultType; this.serDesRunner = serDesRunner; - this.outputContext = serDesRunner == null ? null : new SerDesContext(durableExecutionArn, executionOperationId); + this.outputContext = + serDesRunner == null ? null : new SerDesContext(durableExecutionArn, executionOperationId + "/output"); } /** Returns the execution status (SUCCEEDED, FAILED, or PENDING). */ diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index a382bfac5..c324d3b4d 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -144,7 +144,10 @@ public T deserialize(String data, TypeToken typeToken) { result.getOperation("step").getStepResult(String.class); assertTrue(contexts.stream().allMatch(context -> context.durableExecutionArn() != null)); - assertTrue(contexts.stream().map(SerDesContext::entityId).distinct().count() >= 2); + var entityIds = contexts.stream().map(SerDesContext::entityId).toList(); + assertTrue(entityIds.stream().anyMatch(entityId -> entityId.endsWith("/input"))); + assertTrue(entityIds.stream().anyMatch(entityId -> entityId.endsWith("/output"))); + assertTrue(entityIds.stream().anyMatch(entityId -> entityId.endsWith("/result"))); } @Test diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java index b18417f76..32ffe0d25 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java @@ -49,6 +49,6 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals("step-result", testOperation.getStepResult(String.class)); assertEquals(EXECUTION_ARN, observedContext.get().durableExecutionArn()); - assertEquals("step-id", observedContext.get().entityId()); + assertEquals("step-id/result", observedContext.get().entityId()); } } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java index ba9082e3b..5d19045a9 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -108,7 +108,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { "invoke-result", result.getOperation("invoke").getChainedInvokeDetails().result()); assertEquals( - List.of("invocation-id", "step-id"), + List.of("invocation-id/output", "step-id/result"), observedContexts.stream().map(SerDesContext::entityId).toList()); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 9449e26ce..5144aa1e1 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -62,8 +62,10 @@ public static DurableExecutionOutput execute( var isFirstInvocation = !executionManager.isReplaying(); var requestId = lambdaContext != null ? lambdaContext.getAwsRequestId() : null; var executionArn = input.durableExecutionArn(); - var serDesContext = new SerDesContext( - executionArn, executionManager.getExecutionOperation().id()); + var executionOperationId = executionManager.getExecutionOperation().id(); + var inputSerDesContext = new SerDesContext(executionArn, executionOperationId + "/input"); + var outputSerDesContext = new SerDesContext(executionArn, executionOperationId + "/output"); + var exceptionSerDesContext = new SerDesContext(executionArn, executionOperationId + "/exception"); var serDesRunner = executionManager.getSerDesRunner(); executionManager.registerActiveThread(null); @@ -87,7 +89,7 @@ public static DurableExecutionOutput execute( config.getSerDes(), inputType, serDesRunner, - serDesContext); + inputSerDesContext); } catch (Throwable t) { inputFailure = t; } @@ -180,12 +182,12 @@ public static DurableExecutionOutput execute( cause, pluginExecutionInput.get(), null); - return DurableExecutionOutput.failure( - buildErrorObject(cause, config.getSerDes(), serDesRunner, serDesContext)); + return DurableExecutionOutput.failure(buildErrorObject( + cause, config.getSerDes(), serDesRunner, exceptionSerDesContext)); } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = serDesRunner.serialize(config.getSerDes(), result, serDesContext); + var outputPayload = serDesRunner.serialize(config.getSerDes(), result, outputSerDesContext); var output = DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); fireOnInvocationEnd( diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index b8b4a1706..92cbe1cb3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -115,9 +115,9 @@ public String getName() { return operationIdentifier.name(); } - /** Returns the context used for SerDes calls belonging to this operation. */ - protected SerDesContext getSerDesContext() { - return new SerDesContext(executionManager.getDurableExecutionArn(), getOperationId()); + /** Returns the context used for one durable payload belonging to this operation. */ + protected SerDesContext getSerDesContext(String payloadKind) { + return new SerDesContext(executionManager.getDurableExecutionArn(), getOperationId() + "/" + payloadKind); } /** Returns the invocation-scoped SerDes runner. */ diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index 421123303..f99b6f514 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -70,7 +70,7 @@ private void startInvocation() { .functionName(functionName) .tenantId(invokeConfig.tenantId()) .build()) - .payload(getSerDesRunner().serialize(payloadSerDes, this.payload, getSerDesContext())); + .payload(getSerDesRunner().serialize(payloadSerDes, this.payload, getSerDesContext("invoke-payload"))); sendOperationUpdate(update); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index a8b7c821e..eba38b72b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -86,7 +86,7 @@ protected SerializableDurableOperation( */ protected T deserializeResult(String result) { try { - return getSerDesRunner().deserialize(resultSerDes, result, resultTypeToken, getSerDesContext()); + return getSerDesRunner().deserialize(resultSerDes, result, resultTypeToken, getSerDesContext("result")); } catch (SerDesException e) { logger.warn( "Failed to deserialize {} result for operation name '{}'. Ensure the result is properly encoded.", @@ -106,7 +106,7 @@ protected T deserializeResult(String result) { * @return the serialized string and the deserialized result */ protected SerializedResult serializeAndDeserializeResult(T result) { - var serialized = getSerDesRunner().serialize(resultSerDes, result, getSerDesContext()); + var serialized = getSerDesRunner().serialize(resultSerDes, result, getSerDesContext("result")); var deserialized = shouldDeserializeAfterSerialization() ? deserializeResult(serialized) : result; return new SerializedResult<>(serialized, deserialized); } @@ -119,7 +119,7 @@ protected SerializedResult serializeAndDeserializeResult(T result) { */ @SuppressWarnings("ThrowableNotThrown") protected ErrorObject serializeException(Throwable throwable) { - var errorData = getSerDesRunner().serialize(resultSerDes, throwable, getSerDesContext()); + var errorData = getSerDesRunner().serialize(resultSerDes, throwable, getSerDesContext("exception")); var error = ExceptionHelper.buildErrorObject(throwable, errorData); if (shouldDeserializeAfterSerialization()) { deserializeException(error); @@ -159,7 +159,7 @@ protected Throwable deserializeException(ErrorObject errorObject) { resultSerDes, errorData, TypeToken.get(exceptionClass.asSubclass(Throwable.class)), - getSerDesContext()); + getSerDesContext("exception")); if (original != null) { original.setStackTrace(ExceptionHelper.deserializeStackTrace(errorObject.stackTrace())); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java index caa025079..e07897054 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java @@ -65,7 +65,7 @@ private static Map buildPreview(JsonNode root, PreviewConfig con } var pairs = new ArrayList(); - collect(root, "", config, pairs); + collect(root, "", config, pairs, false); if (pairs.isEmpty()) { return null; } @@ -82,13 +82,18 @@ private static Map buildPreview(JsonNode root, PreviewConfig con return result.isEmpty() ? null : result; } - private static void collect(JsonNode node, String pathPrefix, PreviewConfig config, List pairs) { + private static void collect( + JsonNode node, + String pathPrefix, + PreviewConfig config, + List pairs, + boolean inheritedInclusion) { if (node == null || node.isNull()) { return; } if (node.isArray()) { for (var item : node) { - collect(item, pathPrefix, config, pairs); + collect(item, pathPrefix, config, pairs, inheritedInclusion); } return; } @@ -104,12 +109,12 @@ private static void collect(JsonNode node, String pathPrefix, PreviewConfig conf var path = pathPrefix.isEmpty() ? name : pathPrefix + "." + name; var masked = isMatched(path, config.mask()); var excluded = isMatched(path, config.exclude()); - var visible = !excluded - && (masked || config.mode() == PreviewMode.INCLUDE_ALL || isMatched(path, config.include())); + var included = inheritedInclusion || isMatched(path, config.include()); + var visible = !excluded && (masked || config.mode() == PreviewMode.INCLUDE_ALL || included); if (!visible) { if (!excluded) { - collect(field.getValue(), path, config, pairs); + collect(field.getValue(), path, config, pairs, false); } continue; } @@ -118,7 +123,7 @@ private static void collect(JsonNode node, String pathPrefix, PreviewConfig conf } else if (isScalarArray(field.getValue())) { pairs.add(new PreviewEntry(path, MAPPER.convertValue(field.getValue(), Object.class))); } else if (field.getValue().isContainerNode()) { - collect(field.getValue(), path, config, pairs); + collect(field.getValue(), path, config, pairs, included); } else { pairs.add(new PreviewEntry(path, MAPPER.convertValue(field.getValue(), Object.class))); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 3ac6cd6df..8abde304b 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -4,9 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.ChainedInvokeDetails; @@ -27,6 +31,7 @@ import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDesContext; class InvokeOperationTest { private static final String OPERATION_ID = "2"; @@ -71,6 +76,49 @@ void getDoesNotThrowWhenCalledFromHandlerContext() { assertEquals("cached-result", result); } + @Test + void invokePayloadAndResultUseDistinctPayloadEntityIds() { + var contexts = new CopyOnWriteArrayList(); + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + contexts.add(SerDesContext.getCurrentContext()); + return super.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + contexts.add(SerDesContext.getCurrentContext()); + return super.deserialize(data, typeToken); + } + }; + var completed = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.SUCCEEDED) + .chainedInvokeDetails(ChainedInvokeDetails.builder() + .result("\"cached-result\"") + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(completed); + when(executionManager.sendOperationUpdate(any())).thenReturn(CompletableFuture.completedFuture(null)); + var operation = new InvokeOperation<>( + OPERATION_IDENTIFIER, + "test-function", + "payload", + TypeToken.get(String.class), + InvokeConfig.builder().serDes(serDes).build(), + durableContext); + + operation.start(); + operation.onCheckpointComplete(completed); + + assertEquals("cached-result", operation.get()); + assertEquals( + List.of("2/invoke-payload", "2/result"), + contexts.stream().map(SerDesContext::entityId).toList()); + } + @Test void getInvokeFailedExceptionWhenInvocationFailed() { var op = Operation.builder() diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index 4302f8068..ae4d2858e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -15,6 +15,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,6 +44,7 @@ import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesRunner; class SerializableDurableOperationTest { @@ -552,6 +555,45 @@ public String get() { op.get(); } + @Test + void resultAndExceptionUseDistinctPayloadEntityIds() { + var contexts = new CopyOnWriteArrayList(); + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + contexts.add(SerDesContext.getCurrentContext()); + return super.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + contexts.add(SerDesContext.getCurrentContext()); + return super.deserialize(data, typeToken); + } + }; + SerializableDurableOperation op = + new SerializableDurableOperation<>(OPERATION_IDENTIFIER, RESULT_TYPE, serDes, durableContext) { + @Override + protected void start() {} + + @Override + protected void replay(Operation existing) {} + + @Override + public String get() { + serializeAndDeserializeResult("result"); + serializeException(new RuntimeException("failure")); + return RESULT; + } + }; + + op.get(); + + assertEquals( + List.of("1/result", "1/result", "1/exception", "1/exception"), + contexts.stream().map(SerDesContext::entityId).toList()); + } + @Test void polling() { SerializableDurableOperation op = diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java index 32936c123..2b0d86f1e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java @@ -83,6 +83,47 @@ void pathAndAnywhereMatchingHaveDifferentScopes() { assertEquals("nested@example.com", nested(anywherePreview, "user").get("email")); } + @Test + void selectedObjectIncludesDescendantsWhileApplyingNestedRules() { + var value = Map.of( + "customer", + Map.of( + "name", "Alice", + "email", "alice@example.com", + "secret", "hidden"), + "ignored", + "value"); + var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.path("customer")) + .exclude(PreviewField.path("customer.secret")) + .mask(PreviewField.path("customer.email")) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(Map.of("name", "Alice", "email", "***"), nested(preview, "customer")); + assertFalse(preview.containsKey("ignored")); + } + + @Test + void selectedObjectArrayIncludesFlattenedDescendants() { + var value = Map.of( + "items", + List.of(Map.of("id", "first", "secret", "hidden"), Map.of("email", "second@example.com")), + "ignored", + "value"); + var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.path("items")) + .exclude(PreviewField.path("items.secret")) + .mask(PreviewField.path("items.email")) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(Map.of("id", "first", "email", "***"), nested(preview, "items")); + assertFalse(preview.containsKey("ignored")); + } + @Test void arraysMergeFieldsAtTheirContainingPath() { var value = Map.of("items", List.of(Map.of("id", "first"), Map.of("email", "second@example.com"))); From 124b87c108ee0ef5343aa357fc275d12d7ee424e Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 02:41:25 +0000 Subject: [PATCH 07/11] fix: resolve remaining filesystem SerDes review issues --- docs/adr/005-filesystem-serdes.md | 4 + docs/advanced/serdes.md | 7 +- docs/advanced/testing.md | 12 +++ .../FileSystemSerDesIntegrationTest.java | 15 ++-- .../durable/testing/AsyncExecution.java | 18 +++- .../testing/CloudDurableTestRunner.java | 82 ++++++++++++++++--- .../testing/LocalDurableTestRunner.java | 36 ++++++-- .../testing/OperationSerDesResolver.java | 22 +++++ .../testing/cloud/HistoryEventProcessor.java | 24 +++++- .../local/LocalMemoryExecutionClient.java | 27 +++++- .../cloud/HistoryEventProcessorTest.java | 50 +++++++++++ .../durable/serde/FileSystemSerDes.java | 35 ++++++-- .../lambda/durable/serde/SerDesRunner.java | 32 +++++--- .../durable/serde/FileSystemSerDesTest.java | 20 ++++- .../durable/serde/SerDesRunnerTest.java | 32 ++++++++ 15 files changed, 366 insertions(+), 50 deletions(-) create mode 100644 sdk-testing/src/main/java/software/amazon/lambda/durable/testing/OperationSerDesResolver.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 7069f2e6d..41a875a24 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -86,6 +86,10 @@ The serialized-data hash prevents stale results when the same entity is updated, `waitForCondition` state. Concurrent callers share one in-flight deserialization. Failed deserializations are removed from the cache and can be retried. +Each SerDes/context pair also has an invocation-local serialization generation. The runner advances it after every +serialization attempt, and completed/in-flight cache keys include the current generation. Reusing the same deterministic +external reference after writing new state therefore cannot return the previous cached value. + Repeated reads return the same object instance while the cached value remains reachable. A new invocation creates a new runner and cache. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index 8bdcdba1e..d683e1688 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -81,7 +81,9 @@ can deadlock. Each Lambda invocation owns a `SerDesRunner`. It shares concurrent reads and keeps up to 256 successful deserializations in a weak-reference LRU cache. Cache identity includes the SerDes instance, execution ARN, entity ID, -target type, and serialized payload hash. +target type, serialized payload hash, and the entity's serialization generation. Every serialization advances that +generation after the SerDes call finishes, so a deterministic external reference that is reused for new state cannot +return an older cached object. Local and cloud testing utilities propagate the same contexts and caching behavior through `TestResult`, `TestOperation`, history processing, and asynchronous execution snapshots. @@ -183,7 +185,8 @@ var retryingSerDes = new RetrySerDes( ``` Only `RetryableSerDesException` is retried. Permanent `SerDesException` failures—malformed envelopes, invalid digests, -or incompatible data—fail immediately. Retry delays block the calling thread or the configured SerDes executor thread. +incompatible data, or symbolic-link violations—fail immediately. Retry delays block the calling thread or the +configured SerDes executor thread. ## Testing diff --git a/docs/advanced/testing.md b/docs/advanced/testing.md index 47b986475..29dd2dbd4 100644 --- a/docs/advanced/testing.md +++ b/docs/advanced/testing.md @@ -56,6 +56,18 @@ List succeeded = result.getSucceededOperations(); List failed = result.getFailedOperations(); ``` +When an operation uses a `serDes(...)` override, configure an operation resolver so inspection uses the same SerDes +instead of the runner-wide default: + +```java +var runner = LocalDurableTestRunner.create(Order.class, handler) + .withOperationSerDesResolver((operation, defaultSerDes) -> + "process-payment".equals(operation.name()) ? paymentSerDes : defaultSerDes); +``` + +`CloudDurableTestRunner` provides the same `withOperationSerDesResolver(...)` method. The resolver is also propagated +to `AsyncExecution` history snapshots. + ### Controlling Time in Tests By default, `runUntilComplete()` skips wait durations. For testing time-dependent logic, disable this: diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 3fc37abb9..a78afab49 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -81,17 +81,20 @@ void operationConfigControlsWhereFilesystemStorageIsUsed() throws Exception { var fileSystemSerDes = FileSystemSerDes.builder(tempDir).build(); var config = DurableConfig.builder().withSerDes(new JacksonSerDes()).build(); var runner = LocalDurableTestRunner.create( - String.class, - (input, context) -> context.step( - "filesystem-step", String.class, - stepContext -> input + "-stored", - StepConfig.builder().serDes(fileSystemSerDes).build()), - config); + (input, context) -> context.step( + "filesystem-step", + String.class, + stepContext -> input + "-stored", + StepConfig.builder().serDes(fileSystemSerDes).build()), + config) + .withOperationSerDesResolver((operation, defaultSerDes) -> + "filesystem-step".equals(operation.name()) ? fileSystemSerDes : defaultSerDes); var result = runner.runUntilComplete("value"); assertEquals("value-stored", result.getResult(String.class)); + assertEquals("value-stored", result.getOperation("filesystem-step").getStepResult(String.class)); try (var files = Files.walk(tempDir)) { assertEquals(1, files.filter(Files::isRegularFile).count()); } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java index 06987dcc5..97583a3e3 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java @@ -30,6 +30,7 @@ public class AsyncExecution { private final SerDes serDes; private final Duration pollInterval; private final Duration timeout; + private final OperationSerDesResolver operationSerDesResolver; private final HistoryEventProcessor processor; private List currentHistory; private TestResult currentResult; @@ -41,12 +42,25 @@ public AsyncExecution( SerDes serDes, Duration pollInterval, Duration timeout) { + this(executionArn, lambdaClient, outputType, serDes, pollInterval, timeout, OperationSerDesResolver.DEFAULT); + } + + public AsyncExecution( + String executionArn, + LambdaClient lambdaClient, + TypeToken outputType, + SerDes serDes, + Duration pollInterval, + Duration timeout, + OperationSerDesResolver operationSerDesResolver) { this.executionArn = executionArn; this.lambdaClient = lambdaClient; this.outputType = outputType; this.pollInterval = pollInterval; this.timeout = timeout; this.serDes = serDes; + this.operationSerDesResolver = + java.util.Objects.requireNonNull(operationSerDesResolver, "operationSerDesResolver cannot be null"); this.processor = new HistoryEventProcessor(); } @@ -196,8 +210,8 @@ private void refreshHistory() { .build(); var response = lambdaClient.getDurableExecutionHistory(request); this.currentHistory = response.events(); - this.currentResult = - processor.processEvents(currentHistory, outputType, serDes, new SerDesRunner(null), executionArn); + this.currentResult = processor.processEvents( + currentHistory, outputType, serDes, new SerDesRunner(null), executionArn, operationSerDesResolver); } catch (ResourceNotFoundException e) { // Execution doesn't exist yet - this can happen immediately after async invoke // Leave currentHistory as null, pollUntil will retry diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 5e2d50e42..7ba701e59 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -32,6 +32,7 @@ public class CloudDurableTestRunner { private final Duration timeout; private final InvocationType invocationType; private final SerDes serDes; + private final OperationSerDesResolver operationSerDesResolver; // Store last execution result for operation inspection private TestResult lastResult; @@ -43,7 +44,8 @@ private CloudDurableTestRunner( Duration pollInterval, Duration timeout, InvocationType invocationType, - SerDes serDes) { + SerDes serDes, + OperationSerDesResolver operationSerDesResolver) { this.functionArn = functionArn; this.inputType = inputType; this.outputType = outputType; @@ -53,6 +55,8 @@ private CloudDurableTestRunner( this.timeout = timeout; this.invocationType = invocationType; this.serDes = Objects.requireNonNullElseGet(serDes, JacksonSerDes::new); + this.operationSerDesResolver = + Objects.requireNonNull(operationSerDesResolver, "operationSerDesResolver cannot be null"); } private static LambdaClient createDefaultLambdaClient() { @@ -78,7 +82,8 @@ public static CloudDurableTestRunner create( Duration.ofSeconds(2), Duration.ofSeconds(300), InvocationType.REQUEST_RESPONSE, - null); + null, + OperationSerDesResolver.DEFAULT); } /** Creates a runner with a custom {@link LambdaClient} and Class-based input/output types. */ @@ -98,36 +103,91 @@ public static CloudDurableTestRunner create( Duration.ofSeconds(2), Duration.ofSeconds(300), InvocationType.REQUEST_RESPONSE, - null); + null, + OperationSerDesResolver.DEFAULT); } /** Returns a new runner with the specified lambda client. */ public CloudDurableTestRunner withLambdaClient(LambdaClient lambdaClient) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + serDes, + operationSerDesResolver); } /** Returns a new runner with the specified poll interval between history checks. */ public CloudDurableTestRunner withPollInterval(Duration interval) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, interval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + interval, + timeout, + invocationType, + serDes, + operationSerDesResolver); } /** Returns a new runner with the specified maximum wait time for execution completion. */ public CloudDurableTestRunner withTimeout(Duration timeout) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + serDes, + operationSerDesResolver); } /** Returns a new runner with the specified Lambda invocation type. */ public CloudDurableTestRunner withInvocationType(InvocationType type) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, type, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + type, + serDes, + operationSerDesResolver); } public CloudDurableTestRunner withSerDes(SerDes serDes) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + serDes, + operationSerDesResolver); + } + + /** Resolves operation-specific SerDes overrides when inspecting persisted operation results. */ + public CloudDurableTestRunner withOperationSerDesResolver(OperationSerDesResolver resolver) { + return new CloudDurableTestRunner<>( + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + serDes, + Objects.requireNonNull(resolver)); } /** Invokes the Lambda function, polls execution history until completion, and returns the result. */ @@ -162,7 +222,8 @@ public TestResult run(I input) { // Process events into TestResult var processor = new HistoryEventProcessor(); - var result = processor.processEvents(events, outputType, serDes, new SerDesRunner(null), executionArn); + var result = processor.processEvents( + events, outputType, serDes, new SerDesRunner(null), executionArn, operationSerDesResolver); this.lastResult = result; return result; } catch (Exception e) { @@ -201,7 +262,8 @@ public AsyncExecution startAsync(I input) { // This prevents immediate polling from failing with "execution does not exist" Thread.sleep(100); - return new AsyncExecution<>(executionArn, lambdaClient, outputType, serDes, pollInterval, timeout); + return new AsyncExecution<>( + executionArn, lambdaClient, outputType, serDes, pollInterval, timeout, operationSerDesResolver); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted while starting async execution", e); diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index bc0803917..65262f8b2 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -6,6 +6,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.UUID; import java.util.function.BiFunction; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; @@ -43,6 +44,7 @@ public class LocalDurableTestRunner { private final LocalMemoryExecutionClient storage; private final SerDes serDes; private final DurableConfig customerConfig; + private final OperationSerDesResolver operationSerDesResolver; private final Instant executionStartTime = Instant.now(); // The execution identity is fixed for the whole execution, matching the backend: the ARN and the EXECUTION // operation ID stay stable across reinvocations, while only per-invocation values (the checkpoint token) change. @@ -57,10 +59,21 @@ private LocalDurableTestRunner( TypeToken outputType, BiFunction handlerFn, DurableConfig customerConfig) { + this(inputType, outputType, handlerFn, customerConfig, OperationSerDesResolver.DEFAULT); + } + + private LocalDurableTestRunner( + TypeToken inputType, + TypeToken outputType, + BiFunction handlerFn, + DurableConfig customerConfig, + OperationSerDesResolver operationSerDesResolver) { this.inputType = inputType; this.outputType = outputType; this.handler = handlerFn; this.storage = new LocalMemoryExecutionClient(); + this.operationSerDesResolver = + Objects.requireNonNull(operationSerDesResolver, "operationSerDesResolver cannot be null"); // Create config that uses customer's configuration but overrides the client with in-memory storage if (customerConfig != null) { @@ -199,17 +212,24 @@ public static LocalDurableTestRunner create(Class inputType, Dur * a new runner instance. */ public LocalDurableTestRunner withDurableConfig(DurableConfig config) { - return new LocalDurableTestRunner<>(inputType, outputType, handler, config); + return new LocalDurableTestRunner<>(inputType, outputType, handler, config, operationSerDesResolver); } /** Overrides the output type for this test runner. */ public LocalDurableTestRunner withOutputType(TypeToken outputType) { - return new LocalDurableTestRunner<>(inputType, outputType, handler, customerConfig); + return new LocalDurableTestRunner<>(inputType, outputType, handler, customerConfig, operationSerDesResolver); } /** Overrides the output type for this test runner. */ public LocalDurableTestRunner withOutputType(Class outputType) { - return new LocalDurableTestRunner<>(inputType, TypeToken.get(outputType), handler, customerConfig); + return new LocalDurableTestRunner<>( + inputType, TypeToken.get(outputType), handler, customerConfig, operationSerDesResolver); + } + + /** Resolves operation-specific SerDes overrides when inspecting persisted operation results. */ + public LocalDurableTestRunner withOperationSerDesResolver(OperationSerDesResolver resolver) { + return new LocalDurableTestRunner<>( + inputType, outputType, handler, customerConfig, Objects.requireNonNull(resolver)); } /** @@ -255,7 +275,8 @@ public TestResult run(I input) { var output = DurableExecutor.execute(durableInput, mockLambdaContext(), inputType, handler, customerConfig); - return storage.toTestResult(output, outputType, serDes, serDesRunner, executionArn, executionOperationId); + return storage.toTestResult( + output, outputType, serDes, serDesRunner, executionArn, executionOperationId, operationSerDesResolver); } /** @@ -298,12 +319,17 @@ public TestOperation getOperation(String name) { ? new TestOperation( op, List.of(), - serDes, + resolveOperationSerDes(op), new SerDesRunner(customerConfig.getSerDesExecutorService()), executionArn) : null; } + private SerDes resolveOperationSerDes(Operation operation) { + return Objects.requireNonNull( + operationSerDesResolver.resolve(operation, serDes), "operationSerDesResolver returned null"); + } + /** Get callback ID for a named callback operation. */ public String getCallbackId(String operationName) { return storage.getCallbackId(operationName); diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/OperationSerDesResolver.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/OperationSerDesResolver.java new file mode 100644 index 000000000..c41091cbb --- /dev/null +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/OperationSerDesResolver.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing; + +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.lambda.durable.serde.SerDes; + +/** Resolves the SerDes used to inspect a durable operation's persisted result. */ +@FunctionalInterface +public interface OperationSerDesResolver { + /** Uses the runner-wide SerDes for every operation. */ + OperationSerDesResolver DEFAULT = (operation, defaultSerDes) -> defaultSerDes; + + /** + * Resolves the effective SerDes for an operation. + * + * @param operation operation being inspected + * @param defaultSerDes runner-wide SerDes + * @return the SerDes that encoded this operation's result + */ + SerDes resolve(Operation operation, SerDes defaultSerDes); +} diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java index 13663efe7..02dd6d161 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java @@ -23,6 +23,7 @@ import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.AsyncExecution; import software.amazon.lambda.durable.testing.CloudDurableTestRunner; +import software.amazon.lambda.durable.testing.OperationSerDesResolver; import software.amazon.lambda.durable.testing.TestOperation; import software.amazon.lambda.durable.testing.TestResult; @@ -40,7 +41,7 @@ public class HistoryEventProcessor { * @return a TestResult containing the execution status, output, and operation details */ public TestResult processEvents(List events, TypeToken outputType, SerDes serDes) { - return processEvents(events, outputType, serDes, null, null); + return processEvents(events, outputType, serDes, null, null, OperationSerDesResolver.DEFAULT); } /** @@ -55,6 +56,22 @@ public TestResult processEvents( SerDes serDes, SerDesRunner serDesRunner, String durableExecutionArn) { + return processEvents( + events, outputType, serDes, serDesRunner, durableExecutionArn, OperationSerDesResolver.DEFAULT); + } + + /** + * Processes history with operation-specific SerDes resolution for persisted result inspection. + * + * @param operationSerDesResolver resolves the effective SerDes for each operation + */ + public TestResult processEvents( + List events, + TypeToken outputType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn, + OperationSerDesResolver operationSerDesResolver) { var operations = new HashMap(); var operationEvents = new HashMap>(); var status = ExecutionStatus.PENDING; @@ -283,7 +300,10 @@ public TestResult processEvents( for (var entry : operations.entrySet()) { var opEvents = operationEvents.getOrDefault(entry.getKey(), List.of()); var operation = withEventTimestamps(entry.getValue(), opEvents); - testOperations.add(new TestOperation(operation, opEvents, serDes, serDesRunner, durableExecutionArn)); + var operationSerDes = Objects.requireNonNull( + operationSerDesResolver.resolve(operation, serDes), "operationSerDesResolver returned null"); + testOperations.add( + new TestOperation(operation, opEvents, operationSerDes, serDesRunner, durableExecutionArn)); } return new TestResult<>( diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java index 0cdc85f43..e743039db 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java @@ -25,6 +25,7 @@ import software.amazon.lambda.durable.model.DurableExecutionOutput; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.testing.OperationSerDesResolver; import software.amazon.lambda.durable.testing.TestOperation; import software.amazon.lambda.durable.testing.TestResult; @@ -143,10 +144,34 @@ public TestResult toTestResult( SerDesRunner serDesRunner, String durableExecutionArn, String executionOperationId) { + return toTestResult( + output, + resultType, + serDes, + serDesRunner, + durableExecutionArn, + executionOperationId, + OperationSerDesResolver.DEFAULT); + } + + /** Build TestResult with operation-specific SerDes resolution. */ + public TestResult toTestResult( + DurableExecutionOutput output, + TypeToken resultType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn, + String executionOperationId, + OperationSerDesResolver operationSerDesResolver) { var testOperations = existingOperations.values().stream() .filter(op -> op.type() != OperationType.EXECUTION) .map(op -> new TestOperation( - op, eventProcessor.getEventsForOperation(op.id()), serDes, serDesRunner, durableExecutionArn)) + op, + eventProcessor.getEventsForOperation(op.id()), + java.util.Objects.requireNonNull( + operationSerDesResolver.resolve(op, serDes), "operationSerDesResolver returned null"), + serDesRunner, + durableExecutionArn)) .toList(); return new TestResult<>( output.status(), diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java index 5d19045a9..e2394cd67 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -112,6 +112,56 @@ void deserializesCloudResultsWithDurablePayloadContext() { observedContexts.stream().map(SerDesContext::entityId).toList()); } + @Test + void resolvesOperationSpecificSerDesForCloudHistory() { + var startedAt = Instant.parse("2026-08-24T00:00:00Z"); + var events = List.of( + Event.builder() + .id("step-id") + .name("custom-step") + .subType("Step") + .eventType(EventType.STEP_STARTED) + .eventTimestamp(startedAt) + .stepStartedDetails(StepStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("custom-step") + .subType("Step") + .eventType(EventType.STEP_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(1)) + .stepSucceededDetails(StepSucceededDetails.builder() + .result(EventResult.builder() + .payload("custom:step-result") + .build()) + .build()) + .build()); + var customSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return "custom:" + value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data.substring("custom:".length()); + } + }; + + var result = new HistoryEventProcessor() + .processEvents( + events, + TypeToken.get(String.class), + recordingStringSerDes(new ArrayList<>()), + new SerDesRunner(null), + EXECUTION_ARN, + (operation, defaultSerDes) -> + "custom-step".equals(operation.name()) ? customSerDes : defaultSerDes); + + assertEquals("step-result", result.getOperation("custom-step").getStepResult(String.class)); + } + private static SerDes recordingStringSerDes(List observedContexts) { return new SerDes() { @Override diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index f14afec81..6dd670426 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -20,6 +20,7 @@ import java.nio.file.Path; import java.nio.file.SecureDirectoryStream; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributeView; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -292,6 +293,7 @@ private void writePayload(Path file, String serialized) { try { try (var secureDirectory = openSecureDirectory(file.getParent(), true)) { var created = false; + rejectSymbolicLinkIfPresent(secureDirectory.directory(), file.getFileName(), "payload file"); try (var channel = secureDirectory .directory() .newByteChannel( @@ -330,13 +332,16 @@ private String readPayload(String fileValue) { } try { byte[] storedData; - try (var secureDirectory = openSecureDirectory(file.getParent(), false); - var channel = secureDirectory - .directory() - .newByteChannel( - file.getFileName(), Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)); - var input = Channels.newInputStream(channel)) { - storedData = input.readAllBytes(); + try (var secureDirectory = openSecureDirectory(file.getParent(), false)) { + rejectSymbolicLinkIfPresent(secureDirectory.directory(), file.getFileName(), "payload file"); + try (var channel = secureDirectory + .directory() + .newByteChannel( + file.getFileName(), + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)); + var input = Channels.newInputStream(channel)) { + storedData = input.readAllBytes(); + } } return StandardCharsets.UTF_8 .newDecoder() @@ -365,6 +370,7 @@ private SecureDirectoryHandle openSecureDirectory(Path directory, boolean create for (var component : root.relativize(directory)) { var nextPath = currentPath.resolve(component); DirectoryStream next; + rejectSymbolicLinkIfPresent(current, component, "directory"); try { next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); } catch (NoSuchFileException missing) { @@ -388,6 +394,21 @@ private SecureDirectoryHandle openSecureDirectory(Path directory, boolean create } } + private static void rejectSymbolicLinkIfPresent( + SecureDirectoryStream directory, Path entry, String description) throws IOException { + var attributes = directory.getFileAttributeView(entry, BasicFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + if (attributes == null) { + throw new SerDesException("Filesystem provider cannot inspect " + description + " without following links"); + } + try { + if (attributes.readAttributes().isSymbolicLink()) { + throw new SerDesException("Filesystem SerDes " + description + " cannot be a symbolic link"); + } + } catch (NoSuchFileException ignored) { + // Missing entries are handled by the caller as either creatable directories or retryable read failures. + } + } + @SuppressWarnings("unchecked") private static SecureDirectoryStream requireSecureDirectoryStream( DirectoryStream stream, List> openedStreams) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index 48282361c..3f58410b3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -14,6 +14,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.util.ExceptionHelper; @@ -30,6 +31,7 @@ public final class SerDesRunner { private static final Object NULL_VALUE = new Object(); private final ExecutorService executorService; + private final ConcurrentHashMap contextGenerations = new ConcurrentHashMap<>(); private final ConcurrentHashMap> inFlightDeserializations = new ConcurrentHashMap<>(); private final Map> completedDeserializations = @@ -52,7 +54,13 @@ public SerDesRunner(ExecutorService executorService) { /** Serializes a value with the supplied durable payload context. */ public String serialize(SerDes serDes, Object value, SerDesContext context) { Objects.requireNonNull(serDes, "serDes cannot be null"); - return join(submit(context, () -> serDes.serialize(value))); + Objects.requireNonNull(context, "context cannot be null"); + var contextKey = new ContextKey(serDes, context.durableExecutionArn(), context.entityId()); + try { + return join(submit(context, () -> serDes.serialize(value))); + } finally { + generation(contextKey).incrementAndGet(); + } } /** Deserializes and caches a value for the current invocation using the supplied durable payload context. */ @@ -62,7 +70,8 @@ public T deserialize(SerDes serDes, String data, TypeToken typeToken, Ser Objects.requireNonNull(typeToken, "typeToken cannot be null"); Objects.requireNonNull(context, "context cannot be null"); - var key = new CacheKey(serDes, context.durableExecutionArn(), context.entityId(), typeToken, hash(data)); + var contextKey = new ContextKey(serDes, context.durableExecutionArn(), context.entityId()); + var key = new CacheKey(contextKey, generation(contextKey).get(), typeToken, hash(data)); var cached = getCompleted(key); if (cached != null) { return cached == NULL_VALUE ? null : (T) cached; @@ -113,6 +122,10 @@ private void putCompleted(CacheKey key, Object value) { completedDeserializations.put(key, new WeakReference<>(value)); } + private AtomicLong generation(ContextKey key) { + return contextGenerations.computeIfAbsent(key, ignored -> new AtomicLong()); + } + private CompletableFuture submit(SerDesContext context, Supplier action) { Objects.requireNonNull(context, "context cannot be null"); Objects.requireNonNull(action, "action cannot be null"); @@ -147,25 +160,22 @@ private static String hash(String data) { } } - private record CacheKey( - SerDes serDes, String durableExecutionArn, String entityId, TypeToken typeToken, String dataHash) { + private record ContextKey(SerDes serDes, String durableExecutionArn, String entityId) { @Override public boolean equals(Object other) { - return other instanceof CacheKey that + return other instanceof ContextKey that && serDes == that.serDes && Objects.equals(durableExecutionArn, that.durableExecutionArn) - && Objects.equals(entityId, that.entityId) - && Objects.equals(typeToken, that.typeToken) - && Objects.equals(dataHash, that.dataHash); + && Objects.equals(entityId, that.entityId); } @Override public int hashCode() { int result = System.identityHashCode(serDes); result = 31 * result + Objects.hashCode(durableExecutionArn); - result = 31 * result + Objects.hashCode(entityId); - result = 31 * result + Objects.hashCode(typeToken); - return 31 * result + Objects.hashCode(dataHash); + return 31 * result + Objects.hashCode(entityId); } } + + private record CacheKey(ContextKey context, long generation, TypeToken typeToken, String dataHash) {} } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 763c94338..5bf32974c 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -444,8 +444,9 @@ void rejectsSymbolicLinkDirectoryWhenWriting() throws Exception { Files.createSymbolicLink(tempDir.resolve("test"), outside); var serDes = FileSystemSerDes.builder(tempDir).build(); - assertThrows( + var failure = assertThrows( SerDesException.class, () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + assertFalse(failure instanceof RetryableSerDesException); try (var files = Files.list(outside)) { assertEquals(0, files.count()); } @@ -462,9 +463,10 @@ void rejectsSymbolicLinkDirectoryWhenReading() throws Exception { Files.move(executionDirectory, movedDirectory); Files.createSymbolicLink(executionDirectory, movedDirectory); - assertThrows( + var failure = assertThrows( SerDesException.class, () -> runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + assertFalse(failure instanceof RetryableSerDesException); } @Test @@ -478,9 +480,19 @@ void rejectsSymbolicLinkPayloadFile() throws Exception { Files.delete(file); Files.createSymbolicLink(file, outside); - assertThrows( + var retryDecisions = new AtomicInteger(); + var retryingSerDes = new RetrySerDes( + serDes, + (failure, attempt) -> { + retryDecisions.incrementAndGet(); + return RetryDecision.retry(Duration.ZERO); + }, + delay -> {}); + var failure = assertThrows( SerDesException.class, - () -> runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + () -> runner.deserialize(retryingSerDes, envelope, TypeToken.get(String.class), context)); + assertFalse(failure instanceof RetryableSerDesException); + assertEquals(0, retryDecisions.get()); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index 03dd21eb0..720287892 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -111,6 +112,37 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals(2, calls.get()); } + @Test + void serializationInvalidatesStableExternalReferenceCache() { + var storage = new ConcurrentHashMap(); + var calls = new AtomicInteger(); + var delegate = new JacksonSerDes(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + storage.put("stable", delegate.serialize(value)); + return "reference:stable"; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return delegate.deserialize(storage.get("stable"), typeToken); + } + }; + var context = new SerDesContext("arn:test", "entity"); + + var firstReference = runner.serialize(serDes, new Value("one"), context); + var first = runner.deserialize(serDes, firstReference, TypeToken.get(Value.class), context); + var secondReference = runner.serialize(serDes, new Value("two"), context); + var second = runner.deserialize(serDes, secondReference, TypeToken.get(Value.class), context); + + assertEquals(firstReference, secondReference); + assertEquals(new Value("one"), first); + assertEquals(new Value("two"), second); + assertEquals(2, calls.get()); + } + @Test void failedDeserializationIsNotCachedAndContextIsCleared() throws Exception { var calls = new AtomicInteger(); From 61a939056d2d6d3a452bfb1d1657b5440fe1d0e3 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 04:22:14 +0000 Subject: [PATCH 08/11] fix: tighten filesystem envelope and retry classification --- docs/adr/005-filesystem-serdes.md | 3 + docs/advanced/serdes.md | 4 +- docs/wire-formats/filesystem-serdes.md | 3 +- .../durable/serde/FileSystemSerDes.java | 86 +++++++------------ .../durable/serde/FileSystemSerDesTest.java | 37 ++++++-- 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 41a875a24..04d29ad54 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -145,6 +145,9 @@ Filesystem `IOException`s are surfaced as `RetryableSerDesException`. `RetrySerD an existing `RetryStrategy` only to retryable failures. Permanent `SerDesException` failures propagate immediately. Retry delays run inline or on the configured SerDes executor. +Known structural and permission failures, including symbolic links, non-directory path components, filesystem loops, +and access denial, are permanent rather than retryable. + ### Envelope and file publication Java writes versioned envelopes: diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index d683e1688..cf29aede1 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -185,8 +185,8 @@ var retryingSerDes = new RetrySerDes( ``` Only `RetryableSerDesException` is retried. Permanent `SerDesException` failures—malformed envelopes, invalid digests, -incompatible data, or symbolic-link violations—fail immediately. Retry delays block the calling thread or the -configured SerDes executor thread. +incompatible data, symbolic-link violations, non-directory path components, or access-denied failures—fail +immediately. Retry delays block the calling thread or the configured SerDes executor thread. ## Testing diff --git a/docs/wire-formats/filesystem-serdes.md b/docs/wire-formats/filesystem-serdes.md index 975296831..9e2d2a216 100644 --- a/docs/wire-formats/filesystem-serdes.md +++ b/docs/wire-formats/filesystem-serdes.md @@ -133,7 +133,8 @@ The implementation fails closed when: - the file is missing or unreadable. Security failures represented by invalid envelope or path metadata are permanent. Filesystem I/O failures are -retryable. +retryable unless they identify a structural or permission failure such as a symbolic link, non-directory component, or +access denial. ## Cross-execution references diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 6dd670426..949c7b87d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.serde; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,11 +13,14 @@ import java.nio.channels.Channels; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystemLoopException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; +import java.nio.file.NotDirectoryException; import java.nio.file.Path; import java.nio.file.SecureDirectoryStream; import java.nio.file.StandardOpenOption; @@ -202,65 +206,31 @@ private static boolean isValidEnvelope(JsonNode node) { } private static boolean containsFilesystemMarkerField(String data) { - var index = 0; - while (index < data.length() && Character.isWhitespace(data.charAt(index))) { - index++; - } - if (index == data.length() || data.charAt(index) != '{') { - return false; - } - - var containerDepth = 1; - for (index++; index < data.length() && containerDepth > 0; index++) { - var current = data.charAt(index); - if (current == '{' || current == '[') { - containerDepth++; - } else if (current == '}' || current == ']') { - containerDepth--; - } else if (current == '"') { - var literalStart = index; - var valueStart = ++index; - var escaped = false; - while (index < data.length()) { - var literal = data.charAt(index); - if (escaped) { - escaped = false; - } else if (literal == '\\') { - escaped = true; - } else if (literal == '"') { - break; - } - index++; + try (var parser = ENVELOPE_MAPPER.createParser(data)) { + if (parser.nextToken() != JsonToken.START_OBJECT) { + return false; + } + var depth = 1; + while (parser.nextToken() != null) { + var token = parser.currentToken(); + if (token == JsonToken.FIELD_NAME && depth == 1 && ENVELOPE_MARKER.equals(parser.currentName())) { + return true; } - if (containerDepth == 1 - && index < data.length() - && isFilesystemMarkerLiteral(data, literalStart, valueStart, index)) { - var next = index + 1; - while (next < data.length() && Character.isWhitespace(data.charAt(next))) { - next++; - } - if (next < data.length() && data.charAt(next) == ':') { - return true; + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + depth++; + } else if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) { + depth--; + if (depth == 0) { + return false; } } } + } catch (IOException ignored) { + // The caller delegates malformed input unless a top-level marker field was observed before the failure. } return false; } - private static boolean isFilesystemMarkerLiteral(String data, int literalStart, int valueStart, int literalEnd) { - if (literalEnd - valueStart == ENVELOPE_MARKER.length() - && data.regionMatches(valueStart, ENVELOPE_MARKER, 0, ENVELOPE_MARKER.length())) { - return true; - } - try { - return ENVELOPE_MARKER.equals( - ENVELOPE_MAPPER.readValue(data.substring(literalStart, literalEnd + 1), String.class)); - } catch (JsonProcessingException ignored) { - return false; - } - } - private static void verifyDigest(String serialized, String expected) { if (!expected.equals(sha256(serialized))) { throw new SerDesException("Filesystem SerDes payload digest does not match stored content"); @@ -321,7 +291,7 @@ private void writePayload(Path file, String serialized) { } } } catch (IOException e) { - throw new RetryableSerDesException("Failed to store filesystem SerDes payload", e); + throw classifyFileSystemFailure("store", e); } } @@ -350,8 +320,18 @@ private String readPayload(String fileValue) { .decode(ByteBuffer.wrap(storedData)) .toString(); } catch (IOException e) { - throw new RetryableSerDesException("Failed to load filesystem SerDes payload", e); + throw classifyFileSystemFailure("load", e); + } + } + + private static SerDesException classifyFileSystemFailure(String action, IOException failure) { + var message = "Failed to " + action + " filesystem SerDes payload"; + if (failure instanceof AccessDeniedException + || failure instanceof NotDirectoryException + || failure instanceof FileSystemLoopException) { + return new SerDesException(message, failure); } + return new RetryableSerDesException(message, failure); } private SecureDirectoryHandle openSecureDirectory(Path directory, boolean createMissing) throws IOException { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 5bf32974c..e0ad82b89 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -337,7 +337,9 @@ void recognizesMalformedMarkerRegardlessOfWhitespaceOrFieldOrder() { @Test void markerTextInsideStringDoesNotClaimMalformedJson() { - var value = "{\"message\":\"__durable_execution_filesystem_serdes\"} trailing"; + var values = List.of( + "{\"message\":\"__durable_execution_filesystem_serdes\"} trailing", + "{\"message\":\"__durable_execution_filesystem_serdes\":1}"); var delegate = new SerDes() { @Override public String serialize(Object input) { @@ -351,12 +353,10 @@ public T deserialize(String data, TypeToken typeToken) { } }; - assertEquals( - value, - FileSystemSerDes.builder(tempDir) - .delegate(delegate) - .build() - .deserialize(value, TypeToken.get(String.class))); + var serDes = FileSystemSerDes.builder(tempDir).delegate(delegate).build(); + for (var value : values) { + assertEquals(value, serDes.deserialize(value, TypeToken.get(String.class))); + } } @Test @@ -422,6 +422,29 @@ void missingPayloadFileIsRetryable() throws Exception { assertThrows(RetryableSerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); } + @Test + void nonDirectoryBasePathIsPermanentAndNotRetried() throws Exception { + var baseFile = tempDir.resolve("base-file"); + Files.writeString(baseFile, "not a directory"); + var fileSystemSerDes = + FileSystemSerDes.builder(baseFile.resolve("payloads")).build(); + var retryDecisions = new AtomicInteger(); + var serDes = new RetrySerDes( + fileSystemSerDes, + (failure, attempt) -> { + retryDecisions.incrementAndGet(); + return RetryDecision.retry(Duration.ZERO); + }, + delay -> {}); + + var failure = assertThrows( + SerDesException.class, () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + + assertFalse(failure instanceof RetryableSerDesException); + assertTrue(failure.getCause() instanceof java.nio.file.NotDirectoryException); + assertEquals(0, retryDecisions.get()); + } + @Test void failsClosedWhenProviderLacksSecureDirectoryStreams() throws Exception { var archive = tempDir.resolve("payloads.zip"); From 8786279bf1fa2cd52e97adb216e82f56e8b6993c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 05:10:05 +0000 Subject: [PATCH 09/11] fix: require immutable filesystem publication --- docs/adr/005-filesystem-serdes.md | 14 +++-- docs/advanced/configuration.md | 6 +- docs/advanced/serdes.md | 12 ++-- docs/design.md | 4 +- docs/wire-formats/filesystem-serdes.md | 37 ++++-------- examples/README.md | 3 +- examples/generate-template.py | 2 +- .../examples/CloudBasedIntegrationTest.java | 2 +- examples/test_generate_template.py | 2 +- .../FileSystemSerDesIntegrationTest.java | 26 +++++++++ .../durable/serde/FileSystemSerDes.java | 53 ++++-------------- .../amazon/lambda/durable/serde/SerDes.java | 4 ++ .../durable/serde/FileSystemSerDesTest.java | 56 +++++++++---------- 13 files changed, 109 insertions(+), 112 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 04d29ad54..68969b7a1 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -46,8 +46,8 @@ in `finally`, which supports nesting and prevents context from leaking when exec Entity IDs include a stable payload-kind suffix. Root input, output, and exceptions use `/input`, `/output`, and `/exception`; operation invoke payloads, results/state, and exceptions use `/invoke-payload`, `/result`, and -`/exception`. This prevents deterministic external-storage keys for different payloads on the same operation from -colliding. +`/exception`. This prevents different payload kinds on the same operation from sharing an identity, but external +storage implementations must still publish immutable or versioned references for every serialized value. ### Use an invocation-scoped SerDesRunner @@ -88,7 +88,9 @@ from the cache and can be retried. Each SerDes/context pair also has an invocation-local serialization generation. The runner advances it after every serialization attempt, and completed/in-flight cache keys include the current generation. Reusing the same deterministic -external reference after writing new state therefore cannot return the previous cached value. +external reference after writing new state therefore cannot return the previous cached value in that invocation. This +does not make a mutable reference replay-safe: a reference already persisted in a checkpoint must continue to resolve +the same immutable content after later serialization and checkpoint failures. Repeated reads return the same object instance while the cached value remains reachable. A new invocation creates a new runner and cache. @@ -161,14 +163,15 @@ Java writes versioned envelopes: Only envelopes containing the reserved version marker are interpreted as filesystem payloads. Unmarked JSON, including objects with `data` or `file` fields, is passed to the delegate SerDes unchanged. -File names include the entity ID, serialized-payload digest, and a unique suffix. Files are created with `CREATE_NEW`, +File names include an owner digest derived from the execution ARN and entity ID, the serialized-payload digest, and a +unique suffix. Files are direct children of the pre-provisioned base directory and are created with `CREATE_NEW`, and failed writes are removed before the retryable failure is propagated. This prevents a later serialization from overwriting data referenced by an earlier checkpoint. Deserialization rejects paths outside the configured base directory and verifies the digest. The filesystem provider must support `SecureDirectoryStream`. Directory components are opened relative to held parent handles with symbolic-link following disabled, and file reads/writes use `NOFOLLOW_LINKS`. Providers without secure -directory streams fail closed. +directory streams fail closed. The SDK does not create missing directory components. ### Initial invocation input @@ -207,6 +210,7 @@ Negative: - Do not use Lambda's ephemeral `/tmp` directory. Replay may run in another execution environment. - Use a shared durable mount such as EFS. +- Pre-provision the configured base directory. - S3 Files users must accept its synchronization and crash-durability characteristics. - Verify that the mounted Java filesystem provider supports `SecureDirectoryStream`. - Configure lifecycle cleanup separately; the SDK does not delete persisted payload files. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 261ca1873..2da9a3382 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -104,12 +104,14 @@ var context = SerDesContext.getCurrentContext(); Custom SerDes implementations can use its durable execution ARN and entity ID for external storage. Calls use the configured SerDes executor when present, and successful deserializations are cached for the current Lambda invocation. Root input, output, and exception IDs end in `/input`, `/output`, and `/exception`; operation payload IDs end in -`/invoke-payload`, `/result`, or `/exception`. +`/invoke-payload`, `/result`, or `/exception`. External-storage implementations must return immutable or versioned +references rather than overwriting content reachable through an earlier checkpoint. Do not use Lambda's `/tmp` directory: replay can run in another execution environment. Use a shared durable mount such as EFS. S3 Files users must account for synchronization and crash-durability behavior. A chained-invoke boundary only requires shared storage and compatible filesystem configuration when that boundary explicitly uses `FileSystemSerDes`. -The mounted Java filesystem provider must support `SecureDirectoryStream`; unsupported providers fail closed. +The mounted Java filesystem provider must support `SecureDirectoryStream`; unsupported providers fail closed. The +configured base directory must already exist. See [Serialization and Filesystem Storage](serdes.md) for envelope details, structured previews, retry configuration, testing behavior, and operational guidance. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index cf29aede1..6137db5c2 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -61,8 +61,10 @@ Entity IDs distinguish every persisted payload owned by the same execution or op | Operation result or state | `/result` | | Operation exception | `/exception` | -Custom external-storage SerDes implementations can therefore use `entityId` as part of a deterministic key without a -result or exception overwriting an invoke request or prior operation state. +Custom external-storage SerDes implementations can use `entityId` as part of a key namespace. Every serialization must +still publish immutable content and return a unique or versioned reference. Never overwrite content reachable through +an older reference: an invocation can stop after publishing new content but before its checkpoint update commits, and +replay must continue to resolve the older checkpoint. ## Execution and caching @@ -83,7 +85,8 @@ Each Lambda invocation owns a `SerDesRunner`. It shares concurrent reads and kee deserializations in a weak-reference LRU cache. Cache identity includes the SerDes instance, execution ARN, entity ID, target type, serialized payload hash, and the entity's serialization generation. Every serialization advances that generation after the SerDes call finishes, so a deterministic external reference that is reused for new state cannot -return an older cached object. +return an older cached object during the same invocation. This cache rule does not make mutable references replay-safe; +external references must remain immutable across invocations and checkpoint failures. Local and cloud testing utilities propagate the same contexts and caching behavior through `TestResult`, `TestOperation`, history processing, and asynchronous execution snapshots. @@ -127,7 +130,7 @@ cleaned up. Each envelope includes a SHA-256 digest that is verified when the fi The mounted filesystem provider must support `SecureDirectoryStream`. The SDK traverses every directory relative to an already-open parent with symlink following disabled and performs file I/O with `NOFOLLOW_LINKS`. Providers without this -capability fail closed. +capability fail closed. The configured base directory must already exist; the SDK never creates path components. Only objects containing the reserved version marker are treated as filesystem envelopes. Ordinary JSON with `data` or `file` fields is passed to the configured delegate. @@ -163,6 +166,7 @@ delegate JSON; SDK-managed output and operation payloads can use filesystem stor - Do not use Lambda `/tmp`; replay may run in another execution environment. - Use a shared durable mount such as EFS. +- Pre-provision the configured base directory. - If using S3 Files, account for its synchronization and crash-durability behavior. - Verify that the Java filesystem provider for the mount supports `SecureDirectoryStream`. - Configure retention and cleanup separately; the SDK does not delete completed payload files. diff --git a/docs/design.md b/docs/design.md index c3ab7144e..31bae0192 100644 --- a/docs/design.md +++ b/docs/design.md @@ -671,7 +671,9 @@ ID, target type, and serialized-data hash. The thread-local value is always rest The entity ID combines the execution or operation ID with a payload-kind suffix: `/input`, `/output`, or `/exception` for root execution payloads, and `/invoke-payload`, `/result`, or `/exception` for operation payloads. Distinct durable -payloads therefore remain distinct even for a custom SerDes that uses deterministic external-storage keys. +payload kinds therefore remain distinct. A custom external-storage SerDes must additionally return an immutable or +versioned reference for each serialization; invocation-local cache invalidation cannot make overwritten references +safe across replay. `FileSystemSerDes` uses that context to build collision-free paths on a shared durable filesystem. Calls made before a durable execution ARN exists, such as initial invocation input serialization, fall back to the delegate SerDes without diff --git a/docs/wire-formats/filesystem-serdes.md b/docs/wire-formats/filesystem-serdes.md index 9e2d2a216..2bd65b9a0 100644 --- a/docs/wire-formats/filesystem-serdes.md +++ b/docs/wire-formats/filesystem-serdes.md @@ -72,7 +72,8 @@ Malformed UTF-8 file content and filesystem read failures are `RetryableSerDesEx ## Path construction -All paths are rooted under the configured absolute base path. +All paths are rooted under the configured absolute base path. The base directory and every ancestor must be +pre-provisioned; the SDK does not create directory components. The entity ID used for path construction identifies both the durable owner and payload kind: @@ -80,37 +81,22 @@ The entity ID used for path construction identifies both the durable owner and p - invoke requests use `/invoke-payload`; - operation results/state and exceptions use `/result` and `/exception`. -These suffixes prevent different payloads belonging to one operation from colliding in deterministic external storage. +These suffixes distinguish different payload kinds belonging to one operation. External storage references must also +remain immutable or versioned so an older checkpoint always resolves the same content. -### URI encoding +The owner digest is the lowercase SHA-256 digest of the execution ARN, one zero byte, and the entity ID. -For a durable execution ARN matching: +- `HASH` uses the 64-character owner digest as the filename prefix. +- `URI` uses the first 32 characters of the percent-encoded entity ID, followed by `-` and the owner digest. -```text -arn::lambda:::function::/durable-execution// -``` - -the execution directory is: - -```text -/// -``` - -Other ARNs are encoded into one directory segment. +URI encoding preserves ASCII letters, digits, `-`, `_`, `.`, and `~`; all other UTF-8 bytes are percent encoded. -The file name is: +Every payload is a direct child of the base directory: ```text ---.json +/--.json ``` -URI encoding preserves ASCII letters, digits, `-`, `_`, `.`, and `~`; all other UTF-8 bytes are percent encoded. - -### Hash encoding - -`HASH` replaces the execution ARN and entity ID with lowercase SHA-256 hexadecimal strings. The payload digest and -unique UUID suffix remain in the file name. - ## Publication Files are immutable and published with `CREATE_NEW`. A failed write attempts to delete the partially created file before @@ -128,7 +114,8 @@ directory with `NOFOLLOW_LINKS`. File reads and writes also use `NOFOLLOW_LINKS` The implementation fails closed when: - the provider lacks `SecureDirectoryStream`; -- the base path, an ancestor, execution directory, or payload file is a symbolic link; +- the base path, an ancestor, or payload file is a symbolic link; +- the base directory is missing or is not a directory; - a path is outside the configured base path; - the file is missing or unreadable. diff --git a/examples/README.md b/examples/README.md index 0151edb9a..a2910f13d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -125,11 +125,12 @@ For a deployed function, mount EFS or another compatible shared filesystem at th environment: ```text -FILESYSTEM_SERDES_PATH=/mnt/efs/durable-payloads +FILESYSTEM_SERDES_PATH=/mnt/efs ``` The mounted Java filesystem provider must support `SecureDirectoryStream`. The example uses structured previews, filesystem I/O retries, a durable wait that forces replay, and checksum verification after the payload is loaded again. +The configured path must already exist; the SDK does not create directory components. ## Cleanup diff --git a/examples/generate-template.py b/examples/generate-template.py index 9cb380568..26c5a0128 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -141,7 +141,7 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " LocalMountPath: /mnt/efs", " Environment:", " Variables:", - " FILESYSTEM_SERDES_PATH: /mnt/efs/durable-payloads", + " FILESYSTEM_SERDES_PATH: /mnt/efs", ] ) lines.append("") diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java index 23f0f4095..ae6fb7c68 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java @@ -907,7 +907,7 @@ private static JsonNode assertFileSystemEnvelope(String value) throws Exception var envelope = MAPPER.readTree(value); assertEquals(1, envelope.get(FILE_SYSTEM_ENVELOPE_MARKER).intValue()); assertTrue(envelope.get("sha256").textValue().matches("[0-9a-f]{64}")); - assertTrue(envelope.get("file").textValue().startsWith("/mnt/efs/durable-payloads/")); + assertTrue(envelope.get("file").textValue().startsWith("/mnt/efs/")); return envelope; } diff --git a/examples/test_generate_template.py b/examples/test_generate_template.py index 67dd54fd4..80761fae6 100644 --- a/examples/test_generate_template.py +++ b/examples/test_generate_template.py @@ -35,7 +35,7 @@ def test_file_system_lambda_template_imports_persistent_infrastructure(self) -> self.assertIn("${FileSystemInfrastructureStackName}-SubnetId", template) self.assertIn("${FileSystemInfrastructureStackName}-LambdaSecurityGroupId", template) self.assertIn("${FileSystemInfrastructureStackName}-AccessPointArn", template) - self.assertIn("FILESYSTEM_SERDES_PATH: /mnt/efs/durable-payloads", template) + self.assertIn("FILESYSTEM_SERDES_PATH: /mnt/efs", template) self.assertNotIn("AWS::EFS::FileSystem", template) self.assertNotIn("FileSystemMountTarget", template) diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index a78afab49..a4bbfa164 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -76,6 +77,31 @@ void storesStepAndExecutionResultsAcrossReplay() throws Exception { } } + @Test + void checkpointFailureReplayPreservesPriorFilesystemReference() throws Exception { + var stepRuns = new AtomicInteger(); + var serDes = FileSystemSerDes.builder(tempDir).build(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step( + "persist-version", String.class, stepContext -> "value-" + stepRuns.incrementAndGet()), + config); + + var first = runner.run("input"); + var firstReference = + first.getOperation("persist-version").getStepDetails().result(); + runner.resetCheckpointToStarted("persist-version"); + var replayed = runner.run("input"); + var replayedReference = + replayed.getOperation("persist-version").getStepDetails().result(); + + assertEquals(2, stepRuns.get()); + assertFalse(firstReference.equals(replayedReference)); + assertEquals("value-1", serDes.deserialize(firstReference, TypeToken.get(String.class))); + assertEquals("value-2", serDes.deserialize(replayedReference, TypeToken.get(String.class))); + } + @Test void operationConfigControlsWhereFilesystemStorageIsUsed() throws Exception { var fileSystemSerDes = FileSystemSerDes.builder(tempDir).build(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 949c7b87d..4ea6db038 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -63,8 +63,6 @@ public final class FileSystemSerDes implements SerDes { .reader() .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) .with(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY); - private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( - "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); private static final Pattern SHA_256_DIGEST_PATTERN = Pattern.compile("[0-9a-f]{64}"); private final Path basePath; @@ -238,30 +236,23 @@ private static void verifyDigest(String serialized, String expected) { } private Path payloadPath(SerDesContext context, String digest) { - var directory = executionDirectory(context.durableExecutionArn()); - var fileName = encode(context.entityId()) + "-" + digest + "-" + UUID.randomUUID() + ".json"; - var file = directory.resolve(fileName).toAbsolutePath().normalize(); + var ownerDigest = sha256(context.durableExecutionArn() + "\0" + context.entityId()); + var ownerPrefix = ownerDigest; + if (pathEncoding == FileSystemPathEncoding.URI) { + var encodedEntity = percentEncode(context.entityId()); + ownerPrefix = encodedEntity.substring(0, Math.min(32, encodedEntity.length())) + "-" + ownerDigest; + } + var fileName = ownerPrefix + "-" + digest + "-" + UUID.randomUUID() + ".json"; + var file = basePath.resolve(fileName).toAbsolutePath().normalize(); if (!file.startsWith(basePath)) { throw new SerDesException("Filesystem SerDes path escapes the configured base path"); } return file; } - private Path executionDirectory(String durableExecutionArn) { - if (pathEncoding == FileSystemPathEncoding.URI) { - var match = DURABLE_EXECUTION_ARN_PATTERN.matcher(durableExecutionArn); - if (match.matches()) { - return basePath.resolve(encode(match.group(1))) - .resolve(encode(match.group(2))) - .resolve(encode(match.group(3))); - } - } - return basePath.resolve(encode(durableExecutionArn)); - } - private void writePayload(Path file, String serialized) { try { - try (var secureDirectory = openSecureDirectory(file.getParent(), true)) { + try (var secureDirectory = openSecureDirectory(file.getParent())) { var created = false; rejectSymbolicLinkIfPresent(secureDirectory.directory(), file.getFileName(), "payload file"); try (var channel = secureDirectory @@ -302,7 +293,7 @@ private String readPayload(String fileValue) { } try { byte[] storedData; - try (var secureDirectory = openSecureDirectory(file.getParent(), false)) { + try (var secureDirectory = openSecureDirectory(file.getParent())) { rejectSymbolicLinkIfPresent(secureDirectory.directory(), file.getFileName(), "payload file"); try (var channel = secureDirectory .directory() @@ -334,7 +325,7 @@ private static SerDesException classifyFileSystemFailure(String action, IOExcept return new RetryableSerDesException(message, failure); } - private SecureDirectoryHandle openSecureDirectory(Path directory, boolean createMissing) throws IOException { + private SecureDirectoryHandle openSecureDirectory(Path directory) throws IOException { if (directory == null || !directory.startsWith(basePath)) { throw new SerDesException("Filesystem SerDes directory is outside the configured base path"); } @@ -346,26 +337,10 @@ private SecureDirectoryHandle openSecureDirectory(Path directory, boolean create var openedStreams = new ArrayList>(); try { var current = requireSecureDirectoryStream(Files.newDirectoryStream(root), openedStreams); - var currentPath = root; for (var component : root.relativize(directory)) { - var nextPath = currentPath.resolve(component); - DirectoryStream next; rejectSymbolicLinkIfPresent(current, component, "directory"); - try { - next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); - } catch (NoSuchFileException missing) { - if (!createMissing) { - throw missing; - } - try { - Files.createDirectory(nextPath); - } catch (FileAlreadyExistsException ignored) { - // Validate and open the entry relative to the held parent directory below. - } - next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); - } + var next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); current = requireSecureDirectoryStream(next, openedStreams); - currentPath = nextPath; } return new SecureDirectoryHandle(current, openedStreams); } catch (IOException | RuntimeException failure) { @@ -409,10 +384,6 @@ private static void closeDirectoryStreams(List> streams, T } } - private String encode(String value) { - return pathEncoding == FileSystemPathEncoding.HASH ? sha256(value) : percentEncode(value); - } - private boolean fitsCheckpoint(String envelope) { return envelope.getBytes(StandardCharsets.UTF_8).length <= checkpointEnvelopeLimitBytes; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index b8f39e1c1..42e9097a7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -8,6 +8,10 @@ * Interface for serialization and deserialization of objects. * *

Implementations must support both simple types via {@link Class} and complex generic types via {@link TypeToken}. + * + *

An implementation that publishes payloads to external storage must return an immutable or versioned reference for + * every serialized value. It must not overwrite content reachable through a string that may already be stored in a + * durable checkpoint, because replay can occur after a later serialization attempt fails to checkpoint. */ public interface SerDes { /** diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index e0ad82b89..c9d069e98 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -155,7 +155,7 @@ void supportsHashPathEncodingAndPreview() throws Exception { var node = MAPPER.readTree(runner.serialize(serDes, new Value("preview"), context)); var file = Path.of(node.get("file").textValue()); - assertEquals(64, tempDir.relativize(file).getName(0).toString().length()); + assertEquals(tempDir, file.getParent()); assertTrue(file.getFileName().toString().matches("[0-9a-f]{64}-[0-9a-f]{64}-[0-9a-f-]{36}\\.json")); assertEquals("preview", node.get("preview").get("summary").textValue()); } @@ -386,7 +386,7 @@ void rejectsMalformedUtf8FilePayload() throws Exception { } @Test - void uriEncodingUsesReadableExecutionPathAndFlatUnsafeEntity() throws Exception { + void uriEncodingUsesFlatUnsafeEntityPrefixBoundToExecution() throws Exception { var serDes = FileSystemSerDes.builder(tempDir).build(); var context = new SerDesContext(realisticArn(), "../unsafe/entity"); @@ -394,22 +394,14 @@ void uriEncodingUsesReadableExecutionPathAndFlatUnsafeEntity() throws Exception .get("file") .textValue()); - assertEquals(tempDir.resolve("test").resolve("execution-name").resolve("invocation-id"), file.getParent()); + assertEquals(tempDir, file.getParent()); assertFalse(file.getFileName().toString().contains("/")); assertTrue(file.getFileName().toString().startsWith("..%2Funsafe%2Fentity-")); - } - - @Test - void malformedExecutionArnFallsBackToOneEncodedDirectory() throws Exception { - var serDes = FileSystemSerDes.builder(tempDir).build(); - var arn = "local/test:execution"; - - var file = Path.of(MAPPER.readTree(runner.serialize(serDes, "value", new SerDesContext(arn, "1"))) + var otherExecutionFile = Path.of(MAPPER.readTree(runner.serialize( + serDes, "value", new SerDesContext("local/test:execution", "../unsafe/entity"))) .get("file") .textValue()); - - assertEquals(1, tempDir.relativize(file.getParent()).getNameCount()); - assertEquals("local%2Ftest%3Aexecution", file.getParent().getFileName().toString()); + assertFalse(file.getFileName().equals(otherExecutionFile.getFileName())); } @Test @@ -464,8 +456,9 @@ void failsClosedWhenProviderLacksSecureDirectoryStreams() throws Exception { @Test void rejectsSymbolicLinkDirectoryWhenWriting() throws Exception { var outside = Files.createTempDirectory(tempDir.getParent(), "outside-payloads-"); - Files.createSymbolicLink(tempDir.resolve("test"), outside); - var serDes = FileSystemSerDes.builder(tempDir).build(); + var linkedBase = tempDir.resolve("linked-base"); + Files.createSymbolicLink(linkedBase, outside); + var serDes = FileSystemSerDes.builder(linkedBase).build(); var failure = assertThrows( SerDesException.class, () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); @@ -477,14 +470,14 @@ void rejectsSymbolicLinkDirectoryWhenWriting() throws Exception { @Test void rejectsSymbolicLinkDirectoryWhenReading() throws Exception { - var serDes = FileSystemSerDes.builder(tempDir).build(); + var basePath = Files.createDirectory(tempDir.resolve("base")); + var serDes = FileSystemSerDes.builder(basePath).build(); var context = new SerDesContext(realisticArn(), "1"); var envelope = runner.serialize(serDes, "value", context); - var executionDirectory = tempDir.resolve("test"); var outside = Files.createTempDirectory(tempDir.getParent(), "outside-payloads-"); - var movedDirectory = outside.resolve("test"); - Files.move(executionDirectory, movedDirectory); - Files.createSymbolicLink(executionDirectory, movedDirectory); + var movedDirectory = outside.resolve("base"); + Files.move(basePath, movedDirectory); + Files.createSymbolicLink(basePath, movedDirectory); var failure = assertThrows( SerDesException.class, @@ -520,15 +513,6 @@ void rejectsSymbolicLinkPayloadFile() throws Exception { @Test void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { - var outsideRoot = Files.createTempDirectory(tempDir.getParent(), "outside-root-"); - var linkedRoot = tempDir.resolve("linked-root"); - Files.createSymbolicLink(linkedRoot, outsideRoot); - var rootSerDes = FileSystemSerDes.builder(linkedRoot).build(); - - assertThrows( - SerDesException.class, - () -> runner.serialize(rootSerDes, "value", new SerDesContext(realisticArn(), "1"))); - var outsideAncestor = Files.createTempDirectory(tempDir.getParent(), "outside-ancestor-"); var linkedAncestor = tempDir.resolve("linked-ancestor"); Files.createSymbolicLink(linkedAncestor, outsideAncestor); @@ -541,6 +525,18 @@ void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { assertFalse(Files.exists(outsideAncestor.resolve("payloads"))); } + @Test + void doesNotCreateMissingBasePathComponents() { + var basePath = tempDir.resolve("missing").resolve("payloads"); + var serDes = FileSystemSerDes.builder(basePath).build(); + + assertThrows( + RetryableSerDesException.class, + () -> runner.serialize(serDes, "value", new SerDesContext(realisticArn(), "1"))); + + assertFalse(Files.exists(tempDir.resolve("missing"))); + } + @Test void rejectsTrailingAndDuplicateFieldsInMarkedEnvelope() { var serDes = FileSystemSerDes.builder(tempDir).build(); From 75447029a3e45a916d1b015a190c241b810017ed Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 11:40:21 +0000 Subject: [PATCH 10/11] refactor: pass SerDes context explicitly --- docs/adr/005-filesystem-serdes.md | 55 +++++++++++-------- docs/advanced/configuration.md | 7 ++- docs/advanced/serdes.md | 29 ++++++---- docs/design.md | 8 +-- .../FileSystemSerDesIntegrationTest.java | 38 ++++++++++--- .../testing/LocalDurableTestRunnerTest.java | 6 +- .../durable/testing/TestOperationTest.java | 8 ++- .../cloud/HistoryEventProcessorTest.java | 8 ++- .../durable/serde/FileSystemSerDes.java | 23 ++++++-- .../lambda/durable/serde/RetrySerDes.java | 10 ++++ .../amazon/lambda/durable/serde/SerDes.java | 30 ++++++++++ .../lambda/durable/serde/SerDesContext.java | 30 +--------- .../lambda/durable/serde/SerDesRunner.java | 14 ++--- .../operation/InvokeOperationTest.java | 8 +-- .../SerializableDurableOperationTest.java | 8 +-- .../lambda/durable/serde/RetrySerDesTest.java | 6 +- .../durable/serde/SerDesRunnerTest.java | 20 ++++--- 17 files changed, 195 insertions(+), 113 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 68969b7a1..61e662df8 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -1,4 +1,4 @@ -# ADR-005: Filesystem SerDes with Thread-Local Context +# ADR-005: Filesystem SerDes with Explicit Context Methods **Status:** Accepted **Date:** 2026-08-31 @@ -22,27 +22,36 @@ public interface SerDes { } ``` -Changing those methods would break existing implementations. Filesystem I/O must also avoid blocking the user-operation -executor or the SDK coordination executor, and repeated `DurableFuture.get()` calls must not repeatedly read and decode -the same file. +Replacing those methods would break existing implementations. Adding default context-aware overloads preserves source +and binary compatibility while allowing filesystem storage to receive durable identity explicitly. Filesystem I/O must +also avoid blocking the user-operation executor or the SDK coordination executor, and repeated `DurableFuture.get()` +calls must not repeatedly read and decode the same file. ## Decision -### Preserve the SerDes interface +### Add context-aware default methods -The existing `SerDes` interface remains unchanged. The SDK exposes the active payload identity through: +The original methods remain abstract and unchanged. New default methods accept `SerDesContext` and delegate to the +original methods, so existing implementations require no changes: ```java -public record SerDesContext(String durableExecutionArn, String entityId) { - public static SerDesContext getCurrentContext(); +public interface SerDes { + String serialize(Object value); + + default String serialize(Object value, SerDesContext context) { + return serialize(value); + } + + T deserialize(String data, TypeToken typeToken); + + default T deserialize(String data, TypeToken typeToken, SerDesContext context) { + return deserialize(data, typeToken); + } } ``` -`getCurrentContext()` returns `null` outside an SDK-managed SerDes call. The SDK owns setting and clearing the context; -there is no public setter. - -The implementation uses a plain `ThreadLocal`, not an `InheritableThreadLocal`. Every call restores the previous value -in `finally`, which supports nesting and prevents context from leaking when executor threads are reused. +Context-aware implementations override the new methods. The SDK passes a non-null `SerDesContext` to every managed +call; direct customer calls to the original methods remain context-free. Entity IDs include a stable payload-kind suffix. Root input, output, and exceptions use `/input`, `/output`, and `/exception`; operation invoke payloads, results/state, and exceptions use `/invoke-payload`, `/result`, and @@ -54,10 +63,8 @@ storage implementations must still publish immutable or versioned references for Each `ExecutionManager` creates one `SerDesRunner` for the Lambda invocation. The runner: 1. executes inline or dispatches the SerDes call to the configured SerDes executor; -2. installs `SerDesContext` inside that executor task; -3. invokes the unchanged SerDes method; -4. clears or restores the thread-local context; -5. returns the result or rethrows the original failure. +2. invokes the context-aware default method with `SerDesContext`; +3. returns the result or rethrows the original failure. The configured executor is available through: @@ -175,9 +182,10 @@ directory streams fail closed. The SDK does not create missing directory compone ### Initial invocation input -The durable execution ARN does not exist when a caller serializes the initial Lambda input. Therefore, -`FileSystemSerDes.serialize()` delegates directly when `SerDesContext.getCurrentContext()` is `null`. The initial input -remains ordinary delegate JSON. +The durable execution ARN does not exist when a caller serializes the initial Lambda input. Therefore, the original +context-free `FileSystemSerDes.serialize()` delegates directly and the initial input remains ordinary delegate JSON. +After invocation starts, `DurableExecutor` deserializes that input through the context-aware overload with the root +input identity. After the invocation starts, the SDK routes root input deserialization, operation payloads, exceptions, and root output through `SerDesRunner`. A chained-invoke boundary requires compatible filesystem configuration and shared storage only @@ -200,7 +208,7 @@ Positive: Negative: -- SerDes context is implicit thread-local state. +- The public interface has two additional default overloads that implementations may choose to override. - Configuring a SerDes executor adds an executor boundary to every SDK-managed SerDes call. - Repeated deserialization returns the same object instance within an invocation. - Filesystem retention and cleanup remain the application's responsibility. @@ -217,9 +225,10 @@ Negative: ## Alternatives Rejected -### Change the SerDes method signatures +### Replace the original SerDes method signatures -Rejected because adding context parameters would break every existing implementation. +Rejected because replacing the existing methods would break every implementation. Backward-compatible default +overloads are the accepted approach. ### Add a PayloadOffloader abstraction diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 2da9a3382..53fa5e6f2 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -95,10 +95,13 @@ var config = InvokeConfig.builder() The same pattern applies to `StepConfig.serDes(...)`, callback, child-context, map, parallel, and wait-for-condition configuration. -The SDK supplies a `SerDesContext` through thread-local storage during managed calls: +The SDK supplies a `SerDesContext` explicitly through new default methods during managed calls: ```java -var context = SerDesContext.getCurrentContext(); +@Override +public String serialize(Object value, SerDesContext context) { + return store(value, context.durableExecutionArn(), context.entityId()); +} ``` Custom SerDes implementations can use its durable execution ARN and entity ID for external storage. Calls use the diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index 6137db5c2..d64549ba8 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -37,18 +37,27 @@ selection model applies to steps, callbacks, child contexts, map, parallel, and ## SerDesContext -SDK-managed calls expose durable identity through thread-local storage without changing the `SerDes` interface: +SDK-managed calls pass durable identity explicitly through backward-compatible default methods: ```java -var context = SerDesContext.getCurrentContext(); -if (context != null) { - var executionArn = context.durableExecutionArn(); - var entityId = context.entityId(); +public interface SerDes { + String serialize(Object value); + + default String serialize(Object value, SerDesContext context) { + return serialize(value); + } + + T deserialize(String data, TypeToken typeToken); + + default T deserialize(String data, TypeToken typeToken, SerDesContext context) { + return deserialize(data, typeToken); + } } ``` -The context is installed only while the SDK invokes `serialize` or `deserialize` and is restored in `finally`. -Direct customer calls return `null`. +Existing implementations continue to implement only the original methods. Context-aware implementations override the +new overloads. The SDK supplies a non-null context for managed calls; direct calls to the original methods are +context-free. Entity IDs distinguish every persisted payload owned by the same execution or operation: @@ -158,9 +167,9 @@ skipped because they are ambiguous with path selectors. Use `previewGenerator(.. ### Initial input -The durable execution ARN does not exist before the initial Lambda invocation starts. A direct -`FileSystemSerDes.serialize()` call therefore delegates normally when no `SerDesContext` exists. Root input is ordinary -delegate JSON; SDK-managed output and operation payloads can use filesystem storage after the invocation begins. +The durable execution ARN does not exist while the caller serializes the initial Lambda input. A direct call to the +original `FileSystemSerDes.serialize()` method therefore delegates normally. After invocation begins, root input +deserialization and all persisted output/operation payloads use the context-aware methods. ### Operational requirements diff --git a/docs/design.md b/docs/design.md index 31bae0192..799722823 100644 --- a/docs/design.md +++ b/docs/design.md @@ -353,7 +353,7 @@ software.amazon.lambda.durable │ ├── PreviewConfig # Structured preview selection and byte budget │ ├── RetrySerDes # RetryableSerDesException decorator │ ├── SerDesPreview # Structured preview builder -│ ├── SerDesContext # Thread-local durable payload identity +│ ├── SerDesContext # Explicit durable payload identity │ ├── SerDesRunner # Executor dispatch + invocation cache │ └── AwsSdkV2Module # SDK type support │ @@ -665,9 +665,9 @@ public interface SerDes { ``` SDK-managed calls go through an invocation-scoped `SerDesRunner`. The runner executes inline by default or dispatches -work to the configured SerDes executor, installs a `SerDesContext` in plain thread-local storage for the duration of the -call, and caches successful deserializations in a bounded weak-reference LRU by SerDes identity, execution ARN, entity -ID, target type, and serialized-data hash. The thread-local value is always restored in `finally`. +work to the configured SerDes executor, invokes the context-aware default method with an explicit `SerDesContext`, and +caches successful deserializations in a bounded weak-reference LRU by SerDes identity, execution ARN, entity ID, target +type, and serialized-data hash. The entity ID combines the execution or operation ID with a payload-kind suffix: `/input`, `/output`, or `/exception` for root execution payloads, and `/invoke-payload`, `/result`, or `/exception` for operation payloads. Distinct durable diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index a4bbfa164..aa8f47ef7 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -344,12 +344,22 @@ public String serialize(Object value) { return fileSystemSerDes.serialize(value); } + @Override + public String serialize(Object value, SerDesContext context) { + return fileSystemSerDes.serialize(value, context); + } + @Override public T deserialize(String data, TypeToken typeToken) { - if (typeToken.equals(TypeToken.get(Payload.class)) && SerDesContext.getCurrentContext() != null) { + return fileSystemSerDes.deserialize(data, typeToken); + } + + @Override + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + if (typeToken.equals(TypeToken.get(Payload.class))) { resultDeserializations.incrementAndGet(); } - return fileSystemSerDes.deserialize(data, typeToken); + return fileSystemSerDes.deserialize(data, typeToken, context); } }; var config = DurableConfig.builder().withSerDes(countingSerDes).build(); @@ -481,12 +491,13 @@ private static final class DeterministicExternalSerDes implements SerDes { @Override public String serialize(Object value) { - var context = SerDesContext.getCurrentContext(); - if (context == null) { - return delegate.serialize(value); - } + return delegate.serialize(value); + } + + @Override + public String serialize(Object value, SerDesContext context) { var key = context.durableExecutionArn() + "#" + context.entityId(); - storage.put(key, delegate.serialize(value)); + storage.put(key, delegate.serialize(value, context)); return REFERENCE_PREFIX + key; } @@ -503,6 +514,19 @@ public T deserialize(String data, TypeToken typeToken) { return delegate.deserialize(serialized, typeToken); } + @Override + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + var serialized = data; + if (data != null && data.startsWith(REFERENCE_PREFIX)) { + var key = data.substring(REFERENCE_PREFIX.length()); + serialized = storage.get(key); + if (serialized == null) { + throw new IllegalStateException("Missing external value: " + key); + } + } + return delegate.deserialize(serialized, typeToken, context); + } + Set keys() { return Set.copyOf(storage.keySet()); } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index c324d3b4d..59deefc1b 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -127,10 +127,8 @@ void resultAndOperationInspectionUseDurableSerDesContext() { var contexts = new CopyOnWriteArrayList(); var serDes = new JacksonSerDes() { @Override - public T deserialize(String data, TypeToken typeToken) { - if (SerDesContext.getCurrentContext() != null) { - contexts.add(SerDesContext.getCurrentContext()); - } + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + contexts.add(context); return super.deserialize(data, typeToken); } }; diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java index 32ffe0d25..fbd32fe49 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java @@ -32,7 +32,13 @@ public String serialize(Object value) { @Override @SuppressWarnings("unchecked") public T deserialize(String data, TypeToken typeToken) { - observedContext.set(SerDesContext.getCurrentContext()); + return (T) data; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + observedContext.set(context); return (T) data; } }; diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java index e2394cd67..c9bd0ab0e 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -172,7 +172,13 @@ public String serialize(Object value) { @Override @SuppressWarnings("unchecked") public T deserialize(String data, TypeToken typeToken) { - observedContexts.add(SerDesContext.getCurrentContext()); + return (T) data; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + observedContexts.add(context); return (T) data; } }; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 4ea6db038..8cf1f6590 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -88,12 +88,16 @@ public static Builder builder(Path basePath) { @Override public String serialize(Object value) { - var serialized = delegate.serialize(value); + return delegate.serialize(value); + } + + @Override + public String serialize(Object value, SerDesContext context) { + var serialized = context == null ? delegate.serialize(value) : delegate.serialize(value, context); if (serialized == null) { return null; } - var context = SerDesContext.getCurrentContext(); if (context == null) { return serialized; } @@ -110,23 +114,32 @@ public String serialize(Object value) { @Override public T deserialize(String data, TypeToken typeToken) { + return deserialize(data, typeToken, null); + } + + @Override + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { if (data == null) { return null; } var envelope = parseEnvelope(data); if (envelope == null) { - return delegate.deserialize(data, typeToken); + return deserializeDelegate(data, typeToken, context); } if (envelope.hasNonNull("data")) { var serialized = envelope.get("data").textValue(); verifyDigest(serialized, envelope.get("sha256").textValue()); - return delegate.deserialize(serialized, typeToken); + return deserializeDelegate(serialized, typeToken, context); } var serialized = readPayload(envelope.get("file").textValue()); verifyDigest(serialized, envelope.get("sha256").textValue()); - return delegate.deserialize(serialized, typeToken); + return deserializeDelegate(serialized, typeToken, context); + } + + private T deserializeDelegate(String data, TypeToken typeToken, SerDesContext context) { + return context == null ? delegate.deserialize(data, typeToken) : delegate.deserialize(data, typeToken, context); } private String inlineEnvelope(String serialized) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 6b7cb1119..2b6b5c12f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -37,8 +37,18 @@ public String serialize(Object value) { return retryExecutor.execute("serialization", () -> delegate.serialize(value)); } + @Override + public String serialize(Object value, SerDesContext context) { + return retryExecutor.execute("serialization", () -> delegate.serialize(value, context)); + } + @Override public T deserialize(String data, TypeToken typeToken) { return retryExecutor.execute("deserialization", () -> delegate.deserialize(data, typeToken)); } + + @Override + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + return retryExecutor.execute("deserialization", () -> delegate.deserialize(data, typeToken, context)); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 42e9097a7..487d24455 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -22,6 +22,20 @@ public interface SerDes { */ String serialize(Object value); + /** + * Serializes an object with durable payload context. + * + *

The default implementation preserves compatibility with existing SerDes implementations by delegating to + * {@link #serialize(Object)}. + * + * @param value the object to serialize + * @param context durable payload identity supplied by the SDK + * @return the serialized string, or null if value is null + */ + default String serialize(Object value, SerDesContext context) { + return serialize(value); + } + /** * Deserializes a JSON string to an object of the specified generic type. * @@ -40,4 +54,20 @@ public interface SerDes { * @return the deserialized object, or null if data is null */ T deserialize(String data, TypeToken typeToken); + + /** + * Deserializes a string with durable payload context. + * + *

The default implementation preserves compatibility with existing SerDes implementations by delegating to + * {@link #deserialize(String, TypeToken)}. + * + * @param data the string to deserialize + * @param typeToken target type information + * @param context durable payload identity supplied by the SDK + * @param target type + * @return the deserialized value, or null if data is null + */ + default T deserialize(String data, TypeToken typeToken, SerDesContext context) { + return deserialize(data, typeToken); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java index c68c7dfc0..c44458576 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java @@ -3,45 +3,19 @@ package software.amazon.lambda.durable.serde; import java.util.Objects; -import java.util.function.Supplier; /** * Identifies the durable payload currently being processed by a {@link SerDes}. * - *

The SDK exposes this context through thread-local storage so the existing {@link SerDes} interface remains - * backward compatible. The context is available only while the SDK is invoking a SerDes method. + *

The SDK passes this context explicitly to the context-aware default methods on {@link SerDes}. Existing + * implementations remain compatible because those methods delegate to the original context-free methods by default. * * @param durableExecutionArn ARN of the durable execution * @param entityId stable identifier of the execution or operation payload */ public record SerDesContext(String durableExecutionArn, String entityId) { - private static final ThreadLocal CURRENT = new ThreadLocal<>(); - public SerDesContext { Objects.requireNonNull(durableExecutionArn, "durableExecutionArn cannot be null"); Objects.requireNonNull(entityId, "entityId cannot be null"); } - - /** - * Returns the context for the current SDK-managed SerDes call. - * - * @return the current context, or {@code null} when called outside an SDK-managed SerDes call - */ - public static SerDesContext getCurrentContext() { - return CURRENT.get(); - } - - static T callWithContext(SerDesContext context, Supplier action) { - var previous = CURRENT.get(); - CURRENT.set(Objects.requireNonNull(context, "context cannot be null")); - try { - return action.get(); - } finally { - if (previous == null) { - CURRENT.remove(); - } else { - CURRENT.set(previous); - } - } - } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index 3f58410b3..c1624500f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -20,8 +20,7 @@ import software.amazon.lambda.durable.util.ExceptionHelper; /** - * Executes SDK-managed SerDes calls on a dedicated executor with {@link SerDesContext} installed in thread-local - * storage. + * Executes SDK-managed SerDes calls inline or on a dedicated executor and passes {@link SerDesContext} explicitly. * *

Each runner is scoped to one Lambda invocation. Successful deserializations are cached for that invocation so * repeated reads of the same checkpoint payload do not repeat filesystem or other external I/O. @@ -57,7 +56,7 @@ public String serialize(SerDes serDes, Object value, SerDesContext context) { Objects.requireNonNull(context, "context cannot be null"); var contextKey = new ContextKey(serDes, context.durableExecutionArn(), context.entityId()); try { - return join(submit(context, () -> serDes.serialize(value))); + return join(submit(() -> serDes.serialize(value, context))); } finally { generation(contextKey).incrementAndGet(); } @@ -91,7 +90,7 @@ public T deserialize(SerDes serDes, String data, TypeToken typeToken, Ser return cached == NULL_VALUE ? null : (T) cached; } - var value = join(submit(context, () -> serDes.deserialize(data, typeToken))); + var value = join(submit(() -> serDes.deserialize(data, typeToken, context))); var cacheValue = value == null ? NULL_VALUE : value; putCompleted(key, cacheValue); pending.complete(cacheValue); @@ -126,17 +125,16 @@ private AtomicLong generation(ContextKey key) { return contextGenerations.computeIfAbsent(key, ignored -> new AtomicLong()); } - private CompletableFuture submit(SerDesContext context, Supplier action) { - Objects.requireNonNull(context, "context cannot be null"); + private CompletableFuture submit(Supplier action) { Objects.requireNonNull(action, "action cannot be null"); if (executorService == null) { try { - return CompletableFuture.completedFuture(SerDesContext.callWithContext(context, action)); + return CompletableFuture.completedFuture(action.get()); } catch (Throwable failure) { return CompletableFuture.failedFuture(failure); } } - return CompletableFuture.supplyAsync(() -> SerDesContext.callWithContext(context, action), executorService); + return CompletableFuture.supplyAsync(action, executorService); } private static T join(CompletableFuture future) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 8abde304b..01386e182 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -81,14 +81,14 @@ void invokePayloadAndResultUseDistinctPayloadEntityIds() { var contexts = new CopyOnWriteArrayList(); var serDes = new JacksonSerDes() { @Override - public String serialize(Object value) { - contexts.add(SerDesContext.getCurrentContext()); + public String serialize(Object value, SerDesContext context) { + contexts.add(context); return super.serialize(value); } @Override - public T deserialize(String data, TypeToken typeToken) { - contexts.add(SerDesContext.getCurrentContext()); + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + contexts.add(context); return super.deserialize(data, typeToken); } }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index ae4d2858e..268992256 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -560,14 +560,14 @@ void resultAndExceptionUseDistinctPayloadEntityIds() { var contexts = new CopyOnWriteArrayList(); var serDes = new JacksonSerDes() { @Override - public String serialize(Object value) { - contexts.add(SerDesContext.getCurrentContext()); + public String serialize(Object value, SerDesContext context) { + contexts.add(context); return super.serialize(value); } @Override - public T deserialize(String data, TypeToken typeToken) { - contexts.add(SerDesContext.getCurrentContext()); + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + contexts.add(context); return super.deserialize(data, typeToken); } }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index b510f6a79..c637c25ef 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -82,13 +82,13 @@ public T deserialize(String data, TypeToken typeToken) { } @Test - void retriesKeepTheSameThreadLocalContext() { + void retriesKeepTheSameExplicitContext() { var attempts = new AtomicInteger(); var observed = new AtomicReference(); var delegate = new JacksonSerDes() { @Override - public String serialize(Object value) { - observed.set(SerDesContext.getCurrentContext()); + public String serialize(Object value, SerDesContext context) { + observed.set(context); if (attempts.incrementAndGet() == 1) { throw new RetryableSerDesException("temporary"); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index 720287892..15240065b 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -36,14 +36,14 @@ void tearDown() { } @Test - void executesOnConfiguredExecutorWithThreadLocalContext() { + void executesOnConfiguredExecutorWithExplicitContext() { var observedThread = new AtomicReference(); var observedContext = new AtomicReference(); var serDes = new JacksonSerDes() { @Override - public String serialize(Object value) { + public String serialize(Object value, SerDesContext context) { observedThread.set(Thread.currentThread().getName()); - observedContext.set(SerDesContext.getCurrentContext()); + observedContext.set(context); return super.serialize(value); } }; @@ -52,7 +52,6 @@ public String serialize(Object value) { assertEquals("\"value\"", runner.serialize(serDes, "value", context)); assertEquals("test-serdes", observedThread.get()); assertEquals(context, observedContext.get()); - assertNull(SerDesContext.getCurrentContext()); } @Test @@ -63,16 +62,15 @@ void executesInlineWhenNoExecutorIsConfigured() { var context = new SerDesContext("arn:test", "entity"); var serDes = new JacksonSerDes() { @Override - public String serialize(Object value) { + public String serialize(Object value, SerDesContext suppliedContext) { observedThread.set(Thread.currentThread()); - assertEquals(context, SerDesContext.getCurrentContext()); + assertEquals(context, suppliedContext); return super.serialize(value); } }; assertEquals("\"value\"", inlineRunner.serialize(serDes, "value", context)); assertSame(callingThread, observedThread.get()); - assertNull(SerDesContext.getCurrentContext()); } @Test @@ -154,8 +152,13 @@ public String serialize(Object value) { @Override public T deserialize(String data, TypeToken typeToken) { + throw new AssertionError("context-aware overload should be used"); + } + + @Override + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { calls.incrementAndGet(); - assertEquals(new SerDesContext("arn:test", "entity"), SerDesContext.getCurrentContext()); + assertEquals(new SerDesContext("arn:test", "entity"), context); throw new IllegalStateException("failed"); } }; @@ -169,7 +172,6 @@ public T deserialize(String data, TypeToken typeToken) { () -> runner.deserialize(serDes, "\"value\"", TypeToken.get(String.class), context)); assertEquals(2, calls.get()); - assertNull(executor.submit(SerDesContext::getCurrentContext).get()); assertTrue(executor.submit(() -> Thread.currentThread().getName().startsWith("test-serdes")) .get()); } From 09fbaea9d575b2ff90b22818721f57edda7cec41 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 1 Sep 2026 17:09:51 +0000 Subject: [PATCH 11/11] fix: harden filesystem SerDes boundaries --- docs/adr/005-filesystem-serdes.md | 5 +- docs/advanced/configuration.md | 6 +- docs/advanced/serdes.md | 4 +- docs/wire-formats/filesystem-serdes.md | 2 +- .../durable/serde/FileSystemSerDes.java | 18 ++++- .../durable/serde/FileSystemSerDesTest.java | 74 +++++++++++++++---- 6 files changed, 85 insertions(+), 24 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 61e662df8..3e5af42f2 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -112,7 +112,7 @@ configuration. var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemSerDesMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) - .checkpointEnvelopeLimitBytes(512 * 1024) + .checkpointEnvelopeLimitBytes(256 * 1024 - 1024) .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) .include(PreviewField.anywhere("id")) .mask(PreviewField.anywhere("email")) @@ -138,7 +138,8 @@ Path encodings: | `URI` | Percent-encode readable execution and entity path segments. | | `HASH` | Use fixed-length SHA-256 path segments. | -The default checkpoint-envelope limit is 255 KiB and can be changed with `checkpointEnvelopeLimitBytes(...)`. +The checkpoint-envelope limit defaults to 255 KiB, can be lowered, and cannot be configured above that safe checkpoint +ceiling. The default delegate is `JacksonSerDes`; `.delegate(...)` controls how values are encoded inside files. Existing operation-level SerDes configuration controls which boundaries use filesystem storage. For example, diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 53fa5e6f2..a81d17cb6 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -54,7 +54,7 @@ inline when configured for overflow mode: var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemSerDesMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) - .checkpointEnvelopeLimitBytes(512 * 1024) + .checkpointEnvelopeLimitBytes(256 * 1024 - 1024) .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) .include(PreviewField.anywhere("id"), PreviewField.path("status")) .mask(PreviewField.anywhere("email")) @@ -67,8 +67,8 @@ return DurableConfig.builder() ``` `ALWAYS` writes every SDK-managed payload to a file. `OVERFLOW` stores the payload inline until the complete checkpoint -envelope exceeds the configured limit, which defaults to 255 KiB. `URI` path encoding keeps identifiers readable, -while `HASH` avoids filesystem name-length and character restrictions. +envelope exceeds the configured limit, which defaults to and cannot exceed 255 KiB. `URI` path encoding keeps +identifiers readable, while `HASH` avoids filesystem name-length and character restrictions. `PreviewConfig` supports include-all/exclude-all modes, exact-path or anywhere field matching, masking, and a default 4 KiB preview budget. A custom `previewGenerator(...)` remains available for non-standard preview logic. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index d64549ba8..24b6700c8 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -124,8 +124,8 @@ var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloa | `ALWAYS` | Writes every SDK-managed non-null payload to a file. | | `OVERFLOW` | Keeps the complete envelope inline until it exceeds the configured limit. | -The default checkpoint-envelope limit is 255 KiB and can be changed with -`checkpointEnvelopeLimitBytes(...)`. +The checkpoint-envelope limit defaults to 255 KiB, can be lowered with `checkpointEnvelopeLimitBytes(...)`, and cannot +be increased above that safe checkpoint ceiling. ### Path encoding diff --git a/docs/wire-formats/filesystem-serdes.md b/docs/wire-formats/filesystem-serdes.md index 2bd65b9a0..b593263eb 100644 --- a/docs/wire-formats/filesystem-serdes.md +++ b/docs/wire-formats/filesystem-serdes.md @@ -58,7 +58,7 @@ Text containing the marker inside a JSON string is not recognized as an envelope ``` Exactly one of `data` or `file` is required. `preview` is valid only with `file`. The complete encoded envelope must fit -the configured `checkpointEnvelopeLimitBytes`. +the configured `checkpointEnvelopeLimitBytes`, which cannot exceed the safe 255 KiB checkpoint ceiling. ## Payload and digest diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 8cf1f6590..4f08173b4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -18,6 +18,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystemLoopException; import java.nio.file.Files; +import java.nio.file.InvalidPathException; import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; import java.nio.file.NotDirectoryException; @@ -125,7 +126,7 @@ public T deserialize(String data, TypeToken typeToken, SerDesContext cont var envelope = parseEnvelope(data); if (envelope == null) { - return deserializeDelegate(data, typeToken, context); + return delegate.deserialize(data, typeToken); } if (envelope.hasNonNull("data")) { var serialized = envelope.get("data").textValue(); @@ -300,7 +301,16 @@ private void writePayload(Path file, String serialized) { } private String readPayload(String fileValue) { - var file = basePath.getFileSystem().getPath(fileValue).toAbsolutePath().normalize(); + final Path file; + try { + var candidate = basePath.getFileSystem().getPath(fileValue); + if (!candidate.isAbsolute()) { + throw new SerDesException("Filesystem SerDes file path must be absolute"); + } + file = candidate.normalize(); + } catch (InvalidPathException | SecurityException failure) { + throw new SerDesException("Filesystem SerDes file path is invalid", failure); + } if (!file.startsWith(basePath)) { throw new SerDesException("Filesystem SerDes file is outside the configured base path"); } @@ -507,6 +517,10 @@ public Builder checkpointEnvelopeLimitBytes(int checkpointEnvelopeLimitBytes) { if (checkpointEnvelopeLimitBytes <= 0) { throw new IllegalArgumentException("checkpointEnvelopeLimitBytes must be positive"); } + if (checkpointEnvelopeLimitBytes > DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES) { + throw new IllegalArgumentException( + "checkpointEnvelopeLimitBytes cannot exceed " + DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES); + } this.checkpointEnvelopeLimitBytes = checkpointEnvelopeLimitBytes; return this; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index c9d069e98..2db33b641 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -104,21 +104,11 @@ void overflowModeStoresLargePayload() throws Exception { } @Test - void checkpointEnvelopeLimitCanBeIncreasedForLargerInlinePayloads() throws Exception { - var value = "x".repeat(300_000); - var context = new SerDesContext(realisticArn(), "1"); - var defaultSerDes = FileSystemSerDes.builder(tempDir) - .storageMode(FileSystemSerDesMode.OVERFLOW) - .build(); - var largerEnvelopeSerDes = FileSystemSerDes.builder(tempDir) - .storageMode(FileSystemSerDesMode.OVERFLOW) - .checkpointEnvelopeLimitBytes(512 * 1024) - .build(); + void checkpointEnvelopeLimitCannotExceedSafeCheckpointCeiling() { + var failure = assertThrows(IllegalArgumentException.class, () -> FileSystemSerDes.builder(tempDir) + .checkpointEnvelopeLimitBytes(256 * 1024 - 1024 + 1)); - assertTrue( - MAPPER.readTree(runner.serialize(defaultSerDes, value, context)).hasNonNull("file")); - assertTrue(MAPPER.readTree(runner.serialize(largerEnvelopeSerDes, value, context)) - .hasNonNull("data")); + assertEquals("checkpointEnvelopeLimitBytes cannot exceed 261120", failure.getMessage()); } @Test @@ -300,6 +290,45 @@ void unmarkedDataAndFileObjectsAreDelegatedNormally() { serDes.deserialize("{\"file\":\"value\"}", new TypeToken>() {})); } + @Test + void unmarkedPayloadUsesContextFreeDelegateAndEnvelopeUsesContextualDelegate() { + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + return "raw:" + value; + } + + @Override + public String serialize(Object value, SerDesContext context) { + return "context:" + value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + if (!data.startsWith("raw:")) { + throw new SerDesException("Expected raw encoding"); + } + return (T) data.substring("raw:".length()); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken, SerDesContext context) { + if (!data.startsWith("context:")) { + throw new SerDesException("Expected contextual encoding"); + } + return (T) data.substring("context:".length()); + } + }; + var serDes = FileSystemSerDes.builder(tempDir).delegate(delegate).build(); + var context = new SerDesContext(realisticArn(), "1"); + + assertEquals("value", runner.deserialize(serDes, "raw:value", TypeToken.get(String.class), context)); + var envelope = runner.serialize(serDes, "value", context); + assertEquals("value", runner.deserialize(serDes, envelope, TypeToken.get(String.class), context)); + } + @Test void rejectsFileOutsideConfiguredBasePath() throws Exception { var externalFile = Files.createTempFile("filesystem-serdes", ".json"); @@ -311,6 +340,23 @@ void rejectsFileOutsideConfiguredBasePath() throws Exception { assertThrows(SerDesException.class, () -> serDes.deserialize(envelope, TypeToken.get(String.class))); } + @Test + void rejectsRelativeAndInvalidEnvelopePathsAsPermanentFailures() throws Exception { + var serDes = FileSystemSerDes.builder(tempDir).build(); + var relativeEnvelope = MAPPER.writeValueAsString( + Map.of("__durable_execution_filesystem_serdes", 1, "file", "relative.json", "sha256", "0".repeat(64))); + var invalidEnvelope = MAPPER.writeValueAsString(Map.of( + "__durable_execution_filesystem_serdes", 1, "file", "invalid\u0000path", "sha256", "0".repeat(64))); + + var relativeFailure = assertThrows( + SerDesException.class, () -> serDes.deserialize(relativeEnvelope, TypeToken.get(String.class))); + var invalidFailure = assertThrows( + SerDesException.class, () -> serDes.deserialize(invalidEnvelope, TypeToken.get(String.class))); + + assertTrue(relativeFailure.getMessage().contains("must be absolute")); + assertTrue(invalidFailure.getCause() instanceof java.nio.file.InvalidPathException); + } + @Test void rejectsMalformedRecognizedEnvelope() { var serDes = FileSystemSerDes.builder(tempDir).build();