-
Notifications
You must be signed in to change notification settings - Fork 11
feat: add filesystem payload offloader and retries #681
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
zhongkechen
wants to merge
1
commit into
issue-463-payload-offloader
from
issue-463-payload-offloader-filesystem
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
88 changes: 88 additions & 0 deletions
88
sdk/src/main/java/software/amazon/lambda/durable/offload/PayloadOffloadRetryExecutor.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| package software.amazon.lambda.durable.offload; | ||
|
|
||
| import java.time.Duration; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.function.Supplier; | ||
| import software.amazon.lambda.durable.exception.PayloadOffloadException; | ||
| import software.amazon.lambda.durable.exception.RetryablePayloadOffloadException; | ||
| import software.amazon.lambda.durable.retry.RetryDecision; | ||
| import software.amazon.lambda.durable.retry.RetryStrategy; | ||
|
|
||
| final class PayloadOffloadRetryExecutor { | ||
| static final Sleeper DEFAULT_SLEEPER = delay -> { | ||
| if (delay.getSeconds() > 0) { | ||
| TimeUnit.SECONDS.sleep(delay.getSeconds()); | ||
| } | ||
| if (delay.getNano() > 0) { | ||
| TimeUnit.NANOSECONDS.sleep(delay.getNano()); | ||
| } | ||
| }; | ||
|
|
||
| private final RetryStrategy retryStrategy; | ||
| private final Sleeper sleeper; | ||
|
|
||
| PayloadOffloadRetryExecutor(RetryStrategy retryStrategy, Sleeper sleeper) { | ||
| this.retryStrategy = Objects.requireNonNull(retryStrategy, "retryStrategy cannot be null"); | ||
| this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); | ||
| } | ||
|
|
||
| <T> T execute(String action, Supplier<T> operation) { | ||
| int attempt = 1; | ||
| while (true) { | ||
| try { | ||
| return operation.get(); | ||
| } catch (RetryablePayloadOffloadException failure) { | ||
| var decision = makeRetryDecision(action, failure, attempt); | ||
| if (!decision.shouldRetry()) { | ||
| throw failure; | ||
| } | ||
| waitForRetry(action, failure, attempt, decision.delay()); | ||
| attempt++; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private RetryDecision makeRetryDecision(String action, RetryablePayloadOffloadException failure, int attempt) { | ||
| try { | ||
| var decision = retryStrategy.makeRetryDecision(failure, attempt); | ||
| if (decision == null) { | ||
| throw new PayloadOffloadException( | ||
| String.format("Retry strategy returned null for payload %s attempt %d", action, attempt)); | ||
| } | ||
| return decision; | ||
| } catch (PayloadOffloadException e) { | ||
| throw e; | ||
| } catch (RuntimeException e) { | ||
| throw new PayloadOffloadException( | ||
| String.format("Retry strategy failed for payload %s attempt %d", action, attempt), e); | ||
| } | ||
| } | ||
|
|
||
| private void waitForRetry(String action, RetryablePayloadOffloadException failure, int attempt, Duration delay) { | ||
| if (delay == null || delay.isNegative()) { | ||
| throw new PayloadOffloadException(String.format( | ||
| "Retry strategy returned an invalid delay for payload %s attempt %d", action, attempt)); | ||
| } | ||
| if (delay.isZero()) { | ||
| return; | ||
| } | ||
| try { | ||
| sleeper.sleep(delay); | ||
| } catch (InterruptedException e) { | ||
| Thread.currentThread().interrupt(); | ||
| var interrupted = new RetryablePayloadOffloadException( | ||
| String.format("Interrupted while waiting to retry payload %s after attempt %d", action, attempt), | ||
| e); | ||
| interrupted.addSuppressed(failure); | ||
| throw interrupted; | ||
| } | ||
| } | ||
|
|
||
| @FunctionalInterface | ||
| interface Sleeper { | ||
| void sleep(Duration delay) throws InterruptedException; | ||
| } | ||
| } | ||
32 changes: 32 additions & 0 deletions
32
sdk/src/main/java/software/amazon/lambda/durable/offload/RetryPayloadOffloader.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| package software.amazon.lambda.durable.offload; | ||
|
|
||
| import java.util.Objects; | ||
| import software.amazon.lambda.durable.retry.RetryStrategy; | ||
|
|
||
| /** A payload-offloader decorator that retries explicitly retryable storage failures. */ | ||
| public final class RetryPayloadOffloader implements PayloadOffloader { | ||
| private final PayloadOffloader delegate; | ||
| private final PayloadOffloadRetryExecutor retryExecutor; | ||
|
|
||
| public RetryPayloadOffloader(PayloadOffloader delegate, RetryStrategy retryStrategy) { | ||
| this(delegate, retryStrategy, PayloadOffloadRetryExecutor.DEFAULT_SLEEPER); | ||
| } | ||
|
|
||
| RetryPayloadOffloader( | ||
| PayloadOffloader delegate, RetryStrategy retryStrategy, PayloadOffloadRetryExecutor.Sleeper sleeper) { | ||
| this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); | ||
| retryExecutor = new PayloadOffloadRetryExecutor(retryStrategy, sleeper); | ||
| } | ||
|
|
||
| @Override | ||
| public OffloadedPayload offload(String serializedPayload, PayloadOffloadContext context) { | ||
| return retryExecutor.execute("store", () -> delegate.offload(serializedPayload, context)); | ||
| } | ||
|
|
||
| @Override | ||
| public String load(OffloadedPayload payload, PayloadOffloadContext context) { | ||
| return retryExecutor.execute("load", () -> delegate.load(payload, context)); | ||
| } | ||
| } |
12 changes: 12 additions & 0 deletions
12
sdk/src/main/java/software/amazon/lambda/durable/offload/filesystem/FieldMatchMode.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| package software.amazon.lambda.durable.offload.filesystem; | ||
|
|
||
| /** Controls how a {@link PreviewField} matches a field in a structured value. */ | ||
| public enum FieldMatchMode { | ||
| /** Matches the field name at any depth in the object tree. */ | ||
| ANYWHERE, | ||
|
|
||
| /** Matches the exact dot-separated path from the root object. */ | ||
| PATH | ||
| } |
12 changes: 12 additions & 0 deletions
12
...c/main/java/software/amazon/lambda/durable/offload/filesystem/FileSystemPathEncoding.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| package software.amazon.lambda.durable.offload.filesystem; | ||
|
|
||
| /** Controls how durable execution ownership is represented in payload file names. */ | ||
| public enum FileSystemPathEncoding { | ||
| /** Include a bounded, escaped entity prefix followed by a SHA-256 owner digest. */ | ||
| URI, | ||
|
|
||
| /** Use only the fixed-length SHA-256 owner digest. */ | ||
| HASH | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Codex AI review · Finding
arf_v1_5vmint2jktj7b53xftxnhr7dib[P1] An interrupted backoff is converted to
RetryablePayloadOffloadException.DurableExecutortreats this subtype asRETRYING, so cancellation can trigger a fresh Lambda invocation and repeat storage work. The filesystem classifier similarly marks actual I/O interruptions retryable. Restore the interrupt flag but throw a non-retryablePayloadOffloadException, and classifyClosedByInterruptExceptionor an interrupt-signaledInterruptedIOExceptionas permanent interruption.