diff --git a/.autover/changes/50f64d33-d1b0-445c-aac3-b2c95ae10ff6.json b/.autover/changes/50f64d33-d1b0-445c-aac3-b2c95ae10ff6.json new file mode 100644 index 000000000..8da8a2968 --- /dev/null +++ b/.autover/changes/50f64d33-d1b0-445c-aac3-b2c95ae10ff6.json @@ -0,0 +1,16 @@ +{ + "Projects": [ + { + "Name": "Amazon.Lambda.TestTool", + "Type": "Minor", + "ChangelogMessages": [ + "Add durable-execution service emulator (preview): with --durable-execution, the Test Tool hosts CheckpointDurableExecution and GetDurableExecutionState endpoints alongside the Lambda Runtime API so durable workflows can checkpoint and read state locally. Point the function's AWS_ENDPOINT_URL_LAMBDA at the Test Tool to use it. Timers are time-skipped by default (--durable-time-skip).", + "Support running a durable workflow locally end to end: invoking a function with the X-Amz-Durable-Execution-Name header starts a durable execution that the Test Tool drives to completion, re-invoking the function across the replay cycle (steps, waits, retries, child contexts, parallel/map). Chained durable-to-durable invokes are not yet supported.", + "Add external callback support and a Durable Execution web UI: SendDurableExecutionCallbackSuccess/Failure/Heartbeat endpoints resume a workflow parked on a callback, and a new Durable Execution page lists executions, shows each operation timeline, and can send a callback from the browser.", + "Complete local durable workflow support: chained durable-to-durable invokes (ctx.InvokeAsync) now run the target function as a nested execution and return its result to the caller; StopDurableExecution halts a running execution; and checkpoint payloads exceeding the service size limits are rejected.", + "Start durable executions from the web UI: the Function Tester page adds a 'Start as durable execution' checkbox (with an optional execution name) so a durable workflow can be started without setting the X-Amz-Durable-Execution-Name header manually. Re-Invoke on a durable function now starts a fresh execution from the original user payload instead of replaying an internal envelope. Adds a durable-execution sample request and documents the feature in the README and in-app documentation.", + "Fix --durable-time-skip defaulting to false (timers were never skipped); it now defaults to true and accepts an explicit value (--durable-time-skip false). Add a live 'Time-skip timers' toggle to the Durable Execution page. Completed WAIT operations now show SUCCEEDED in the timeline instead of remaining STARTED after their timer elapses." + ] + } + ] +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/docs/design/testtool-integration-plan.md b/Libraries/src/Amazon.Lambda.DurableExecution/docs/design/testtool-integration-plan.md new file mode 100644 index 000000000..5f2a4784d --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution/docs/design/testtool-integration-plan.md @@ -0,0 +1,293 @@ +# Durable Execution support in Lambda Test Tool v2 — Design + +**Status:** Proposed · **Phase 0 spike:** ✅ complete (endpoint redirection verified) +**Audience:** contributors to `Tools/LambdaTestTool-v2` and `Amazon.Lambda.DurableExecution*` + +## 1. Goal + +Let a developer run a **real** durable Lambda function locally (`dotnet run`, debugger attached) and +exercise a full multi-invocation durable workflow — steps, waits, retries, callbacks, child contexts, +parallel/map — without deploying to AWS and without a real durable-execution service. The Test Tool +already emulates the Lambda Runtime API for ordinary functions; this adds the **durable-execution +service data plane** as a sibling emulator on the same host. + +Two experiences result: +- **Headless** — invoke a durable workflow and watch it drive to completion (time-skipped by default). +- **Interactive** — step through the replay cycle in a debugger; inspect the operation timeline and fire + callbacks from the Blazor UI. + +The in-process `DurableTestRunner` (in `Amazon.Lambda.DurableExecution.Testing`) already covers automated +unit/integration testing. It is **not** a substitute for this: it drives an in-process C# delegate, so it +cannot run the developer's real, separately-launched executable under a debugger — which is the entire +point of the Test Tool. See §3 for why the orchestrator cannot simply be reused. + +## 2. How the two systems work today (verified) + +### 2.1 Test Tool v2 (`Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool`) + +- Accepts SDK invokes at `POST /2015-03-31/functions/{functionName}/invocations` and a fixed + `/functions/function/invocations` default variant (`LambdaRuntimeAPI.cs:25-26`). +- Queues events into a per-function `RuntimeApiDataStore`, partitioned **by URL path** through + `RuntimeApiDataStoreManager` (a `ConcurrentDictionary` keyed by function name, `:40-52`). The partition + is selected by the path, **not** by `AWS_LAMBDA_FUNCTION_NAME`. +- Exposes the Runtime API a local bootstrap polls: `GET /{fn}/2018-06-01/runtime/invocation/next` + (`GetNextInvocation`, polls `TryActivateEvent` every 100 ms, `:120`) and + `POST .../invocation/{awsRequestId}/response` (`PostInvocationResponse` → `ReportSuccess`, `:189`). +- `EventContainer` tracks a request/response pair and blocks the caller in RequestResponse mode via + `WaitForCompletion` — a **fail-safe 15-minute bounded wait**, not indefinite (`EventContainer.cs:86-96`). + RequestResponse is the default unless `X-Amz-Invocation-Type` says otherwise (`LambdaRuntimeAPI.cs:56-60`). +- **Hosting model:** `TestToolProcess` is a plain class with a static `Startup(...)` factory (not a + `BackgroundService`, not a base class); endpoints register from `TestToolProcess.cs:108`. Only the SQS / + DynamoDB event sources use real `BackgroundService`. The API Gateway emulator uses a + `app.Map("/{**catchAll}", ...)` catch-all (`ApiGatewayEmulatorProcess.cs:86`). Subsystems are wired into + the run from `RunCommand.cs:49-97` — which starts **only the tool's own web app and a browser**; it does + **not** launch the user's function. + +### 2.2 Durable Execution SDK (`Libraries/src/Amazon.Lambda.DurableExecution`) + +- The function's handler delegates to `DurableFunction.WrapAsync`, which hydrates `ExecutionState` from the + invocation envelope's `InitialExecutionState.Operations` (+ `NextMarker` paging), runs the workflow, and + returns a `DurableExecutionInvocationOutput` with status **Succeeded / Failed / Pending**. +- During a run it calls exactly **two** data-plane RPCs through `IDurableServiceClient` + (`Services/IDurableServiceClient.cs`): `CheckpointAsync` (flush `OperationUpdate`s) and + `GetExecutionStateAsync` (page state). The production adapter `LambdaDurableServiceClient` calls + `IAmazonLambda.CheckpointDurableExecutionAsync` / `GetDurableExecutionStateAsync`. +- **The SDK owns no timers.** Suspension = returning a never-completing Task and emitting `Pending` + (`TerminationManager`). Timers, retry backoff, and re-invocation are the **service's** responsibility. +- By default `DurableFunction.cs:30-31` constructs `new AmazonLambdaClient()` with **no config**, cached in + a `Lazy`. There is also a `WrapAsync(..., IAmazonLambda lambdaClient)` overload (`DurableFunction.cs:46-52`) + for supplying a custom client. + +### 2.3 Testing package (`Amazon.Lambda.DurableExecution.Testing`) + +- `ExecutionOrchestrator` plays the service role **in-process**: it repeatedly calls + `DurableFunction.WrapAsync` with an in-memory `IDurableServiceClient`, interprets `Pending`, and drives + re-invocation until terminal (`ExecutionOrchestrator.cs:83-136`). +- Time-skipping lives in `CheckpointProcessor.ApplyTimeSkipping` (`CheckpointProcessor.cs:273-298`), which + mutates stored ops (`WAIT → Succeeded`, retry `STEP → Ready`) **at checkpoint-application time**. +- The reusable backend trio — `InMemoryOperationStore`, `CheckpointProcessor`, `InMemoryDurableServiceClient` + — are all `internal sealed`, wired at `DurableTestRunner.cs:60-62`. They already traffic in + `Amazon.Lambda.Model.OperationUpdate` / `Operation`. + +### 2.4 Wire format (`aws-sdk-net` `generator/ServiceModels/lambda/lambda-2015-03-31.normal.json`) + +- REST-JSON with path templates. `CheckpointDurableExecution` → + `POST /2025-12-01/durable-executions/{DurableExecutionArn}/checkpoint`; `GetDurableExecutionState` → + `GET /2025-12-01/durable-executions/{DurableExecutionArn}/state`; callbacks → + `POST /2025-12-01/durable-execution-callbacks/{CallbackId}/{succeed|fail|heartbeat}`. +- **There is no `StartDurableExecution` operation** and **no named `DurableExecutionInvocationInput/Output` + wire shape.** An execution starts as an ordinary `Invoke` carrying the `X-Amz-Durable-Execution-Name` + header; the response returns `X-Amz-Durable-Execution-Arn`. The `InvocationInput/Output` envelope is an + **SDK-side POCO** that rides the opaque Runtime-API event/response. +- `DurableExecutionArn` **contains slashes** (`arn:...:function:NAME:QUALIFIER/durable-execution/GROUP/NAME`), + so an emulator router must use a catch-all and handle URL-encoded slashes. +- Payload caps: 6 MB max `OperationPayload`/`InputPayload` string length; effective caps are 256 KB + (CONTEXT/STEP/WAIT/CALLBACK) and 1 MB (async/chained). `ReplayChildren` exists on both checkpoint updates + (`ContextOptions.ReplayChildren`) and state responses (`ContextDetails.ReplayChildren`). +- **ID allocation is split:** the **server (emulator)** allocates the `DurableExecutionArn` and the rotating + `CheckpointToken`; the **SDK (client)** allocates operation IDs and callback IDs. The emulator only mints + ARNs and tokens. + +## 3. Corrections to the original investigation + +| Claim | Verdict | Correction | +|---|---|---| +| Reuse `ExecutionOrchestrator` in the Test Tool | **Rejected** | It holds a generic in-process `Func>` (`ExecutionOrchestrator.cs:16`, invoked `:95`) with no serialization boundary. It cannot drive a separate process. Reuse the **backend trio**; **re-write** the ~50-line drive loop over the wire. | +| Test Tool injects env vars into the function process | **False** | The Test Tool never launches the function (`RunCommand.cs:49-97`). The **user** sets the env vars on their own `dotnet run`, matching today's `AWS_LAMBDA_RUNTIME_API` UX. | +| Redirect the durable client via a ServiceURL hook in the SDK | **Resolved (see §4)** | No bespoke SDK hook; relies on stock AWSSDK `AWS_ENDPOINT_URL_LAMBDA` resolution. **Verified working** by the Phase-0 spike. | +| Feed `CheckpointDurableExecutionRequest` straight into `CheckpointProcessor.Process` | **False as written** | `Process` takes `Amazon.Lambda.Model.OperationUpdate`, whose `Action`/`Type` are AWSSDK `ConstantClass` types with **no System.Text.Json contract**. The emulator must define its own rest-json DTOs and **hand-map** them into the model types. This mapper is the bulk of the data-plane work. | +| `TestToolProcess`-style background services | **Partly** | `TestToolProcess` is not a `BackgroundService` and not a base class. The durable driver should follow the Runtime-API model — a per-execution `Task` — not `AddHostedService`. | +| Time-skipping incompatible with a wall-clock re-invoke loop | **Resolved** | True only against the **real** service. Here the Test Tool **owns** the store the function reads back on replay, so `ApplyTimeSkipping` is authoritative and is the correct fast default. | +| IVT already lets the Test Tool reuse the backend trio | **False** | The trio is `internal sealed`; `.Testing` grants IVT only to `.Testing.Tests`. A **new IVT entry (or public facade)** plus matching strong-name is required. Prefer a small public facade to avoid coupling to 0.x internals. | + +## 4. Phase 0 spike — endpoint redirection (COMPLETE ✅) + +**Question:** does the bundled `AWSSDK.Lambda 4.0.13.1` route `CheckpointDurableExecutionAsync` to +`AWS_ENDPOINT_URL_LAMBDA` when the client is `new AmazonLambdaClient()` with no config — i.e. exactly the +`DurableFunction` default path? + +**Method** (`C:\tmp\durable-endpoint-spike`): a local `HttpListener` stub; set only +`AWS_ENDPOINT_URL_LAMBDA`, `AWS_REGION`, and dummy creds; construct `new AmazonLambdaClient()`; call +`CheckpointDurableExecutionAsync` and observe where the request lands. + +**Result: SUCCESS.** The SDK resolved `client.Config.ServiceURL` to the stub and sent: +``` +POST /2025-12-01/durable-executions/arn%3Aaws%3Alambda%3A...%2Fdurable-execution%2Fg%2Fe/checkpoint +User-Agent: aws-sdk-dotnet-coreclr/4.0.13.1 ... api/Lambda#4.0.13.1 +``` +Confirmed in AWSSDK.Core: `ClientConfig.ServiceURL` reads the service-specific +`AWS_ENDPOINT_URL_LAMBDA` (then global `AWS_ENDPOINT_URL`) whenever `IgnoreConfiguredEndpointUrls == false` +(default) and `ServiceId != null` (`ClientConfig.cs:340-395`). + +**Implications:** +- The **zero-user-code-change** story holds: the developer sets env vars, no SDK modification needed. +- Note the ARN's slashes arrive **URL-encoded** (`%2F`) — the emulator router must decode them. +- The user must also set `AWS_REGION` + dummy `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, or SigV4 signing + throws before the request leaves the process; the emulator ignores the signature. +- **Fallback (not needed):** if a future SDK build regressed this, the `WrapAsync(IAmazonLambda)` overload + (`DurableFunction.cs:46-52`) with a caller-built `AmazonLambdaConfig { ServiceURL }` would work but require + a user code change. + +## 5. Recommended architecture + +Single Kestrel `WebApplication` (the existing `TestToolProcess`), one port, three route groups, one +out-of-process function. + +**Env vars the user sets on their `dotnet run`:** +- `AWS_LAMBDA_RUNTIME_API=127.0.0.1:{port}/{functionName}` — the bootstrap poll target. **Must include the + function-name path prefix** so the bootstrap's `BaseUrl` hits the named partition (`RuntimeApiDataStoreManager`). +- `AWS_ENDPOINT_URL_LAMBDA=http://127.0.0.1:{port}` — redirects checkpoint/state/callback traffic (verified §4). +- `AWS_REGION` + dummy `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` — required for signing; ignored by the emulator. + +**Route group A — Runtime API (existing, unchanged).** Transports the durable envelope in/out via the +event queue + `EventContainer`. + +**Route group B — Durable data plane (NEW).** Registered via `SetupDurableServiceEndpoints(WebApplication)` +called next to the Runtime-API setup at `TestToolProcess.cs:108`: +- `POST /2025-12-01/durable-executions/{**path}` (suffix `/checkpoint`, `/stop`) +- `GET /2025-12-01/durable-executions/{**path}` (bare, `/state`, `/history`) +- `POST /2025-12-01/durable-execution-callbacks/{callbackId}/{succeed|fail|heartbeat}` + +Uses a catch-all `{**path}` + suffix parse (the `ApiGatewayEmulatorProcess.cs:86` pattern) because the ARN +contains encoded slashes. Each handler deserializes into **emulator-owned STJ DTOs**, hand-maps to +`Amazon.Lambda.Model.OperationUpdate`, calls `CheckpointProcessor.Process(arn, token, updates)` over a +shared `InMemoryOperationStore`, and returns `NewExecutionState { Operations, NextMarker }` with a +**freshly minted CheckpointToken**. + +**Route group C — Durable start (NEW).** Extend `PostEvent` (`LambdaRuntimeAPI.cs:51`) to branch on the +`X-Amz-Durable-Execution-Name` header. It does **not** take the blocking `EventContainer` path; it allocates +the ARN + initial token, seeds an `EXECUTION`-type op with the input payload, launches the drive loop as a +detached `Task`, and returns **202 immediately** with `X-Amz-Durable-Execution-Arn`. Idempotency: same +name + same payload → existing ARN; differing payload → `DurableExecutionAlreadyStartedException`. + +**Drive loop (`DurableExecutionDriver`, re-expressing `ExecutionOrchestrator.cs:83-136` over the wire):** +1. Read the **current** CheckpointToken from the store (rotates each checkpoint — never reuse the prior). +2. Build the invocation envelope with a **bounded first page** of `InitialExecutionState.Operations` + + `NextMarker` (honoring `ReplayChildren`). **Do not inline full history** — it breaches the 6 MB + queued-event cap (`LambdaRuntimeAPI.cs:14,65`); page the rest via `GET .../state`. +3. `GetLambdaRuntimeDataStore(fn).QueueEvent(inputJson, isRequestResponseMode:true)` + (`RuntimeApiDataStore.cs:92`), then block on `EventContainer.WaitForCompletion` — the **driver** blocks, + not the end caller. +4. While the function runs, its redirected client hits route group B; with time-skip on, + `ApplyTimeSkipping` flips `WAIT → Succeeded` / retry `STEP → Ready` in the store **at checkpoint time**, + before the function posts its Pending response — so on `WaitForCompletion` return the store shows no + pending timer. +5. Parse `DurableExecutionInvocationOutput.Status`: `Succeeded`/`Failed` → terminal, store result/error, + stop. `Pending` → re-drive from step 1. **Serialize passes per ARN** so the abandoned-event resend + (`LambdaRuntimeAPI.cs:120-134`) can't double-deliver. + +**Timers.** Default: time-skip. Optional wall-clock mode: `Task.Delay` to the min pending +`WaitDetails.ScheduledEndTimestamp` (mirrors `ExecutionOrchestrator.cs:123-135`) to watch real backoff. + +**Callbacks.** Inbound endpoints are keyed by `{CallbackId}`, **not** ARN. Maintain a `CallbackId → ARN` +reverse map to wake the correct parked driver. The `onNewOperations` cb-id feedback survives the HTTP hop as +long as the checkpoint response's `NewExecutionState.Operations` is populated — `LambdaDurableServiceClient` +already forwards minted cb-ids back into `ExecutionState`. + +## 6. Reused vs new + +**Reused verbatim:** +- Runtime-API transport: `LambdaRuntimeAPI.cs:26-38,120,189`, `RuntimeApiDataStore.QueueEvent` (`:92`), + `EventContainer.WaitForCompletion` (`:86-96`), `RuntimeApiDataStoreManager` (`:40-52`). +- Durable state machine: `CheckpointProcessor` (incl. `ApplyTimeSkipping` `:273-298`, cb minting `:167-170`), + `InMemoryOperationStore`, `InMemoryDurableServiceClient`. +- Public STJ envelope POCOs: `DurableExecutionInvocationInput/Output`, `InitialExecutionState`, `Operation`. +- Hosting/DI seams: `TestToolProcess.cs:43-84,108`; catch-all routing `ApiGatewayEmulatorProcess.cs:86`; + task-list wiring `RunCommand.cs:49-97`. +- Stock AWSSDK.Lambda `AWS_ENDPOINT_URL_LAMBDA` resolution (verified §4). + +**New:** +- `Services/DurableExecution/DurableServiceApi.cs` — `SetupDurableServiceEndpoints`, route group B. +- `Services/DurableExecution/DurableRestJsonDtos.cs` + mapper — emulator-owned STJ DTOs ⇄ + `Amazon.Lambda.Model.OperationUpdate/Operation` (**the largest new component**; must track + `Action{START/SUCCEED/FAIL/RETRY/CANCEL}`, `Type`, `Status` against `lambda-2015-03-31.normal.json`). +- `Services/DurableExecution/DurableExecutionDriver` (+ `IDurableExecutionDriver`) — start + per-ARN drive + loop, `ConcurrentDictionary`, callback reverse-map, `Stop`. +- `DurableExecutionStore` singleton — owns `CheckpointProcessor` + `InMemoryOperationStore` keyed by ARN; + mints ARNs + rotates tokens. +- Extension of `PostEvent` (`LambdaRuntimeAPI.cs:51`) for the `X-Amz-Durable-Execution-Name` branch. +- `RunCommandSettings` flags: `--durable-execution`, `--durable-time-skip` (default true). +- New **IVT entry / public facade** on the `.Testing` assembly for the Test Tool (+ strong-name match). +- (Phase 3) `Components/Pages/DurableExecutionPanel.razor` — timeline + callback buttons + time control, + using the existing `StateChange → StateHasChanged` pattern (`Home.razor.cs:114,218`). + +## 7. Risks & open questions + +1. ~~**Endpoint redirection**~~ — **resolved by the Phase-0 spike (§4).** +2. **rest-json ⇄ SDK-model mapping fidelity.** `ConstantClass` blocks STJ; the hand-map across every nested + option (Step/Wait/Callback/ChainedInvoke/Context, Error, `ReplayChildren`) is the real cost and must stay + in sync with the service model. **The most under-estimated piece of work.** +3. **Chained durable-to-durable invokes are not v1.** Out-of-process, `ctx.Invoke` would loop forever on + `InvokePending`. **Detect and fail fast** with a clear "chained invoke not yet supported" error until Phase 4. +4. **IVT + strong-naming coupling.** Locks the Test Tool version to the 0.x `.Testing` package. Prefer a + small public facade over raw IVT. +5. **Checkpoint-before-suspend ordering.** The driver time-skips on store state, so `WrapAsyncCore` must + flush the WAIT/retry op over HTTP **before** posting Pending, not on async dispose after. Verify. +6. **15-min `WaitForCompletion` cap** at a breakpoint — dev-acceptable, but the driver needs a clean + abort/re-attach path. +7. **Payload caps / `ReplayChildren`** tiered enforcement (6 MB / 1 MB / 256 KB) — fidelity, deferrable past + first ship but required before "high-fidelity" is claimed. + +## 8. Phased plan + +- **Phase 0 — Spike: endpoint redirection (S).** ✅ **Done.** Verified §4. +- **Phase 1 — Data plane + backend reuse (M).** ✅ **Done.** Chose to **copy** `CheckpointProcessor` + + `InMemoryOperationStore` into the Test Tool (retyped to plain-STJ DTOs) rather than IVT/facade, keeping + it decoupled from the 0.x preview internals. `DurableServiceApi` catch-all endpoints; `DurableExecutionStore` + over the copied backend. Delivered under `Services/DurableExecution/`, gated by `--durable-execution`. + Tests (`DurableServiceApiTests`) exercise checkpoint/state round-trips through a real `AmazonLambdaClient`. + **Bug found & fixed:** the checkpoint/state *response* is read by the AWSSDK rest-json unmarshaller (not + System.Text.Json), whose `timestamp` members are unix **seconds** while the `Operation` POCO emits epoch + **millis** — the direct-POCO serialization overflowed `DateTime`. Fixed with dedicated `WireOperation` + response DTOs that convert millis→seconds (validates §7 risk #2). +- **Phase 2 — Start hook + drive loop (M).** ✅ **Done.** `X-Amz-Durable-Execution-Name` branch in + `PostEvent`; `DurableExecutionDriver` (QueueEvent + WaitForCompletion, per-ARN idempotency by name + + payload hash, paged envelope, time-skip default, fail-fast on chained invokes, park on pending callback). + Registered when `--durable-execution` is set. `DurableExecutionDriverTests` drives a wait-then-step + workflow end to end (start → Pending → time-skip → Succeeded) through the real Runtime API + SDK wire. + **Bug found & fixed:** the ARN's internal slashes arrive percent-encoded (`%2F`) and ASP.NET's catch-all + preserves them un-decoded, so the checkpoint handler keyed the store under the encoded ARN while the driver + reads by the raw ARN — the two never met. Fixed by `Uri.UnescapeDataString` on the captured ARN in + `DurableServiceApi.TrySplit`. (Phase 1 tests didn't catch it because checkpoint+getstate were both + SDK-encoded and thus internally consistent.) **First genuinely useful ship** — single-function workflows + headless, debugger-attached. +- **Phase 3 — Callbacks + Blazor UI (M).** ✅ **Done.** Callback endpoints keyed by `{CallbackId}` + + a `CallbackId → (arn, opId)` index; the driver parks on a pending callback and resumes on + `NotifyCallbackResolved` (lock-guarded against a resolve racing the park). `DurableExecution.razor` lists + executions, shows the operation timeline, and offers a Send-Callback action. E2E test covers start → park + → SendCallbackSuccess → resume → Succeeded. +- **Phase 4 — Chained invokes + fidelity polish (L).** ✅ **Done.** `CHAINED_INVOKE` now resolves by + starting the sibling as a **nested durable execution** (`DurableExecutionDriver.ResolveChainedInvokesAsync`) + and stamping its result/error onto the parent op; the sibling must be polling the Runtime API under its own + function name. `StopDurableExecution` (`/stop` endpoint → `driver.Stop`) cancels the drive loop and marks + the EXECUTION op STOPPED. Checkpoint payloads over the tiered caps (256 KB general / 1 MB chained-invoke) + are rejected with 413. (Idempotency via `DurableExecutionAlreadyStartedException` landed in Phase 2.) + **Bug found & fixed:** the drive loop awaited `EventContainer.WaitForCompletion` on a threadpool thread — + a blocking 15-min wait that ignored cancellation, so `Stop` couldn't interrupt an in-flight invocation and + serialized tests leaked pinned threads. Replaced with a cancellation-aware status poll. The durable + web-host tests were also moved into a non-parallel xUnit collection (each starts a full Kestrel host + + 100 ms poll loop; running them concurrently starved the polls under CPU pressure). + +## Status + +All four phases are implemented, tested, and committed on `feature/testtool-durable-execution`. Remaining +work before GA is quality/fidelity, not new capability: a real out-of-process-executable integration test +(the E2E tests use a hand-written wire-protocol fake, not the DurableExecution SDK engine — see §"test-design +note" below), `ReplayChildren`/large-result paging fidelity, and `GetDurableExecution`/`ListDurableExecutionsByFunction` +read operations if the UI needs them. + +### Test-design note + +The end-to-end tests drive a **hand-written fake durable function** that speaks the wire protocol, not the +real `Amazon.Lambda.DurableExecution` SDK engine. This is deliberate: the SDK caches its checkpoint client in +a process-global `Lazy` keyed off a global env var, which would leak across xUnit tests in +one process. The SDK's replay engine is covered in its own package; these tests verify the Test Tool's +endpoints + driver. The gap — a mismatch between the real SDK's exact checkpoint sequence and the emulator — +is best closed by a separate integration test using a real out-of-process function executable (as the +DurableExecution integ tests already do). + +## Appendix — Phase-0 spike source + +Kept at `C:\tmp\durable-endpoint-spike\` (throwaway; not checked in). `Program.cs` stands up an +`HttpListener`, sets only `AWS_ENDPOINT_URL_LAMBDA` + region + dummy creds, constructs +`new AmazonLambdaClient()`, and calls `CheckpointDurableExecutionAsync`. Reproduce with `dotnet run -c Release`. diff --git a/Tools/LambdaTestTool-v2/README.md b/Tools/LambdaTestTool-v2/README.md index ba025aefa..cf5d50a86 100644 --- a/Tools/LambdaTestTool-v2/README.md +++ b/Tools/LambdaTestTool-v2/README.md @@ -78,6 +78,33 @@ Use this mode when you want to test Lambda functions directly without API Gatewa dotnet lambda-test-tool start --lambda-emulator-port 5050 ``` +### Durable Execution Mode (preview) +Use this mode to run [durable Lambda functions](../../Libraries/src/Amazon.Lambda.DurableExecution) locally. The Test Tool emulates the durable-execution service: it stores checkpoints, drives the workflow across the replay cycle, resolves timers, and manages callbacks — so you can run a full multi-step workflow (and attach a debugger) without deploying to AWS. + +Enable it with `--durable-execution`: + +``` +# Start the Lambda emulator with the durable-execution service emulator +dotnet lambda-test-tool start --lambda-emulator-port 5050 --durable-execution +``` + +By default, timers (`WaitAsync` and retry backoff) are **time-skipped** so workflows advance immediately. Pass `--durable-time-skip false` to wait real wall-clock time instead. + +**Point your function at the emulator.** Run your durable function as a normal local process (`dotnet run`) with these environment variables set: + +| Variable | Value | Why | +|---|---|---| +| `AWS_LAMBDA_RUNTIME_API` | `localhost:5050/` | Runtime API endpoint. **Must include the `/` suffix** — the tool partitions events by that path segment, and it must match the function name you invoke with. | +| `AWS_ENDPOINT_URL_LAMBDA` | `http://localhost:5050` | Redirects the durable checkpoint/state/callback calls to the Test Tool. | +| `AWS_REGION` | e.g. `us-east-1` | Required for request signing. | +| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | any dummy value | Required for signing; the emulator ignores the signature. | + +**Start a durable execution.** In the Test Tool web UI, select your function, check **Start as durable execution** (optionally name it), and click **Invoke**. Then open the **Durable Execution** page to watch the operation timeline (steps, waits, callbacks) as the workflow runs. + +**Callbacks.** When a workflow parks on `WaitForCallbackAsync`, the Durable Execution page shows a **Send Callback** action to resolve it and resume the workflow. Callbacks can also be sent with the AWS SDK's `SendDurableExecutionCallbackSuccess`/`Failure`/`Heartbeat` against the tool's endpoint. + +**Current limitations (preview):** chained durable-to-durable invokes run each target as a nested execution against a sibling function that must also be running locally; large-payload paging fidelity is approximate. + ### API Gateway Emulator Mode Use this mode when you want to test Lambda functions through API Gateway endpoints. **Note: Running this mode by itself will not work, you will still need have the lambda runtime client running elsewhere and reference it in the `Endpoint` parameter in the `APIGATEWAY_EMULATOR_ROUTE_CONFIG` env varible (see below [Required Configuration](#required-configuration))** Api gateway mode requires additional configuration through environment variables. diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj index b2ae28208..16c990bbb 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj @@ -23,6 +23,11 @@ + + diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs index f4ab3d57f..6cef95d26 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs @@ -97,6 +97,33 @@ public sealed class RunCommandSettings : CommandSettings [Description("The absolute path used to save global settings and saved requests. You will need to specify a path in order to enable saving global settings and requests.")] public string? ConfigStoragePath { get; set; } + /// + /// Enables the durable-execution service emulator: HTTP endpoints emulating the Lambda + /// durable-execution data plane (checkpoint / get-state) are hosted alongside the Runtime + /// API. Point the function's AWS_ENDPOINT_URL_LAMBDA at the emulator to run a durable + /// workflow locally. + /// + [CommandOption("--durable-execution")] + [Description("Enable the durable-execution service emulator (checkpoint/get-state endpoints) hosted alongside the Lambda Runtime API.")] + public bool DurableExecution { get; set; } + + /// + /// When the durable-execution emulator is enabled, resolves timers (WaitAsync, retry + /// backoff) immediately instead of waiting for wall-clock time. Defaults to true so + /// local workflows advance without real delays. Use --durable-time-skip false to wait + /// real wall-clock time. + /// + /// + /// Declared as a nullable bool with an explicit so the + /// Spectre.Console.Cli binder honors the "default true" behavior. A plain bool option + /// binds presence→true / absence→false regardless of a C# field initializer, which would make + /// the default effectively false and prevent --durable-time-skip false from parsing. + /// + [CommandOption("--durable-time-skip ")] + [Description("When durable execution is enabled, resolve timers/retry backoff immediately rather than waiting for wall-clock time. Default: true.")] + [DefaultValue(true)] + public bool DurableTimeSkip { get; set; } = true; + /// /// Validate that is an absolute path. /// diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Layout/MainLayout.razor b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Layout/MainLayout.razor index dd2e997fc..873af3d88 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Layout/MainLayout.razor +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Layout/MainLayout.razor @@ -25,6 +25,14 @@ Function Tester +
  • + + + + + Durable Execution + +
  • diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Documentation.razor b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Documentation.razor index 2cff94fee..280b1d049 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Documentation.razor +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Documentation.razor @@ -11,3 +11,25 @@ Please see the README file here for more information.

    +

    Durable Execution (preview)

    +
    +

    + Run durable Lambda functions + locally. Start the tool with --durable-execution and the tool emulates the durable-execution + service: it stores checkpoints, drives the workflow across its replay cycle, time-skips timers, and manages callbacks. +

    +
      +
    1. Start the tool with --durable-execution (timers are time-skipped by default; use --durable-time-skip false for real waits).
      2. +
    2. + Run your durable function as a local process with these environment variables: +
        +
      • AWS_LAMBDA_RUNTIME_API=localhost:<port>/<FunctionName> — must include the /<FunctionName> suffix.
        • +
      • AWS_ENDPOINT_URL_LAMBDA=http://localhost:<port> — redirects checkpoint/state/callback calls to the tool.
        • +
      • AWS_REGION plus dummy AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY — required for request signing; the emulator ignores the signature.
        • +
      +
      3. +
    3. On the Function Tester page, select your function, check Start as durable execution (optionally name it), and click Invoke.
      4. +
    4. Open the Durable Execution page to watch the operation timeline. When a workflow parks on a callback, use Send Callback to resume it.
      5. +
    +
    + diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/DurableExecution.razor b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/DurableExecution.razor new file mode 100644 index 000000000..b74a7fa45 --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/DurableExecution.razor @@ -0,0 +1,154 @@ +@page "/durable-execution" +@using Amazon.Lambda.DurableExecution + +Durable Execution + +
    +

    Durable Execution

    + + @if (Driver is null) + { + + } + else + { +

    + Durable executions started against this Test Tool. Invoke your function with the + X-Amz-Durable-Execution-Name header (the AWS SDK sets this from + DurableExecutionName) to start one. +

    + +
    + + +
    + +
    +
    +
    Executions
    + @if (_executions.Count == 0) + { +
    No executions yet.
    + } + else + { +
    + @foreach (var exec in _executions) + { + + } +
    + } +
    + +
    + @if (_selectedArn is null) + { +
    Select an execution to view its timeline.
    + } + else + { +
    Timeline
    + + + + + + + + + + + @foreach (var op in _operations) + { + + + + + + + } + +
    OperationTypeStatusDetail
    @op.Id@if (!string.IsNullOrEmpty(op.Name)) { (@op.Name) }@op.Type@if (!string.IsNullOrEmpty(op.SubType)) { /@op.SubType }@op.Status + @{ var pendingCb = PendingCallbackId(op); } + @if (pendingCb is not null) + { + + } + else + { + @DetailText(op) + } +
    + +
    +
    Send Callback
    +
    +
    + + +
    +
    + + +
    + + @if (_callbackFeedback is not null) + { +
    @_callbackFeedback
    + } +
    +
    + } +
    +
    + } +
    + +@code { + private static string ShortName(string arn) + { + // Show the trailing "GROUP/NAME" of the durable-execution ARN for readability. + var idx = arn.IndexOf("/durable-execution/", StringComparison.Ordinal); + return idx >= 0 ? arn[(idx + "/durable-execution/".Length)..] : arn; + } + + private static string DetailText(Operation op) + { + if (op.StepDetails?.Result is { } r) return Truncate(r); + if (op.ContextDetails?.Result is { } cr) return Truncate(cr); + if (op.CallbackDetails?.Result is { } cbr) return Truncate(cbr); + if (op.ChainedInvokeDetails?.Result is { } ir) return Truncate(ir); + if (op.ExecutionDetails?.InputPayload is { } ip) return "input: " + Truncate(ip); + if (op.StepDetails?.Error?.ErrorMessage is { } em) return "error: " + em; + return string.Empty; + } + + private static string Truncate(string s) => s.Length <= 80 ? s : s[..77] + "..."; +} diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/DurableExecution.razor.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/DurableExecution.razor.cs new file mode 100644 index 000000000..585a7883f --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/DurableExecution.razor.cs @@ -0,0 +1,145 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.TestTool.Services.DurableExecution; +using Microsoft.AspNetCore.Components; + +namespace Amazon.Lambda.TestTool.Components.Pages; + +/// +/// Web UI for the durable-execution emulator: lists local durable executions, shows each one's +/// operation timeline, and lets the developer resolve a parked callback ("Send Callback"). +/// Only meaningful when the Test Tool was started with --durable-execution; otherwise the +/// driver is not registered and the page shows a hint. +/// +public partial class DurableExecution : ComponentBase, IDisposable +{ + // Resolve the driver optionally: it is only registered when --durable-execution was passed, + // and [Inject] on a missing service would throw. Pulling it from the provider lets the page + // render a friendly "not enabled" state instead. + [Inject] public required IServiceProvider ServiceProvider { get; set; } + + internal DurableExecutionDriver? Driver { get; private set; } + + private IReadOnlyList _executions = new List(); + private string? _selectedArn; + private IReadOnlyList _operations = new List(); + + // Send-callback form state. + private string _callbackId = string.Empty; + private string _callbackResult = "\"approved\""; + private string? _callbackFeedback; + + // Mirrors the driver's runtime time-skip setting for the toggle. + private bool _skipTime; + + protected override void OnInitialized() + { + Driver = (DurableExecutionDriver?)ServiceProvider.GetService(typeof(DurableExecutionDriver)); + if (Driver is not null) + { + _skipTime = Driver.SkipTime; + Driver.StateChange += OnDriverStateChange; + Refresh(); + } + } + + private void OnSkipTimeToggled(ChangeEventArgs e) + { + _skipTime = e.Value is true; + if (Driver is not null) + Driver.SkipTime = _skipTime; + StateHasChanged(); + } + + private void OnDriverStateChange(object? sender, EventArgs e) => InvokeAsync(() => + { + Refresh(); + StateHasChanged(); + }); + + private void Refresh() + { + if (Driver is null) + return; + + _executions = Driver.GetExecutions() + .OrderBy(r => r.Arn, StringComparer.Ordinal) + .ToList(); + + // Keep a selection; default to the first execution. + _selectedArn ??= _executions.FirstOrDefault()?.Arn; + if (_selectedArn is not null) + _operations = Driver.GetOperations(_selectedArn); + } + + private void SelectExecution(string arn) + { + _selectedArn = arn; + _callbackFeedback = null; + if (Driver is not null) + _operations = Driver.GetOperations(arn); + StateHasChanged(); + } + + private void PrefillCallback(string callbackId) + { + _callbackId = callbackId; + _callbackFeedback = null; + StateHasChanged(); + } + + private void SendCallbackSuccess() + { + if (Driver is null) + return; + + if (string.IsNullOrWhiteSpace(_callbackId)) + { + _callbackFeedback = "Enter a callback id first."; + StateHasChanged(); + return; + } + + var outcome = Driver.SendCallback(_callbackId, _callbackResult, error: null); + _callbackFeedback = outcome switch + { + DurableExecutionStore.CallbackResolution.Resolved => $"Callback '{_callbackId}' resolved; execution resumed.", + DurableExecutionStore.CallbackResolution.AlreadyResolved => $"Callback '{_callbackId}' was already completed.", + _ => $"Unknown callback id '{_callbackId}'." + }; + Refresh(); + StateHasChanged(); + } + + private static string PhaseBadgeClass(DurableExecutionDriver.ExecutionPhase phase) => phase switch + { + DurableExecutionDriver.ExecutionPhase.Succeeded => "text-bg-success", + DurableExecutionDriver.ExecutionPhase.Failed => "text-bg-danger", + DurableExecutionDriver.ExecutionPhase.ParkedOnCallback => "text-bg-warning", + _ => "text-bg-secondary" + }; + + private static string StatusBadgeClass(string? status) => status switch + { + OperationStatuses.Succeeded => "text-bg-success", + OperationStatuses.Failed => "text-bg-danger", + OperationStatuses.Cancelled or OperationStatuses.Stopped or OperationStatuses.TimedOut => "text-bg-danger", + OperationStatuses.Started or OperationStatuses.Pending => "text-bg-warning", + OperationStatuses.Ready => "text-bg-info", + _ => "text-bg-secondary" + }; + + /// The callback id awaiting resolution for an op, if this is a started CALLBACK. + private static string? PendingCallbackId(Operation op) => + op.Type == OperationTypes.Callback && op.Status == OperationStatuses.Started + ? op.CallbackDetails?.CallbackId + : null; + + public void Dispose() + { + if (Driver is not null) + Driver.StateChange -= OnDriverStateChange; + } +} diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Home.razor b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Home.razor index 43093e3d6..def1ff4c0 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Home.razor +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Components/Pages/Home.razor @@ -59,6 +59,26 @@ else + @if (DurableExecutionEnabled) + { +
    +
    + + +
    + @if (_durableExecution) + { + + + Watch progress on the Durable Execution page. + + } +
    + }