a valid extracted backend context anchors the execution trace (this is what makes the durable spans share one
@@ -34,4 +36,15 @@ public interface ContextExtractor {
* @return the extracted context, or {@code null} if no context is available
*/
ExtractedContext extract();
+
+ /**
+ * Extracts context using the invocation and its immutable X-Ray header snapshot. Existing extractors retain their
+ * no-argument behavior; implementations that need invocation-local carriers can override this method.
+ *
+ * A null snapshot denotes an unavailable runtime accessor and permits legacy fallback. An empty string denotes
+ * an available accessor with no invocation header; the built-in extractor treats it as authoritative absence.
+ */
+ default ExtractedContext extract(InvocationInfo info) {
+ return extract();
+ }
}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java
index b23bb19835..617bab5bb5 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java
@@ -21,11 +21,13 @@
import java.time.Instant;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.ExclusivePluginGroup;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.HandlerScoped;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.OperationEndInfo;
@@ -63,10 +65,21 @@
*
*
The Workflow and Invocation spans share one execution trace, anchored at the execution ancestor resolved at
* invocation start: a valid propagated remote server span becomes that ancestor directly, otherwise a synthetic
- * execution root anchors the trace. The trace ID is stable across invocations of the same execution. When using
- * {@link #ExecutionOtelPlugin()}, the plugin resolves the global provider at invocation start. If the OpenTelemetry
- * Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider resolution is
- * retried on the next invocation.
+ * execution root anchors the trace. The trace ID is stable across invocations of the same execution, because it is
+ * derived from the execution ARN and start time rather than carried in the plugin.
+ *
+ *
Lifetime. One instance serves exactly one Lambda invocation: {@link #factory()} and its overloads return a
+ * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation. Invocation-level references are
+ * released at return; an unfinished operation retains its matching cleanup until its owner unwinds. Everything about
+ * the invocation — the execution ARN, the resolved execution trace and ancestor, the sampling intent, the Invocation
+ * span, the deferred Workflow span context — is therefore a {@code final} field, resolved in the constructor from the
+ * {@link InvocationInfo} the factory receives. Nothing is reset between invocations because nothing is carried between
+ * them.
+ *
+ *
What belongs to the execution environment stays in the factory's {@link OtelPluginEnvironment}: the configuration,
+ * the ID generator, and either the application-owned tracer provider (built once) or the lazily resolved ADOT global
+ * provider. An invocation whose instance cannot resolve the global provider emits no telemetry at all, and the next
+ * invocation's instance resolves it again.
*
*
Status mapping (parity with the Python/JS references):
*
@@ -86,47 +99,63 @@
* current, so {@code Span.current()} enrichment is not recorded on the final operation span. The placeholder uses the
* Invocation span's resolved sampling metadata when available.
*
- *
Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple
- * threads.
+ *
Thread-safe within its invocation: the SDK runs user code on multiple threads, so the open-span registries are
+ * {@link ConcurrentHashMap}s. The invocation's identity needs no such protection — it is final state written before the
+ * SDK publishes the instance to those threads.
*/
-@ExclusivePluginGroup("durable-otel-view")
-public class ExecutionOtelPlugin implements DurableExecutionPlugin {
+@HandlerScoped(ExecutionOtelPlugin.HandlerScopeOpener.class)
+public final class ExecutionOtelPlugin implements DurableExecutionPlugin {
private static final Logger logger = LoggerFactory.getLogger(ExecutionOtelPlugin.class);
- private volatile SdkTracerProvider sdkTracerProvider;
- private volatile Tracer tracer;
+ // ─── Environment lifetime (shared with every other invocation's instance) ─────────────
+
private final DeterministicIdGenerator idGenerator;
- private final ContextExtractor contextExtractor;
private final boolean enableMdc;
private final String workflowSpanName;
- private final String instrumentationName;
-
- // Per-invocation state
- private volatile boolean tracingEnabled;
- private volatile Span invocationSpan;
- private volatile Span executionRootSpan;
- // The root handler opens and closes this context through the base invocation lifecycle.
- private Scope handlerScope;
- private volatile String durableExecutionArn;
-
- // Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID
- // with mismatched flags.
- private volatile ExecutionTrace executionTrace;
- // The execution's single sampling intent for this invocation, computed once at onInvocationStart and attached to
- // every durable span's parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to
- // its own delegate) without re-invoking the configured sampler per span.
- private volatile DurableSamplingDecision.Intent samplingIntent;
-
- /** Immutable snapshot of the resolved execution trace, read atomically through a single volatile reference. */
- private record ExecutionTrace(String traceId, TraceFlags flags) {}
- // Between invocations the Workflow span exists only as a deterministic context that operations parent onto; the
- // recording span is started and ended in a single call on the terminal invocation, so it is never left open. The
- // execution ancestor and start time are retained so that span can be built at invocation end.
- private volatile SpanContext workflowSpanContext;
- private volatile SpanContext executionAncestor;
- private volatile Instant executionStartTime;
+ // ─── This invocation, all resolved in the constructor from its InvocationInfo ─────────
+
+ /** The provider used to flush before Lambda freezes; null when it is not visible to the application. */
+ private final SdkTracerProvider sdkTracerProvider;
+
+ /** Null when telemetry is disabled for this invocation, which makes every hook on this instance a no-op. */
+ private final Tracer tracer;
+
+ private final String durableExecutionArn;
+ private final Instant executionStartTime;
+
+ /** Trace ID and flags of the execution trace, resolved together so they can never be paired mismatched. */
+ private final ExecutionTrace executionTrace;
+
+ /**
+ * The execution's single sampling intent for this invocation, resolved once and attached to every durable span's
+ * parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to its own delegate)
+ * without re-invoking the configured sampler per span.
+ */
+ private final DurableSamplingDecision.Intent samplingIntent;
+
+ private final Span invocationSpan;
+ private final Span executionRootSpan;
+
+ /**
+ * The Workflow span exists as a deterministic context that operations parent onto; the recording span is started
+ * and ended in a single call on the terminal invocation, so it is never left open. The execution ancestor and start
+ * time are held so that span can be built at invocation end.
+ */
+ private final SpanContext workflowSpanContext;
+
+ private final SpanContext executionAncestor;
+
+ /**
+ * Set when this invocation ends; never cleared, because an instance is never reused. Read by the operation and user
+ * function hooks, which may run on other threads of this invocation, so that a straggler hook arriving after the
+ * spans have been ended does not open a new one — volatile for that publication.
+ */
+ private volatile boolean ended;
+
+ /** Immutable snapshot of the resolved execution trace. */
+ private record ExecutionTrace(String traceId, TraceFlags flags) {}
// Thread-safe storage for attempt spans/scopes (keyed by operationId + "-" + attempt)
private final ConcurrentHashMap attemptSpans = new ConcurrentHashMap<>();
@@ -141,91 +170,128 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {}
private final ConcurrentHashMap operationStartTimes = new ConcurrentHashMap<>();
/**
- * Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction, MDC enabled, root span
- * named {@code "Workflow"}.
+ * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider,
+ * with default settings: X-Ray context extraction, MDC enabled, root span named {@code "Workflow"}.
*
- * Uses the provided tracer provider builder. For ADOT Java agent usage, prefer {@link #ExecutionOtelPlugin()}
- * with the plugin jar configured through {@code OTEL_JAVAAGENT_EXTENSIONS}.
+ *
{@code
+ * DurableConfig.builder().withPlugins(ExecutionOtelPlugin.factory()).build();
+ * }
+ *
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
+ */
+ public static DurableExecutionPluginFactory factory() {
+ return factory(OtelPluginConfig.defaults());
+ }
+
+ /**
+ * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider.
+ *
+ * The global provider is resolved when the first invocation's instance needs it. If the agent has not
+ * initialized it yet, that invocation emits no telemetry and the next invocation's instance resolves it again.
*
- * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
+ * @param config the plugin configuration
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
*/
- public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
- this(tracerProviderBuilder, OtelPluginConfig.defaults());
+ public static DurableExecutionPluginFactory factory(OtelPluginConfig config) {
+ var environment = OtelPluginEnvironment.forGlobalProvider(config);
+ return invocationFactory(environment);
}
/**
- * Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction and MDC enabled.
+ * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider,
+ * with default settings: X-Ray context extraction, MDC enabled, root span named {@code "Workflow"}.
+ *
+ *
Customers configure exporters and span processors on the builder — the plugin handles ID generation. The
+ * provider is built once, here, and shared by every invocation's instance.
*
- *
Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet,
- * telemetry is disabled for that invocation and resolution is retried on the next invocation.
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped)
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
*/
- public ExecutionOtelPlugin() {
- this(OtelPluginConfig.defaults());
+ public static DurableExecutionPluginFactory factory(SdkTracerProviderBuilder tracerProviderBuilder) {
+ return factory(tracerProviderBuilder, OtelPluginConfig.defaults());
}
/**
- * Creates a Workflow-rooted OTel plugin from the given tracer provider builder and configuration.
+ * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider.
*
*
Customers configure exporters and span processors on the builder; all other tunables (context extractor, MDC
- * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}. Use
- * {@link OtelPluginConfig#builder()} for readable, named configuration:
+ * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}:
*
*
{@code
- * var plugin = new ExecutionOtelPlugin(
+ * var factory = ExecutionOtelPlugin.factory(
* SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
* OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
* }
*
- * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped)
* @param config the plugin configuration
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
*/
- public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
- OtelPluginSupport.requireSameThreadInvocationHooks();
- this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
- // Wrap the configured sampler so durable spans use the execution's single precomputed decision.
- DurableSampler.installOn(tracerProviderBuilder);
-
- this.sdkTracerProvider = tracerProviderBuilder.build();
- this.tracer = sdkTracerProvider.get(config.instrumentationName());
- this.contextExtractor = config.contextExtractor();
- this.enableMdc = config.enableMdc();
- this.workflowSpanName = config.workflowSpanName();
- this.instrumentationName = config.instrumentationName();
+ public static DurableExecutionPluginFactory factory(
+ SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
+ var environment = OtelPluginEnvironment.forProviderBuilder(tracerProviderBuilder, config);
+ return invocationFactory(environment);
+ }
+
+ private static DurableExecutionPluginFactory invocationFactory(OtelPluginEnvironment environment) {
+ return new DurableExecutionPluginFactory() {
+ @Override
+ public String getExclusiveGroup() {
+ return "durable-otel-view";
+ }
+
+ @Override
+ public String toString() {
+ return "ExecutionOtelPlugin";
+ }
+
+ @Override
+ public DurableExecutionPlugin createPlugin(InvocationInfo info) {
+ return environment.createPlugin(() -> new ExecutionOtelPlugin(environment, info));
+ }
+ };
}
/**
- * Creates a Workflow-rooted OTel plugin from configuration alone (no caller-supplied tracer provider builder).
+ * Creates the instance that serves one invocation.
*
- * The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the
- * two-arg constructor for an application-owned provider.
+ *
Everything this invocation's spans are keyed by is resolved here, from the {@code info} the factory received:
+ * the tracer binding, the extracted context, the canonical execution trace and its ancestor, the single sampling
+ * intent, the Invocation span, and the deferred Workflow span context. Resolving them in the constructor — before
+ * the SDK publishes this instance to the operation and user function threads — is what lets them be {@code final}
+ * rather than volatile per-invocation state.
*
- * @param config the plugin configuration
+ *
When the tracer cannot be bound, telemetry is disabled for this invocation: the span fields stay null and
+ * every hook returns immediately. The next invocation gets a new instance, which binds again.
*/
- public ExecutionOtelPlugin(OtelPluginConfig config) {
- OtelPluginSupport.requireSameThreadInvocationHooks();
- this.contextExtractor = config.contextExtractor();
+ private ExecutionOtelPlugin(OtelPluginEnvironment environment, InvocationInfo info) {
+ var config = environment.config();
+ this.idGenerator = environment.idGenerator();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
- this.instrumentationName = config.instrumentationName();
- this.idGenerator = OtelPluginSupport.createDefaultIdGenerator();
- }
-
- // ─── Invocation hooks ────────────────────────────────────────────────
-
- @Override
- public void onInvocationStart(InvocationInfo info) {
- tracingEnabled = false;
- if (!bindTracer()) {
+ this.durableExecutionArn = info.durableExecutionArn();
+ this.executionStartTime = info.executionStartTime();
+
+ var setup = environment.bind("ExecutionOtelPlugin");
+ if (setup == null) {
+ this.sdkTracerProvider = null;
+ this.tracer = null;
+ this.samplingIntent = null;
+ this.executionTrace = null;
+ this.executionAncestor = null;
+ this.invocationSpan = null;
+ this.executionRootSpan = null;
+ this.workflowSpanContext = null;
return;
}
-
- this.durableExecutionArn = info.durableExecutionArn();
+ this.sdkTracerProvider = setup.sdkTracerProvider();
+ this.tracer = setup.tracer();
// Resolve the one execution ancestor both spans parent onto, so they share a stable-per-execution trace and a
// sampling decision.
- var extracted = contextExtractor.extract();
+ var extracted = config.contextExtractor().extract(info);
var canonicalTraceId =
- ExecutionTraceContext.canonicalTraceId(extracted, arn(), info.executionStartTime(), idGenerator);
+ ExecutionTraceContext.canonicalTraceId(extracted, arn(), executionStartTime, idGenerator);
// Resolve the execution's sampling decision once for this invocation as a full SamplingResult, then apply it to
// every durable span via DurableSampler. The execution ancestor's trace flags are derived from the same
// decision so a parent-based sampler stays consistent with it.
@@ -238,36 +304,31 @@ public void onInvocationStart(InvocationInfo info) {
Attributes.of(DURABLE_EXECUTION_ARN, arn()));
// An unresolved decision is supplied by the actual fallback root or the agent-side sampler. Do not invent
// a decision that bypasses an installed drop/rate-limit policy.
- samplingIntent = decision != null
+ this.samplingIntent = decision != null
? DurableSamplingDecision.Intent.resolved(decision)
: DurableSamplingDecision.Intent.deferred(canonicalTraceId);
var sampled = OtelPluginSupport.isSampled(decision);
var execCtx = ExecutionTraceContext.resolve(extracted, canonicalTraceId, arn(), idGenerator, () -> sampled);
- executionStartTime = info.executionStartTime();
+ // Materialize and immediately end the SDK-owned anchor before creating descendants or user code.
var root = DurableExecutionRoot.start(
tracer,
idGenerator,
execCtx.executionAncestor(),
- info.durableExecutionArn(),
+ durableExecutionArn,
executionStartTime,
samplingIntent,
OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider),
OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider));
- try {
- DurableExecutionRoot.end(root, executionStartTime);
- } finally {
- // Publish only after end callbacks, so a reentrant End cannot flush from inside root.end().
- // Retain the ended root as a flush marker even if an onEnd processor throws during Start.
- executionRootSpan = root;
- }
+ DurableExecutionRoot.end(root, executionStartTime);
+ this.executionRootSpan = root;
// An unresolved plain fallback uses the real root's decision; explicit/ambient decisions and the
// legacy remote-parent path retain their existing behavior. Plain samplers still own each child decision.
if (executionRootSpan != null
&& (OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider) || decision == null)) {
execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext());
}
- executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags());
- executionAncestor = execCtx.executionAncestor();
+ this.executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags());
+ this.executionAncestor = execCtx.executionAncestor();
// Invocation span — child of the ambient Lambda span when it is on the execution trace, otherwise a child of
// the execution ancestor so it stays within the same trace.
@@ -275,55 +336,65 @@ public void onInvocationStart(InvocationInfo info) {
var spanBuilder = tracer.spanBuilder("Invocation")
.setSpanKind(SpanKind.INTERNAL)
.setParent(invocationParent)
- .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn())
+ .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn)
.setAttribute(DURABLE_FIRST_INVOCATION, info.isFirstInvocation());
if (info.requestId() != null) {
spanBuilder.setAttribute(AttributeKey.stringKey("faas.invocation_id"), info.requestId());
}
- invocationSpan = startDurableSpan(spanBuilder);
+ this.invocationSpan = startDurableSpan(spanBuilder);
// Defer the recording Workflow span until terminal completion. The placeholder uses the Invocation span's
// resolved sampling metadata so operation parents/links match the span that is eventually exported.
- var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn());
var invocationContext = invocationSpan.getSpanContext();
- workflowSpanContext = SpanContext.create(
- canonicalTraceId, workflowSpanId, invocationContext.getTraceFlags(), invocationContext.getTraceState());
+ this.workflowSpanContext = SpanContext.create(
+ canonicalTraceId,
+ idGenerator.generateWorkflowSpanId(durableExecutionArn),
+ invocationContext.getTraceFlags(),
+ invocationContext.getTraceState());
+ }
- // Inject MDC on the handler thread so handler-level logs (between steps) have trace context.
- if (enableMdc) {
- MDC.put(
- MdcSpanEnricher.MDC_TRACE_ID,
- invocationSpan.getSpanContext().getTraceId());
+ // ─── Invocation hooks ────────────────────────────────────────────────
+
+ @Override
+ public void onInvocationStart(InvocationInfo info) {
+ // This invocation's identity, its Invocation span and its Workflow span context were resolved in the
+ // constructor, from the very InvocationInfo this hook receives. What is left is the MDC injection, which
+ // belongs
+ // here because it must run on the handler thread so handler-level logs between steps carry trace context.
+ if (invocationSpan == null || !enableMdc) {
+ return;
}
- tracingEnabled = true;
- handlerScope = activateHandlerContext();
+ MDC.put(MdcSpanEnricher.MDC_TRACE_ID, invocationSpan.getSpanContext().getTraceId());
}
- private Scope activateHandlerContext() {
+ /** JDK-only scope bridge; does not dispatch to coincidental subclass methods. */
+ public static final class HandlerScopeOpener implements Function {
+ @Override
+ public AutoCloseable apply(ExecutionOtelPlugin plugin) {
+ return plugin.activateHandlerContext();
+ }
+ }
+
+ private AutoCloseable activateHandlerContext() {
var trace = executionTrace;
- if (!tracingEnabled || trace == null) return null;
+ if (ended || tracer == null || trace == null) return null;
var ambient = Span.current().getSpanContext();
+ // Preserve a compatible ambient Lambda span. An absent or unrelated ambient span must not leave
+ // handler instrumentation outside the durable execution's canonical trace.
if (ambient.isValid() && trace.traceId().equals(ambient.getTraceId())) return Scope.noop();
return Span.wrap(workflowSpanContext).makeCurrent();
}
@Override
public void onInvocationEnd(InvocationEndInfo info) {
- var scope = handlerScope;
- handlerScope = null;
- OtelPluginSupport.runInvocationEnd(scope, () -> endInvocation(info));
- }
-
- private void endInvocation(InvocationEndInfo info) {
- var root = executionRootSpan;
- var provider = sdkTracerProvider;
- var shouldFlush = tracingEnabled || root != null;
- executionRootSpan = null; // The anchor already ended at Start; flush this invocation only once.
+ if (ended) return;
+ ended = true;
+ var shouldFlush = invocationSpan != null || executionRootSpan != null;
OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> {
- if (shouldFlush && provider != null) {
- var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS);
+ if (shouldFlush && sdkTracerProvider != null) {
+ var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS);
if (!flushResult.isSuccess())
logger.warn("OTel span flush failed or timed out — some spans may be lost");
}
@@ -331,39 +402,28 @@ private void endInvocation(InvocationEndInfo info) {
}
private void finishInvocationEnd(InvocationEndInfo info) {
- if (!tracingEnabled) {
- return;
- }
- tracingEnabled = false;
-
+ if (invocationSpan == null) return;
// Clear invocation-level MDC
if (enableMdc) {
MdcSpanEnricher.clear();
}
- // Drop placeholder state. Open operations have no recording span to abandon.
- operationContexts.clear();
- operationStartTimes.clear();
-
- // End open recording spans now, but leave a running worker's thread-bound scope for its own user-function End.
- // Only scopes owned by this finalization thread can be closed here.
+ // Close scopes owned by this thread, retaining other owners' scopes until their matching end hooks run.
attemptScopes.closeCurrentThread();
for (var span : attemptSpans.values()) {
span.end();
}
- attemptSpans.clear();
+ // Every attempt span has ended. Only unfinished owners may retain this invocation's instance for scope
+ // cleanup; these registries are never recycled for another invocation.
// End the invocation span every invocation.
- if (invocationSpan != null) {
- invocationSpan.setAttribute(
- DURABLE_INVOCATION_STATUS, info.invocationStatus().name());
- applyInvocationStatus(invocationSpan, info);
- invocationSpan.end();
- invocationSpan = null;
- }
+ invocationSpan.setAttribute(
+ DURABLE_INVOCATION_STATUS, info.invocationStatus().name());
+ applyInvocationStatus(invocationSpan, info);
+ invocationSpan.end();
// Materialize the Workflow span only on terminal status.
- if (isTerminal(info) && workflowSpanContext != null && executionAncestor != null) {
+ if (isTerminal(info)) {
var workflowSpanBuilder = tracer.spanBuilder(workflowSpanName)
.setSpanKind(SpanKind.INTERNAL)
.setParent(withDurableDecision(Context.root().with(Span.wrap(executionAncestor))))
@@ -387,17 +447,13 @@ private void finishInvocationEnd(InvocationEndInfo info) {
}
workflowSpan.end();
}
- workflowSpanContext = null;
- executionAncestor = null;
- executionStartTime = null;
- samplingIntent = null;
}
/** Produces metadata without creating spans or changing the configured provider/resource. */
@Override
public String providePropagationMetadata(
String executionArn, String operationId, String parentOperationId, String targetFunctionName) {
- if (!tracingEnabled || !executionArn.equals(durableExecutionArn)) return null;
+ if (disabled() || !executionArn.equals(durableExecutionArn)) return null;
var trace = executionTrace;
if (trace == null) return null;
// These contexts are deferred placeholders, not sampled operation spans. A replacement sampler may decide
@@ -419,7 +475,7 @@ public String providePropagationMetadata(
@Override
public void onOperationStart(OperationInfo info) {
- if (!tracingEnabled) return;
+ if (disabled()) return;
if (info.id() == null) return;
// Retain only a deterministic placeholder. Its flags/state come from the Invocation span's resolved sampling
@@ -437,7 +493,7 @@ public void onOperationStart(OperationInfo info) {
@Override
public void onOperationEnd(OperationEndInfo info) {
- if (!tracingEnabled) return;
+ if (disabled()) return;
if (info.id() == null) return;
// Start and end the operation's single span here, using its deterministic span ID and linking to the
@@ -490,7 +546,7 @@ public void onOperationEnd(OperationEndInfo info) {
@Override
public void onUserFunctionStart(UserFunctionStartInfo info) {
- if (!tracingEnabled) return;
+ if (disabled()) return;
// Skip attempt spans for CONTEXT operations — they are a scoping construct, not a retriable unit of work. Still
// make the operation's context current so auto-instrumented calls become children of the (deferred) operation
@@ -498,7 +554,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
if ("CONTEXT".equals(info.type())) {
var operationContext = operationContexts.get(info.id());
if (operationContext != null) {
- var scope = Span.wrap(operationContext).makeCurrent();
+ var scope = OtelPluginSupport.onOwnerThread(
+ Span.wrap(operationContext).makeCurrent());
var key = attemptKey(info.id(), info.attempt());
attemptScopes.put(key, scope);
}
@@ -538,7 +595,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
attemptSpans.put(key, span);
// Make span current on this thread so auto-instrumented calls become children
- var scope = span.makeCurrent();
+ var scope = OtelPluginSupport.onOwnerThread(span.makeCurrent());
attemptScopes.put(key, scope);
if (enableMdc) {
@@ -548,10 +605,14 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (invocationSpan == null) return;
+
var key = attemptKey(info.id(), info.attempt());
- // A running user function can finish after invocation End; its scope still belongs to this thread.
+
+ // A late user-function End still closes the scope on the owning thread.
attemptScopes.close(key);
- if (!tracingEnabled) return;
+
+ if (ended) return;
// CONTEXT operations don't have attempt spans — scope cleanup is all we need
if ("CONTEXT".equals(info.type())) {
@@ -581,22 +642,12 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
// ─── Helpers ─────────────────────────────────────────────────────────
- private boolean bindTracer() {
- if (tracer != null) {
- return true;
- }
- synchronized (this) {
- if (tracer != null) {
- return true;
- }
- var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "ExecutionOtelPlugin");
- if (setup == null) {
- return false;
- }
- sdkTracerProvider = setup.sdkTracerProvider();
- tracer = setup.tracer();
- return true;
- }
+ /**
+ * True when this instance emits no telemetry: either the tracer could not be bound for this invocation, or the
+ * invocation has already ended and its spans are closed.
+ */
+ private boolean disabled() {
+ return invocationSpan == null || ended;
}
private void applyInvocationStatus(Span span, InvocationEndInfo info) {
@@ -695,17 +746,11 @@ private Context withDurableDecision(Context context) {
}
private TraceFlags effectiveTraceFlags() {
- var invocation = invocationSpan;
- if (invocation != null) {
- return invocation.getSpanContext().getTraceFlags();
- }
- var trace = executionTrace;
- return trace != null ? trace.flags() : TraceFlags.getDefault();
+ return invocationSpan.getSpanContext().getTraceFlags();
}
private TraceState effectiveTraceState() {
- var invocation = invocationSpan;
- return invocation != null ? invocation.getSpanContext().getTraceState() : TraceState.getDefault();
+ return invocationSpan.getSpanContext().getTraceState();
}
/**
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java
index 011e1dc028..01aee63c3c 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java
@@ -3,30 +3,32 @@
package software.amazon.lambda.durable.otel;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
/**
* Dynamically loads {@link ExecutionOtelPlugin} when {@code DURABLE_EXECUTION_PLUGINS} contains {@code otel-execution}.
+ *
+ * The provider is itself the per-invocation factory: it holds the environment-lifetime state (the ADOT global
+ * provider binding, the ID generator) once and creates one plugin instance per invocation from it.
*/
public final class ExecutionOtelPluginProvider implements DurableExecutionPluginProvider {
+ private final DurableExecutionPluginFactory factory = ExecutionOtelPlugin.factory();
+
@Override
public String getName() {
return "otel-execution";
}
@Override
- public int getApiVersion() {
- return API_VERSION;
- }
-
- @Override
- public Class getPluginType() {
- return ExecutionOtelPlugin.class;
+ public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) {
+ return factory.createPlugin(invocationInfo);
}
@Override
- public DurableExecutionPlugin createPlugin() {
- return new ExecutionOtelPlugin();
+ public String getExclusiveGroup() {
+ return factory.getExclusiveGroup();
}
}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
index a821787978..c1ecbc4e6d 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
@@ -22,11 +22,13 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.ExclusivePluginGroup;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.HandlerScoped;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.OperationEndInfo;
@@ -62,9 +64,18 @@
*
Tracing: Active (to populate {@code _X_AMZN_TRACE_ID})
*
*
- * When using {@link #InvocationOtelPlugin()}, the plugin resolves the global provider at invocation start. If the
- * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider
- * resolution is retried on the next invocation.
+ *
Lifetime. One instance serves exactly one Lambda invocation: {@link #factory()} and its overloads return a
+ * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation. Invocation-level references are
+ * released at return; an unfinished operation retains its matching cleanup until its owner unwinds. Everything about
+ * the invocation — the execution ARN, the resolved execution trace and ancestor, the sampling intent, the Invocation
+ * span, the deferred Workflow span context — is therefore a {@code final} field, resolved in the constructor from the
+ * {@link InvocationInfo} the factory receives (the same instance {@link #onInvocationStart(InvocationInfo)} then
+ * receives). Nothing is reset between invocations because nothing is carried between them.
+ *
+ *
What belongs to the execution environment stays in the factory's {@link OtelPluginEnvironment}: the configuration,
+ * the ID generator, and either the application-owned tracer provider (built once) or the lazily resolved ADOT global
+ * provider. On the agent path, an invocation whose instance cannot resolve the global provider emits no telemetry at
+ * all, and the next invocation's instance resolves it again.
*
*
X-Ray console limitation: In the X-Ray "Segments Timeline" ungrouped view, the plugin's spans (Invocation,
* operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of
@@ -73,43 +84,58 @@
* view to inspect parent-child relationships within the shared execution trace and the links between operation spans
* and the Workflow span.
*
- *
Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple
- * threads.
+ *
Thread-safe within its invocation: the SDK runs user code on multiple threads, so the open-span registries are
+ * {@link ConcurrentHashMap}s. The invocation's identity needs no such protection — it is final state written before the
+ * SDK publishes the instance to those threads.
*/
-@ExclusivePluginGroup("durable-otel-view")
-public class InvocationOtelPlugin implements DurableExecutionPlugin {
+@HandlerScoped(InvocationOtelPlugin.HandlerScopeOpener.class)
+public final class InvocationOtelPlugin implements DurableExecutionPlugin {
private static final Logger logger = LoggerFactory.getLogger(InvocationOtelPlugin.class);
- private volatile SdkTracerProvider sdkTracerProvider;
- private volatile Tracer tracer;
+ // ─── Environment lifetime (shared with every other invocation's instance) ─────────────
+
private final DeterministicIdGenerator idGenerator;
- private final ContextExtractor contextExtractor;
private final boolean enableMdc;
private final String workflowSpanName;
- private final String instrumentationName;
-
- // Per-invocation state
- private volatile boolean tracingEnabled;
- private volatile Span invocationSpan;
- private volatile Span executionRootSpan;
- // The root handler opens and closes this context through the base invocation lifecycle.
- private Scope handlerScope;
- private volatile String durableExecutionArn;
- // Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID
- // with mismatched flags.
- private volatile ExecutionTrace executionTrace;
- // The execution's single sampling intent for this invocation, computed once at onInvocationStart and attached to
- // every durable span's parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to
- // its own delegate) without re-invoking the configured sampler per span.
- private volatile DurableSamplingDecision.Intent samplingIntent;
-
- // Deferred Workflow placeholder; the recording span is emitted only on terminal invocation.
- private volatile SpanContext workflowSpanContext;
- private volatile SpanContext executionAncestor;
- private volatile Instant executionStartTime;
-
- /** Immutable snapshot of the resolved execution trace, read atomically through a single volatile reference. */
+
+ // ─── This invocation, all resolved in the constructor from its InvocationInfo ─────────
+
+ /** The provider used to flush before Lambda freezes; null when it is not visible to the application. */
+ private final SdkTracerProvider sdkTracerProvider;
+
+ /** Null when telemetry is disabled for this invocation, which makes every hook on this instance a no-op. */
+ private final Tracer tracer;
+
+ private final String durableExecutionArn;
+ private final Instant executionStartTime;
+
+ /** Trace ID and flags of the execution trace, resolved together so they can never be paired mismatched. */
+ private final ExecutionTrace executionTrace;
+
+ /**
+ * The execution's single sampling intent for this invocation, resolved once and attached to every durable span's
+ * parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to its own delegate)
+ * without re-invoking the configured sampler per span.
+ */
+ private final DurableSamplingDecision.Intent samplingIntent;
+
+ private final SpanContext executionAncestor;
+
+ private final Span invocationSpan;
+ private final Span executionRootSpan;
+
+ /** Deferred Workflow placeholder; the recording span is emitted only on terminal invocation. */
+ private final SpanContext workflowSpanContext;
+
+ /**
+ * Set when this invocation ends; never cleared, because an instance is never reused. Read by the operation and user
+ * function hooks, which may run on other threads of this invocation, so that a straggler hook arriving after the
+ * spans have been ended does not open a new one — volatile for that publication.
+ */
+ private volatile boolean ended;
+
+ /** Immutable snapshot of the resolved execution trace. */
private record ExecutionTrace(String traceId, TraceFlags flags) {}
// Thread-safe storage for operation spans (keyed by operationId) — open spans that need ending
@@ -126,99 +152,134 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {}
private final ConcurrentLinkedDeque operationStartOrder = new ConcurrentLinkedDeque<>();
/**
- * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled.
- *
- * Uses the provided tracer provider builder. Customers configure exporters and span processors on the builder —
- * the plugin handles ID generation.
- *
- *
For ADOT Java agent usage, prefer {@link #InvocationOtelPlugin()} with the plugin jar configured through
- * {@code OTEL_JAVAAGENT_EXTENSIONS}. Use this builder constructor when you want to own the exporter pipeline:
+ * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider,
+ * with default settings: X-Ray context extraction and MDC enabled.
*
*
{@code
- * var exporter = LoggingSpanExporter.create();
- * var plugin = new InvocationOtelPlugin(
- * SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
+ * DurableConfig.builder().withPlugins(InvocationOtelPlugin.factory()).build();
* }
*
- * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
+ */
+ public static DurableExecutionPluginFactory factory() {
+ return factory(OtelPluginConfig.defaults());
+ }
+
+ /**
+ * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider.
+ *
+ * The global provider is resolved when the first invocation's instance needs it. If the agent has not
+ * initialized it yet, that invocation emits no telemetry and the next invocation's instance resolves it again.
+ *
+ * @param config the plugin configuration
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
*/
- public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
- this(tracerProviderBuilder, OtelPluginConfig.defaults());
+ public static DurableExecutionPluginFactory factory(OtelPluginConfig config) {
+ var environment = OtelPluginEnvironment.forGlobalProvider(config);
+ return invocationFactory(environment);
}
/**
- * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled.
+ * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider,
+ * with default settings: X-Ray context extraction and MDC enabled.
+ *
+ *
Customers configure exporters and span processors on the builder — the plugin handles ID generation. The
+ * provider is built once, here, and shared by every invocation's instance:
+ *
+ *
{@code
+ * var exporter = LoggingSpanExporter.create();
+ * var factory = InvocationOtelPlugin.factory(
+ * SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
+ * }
*
- * Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet,
- * telemetry is disabled for that invocation and resolution is retried on the next invocation.
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped)
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
*/
- public InvocationOtelPlugin() {
- this(OtelPluginConfig.defaults());
+ public static DurableExecutionPluginFactory factory(SdkTracerProviderBuilder tracerProviderBuilder) {
+ return factory(tracerProviderBuilder, OtelPluginConfig.defaults());
}
/**
- * Creates an OTel plugin from the given tracer provider builder and configuration.
+ * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider.
*
*
Customers configure exporters and span processors on the builder; all other tunables (context extractor, MDC
- * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}. Use
- * {@link OtelPluginConfig#builder()} for readable, named configuration:
+ * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}:
*
*
{@code
- * var plugin = new InvocationOtelPlugin(
+ * var factory = InvocationOtelPlugin.factory(
* SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
* OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
* }
*
- * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped)
* @param config the plugin configuration
+ * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins}
*/
- public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
- OtelPluginSupport.requireSameThreadInvocationHooks();
- this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
- // Wrap the configured sampler so durable spans use the execution's single precomputed decision.
- DurableSampler.installOn(tracerProviderBuilder);
-
- this.sdkTracerProvider = tracerProviderBuilder.build();
- this.tracer = sdkTracerProvider.get(config.instrumentationName());
- this.contextExtractor = config.contextExtractor();
- this.enableMdc = config.enableMdc();
- this.workflowSpanName = config.workflowSpanName();
- this.instrumentationName = config.instrumentationName();
+ public static DurableExecutionPluginFactory factory(
+ SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
+ var environment = OtelPluginEnvironment.forProviderBuilder(tracerProviderBuilder, config);
+ return invocationFactory(environment);
+ }
+
+ private static DurableExecutionPluginFactory invocationFactory(OtelPluginEnvironment environment) {
+ return new DurableExecutionPluginFactory() {
+ @Override
+ public String getExclusiveGroup() {
+ return "durable-otel-view";
+ }
+
+ @Override
+ public String toString() {
+ return "InvocationOtelPlugin";
+ }
+
+ @Override
+ public DurableExecutionPlugin createPlugin(InvocationInfo info) {
+ return environment.createPlugin(() -> new InvocationOtelPlugin(environment, info));
+ }
+ };
}
/**
- * Creates an OTel plugin from configuration alone (no caller-supplied tracer provider builder).
+ * Creates the instance that serves one invocation.
*
- * The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the
- * two-arg constructor for an application-owned provider.
+ *
Everything this invocation's spans are keyed by is resolved here, from the {@code info} the factory received:
+ * the tracer binding, the extracted context, the canonical execution trace and its ancestor, the single sampling
+ * intent, the Invocation span, and the deferred Workflow span context. Resolving them in the constructor — before
+ * the SDK publishes this instance to the operation and user function threads — is what lets them be {@code final}
+ * rather than volatile per-invocation state.
*
- * @param config the plugin configuration
+ *
When the tracer cannot be bound, telemetry is disabled for this invocation: the span fields stay null and
+ * every hook returns immediately. The next invocation gets a new instance, which binds again.
*/
- public InvocationOtelPlugin(OtelPluginConfig config) {
- OtelPluginSupport.requireSameThreadInvocationHooks();
- this.contextExtractor = config.contextExtractor();
+ private InvocationOtelPlugin(OtelPluginEnvironment environment, InvocationInfo info) {
+ var config = environment.config();
+ this.idGenerator = environment.idGenerator();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
- this.instrumentationName = config.instrumentationName();
- this.idGenerator = OtelPluginSupport.createDefaultIdGenerator();
- }
-
- // ─── Invocation hooks ────────────────────────────────────────────────
-
- @Override
- public void onInvocationStart(InvocationInfo info) {
- tracingEnabled = false;
- if (!bindTracer()) {
+ this.durableExecutionArn = info.durableExecutionArn();
+ this.executionStartTime = info.executionStartTime();
+
+ var setup = environment.bind("InvocationOtelPlugin");
+ if (setup == null) {
+ this.sdkTracerProvider = null;
+ this.tracer = null;
+ this.samplingIntent = null;
+ this.executionTrace = null;
+ this.executionAncestor = null;
+ this.invocationSpan = null;
+ this.executionRootSpan = null;
+ this.workflowSpanContext = null;
return;
}
+ this.sdkTracerProvider = setup.sdkTracerProvider();
+ this.tracer = setup.tracer();
- this.durableExecutionArn = info.durableExecutionArn();
-
- var extracted = contextExtractor.extract();
+ var extracted = config.contextExtractor().extract(info);
// Resolve the execution ancestor the Workflow span parents onto so it joins the stable-per-execution trace.
- var canonicalTraceId = ExecutionTraceContext.canonicalTraceId(
- extracted, info.durableExecutionArn(), info.executionStartTime(), idGenerator);
+ var canonicalTraceId =
+ ExecutionTraceContext.canonicalTraceId(extracted, durableExecutionArn, executionStartTime, idGenerator);
// Resolve the execution's sampling decision once for this invocation as a full SamplingResult, then apply it to
// every durable span via DurableSampler (see below). The execution ancestor's trace flags are derived from the
// same decision so a parent-based sampler stays consistent with it.
@@ -231,37 +292,32 @@ public void onInvocationStart(InvocationInfo info) {
Attributes.of(DURABLE_EXECUTION_ARN, info.durableExecutionArn()));
// An unresolved decision is supplied by the actual fallback root or the agent-side sampler. Do not invent
// a decision that bypasses an installed drop/rate-limit policy.
- samplingIntent = decision != null
+ this.samplingIntent = decision != null
? DurableSamplingDecision.Intent.resolved(decision)
: DurableSamplingDecision.Intent.deferred(canonicalTraceId);
var sampled = OtelPluginSupport.isSampled(decision);
var execCtx = ExecutionTraceContext.resolve(
- extracted, canonicalTraceId, info.durableExecutionArn(), idGenerator, () -> sampled);
- executionStartTime = info.executionStartTime();
+ extracted, canonicalTraceId, durableExecutionArn, idGenerator, () -> sampled);
+ // Materialize and immediately end the SDK-owned anchor before creating descendants or user code.
var root = DurableExecutionRoot.start(
tracer,
idGenerator,
execCtx.executionAncestor(),
- info.durableExecutionArn(),
+ durableExecutionArn,
executionStartTime,
samplingIntent,
OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider),
OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider));
- try {
- DurableExecutionRoot.end(root, executionStartTime);
- } finally {
- // Publish only after end callbacks, so a reentrant End cannot flush from inside root.end().
- // Retain the ended root as a flush marker even if an onEnd processor throws during Start.
- executionRootSpan = root;
- }
+ DurableExecutionRoot.end(root, executionStartTime);
+ this.executionRootSpan = root;
// An unresolved plain fallback uses the real root's decision; explicit/ambient decisions and the
// legacy remote-parent path retain their existing behavior. Plain samplers still own each child decision.
if (executionRootSpan != null
&& (OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider) || decision == null)) {
execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext());
}
- executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags());
- executionAncestor = execCtx.executionAncestor();
+ this.executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags());
+ this.executionAncestor = execCtx.executionAncestor();
// Invocation span parent — the same-trace ambient span when available, then the execution ancestor, so the
// Invocation span stays on the execution trace.
@@ -271,56 +327,65 @@ public void onInvocationStart(InvocationInfo info) {
var spanBuilder = tracer.spanBuilder("Invocation")
.setSpanKind(SpanKind.INTERNAL)
.setParent(parentContext)
- .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn())
+ .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn)
.setAttribute(DURABLE_FIRST_INVOCATION, info.isFirstInvocation());
if (info.requestId() != null) {
spanBuilder.setAttribute(AttributeKey.stringKey("faas.invocation_id"), info.requestId());
}
- invocationSpan = startDurableSpan(spanBuilder);
+ this.invocationSpan = startDurableSpan(spanBuilder);
// Defer the recording Workflow span until terminal completion. The placeholder uses the Invocation span's
// resolved sampling metadata so operation links match the span that is eventually exported.
- var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn());
var invocationContext = invocationSpan.getSpanContext();
- workflowSpanContext = SpanContext.create(
- canonicalTraceId, workflowSpanId, invocationContext.getTraceFlags(), invocationContext.getTraceState());
+ this.workflowSpanContext = SpanContext.create(
+ canonicalTraceId,
+ idGenerator.generateWorkflowSpanId(durableExecutionArn),
+ invocationContext.getTraceFlags(),
+ invocationContext.getTraceState());
+ }
- // Inject MDC on the handler thread so handler-level logs (between steps) have trace context.
- // This runs on the same thread as context.getLogger() calls in the handler.
- if (enableMdc) {
- MDC.put(
- MdcSpanEnricher.MDC_TRACE_ID,
- invocationSpan.getSpanContext().getTraceId());
+ // ─── Invocation hooks ────────────────────────────────────────────────
+
+ @Override
+ public void onInvocationStart(InvocationInfo info) {
+ // This invocation's identity and its Invocation span were resolved in the constructor, from the very
+ // InvocationInfo this hook receives. What is left is the MDC injection, which belongs here because it must run
+ // on the handler thread — the same thread as the context.getLogger() calls in the handler — so handler-level
+ // logs between steps carry trace context.
+ if (invocationSpan == null || !enableMdc) {
+ return;
}
- tracingEnabled = true;
- handlerScope = activateHandlerContext();
+ MDC.put(MdcSpanEnricher.MDC_TRACE_ID, invocationSpan.getSpanContext().getTraceId());
}
- private Scope activateHandlerContext() {
+ /** JDK-only scope bridge; does not dispatch to coincidental subclass methods. */
+ public static final class HandlerScopeOpener implements Function {
+ @Override
+ public AutoCloseable apply(InvocationOtelPlugin plugin) {
+ return plugin.activateHandlerContext();
+ }
+ }
+
+ private AutoCloseable activateHandlerContext() {
var trace = executionTrace;
- if (!tracingEnabled || trace == null) return null;
+ if (ended || tracer == null || trace == null) return null;
var ambient = Span.current().getSpanContext();
+ // Preserve a compatible ambient Lambda span. An absent or unrelated ambient span must not leave
+ // handler instrumentation outside the durable execution's canonical trace.
if (ambient.isValid() && trace.traceId().equals(ambient.getTraceId())) return Scope.noop();
return invocationSpan.makeCurrent();
}
@Override
public void onInvocationEnd(InvocationEndInfo info) {
- var scope = handlerScope;
- handlerScope = null;
- OtelPluginSupport.runInvocationEnd(scope, () -> endInvocation(info));
- }
-
- private void endInvocation(InvocationEndInfo info) {
- var root = executionRootSpan;
- var provider = sdkTracerProvider;
- var shouldFlush = tracingEnabled || root != null;
- executionRootSpan = null; // The anchor already ended at Start; flush this invocation only once.
+ if (ended) return;
+ ended = true;
+ var shouldFlush = invocationSpan != null || executionRootSpan != null;
OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> {
- if (shouldFlush && provider != null) {
- var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS);
+ if (shouldFlush && sdkTracerProvider != null) {
+ var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS);
if (!flushResult.isSuccess())
logger.warn("OTel span flush failed or timed out — some spans may be lost");
}
@@ -328,18 +393,12 @@ private void endInvocation(InvocationEndInfo info) {
}
private void finishInvocationEnd(InvocationEndInfo info) {
- if (!tracingEnabled) {
- return;
- }
- tracingEnabled = false;
-
+ if (invocationSpan == null) return;
// Clear invocation-level MDC (set in onInvocationStart on the handler thread)
if (enableMdc) {
MdcSpanEnricher.clear();
}
- if (invocationSpan == null) return;
-
endOpenSpansChildFirst();
// End invocation span
@@ -369,10 +428,9 @@ private void finishInvocationEnd(InvocationEndInfo info) {
}
invocationSpan.end();
- invocationSpan = null;
// Materialize the Workflow span only on terminal status.
- if (isTerminal(info) && workflowSpanContext != null && executionAncestor != null) {
+ if (isTerminal(info)) {
var workflowSpanBuilder = tracer.spanBuilder(workflowSpanName)
.setSpanKind(SpanKind.INTERNAL)
.setParent(withDurableDecision(Context.root().with(Span.wrap(executionAncestor))))
@@ -396,17 +454,13 @@ private void finishInvocationEnd(InvocationEndInfo info) {
}
workflowSpan.end();
}
- workflowSpanContext = null;
- executionAncestor = null;
- executionStartTime = null;
- samplingIntent = null;
}
/** Produces metadata without creating spans or changing the configured provider/resource. */
@Override
public String providePropagationMetadata(
String executionArn, String operationId, String parentOperationId, String targetFunctionName) {
- if (!tracingEnabled || !executionArn.equals(durableExecutionArn)) return null;
+ if (disabled() || !executionArn.equals(durableExecutionArn)) return null;
var trace = executionTrace;
if (trace == null) return null;
// An observed operation's actual context is authoritative, including invocation-view continuation segments.
@@ -430,7 +484,7 @@ public String providePropagationMetadata(
@Override
public void onOperationStart(OperationInfo info) {
- if (!tracingEnabled) return;
+ if (disabled()) return;
if (info.id() == null) return;
var parentContext = resolveParentContext(info.parentId());
@@ -471,7 +525,7 @@ public void onOperationStart(OperationInfo info) {
@Override
public void onOperationEnd(OperationEndInfo info) {
- if (!tracingEnabled) return;
+ if (disabled()) return;
if (info.id() == null) return;
var span = operationSpans.remove(info.id());
@@ -526,7 +580,7 @@ public void onOperationEnd(OperationEndInfo info) {
@Override
public void onUserFunctionStart(UserFunctionStartInfo info) {
- if (!tracingEnabled) return;
+ if (disabled()) return;
// Skip attempt spans for CONTEXT operations — they are a scoping construct, not a
// retriable unit of work, so attempt number/outcome attributes don't apply.
@@ -535,7 +589,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
// Still set the operation span as current so auto-instrumented calls become children
var operationSpan = operationSpans.get(info.id());
if (operationSpan != null) {
- var scope = operationSpan.makeCurrent();
+ var scope = OtelPluginSupport.onOwnerThread(operationSpan.makeCurrent());
var key = attemptKey(info.id(), info.attempt());
attemptScopes.put(key, scope);
}
@@ -575,7 +629,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
attemptSpans.put(key, span);
// Make span current on this thread so auto-instrumented calls become children
- var scope = span.makeCurrent();
+ var scope = OtelPluginSupport.onOwnerThread(span.makeCurrent());
attemptScopes.put(key, scope);
// Inject trace context into MDC for log-trace correlation
@@ -586,16 +640,21 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (invocationSpan == null) return;
+
var key = attemptKey(info.id(), info.attempt());
- // A running user function can finish after invocation End; its scope still belongs to this thread.
+
+ // A late user-function End still closes the scope on the owning thread.
attemptScopes.close(key);
- if (!tracingEnabled) return;
// Clear span-level MDC after user function completes (keep trace_id for handler-level logs between steps)
if (enableMdc) {
MDC.remove(MdcSpanEnricher.MDC_SPAN_ID);
}
+ // A bounded fatal return can finish spans before this operation owner finally unwinds.
+ if (ended) return;
+
// CONTEXT operations don't have attempt spans — scope cleanup is all we need
if ("CONTEXT".equals(info.type())) {
return;
@@ -628,31 +687,20 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
// ─── Helpers ─────────────────────────────────────────────────────────
- private boolean bindTracer() {
- if (tracer != null) {
- return true;
- }
- synchronized (this) {
- if (tracer != null) {
- return true;
- }
- var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "InvocationOtelPlugin");
- if (setup == null) {
- return false;
- }
- sdkTracerProvider = setup.sdkTracerProvider();
- tracer = setup.tracer();
- return true;
- }
+ /**
+ * True when this instance emits no telemetry: either the tracer could not be bound for this invocation, or the
+ * invocation has already ended and its spans are closed.
+ */
+ private boolean disabled() {
+ return invocationSpan == null || ended;
}
private void endOpenSpansChildFirst() {
- // Attempt spans are children of operation spans.
+ // Close scopes owned by this thread, retaining other owners' scopes for their matching end hooks.
attemptScopes.closeCurrentThread();
for (var span : attemptSpans.values()) {
span.end();
}
- attemptSpans.clear();
// End still-open operation spans with the STARTED status set in onOperationStart.
// A later invocation's onOperationEnd emits a continuation span with the real terminal status.
@@ -663,8 +711,8 @@ private void endOpenSpansChildFirst() {
span.end();
}
}
- operationSpans.clear();
- operationContexts.clear();
+ // Every span has ended. Only unfinished owners may retain this invocation's instance for scope cleanup;
+ // these registries are never recycled for another invocation.
}
/**
@@ -767,17 +815,11 @@ private void addInitialOperationLink(SpanBuilder spanBuilder, String operationId
}
private TraceFlags effectiveTraceFlags() {
- var invocation = invocationSpan;
- if (invocation != null) {
- return invocation.getSpanContext().getTraceFlags();
- }
- var trace = executionTrace;
- return trace != null ? trace.flags() : TraceFlags.getDefault();
+ return invocationSpan.getSpanContext().getTraceFlags();
}
private TraceState effectiveTraceState() {
- var invocation = invocationSpan;
- return invocation != null ? invocation.getSpanContext().getTraceState() : TraceState.getDefault();
+ return invocationSpan.getSpanContext().getTraceState();
}
private static boolean isTerminal(InvocationEndInfo info) {
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java
index 165ed88fcd..caa5a0c0a8 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java
@@ -3,31 +3,33 @@
package software.amazon.lambda.durable.otel;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
/**
* Dynamically loads {@link InvocationOtelPlugin} when {@code DURABLE_EXECUTION_PLUGINS} contains
* {@code otel-invocation}.
+ *
+ * The provider is itself the per-invocation factory: it holds the environment-lifetime state (the ADOT global
+ * provider binding, the ID generator) once and creates one plugin instance per invocation from it.
*/
public final class InvocationOtelPluginProvider implements DurableExecutionPluginProvider {
+ private final DurableExecutionPluginFactory factory = InvocationOtelPlugin.factory();
+
@Override
public String getName() {
return "otel-invocation";
}
@Override
- public int getApiVersion() {
- return API_VERSION;
- }
-
- @Override
- public Class getPluginType() {
- return InvocationOtelPlugin.class;
+ public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) {
+ return factory.createPlugin(invocationInfo);
}
@Override
- public DurableExecutionPlugin createPlugin() {
- return new InvocationOtelPlugin();
+ public String getExclusiveGroup() {
+ return factory.getExclusiveGroup();
}
}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
index ed4ac9be58..8fc5b136ac 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
@@ -10,8 +10,8 @@
* mirrors the {@code OtelPluginConfig} object in the JavaScript SDK and the {@code OtelPluginConfig} dataclass in the
* Python SDK for cross-SDK parity.
*
- *
Construct via {@link #builder()} and pass to a plugin's {@code (SdkTracerProviderBuilder, OtelPluginConfig)}
- * constructor:
+ *
Construct via {@link #builder()} and pass to a plugin's {@code factory(SdkTracerProviderBuilder,
+ * OtelPluginConfig)}:
*
*
{@code
* var config = OtelPluginConfig.builder()
@@ -20,7 +20,7 @@
* .workflowSpanName("Workflow")
* .instrumentationName("my-scope")
* .build();
- * var plugin = new InvocationOtelPlugin(tracerProviderBuilder, config);
+ * var factory = InvocationOtelPlugin.factory(tracerProviderBuilder, config);
* }
*
* Defaults: {@code contextExtractor = new XRayContextExtractor()}, {@code enableMdc = true}, {@code workflowSpanName
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java
new file mode 100644
index 0000000000..0593a3927a
--- /dev/null
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java
@@ -0,0 +1,146 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+/**
+ * Everything the OTel plugins need that belongs to the execution environment rather than to one invocation.
+ *
+ *
A plugin instance now serves exactly one Lambda invocation, so the objects that must exist once per environment
+ * live here: the resolved {@link OtelPluginConfig}, the {@link DeterministicIdGenerator}, and — for an
+ * application-owned tracer provider — the built provider and its tracer. {@code InvocationOtelPlugin.factory(...)} and
+ * {@code ExecutionOtelPlugin.factory(...)} create one of these and hand the same instance to every plugin instance they
+ * create, so the provider is built (and its ID generator and sampler installed) once per environment rather than once
+ * per invocation.
+ *
+ *
On the ADOT Java agent path there is no provider to build here: the global provider is resolved on first use and
+ * then cached. An invocation that runs before the agent has finished initializing therefore disables telemetry for
+ * itself only, and the next invocation's instance resolves the provider again.
+ */
+final class OtelPluginEnvironment {
+
+ private static final Logger logger = LoggerFactory.getLogger(OtelPluginEnvironment.class);
+
+ private final OtelPluginConfig config;
+ private final DeterministicIdGenerator idGenerator;
+
+ /** The application-owned provider and tracer, or null on the Java agent path. */
+ private final OtelPluginSupport.ProviderSetup ownedSetup;
+
+ /**
+ * The global provider and tracer, once resolved. Environment-lifetime state shared by every invocation's instance,
+ * hence volatile; a lost race only resolves the same global provider twice.
+ */
+ private volatile OtelPluginSupport.ProviderSetup resolvedGlobalSetup;
+
+ /**
+ * @throws NullPointerException if the config is null. The check belongs here because every factory overload on both
+ * plugins reaches this constructor, and because the alternative is silence: the global-provider path only
+ * stores the config, so a null one would first be dereferenced when an invocation's plugin instance is built,
+ * where {@code PluginRunner} contains the failure. The function would then run without the telemetry it asked
+ * for, reporting one warning per invocation. Registration is where a caller can still act on it.
+ */
+ private OtelPluginEnvironment(
+ OtelPluginConfig config, DeterministicIdGenerator idGenerator, OtelPluginSupport.ProviderSetup ownedSetup) {
+ this.config = Objects.requireNonNull(config, "config must not be null");
+ this.idGenerator = idGenerator;
+ this.ownedSetup = ownedSetup;
+ }
+
+ /**
+ * Builds the application-owned provider once: wraps the builder's ID generator and sampler, builds the provider and
+ * gets the tracer. Every invocation's plugin instance then shares them.
+ */
+ static OtelPluginEnvironment forProviderBuilder(
+ SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
+ // Checked before anything is consumed. The constructor below checks it too, but by then this method has
+ // installed the ID generator and the sampler on the caller's builder and built a provider -- and a provider
+ // that fails validation is unreachable, so its span processors and their worker threads are never shut down.
+ Objects.requireNonNull(config, "config must not be null");
+ var idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
+ // Wrap the configured sampler so durable spans use the execution's single precomputed decision.
+ DurableSampler.installOn(tracerProviderBuilder);
+ var sdkTracerProvider = tracerProviderBuilder.build();
+ var setup = new OtelPluginSupport.ProviderSetup(
+ sdkTracerProvider, sdkTracerProvider.get(config.instrumentationName()));
+ return new OtelPluginEnvironment(config, idGenerator, setup);
+ }
+
+ /** The Java agent path: the global provider is resolved lazily, when an invocation's instance first needs it. */
+ static OtelPluginEnvironment forGlobalProvider(OtelPluginConfig config) {
+ return new OtelPluginEnvironment(config, OtelPluginSupport.createDefaultIdGenerator(), null);
+ }
+
+ /** Nonfatal failed construction can flush completed spans; known JVM fatals propagate without added cleanup. */
+ @SuppressWarnings("removal")
+ T createPlugin(Supplier constructor) {
+ try {
+ return constructor.get();
+ } catch (Throwable failure) {
+ var originalCause = ExceptionHelper.unwrapAsyncFailure(failure);
+ if (originalCause instanceof VirtualMachineError fatal) throw fatal;
+ if (originalCause instanceof ThreadDeath fatal) throw fatal;
+ flushAfterNonfatalConstructionFailure(failure);
+ throw failure;
+ }
+ }
+
+ @SuppressWarnings("removal")
+ private void flushAfterNonfatalConstructionFailure(Throwable failure) {
+ try {
+ // Do not rebind or reach through an opaque provider. Use only a setup the constructor could already use.
+ var setup = ownedSetup != null ? ownedSetup : resolvedGlobalSetup;
+ if (setup == null || setup.sdkTracerProvider() == null) return;
+ var result = setup.sdkTracerProvider().forceFlush().join(5, TimeUnit.SECONDS);
+ if (!result.isSuccess())
+ logger.warn("OTel startup cleanup flush failed or timed out — some spans may be lost");
+ } catch (Throwable cleanupFailure) {
+ if (failure == cleanupFailure) return;
+ var cleanupCause = ExceptionHelper.unwrapAsyncFailure(cleanupFailure);
+ if (cleanupCause instanceof VirtualMachineError || cleanupCause instanceof ThreadDeath) {
+ cleanupCause.addSuppressed(failure);
+ throw (Error) cleanupCause;
+ }
+ // Nonfatal cleanup must not replace the original nonfatal construction failure.
+ failure.addSuppressed(cleanupFailure);
+ }
+ }
+
+ OtelPluginConfig config() {
+ return config;
+ }
+
+ DeterministicIdGenerator idGenerator() {
+ return idGenerator;
+ }
+
+ /**
+ * The provider and tracer one invocation's plugin instance should use, or {@code null} when telemetry must be
+ * disabled for that invocation because the agent's global provider is not available yet.
+ *
+ * @param pluginName the plugin name used in diagnostics
+ */
+ OtelPluginSupport.ProviderSetup bind(String pluginName) {
+ if (ownedSetup != null) {
+ return ownedSetup;
+ }
+ var alreadyResolved = resolvedGlobalSetup;
+ if (alreadyResolved != null) {
+ return alreadyResolved;
+ }
+ var setup = OtelPluginSupport.tryResolveGlobalProvider(config.instrumentationName(), pluginName);
+ if (setup != null) {
+ // Resolution succeeded, so it holds for the rest of this environment's life: cache it so later invocations
+ // neither re-resolve nor re-log it. A failure is not cached — that is what makes the retry per invocation.
+ resolvedGlobalSetup = setup;
+ }
+ return setup;
+ }
+}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java
index 6ce572c6d9..7687fc2715 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java
@@ -22,13 +22,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.lambda.durable.exception.DurableOperationException;
-import software.amazon.lambda.durable.execution.DurableExecutor;
import software.amazon.lambda.durable.plugin.OperationEndInfo;
-/** Shared provider setup, sampling resolution, and invocation cleanup support. */
+/** Shared utilities for the OTel plugins' ADOT Java agent SPI path. */
final class OtelPluginSupport {
- /** Always runs invocation cleanup without hiding an earlier failure or a later JVM fatal. */
+ /** Always attempt provider cleanup, retaining the approved plugin-local failure precedence. */
static void finishWithRootCleanup(Runnable finish, Runnable cleanup) {
Throwable primary = null;
try {
@@ -59,32 +58,24 @@ private static int endFailurePriority(Throwable failure) {
return 1;
}
- /** Applies the base lifecycle's scope restoration with the same cleanup failure precedence. */
- static void runInvocationEnd(Scope scope, Runnable end) {
- finishWithRootCleanup(end, () -> {
- if (scope != null) scope.close();
- });
- }
-
- /** Rejects an older core before a plugin builds a provider or activates thread-local context. */
- static void requireSameThreadInvocationHooks() {
- var message = "This OpenTelemetry plugin requires same-thread invocation hooks from the Durable Execution"
- + " core. Upgrade the core together with the plugin layer.";
- try {
- if (!DurableExecutor.supportsSameThreadInvocationHooks()) throw new IllegalStateException(message);
- } catch (NoSuchMethodError missingCapability) {
- throw new IllegalStateException(message, missingCapability);
- }
- }
-
private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class);
private OtelPluginSupport() {}
- /** A visible replacement sampler cannot consume our bridge; an opaque agent provider may still need it. */
- static boolean usesDurableSamplingBridge(SdkTracerProvider provider) {
- // This identifies only the cross-loader wire consumer; context-key ownership still requires instanceof.
- return provider == null || provider.getSampler().getClass().getName().equals(DurableSampler.class.getName());
+ /** Keeps finalization on another thread from consuming an operation owner's eventual scope cleanup. */
+ static Scope onOwnerThread(Scope scope) {
+ var owner = Thread.currentThread();
+ return new Scope() {
+ // Only the owner can read or mutate this flag; other threads return before accessing it.
+ private boolean closed;
+
+ @Override
+ public void close() {
+ if (Thread.currentThread() != owner || closed) return;
+ closed = true;
+ scope.close();
+ }
+ };
}
/** Only the same DurableSampler class copy can consume this loader's parent-context holder. */
@@ -92,6 +83,11 @@ static boolean usesLocalDurableSampler(SdkTracerProvider provider) {
return provider != null && provider.getSampler() instanceof DurableSampler;
}
+ /** A visible replacement sampler cannot consume our bridge; an opaque agent provider may still need it. */
+ static boolean usesDurableSamplingBridge(SdkTracerProvider provider) {
+ return provider == null || provider.getSampler().getClass().getName().equals(DurableSampler.class.getName());
+ }
+
/** Terminal backend status and exception details are independent telemetry signals. */
static void applyOperationStatus(Span span, OperationEndInfo info) {
applyOperationStatus(span, info, null);
@@ -176,7 +172,7 @@ static DeterministicIdGenerator createDefaultIdGenerator() {
* sampled span yields {@code RECORD_AND_SAMPLE}; an unsampled but recording span yields {@code RECORD_ONLY}
* (its spans still reach processors); only an unsampled, non-recording span yields {@code DROP};
* Application-owned provider: configured sampler, once. When the tracer provider is reachable (the
- * two-argument constructor path), its sampler is read directly and evaluated a single time with
+ * application-owned provider path), its sampler is read directly and evaluated a single time with
* {@code ROOT_CONTEXT} (so a parent-based sampler applies its root policy), the canonical trace ID, span
* name, and attributes, and its full result is returned;
* Java-agent path: defer to the installed sampler. When the provider is not visible
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
index 05a1110ab5..b15c978393 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
@@ -7,9 +7,11 @@
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
/**
- * Extracts OTel trace context from the AWS X-Ray {@code _X_AMZN_TRACE_ID} environment variable.
+ * Extracts OTel trace context from the invocation-local X-Ray header, using ordinary Lambda carriers only when no
+ * runtime snapshot is available.
*
* This extractor parses the Lambda/X-Ray header and returns the trace ID in OTel format (32 hex chars) along with
* the parent span ID (16 hex chars). Plugins use it as a fallback parent for Invocation spans; the deterministic
@@ -28,8 +30,40 @@ public class XRayContextExtractor implements ContextExtractor {
private static final Pattern HEX_32 = Pattern.compile("[0-9a-f]{32}");
private static final Pattern HEX_16 = Pattern.compile("[0-9a-f]{16}");
+ // Scoped only around extraction so legacy no-argument overrides retain virtual dispatch.
+ private final ThreadLocal invocationTraceHeader = new ThreadLocal<>();
+
+ @Override
+ public ExtractedContext extract(InvocationInfo info) {
+ var previous = invocationTraceHeader.get();
+ invocationTraceHeader.set(runtimeHeader(info));
+ try {
+ return extract();
+ } finally {
+ if (previous == null) {
+ invocationTraceHeader.remove();
+ } else {
+ invocationTraceHeader.set(previous);
+ }
+ }
+ }
+
+ private static String runtimeHeader(InvocationInfo info) {
+ try {
+ return info.xRayTraceId();
+ } catch (NoSuchMethodError olderCore) {
+ // Preserve extractor compatibility when InvocationInfo predates the optional accessor.
+ return null;
+ }
+ }
+
@Override
public ExtractedContext extract() {
+ var invocationHeader = invocationTraceHeader.get();
+ // A captured snapshot is authoritative even when empty or malformed; only null permits legacy fallback.
+ if (invocationHeader != null) {
+ return parseHeader(invocationHeader);
+ }
// Try system property first — the Lambda runtime interface client updates this per invocation, so it reflects
// the current invocation and avoids the JVM's process-lifetime environment-variable caching.
var traceHeader = System.getProperty(XRAY_SYSTEM_PROPERTY);
@@ -42,6 +76,10 @@ public ExtractedContext extract() {
return null;
}
+ return parseHeader(traceHeader);
+ }
+
+ private static ExtractedContext parseHeader(String traceHeader) {
String root = null;
String parent = null;
String sampled = null;
diff --git a/otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java b/otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java
new file mode 100644
index 0000000000..096c7e9d96
--- /dev/null
+++ b/otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java
@@ -0,0 +1,76 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+import java.util.ServiceLoader;
+import software.amazon.lambda.durable.plugin.*;
+
+public class PluginLayerCompatibilityProbe {
+ public static void main(String[] args) throws Exception {
+ var path = Path.of(args[0]);
+ var view = args[1];
+ var header = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1";
+ var old = System.getProperty("com.amazonaws.xray.traceHeader");
+ try (var layer = new URLClassLoader(new URL[] {path.toUri().toURL()},
+ PluginLayerCompatibilityProbe.class.getClassLoader())) {
+ System.setProperty("com.amazonaws.xray.traceHeader", header);
+ var provider = ServiceLoader.load(DurableExecutionPluginProvider.class, layer).stream()
+ .map(ServiceLoader.Provider::get).filter(p -> p.getName().equals(view)).findFirst().orElseThrow();
+ if (provider.getApiVersion() != DurableExecutionPluginProvider.API_VERSION) throw new AssertionError("provider API");
+ var type = provider.getPluginType();
+ if (type.getClassLoader() != layer) throw new AssertionError("plugin did not load from separate layer");
+ if (provider.createPlugin() == null) throw new AssertionError("dynamic provider constructor");
+ var configType = layer.loadClass("software.amazon.lambda.durable.otel.OtelPluginConfig");
+ var configBuilder = configType.getMethod("builder").invoke(null);
+ configBuilder.getClass().getMethod("enableMdc", boolean.class).invoke(configBuilder, false);
+ var config = configBuilder.getClass().getMethod("build").invoke(configBuilder);
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var plugin = (DurableExecutionPlugin) type.getConstructor(SdkTracerProviderBuilder.class, configType)
+ .newInstance(builder, config);
+ var runner = new PluginRunner(List.of(plugin));
+ runner.onInvocationStart(new InvocationInfo("request", "arn:exec", true, Instant.EPOCH));
+ runner.onInvocationEnd(new InvocationEndInfo("request", "arn:exec", true, InvocationStatus.SUCCEEDED, null));
+ var spans = exporter.getFinishedSpanItems();
+ if (spans.size() != 2 || spans.stream().anyMatch(s -> !s.getTraceId().equals("6955b900123456789012345678901234")))
+ throw new AssertionError("legacy tracing failed: " + spans);
+ type.getMethods(); // Public reflection must also work with an older core.
+ checkOptionalHeaderDispatch(runner, plugin, exporter, type);
+ System.out.println("PASS " + view + " core=" + DurableExecutionPlugin.class.getProtectionDomain()
+ .getCodeSource().getLocation() + " layer=" + path);
+ } finally {
+ if (old == null) System.clearProperty("com.amazonaws.xray.traceHeader");
+ else System.setProperty("com.amazonaws.xray.traceHeader", old);
+ }
+ }
+ private static void checkOptionalHeaderDispatch(PluginRunner runner, DurableExecutionPlugin plugin,
+ InMemorySpanExporter exporter, Class type) throws Exception {
+ Method hook;
+ try {
+ hook = PluginRunner.class.getMethod("onInvocationStart", InvocationInfo.class, String.class);
+ } catch (NoSuchMethodException olderCore) {
+ return; // The older core retains the original hook and ordinary carriers.
+ }
+ boolean supportsHeader;
+ try {
+ type.getDeclaredMethod("onInvocationStart", InvocationInfo.class, String.class);
+ supportsHeader = true;
+ } catch (NoSuchMethodException olderPlugin) {
+ supportsHeader = false;
+ }
+ exporter.reset();
+ hook.invoke(runner, new InvocationInfo("request-2", "arn:exec", false, Instant.EPOCH),
+ "Root=1-6955b900-aaaaaaaaaaaaaaaaaaaaaaaa;Parent=abcdefabcdefabcd;Sampled=0");
+ runner.onInvocationEnd(new InvocationEndInfo("request-2", "arn:exec", false, InvocationStatus.SUCCEEDED, null));
+ if (exporter.getFinishedSpanItems().size() != (supportsHeader ? 0 : 2))
+ throw new AssertionError("Optional dispatch did not preserve new/legacy behavior");
+ }
+}
diff --git a/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java b/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java
index 26721887b4..95008149f4 100644
--- a/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java
+++ b/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java
@@ -2,8 +2,8 @@
// SPDX-License-Identifier: Apache-2.0
package software.amazon.lambda.durable.otel;
-import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import io.opentelemetry.api.GlobalOpenTelemetry;
@@ -17,7 +17,6 @@
import java.time.Duration;
import java.util.List;
import java.util.ServiceLoader;
-import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.LoggerFactory;
import software.amazon.lambda.durable.model.ExecutionStatus;
@@ -26,9 +25,8 @@
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-import software.amazon.lambda.durable.testing.TestResult;
-/** Compiled against the actual released 2.2.1 SPI, then run unchanged in fresh matrix JVMs. */
+/** Compiled against the current 3.x factory API, then run in fresh JVMs with each visible OTel API. */
public final class InstalledApiProbe {
private static final AtomicInteger HEALTHY_CREATED = new AtomicInteger();
private static final AtomicInteger HEALTHY_STARTS = new AtomicInteger();
@@ -44,9 +42,8 @@ public static void main(String[] args) throws Exception {
logs.start();
root.addAppender(logs);
try {
- if (Boolean.parseBoolean(args[7])) assertOldCoreRejected(view);
- else exercise(view, Boolean.parseBoolean(args[5]), Boolean.parseBoolean(args[6]), logs);
- System.out.println("COMPAT_PASS " + view + " negative=" + args[5] + " api=" + args[2]
+ exercise(view, Boolean.parseBoolean(args[5]), logs);
+ System.out.println("COMPAT_PASS " + view + " api=" + args[2]
+ " core=" + args[0] + " plugin=" + args[1]);
} finally {
root.detachAppender(logs);
@@ -64,61 +61,20 @@ private static void verifyArtifacts(String[] args) throws Exception {
.map(ServiceLoader.Provider::get)
.filter(value -> value.getName().equals(args[4]))
.findFirst().orElseThrow();
- check(provider.getApiVersion() == DurableExecutionPluginProvider.API_VERSION, "released SPI version");
- checkSource(provider.getPluginType(), Path.of(args[1]));
+ checkSource(provider.getClass(), Path.of(args[1]));
+ checkSource(LocalDurableTestRunner.class, Path.of(args[6]));
}
- private static void assertOldCoreRejected(String view) throws Exception {
- var provider = ServiceLoader.load(DurableExecutionPluginProvider.class).stream()
- .map(ServiceLoader.Provider::get).filter(value -> value.getName().equals(view))
- .findFirst().orElseThrow();
- expectCoreRejection(provider::createPlugin);
- var constructors = provider.getPluginType().getConstructors();
- check(constructors.length == 4, "exercise every published plugin constructor");
- for (var constructor : constructors) {
- var types = constructor.getParameterTypes();
- var parameters = new Object[types.length];
- for (int i = 0; i < types.length; i++) {
- parameters[i] = types[i].getName().endsWith("OtelPluginConfig")
- ? types[i].getMethod("defaults").invoke(null) : SdkTracerProvider.builder();
- }
- expectCoreRejection(() -> constructor.newInstance(parameters));
- }
- var handlerCalls = new AtomicInteger();
- expectCoreRejection(() -> createRunner(handlerCalls, new AtomicInteger()));
- check(handlerCalls.get() == 0 && HEALTHY_STARTS.get() == 0 && HEALTHY_ENDS.get() == 0,
- "configuration rejection must precede handler execution and lifecycle hooks");
- System.out.println("CORE_LIFECYCLE_REJECTION_CONFIRMED " + view);
- }
-
- private static void expectCoreRejection(Callable construction) {
- Throwable rejected = null;
- try {
- construction.call();
- } catch (Throwable failure) {
- rejected = failure;
- }
- check(rejected != null, "new plugin silently accepted the released core without its lifecycle guarantee");
- for (var failure = rejected; failure != null; failure = failure.getCause()) {
- if (failure instanceof IllegalStateException && failure.getMessage() != null
- && failure.getMessage().contains("requires same-thread invocation hooks")) return;
- }
- throw new AssertionError("missing explicit core compatibility diagnostic", rejected);
- }
-
- private static void exercise(String view, boolean negative, boolean compatible, ListAppender logs) {
+ private static void exercise(String view, boolean compatible, ListAppender logs) {
GlobalOpenTelemetry.resetForTest();
// Reproduce #763's documented visible-API skew, not a claim of a deployed agent test.
OtelPluginAutoConfigurationState.markInstalled();
var handlerCalls = new AtomicInteger();
var sideEffects = new AtomicInteger();
var runner = createRunner(handlerCalls, sideEffects);
- check(HEALTHY_CREATED.get() == 1, "existing SPI must create the healthy plugin once per configuration");
+ check(HEALTHY_CREATED.get() == 0, "configuration must not create an invocation-owned plugin");
var first = runner.run("compatibility-input");
- if (negative) {
- assertNegative(first, handlerCalls, sideEffects);
- return;
- }
+ check(HEALTHY_CREATED.get() == 1, "first invocation must create its healthy plugin");
check(first.getStatus() == ExecutionStatus.PENDING, "plugin failure must preserve first invocation");
check(HEALTHY_STARTS.get() == 1 && HEALTHY_ENDS.get() == 1 && handlerCalls.get() == 1,
"healthy plugin and handler must run");
@@ -143,18 +99,6 @@ private static LocalDurableTestRunner createRunner(
});
}
- private static void assertNegative(
- TestResult result, AtomicInteger handlerCalls, AtomicInteger sideEffects) {
- check(result.getStatus() == ExecutionStatus.FAILED, "old/old older API must reproduce customer failure");
- var failure = result.getError().orElseThrow();
- check(failure.errorType().endsWith("NoSuchMethodError")
- && failure.errorMessage().contains("GlobalOpenTelemetry.isSet"),
- "negative control must reproduce the exact unsupported API: " + failure);
- check(HEALTHY_STARTS.get() == 0 && handlerCalls.get() == 0 && sideEffects.get() == 0,
- "old linkage failure must precede the healthy start hook and handler");
- System.out.println("NEGATIVE_CONTROL_REPRODUCED NoSuchMethodError GlobalOpenTelemetry.isSet");
- }
-
private static void resumeAndCheck(LocalDurableTestRunner runner, String view, boolean compatible,
AtomicInteger handlerCalls, AtomicInteger sideEffects) {
var exporter = InMemorySpanExporter.create();
@@ -167,7 +111,7 @@ private static void resumeAndCheck(LocalDurableTestRunner runner
check(HEALTHY_STARTS.get() == 2 && HEALTHY_ENDS.get() == 2 && handlerCalls.get() == 2,
"healthy hooks and handler must remain active on resume");
check(sideEffects.get() == 1, "completed user step must not repeat on resume");
- check(HEALTHY_CREATED.get() == 1, "resume must preserve the existing 2.x plugin instance lifetime");
+ check(HEALTHY_CREATED.get() == 2, "resume must create a new invocation-owned healthy plugin");
var spans = exporter.getFinishedSpanItems();
if (compatible) check(spans.stream().anyMatch(span -> span.getName().equals("Workflow")),
"compatible global provider must export Workflow spans in " + view);
@@ -191,19 +135,13 @@ private static SdkTracerProvider registerLateGlobal(boolean compatible, InMemory
return tracing;
}
- /** A real old-SPI service provider, discovered alongside the actual released/candidate OTel provider. */
+ /** A real factory service provider, discovered alongside the current OTel provider. */
public static final class HealthyProvider implements DurableExecutionPluginProvider {
@Override
public String getName() { return "compat-healthy"; }
@Override
- public int getApiVersion() { return API_VERSION; }
-
- @Override
- public Class getPluginType() { return HealthyPlugin.class; }
-
- @Override
- public DurableExecutionPlugin createPlugin() {
+ public DurableExecutionPlugin createPlugin(InvocationInfo info) {
HEALTHY_CREATED.incrementAndGet();
return new HealthyPlugin();
}
diff --git a/otel-plugin/src/test/compatibility/b1/README.md b/otel-plugin/src/test/compatibility/b1/README.md
index a0d8ab922f..b3d21b06e5 100644
--- a/otel-plugin/src/test/compatibility/b1/README.md
+++ b/otel-plugin/src/test/compatibility/b1/README.md
@@ -1,39 +1,33 @@
# Installed OpenTelemetry API compatibility
-Run after building the SDK and OTel plugin:
+Run after building the current core, testing SDK, and OTel plugin:
```sh
python3 .github/scripts/verify_otel_api_compatibility.py
```
-The driver resolves real released core, testing, and plugin **2.2.1** artifacts
-through Maven. It compiles one probe against the released SPI and runs fresh JVMs
-with old/old, new/old, old/new, and new/new core/plugin pairs, in both OTel views,
-using actual API/context **1.49.0** and **1.66.0** jars. Candidate jars come from
-the reactor build, or explicit `--new-core` / `--new-plugin` paths.
+This 3.x branch compiles its service-provider probe against the current factory
+API. It runs the current core, plugin, and testing SDK together in fresh JVMs,
+in both OTel views, with actual OpenTelemetry API/context **1.49.0** and **1.66.0**.
+The four required cases check:
-The sixteen required cases verify:
+- Core, plugin, testing SDK, API, and context classes load from the intended jars.
+- ServiceLoader discovers real providers; configuration creates no invocation
+ instances, and each initial/resumed invocation creates its own healthy plugin.
+- An unsupported visible API disables the affected instrumentation with a
+ diagnostic while healthy hooks, handler output, and completed-step replay work.
+- Early global-provider lookup does not install a no-op global. Later registration
+ succeeds, and API 1.66 exports real Workflow spans after resume.
-- Selected core, plugin, API, and context classes load from the intended jars.
-- The existing service-provider API discovers and creates a healthy plugin, with
- the same instance lifetime across suspension/resume.
-- Old/old plus API 1.49 reproduces the exact `GlobalOpenTelemetry.isSet`
- `NoSuchMethodError` and customer failure, before healthy start hooks/user code.
-- Fixed combinations isolate that mismatch, report a diagnostic, and preserve
- healthy hooks, handler output, and completed-step replay behavior.
-- An early unsupported/uninitialized global provider does not install a no-op
- global: subsequent registration must succeed. API 1.66 then exports real
- Workflow spans; unsupported API 1.49 disables the affected instrumentation.
+The independent fixture POM resolves the common dependency graph from released
+2.2.1 artifacts. The driver **replaces all three durable artifacts** with the
+current reactor jars before compiling or running. Explicit `--new-core`,
+`--new-plugin`, and `--new-testing` paths are available for an identical built tree.
+No 2.x/3.x interoperability is claimed: 3.x requires rebuilt factory providers.
+The 2.x branch retains its separate sixteen-case old/new compatibility matrix.
-Resolution, compilation, timeout, and probe failures all fail the command. There
-is no network-dependent skip. Logs, artifact SHA-256 hashes, negative-control
-results, and the case summary are written to `target/otel-api-compatibility`.
-The fixture POM is independent of the reactor and changes no production version
-or dependency floor.
-
-This matrix reproduces visible-API/classpath skew with real artifacts. Its test
-marker enables the documented plugin auto-configuration path; it does **not**
-claim to deploy or validate every Java-agent version. Actual agent validation
-must identify the released agent version, extension jar, visible API, runtime,
-and observed behavior separately. Provider registration remains fail-fast for
-invalid configuration; the fixture does not introduce invocation factories.
+Resolution, compilation, timeout, and probe failures fail the command; there is
+no network-dependent skip. Logs, jar hashes, and results go to
+`target/otel-api-compatibility`. No production dependencies or versions change.
+This is a visible-API test with an explicit auto-configuration marker, not a
+claim of deployed Java-agent coverage.
diff --git a/otel-plugin/src/test/compatibility/run_matrix.py b/otel-plugin/src/test/compatibility/run_matrix.py
new file mode 100644
index 0000000000..d38b5d629e
--- /dev/null
+++ b/otel-plugin/src/test/compatibility/run_matrix.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Run 2.x-only installed core/plugin compatibility with an actual separate layer loader and a legacy-compiled caller.
+
+No downloads or AWS calls. Supply released core/plugin jars, built current artifacts/directories, and an existing
+classpath file containing their common dependencies. No SDK production dependencies are added for this check.
+"""
+import argparse
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import zipfile
+import xml.etree.ElementTree as ET
+
+parser = argparse.ArgumentParser(description=__doc__)
+for name in ("old-core", "old-plugin", "new-core", "new-plugin", "dependencies-classpath"):
+ parser.add_argument("--" + name, required=True, type=Path)
+parser.add_argument("--java-home", required=True, type=Path)
+args = parser.parse_args()
+def artifact_version(path):
+ if path.is_dir():
+ version_file = path / "version.prop"
+ if version_file.exists():
+ properties = version_file.read_text()
+ else:
+ pom = path.parent.parent / "pom.xml"
+ if not pom.exists():
+ parser.error(f"Cannot verify SDK version for {path}; supply a Maven artifact")
+ root = ET.parse(pom).getroot()
+ ns = {"m": "http://maven.apache.org/POM/4.0.0"}
+ version = root.find("m:version", ns)
+ if version is None:
+ version = root.find("m:parent/m:version", ns)
+ return version.text if version is not None else None
+ else:
+ with zipfile.ZipFile(path) as jar:
+ names = [name for name in jar.namelist()
+ if name.startswith("META-INF/maven/software.amazon.lambda.durable/")
+ and name.endswith("/pom.properties")]
+ if not names:
+ parser.error(f"Cannot verify SDK version for {path}")
+ properties = jar.read(names[0]).decode()
+ for line in properties.splitlines():
+ if line.startswith("version="):
+ return line.split("=", 1)[1].strip()
+ return None
+
+for artifact in (args.old_core, args.old_plugin, args.new_core, args.new_plugin):
+ version = artifact_version(artifact)
+ if not version or not version.startswith("2."):
+ parser.error(f"This legacy ABI harness accepts only 2.x artifacts; {artifact} has version {version!r}. "
+ "Use the factory/provider migration tests for 3.x.")
+
+known = {p.resolve() for p in (args.old_core, args.old_plugin, args.new_core, args.new_plugin)}
+entries = args.dependencies_classpath.read_text().strip().split(os.pathsep)
+deps = [p for p in entries if Path(p).resolve() not in known
+ and not Path(p).name.startswith("aws-durable-execution-sdk-java")]
+source = Path(__file__).with_name("PluginLayerCompatibilityProbe.java")
+target = Path.cwd() / "target"
+target.mkdir(exist_ok=True)
+with tempfile.TemporaryDirectory(prefix="layer-compat-", dir=target) as output:
+ subprocess.run([str(args.java_home / "bin/javac"), "--release", "17", "-cp",
+ os.pathsep.join([str(args.old_core), *deps]), "-d", output, str(source)], check=True)
+ for label, core, plugin in [("old/old", args.old_core, args.old_plugin),
+ ("old/new", args.old_core, args.new_plugin),
+ ("new/old", args.new_core, args.old_plugin),
+ ("new/new", args.new_core, args.new_plugin)]:
+ for view in ("otel-invocation", "otel-execution"):
+ result = subprocess.run([str(args.java_home / "bin/java"), "-cp",
+ os.pathsep.join([output, str(core), *deps]), "PluginLayerCompatibilityProbe", str(plugin), view],
+ capture_output=True, text=True)
+ if result.returncode:
+ raise RuntimeError(f"{label} {view} failed:\n{result.stdout}{result.stderr}")
+ print(label, result.stdout.strip())
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java
index b90217c375..58494dc79d 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java
@@ -12,8 +12,6 @@
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
-import java.io.File;
-import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
@@ -24,8 +22,8 @@
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.MDC;
-import software.amazon.lambda.durable.context.DurableContextImpl;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.otel.ExecutionOtelPlugin;
import software.amazon.lambda.durable.otel.ExecutionOtelPluginProvider;
@@ -33,53 +31,33 @@
import software.amazon.lambda.durable.otel.InvocationOtelPluginProvider;
import software.amazon.lambda.durable.otel.OtelPluginConfig;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.ExclusivePluginGroup;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
class OtelViewRegistrationTest {
@Test
- void existingSubclassGroupMethodsRemainCompatible() {
- // Downstream subclasses could already declare this method before group metadata joined the plugin API.
- var invocation = new InvocationOtelPlugin() {
+ void factoryMetadataRejectsConflictsBeforeCreatingInstances() {
+ var created = new AtomicInteger();
+ var factory = new DurableExecutionPluginFactory() {
+ @Override
public String getExclusiveGroup() {
- return "legacy application group";
+ return "custom-tracing";
}
- };
- var execution = new ExecutionOtelPlugin() {
- public String getExclusiveGroup() {
- return null;
+
+ @Override
+ public DurableExecutionPlugin createPlugin(InvocationInfo info) {
+ created.incrementAndGet();
+ return new DurableExecutionPlugin() {};
}
};
- assertEquals("legacy application group", invocation.getExclusiveGroup());
- assertNull(execution.getExclusiveGroup());
- var error = assertThrows(
- IllegalStateException.class,
- () -> DurableConfig.builder().withPlugins(invocation, execution).build());
- assertTrue(error.getMessage().contains(invocation.getClass().getName()));
- assertTrue(error.getMessage().contains(execution.getClass().getName()));
- assertDoesNotThrow(() -> DurableConfig.builder().withPlugins(invocation).build());
- }
-
- @ParameterizedTest
- @CsvSource({"true,false", "true,true", "false,false", "false,true"})
- void reannotatedSubclassesRetainViewExclusivity(boolean executionSubclass, boolean reversed) {
- DurableExecutionPlugin custom =
- executionSubclass ? new ReannotatedExecutionPlugin() : new ReannotatedInvocationPlugin();
- DurableExecutionPlugin opposite = executionSubclass ? new InvocationOtelPlugin() : new ExecutionOtelPlugin();
- var first = reversed ? opposite : custom;
- var second = reversed ? custom : opposite;
- var error = assertThrows(
- IllegalStateException.class,
- () -> DurableConfig.builder().withPlugins(first, second).build());
- assertTrue(error.getMessage().contains("durable-otel-view"));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> DurableConfig.builder().withPlugins(factory, factory).build());
+ assertDoesNotThrow(() -> DurableConfig.builder().withPlugins(factory).build());
+ assertEquals(0, created.get());
}
- @ExclusivePluginGroup("application-instrumentation")
- private static class ReannotatedInvocationPlugin extends InvocationOtelPlugin {}
-
- @ExclusivePluginGroup("application-instrumentation")
- private static class ReannotatedExecutionPlugin extends ExecutionOtelPlugin {}
-
@ParameterizedTest
@CsvSource({"explicit,false", "explicit,true", "dynamic,false", "dynamic,true", "mixed,false", "mixed,true"})
void rejectsBothViewsBeforeEmissionAndLeavesContextUntouched(String path, boolean reversed) {
@@ -89,9 +67,9 @@ void rejectsBothViewsBeforeEmissionAndLeavesContextUntouched(String path, boolea
var providers = List.of(new ExecutionOtelPluginProvider(), new InvocationOtelPluginProvider());
var names = reversed ? "otel-execution,otel-invocation" : "otel-invocation,otel-execution";
var plugins = switch (path) {
- case "dynamic" -> DynamicPluginLoader.loadConfiguredPlugins(names, providers, List.of());
+ case "dynamic" -> DynamicPluginLoader.loadConfiguredPluginFactories(names, providers, List.of());
case "mixed" ->
- DynamicPluginLoader.loadConfiguredPlugins(
+ DynamicPluginLoader.loadConfiguredPluginFactories(
reversed ? "otel-execution" : "otel-invocation", providers, List.of(second));
default -> List.of(first, second);
};
@@ -105,11 +83,10 @@ void rejectsBothViewsBeforeEmissionAndLeavesContextUntouched(String path, boolea
MDC.put("trace_id", "existing");
try {
var error = assertThrows(
- IllegalStateException.class,
+ IllegalArgumentException.class,
() -> DurableConfig.builder()
- .withPlugins(plugins.toArray(DurableExecutionPlugin[]::new))
+ .withPlugins(plugins.toArray(DurableExecutionPluginFactory[]::new))
.build());
- assertTrue(error.getMessage().startsWith("Dynamic plugin configuration failed: "));
assertTrue(error.getMessage().contains("InvocationOtelPlugin"));
assertTrue(error.getMessage().contains("ExecutionOtelPlugin"));
assertTrue(error.getMessage().contains("only one"));
@@ -122,30 +99,13 @@ void rejectsBothViewsBeforeEmissionAndLeavesContextUntouched(String path, boolea
}
}
- @ParameterizedTest
- @CsvSource({"true", "false"})
- void repeatedExplicitViewInstancesAreRejectedBeforeEmission(boolean executionView) {
- var exporter = InMemorySpanExporter.create();
- var first = plugin(executionView, exporter);
- var second = plugin(executionView, exporter);
- var ambient = Context.current();
- assertThrows(
- IllegalStateException.class,
- () -> DurableConfig.builder().withPlugins(first, second).build());
- assertThrows(
- IllegalStateException.class,
- () -> DurableConfig.builder().withPlugins(first, first).build());
- assertTrue(exporter.getFinishedSpanItems().isEmpty());
- assertSame(ambient, Context.current());
- }
-
@ParameterizedTest
@CsvSource({"true,true", "true,false", "false,true", "false,false"})
void singleViewWithUnrelatedPluginPreservesResumeAndOutcome(boolean executionView, boolean success) {
var exporter = InMemorySpanExporter.create();
var view = plugin(executionView, exporter);
var config = DurableConfig.builder()
- .withPlugins(view, new DurableExecutionPlugin() {})
+ .withPlugins(view, info -> new DurableExecutionPlugin() {})
.build();
var effects = new AtomicInteger();
var runner = LocalDurableTestRunner.create(
@@ -175,18 +135,13 @@ void singleViewWithUnrelatedPluginPreservesResumeAndOutcome(boolean executionVie
}
@ParameterizedTest
- @CsvSource({"otel-invocation,false", "otel-execution,false", "otel-invocation,true", "otel-execution,true"})
- void environmentSelectedSingleViewCanBeCopiedIntoLocalRunner(
- String provider, boolean releasedTestingSdk, @TempDir Path directory) throws Exception {
+ @ValueSource(strings = {"otel-invocation", "otel-execution"})
+ void environmentSelectedSingleViewCanBeCopiedIntoLocalRunner(String provider, @TempDir Path directory)
+ throws Exception {
var java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
var classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"));
- var testingLocation =
- releasedTestingSdk ? Path.of(System.getProperty("releasedTestingSdkJar")) : testingSdkLocation();
- assertTrue(Files.exists(testingLocation), "Testing SDK artifact must exist");
- classpath = testingLocation + File.pathSeparator + classpath;
var output = directory.resolve("child.log");
- var builder = new ProcessBuilder(
- java, "-cp", classpath, EnvironmentRunnerCheck.class.getName(), testingLocation.toString())
+ var builder = new ProcessBuilder(java, "-cp", classpath, EnvironmentRunnerCheck.class.getName())
.redirectErrorStream(true)
.redirectOutput(output.toFile());
builder.environment().put("DURABLE_EXECUTION_PLUGINS", provider);
@@ -199,14 +154,6 @@ void environmentSelectedSingleViewCanBeCopiedIntoLocalRunner(
}
}
- private static Path testingSdkLocation() throws URISyntaxException {
- return Path.of(LocalDurableTestRunner.class
- .getProtectionDomain()
- .getCodeSource()
- .getLocation()
- .toURI());
- }
-
private static String read(Path path) {
try {
return Files.readString(path);
@@ -216,78 +163,46 @@ private static String read(Path path) {
}
public static class EnvironmentRunnerCheck {
- public static void main(String[] args) throws Exception {
- // Ensure the old-testing cases execute the released bytecode, not the reactor's updated runner.
- assertEquals(Path.of(args[0]).toRealPath(), testingSdkLocation().toRealPath());
+ public static void main(String[] args) {
var config = DurableConfig.builder()
.withDeserializeAfterSerialization(false)
.build();
- assertEquals(1, config.getPluginRunner().getPlugins().size());
+ assertEquals(1, config.getPluginFactories().size());
var copy = config.toBuilder().build();
- assertEquals(
- config.getPluginRunner().getPlugins(),
- copy.getPluginRunner().getPlugins());
+ assertEquals(config.getPluginFactories(), copy.getPluginFactories());
assertFalse(copy.shouldDeserializeAfterSerialization());
- assertCopiedBuilderReplacementDoesNotRediscover(config);
- var effects = new AtomicInteger();
+ assertTrue(config.toBuilder()
+ .withPlugins()
+ .build()
+ .getPluginFactories()
+ .isEmpty());
+ DurableExecutionPluginFactory replacement = info -> new DurableExecutionPlugin() {};
+ assertEquals(
+ List.of(replacement),
+ config.toBuilder().withPlugins(replacement).build().getPluginFactories());
+ assertEquals(1, DurableConfig.builder().build().getPluginFactories().size());
var runner = LocalDurableTestRunner.create(
String.class,
(input, ctx) -> {
- var effective = ((DurableContextImpl) ctx)
- .getDurableConfig()
- .getPluginRunner()
- .getPlugins();
- assertEquals(1, effective.size(), "Released and current runner copies must retain one view");
- assertSame(config.getPluginRunner().getPlugins().get(0), effective.get(0));
- ctx.step("once", Integer.class, step -> effects.incrementAndGet());
ctx.wait("pause", Duration.ofSeconds(1));
return input;
},
config);
assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
runner.advanceTime();
- var result = runner.runUntilComplete("input");
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertEquals("input", result.getResult(String.class));
- assertEquals(1, effects.get());
- }
-
- private static void assertCopiedBuilderReplacementDoesNotRediscover(DurableConfig config) {
- var replacement = new DurableExecutionPlugin() {};
- var copiedBuilder = config.toBuilder();
- assertTrue(copiedBuilder
- .withPlugins()
- .build()
- .getPluginRunner()
- .getPlugins()
- .isEmpty());
- assertEquals(
- List.of(replacement),
- copiedBuilder
- .withPlugins(replacement)
- .build()
- .getPluginRunner()
- .getPlugins());
-
- var fresh = DurableConfig.builder()
- .withPlugins(replacement)
- .build()
- .getPluginRunner()
- .getPlugins();
- assertEquals(2, fresh.size(), "Fresh builders continue to honor environment discovery");
assertEquals(
- config.getPluginRunner().getPlugins().get(0).getClass(),
- fresh.get(0).getClass());
- assertSame(replacement, fresh.get(1));
+ ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus());
}
}
- private static DurableExecutionPlugin plugin(boolean executionView, InMemorySpanExporter exporter) {
+ private static DurableExecutionPluginFactory plugin(boolean executionView, InMemorySpanExporter exporter) {
var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
var config = OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(false)
.build();
- return executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
+ return executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
}
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java
index 5c91d0413b..328dc35cf7 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java
@@ -29,6 +29,7 @@
import software.amazon.lambda.durable.model.DurableExecutionInput;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.OperationEndInfo;
@Timeout(15)
@@ -98,9 +99,9 @@ public GetDurableExecutionStateResponse getExecutionState(String arn, String tok
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(builder, settings)
- : new InvocationOtelPlugin(builder, settings);
+ DurableExecutionPluginFactory otel = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
var ends = new CopyOnWriteArrayList();
var observer = new DurableExecutionPlugin() {
public void onOperationEnd(OperationEndInfo info) {
@@ -110,7 +111,7 @@ public void onOperationEnd(OperationEndInfo info) {
var caught = new AtomicInteger();
var config = DurableConfig.builder()
.withDurableExecutionClient(client)
- .withPlugins(otel, observer)
+ .withPlugins(otel, ignored -> observer)
.build();
for (int invocation = 0; invocation < 2; invocation++) {
var firstLive = live && invocation == 0;
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java
index b79d439ed5..546e55611a 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java
@@ -21,6 +21,7 @@
import software.amazon.lambda.durable.exception.DurableOperationException;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.OperationEndInfo;
@@ -106,9 +107,9 @@ void publicCallbackPreservesStoredFailureButReportsOnlyPresentErrorDetails(
.contextExtractor(() -> null)
.enableMdc(false)
.build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(provider, options)
- : new InvocationOtelPlugin(provider, options);
+ DurableExecutionPluginFactory otel = executionView
+ ? ExecutionOtelPlugin.factory(provider, options)
+ : InvocationOtelPlugin.factory(provider, options);
var starts = new CopyOnWriteArrayList();
var ends = new CopyOnWriteArrayList();
var operationEnds = new CopyOnWriteArrayList();
@@ -133,7 +134,9 @@ public void onOperationEnd(OperationEndInfo info) {
String.class,
(input, context) -> context.waitForCallback(
"external-failure", String.class, (callbackId, step) -> submitterCalls.incrementAndGet()),
- DurableConfig.builder().withPlugins(otel, observer).build());
+ DurableConfig.builder()
+ .withPlugins(otel, ignored -> observer)
+ .build());
assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
var callbackId = runner.getCallbackId(CALLBACK);
if (timeout) runner.timeoutCallback(callbackId);
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtCallbackFailureStatusTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtCallbackFailureStatusTest.java
index ffdf7c98d1..3198a48e44 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtCallbackFailureStatusTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtCallbackFailureStatusTest.java
@@ -26,6 +26,7 @@
import software.amazon.lambda.durable.exception.DurableOperationException;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.OperationEndInfo;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
@@ -59,9 +60,9 @@ void caughtCallbackFailureIsErrorWhileHandlerSucceedsAndReplaySkipsSavedEffects(
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(builder, settings)
- : new InvocationOtelPlugin(builder, settings);
+ DurableExecutionPluginFactory otel = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
var ends = new CopyOnWriteArrayList();
var observer = new DurableExecutionPlugin() {
@Override
@@ -92,7 +93,9 @@ public void onOperationEnd(OperationEndInfo info) {
context.wait("resume-once", Duration.ofSeconds(1));
return "handled";
},
- DurableConfig.builder().withPlugins(otel, observer).build());
+ DurableConfig.builder()
+ .withPlugins(otel, ignored -> observer)
+ .build());
runnerRef.set(runner);
assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
if (!live) {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtFailureConformanceHandlerTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtFailureConformanceHandlerTest.java
index cfbc85c2eb..fc9a120741 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtFailureConformanceHandlerTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CaughtFailureConformanceHandlerTest.java
@@ -414,9 +414,9 @@ private Fixture(boolean execution, Path classes, String handlerName) throws Exce
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- DurableExecutionPlugin otel = execution
- ? new ExecutionOtelPlugin(provider, options)
- : new InvocationOtelPlugin(provider, options);
+ DurableExecutionPluginFactory otel = execution
+ ? ExecutionOtelPlugin.factory(provider, options)
+ : InvocationOtelPlugin.factory(provider, options);
var observer = new DurableExecutionPlugin() {
public void onOperationEnd(OperationEndInfo info) {
operationEnds.add(info);
@@ -443,7 +443,7 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
};
config = DurableConfig.builder()
.withExecutorService(workers)
- .withPlugins(otel, observer)
+ .withPlugins(otel, ignored -> observer)
.build();
runner = LocalDurableTestRunner.create(
new TypeToken>() {}, handler::handleRequest, config);
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ConfigurationCopyDiscoveryTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ConfigurationCopyDiscoveryTest.java
new file mode 100644
index 0000000000..87c8cc96ed
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ConfigurationCopyDiscoveryTest.java
@@ -0,0 +1,77 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+class ConfigurationCopyDiscoveryTest {
+ @Test
+ @Timeout(20)
+ void migrationCopyRetainsOneEnvironmentSelectedFactory(@TempDir Path directory) throws Exception {
+ var output = directory.resolve("environment-copy.log");
+ var process = new ProcessBuilder(
+ Path.of(System.getProperty("java.home"), "bin", "java").toString(),
+ "-cp",
+ System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")),
+ EnvironmentCopy.class.getName())
+ .redirectErrorStream(true)
+ .redirectOutput(output.toFile());
+ process.environment().put("DURABLE_EXECUTION_PLUGINS", "otel-invocation");
+ var child = process.start();
+ try {
+ assertTrue(child.waitFor(10, TimeUnit.SECONDS), "environment copy fixture timed out");
+ assertEquals(0, child.exitValue(), () -> read(output));
+ assertTrue(read(output).contains("PASS selected=1 copied=1 handler=SUCCEEDED"));
+ System.out.println(read(output));
+ } finally {
+ child.destroyForcibly();
+ }
+ }
+
+ private static String read(Path file) {
+ try {
+ return Files.readString(file);
+ } catch (Exception failure) {
+ throw new AssertionError(failure);
+ }
+ }
+
+ public static class EnvironmentCopy {
+ public static void main(String[] args) {
+ assertEquals("otel-invocation", System.getenv("DURABLE_EXECUTION_PLUGINS"));
+ var config = DurableConfig.builder().build();
+ assertEquals(1, config.getPluginFactories().size());
+ var duplicate = assertThrows(
+ IllegalArgumentException.class,
+ () -> DurableConfig.builder()
+ .withPlugins(config.getPluginFactories().toArray(new DurableExecutionPluginFactory[0]))
+ .build());
+ assertTrue(duplicate.getMessage().contains("durable-otel-view"));
+ var testClient = new LocalMemoryExecutionClient();
+ var derived =
+ config.toBuilder().withDurableExecutionClient(testClient).build();
+ assertSame(testClient, derived.getDurableExecutionClient());
+ assertEquals(1, derived.getPluginFactories().size());
+ assertSame(
+ config.getPluginFactories().get(0),
+ derived.getPluginFactories().get(0));
+ var result = LocalDurableTestRunner.create(String.class, (input, context) -> input, derived)
+ .run("ok");
+ assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
+ assertEquals("ok", result.getResult(String.class));
+ System.out.println("PASS selected=1 copied=1 handler=SUCCEEDED");
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java
new file mode 100644
index 0000000000..282e415f17
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java
@@ -0,0 +1,137 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.api.trace.TracerProvider;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+
+class CrossLoaderParentSamplingTest {
+ @AfterEach
+ void clearGlobalState() {
+ GlobalOpenTelemetry.resetForTest();
+ DurableSamplingDecision.clearSharedStateForTest();
+ DeterministicIdGenerator.clearSharedStateForTest();
+ OtelPluginAutoConfigurationState.resetInstalledForTest();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void agentProcessorCannotForwardDurableDecisionToApplicationProvider(boolean executionView) throws Exception {
+ var observations = new ArrayList();
+ try (var agentLoader = DurableSamplingDecisionClassLoaderTest.pluginClassLoader();
+ var application = SdkTracerProvider.builder()
+ .setSampler(DurableSampler.wrap(Sampler.alwaysOff()))
+ .build();
+ var agent = agentProvider(agentLoader, application.get("application"), observations)) {
+ GlobalOpenTelemetry.set(opaqueTelemetry(agent));
+ OtelPluginAutoConfigurationState.markInstalled();
+ var config = OtelPluginConfig.builder()
+ .contextExtractor(() -> new ExtractedContext(
+ "aabbccddee112233445566778899aabb", null, ExtractedContext.Sampling.SAMPLED))
+ .enableMdc(false)
+ .build();
+ var factory = executionView ? ExecutionOtelPlugin.factory(config) : InvocationOtelPlugin.factory(config);
+ var arn = "arn:aws:lambda:us-east-1:123:function:test/durable/exec";
+ var info = new InvocationInfo("request", arn, true, Instant.EPOCH);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
+ plugin.onInvocationEnd(new InvocationEndInfo("request", arn, true, InvocationStatus.SUCCEEDED, null));
+ assertTrue(observations.stream().anyMatch(o -> o.name().equals("Workflow")));
+ assertTrue(observations.stream().anyMatch(o -> o.name().equals("DurableExecutionRoot")));
+ for (var observation : observations) {
+ assertFalse(
+ observation.auxiliaryRecording(), "forwarded parent leaked sampling for " + observation.name());
+ }
+ }
+ }
+
+ private static SdkTracerProvider agentProvider(
+ ClassLoader loader, Tracer application, List observations) throws Exception {
+ var samplerClass = Class.forName(DurableSampler.class.getName(), true, loader);
+ assertNotSame(DurableSampler.class, samplerClass);
+ var wrap = samplerClass.getDeclaredMethod("wrap", Sampler.class);
+ wrap.setAccessible(true);
+ var sampler = (Sampler) wrap.invoke(null, Sampler.alwaysOn());
+ var builder = SdkTracerProvider.builder()
+ .setSampler(sampler)
+ .addSpanProcessor(new ForwardingProcessor(application, observations));
+ var ids = Class.forName(DeterministicIdGenerator.class.getName(), true, loader);
+ var install = ids.getDeclaredMethod("installOn", SdkTracerProviderBuilder.class);
+ install.setAccessible(true);
+ install.invoke(null, builder);
+ return builder.build();
+ }
+
+ private static OpenTelemetry opaqueTelemetry(SdkTracerProvider provider) {
+ var opaque = new TracerProvider() {
+ @Override
+ public Tracer get(String name) {
+ return provider.get(name);
+ }
+
+ @Override
+ public Tracer get(String name, String version) {
+ return provider.get(name, version);
+ }
+ };
+ return new OpenTelemetry() {
+ @Override
+ public TracerProvider getTracerProvider() {
+ return opaque;
+ }
+
+ @Override
+ public ContextPropagators getPropagators() {
+ return ContextPropagators.noop();
+ }
+ };
+ }
+
+ private record Observation(String name, boolean auxiliaryRecording) {}
+
+ private record ForwardingProcessor(Tracer application, List observations) implements SpanProcessor {
+ @Override
+ public void onStart(Context parent, ReadWriteSpan span) {
+ var auxiliary = application
+ .spanBuilder("processor-auxiliary")
+ .setParent(parent)
+ .startSpan();
+ observations.add(new Observation(span.getName(), auxiliary.isRecording()));
+ auxiliary.end();
+ }
+
+ @Override
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ @Override
+ public void onEnd(ReadableSpan span) {}
+
+ @Override
+ public boolean isEndRequired() {
+ return false;
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java
new file mode 100644
index 0000000000..a762d23e43
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java
@@ -0,0 +1,57 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+class DelayedFlush implements SpanProcessor {
+ public void onStart(Context parent, ReadWriteSpan span) {}
+
+ final HandlerScopeFinalizationTest.Deadline deadline;
+ final AtomicInteger calls = new AtomicInteger();
+ final AtomicInteger remainingAtFlush = new AtomicInteger();
+ final AtomicLong flushMillis = new AtomicLong();
+ final ScheduledExecutorService clock;
+
+ DelayedFlush(HandlerScopeFinalizationTest.Deadline deadline, ScheduledExecutorService clock) {
+ this.deadline = deadline;
+ this.clock = clock;
+ }
+
+ public boolean isStartRequired() {
+ return false;
+ }
+
+ public void onEnd(ReadableSpan span) {}
+
+ public boolean isEndRequired() {
+ return false;
+ }
+
+ public CompletableResultCode shutdown() {
+ return CompletableResultCode.ofSuccess();
+ }
+
+ public CompletableResultCode forceFlush() {
+ calls.incrementAndGet();
+ remainingAtFlush.set(deadline.remaining());
+ var result = new CompletableResultCode();
+ long start = System.nanoTime();
+ clock.schedule(
+ () -> {
+ flushMillis.set(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
+ result.succeed();
+ },
+ 250,
+ TimeUnit.MILLISECONDS);
+ return result;
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java
index f6126492e5..f84338d2e1 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java
@@ -6,6 +6,7 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static software.amazon.lambda.durable.otel.Invocations.started;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.common.Attributes;
@@ -199,7 +200,7 @@ private static Sampler captureInstalledSampler(Sampler effectiveSampler) {
void configuredSampler_isEvaluatedAtMostOncePerInvocation() {
var delegate = new CountingSampler(Sampler.alwaysOn());
var exporter = InMemorySpanExporter.create();
- var plugin = new InvocationOtelPlugin(
+ var pluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().setSampler(delegate).addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -207,7 +208,7 @@ void configuredSampler_isEvaluatedAtMostOncePerInvocation() {
.build());
// A full invocation with a Workflow span, Invocation span, operation span, and attempt span.
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(pluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onUserFunctionStart(
@@ -253,7 +254,7 @@ void explicitNotSampled_winsOverConfiguredAlwaysOn() {
private InMemorySpanExporter exportedWith(Sampler configuredSampler, ExtractedContext.Sampling sampling) {
var exporter = InMemorySpanExporter.create();
var extracted = new ExtractedContext(TRACE_ID, SPAN_ID, sampling);
- var plugin = new InvocationOtelPlugin(
+ var pluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(configuredSampler)
.addSpanProcessor(SimpleSpanProcessor.create(exporter)),
@@ -262,7 +263,7 @@ private InMemorySpanExporter exportedWith(Sampler configuredSampler, ExtractedCo
.enableMdc(false)
.build());
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(pluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
return exporter;
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java
index 613e8e61cc..68f29f4c0f 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java
@@ -36,13 +36,14 @@
import java.net.URLClassLoader;
import java.time.Instant;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -141,13 +142,23 @@ public ContextPropagators getPropagators() {
return ContextPropagators.noop();
}
});
- var plugin = (DurableExecutionPlugin)
+ var factory = (DurableExecutionPluginFactory)
Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader)
- .getConstructor()
- .newInstance();
+ .getMethod("factory")
+ .invoke(null);
var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/custom/id";
for (var first : new boolean[] {true, false}) {
- plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10)));
+ var info = new InvocationInfo(
+ "request",
+ arn,
+ first,
+ Instant.ofEpochSecond(10),
+ null,
+ Map.of(),
+ Map.of(),
+ System.getProperty("com.amazonaws.xray.traceHeader"));
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onInvocationEnd(new InvocationEndInfo(
"request",
arn,
@@ -282,17 +293,27 @@ public ContextPropagators getPropagators() {
return ContextPropagators.noop();
}
});
- var plugin = (DurableExecutionPlugin)
+ var factory = (DurableExecutionPluginFactory)
Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader)
- .getConstructor()
- .newInstance();
+ .getMethod("factory")
+ .invoke(null);
var executionCount = sharedTraceExecutions ? 2 : 1;
for (var index = 0; index < executionCount; index++) {
var executionName = index == 0 ? "first" : "second";
var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/" + executionName
+ "/id";
for (var first : new boolean[] {true, false}) {
- plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10)));
+ var info = new InvocationInfo(
+ "request",
+ arn,
+ first,
+ Instant.ofEpochSecond(10),
+ null,
+ Map.of(),
+ Map.of(),
+ System.getProperty("com.amazonaws.xray.traceHeader"));
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onInvocationEnd(
new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null));
}
@@ -494,10 +515,10 @@ public ContextPropagators getPropagators() {
return ContextPropagators.noop();
}
});
- var plugin = (DurableExecutionPlugin)
+ var factory = (DurableExecutionPluginFactory)
Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader)
- .getConstructor()
- .newInstance();
+ .getMethod("factory")
+ .invoke(null);
var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id";
var previousAmbient = Span.current();
var ambient = appProvider
@@ -506,7 +527,17 @@ public ContextPropagators getPropagators() {
.startSpan();
try (var ambientScope = ambient.makeCurrent()) {
for (var first : new boolean[] {true, false}) {
- plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.EPOCH));
+ var info = new InvocationInfo(
+ "request",
+ arn,
+ first,
+ Instant.EPOCH,
+ null,
+ Map.of(),
+ Map.of(),
+ System.getProperty("com.amazonaws.xray.traceHeader"));
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onInvocationEnd(
new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null));
assertSame(ambient, Span.current(), "Plugin cleanup must preserve the caller's ambient span");
@@ -534,7 +565,7 @@ public ContextPropagators getPropagators() {
* holds its own copies, mirroring the two plugin class loaders) while delegating OpenTelemetry and JDK classes to
* the parent so those types are shared and interoperable across loaders.
*/
- private static URLClassLoader pluginClassLoader() {
+ static URLClassLoader pluginClassLoader() {
var classesDir = DurableSamplingDecisionClassLoaderTest.class
.getProtectionDomain()
.getCodeSource()
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java
index 175e7ea9d3..24311c923b 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java
@@ -42,14 +42,15 @@ void setUp() {
OtelPluginAutoConfigurationState.resetInstalledForTest();
spanExporter = InMemorySpanExporter.create();
- var plugin = new ExecutionOtelPlugin(
+ // One factory for the environment; the SDK creates one plugin instance per invocation from it.
+ var factory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(false)
.build());
- otelConfig = DurableConfig.builder().withPlugins(plugin).build();
+ otelConfig = DurableConfig.builder().withPlugins(factory).build();
}
@AfterEach
@@ -168,8 +169,9 @@ public ContextPropagators getPropagators() {
}
});
- var defaultConfig =
- DurableConfig.builder().withPlugins(new ExecutionOtelPlugin()).build();
+ var defaultConfig = DurableConfig.builder()
+ .withPlugins(ExecutionOtelPlugin.factory())
+ .build();
var runner = LocalDurableTestRunner.create(
String.class,
(input, ctx) -> ctx.step("wrapped-step", String.class, stepCtx -> "Hello " + input),
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
index 95817a05ee..f15f73871f 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
@@ -3,6 +3,7 @@
package software.amazon.lambda.durable.otel;
import static org.junit.jupiter.api.Assertions.*;
+import static software.amazon.lambda.durable.otel.Invocations.started;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.common.AttributeKey;
@@ -37,7 +38,9 @@ class ExecutionOtelPluginTest {
private static final String CONFIGURED_SERVICE_NAME = "durable-execution-conformance";
private InMemorySpanExporter spanExporter;
- private ExecutionOtelPlugin plugin;
+
+ /** The environment's plugin factory; each test creates one instance per invocation from it. */
+ private DurableExecutionPluginFactory factory;
@BeforeEach
void setUp() {
@@ -46,7 +49,7 @@ void setUp() {
OtelPluginAutoConfigurationState.resetInstalledForTest();
spanExporter = InMemorySpanExporter.create();
var resource = Resource.create(Attributes.of(SERVICE_NAME, CONFIGURED_SERVICE_NAME));
- plugin = new ExecutionOtelPlugin(
+ factory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder()
.setResource(resource)
.addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
@@ -65,12 +68,12 @@ void tearDown() {
OtelPluginAutoConfigurationState.resetInstalledForTest();
}
- // ─── Default constructor ─────────────────────────────────────────────
+ // ─── Java agent path (global provider) ───────────────────────────────
@Test
void customInstrumentationName_isUsedForTracerScope() {
var exporter = InMemorySpanExporter.create();
- var customPlugin = new ExecutionOtelPlugin(
+ var customPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -78,7 +81,7 @@ void customInstrumentationName_isUsedForTracerScope() {
.workflowSpanName("Workflow")
.instrumentationName("my-custom-scope")
.build());
- customPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var customPlugin = started(customPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
customPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = exporter.getFinishedSpanItems();
@@ -89,12 +92,13 @@ void customInstrumentationName_isUsedForTracerScope() {
}
@Test
- void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
+ void agentPathFactory_bindsGlobalProviderOnALaterInvocationsInstance() {
GlobalOpenTelemetry.resetForTest();
OtelPluginAutoConfigurationState.markInstalled();
- var defaultPlugin = new ExecutionOtelPlugin();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now()));
+ var defaultPluginFactory = ExecutionOtelPlugin.factory();
+ var defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now()));
defaultPlugin.onOperationStart(new OperationInfo(
"op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -121,7 +125,8 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now()));
+ defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now()));
defaultPlugin.onOperationStart(new OperationInfo(
"op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -147,8 +152,8 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
}
@Test
- void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
- var defaultPlugin = new ExecutionOtelPlugin();
+ void agentPathFactory_usesGlobalSdkTracerProviderDirectly() {
+ var defaultPluginFactory = ExecutionOtelPlugin.factory();
assertFalse(GlobalOpenTelemetry.isSet());
OtelPluginAutoConfigurationState.markInstalled();
@@ -158,7 +163,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
defaultPlugin.onOperationStart(
new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -194,15 +200,22 @@ void executionOtelPluginProvider_isRegisteredAsServiceProvider() {
.get();
assertEquals("otel-execution", provider.getName());
- assertEquals(DurableExecutionPluginProvider.API_VERSION, provider.getApiVersion());
- assertEquals(ExecutionOtelPlugin.class, provider.getPluginType());
+
+ // The provider is the per-invocation factory: it creates an ExecutionOtelPlugin for the invocation it is
+ // handed,
+ // and a distinct instance for the next one.
+ var first = provider.createPlugin(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var second = provider.createPlugin(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ assertInstanceOf(ExecutionOtelPlugin.class, first);
+ assertInstanceOf(ExecutionOtelPlugin.class, second);
+ assertNotSame(first, second, "Each invocation gets its own plugin instance");
}
// ─── Workflow root span lifecycle ────────────────────────────────────
@Test
void terminalInvocation_exportsWorkflowAndInvocationSpans() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -217,7 +230,7 @@ void terminalInvocation_exportsWorkflowAndInvocationSpans() {
@Test
void spans_preserveConfiguredServiceName() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
for (var span : spanExporter.getFinishedSpanItems()) {
@@ -231,7 +244,7 @@ void spans_preserveConfiguredServiceName() {
@Test
void workflowSpan_startsAtExecutionStartTime() {
var start = Instant.parse("2026-01-15T08:00:00Z");
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, start));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, start));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var workflowSpan = spanByName(spanExporter.getFinishedSpanItems(), "Workflow");
@@ -243,7 +256,7 @@ void workflowSpan_startsAtExecutionStartTime() {
@Test
void workflowSpan_hasInternalKind() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
assertEquals(
@@ -254,7 +267,7 @@ void workflowSpan_hasInternalKind() {
@Test
void workflowAndInvocationSpans_shareExecutionTrace_withoutAmbientContext() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -281,7 +294,7 @@ void invocationStart_joinsAmbientTrace_whenAmbientIsOnExecutionTrace() {
// Drive an invocation to learn the canonical execution trace ID, then start a fresh invocation with an ambient
// span on that same trace: the Invocation span joins the ambient span directly.
var executionStart = Instant.parse("2026-01-01T00:00:00Z");
- plugin.onInvocationStart(new InvocationInfo("req-0", ARN, true, executionStart));
+ var plugin = started(factory, new InvocationInfo("req-0", ARN, true, executionStart));
plugin.onInvocationEnd(new InvocationEndInfo("req-0", ARN, true, InvocationStatus.SUCCEEDED, null));
var canonicalTraceId =
spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getTraceId();
@@ -291,9 +304,9 @@ void invocationStart_joinsAmbientTrace_whenAmbientIsOnExecutionTrace() {
var ambient =
SpanContext.create(canonicalTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault());
try (var ignored = Span.wrap(ambient).makeCurrent()) {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, false, executionStart));
- plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, false, InvocationStatus.SUCCEEDED, null));
+ plugin = started(factory, new InvocationInfo("req-1", ARN, false, executionStart));
}
+ plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, false, InvocationStatus.SUCCEEDED, null));
var invocationSpan = spanByName(spanExporter.getFinishedSpanItems(), "Invocation");
assertEquals(canonicalTraceId, invocationSpan.getTraceId());
@@ -310,10 +323,13 @@ void invocationStart_staysOnExecutionTrace_withoutLinkingAmbientSpan() {
var ambientSpanId = "1111111111111111";
var ambient =
SpanContext.create(ambientTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault());
+ // The instance is created inside the ambient scope because the invocation's parent resolution happens when the
+ // factory creates it, not later.
+ DurableExecutionPlugin plugin;
try (var ignored = Span.wrap(ambient).makeCurrent()) {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
- plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
+ plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
}
+ plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
var workflowSpan = spanByName(spans, "Workflow");
@@ -331,7 +347,7 @@ void contextExtractor_isInvokedEveryInvocation_evenWithAmbientSpan_andBackendCon
var backendParentId = "2222222222222222";
var extractCalls = new AtomicInteger();
var exporter = InMemorySpanExporter.create();
- var extractorPlugin = new ExecutionOtelPlugin(
+ var extractorPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> {
@@ -348,11 +364,11 @@ void contextExtractor_isInvokedEveryInvocation_evenWithAmbientSpan_andBackendCon
"1111111111111111",
TraceFlags.getSampled(),
TraceState.getDefault());
+ DurableExecutionPlugin extractorPlugin;
try (var ignored = Span.wrap(ambient).makeCurrent()) {
- extractorPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
- extractorPlugin.onInvocationEnd(
- new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
+ extractorPlugin = started(extractorPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
}
+ extractorPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
assertEquals(1, extractCalls.get(), "Extractor is invoked even when a valid ambient span is active");
var spans = exporter.getFinishedSpanItems();
@@ -377,18 +393,20 @@ void executionTrace_isStableAcrossReinvocations_withDifferentAmbientTraces() {
TraceState.getDefault());
var startTime = Instant.now();
+ // Two invocations of the same execution, so two instances from the same factory.
+ DurableExecutionPlugin plugin;
try (var ignored = Span.wrap(ambientA).makeCurrent()) {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, startTime));
- plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
+ plugin = started(factory, new InvocationInfo("req-1", ARN, true, startTime));
}
+ plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
var firstInvocationTrace =
spanByName(spanExporter.getFinishedSpanItems(), "Invocation").getTraceId();
spanExporter.reset();
try (var ignored = Span.wrap(ambientB).makeCurrent()) {
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, startTime));
- plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null));
+ plugin = started(factory, new InvocationInfo("req-2", ARN, false, startTime));
}
+ plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null));
var secondInvocationTrace =
spanByName(spanExporter.getFinishedSpanItems(), "Invocation").getTraceId();
@@ -400,7 +418,7 @@ void executionTrace_isStableAcrossReinvocations_withDifferentAmbientTraces() {
@Test
void nonTerminalInvocation_doesNotExportWorkflowSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -416,7 +434,7 @@ void nonTerminalInvocation_doesNotExportWorkflowSpan() {
@Test
void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() {
// Invocation 1: non-terminal → no Workflow span exported
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
assertTrue(
spanExporter.getFinishedSpanItems().stream()
@@ -425,7 +443,7 @@ void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() {
spanExporter.reset();
// Invocation 2: terminal → Workflow span exported with the deterministic ID
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null));
var workflowSpan = spanByName(spanExporter.getFinishedSpanItems(), "Workflow");
@@ -437,7 +455,7 @@ void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() {
@Test
void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(
new InvocationEndInfo("req-1", ARN, true, InvocationStatus.FAILED, new RuntimeException("boom")));
@@ -449,7 +467,7 @@ void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() {
@Test
void retryingInvocation_invocationSpanUnset_workflowNotExported() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req-1", ARN, true, InvocationStatus.RETRYING, new RuntimeException("transient")));
@@ -467,7 +485,7 @@ void retryingInvocation_invocationSpanUnset_workflowNotExported() {
@Test
void operationSpan_carriesAttemptNumberAtEnd() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "flaky", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -496,7 +514,7 @@ void operationSpan_carriesAttemptNumberAtEnd() {
@Test
void continuationOperationSpan_carriesAttemptNumber() {
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now()));
// No matching onOperationStart in this invocation — continuation branch.
plugin.onOperationEnd(new OperationEndInfo(
"op-1",
@@ -526,7 +544,7 @@ void continuationOperationSpan_carriesAttemptNumber() {
void operationSpan_startsAtOperationStartTimestamp() {
var opStart = Instant.parse("2026-02-01T10:00:00Z");
var opEnd = Instant.parse("2026-02-01T10:00:03Z");
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(new OperationInfo("op-1", "step-a", "STEP", "Step", null, opStart, null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
"op-1", "step-a", "STEP", "Step", null, opStart, opEnd, "SUCCEEDED", null, false, null, null));
@@ -541,7 +559,7 @@ void operationSpan_startsAtOperationStartTimestamp() {
@Test
void operationSpan_parentedToWorkflow_linkedToInvocation() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -576,7 +594,7 @@ void operationSpan_parentedToWorkflow_linkedToInvocation() {
@Test
void attemptSpan_childOfOperation_linkedToInvocation() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onUserFunctionStart(
@@ -629,7 +647,7 @@ void attemptSpan_childOfOperation_linkedToInvocation() {
@Test
void attemptSpan_carriesOperationSubtype() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "process-order", "STEP", "Step", null, Instant.now(), false, 1));
plugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -658,7 +676,7 @@ void attemptSpan_carriesOperationSubtype() {
@Test
void childOperation_parentedToParentOperationSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(new OperationInfo(
"op-parent", "my-context", "CONTEXT", "RunInChildContext", null, Instant.now(), null, null, false));
plugin.onOperationStart(new OperationInfo(
@@ -707,7 +725,7 @@ void contextOperation_currentContextCarriesResolvedFlags_withAlwaysOffSampler()
// DurableSampler), so descendants inherit the resolved decision. With always_off the resolved decision is
// unsampled, so the current context inside the context body must be unsampled — not a provisional sampled bit.
var exporter = InMemorySpanExporter.create();
- var offPlugin = new ExecutionOtelPlugin(
+ var offPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(Sampler.alwaysOff())
.addSpanProcessor(SimpleSpanProcessor.create(exporter)),
@@ -717,7 +735,7 @@ void contextOperation_currentContextCarriesResolvedFlags_withAlwaysOffSampler()
.workflowSpanName("Workflow")
.build());
- offPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var offPlugin = started(offPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
offPlugin.onOperationStart(
new OperationInfo("ctx-1", "my-ctx", "CONTEXT", "Context", null, Instant.now(), null, null, false));
offPlugin.onUserFunctionStart(
@@ -762,7 +780,7 @@ void contextOperation_currentContextCarriesResolvedFlags_withAlwaysOffSampler()
@Test
void userFunctionFailure_setsErrorOnAttemptSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "failing", "STEP", "Step", null, Instant.now(), false, 1));
plugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -788,7 +806,7 @@ void userFunctionFailure_setsErrorOnAttemptSpan() {
@Test
void userFunctionSuccess_setsOkOnAttemptSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1));
plugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -814,7 +832,7 @@ void userFunctionSuccess_setsOkOnAttemptSpan() {
@Test
void userFunctionIncomplete_leavesAttemptSpanUnset() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "waiting", "STEP", "Step", null, Instant.now(), false, 1));
plugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -839,7 +857,7 @@ void userFunctionIncomplete_leavesAttemptSpanUnset() {
@Test
void operationSuccess_setsOkOnOperationSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step-ok", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -864,7 +882,7 @@ void operationSuccess_setsOkOnOperationSpan() {
@Test
void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() {
// Cancellation without diagnostic details retains UNSET; it is not blanket-classified as failure.
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-cancel", "step-cancel", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -889,7 +907,7 @@ void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() {
@Test
void operationEnd_withoutStart_timeoutWithoutError_setsError() {
// An explicit timeout remains an error even when the backend supplied no exception details.
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now()));
plugin.onOperationEnd(new OperationEndInfo(
"op-cb-timeout",
"my-callback",
@@ -914,7 +932,7 @@ void operationEnd_withoutStart_timeoutWithoutError_setsError() {
void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() {
// A successful statusless virtual (FLAT CONTEXT) operation fires onOperationEnd with a null operation ->
// null status and null error. This is genuine success and must be stamped OK.
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-ctx", "my-ctx", "CONTEXT", null, null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -938,7 +956,7 @@ void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() {
@Test
void operationNotCompleted_notEndedAtInvocationEnd() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, null, false));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
@@ -958,7 +976,7 @@ void operationNotCompleted_notEndedAtInvocationEnd() {
void openAttemptSpan_isEndedAtInvocationEnd_notAbandoned() {
// A user function that starts but never ends (e.g. the execution suspends mid-attempt) must not leave a
// recording span abandoned: onInvocationEnd force-ends it so it is exported.
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "stuck", "STEP", "Step", null, Instant.now(), false, 1));
// No onUserFunctionEnd — the invocation suspends.
@@ -974,7 +992,7 @@ void openAttemptSpan_isEndedAtInvocationEnd_notAbandoned() {
@Test
void everyRecordingSpanIsEnded_onNonTerminalInvocation() {
// No recording span may be left un-ended when the execution returns a non-terminal status.
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onUserFunctionStart(
@@ -1009,7 +1027,7 @@ void noRecordingSpanIsLeftOpen_onRetrying_trackedByLifecycleProcessor() {
*/
private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) {
var lifecycle = new LifecycleTrackingSpanProcessor();
- var trackingPlugin = new ExecutionOtelPlugin(
+ var trackingPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(lifecycle),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -1017,7 +1035,7 @@ private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) {
.workflowSpanName("Workflow")
.build());
- trackingPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var trackingPlugin = started(trackingPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
trackingPlugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
trackingPlugin.onUserFunctionStart(
@@ -1040,7 +1058,7 @@ private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) {
@Test
void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() {
// Invocation 1: operation opens but does not complete.
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, null, false));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
@@ -1051,7 +1069,7 @@ void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() {
spanExporter.reset();
// Invocation 2: the operation completes → materialized once via onOperationEnd, linked to this invocation.
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now()));
plugin.onOperationEnd(new OperationEndInfo(
"op-1",
"my-wait",
@@ -1083,7 +1101,7 @@ void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() {
@Test
void executionTraceIsStableAcrossInvocations_andSharedByInvocationSpans() {
var executionStartTime = Instant.parse("2026-08-15T00:00:00Z");
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, executionStartTime));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, executionStartTime));
plugin.onOperationStart(
new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -1105,7 +1123,7 @@ void executionTraceIsStableAcrossInvocations_andSharedByInvocationSpans() {
var firstInvocationTraceId = spanByName(firstSpans, "Invocation").getTraceId();
spanExporter.reset();
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, executionStartTime));
+ plugin = started(factory, new InvocationInfo("req-2", ARN, false, executionStartTime));
plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null));
var secondSpans = spanExporter.getFinishedSpanItems();
var workflowSpan = spanByName(secondSpans, "Workflow");
@@ -1119,7 +1137,7 @@ void executionTraceIsStableAcrossInvocations_andSharedByInvocationSpans() {
@Test
void operationEnd_withoutStart_createsContinuationSpanWithLink() {
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now()));
// Operation completed between invocations — no matching onOperationStart in this invocation.
plugin.onOperationEnd(new OperationEndInfo(
"op-wait-1",
@@ -1148,7 +1166,7 @@ void operationEnd_withoutStart_createsContinuationSpanWithLink() {
@Test
void deterministicWorkflowSpanId_stableAcrossInvocations() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var firstWorkflowSpanId =
spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getSpanId();
@@ -1156,14 +1174,14 @@ void deterministicWorkflowSpanId_stableAcrossInvocations() {
// A second (independent) plugin for the same execution ARN must derive the same Workflow span ID.
var exporter2 = InMemorySpanExporter.create();
- var plugin2 = new ExecutionOtelPlugin(
+ var plugin2Factory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter2)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(false)
.workflowSpanName("Workflow")
.build());
- plugin2.onInvocationStart(new InvocationInfo("req-9", ARN, true, Instant.now()));
+ var plugin2 = started(plugin2Factory, new InvocationInfo("req-9", ARN, true, Instant.now()));
plugin2.onInvocationEnd(new InvocationEndInfo("req-9", ARN, true, InvocationStatus.SUCCEEDED, null));
var secondWorkflowSpanId =
spanByName(exporter2.getFinishedSpanItems(), "Workflow").getSpanId();
@@ -1179,7 +1197,7 @@ void deterministicWorkflowSpanId_stableAcrossInvocations() {
@Test
void sampling_disabled_producesNoSpans() {
var exporter = InMemorySpanExporter.create();
- var sampledPlugin = new ExecutionOtelPlugin(
+ var sampledPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOff())
.addSpanProcessor(SimpleSpanProcessor.create(exporter)),
@@ -1188,7 +1206,7 @@ void sampling_disabled_producesNoSpans() {
.enableMdc(false)
.workflowSpanName("Workflow")
.build());
- sampledPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var sampledPlugin = started(sampledPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
sampledPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
assertTrue(exporter.getFinishedSpanItems().isEmpty(), "No spans should be exported with 0% sampling");
}
@@ -1203,7 +1221,7 @@ void xrayExtraction_undecidedSampling_remoteParentIsAncestor_flagUnset() {
// Two-arg context → UNDECIDED sampling: the valid remote parent is still the authoritative ancestor. A
// non-parent-based alwaysOn sampler exports the spans so the topology is observable (a plain parent-based
// sampler would drop them, since the remote parent's sampled flag is left unset).
- var xrayPlugin = new ExecutionOtelPlugin(
+ var xrayPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(Sampler.alwaysOn())
.addSpanProcessor(SimpleSpanProcessor.create(exporter)),
@@ -1212,7 +1230,7 @@ void xrayExtraction_undecidedSampling_remoteParentIsAncestor_flagUnset() {
.enableMdc(false)
.workflowSpanName("Workflow")
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
xrayPlugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
xrayPlugin.onOperationEnd(new OperationEndInfo(
@@ -1252,7 +1270,7 @@ void xrayExtraction_undecidedSampling_parentBasedSampler_defersToSamplerAndExpor
var xrayTraceId = "aabbccddee112233445566778899aabb";
var parentSpanId = "53995c3f42cd8ad8";
var exporter = InMemorySpanExporter.create();
- var xrayPlugin = new ExecutionOtelPlugin(
+ var xrayPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(Sampler.parentBased(Sampler.alwaysOn()))
.addSpanProcessor(SimpleSpanProcessor.create(exporter)),
@@ -1261,7 +1279,7 @@ void xrayExtraction_undecidedSampling_parentBasedSampler_defersToSamplerAndExpor
.enableMdc(false)
.workflowSpanName("Workflow")
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = exporter.getFinishedSpanItems();
@@ -1282,7 +1300,7 @@ void xrayExtraction_undecidedSampling_parentBasedNeverSampler_dropsExecutionTrac
var xrayTraceId = "aabbccddee112233445566778899aabb";
var parentSpanId = "53995c3f42cd8ad8";
var exporter = InMemorySpanExporter.create();
- var xrayPlugin = new ExecutionOtelPlugin(
+ var xrayPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(Sampler.parentBased(Sampler.alwaysOff()))
.addSpanProcessor(SimpleSpanProcessor.create(exporter)),
@@ -1291,7 +1309,7 @@ void xrayExtraction_undecidedSampling_parentBasedNeverSampler_dropsExecutionTrac
.enableMdc(false)
.workflowSpanName("Workflow")
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
assertTrue(
@@ -1305,7 +1323,7 @@ void xrayExtraction_explicitSampled_remoteParentIsExecutionAncestor() {
var parentSpanId = "53995c3f42cd8ad8";
var exporter = InMemorySpanExporter.create();
// Explicit Sampled=1 with a complete parent → the remote context is the execution ancestor directly.
- var xrayPlugin = new ExecutionOtelPlugin(
+ var xrayPluginFactory = ExecutionOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() ->
@@ -1314,7 +1332,7 @@ void xrayExtraction_explicitSampled_remoteParentIsExecutionAncestor() {
.workflowSpanName("Workflow")
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = exporter.getFinishedSpanItems();
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java
deleted file mode 100644
index 835267fbf2..0000000000
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java
+++ /dev/null
@@ -1,277 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import io.opentelemetry.context.Context;
-import io.opentelemetry.sdk.common.CompletableResultCode;
-import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
-import io.opentelemetry.sdk.trace.ReadWriteSpan;
-import io.opentelemetry.sdk.trace.ReadableSpan;
-import io.opentelemetry.sdk.trace.SdkTracerProvider;
-import io.opentelemetry.sdk.trace.SpanProcessor;
-import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
-import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.List;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-import java.util.stream.Stream;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.MethodSource;
-import software.amazon.awssdk.services.lambda.model.*;
-import software.amazon.lambda.durable.DurableConfig;
-import software.amazon.lambda.durable.TypeToken;
-import software.amazon.lambda.durable.execution.DurableExecutor;
-import software.amazon.lambda.durable.model.DurableExecutionInput;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.*;
-import software.amazon.lambda.durable.serde.JacksonSerDes;
-import software.amazon.lambda.durable.serde.SerDes;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
-import software.amazon.lambda.durable.util.ExceptionHelper;
-
-class ExecutionRootDeliveryFailureTest {
- @ParameterizedTest
- @CsvSource({
- "false,serialize",
- "true,serialize",
- "false,checkpoint",
- "true,checkpoint",
- "false,root-end",
- "true,root-end",
- "false,success",
- "true,success"
- })
- void outputPreparationAndOrdinaryRootEndFailuresStillReleaseAndFlush(boolean executionView, String mode)
- throws Exception {
- var originalContext = Context.current();
- var failure = new IllegalStateException("controlled " + mode);
- var failOnce = new AtomicBoolean(true);
- var bodyCalls = new AtomicInteger();
- var rootEnds = new AtomicInteger();
- var flushes = new AtomicInteger();
- var roots = new CopyOnWriteArrayList();
- var ends = new CopyOnWriteArrayList();
- var observer = new SpanProcessor() {
- public void onStart(Context parent, ReadWriteSpan span) {
- if (span.getName().equals("DurableExecutionRoot")) roots.add(span);
- }
-
- public boolean isStartRequired() {
- return true;
- }
-
- public void onEnd(ReadableSpan span) {
- if (span.getName().equals("DurableExecutionRoot")) {
- rootEnds.incrementAndGet();
- if (mode.equals("root-end") && failOnce.compareAndSet(true, false)) throw failure;
- }
- }
-
- public boolean isEndRequired() {
- return true;
- }
-
- public CompletableResultCode forceFlush() {
- flushes.incrementAndGet();
- return CompletableResultCode.ofSuccess();
- }
- };
- var client = new LocalMemoryExecutionClient() {
- @Override
- public CheckpointDurableExecutionResponse checkpoint(
- String arn, String token, List updates) {
- if (mode.equals("checkpoint")
- && updates.stream().anyMatch(u -> u.type() == OperationType.EXECUTION)
- && failOnce.compareAndSet(true, false)) throw failure;
- return super.checkpoint(arn, token, updates);
- }
- };
- var serDes = new SerDes() {
- private final JacksonSerDes delegate = new JacksonSerDes();
-
- public String serialize(Object value) {
- if (mode.equals("serialize") && "result".equals(value) && failOnce.compareAndSet(true, false))
- throw failure;
- return delegate.serialize(value);
- }
-
- public T deserialize(String data, TypeToken type) {
- return delegate.deserialize(data, type);
- }
- };
- var caller = Executors.newSingleThreadExecutor();
- var workers = Executors.newCachedThreadPool();
- try (var exporter = InMemorySpanExporter.create();
- var batch = BatchSpanProcessor.builder(exporter)
- .setScheduleDelay(Duration.ofDays(1))
- .build()) {
- var builder = SdkTracerProvider.builder().addSpanProcessor(observer).addSpanProcessor(batch);
- var otelConfig = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(() -> null)
- .build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(builder, otelConfig)
- : new InvocationOtelPlugin(builder, otelConfig);
- var config = DurableConfig.builder()
- .withDurableExecutionClient(client)
- .withExecutorService(workers)
- .withCheckpointDelay(Duration.ZERO)
- .withSerDes(serDes)
- .withPlugins(otel, new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add(info);
- }
- })
- .build();
- try {
- var first = caller.submit(() -> DurableExecutor.execute(
- input(),
- null,
- TypeToken.get(String.class),
- (value, context) -> bodyCalls.incrementAndGet() == 1 && mode.equals("checkpoint")
- ? "x".repeat(6 * 1024 * 1024)
- : "result",
- config));
- if (mode.equals("serialize") || mode.equals("checkpoint")) {
- var thrown = assertThrows(ExecutionException.class, () -> first.get(5, TimeUnit.SECONDS));
- assertSame(failure, thrown.getCause(), "Output preparation failure remains caller-visible");
- } else
- assertEquals(
- ExecutionStatus.SUCCEEDED,
- first.get(5, TimeUnit.SECONDS).status());
- var exported = exporter.getFinishedSpanItems();
- System.out.println("ROOT_DELIVERY view=" + executionView + " mode=" + mode + " ends=" + ends.size()
- + " rootEnds=" + rootEnds.get() + " rootRecording="
- + roots.get(0).isRecording()
- + " flushes=" + flushes.get() + " exported="
- + exported.stream().map(s -> s.getName()).toList());
- assertEquals(1, ends.size(), "Output preparation failure still has exactly one End");
- assertEquals(
- mode.equals("serialize") || mode.equals("checkpoint")
- ? InvocationStatus.RETRYING
- : InvocationStatus.SUCCEEDED,
- ends.get(0).invocationStatus());
- assertEquals(1, rootEnds.get());
- assertFalse(roots.get(0).isRecording());
- assertEquals(1, flushes.get(), "Root onEnd failure does not skip flushing completed spans");
- assertTrue(
- exported.stream().anyMatch(s -> s.getName().equals("Invocation")),
- "The BatchSpanProcessor must be flushed before invocation return");
- var second = caller.submit(() -> DurableExecutor.execute(
- input(), null, TypeToken.get(String.class), (value, context) -> "result", config));
- assertEquals(
- ExecutionStatus.SUCCEEDED,
- second.get(5, TimeUnit.SECONDS).status());
- assertEquals(2, ends.size());
- assertEquals(2, rootEnds.get());
- assertEquals(2, flushes.get());
- assertEquals(roots.get(0).getSpanContext(), roots.get(1).getSpanContext());
- assertTrue(roots.stream().noneMatch(ReadWriteSpan::isRecording));
- assertSame(originalContext, Context.current());
- } finally {
- caller.shutdownNow();
- workers.shutdownNow();
- assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
- assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
- }
-
- private static DurableExecutionInput input() {
- var operation = Operation.builder()
- .id("execution")
- .type(OperationType.EXECUTION)
- .status(OperationStatus.STARTED)
- .startTimestamp(Instant.ofEpochSecond(10))
- .executionDetails(
- ExecutionDetails.builder().inputPayload("\"input\"").build())
- .build();
- return new DurableExecutionInput(
- "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/execution",
- "token",
- CheckpointUpdatedExecutionState.builder().operations(operation).build());
- }
-
- static Stream preparationEndFailures() {
- return Stream.of(false, true)
- .flatMap(view -> Stream.of("ordinary", "vm", "death", "wrapped-vm")
- .flatMap(preparation -> Stream.of("assertion", "vm", "death")
- .map(end -> Arguments.of(view, preparation, end))));
- }
-
- @SuppressWarnings("removal")
- @ParameterizedTest
- @MethodSource("preparationEndFailures")
- void preparationAndEndFailuresRetainCallerIdentityAndCleanupDiagnostics(
- boolean executionView, String preparationKind, String endKind) {
- Throwable original = switch (preparationKind) {
- case "vm", "wrapped-vm" -> new InternalError("preparation fatal");
- case "death" -> new ThreadDeath();
- default -> new IllegalStateException("preparation ordinary");
- };
- Throwable preparation = preparationKind.equals("wrapped-vm") ? new CompletionException(original) : original;
- Error cleanup = switch (endKind) {
- case "vm" -> new InternalError("End fatal");
- case "death" -> new ThreadDeath();
- default -> new AssertionError("End error");
- };
- var ends = new AtomicInteger();
- var seen = new AtomicReference();
- var serDes = new SerDes() {
- private final JacksonSerDes delegate = new JacksonSerDes();
-
- public String serialize(Object value) {
- if ("result".equals(value)) ExceptionHelper.sneakyThrow(preparation);
- return delegate.serialize(value);
- }
-
- public T deserialize(String value, TypeToken type) {
- return delegate.deserialize(value, type);
- }
- };
- try (var exporter = InMemorySpanExporter.create()) {
- var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
- var otelConfig = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(() -> null)
- .build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(builder, otelConfig)
- : new InvocationOtelPlugin(builder, otelConfig);
- var runner = LocalDurableTestRunner.create(
- String.class,
- (value, context) -> "result",
- DurableConfig.builder()
- .withSerDes(serDes)
- .withPlugins(otel, new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.incrementAndGet();
- seen.set(info);
- throw cleanup;
- }
- })
- .build());
- Throwable expected = original instanceof VirtualMachineError || original instanceof ThreadDeath
- ? original
- : cleanup instanceof VirtualMachineError || cleanup instanceof ThreadDeath ? cleanup : original;
- assertSame(expected, assertThrows(Throwable.class, () -> runner.run("input")));
- assertEquals(List.of(expected == original ? cleanup : original), List.of(expected.getSuppressed()));
- assertEquals(1, ends.get());
- assertEquals(InvocationStatus.RETRYING, seen.get().invocationStatus());
- assertSame(original, seen.get().executionError());
- var roots = exporter.getFinishedSpanItems().stream()
- .filter(span -> span.getName().equals("DurableExecutionRoot"))
- .toList();
- assertEquals(1, roots.size());
- assertEquals(roots.get(0).getStartEpochNanos(), roots.get(0).getEndEpochNanos());
- }
- }
-}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java
deleted file mode 100644
index 3cb14b92d7..0000000000
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java
+++ /dev/null
@@ -1,187 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import io.opentelemetry.context.Context;
-import io.opentelemetry.sdk.common.CompletableResultCode;
-import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
-import io.opentelemetry.sdk.trace.ReadWriteSpan;
-import io.opentelemetry.sdk.trace.ReadableSpan;
-import io.opentelemetry.sdk.trace.SdkTracerProvider;
-import io.opentelemetry.sdk.trace.SpanProcessor;
-import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
-import software.amazon.lambda.durable.plugin.InvocationInfo;
-import software.amazon.lambda.durable.plugin.InvocationStatus;
-import software.amazon.lambda.durable.util.ExceptionHelper;
-
-class ExecutionRootLifecycleTest {
- private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/id";
- private static final Instant START = Instant.ofEpochSecond(10);
-
- @ParameterizedTest
- @CsvSource({
- "false,start-failure", "true,start-failure",
- "false,end-failure", "true,end-failure",
- "false,fatal-root-cleanup", "true,fatal-root-cleanup",
- "false,reentrant-end", "true,reentrant-end"
- })
- void rootIsClosedOnceAcrossPartialStartEndFailureReentryAndReuse(boolean executionView, String mode) {
- var original = Context.current();
- var plugin = new AtomicReference();
- var failOnce = new AtomicBoolean(true);
- var rootFatalOnce = new AtomicBoolean(true);
- var inRootCallback = new AtomicBoolean();
- var rootsEnded = new AtomicInteger();
- var flushes = new AtomicInteger();
- var starts = new ArrayList();
- var roots = new ArrayList();
- var primary = new AssertionError("invocation processor failure");
- var cleanupFatal = new InternalError("root cleanup fatal");
- var observer = new SpanProcessor() {
- public void onStart(Context parent, ReadWriteSpan span) {
- starts.add(span.getName());
- if (span.getName().equals("DurableExecutionRoot")) roots.add(span);
- if (span.getName().equals("Invocation")
- && mode.equals("start-failure")
- && failOnce.compareAndSet(true, false)) throw primary;
- }
-
- public boolean isStartRequired() {
- return true;
- }
-
- public void onEnd(ReadableSpan span) {
- if (span.getName().equals("Invocation")
- && mode.contains("failure")
- && !mode.equals("start-failure")
- && failOnce.compareAndSet(true, false)) throw primary;
- if (!span.getName().equals("DurableExecutionRoot")) return;
- rootsEnded.incrementAndGet();
- inRootCallback.set(true);
- try {
- if (mode.equals("reentrant-end")) plugin.get().onInvocationEnd(end(false));
- if (mode.equals("fatal-root-cleanup") && rootFatalOnce.compareAndSet(true, false))
- throw cleanupFatal;
- } finally {
- inRootCallback.set(false);
- }
- }
-
- public boolean isEndRequired() {
- return true;
- }
-
- public CompletableResultCode forceFlush() {
- assertFalse(inRootCallback.get(), "Reentrant End must not flush recursively from root end callbacks");
- flushes.incrementAndGet();
- return CompletableResultCode.ofSuccess();
- }
- };
- try (var exporter = InMemorySpanExporter.create()) {
- var builder = SdkTracerProvider.builder()
- .addSpanProcessor(SimpleSpanProcessor.create(exporter))
- .addSpanProcessor(observer);
- var config = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(() -> null)
- .build();
- plugin.set(
- executionView
- ? new ExecutionOtelPlugin(builder, config)
- : new InvocationOtelPlugin(builder, config));
- var first = new InvocationInfo("first", ARN, true, START);
- if (mode.equals("start-failure"))
- assertSame(
- primary,
- assertThrows(AssertionError.class, () -> plugin.get().onInvocationStart(first)));
- else if (mode.equals("fatal-root-cleanup"))
- assertSame(
- cleanupFatal,
- assertThrows(InternalError.class, () -> plugin.get().onInvocationStart(first)));
- else plugin.get().onInvocationStart(first);
- assertEquals("DurableExecutionRoot", starts.get(0));
- assertFalse(roots.get(0).isRecording(), "The anchor ended before the Start hook returned or failed");
- if (mode.equals("end-failure"))
- assertSame(
- primary,
- assertThrows(AssertionError.class, () -> plugin.get().onInvocationEnd(end(true))));
- else plugin.get().onInvocationEnd(end(true));
- assertEquals(1, rootsEnded.get());
- assertFalse(roots.get(0).isRecording());
- assertEquals(
- 1,
- flushes.get(),
- "End flushes completed spans, including cleanup after a partially completed Start");
- var beforeDuplicateEnd = flushes.get();
- plugin.get().onInvocationEnd(end(true));
- assertEquals(beforeDuplicateEnd, flushes.get());
- assertEquals(1, rootsEnded.get());
-
- plugin.get().onInvocationStart(new InvocationInfo("second", ARN, false, START));
- plugin.get().onInvocationEnd(end(false));
- assertEquals(2, rootsEnded.get());
- assertEquals(roots.get(0).getSpanContext(), roots.get(1).getSpanContext());
- var exportedRoots = exporter.getFinishedSpanItems().stream()
- .filter(span -> span.getName().equals("DurableExecutionRoot"))
- .toList();
- assertEquals(2, exportedRoots.size());
- for (var root : exportedRoots) {
- assertEquals(10_000_000_000L, root.getStartEpochNanos());
- assertEquals(root.getStartEpochNanos(), root.getEndEpochNanos());
- }
- assertSame(original, Context.current());
- }
- }
-
- private static InvocationEndInfo end(boolean first) {
- return new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null);
- }
-
- @SuppressWarnings("removal")
- @ParameterizedTest
- @CsvSource({
- "runtime,ordinary",
- "runtime,fatal",
- "linkage,ordinary",
- "linkage,fatal",
- "assertion,ordinary",
- "vm,ordinary",
- "death,ordinary"
- })
- void flushPreservesTheExistingEndFailurePriority(String rootKind, String flushKind) {
- Throwable rootFailure = switch (rootKind) {
- case "linkage" -> new LinkageError("root linkage");
- case "assertion" -> new AssertionError("root assertion");
- case "vm" -> new InternalError("root fatal");
- case "death" -> new ThreadDeath();
- default -> new IllegalStateException("root ordinary");
- };
- Throwable flushFailure = flushKind.equals("fatal")
- ? new InternalError("flush fatal")
- : new IllegalArgumentException("flush ordinary");
- var flushes = new AtomicInteger();
- var observed = assertThrows(
- Throwable.class,
- () -> OtelPluginSupport.finishWithRootCleanup(() -> ExceptionHelper.sneakyThrow(rootFailure), () -> {
- flushes.incrementAndGet();
- ExceptionHelper.sneakyThrow(flushFailure);
- }));
- var isolated = rootKind.equals("runtime") || rootKind.equals("linkage");
- assertEquals(1, flushes.get());
- var expected = isolated && flushKind.equals("fatal") ? flushFailure : rootFailure;
- assertSame(expected, observed);
- assertEquals(List.of(expected == rootFailure ? flushFailure : rootFailure), List.of(observed.getSuppressed()));
- }
-}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java
index 6a556d0c11..959d00da30 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java
@@ -31,7 +31,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -45,7 +45,8 @@ class ExecutionRootParityTest {
@ValueSource(booleans = {false, true})
void sampledAnchorEndsBeforeTheStartHookReturns(boolean executionView) {
try (var exporter = InMemorySpanExporter.create()) {
- var plugin = plugin(executionView, exporter, Sampler.alwaysOn());
+ var factory = plugin(executionView, exporter, Sampler.alwaysOn());
+ var plugin = factory.createPlugin(start(true));
plugin.onInvocationStart(start(true));
var roots = roots(exporter);
assertEquals(1, roots.size(), "The anchor is available to processors before handler execution");
@@ -77,8 +78,9 @@ public TraceState getUpdatedTraceState(TraceState parent) {
};
});
try (var exporter = InMemorySpanExporter.create()) {
- var plugin = plugin(executionView, exporter, sampler);
+ var factory = plugin(executionView, exporter, sampler);
for (var first : new boolean[] {true, false}) {
+ var plugin = factory.createPlugin(start(first));
plugin.onInvocationStart(start(first));
plugin.onInvocationEnd(end(first));
}
@@ -113,7 +115,8 @@ public TraceState getUpdatedTraceState(TraceState parent) {
@ValueSource(booleans = {false, true})
void recordOnlyStillRecordsChildrenWithoutAnAnchor(boolean executionView) {
try (var exporter = InMemorySpanExporter.create()) {
- var plugin = plugin(executionView, exporter, sampler(name -> SamplingResult.recordOnly()));
+ var factory = plugin(executionView, exporter, sampler(name -> SamplingResult.recordOnly()));
+ var plugin = factory.createPlugin(start(true));
plugin.onInvocationStart(start(true));
plugin.onInvocationEnd(end(false));
assertTrue(roots(exporter).isEmpty());
@@ -143,8 +146,9 @@ void plainReplacementRetainsIndependentPerSpanSampling(boolean executionView, bo
provider,
OtelPluginSupport.tryResolveGlobalProvider("test", "test").sdkTracerProvider());
assertFalse(provider.getSampler() instanceof DurableSampler);
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(config()) : new InvocationOtelPlugin(config());
+ DurableExecutionPluginFactory factory =
+ executionView ? ExecutionOtelPlugin.factory(config()) : InvocationOtelPlugin.factory(config());
+ var plugin = factory.createPlugin(start(true));
plugin.onInvocationStart(start(true));
plugin.onInvocationEnd(end(false));
assertEquals(sampleAnchor ? 1 : 0, roots(exporter).size());
@@ -179,8 +183,9 @@ void plainFallbackQuotaIsSpentOnlyOnActualSpanCreation(boolean executionView) {
provider,
OtelPluginSupport.tryResolveGlobalProvider("test", "test").sdkTracerProvider());
assertFalse(provider.getSampler() instanceof DurableSampler);
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(config()) : new InvocationOtelPlugin(config());
+ DurableExecutionPluginFactory factory =
+ executionView ? ExecutionOtelPlugin.factory(config()) : InvocationOtelPlugin.factory(config());
+ var plugin = factory.createPlugin(start(true));
plugin.onInvocationStart(start(true));
plugin.onInvocationEnd(end(false));
assertEquals(List.of("DurableExecutionRoot", "Invocation", "Workflow"), calls);
@@ -217,8 +222,9 @@ void completeRemoteParentRetainsItsLegacyAncestorPolicy(boolean executionView) {
.enableMdc(false)
.contextExtractor(() -> new ExtractedContext(traceId, parentId))
.build();
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(config) : new InvocationOtelPlugin(config);
+ DurableExecutionPluginFactory factory =
+ executionView ? ExecutionOtelPlugin.factory(config) : InvocationOtelPlugin.factory(config);
+ var plugin = factory.createPlugin(start(true));
plugin.onInvocationStart(start(true));
plugin.onInvocationEnd(end(false));
assertEquals(List.of("Workflow", "Invocation", "Workflow"), calls);
@@ -245,14 +251,16 @@ public ContextPropagators getPropagators() {
};
}
- private static DurableExecutionPlugin plugin(
+ private static DurableExecutionPluginFactory plugin(
boolean executionView, InMemorySpanExporter exporter, Sampler sampler) {
var builder = SdkTracerProvider.builder()
.setSampler(sampler)
.addSpanProcessor(SimpleSpanProcessor.builder(exporter)
.setExportUnsampledSpans(true)
.build());
- return executionView ? new ExecutionOtelPlugin(builder, config()) : new InvocationOtelPlugin(builder, config());
+ return executionView
+ ? ExecutionOtelPlugin.factory(builder, config())
+ : InvocationOtelPlugin.factory(builder, config());
}
private static OtelPluginConfig config() {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java
index 2a00910d54..0403fb7b8a 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java
@@ -37,7 +37,7 @@
import org.junit.jupiter.params.provider.ValueSource;
import software.amazon.lambda.durable.DurableConfig;
import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -252,10 +252,10 @@ void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView, bo
.enableMdc(false)
.contextExtractor(() -> new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED))
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(builder, config)
- : new InvocationOtelPlugin(builder, config);
- invoke(plugin, ARN, true, InvocationStatus.PENDING);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ invoke(factory, ARN, true, InvocationStatus.PENDING);
assertTrue(named(exporter.getFinishedSpanItems(), "DurableExecutionRoot")
.getSpanContext()
.isSampled());
@@ -331,29 +331,34 @@ private static SpanData named(List spans, String name) {
.orElseThrow();
}
- private static void invoke(DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status) {
- invoke(plugin, arn, first, status, START);
+ private static void invoke(
+ DurableExecutionPluginFactory factory, String arn, boolean first, InvocationStatus status) {
+ invoke(factory, arn, first, status, START);
}
private static void invoke(
- DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status, Instant start) {
- plugin.onInvocationStart(new InvocationInfo("request", arn, first, start));
+ DurableExecutionPluginFactory factory, String arn, boolean first, InvocationStatus status, Instant start) {
+ var info = new InvocationInfo("request", arn, first, start);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onInvocationEnd(new InvocationEndInfo("request", arn, first, status, null));
}
- private static DurableExecutionPlugin plugin(
+ private static DurableExecutionPluginFactory plugin(
boolean executionView, InMemorySpanExporter exporter, ExtractedContext extracted, Sampler sampler) {
var builder =
SdkTracerProvider.builder().setSampler(sampler).addSpanProcessor(SimpleSpanProcessor.create(exporter));
return plugin(executionView, builder, extracted);
}
- private static DurableExecutionPlugin plugin(
+ private static DurableExecutionPluginFactory plugin(
boolean executionView, SdkTracerProviderBuilder builder, ExtractedContext extracted) {
var config = OtelPluginConfig.builder()
.contextExtractor(() -> extracted)
.enableMdc(false)
.build();
- return executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
+ return executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
}
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExternalCompletionConformanceHandlerTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExternalCompletionConformanceHandlerTest.java
index f373ef3f0c..0819726f61 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExternalCompletionConformanceHandlerTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExternalCompletionConformanceHandlerTest.java
@@ -29,6 +29,7 @@
import software.amazon.lambda.durable.TypeToken;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
import software.amazon.lambda.durable.plugin.UserFunctionEndInfo;
@@ -62,9 +63,9 @@ void actualHandlerExportsFirstCompletionBeforeTwoLaterReplays(boolean executionV
.contextExtractor(() -> null)
.enableMdc(false)
.build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(provider, options)
- : new InvocationOtelPlugin(provider, options);
+ DurableExecutionPluginFactory otel = executionView
+ ? ExecutionOtelPlugin.factory(provider, options)
+ : InvocationOtelPlugin.factory(provider, options);
var ends = new CopyOnWriteArrayList();
var markerBodies = new AtomicInteger();
var observer = new DurableExecutionPlugin() {
@@ -83,7 +84,7 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
handler::handleRequest,
DurableConfig.builder()
.withExecutorService(workers)
- .withPlugins(otel, observer)
+ .withPlugins(otel, ignored -> observer)
.build());
Map event = Map.of("scenario", "external-callback-completion-replay");
var first = runner.run(event);
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FactoryFatalStartupHandoffTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FactoryFatalStartupHandoffTest.java
new file mode 100644
index 0000000000..b2da3887fc
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FactoryFatalStartupHandoffTest.java
@@ -0,0 +1,184 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import java.time.Duration;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.DurableHandler;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+@Timeout(15)
+class FactoryFatalStartupHandoffTest {
+ static Stream cases() {
+ return Stream.of(false, true)
+ .flatMap(execution -> Stream.of(false, true)
+ .flatMap(death ->
+ Stream.of(false, true).map(wrapped -> Arguments.of(execution, death, wrapped))));
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @MethodSource("cases")
+ void originalFatalReachesCallerAndEscapesWorkerBeforeAnyAdditionalFlush(
+ boolean executionView, boolean death, boolean wrapped) throws Exception {
+ Error original = death ? new ThreadDeath() : new InternalError("original startup VM fatal");
+ var throwing = new CountDownLatch(1);
+ var releaseFlush = new CountDownLatch(1);
+ var escaped = new CountDownLatch(1);
+ var flushEntered = new AtomicInteger();
+ var flushReturned = new AtomicInteger();
+ var workerFailure = new AtomicReference();
+ var bodies = new AtomicInteger();
+ var published = new AtomicInteger();
+ var workers = Executors.newSingleThreadExecutor(task -> {
+ var worker = new Thread(task, "factory-fatal-owner");
+ worker.setDaemon(true);
+ worker.setUncaughtExceptionHandler((thread, failure) -> {
+ workerFailure.set(failure);
+ escaped.countDown();
+ });
+ return worker;
+ });
+ var caller = Executors.newSingleThreadExecutor();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofHours(1))
+ .build()) {
+ var processor = new SpanProcessor() {
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (!span.getName().equals("Invocation")) return;
+ throwing.countDown();
+ if (wrapped) throw new CompletionException(new ExecutionException(original));
+ throw original;
+ }
+
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ public void onEnd(ReadableSpan span) {}
+
+ public boolean isEndRequired() {
+ return false;
+ }
+
+ public CompletableResultCode forceFlush() {
+ flushEntered.incrementAndGet();
+ try {
+ if (!releaseFlush.await(10, TimeUnit.SECONDS))
+ throw new AssertionError("flush gate not released");
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ flushReturned.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ var builder = SdkTracerProvider.builder().addSpanProcessor(batch).addSpanProcessor(processor);
+ var settings = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ var actual = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
+ DurableExecutionPluginFactory observing = info -> {
+ var instance = actual.createPlugin(info);
+ published.incrementAndGet();
+ return instance;
+ };
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withPlugins(observing)
+ .build();
+ var handler = new DurableHandler() {
+ @Override
+ public DurableConfig getConfiguration() {
+ return config;
+ }
+
+ @Override
+ public String handleRequest(String input, DurableContext context) {
+ bodies.incrementAndGet();
+ return "unreachable";
+ }
+ };
+ var runner = LocalDurableTestRunner.create(String.class, handler);
+ var result = caller.submit(() -> runner.run("input"));
+ try {
+ assertTrue(throwing.await(3, TimeUnit.SECONDS), "actual factory never reached Invocation start");
+ Throwable beforeRelease = null;
+ boolean timedOut = false;
+ try {
+ result.get(2, TimeUnit.SECONDS);
+ fail("startup fatal unexpectedly returned a handler result");
+ } catch (ExecutionException failure) {
+ beforeRelease = ExceptionHelper.unwrapAsyncFailure(failure);
+ } catch (TimeoutException blocked) {
+ timedOut = true;
+ }
+ boolean workerBeforeRelease = !timedOut && escaped.await(2, TimeUnit.SECONDS);
+ int enteredBeforeRelease = flushEntered.get();
+ int returnedBeforeRelease = flushReturned.get();
+ assertEquals(1, releaseFlush.getCount(), "observations must precede latch release");
+ releaseFlush.countDown();
+ var afterRelease = assertThrows(ExecutionException.class, () -> result.get(3, TimeUnit.SECONDS));
+ assertSame(original, ExceptionHelper.unwrapAsyncFailure(afterRelease));
+ assertTrue(escaped.await(3, TimeUnit.SECONDS));
+ assertSame(original, workerFailure.get(), "original fatal must escape the actual asynchronous worker");
+ assertEquals(0, bodies.get());
+ assertEquals(0, published.get());
+ System.out.printf(
+ "HANDOFF execution=%s death=%s wrapped=%s callerBefore=%s workerBefore=%s flushEntered=%s flushReturned=%s callerAfterSame=true workerSame=true%n",
+ executionView,
+ death,
+ wrapped,
+ beforeRelease == original,
+ workerBeforeRelease,
+ enteredBeforeRelease,
+ returnedBeforeRelease);
+ assertSame(
+ original,
+ beforeRelease,
+ "caller must observe the original fatal before the flush gate is released");
+ assertTrue(workerBeforeRelease, "worker fatal escape must precede additional cleanup");
+ assertEquals(0, enteredBeforeRelease, "known original fatal must not call additional forceFlush");
+ } finally {
+ releaseFlush.countDown();
+ }
+ } finally {
+ releaseFlush.countDown();
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FactoryStartupFailureTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FactoryStartupFailureTest.java
new file mode 100644
index 0000000000..818d0dba1a
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FactoryStartupFailureTest.java
@@ -0,0 +1,302 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.DurableHandler;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+@Timeout(20)
+class FactoryStartupFailureTest {
+ @ParameterizedTest
+ @CsvSource({"false,true", "true,true", "false,false", "true,false"})
+ void completedAnchorFlushesAtFailedFactoryBoundaryAndProviderIsReusable(boolean executionView, boolean failStartup)
+ throws Exception {
+ var original = new IllegalStateException("Invocation processor start failed");
+ var failOnce = new AtomicBoolean(failStartup);
+ var rootsEnded = new AtomicInteger();
+ var flushes = new AtomicInteger();
+ var shutdowns = new AtomicInteger();
+ var bufferedAtFailure = new AtomicInteger();
+ var observed = new AtomicReference();
+ var published = new CopyOnWriteArrayList();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofHours(1))
+ .build()) {
+ var processor = new SpanProcessor() {
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (span.getName().equals("Invocation") && failOnce.compareAndSet(true, false)) {
+ assertEquals(1, rootsEnded.get(), "the root already completed before Invocation starts");
+ assertTrue(exporter.getFinishedSpanItems().isEmpty(), "healthy BSP still holds the root");
+ bufferedAtFailure.incrementAndGet();
+ throw original;
+ }
+ }
+
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ public void onEnd(ReadableSpan span) {
+ if (span.getName().equals("DurableExecutionRoot")) rootsEnded.incrementAndGet();
+ }
+
+ public boolean isEndRequired() {
+ return true;
+ }
+
+ public CompletableResultCode forceFlush() {
+ flushes.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+
+ public CompletableResultCode shutdown() {
+ shutdowns.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ var builder = SdkTracerProvider.builder().addSpanProcessor(batch).addSpanProcessor(processor);
+ var settings = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ var actual = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
+ DurableExecutionPluginFactory observing = info -> {
+ try {
+ var instance = actual.createPlugin(info);
+ published.add(instance);
+ return instance;
+ } catch (Throwable failure) {
+ observed.set(failure);
+ throw failure;
+ }
+ };
+ var config = DurableConfig.builder().withPlugins(observing).build();
+ assertSuccessfulHandler(config);
+ if (failStartup) {
+ assertSame(original, observed.get());
+ assertEquals(1, bufferedAtFailure.get());
+ assertTrue(published.isEmpty(), "no partially initialized plugin is published");
+ assertEquals(1, exporter.getFinishedSpanItems().size(), "completed root must export before return");
+ assertEquals(
+ "DurableExecutionRoot",
+ exporter.getFinishedSpanItems().get(0).getName());
+ } else {
+ assertNull(observed.get());
+ assertEquals(1, published.size());
+ assertEquals(3, exporter.getFinishedSpanItems().size());
+ }
+ assertEquals(1, flushes.get());
+ assertEquals(0, shutdowns.get());
+ awaitBatchFlushRetired(batch);
+ assertSuccessfulHandler(config);
+ assertEquals(failStartup ? 1 : 2, published.size());
+ assertEquals(
+ failStartup ? 4 : 6,
+ exporter.getFinishedSpanItems().size(),
+ "same provider accepts the next invocation");
+ assertEquals(2, rootsEnded.get());
+ assertEquals(2, flushes.get());
+ assertEquals(0, shutdowns.get(), "factory cleanup must never shut down the environment provider");
+ }
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @CsvSource({
+ "false,ordinary,ordinary", "true,ordinary,ordinary",
+ "false,ordinary,assertion", "true,ordinary,assertion",
+ "false,ordinary,vm", "true,ordinary,vm",
+ "false,ordinary,wrapped-vm", "true,ordinary,wrapped-vm",
+ "false,ordinary,death", "true,ordinary,death",
+ "false,vm,ordinary", "true,vm,ordinary",
+ "false,vm,vm", "true,vm,vm",
+ "false,wrapped-vm,vm", "true,wrapped-vm,vm",
+ "false,vm,same", "true,vm,same",
+ "false,ordinary,same", "true,ordinary,same",
+ "false,ordinary,failed-result", "true,ordinary,failed-result"
+ })
+ void constructionFailureIdentityAndFatalCleanupPrecedenceStayLocal(
+ boolean executionView, String primaryKind, String cleanupKind) throws Exception {
+ var firstFatal = new InternalError("startup VM fatal");
+ var diagnosticReads = new AtomicInteger();
+ Throwable primary = switch (primaryKind) {
+ case "vm" -> firstFatal;
+ case "wrapped-vm" ->
+ new CompletionException("wrapped startup fatal", firstFatal) {
+ @Override
+ public synchronized Throwable getCause() {
+ if (diagnosticReads.incrementAndGet() != 1)
+ throw new IllegalStateException("cause already read");
+ return firstFatal;
+ }
+ };
+ default -> new IllegalStateException("ordinary startup failure");
+ };
+ Error cleanupFatal = cleanupKind.equals("death") ? new ThreadDeath() : new InternalError("flush VM fatal");
+ Throwable cleanup = switch (cleanupKind) {
+ case "vm", "death" -> cleanupFatal;
+ case "wrapped-vm" -> new CompletionException(cleanupFatal);
+ case "assertion" -> new AssertionError("ordinary factory cleanup Error");
+ case "same" -> primary;
+ case "failed-result" -> null;
+ default -> new IllegalArgumentException("ordinary flush failure");
+ };
+ var failStart = new AtomicBoolean(true);
+ var failFlush = new AtomicBoolean(true);
+ var flushes = new AtomicInteger();
+ var shutdowns = new AtomicInteger();
+ var observed = new AtomicReference();
+ var suppressedAtFactory = new AtomicReference();
+ var published = new AtomicInteger();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofHours(1))
+ .build()) {
+ var processor = new SpanProcessor() {
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (span.getName().equals("Invocation") && failStart.compareAndSet(true, false)) {
+ if (primary instanceof Error error) throw error;
+ throw (RuntimeException) primary;
+ }
+ }
+
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ public void onEnd(ReadableSpan span) {}
+
+ public boolean isEndRequired() {
+ return false;
+ }
+
+ public CompletableResultCode forceFlush() {
+ flushes.incrementAndGet();
+ if (failFlush.compareAndSet(true, false)) {
+ if (cleanup instanceof Error error) throw error;
+ if (cleanup != null) throw (RuntimeException) cleanup;
+ return CompletableResultCode.ofFailure();
+ }
+ return CompletableResultCode.ofSuccess();
+ }
+
+ public CompletableResultCode shutdown() {
+ shutdowns.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ var builder = SdkTracerProvider.builder().addSpanProcessor(batch).addSpanProcessor(processor);
+ var settings = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ var actual = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
+ DurableExecutionPluginFactory observing = info -> {
+ try {
+ var plugin = actual.createPlugin(info);
+ published.incrementAndGet();
+ return plugin;
+ } catch (Throwable failure) {
+ observed.set(failure);
+ suppressedAtFactory.set(failure.getSuppressed());
+ throw failure;
+ }
+ };
+ var config = DurableConfig.builder().withPlugins(observing).build();
+ boolean fatalCleanup =
+ cleanupKind.equals("vm") || cleanupKind.equals("wrapped-vm") || cleanupKind.equals("death");
+ boolean originalFatal = !primaryKind.equals("ordinary");
+ Throwable expected = originalFatal ? firstFatal : fatalCleanup ? cleanupFatal : primary;
+ if (expected instanceof Error)
+ assertSame(expected, assertThrows(Error.class, () -> assertSuccessfulHandler(config)));
+ else assertSuccessfulHandler(config);
+ assertSame(expected, observed.get());
+ if (primaryKind.equals("wrapped-vm")) assertEquals(1, diagnosticReads.get());
+ assertEquals(0, published.get());
+ assertEquals(originalFatal ? 0 : 1, flushes.get(), "only nonfatal startup failures attempt cleanup flush");
+ if (originalFatal) {
+ assertEquals(0, suppressedAtFactory.get().length, "no cleanup ran before original fatal propagation");
+ // The startup-cleanup poison was never invoked. Keep the later ordinary End a healthy reuse control.
+ failFlush.set(false);
+ } else if (expected == cleanupFatal)
+ assertTrue(Arrays.asList(suppressedAtFactory.get()).contains(primary));
+ else if (cleanup != null && cleanup != primary)
+ assertTrue(Arrays.asList(suppressedAtFactory.get()).contains(cleanup));
+ else
+ assertEquals(
+ 0, suppressedAtFactory.get().length, "no factory self suppression or synthetic flush failure");
+ assertEquals(0, shutdowns.get());
+ awaitBatchFlushRetired(batch);
+ assertSuccessfulHandler(config);
+ assertEquals(1, published.get());
+ assertEquals(originalFatal ? 1 : 2, flushes.get());
+ assertEquals(4, exporter.getFinishedSpanItems().size(), "next End can flush any prior completed spans too");
+ assertEquals(0, shutdowns.get());
+ }
+ }
+
+ private static void awaitBatchFlushRetired(BatchSpanProcessor batch) throws Exception {
+ // BSP 1.67 completes forceFlush before clearing its in-progress slot. A new invocation can otherwise reuse
+ // that completed result without flushing its spans. Keep both export-at-return assertions strict, but wait
+ // for this test-owned processor's prior flush to retire before exercising sequential provider reuse.
+ var workerField = BatchSpanProcessor.class.getDeclaredField("worker");
+ workerField.setAccessible(true);
+ var worker = workerField.get(batch);
+ var pendingField = worker.getClass().getDeclaredField("flushRequested");
+ pendingField.setAccessible(true);
+ var pending = (AtomicReference) pendingField.get(worker);
+ var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (pending.get() != null && System.nanoTime() < deadline) {
+ TimeUnit.MILLISECONDS.sleep(1);
+ }
+ assertNull(pending.get(), "previous BSP flush must retire before the provider reuse control");
+ }
+
+ private static void assertSuccessfulHandler(DurableConfig config) {
+ var handler = new DurableHandler() {
+ @Override
+ public DurableConfig getConfiguration() {
+ return config;
+ }
+
+ @Override
+ public String handleRequest(String input, DurableContext context) {
+ return "done";
+ }
+ };
+ var result = LocalDurableTestRunner.create(String.class, handler).run("input");
+ assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
+ assertEquals("done", result.getResult(String.class));
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java
new file mode 100644
index 0000000000..14858a3e45
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java
@@ -0,0 +1,142 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.HandlerScoped;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+class FatalScopeCleanupBudgetTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ @SuppressWarnings("removal")
+ void laterFatalStillWaitsForEarlierScopeCleanupBeforeEndDispatch(boolean threadDeath) throws Exception {
+ var enteredEarlier = new CountDownLatch(1);
+ var releaseEarlier = new CountDownLatch(1);
+ var endCalled = new CountDownLatch(1);
+ var scopeClosed = new AtomicBoolean();
+ var endedBeforeClose = new AtomicBoolean();
+ var ends = new AtomicInteger();
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("later scope close");
+ var callers = Executors.newSingleThreadExecutor();
+ var workers = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>()) {
+ @Override
+ public void execute(Runnable command) {
+ super.execute(command);
+ // Let both the fatal publication and earlier blocking close happen before caller finalization begins.
+ await(enteredEarlier);
+ }
+ };
+ try {
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withPlugins(
+ info -> new ScopePlugin(
+ () -> {
+ enteredEarlier.countDown();
+ await(releaseEarlier);
+ scopeClosed.set(true);
+ },
+ scopeClosed,
+ endedBeforeClose,
+ ends,
+ endCalled,
+ fatal),
+ info -> new ScopePlugin(
+ () -> {
+ throw fatal;
+ },
+ scopeClosed,
+ endedBeforeClose,
+ ends,
+ endCalled,
+ fatal))
+ .build();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ context.wait("pause", Duration.ofSeconds(1));
+ return input;
+ },
+ config);
+ var result = callers.submit(() -> runner.run("input"));
+ assertTrue(enteredEarlier.await(3, TimeUnit.SECONDS));
+ assertFalse(endCalled.await(100, TimeUnit.MILLISECONDS), "earlier cleanup must receive its handoff budget");
+ releaseEarlier.countDown();
+ var failure = assertThrows(ExecutionException.class, () -> result.get(3, TimeUnit.SECONDS));
+ assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure));
+ assertFalse(endedBeforeClose.get(), "finalization must follow completed in-budget scope cleanup");
+ assertEquals(2, ends.get(), "both plugins finalize once with the original fatal");
+ } finally {
+ releaseEarlier.countDown();
+ callers.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ }
+ }
+
+ @HandlerScoped(ScopeOpener.class)
+ public static final class ScopePlugin implements DurableExecutionPlugin {
+ private final AutoCloseable closer;
+ private final AtomicBoolean closed;
+ private final AtomicBoolean endedBeforeClose;
+ private final AtomicInteger ends;
+ private final CountDownLatch endCalled;
+ private final Error fatal;
+
+ ScopePlugin(
+ AutoCloseable closer,
+ AtomicBoolean closed,
+ AtomicBoolean endedBeforeClose,
+ AtomicInteger ends,
+ CountDownLatch endCalled,
+ Error fatal) {
+ this.closer = closer;
+ this.closed = closed;
+ this.endedBeforeClose = endedBeforeClose;
+ this.ends = ends;
+ this.endCalled = endCalled;
+ this.fatal = fatal;
+ }
+
+ @Override
+ public void onInvocationEnd(InvocationEndInfo info) {
+ if (!closed.get()) endedBeforeClose.set(true);
+ if (info.invocationStatus() != InvocationStatus.RETRYING || info.executionError() != fatal) {
+ endedBeforeClose.set(true);
+ }
+ ends.incrementAndGet();
+ endCalled.countDown();
+ }
+ }
+
+ public static final class ScopeOpener implements Function {
+ @Override
+ public AutoCloseable apply(Object value) {
+ return ((ScopePlugin) value).closer;
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ if (!latch.await(3, TimeUnit.SECONDS)) throw new AssertionError("latch timed out");
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java
index faa242bc02..e69f1a6cef 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java
@@ -9,8 +9,6 @@
import io.opentelemetry.api.trace.TraceFlags;
import io.opentelemetry.api.trace.TraceState;
import io.opentelemetry.context.Context;
-import io.opentelemetry.context.ContextKey;
-import io.opentelemetry.context.Scope;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
@@ -23,17 +21,65 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.slf4j.MDC;
import software.amazon.lambda.durable.DurableConfig;
import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationInfo;
-import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.plugin.PluginRunner;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
class HandlerContextIntegrationTest {
private static final String TRACE_ID = "12345678901234567890123456789012";
+ @ParameterizedTest
+ @CsvSource({"true,success", "false,success", "true,failure", "false,failure", "true,suspension", "false,suspension"
+ })
+ void reusedHandlerWorkerHasNoLeakedMdc(boolean executionView, String outcome) throws Exception {
+ var pluginConfig = OtelPluginConfig.builder()
+ .enableMdc(true)
+ .contextExtractor(
+ () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ DurableExecutionPluginFactory plugin = executionView
+ ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), pluginConfig)
+ : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), pluginConfig);
+ var executor = Executors.newSingleThreadExecutor();
+ var config = DurableConfig.builder()
+ .withExecutorService(executor)
+ .withPlugins(plugin)
+ .build();
+ try {
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, ctx) -> {
+ assertEquals(TRACE_ID, MDC.get(MdcSpanEnricher.MDC_TRACE_ID));
+ if (outcome.equals("failure")) throw new IllegalStateException("body");
+ if (outcome.equals("suspension")) ctx.wait("pause", Duration.ofSeconds(1));
+ return "done";
+ },
+ config);
+ var result = runner.run("input");
+ assertEquals(
+ switch (outcome) {
+ case "failure" -> ExecutionStatus.FAILED;
+ case "suspension" -> ExecutionStatus.PENDING;
+ default -> ExecutionStatus.SUCCEEDED;
+ },
+ result.getStatus());
+ executor.submit(() -> {
+ assertNull(
+ MDC.get(MdcSpanEnricher.MDC_TRACE_ID),
+ "a reused worker must not inherit the prior trace");
+ assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID));
+ assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED));
+ })
+ .get(2, TimeUnit.SECONDS);
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
@ParameterizedTest
@CsvSource({"true,success", "false,success", "true,failure", "false,failure", "true,suspension", "false,suspension"
})
@@ -45,13 +91,13 @@ void reusedHandlerWorkerHasNoLeakedScope(boolean executionView, String outcome)
.contextExtractor(
() -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED))
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(builder, pluginConfig)
- : new InvocationOtelPlugin(builder, pluginConfig);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, pluginConfig)
+ : InvocationOtelPlugin.factory(builder, pluginConfig);
var executor = Executors.newSingleThreadExecutor();
var config = DurableConfig.builder()
.withExecutorService(executor)
- .withPlugins(plugin)
+ .withPlugins(factory)
.build();
try {
var runner = LocalDurableTestRunner.create(
@@ -74,7 +120,7 @@ void reusedHandlerWorkerHasNoLeakedScope(boolean executionView, String outcome)
assertFalse(
executor.submit(() -> Span.current().getSpanContext().isValid())
.get(5, TimeUnit.SECONDS),
- "invocation-end cleanup must restore the handler worker before the response returns");
+ "cleanup must run on the worker, even if invocation finalization ran on another thread");
} finally {
executor.shutdownNow();
}
@@ -90,9 +136,9 @@ void preservesCompatibleAmbientAndRestoresUnrelatedAmbientAfterFailure(boolean e
.contextExtractor(
() -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED))
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(builder, pluginConfig)
- : new InvocationOtelPlugin(builder, pluginConfig);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, pluginConfig)
+ : InvocationOtelPlugin.factory(builder, pluginConfig);
var ambient = SpanContext.create(
sameTrace ? TRACE_ID : "abcdefabcdefabcdefabcdefabcdefab",
"abcdefabcdefabcd",
@@ -100,20 +146,24 @@ void preservesCompatibleAmbientAndRestoresUnrelatedAmbientAfterFailure(boolean e
TraceState.getDefault());
var previous = Context.current();
try (var ignored = Span.wrap(ambient).makeCurrent()) {
- plugin.onInvocationStart(new InvocationInfo(
+ var runner = new PluginRunner(List.of(factory));
+ runner.onInvocationStart(new InvocationInfo(
"req",
"arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id",
true,
Instant.now()));
- try {
- var active = Span.current().getSpanContext();
- assertEquals(TRACE_ID, active.getTraceId());
- if (sameTrace) assertEquals(ambient, active);
- else assertNotEquals(ambient.getSpanId(), active.getSpanId());
- } finally {
- plugin.onInvocationEnd(new InvocationEndInfo(
- "req", "arn", true, InvocationStatus.FAILED, new IllegalStateException("handler failure")));
- }
+ var error = new IllegalStateException("handler failure");
+ assertSame(
+ error,
+ assertThrows(
+ IllegalStateException.class,
+ () -> runner.runHandler(() -> {
+ var active = Span.current().getSpanContext();
+ assertEquals(TRACE_ID, active.getTraceId());
+ if (sameTrace) assertEquals(ambient, active);
+ else assertNotEquals(ambient.getSpanId(), active.getSpanId());
+ throw error;
+ })));
assertEquals(ambient, Span.current().getSpanContext());
}
assertSame(previous, Context.current());
@@ -131,13 +181,13 @@ void rootContextIsValidAndRestoredAcrossNestedWorkAndResume(boolean executionVie
"1234567890123456",
sampled ? ExtractedContext.Sampling.SAMPLED : ExtractedContext.Sampling.NOT_SAMPLED))
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(builder, pluginConfig)
- : new InvocationOtelPlugin(builder, pluginConfig);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, pluginConfig)
+ : InvocationOtelPlugin.factory(builder, pluginConfig);
var executor = Executors.newCachedThreadPool();
var config = DurableConfig.builder()
.withExecutorService(executor)
- .withPlugins(plugin)
+ .withPlugins(factory)
.build();
var userProvider = SdkTracerProvider.builder()
.addSpanProcessor(SimpleSpanProcessor.create(exporter))
@@ -215,76 +265,4 @@ void rootContextIsValidAndRestoredAcrossNestedWorkAndResume(boolean executionVie
userProvider.close();
}
}
-
- @ParameterizedTest
- @CsvSource({"true,true", "true,false", "false,true", "false,false"})
- void customPluginScopesUnwindBeforeOtelScopeOnTheReusedWorker(boolean executionView, boolean suspend)
- throws Exception {
- var key = ContextKey.named("application-context");
- var exporter = InMemorySpanExporter.create();
- var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
- var settings = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(
- () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED))
- .build();
- DurableExecutionPlugin delegate = executionView
- ? new ExecutionOtelPlugin(builder, settings)
- : new InvocationOtelPlugin(builder, settings);
- var ends = new ArrayList();
- var otel = new DurableExecutionPlugin() {
- @Override
- public void onInvocationStart(InvocationInfo info) {
- delegate.onInvocationStart(info);
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add("otel");
- delegate.onInvocationEnd(info);
- }
- };
- var custom = new DurableExecutionPlugin() {
- private Scope scope;
-
- @Override
- public void onInvocationStart(InvocationInfo info) {
- scope = Context.current().with(key, "active").makeCurrent();
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add("custom");
- assertEquals("active", Context.current().get(key));
- scope.close();
- }
- };
- var workers = Executors.newSingleThreadExecutor();
- try {
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, context) -> {
- assertEquals("active", Context.current().get(key));
- assertEquals(TRACE_ID, Span.current().getSpanContext().getTraceId());
- if (suspend) context.wait("pause", Duration.ofSeconds(1));
- return "done";
- },
- DurableConfig.builder()
- .withExecutorService(workers)
- .withPlugins(otel, custom)
- .build());
- for (var invocation = 0; invocation < 2; invocation++) {
- runner.run("input");
- assertEquals(List.of("custom", "otel"), ends);
- ends.clear();
- assertFalse(workers.submit(() -> Span.current().getSpanContext().isValid())
- .get(3, TimeUnit.SECONDS));
- assertNull(workers.submit(() -> Context.current().get(key)).get(3, TimeUnit.SECONDS));
- runner.advanceTime();
- }
- } finally {
- workers.shutdownNow();
- assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
- }
- }
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java
index 1369254cd3..dc28b137d7 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java
@@ -7,7 +7,9 @@
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import java.time.Duration;
import java.util.Arrays;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -24,6 +26,7 @@
import software.amazon.lambda.durable.TypeToken;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.serde.JacksonSerDes;
@@ -44,7 +47,7 @@ void noPluginPathRetainsExistingMdcBehavior(String outcome) throws Exception {
executor.submit(() -> MDC.setContextMap(workerMdc)).get(5, TimeUnit.SECONDS);
MDC.setContextMap(callerMdc);
runWithoutPlugins(executor, outcome);
- // The existing logger clears MDC once entered; failed input never enters that scope.
+ // No-plugin setup does not own MDC before the durable logger scope is established.
var expectedWorker = outcome.equals("inputFailure") ? workerMdc : Map.of();
assertRestored(executor, expectedWorker, callerMdc);
} finally {
@@ -54,12 +57,72 @@ void noPluginPathRetainsExistingMdcBehavior(String outcome) throws Exception {
}
}
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false", "true,true"})
+ void noPluginInputFailurePreservesAmbientMdcBeforeLoggerScope(boolean direct, boolean ambient) throws Exception {
+ var delegate = Executors.newSingleThreadExecutor();
+ var executor = new AbstractExecutorService() {
+ public void execute(Runnable task) {
+ if (direct) task.run();
+ else delegate.execute(task);
+ }
+
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+ };
+ var previous = MDC.getCopyOfContextMap();
+ var workerMap = ambient ? ambientMdc() : Map.of();
+ var callerMap = direct ? workerMap : Map.of("caller", "retained");
+ var bodies = new AtomicInteger();
+ try {
+ executor.submit(() -> MDC.setContextMap(workerMap)).get(3, TimeUnit.SECONDS);
+ MDC.setContextMap(callerMap);
+ var config = DurableConfig.builder()
+ .withExecutorService(executor)
+ .withSerDes(failingInputSerDes())
+ .build();
+ assertTrue(config.getPluginFactories().isEmpty());
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ bodies.incrementAndGet();
+ return "unreachable";
+ },
+ config);
+ assertEquals(ExecutionStatus.FAILED, runner.run("input").getStatus());
+ assertEquals(0, bodies.get());
+ assertRestored(executor, workerMap, callerMap);
+ } finally {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS));
+ if (previous == null) MDC.clear();
+ else MDC.setContextMap(previous);
+ }
+ }
+
private static void runWithoutPlugins(ExecutorService executor, String outcome) {
var config = DurableConfig.builder()
.withExecutorService(executor)
.withSerDes(outcome.equals("inputFailure") ? failingInputSerDes() : new JacksonSerDes())
.build();
- assertTrue(config.getPluginRunner().isEmpty());
+ assertTrue(config.getPluginFactories().isEmpty());
var runner = LocalDurableTestRunner.create(
String.class,
(input, ctx) -> {
@@ -153,9 +216,9 @@ private static DurableConfig config(ExecutorService executor, boolean executionV
.contextExtractor(
() -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED))
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(SdkTracerProvider.builder(), settings)
- : new InvocationOtelPlugin(SdkTracerProvider.builder(), settings);
+ DurableExecutionPluginFactory plugin = executionView
+ ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), settings)
+ : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), settings);
return DurableConfig.builder()
.withExecutorService(executor)
.withPlugins(plugin)
@@ -217,7 +280,7 @@ public void onInvocationEnd(InvocationEndInfo info) {
worker.submit(() -> MDC.setContextMap(ambient)).get(3, TimeUnit.SECONDS);
var config = DurableConfig.builder()
.withExecutorService(worker)
- .withPlugins(plugin)
+ .withPlugins(info -> plugin)
.withSerDes(outcome.equals("inputFailure") ? failingInputSerDes() : new JacksonSerDes())
.build();
var runner = LocalDurableTestRunner.create(
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java
new file mode 100644
index 0000000000..52c71ec62b
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java
@@ -0,0 +1,378 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import java.lang.reflect.Proxy;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.function.Function;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.DurableExecutionOutput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.HandlerScoped;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+class HandlerScopeFinalizationTest {
+ @ParameterizedTest
+ @CsvSource({
+ "false,false,false",
+ "false,false,true",
+ "false,true,false",
+ "false,true,true",
+ "true,false,false",
+ "true,false,true",
+ "true,true,false",
+ "true,true,true"
+ })
+ void ownerCleanupPrecedesRealOtelFinalization(boolean executionView, boolean retry, boolean hasScope)
+ throws Exception {
+ var deadline = new Deadline();
+ var clock = Executors.newSingleThreadScheduledExecutor(r -> daemon(r, "flush-clock"));
+ var flush = new DelayedFlush(deadline, clock);
+ var builder = SdkTracerProvider.builder().addSpanProcessor(flush);
+ var settings = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> new ExtractedContext(
+ "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ var factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
+ DurableExecutionPluginFactory plugin = info -> {
+ var delegate = factory.createPlugin(info);
+ return new ScopedPlugin() {
+ @Override
+ public AutoCloseable openHandlerScope() {
+ if (!hasScope) return null;
+ if (delegate instanceof InvocationOtelPlugin invocation)
+ return new InvocationOtelPlugin.HandlerScopeOpener().apply(invocation);
+ return new ExecutionOtelPlugin.HandlerScopeOpener().apply((ExecutionOtelPlugin) delegate);
+ }
+
+ @Override
+ public void onInvocationStart(InvocationInfo invocation) {
+ delegate.onInvocationStart(invocation);
+ }
+
+ @Override
+ public void onInvocationEnd(InvocationEndInfo invocation) {
+ delegate.onInvocationEnd(invocation);
+ }
+ };
+ };
+ var workers = Executors.newCachedThreadPool(r -> daemon(r, "handler-owner"));
+ var callers = Executors.newSingleThreadExecutor(r -> daemon(r, "invocation-caller"));
+ var enteredFinally = new CountDownLatch(1);
+ var releaseFinally = new CountDownLatch(1);
+ var original = retryError();
+ var cfg = config(workers, plugin);
+ try {
+ var response = callers.submit(() -> DurableExecutor.execute(
+ input(),
+ deadline.context(),
+ TypeToken.get(String.class),
+ (value, ctx) -> {
+ deadline.arm(400);
+ try {
+ if (retry)
+ ctx.step("retry", String.class, step -> {
+ throw original;
+ });
+ else ctx.wait("pause", Duration.ofSeconds(1));
+ return "done";
+ } finally {
+ enteredFinally.countDown();
+ await(releaseFinally);
+ }
+ },
+ cfg));
+ check(enteredFinally.await(2, TimeUnit.SECONDS), "handler finally not reached");
+ assertThrows(TimeoutException.class, () -> response.get(600, TimeUnit.MILLISECONDS));
+ check(flush.calls.get() == 0, "End cannot flush while handler cleanup is still active");
+ releaseFinally.countDown();
+ outcome(response, retry, original);
+ long total = deadline.elapsedMillis();
+ check(flush.calls.get() == 1, "expected one actual bundled OTel forceFlush");
+ check(flush.flushMillis.get() >= 240, "actual bundled OTel join did not await forceFlush result");
+ check(total >= 600, "The caller must await the actual handler owner, even past the old handoff budget");
+ check(releaseFinally.getCount() == 0, "handler cleanup must finish before End");
+
+ } finally {
+ releaseFinally.countDown();
+ workers.shutdown();
+ workers.awaitTermination(2, TimeUnit.SECONDS);
+ callers.shutdownNow();
+ clock.shutdownNow();
+ }
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @CsvSource({
+ "false,false,false",
+ "false,false,true",
+ "false,true,false",
+ "false,true,true",
+ "true,false,false",
+ "true,false,true",
+ "true,true,false",
+ "true,true,true"
+ })
+ void scopeFatal(boolean retry, boolean wrapped, boolean threadDeath) throws Exception {
+ scopeFatal(retry, wrapped, threadDeath ? new ThreadDeath() : new InternalError("scope fatal"), false);
+ }
+
+ @Test
+ void losingBodyFinallyFatalPreservesTheOriginalWinner() throws Exception {
+ scopeFatal(false, false, new InternalError("body finally fatal"), true);
+ }
+
+ @Test
+ void fatalAfterTheOldHandoffBudgetStillPrecedesEnd() throws Exception {
+ scopeFatal(false, false, new InternalError("late scope fatal"), false, true);
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void scopeFatalAfterOldBudgetIsReportedBeforeOwnerEnd(boolean threadDeath) throws Exception {
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("owner cleanup failure");
+ scopeFatal(false, false, fatal, false, true);
+ }
+
+ static void scopeFatal(boolean retry, boolean wrapped, Error fatal, boolean fromBodyFinally) throws Exception {
+ scopeFatal(retry, wrapped, fatal, fromBodyFinally, false);
+ }
+
+ static void scopeFatal(boolean retry, boolean wrapped, Error fatal, boolean fromBodyFinally, boolean afterTimeout)
+ throws Exception {
+ var deadline = new Deadline();
+ var closeEntered = new CountDownLatch(1);
+ var releaseFatal = new CountDownLatch(1);
+ var fatalRaised = new CountDownLatch(1);
+ var fatalBeforeEnd = new AtomicBoolean();
+ var endCalls = new AtomicInteger();
+ var endStatus = new AtomicReference();
+ var endError = new AtomicReference();
+ var plugin = new ScopedPlugin() {
+ public AutoCloseable openHandlerScope() {
+ var owner = Thread.currentThread();
+ return () -> {
+ check(owner == Thread.currentThread(), "wrong scope close thread");
+ if (!fromBodyFinally) raiseFatal(closeEntered, releaseFatal, fatalRaised, wrapped, fatal);
+ };
+ }
+
+ public void onInvocationEnd(InvocationEndInfo info) {
+ fatalBeforeEnd.set(fatalRaised.getCount() == 0);
+ endCalls.incrementAndGet();
+ endStatus.set(info.invocationStatus());
+ endError.set(info.executionError());
+ }
+ };
+ var workers = Executors.newCachedThreadPool(r -> daemon(r, "fatal-owner"));
+ var callers = Executors.newSingleThreadExecutor(r -> daemon(r, "fatal-caller"));
+ var original = retryError();
+ var cfg = config(workers, info -> plugin);
+ try {
+ var response = callers.submit(() -> DurableExecutor.execute(
+ input(),
+ deadline.context(),
+ TypeToken.get(String.class),
+ (value, ctx) -> {
+ deadline.arm(30_000);
+ try {
+ if (retry)
+ ctx.step("retry", String.class, step -> {
+ throw original;
+ });
+ else ctx.wait("pause", Duration.ofSeconds(1));
+ return "done";
+ } finally {
+ if (fromBodyFinally) raiseFatal(closeEntered, releaseFatal, fatalRaised, wrapped, fatal);
+ }
+ },
+ cfg));
+ check(closeEntered.await(2, TimeUnit.SECONDS), "fatal site not reached");
+ assertThrows(TimeoutException.class, () -> response.get(100, TimeUnit.MILLISECONDS));
+ assertEquals(0, endCalls.get(), "End must not precede handler cleanup");
+ if (afterTimeout) {
+ assertThrows(TimeoutException.class, () -> response.get(600, TimeUnit.MILLISECONDS));
+ assertEquals(0, endCalls.get(), "No caller-thread End when the old budget expires");
+ }
+ releaseFatal.countDown();
+ if (fromBodyFinally) {
+ outcome(response, retry, original);
+ assertTrue(fatalBeforeEnd.get());
+ } else {
+ var thrown = assertThrows(ExecutionException.class, () -> response.get(3, TimeUnit.SECONDS));
+ assertSame(fatal, thrown.getCause(), "scope-owned fatal must escape the invocation caller");
+ assertEquals(1, endCalls.get(), "major invocation lifetime finalizes created plugins once");
+ assertEquals(InvocationStatus.RETRYING, endStatus.get());
+ assertSame(fatal, endError.get());
+ }
+
+ } finally {
+ releaseFatal.countDown();
+ workers.shutdown();
+ workers.awaitTermination(2, TimeUnit.SECONDS);
+ callers.shutdownNow();
+ }
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @CsvSource({"false", "true"})
+ void majorBodyFatalRetainsCallerPropagation(boolean threadDeath) {
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("body fatal");
+ var workers = Executors.newCachedThreadPool(r -> daemon(r, "body-fatal"));
+ try {
+ assertSame(
+ fatal,
+ assertThrows(
+ Error.class,
+ () -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, ctx) -> {
+ throw fatal;
+ },
+ config(workers))));
+ } finally {
+ workers.shutdownNow();
+ }
+ }
+
+ static void raiseFatal(
+ CountDownLatch entered, CountDownLatch release, CountDownLatch raised, boolean wrapped, Error fatal) {
+ entered.countDown();
+ await(release);
+ raised.countDown();
+ if (wrapped) throw new CompletionException(new ExecutionException(fatal));
+ throw fatal;
+ }
+
+ static void outcome(Future response, boolean retry, Throwable original) throws Exception {
+ if (retry) {
+ try {
+ response.get(3, TimeUnit.SECONDS);
+ throw new AssertionError("retry returned output");
+ } catch (ExecutionException e) {
+ check(e.getCause() == original, "retry failure changed to " + e.getCause());
+ }
+ } else check(response.get(3, TimeUnit.SECONDS).status() == ExecutionStatus.PENDING, "suspension changed");
+ }
+
+ static DurableConfig config(ExecutorService workers, DurableExecutionPluginFactory... plugins) {
+ return DurableConfig.builder()
+ .withDurableExecutionClient(new LocalMemoryExecutionClient())
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(plugins)
+ .build();
+ }
+
+ static UnrecoverableDurableExecutionException retryError() {
+ return new UnrecoverableDurableExecutionException(
+ ErrorObject.builder().errorMessage("original retry").build(), true);
+ }
+
+ static DurableExecutionInput input() {
+ var operation = Operation.builder()
+ .id("id")
+ .name("test")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.now())
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id",
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operation).build());
+ }
+
+ static Thread daemon(Runnable runnable, String name) {
+ var thread = new Thread(runnable, name);
+ thread.setDaemon(true);
+ return thread;
+ }
+
+ static void await(CountDownLatch latch) {
+ try {
+ check(latch.await(3, TimeUnit.SECONDS), "fixture latch timed out");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("owner interrupted", e);
+ }
+ }
+
+ static void check(boolean condition, String message) {
+ if (!condition) throw new AssertionError(message);
+ }
+
+ static class Deadline {
+ volatile long start;
+ volatile long end;
+ final CountDownLatch budgetRead = new CountDownLatch(1);
+
+ void arm(int millis) {
+ start = System.nanoTime();
+ end = start + TimeUnit.MILLISECONDS.toNanos(millis);
+ }
+
+ long elapsedMillis() {
+ return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+ }
+
+ int remaining() {
+ return end == 0 ? 30_000 : (int) Math.max(0L, TimeUnit.NANOSECONDS.toMillis(end - System.nanoTime()));
+ }
+
+ Context context() {
+ return (Context) Proxy.newProxyInstance(
+ Context.class.getClassLoader(), new Class[] {Context.class}, (proxy, method, args) -> {
+ if (method.getName().equals("getRemainingTimeInMillis")) {
+ budgetRead.countDown();
+ return remaining();
+ }
+ if (method.getName().equals("getAwsRequestId")) return "test-request";
+ if (method.getName().equals("getMemoryLimitInMB")) return 512;
+ return null;
+ });
+ }
+ }
+
+ @HandlerScoped(ScopedPlugin.Opener.class)
+ private abstract static class ScopedPlugin implements DurableExecutionPlugin {
+ public abstract AutoCloseable openHandlerScope();
+
+ public static class Opener implements Function {
+ public AutoCloseable apply(ScopedPlugin plugin) {
+ return plugin.openHandlerScope();
+ }
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java
index da378fd869..a3ba426ae1 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java
@@ -35,7 +35,7 @@ public long nanoTime() {
}
};
try (var exporter = InMemorySpanExporter.create()) {
- var plugin = new InvocationOtelPlugin(
+ var factory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().setClock(clock).addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -43,7 +43,9 @@ public long nanoTime() {
.build());
var start = Instant.ofEpochSecond(10);
var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/clock/execution";
- plugin.onInvocationStart(new InvocationInfo("request", arn, true, start));
+ var info = new InvocationInfo("request", arn, true, start);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onOperationStart(new OperationInfo(
"parent", "parent", "CONTEXT", "WaitForCallback", null, start, null, null, false));
plugin.onOperationStart(
@@ -88,7 +90,7 @@ public long nanoTime() {
@Test
void endedParentRetainsItsCachedContextForAReplayedChildWithoutReexport() {
try (var exporter = InMemorySpanExporter.create()) {
- var plugin = new InvocationOtelPlugin(
+ var factory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -96,7 +98,9 @@ void endedParentRetainsItsCachedContextForAReplayedChildWithoutReexport() {
.build());
var start = Instant.now();
var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fallback/execution";
- plugin.onInvocationStart(new InvocationInfo("request", arn, false, start));
+ var info = new InvocationInfo("request", arn, false, start);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onOperationStart(new OperationInfo(
"parent", "parent", "CONTEXT", "RunInChildContext", null, start, null, null, true));
plugin.onOperationEnd(new OperationEndInfo(
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java
index 863ca67b5b..3aae2db5d9 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java
@@ -85,7 +85,7 @@ public CheckpointDurableExecutionResponse checkpoint(
}
};
var exporter = InMemorySpanExporter.create();
- var otel = new InvocationOtelPlugin(
+ var otel = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -118,7 +118,7 @@ public void onOperationEnd(OperationEndInfo info) {
if (ending.get()) lateHooks.incrementAndGet();
}
};
- // The merged lifecycle unwinds End in reverse order, so OTel must be registered after the gate.
+ // The lifecycle unwinds End in reverse order, so OTel is registered after the gate.
// This fixture pauses a continuation after admission. Draining must stop its owner and join the
// continuation before End. A predicate that already started must finish its checkpoint first.
var config = DurableConfig.builder()
@@ -126,7 +126,7 @@ public void onOperationEnd(OperationEndInfo info) {
.withDurableExecutionClient(client)
.withCheckpointDelay(Duration.ZERO)
.withPollingStrategy(attempt -> Duration.ZERO)
- .withPlugins(gate, otel)
+ .withPlugins(ignoredInfo -> gate, otel)
.build();
var caller = Executors.newSingleThreadExecutor();
try {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java
new file mode 100644
index 0000000000..f156588fa5
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java
@@ -0,0 +1,77 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class InvocationEndCleanupTest {
+ @SuppressWarnings("removal")
+ static Stream fatalCases() {
+ return Stream.of(false, true)
+ .flatMap(executionView -> Stream.of(false, true)
+ .flatMap(wrapped -> Stream.of(new InternalError("first end hook"), new ThreadDeath())
+ .map(fatal -> Arguments.of(executionView, wrapped, fatal))));
+ }
+
+ @ParameterizedTest
+ @MethodSource("fatalCases")
+ void remainingOtelAndObserverFinalizeAfterFirstReverseEndFatal(
+ boolean executionView, boolean wrapped, Error fatal) {
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> new ExtractedContext(
+ "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ var otel = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ var calls = new ArrayList();
+ var snapshots = new ArrayList();
+ DurableExecutionPluginFactory first = info -> new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo end) {
+ calls.add("first");
+ snapshots.add(end);
+ if (wrapped) throw new CompletionException(new ExecutionException(fatal));
+ throw fatal;
+ }
+ };
+ DurableExecutionPluginFactory last = info -> new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo end) {
+ calls.add("last");
+ snapshots.add(end);
+ }
+ };
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> "done",
+ DurableConfig.builder().withPlugins(last, otel, first).build());
+ assertSame(fatal, assertThrows(Error.class, () -> runner.run("input")));
+ assertEquals(List.of("first", "last"), calls);
+ assertSame(snapshots.get(0), snapshots.get(1), "each plugin receives the same single dispatch snapshot");
+ assertEquals(InvocationStatus.SUCCEEDED, snapshots.get(1).invocationStatus());
+ assertTrue(
+ exporter.getFinishedSpanItems().stream()
+ .anyMatch(span -> span.getName().equals("Invocation")),
+ "the later OTel plugin must finish and export its span before the fatal reaches the caller");
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndFailureTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndFailureTest.java
deleted file mode 100644
index d44a73a91a..0000000000
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndFailureTest.java
+++ /dev/null
@@ -1,215 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import io.opentelemetry.api.trace.Span;
-import io.opentelemetry.api.trace.SpanContext;
-import io.opentelemetry.api.trace.TraceFlags;
-import io.opentelemetry.api.trace.TraceState;
-import io.opentelemetry.context.Context;
-import io.opentelemetry.context.Scope;
-import io.opentelemetry.sdk.common.CompletableResultCode;
-import io.opentelemetry.sdk.trace.ReadWriteSpan;
-import io.opentelemetry.sdk.trace.ReadableSpan;
-import io.opentelemetry.sdk.trace.SdkTracerProvider;
-import io.opentelemetry.sdk.trace.SpanProcessor;
-import java.time.Instant;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
-import software.amazon.lambda.durable.plugin.InvocationInfo;
-import software.amazon.lambda.durable.plugin.InvocationStatus;
-import software.amazon.lambda.durable.plugin.PluginRunner;
-
-class InvocationEndFailureTest {
- @ParameterizedTest
- @MethodSource("failures")
- @SuppressWarnings("removal")
- void restoresContextWhenInvocationEndFails(boolean executionView, String phase, String failureKind)
- throws Exception {
- Throwable failure = switch (failureKind) {
- case "exception" -> new IllegalStateException("telemetry failed");
- case "linkage" -> new NoClassDefFoundError("incompatible telemetry dependency");
- case "fatal" -> new InternalError("fatal telemetry failure");
- default -> new ThreadDeath();
- };
- var processor = new FailingProcessor(phase, failure);
- var builder = SdkTracerProvider.builder().addSpanProcessor(processor);
- var config = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(() -> new ExtractedContext(
- "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
- .build();
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
- var healthyEnds = new AtomicInteger();
- var healthy = new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- healthyEnds.incrementAndGet();
- }
- };
- var runner = new PluginRunner(List.of(healthy, plugin));
- var ambient = Span.wrap(SpanContext.create(
- "abcdefabcdefabcdefabcdefabcdefab",
- "abcdefabcdefabcd",
- TraceFlags.getSampled(),
- TraceState.getDefault()));
- try (var ignored = ambient.makeCurrent()) {
- var original = Context.current();
- runner.onInvocationStart(new InvocationInfo("req", "arn", true, Instant.now()));
- assertNotSame(original, Context.current());
- if (phase.equals("scope")) {
- var field = plugin.getClass().getDeclaredField("handlerScope");
- field.setAccessible(true);
- var scope = (Scope) field.get(plugin);
- field.set(plugin, (Scope) () -> {
- scope.close();
- raise(failure);
- });
- }
- var end = new InvocationEndInfo("req", "arn", true, InvocationStatus.SUCCEEDED, null);
- var fatal = failureKind.equals("fatal") || failureKind.equals("thread-death");
- if (fatal) assertSame(failure, assertThrows(Error.class, () -> runner.onInvocationEnd(end)));
- else assertDoesNotThrow(() -> runner.onInvocationEnd(end));
- assertSame(original, Context.current(), "telemetry errors must not leave the handler context attached");
- assertEquals(1, healthyEnds.get(), "remaining invocation-end hooks must always release their resources");
- // Cleanup state is consumed even if the scope's close implementation throws.
- assertDoesNotThrow(() -> plugin.onInvocationEnd(end));
- assertSame(original, Context.current());
- }
- }
-
- @ParameterizedTest
- @MethodSource("combinedFailures")
- void preservesCombinedFinalizationAndScopeFailures(boolean executionView, String phase, String pair)
- throws Exception {
- Throwable primary = switch (pair.split("-")[0]) {
- case "fatal" -> new InternalError("primary finalization");
- case "assert" -> new AssertionError("primary finalization");
- case "linkage" -> new NoClassDefFoundError("primary finalization");
- default -> new IllegalStateException("primary finalization");
- };
- Throwable cleanup = switch (pair.split("-")[1]) {
- case "fatal" -> new InternalError("scope cleanup");
- case "assert" -> new AssertionError("scope cleanup");
- case "same" -> primary;
- default -> new IllegalStateException("scope cleanup");
- };
- var cleanupWins = List.of("runtime-fatal", "assert-fatal", "runtime-assert", "linkage-assert")
- .contains(pair);
- var expected = cleanupWins ? cleanup : primary;
- var secondary = cleanupWins ? primary : cleanup;
- var processor = new FailingProcessor(phase, primary);
- var config = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(() -> new ExtractedContext(
- "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
- .build();
- var builder = SdkTracerProvider.builder().addSpanProcessor(processor);
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
- var healthyEnds = new AtomicInteger();
- var closed = new AtomicInteger();
- var healthy = new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- healthyEnds.incrementAndGet();
- }
- };
- var runner = new PluginRunner(List.of(healthy, plugin));
- var ambient = Span.wrap(SpanContext.create(
- "abcdefabcdefabcdefabcdefabcdefab",
- "abcdefabcdefabcd",
- TraceFlags.getSampled(),
- TraceState.getDefault()));
- try (var ignored = ambient.makeCurrent()) {
- var original = Context.current();
- runner.onInvocationStart(new InvocationInfo("req", "arn", true, Instant.now()));
- var field = plugin.getClass().getDeclaredField("handlerScope");
- field.setAccessible(true);
- var scope = (Scope) field.get(plugin);
- field.set(plugin, (Scope) () -> {
- closed.incrementAndGet();
- scope.close();
- raise(cleanup);
- });
- var end = new InvocationEndInfo("req", "arn", true, InvocationStatus.SUCCEEDED, null);
- if (expected instanceof RuntimeException || expected instanceof LinkageError) {
- assertDoesNotThrow(() -> runner.onInvocationEnd(end));
- } else assertSame(expected, assertThrows(Error.class, () -> runner.onInvocationEnd(end)));
- assertEquals(expected == secondary ? List.of() : List.of(secondary), List.of(expected.getSuppressed()));
- assertSame(original, Context.current());
- assertEquals(1, healthyEnds.get());
- assertEquals(1, closed.get());
- assertDoesNotThrow(() -> plugin.onInvocationEnd(end));
- assertEquals(1, closed.get(), "scope cleanup is one-shot even when both phases throw");
- }
- }
-
- private static Stream combinedFailures() {
- return Stream.of(false, true)
- .flatMap(view -> Stream.of("span", "flush")
- .flatMap(phase -> Stream.of(
- "fatal-runtime",
- "assert-runtime",
- "runtime-fatal",
- "assert-fatal",
- "fatal-fatal",
- "fatal-same",
- "runtime-assert",
- "linkage-assert",
- "runtime-runtime")
- .map(pair -> Arguments.of(view, phase, pair))));
- }
-
- private static Stream failures() {
- return Stream.of(false, true)
- .flatMap(executionView -> Stream.of("span", "flush", "scope")
- .flatMap(phase -> Stream.of("exception", "linkage", "fatal", "thread-death")
- .map(kind -> Arguments.of(executionView, phase, kind))));
- }
-
- private static void raise(Throwable failure) {
- if (failure instanceof RuntimeException exception) throw exception;
- throw (Error) failure;
- }
-
- private record FailingProcessor(String phase, Throwable failure) implements SpanProcessor {
- @Override
- public void onStart(Context parent, ReadWriteSpan span) {}
-
- @Override
- public boolean isStartRequired() {
- return false;
- }
-
- @Override
- public void onEnd(ReadableSpan span) {
- if (phase.equals("span")) raise(failure);
- }
-
- @Override
- public boolean isEndRequired() {
- return true;
- }
-
- @Override
- public CompletableResultCode forceFlush() {
- if (phase.equals("flush")) raise(failure);
- return CompletableResultCode.ofSuccess();
- }
-
- @Override
- public CompletableResultCode shutdown() {
- return CompletableResultCode.ofSuccess();
- }
- }
-}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndFinalizationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndFinalizationTest.java
deleted file mode 100644
index 5da60b8689..0000000000
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndFinalizationTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import io.opentelemetry.api.trace.Span;
-import io.opentelemetry.context.Context;
-import io.opentelemetry.sdk.common.CompletableResultCode;
-import io.opentelemetry.sdk.trace.ReadWriteSpan;
-import io.opentelemetry.sdk.trace.ReadableSpan;
-import io.opentelemetry.sdk.trace.SdkTracerProvider;
-import io.opentelemetry.sdk.trace.SpanProcessor;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import software.amazon.awssdk.services.lambda.model.*;
-import software.amazon.lambda.durable.DurableConfig;
-import software.amazon.lambda.durable.TypeToken;
-import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
-import software.amazon.lambda.durable.execution.DurableExecutor;
-import software.amazon.lambda.durable.model.DurableExecutionInput;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
-import software.amazon.lambda.durable.plugin.InvocationInfo;
-import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
-
-class InvocationEndFinalizationTest {
- @ParameterizedTest
- @CsvSource({"false,false", "false,true", "true,false", "true,true"})
- void waitsForHandlerFinallyAndRealOtelFlushBeforeResponding(boolean executionView, boolean retry) throws Exception {
- var flush = new BlockingFlush();
- var builder = SdkTracerProvider.builder().addSpanProcessor(flush);
- var settings = OtelPluginConfig.builder()
- .enableMdc(false)
- .contextExtractor(() -> new ExtractedContext(
- "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
- .build();
- DurableExecutionPlugin delegate = executionView
- ? new ExecutionOtelPlugin(builder, settings)
- : new InvocationOtelPlugin(builder, settings);
- var owner = new AtomicReference();
- var endOwner = new AtomicReference();
- var ends = new AtomicInteger();
- var plugin = new DurableExecutionPlugin() {
- @Override
- public void onInvocationStart(InvocationInfo info) {
- owner.set(Thread.currentThread());
- delegate.onInvocationStart(info);
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- endOwner.set(Thread.currentThread());
- delegate.onInvocationEnd(info);
- assertFalse(
- Span.current().getSpanContext().isValid(), "End must restore ambient context before returning");
- ends.incrementAndGet();
- }
- };
- var workers = Executors.newCachedThreadPool();
- var callers = Executors.newSingleThreadExecutor();
- var enteredFinally = new CountDownLatch(1);
- var releaseFinally = new CountDownLatch(1);
- var original = new UnrecoverableDurableExecutionException(
- ErrorObject.builder().errorMessage("original retry").build(), true);
- var config = DurableConfig.builder()
- .withDurableExecutionClient(new LocalMemoryExecutionClient())
- .withExecutorService(workers)
- .withCheckpointDelay(Duration.ZERO)
- .withPlugins(plugin)
- .build();
- try {
- var response = callers.submit(() -> DurableExecutor.execute(
- input(),
- null,
- TypeToken.get(String.class),
- (value, ctx) -> {
- assertSame(owner.get(), Thread.currentThread());
- assertTrue(Span.current().getSpanContext().isValid());
- try {
- if (retry)
- ctx.step("retry", String.class, step -> {
- throw original;
- });
- else ctx.wait("pause", Duration.ofSeconds(1));
- return "done";
- } finally {
- enteredFinally.countDown();
- await(releaseFinally);
- }
- },
- config));
- assertTrue(enteredFinally.await(3, TimeUnit.SECONDS));
- assertThrows(
- TimeoutException.class,
- () -> response.get(600, TimeUnit.MILLISECONDS),
- "suspension or termination cannot bypass a blocked handler finally");
- assertEquals(0, flush.calls.get(), "End must wait for the handler to unwind");
- assertEquals(0, ends.get());
- releaseFinally.countDown();
- assertTrue(flush.entered.await(3, TimeUnit.SECONDS));
- assertSame(owner.get(), endOwner.get());
- assertSame(owner.get(), flush.owner.get());
- assertThrows(
- TimeoutException.class,
- () -> response.get(100, TimeUnit.MILLISECONDS),
- "the invocation response must wait for the plugin's actual forceFlush result");
- flush.result.succeed();
- if (retry) {
- var thrown = assertThrows(ExecutionException.class, () -> response.get(3, TimeUnit.SECONDS));
- assertSame(original, thrown.getCause());
- } else {
- assertEquals(
- ExecutionStatus.PENDING,
- response.get(3, TimeUnit.SECONDS).status());
- }
- assertEquals(1, flush.calls.get());
- assertEquals(1, ends.get());
- } finally {
- releaseFinally.countDown();
- flush.result.succeed();
- workers.shutdown();
- callers.shutdown();
- assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
- assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS));
- }
- }
-
- private static DurableExecutionInput input() {
- var operation = Operation.builder()
- .id("id")
- .name("test")
- .type(OperationType.EXECUTION)
- .status(OperationStatus.STARTED)
- .startTimestamp(Instant.now())
- .executionDetails(
- ExecutionDetails.builder().inputPayload("\"input\"").build())
- .build();
- return new DurableExecutionInput(
- "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id",
- "token",
- CheckpointUpdatedExecutionState.builder().operations(operation).build());
- }
-
- private static void await(CountDownLatch latch) {
- try {
- assertTrue(latch.await(5, TimeUnit.SECONDS));
- } catch (InterruptedException error) {
- Thread.currentThread().interrupt();
- throw new AssertionError(error);
- }
- }
-
- private static final class BlockingFlush implements SpanProcessor {
- final AtomicInteger calls = new AtomicInteger();
- final AtomicReference owner = new AtomicReference<>();
- final CountDownLatch entered = new CountDownLatch(1);
- final CompletableResultCode result = new CompletableResultCode();
-
- @Override
- public CompletableResultCode forceFlush() {
- calls.incrementAndGet();
- owner.set(Thread.currentThread());
- entered.countDown();
- return result;
- }
-
- @Override
- public void onStart(Context parent, ReadWriteSpan span) {}
-
- @Override
- public boolean isStartRequired() {
- return false;
- }
-
- @Override
- public void onEnd(ReadableSpan span) {}
-
- @Override
- public boolean isEndRequired() {
- return false;
- }
-
- @Override
- public CompletableResultCode shutdown() {
- return CompletableResultCode.ofSuccess();
- }
- }
-}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java
new file mode 100644
index 0000000000..1f4b1577ea
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java
@@ -0,0 +1,171 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Instant;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+
+class InvocationHeaderTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void overlappingInvocationsAndResumeKeepTheirOwnHeader(boolean executionView) throws Exception {
+ var barrier = new CyclicBarrier(2);
+ var sampled = CompletableFuture.runAsync(() -> runInvocations(executionView, true, barrier));
+ var unsampled = CompletableFuture.runAsync(() -> runInvocations(executionView, false, barrier));
+ CompletableFuture.allOf(sampled, unsampled).get(20, TimeUnit.SECONDS);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void factoryHeaderPrecedesSpanCreationAndPreservesExtractorOverride(boolean executionView) {
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder()
+ .setSampler(Sampler.alwaysOn())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var calls = new AtomicInteger();
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(new XRayContextExtractor() {
+ @Override
+ public ExtractedContext extract() {
+ calls.incrementAndGet();
+ return super.extract();
+ }
+ })
+ .build();
+ var factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ var info =
+ invocation("request", "arn", true, Instant.EPOCH, "Root=1-6955b900-123456789012345678901234;Sampled=0");
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationEnd(new InvocationEndInfo("request", "arn", true, InvocationStatus.SUCCEEDED, null));
+ assertEquals(1, calls.get());
+ assertTrue(exporter.getFinishedSpanItems().isEmpty());
+ var next = factory.createPlugin(new InvocationInfo("request", "arn", true, Instant.EPOCH));
+ next.onInvocationEnd(new InvocationEndInfo("request", "arn", true, InvocationStatus.SUCCEEDED, null));
+ assertEquals(2, calls.get());
+ assertTrue(exporter.getFinishedSpanItems().stream()
+ .anyMatch(span -> span.getName().equals("Workflow")));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void capturedMissingHeaderDoesNotBorrowTheOverlappingGlobalCarrier(boolean executionView) throws Exception {
+ var property = "com.amazonaws.xray.traceHeader";
+ var previous = System.getProperty(property);
+ var conflictingTrace = "6955b900aaaaaaaaaaaaaaaaaaaaaaaa";
+ System.setProperty(property, "Root=1-6955b900-aaaaaaaaaaaaaaaaaaaaaaaa;Sampled=0");
+ var barrier = new CyclicBarrier(2);
+ var sharedExtractor = new XRayContextExtractor();
+ try {
+ var missing = CompletableFuture.runAsync(() -> {
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder()
+ .setSampler(Sampler.alwaysOn())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(sharedExtractor)
+ .build();
+ var factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ var info = invocation("missing", "arn:missing", true, Instant.EPOCH, "");
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info); // API available, but this invocation has no header.
+ await(barrier);
+ plugin.onInvocationEnd(
+ new InvocationEndInfo("missing", "arn:missing", true, InvocationStatus.SUCCEEDED, null));
+ var spans = exporter.getFinishedSpanItems();
+ assertEquals(
+ 3,
+ spans.size(),
+ "the synthetic anchor and both SDK spans must retain their own sampling decision");
+ assertTrue(spans.stream().allMatch(span -> !conflictingTrace.equals(span.getTraceId())));
+ });
+ var present = CompletableFuture.runAsync(() -> {
+ var extracted = sharedExtractor.extract(invocation(
+ "present",
+ "arn:present",
+ true,
+ Instant.EPOCH,
+ "Root=1-6955b900-123456789012345678901234;Sampled=1"));
+ assertEquals("6955b900123456789012345678901234", extracted.traceId());
+ assertEquals(ExtractedContext.Sampling.SAMPLED, extracted.sampling());
+ await(barrier);
+ });
+ CompletableFuture.allOf(missing, present).get(20, TimeUnit.SECONDS);
+ assertNull(sharedExtractor.extract(invocation("missing", "arn:missing", true, Instant.EPOCH, "")));
+ assertEquals(
+ conflictingTrace,
+ sharedExtractor.extract().traceId(),
+ "legacy direct extraction keeps its fallback");
+ } finally {
+ if (previous == null) System.clearProperty(property);
+ else System.setProperty(property, previous);
+ }
+ }
+
+ private static void runInvocations(boolean executionView, boolean sampled, CyclicBarrier barrier) {
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder()
+ .setSampler(Sampler.alwaysOn())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder().enableMdc(false).build();
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ var traceId = sampled ? "6955b900123456789012345678901234" : "6955b900aaaaaaaaaaaaaaaaaaaaaaaa";
+ var header = "Root=1-" + traceId.substring(0, 8) + "-" + traceId.substring(8)
+ + ";Parent=1234567890123456;Sampled=" + (sampled ? "1" : "0");
+ var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/" + sampled;
+ for (int invocation = 0; invocation < 2; invocation++) {
+ var plugin = factory.createPlugin(invocation(
+ "request-" + invocation, arn, invocation == 0, Instant.parse("2026-10-02T00:00:00Z"), header));
+ await(barrier);
+ plugin.onInvocationEnd(new InvocationEndInfo(
+ "request-" + invocation,
+ arn,
+ invocation == 0,
+ invocation == 0 ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED,
+ null));
+ }
+ var spans = exporter.getFinishedSpanItems();
+ if (!sampled) {
+ assertTrue(spans.isEmpty(), "Upstream Sampled=0 overrides always-on fallback");
+ return;
+ }
+ assertEquals(3, spans.size());
+ assertTrue(spans.stream().allMatch(span -> traceId.equals(span.getTraceId())));
+ assertTrue(spans.stream().allMatch(span -> "1234567890123456".equals(span.getParentSpanId())));
+ }
+
+ private static InvocationInfo invocation(String request, String arn, boolean first, Instant start, String header) {
+ return new InvocationInfo(request, arn, first, start, null, Map.of(), Map.of(), header);
+ }
+
+ private static void await(CyclicBarrier barrier) {
+ try {
+ barrier.await(10, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java
new file mode 100644
index 0000000000..5d58defa8c
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java
@@ -0,0 +1,168 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Instant;
+import java.util.Map;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState;
+import software.amazon.awssdk.services.lambda.model.ExecutionDetails;
+import software.amazon.awssdk.services.lambda.model.Operation;
+import software.amazon.awssdk.services.lambda.model.OperationStatus;
+import software.amazon.awssdk.services.lambda.model.OperationType;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.client.DurableExecutionClient;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+
+class InvocationInfoHeaderIntegrationTest {
+ private static final String TRACE = "6955b900123456789012345678901234";
+ private static final String PARENT = "1234567890123456";
+ private static final String HEADER = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1";
+ private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id";
+
+ @ParameterizedTest
+ @CsvSource({
+ "factory,unreadable-cause",
+ "hook,unreadable-cause",
+ "factory,present",
+ "hook,present",
+ "factory,empty",
+ "hook,empty",
+ "factory,null",
+ "hook,null",
+ "factory,unavailable",
+ "hook,unavailable"
+ })
+ void ordinaryFactoryAndHookCanReadTheCapturedHeader(String reader, String carrier) {
+ var runtimeThread = Thread.currentThread();
+ var capturedThread = new AtomicReference();
+ var factoryInfo = new AtomicReference();
+ var hookInfo = new AtomicReference();
+ var observed = new AtomicReference<>("callback did not run");
+ var runtime = "unavailable".equals(carrier) ? null : mock(RuntimeContext.class);
+ if (runtime != null) {
+ when(runtime.getRemainingTimeInMillis()).thenReturn(30000);
+ when(runtime.getXrayTraceId()).thenAnswer(ignored -> {
+ capturedThread.set(Thread.currentThread());
+ return switch (carrier) {
+ case "unreadable-cause" ->
+ throw new CompletionException("unreadable cause", null) {
+ @Override
+ public synchronized Throwable getCause() {
+ throw new IllegalStateException("cause accessor failed");
+ }
+ };
+ case "present" -> HEADER;
+ case "empty" -> "";
+ default -> null;
+ };
+ });
+ }
+ var config = DurableConfig.builder()
+ .withDurableExecutionClient(mock(DurableExecutionClient.class))
+ .withPlugins(info -> {
+ factoryInfo.set(info);
+ if (reader.equals("factory")) observed.set(info.xRayTraceId());
+ return new DurableExecutionPlugin() {
+ @Override
+ public void onInvocationStart(InvocationInfo info) {
+ hookInfo.set(info);
+ if (reader.equals("hook")) observed.set(info.xRayTraceId());
+ }
+ };
+ })
+ .build();
+ var output =
+ DurableExecutor.execute(input(), runtime, TypeToken.get(String.class), (value, ctx) -> value, config);
+ assertEquals(ExecutionStatus.SUCCEEDED, output.status());
+ assertEquals("\"input\"", output.result());
+ assertNotNull(factoryInfo.get());
+ assertSame(factoryInfo.get(), hookInfo.get(), "Factory and hook must receive the same immutable snapshot");
+ var expected = switch (carrier) {
+ case "present" -> HEADER;
+ case "unavailable" -> null;
+ default -> "";
+ };
+ assertEquals(expected, observed.get(), reader + " must read the captured value from InvocationInfo");
+ if (runtime != null) {
+ verify(runtime, times(1)).getXrayTraceId();
+ assertSame(runtimeThread, capturedThread.get());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void directFactoryDispatchesTheInvocationAwareCustomExtractor(boolean executionView) {
+ var seen = new AtomicReference();
+ var exporter = InMemorySpanExporter.create();
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(new ContextExtractor() {
+ @Override
+ public ExtractedContext extract() {
+ return null;
+ }
+
+ @Override
+ public ExtractedContext extract(InvocationInfo info) {
+ seen.set(info);
+ assertEquals(HEADER, info.xRayTraceId());
+ return new ExtractedContext(TRACE, PARENT, ExtractedContext.Sampling.SAMPLED);
+ }
+ })
+ .build();
+ var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var factory = executionView
+ ? ExecutionOtelPlugin.factory(provider, config)
+ : InvocationOtelPlugin.factory(provider, config);
+ var info = new InvocationInfo("request", ARN, true, Instant.EPOCH, null, Map.of(), Map.of(), HEADER);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
+ plugin.onInvocationEnd(new InvocationEndInfo("request", ARN, true, InvocationStatus.SUCCEEDED, null));
+ assertSame(info, seen.get(), "The extract(info) extension must run before factory span creation");
+ var spans = exporter.getFinishedSpanItems();
+ assertEquals(2, spans.size(), "A complete remote parent is external, not a synthetic root");
+ assertTrue(spans.stream().allMatch(span -> TRACE.equals(span.getTraceId())));
+ assertTrue(spans.stream().allMatch(span -> PARENT.equals(span.getParentSpanId())));
+ }
+
+ private static DurableExecutionInput input() {
+ var operation = Operation.builder()
+ .id("id")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ return new DurableExecutionInput(
+ ARN,
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operation).build());
+ }
+
+ private abstract static class RuntimeContext implements Context {
+ @Override
+ public String getXrayTraceId() {
+ return null;
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java
index b6bb682aaa..9948dbc786 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java
@@ -42,7 +42,7 @@ void alreadyRunningStepCompletesBeforeEnd(boolean earlyParallel) throws Exceptio
var batch = BatchSpanProcessor.builder(exporter)
.setScheduleDelay(Duration.ofDays(1))
.build()) {
- var otel = new InvocationOtelPlugin(
+ var otel = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(batch),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -52,15 +52,17 @@ void alreadyRunningStepCompletesBeforeEnd(boolean earlyParallel) throws Exceptio
.withDurableExecutionClient(client)
.withExecutorService(workers)
.withCheckpointDelay(Duration.ZERO)
- .withPlugins(otel, new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- ended.countDown();
- }
+ .withPlugins(
+ ignoredInfo -> new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ended.countDown();
+ }
- public void onOperationEnd(OperationEndInfo info) {
- if (ended.getCount() == 0) lateOperationEnds.incrementAndGet();
- }
- })
+ public void onOperationEnd(OperationEndInfo info) {
+ if (ended.getCount() == 0) lateOperationEnds.incrementAndGet();
+ }
+ },
+ otel)
.build();
try {
var call = caller.submit(() -> DurableExecutor.execute(
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java
new file mode 100644
index 0000000000..3b2ca0cb87
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java
@@ -0,0 +1,122 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.slf4j.MDC;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.serde.JacksonSerDes;
+import software.amazon.lambda.durable.serde.SerDes;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class InvocationMdcCleanupTest {
+ private static final String TRACE_ID = "12345678901234567890123456789012";
+
+ @ParameterizedTest
+ @CsvSource({"true,input", "false,input", "true,factory", "false,factory", "true,start", "false,start"})
+ void startupFailureClearsMdcOnTheOwningWorker(boolean executionView, String stage) throws Exception {
+ var fatal = new InternalError("startup hook");
+ var uncaught = new AtomicReference();
+ var escapedMdc = new AtomicReference();
+ var escaped = new CountDownLatch(1);
+ var executor = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "startup-mdc-owner");
+ thread.setDaemon(true);
+ thread.setUncaughtExceptionHandler((owner, failure) -> {
+ escapedMdc.set(MDC.get(MdcSpanEnricher.MDC_TRACE_ID));
+ uncaught.set(failure);
+ escaped.countDown();
+ });
+ return thread;
+ });
+ var config = OtelPluginConfig.builder()
+ .enableMdc(true)
+ .contextExtractor(
+ () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ var otel = executionView
+ ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), config)
+ : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), config);
+ var startMdc = new AtomicReference();
+ var ends = new AtomicInteger();
+ var end = new AtomicReference();
+ DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() {
+ public void onInvocationStart(InvocationInfo value) {
+ startMdc.set(MDC.get(MdcSpanEnricher.MDC_TRACE_ID));
+ }
+
+ public void onInvocationEnd(InvocationEndInfo value) {
+ ends.incrementAndGet();
+ end.set(value);
+ }
+ };
+ DurableExecutionPluginFactory faulty = info -> {
+ if (stage.equals("factory")) throw fatal;
+ return new DurableExecutionPlugin() {
+ public void onInvocationStart(InvocationInfo value) {
+ if (stage.equals("start")) throw fatal;
+ }
+ };
+ };
+ var builder = DurableConfig.builder().withExecutorService(executor).withPlugins(otel, observer, faulty);
+ if (stage.equals("input"))
+ builder.withSerDes(new SerDes() {
+ private final JacksonSerDes delegate = new JacksonSerDes();
+
+ public String serialize(Object value) {
+ return delegate.serialize(value);
+ }
+
+ public T deserialize(String data, TypeToken type) {
+ throw new IllegalArgumentException("bad input");
+ }
+ });
+ var bodies = new AtomicInteger();
+ try {
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ bodies.incrementAndGet();
+ return "unreachable";
+ },
+ builder.build());
+ if (stage.equals("input")) {
+ assertEquals(ExecutionStatus.FAILED, runner.run("input").getStatus());
+ assertNull(
+ executor.submit(() -> MDC.get(MdcSpanEnricher.MDC_TRACE_ID))
+ .get(2, TimeUnit.SECONDS),
+ "startup failure must not leave a trace on the reused handler worker");
+ } else {
+ assertSame(fatal, assertThrows(InternalError.class, () -> runner.run("input")));
+ assertTrue(escaped.await(2, TimeUnit.SECONDS));
+ assertSame(fatal, uncaught.get());
+ assertNull(escapedMdc.get(), "cleanup must run before the fatal escapes its owner thread");
+ }
+ assertEquals(TRACE_ID, startMdc.get(), "later startup plugins must still see the invocation trace");
+ assertEquals(0, bodies.get());
+ assertEquals(1, ends.get());
+ assertEquals(
+ stage.equals("input") ? InvocationStatus.FAILED : InvocationStatus.RETRYING,
+ end.get().invocationStatus());
+ if (!stage.equals("input")) assertSame(fatal, end.get().executionError());
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java
index f9ff81d20f..fe54f3499e 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java
@@ -45,14 +45,15 @@ void setUp() {
OtelPluginAutoConfigurationState.resetInstalledForTest();
spanExporter = InMemorySpanExporter.create();
- var plugin = new InvocationOtelPlugin(
+ // One factory for the environment; the SDK creates one plugin instance per invocation from it.
+ var factory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(false)
.build());
- otelConfig = DurableConfig.builder().withPlugins(plugin).build();
+ otelConfig = DurableConfig.builder().withPlugins(factory).build();
}
@AfterEach
@@ -337,7 +338,7 @@ void failedStep_producesErrorSpan() {
void sampling_off_producesNoSpans() {
var sampledExporter = InMemorySpanExporter.create();
- var noSamplePlugin = new InvocationOtelPlugin(
+ var noSampleFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(Sampler.alwaysOff())
.addSpanProcessor(SimpleSpanProcessor.create(sampledExporter)),
@@ -346,7 +347,8 @@ void sampling_off_producesNoSpans() {
.enableMdc(false)
.build());
- var noSampleConfig = DurableConfig.builder().withPlugins(noSamplePlugin).build();
+ var noSampleConfig =
+ DurableConfig.builder().withPlugins(noSampleFactory).build();
var runner = LocalDurableTestRunner.create(
String.class, (input, ctx) -> ctx.step("step", String.class, stepCtx -> "result"), noSampleConfig);
@@ -545,8 +547,8 @@ void waitForCondition_producesSpansWithAttempts() {
}
@Test
- void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() {
- var defaultPlugin = new InvocationOtelPlugin();
+ void agentPathFactory_bindsGlobalSdkTracerProviderWhenTheInvocationsInstanceIsCreated() {
+ var defaultFactory = InvocationOtelPlugin.factory();
assertFalse(GlobalOpenTelemetry.isSet());
OtelPluginAutoConfigurationState.markInstalled();
@@ -556,7 +558,7 @@ void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() {
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- var defaultConfig = DurableConfig.builder().withPlugins(defaultPlugin).build();
+ var defaultConfig = DurableConfig.builder().withPlugins(defaultFactory).build();
var runner = LocalDurableTestRunner.create(
String.class,
(input, ctx) -> ctx.step("global-step", String.class, stepCtx -> "Hello " + input),
@@ -573,7 +575,7 @@ void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() {
}
@Test
- void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() {
+ void agentPathFactory_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() {
OtelPluginAutoConfigurationState.markInstalled();
GlobalOpenTelemetry.resetForTest();
var globalExporter = InMemorySpanExporter.create();
@@ -595,8 +597,9 @@ public ContextPropagators getPropagators() {
}
});
- var defaultConfig =
- DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build();
+ var defaultConfig = DurableConfig.builder()
+ .withPlugins(InvocationOtelPlugin.factory())
+ .build();
var runner = LocalDurableTestRunner.create(
String.class,
(input, ctx) -> ctx.step("javaagent-step", String.class, stepCtx -> "Hello " + input),
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
index 36ac782a88..4d997a5d24 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
@@ -9,6 +9,7 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static software.amazon.lambda.durable.otel.Invocations.started;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.OpenTelemetry;
@@ -47,7 +48,9 @@
class InvocationOtelPluginTest {
private InMemorySpanExporter spanExporter;
- private InvocationOtelPlugin plugin;
+
+ /** The environment's plugin factory; each test creates one instance per invocation from it. */
+ private DurableExecutionPluginFactory factory;
@BeforeEach
void setUp() {
@@ -56,7 +59,7 @@ void setUp() {
OtelPluginAutoConfigurationState.resetInstalledForTest();
spanExporter = InMemorySpanExporter.create();
- plugin = new InvocationOtelPlugin(
+ factory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -73,12 +76,13 @@ void tearDown() {
}
@Test
- void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
+ void agentPathFactory_bindsGlobalProviderOnALaterInvocationsInstance() {
GlobalOpenTelemetry.resetForTest();
OtelPluginAutoConfigurationState.markInstalled();
- var defaultPlugin = new InvocationOtelPlugin();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now()));
+ var defaultPluginFactory = InvocationOtelPlugin.factory();
+ var defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now()));
defaultPlugin.onOperationStart(new OperationInfo(
"op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -105,7 +109,8 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now()));
+ defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now()));
defaultPlugin.onOperationStart(new OperationInfo(
"op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -131,8 +136,8 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
}
@Test
- void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
- var defaultPlugin = new InvocationOtelPlugin();
+ void agentPathFactory_usesGlobalSdkTracerProviderDirectly() {
+ var defaultPluginFactory = InvocationOtelPlugin.factory();
assertFalse(GlobalOpenTelemetry.isSet());
OtelPluginAutoConfigurationState.markInstalled();
@@ -142,7 +147,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
defaultPlugin.onOperationStart(
new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -168,7 +174,7 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
}
@Test
- void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() {
+ void agentPathFactory_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() {
OtelPluginAutoConfigurationState.markInstalled();
GlobalOpenTelemetry.resetForTest();
var globalExporter = InMemorySpanExporter.create();
@@ -190,8 +196,9 @@ public ContextPropagators getPropagators() {
}
});
- var defaultPlugin = new InvocationOtelPlugin();
- defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var defaultPluginFactory = InvocationOtelPlugin.factory();
+ var defaultPlugin =
+ started(defaultPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
defaultPlugin.onOperationStart(
new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));
defaultPlugin.onOperationEnd(new OperationEndInfo(
@@ -295,8 +302,14 @@ void invocationOtelPluginProvider_isRegisteredAsServiceProvider() {
.get();
assertEquals("otel-invocation", provider.getName());
- assertEquals(DurableExecutionPluginProvider.API_VERSION, provider.getApiVersion());
- assertEquals(InvocationOtelPlugin.class, provider.getPluginType());
+
+ // The provider is the per-invocation factory: it creates an InvocationOtelPlugin for the invocation it is
+ // handed, and a distinct instance for the next one.
+ var first = provider.createPlugin(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var second = provider.createPlugin(new InvocationInfo("req-2", "arn:exec1", false, Instant.now()));
+ assertInstanceOf(InvocationOtelPlugin.class, first);
+ assertInstanceOf(InvocationOtelPlugin.class, second);
+ assertNotSame(first, second, "Each invocation gets its own plugin instance");
}
@Test
@@ -310,10 +323,13 @@ void invocationStart_staysOnExecutionTrace_withoutLinkingAmbientSpan() {
var ambientSpanContext =
SpanContext.create(ambientTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault());
+ // The instance is created inside the ambient scope because the invocation's parent resolution happens when the
+ // factory creates it, not later.
+ DurableExecutionPlugin plugin;
try (var ignored = Span.wrap(ambientSpanContext).makeCurrent()) {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
- plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
+ plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
}
+ plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var invocationSpan = spanByName("Invocation");
var workflowSpan = spanByName("Workflow");
@@ -324,8 +340,13 @@ void invocationStart_staysOnExecutionTrace_withoutLinkingAmbientSpan() {
@Test
void invocationStart_and_end_createsSpan() {
- plugin.onInvocationStart(new InvocationInfo(
- "req-123", "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1", true, Instant.now()));
+ var plugin = started(
+ factory,
+ new InvocationInfo(
+ "req-123",
+ "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1",
+ true,
+ Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req-123",
"arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1",
@@ -344,7 +365,7 @@ void invocationStart_and_end_createsSpan() {
@Test
void customInstrumentationName_isUsedForTracerScope() {
var exporter = InMemorySpanExporter.create();
- var customPlugin = new InvocationOtelPlugin(
+ var customPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -352,7 +373,7 @@ void customInstrumentationName_isUsedForTracerScope() {
.workflowSpanName("Workflow")
.instrumentationName("my-custom-scope")
.build());
- customPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var customPlugin = started(customPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
customPlugin.onInvocationEnd(
new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
@@ -365,11 +386,14 @@ void customInstrumentationName_isUsedForTracerScope() {
@Test
void explicitProvider_unrelatedRootSpansKeepFreshTraceIds() {
+ // This invocation's instance and the unrelated library share the one provider the factory built.
+ var info = new InvocationInfo("req-1", "arn:exec1", true, Instant.now());
+ var plugin = (InvocationOtelPlugin) factory.createPlugin(info);
var provider = sdkTracerProvider(plugin);
var unrelatedTracer = provider.get("unrelated-library");
var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan();
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ plugin.onInvocationStart(info);
var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan();
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan();
@@ -398,8 +422,8 @@ void globalProvider_unrelatedRootSpansKeepFreshTraceIds() {
var unrelatedTracer = provider.get("unrelated-library");
var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan();
- var globalPlugin = new InvocationOtelPlugin();
- globalPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var globalPluginFactory = InvocationOtelPlugin.factory();
+ var globalPlugin = started(globalPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan();
globalPlugin.onInvocationEnd(
new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
@@ -419,7 +443,7 @@ void globalProvider_unrelatedRootSpansKeepFreshTraceIds() {
@Test
void invocationSpan_hasInternalKind() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var span = spanByName("Invocation");
@@ -428,7 +452,7 @@ void invocationSpan_hasInternalKind() {
@Test
void operationSpanName_usesOperationName_withoutPrefix() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "create-greeting", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -458,7 +482,7 @@ void operationSpanName_usesOperationName_withoutPrefix() {
@Test
void attemptSpanName_usesOperationNameWithAttemptNumber() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "process-order", "STEP", "Step", null, Instant.now(), false, 1));
plugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -487,7 +511,7 @@ void attemptSpanName_usesOperationNameWithAttemptNumber() {
@Test
void operationEnd_withAttempt_stampsAttemptNumberOnOperationSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "flaky", "STEP", "Step", null, Instant.now(), null, null, false));
@@ -518,7 +542,7 @@ void operationEnd_withAttempt_stampsAttemptNumberOnOperationSpan() {
@Test
void attemptSpan_carriesOperationSubtype() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "process-order", "STEP", "Step", null, Instant.now(), false, 1));
plugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -547,7 +571,7 @@ void attemptSpan_carriesOperationSubtype() {
@Test
void operationEnd_withoutMatchingStart_stampsAttemptNumberOnContinuationSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
// No onOperationStart in this invocation → onOperationEnd takes the continuation-span branch.
plugin.onOperationEnd(new OperationEndInfo(
@@ -581,7 +605,7 @@ void operationEnd_withoutMatchingStart_stampsAttemptNumberOnContinuationSpan() {
@Test
void invocationEnd_withFailure_setsErrorStatus() {
- plugin.onInvocationStart(new InvocationInfo("req-123", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-123", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req-123", "arn:exec1", true, InvocationStatus.FAILED, new RuntimeException("boom")));
@@ -592,7 +616,7 @@ void invocationEnd_withFailure_setsErrorStatus() {
@Test
void invocationEnd_withRetrying_leavesStatusUnset() {
- plugin.onInvocationStart(new InvocationInfo("req-123", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-123", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req-123", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient")));
@@ -606,7 +630,7 @@ void invocationEnd_withRetrying_leavesStatusUnset() {
@Test
void operationStart_createsSpan_operationEnd_endsIt() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
var start = Instant.parse("2026-06-01T10:00:00Z");
var end = Instant.parse("2026-06-01T10:00:05Z");
@@ -633,7 +657,7 @@ void operationStart_createsSpan_operationEnd_endsIt() {
@Test
void userFunctionStart_and_end_createsAttemptSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1));
@@ -666,7 +690,7 @@ void userFunctionStart_and_end_createsAttemptSpan() {
@Test
void userFunctionEnd_withFailure_setsErrorOnAttemptSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "failing", "STEP", "Step", null, Instant.now(), false, 1));
@@ -695,7 +719,7 @@ void userFunctionEnd_withFailure_setsErrorOnAttemptSpan() {
@Test
void userFunctionEnd_withSuccess_setsOkOnAttemptSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1));
@@ -723,7 +747,7 @@ void userFunctionEnd_withSuccess_setsOkOnAttemptSpan() {
@Test
void userFunctionEnd_withIncomplete_leavesAttemptSpanUnset() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "waiting", "STEP", "Step", null, Instant.now(), false, 1));
@@ -753,7 +777,7 @@ void userFunctionEnd_withIncomplete_leavesAttemptSpanUnset() {
@Test
void operationEnd_withSuccess_setsOkOnOperationSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step-ok", "STEP", "Step", null, Instant.now(), null, null, false));
@@ -785,7 +809,7 @@ void operationEnd_withSuccess_setsOkOnOperationSpan() {
@Test
void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() {
// Cancellation without diagnostic details retains UNSET; it is not blanket-classified as failure.
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-cancel", "step-cancel", "STEP", "Step", null, Instant.now(), null, null, false));
@@ -815,7 +839,7 @@ void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() {
@Test
void operationEnd_withoutMatchingStart_timeoutWithoutError_setsError() {
// An explicit timeout remains an error even when the backend supplied no exception details.
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationEnd(new OperationEndInfo(
"op-cb-timeout",
@@ -845,7 +869,7 @@ void operationEnd_withoutMatchingStart_timeoutWithoutError_setsError() {
void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() {
// A successful statusless virtual (FLAT CONTEXT) operation fires onOperationEnd with a null operation ->
// null status and null error. This is genuine success and must be stamped OK.
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-ctx", "my-ctx", "CONTEXT", null, null, Instant.now(), null, null, false));
@@ -875,7 +899,7 @@ void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() {
@Test
void fullLifecycle_producesCorrectSpanHierarchy() {
var arn = "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1";
- plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", arn, true, Instant.now()));
// Step 1: operation starts, user function runs, operation completes
plugin.onOperationStart(
@@ -958,14 +982,14 @@ void invocationRoots_sameExecutionShareExecutionTrace() {
// Same execution start time across invocations so the ARN-derived canonical trace ID is reproducible.
var startTime = Instant.now();
- plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, startTime));
+ var plugin = started(factory, new InvocationInfo("req-1", arn, true, startTime));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null));
var firstTraceId = spanByName("Invocation").getTraceId();
spanExporter.reset();
// Second invocation of same execution
- plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, startTime));
+ plugin = started(factory, new InvocationInfo("req-2", arn, false, startTime));
plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null));
var secondTraceId = spanByName("Invocation").getTraceId();
@@ -979,7 +1003,7 @@ void invocationRoots_sameExecutionShareExecutionTrace() {
@Test
void operationNotCompleted_spanEndedAtInvocationEnd() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
// Operation starts but never completes (e.g., wait operation, invocation suspends)
plugin.onOperationStart(
@@ -1002,7 +1026,7 @@ void operationNotCompleted_spanEndedAtInvocationEnd() {
@Test
void operationStart_withStatus_preservesStatus() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", false, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", false, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "my-step", "STEP", "Step", null, Instant.now(), null, "PENDING", true));
@@ -1028,7 +1052,7 @@ void invocationEnd_closesNestedSpansChildFirst() {
})
.when(clockAdvancer)
.onEnd(any());
- var timedPlugin = new InvocationOtelPlugin(
+ var timedFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder()
.setClock(clock)
.addSpanProcessor(SimpleSpanProcessor.create(spanExporter))
@@ -1039,16 +1063,16 @@ void invocationEnd_closesNestedSpansChildFirst() {
.build());
var parentId = "op-parent";
var childId = "op-child";
- timedPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, start));
- timedPlugin.onOperationStart(new OperationInfo(
+ var plugin = started(timedFactory, new InvocationInfo("req-1", "arn:exec1", true, start));
+ plugin.onOperationStart(new OperationInfo(
parentId, "parent-context", "CONTEXT", "RunInChildContext", null, start, null, null, false));
- timedPlugin.onOperationStart(
+ plugin.onOperationStart(
new OperationInfo(childId, "child-step", "STEP", "Step", parentId, start, null, null, false));
- timedPlugin.onUserFunctionStart(
+ plugin.onUserFunctionStart(
new UserFunctionStartInfo(childId, "child-step", "STEP", "Step", parentId, start, false, 1));
clock.advance(1, TimeUnit.SECONDS);
- timedPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null));
+ plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null));
var parentSpan = spanByName("parent-context");
var childSpan = spanByName("child-step");
@@ -1071,7 +1095,7 @@ void invocationEnd_closesNestedSpansChildFirst() {
@Test
void sampling_disabled_producesNoSpans() {
spanExporter = InMemorySpanExporter.create();
- var sampledPlugin = new InvocationOtelPlugin(
+ var sampledPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder()
.setSampler(io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOff())
.addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
@@ -1080,7 +1104,8 @@ void sampling_disabled_producesNoSpans() {
.enableMdc(false)
.build());
- sampledPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var sampledPlugin =
+ started(sampledPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
sampledPlugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "step", "STEP", "Step", null, Instant.now(), false, 1));
sampledPlugin.onUserFunctionEnd(new UserFunctionEndInfo(
@@ -1122,14 +1147,14 @@ void xrayExtraction_withoutParentDoesNotForceTraceId() {
var extractedContext = new ExtractedContext(xrayTraceId, null);
spanExporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> extractedContext)
.enableMdc(false)
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -1154,14 +1179,14 @@ void xrayExtraction_invocationTreeUsesExtractedTraceId_workflowJoinsExecutionTra
var extractedContext = new ExtractedContext(xrayTraceId, parentSpanId);
spanExporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> extractedContext)
.enableMdc(false)
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
xrayPlugin.onUserFunctionStart(
@@ -1208,14 +1233,14 @@ void xrayExtraction_withParentSpanId_invocationSpanHasCorrectParent() {
var extractedContext = new ExtractedContext(xrayTraceId, parentSpanId, ExtractedContext.Sampling.SAMPLED);
spanExporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> extractedContext)
.enableMdc(false)
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -1242,14 +1267,14 @@ void xrayExtraction_withoutParentSpanId_invocationSpanParentsOntoSyntheticRoot()
var extractedContext = new ExtractedContext(xrayTraceId, null);
spanExporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> extractedContext)
.enableMdc(false)
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -1271,7 +1296,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() {
var extractedContext = new ExtractedContext(xrayTraceId, "53995c3f42cd8ad8", ExtractedContext.Sampling.SAMPLED);
spanExporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> extractedContext)
@@ -1279,7 +1304,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() {
.build());
// First invocation
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onOperationStart(
new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, null, false));
xrayPlugin.onOperationEnd(new OperationEndInfo(
@@ -1298,7 +1323,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() {
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null));
// Second invocation (same execution, same X-Ray Root from backend)
- xrayPlugin.onInvocationStart(new InvocationInfo("req-2", "arn:exec1", false, Instant.now()));
+ xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-2", "arn:exec1", false, Instant.now()));
xrayPlugin.onOperationStart(
new OperationInfo("op-2", "step-2", "STEP", "Step", null, Instant.now(), null, null, false));
xrayPlugin.onOperationEnd(new OperationEndInfo(
@@ -1329,7 +1354,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() {
@Test
void xrayExtraction_nullExtractor_sharesArnDerivedExecutionTrace() {
spanExporter = InMemorySpanExporter.create();
- var noXrayPlugin = new InvocationOtelPlugin(
+ var noXrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
@@ -1337,7 +1362,7 @@ void xrayExtraction_nullExtractor_sharesArnDerivedExecutionTrace() {
.build());
var arn = "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1";
- noXrayPlugin.onInvocationStart(new InvocationInfo("req-1", arn, true, Instant.now()));
+ var noXrayPlugin = started(noXrayPluginFactory, new InvocationInfo("req-1", arn, true, Instant.now()));
noXrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
@@ -1369,14 +1394,14 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() {
// the spans export.
var extractedContext = new ExtractedContext(convertedId, "53995c3f42cd8ad8", ExtractedContext.Sampling.SAMPLED);
spanExporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> extractedContext)
.enableMdc(false)
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
assertEquals(expectedOtelTraceId, spanByName("Invocation").getTraceId());
@@ -1388,7 +1413,7 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() {
@Test
void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
// onOperationEnd without a prior onOperationStart — operation completed between invocations
plugin.onOperationEnd(new OperationEndInfo(
@@ -1420,7 +1445,7 @@ void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() {
@Test
void operationEnd_withoutMatchingStart_startsWithinCurrentInvocation() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
var operationStart = Instant.EPOCH;
var operationEnd = operationStart.plusSeconds(60);
@@ -1459,7 +1484,7 @@ void operationEnd_withoutMatchingStart_startsWithinCurrentInvocation() {
@Test
void operationEnd_withoutMatchingStart_withError_setsErrorStatus() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onOperationEnd(new OperationEndInfo(
"op-cb-1",
@@ -1489,7 +1514,7 @@ void operationEnd_withoutMatchingStart_withError_setsErrorStatus() {
@Test
void contextOperation_doesNotCreateAttemptSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
// Create operation span first so the CONTEXT user function has a parent
plugin.onOperationStart(new OperationInfo(
@@ -1524,7 +1549,7 @@ void contextOperation_doesNotCreateAttemptSpan() {
@Test
void attemptSpan_endedAtInvocationEnd_whenUserFunctionEndNotCalled() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
// Start attempt but never call onUserFunctionEnd (simulates crash before end hook)
plugin.onUserFunctionStart(
@@ -1545,7 +1570,7 @@ void attemptSpan_endedAtInvocationEnd_whenUserFunctionEndNotCalled() {
@Test
void childOperation_parentedToParentOperationSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
// Parent context operation
plugin.onOperationStart(new OperationInfo(
@@ -1611,7 +1636,7 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() {
var startTime = Instant.now();
// Invocation 1: step completes, wait starts
- plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, startTime));
+ var plugin = started(factory, new InvocationInfo("req-1", arn, true, startTime));
plugin.onOperationStart(
new OperationInfo("op-1", "step-A", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onUserFunctionStart(
@@ -1662,7 +1687,7 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() {
spanExporter.reset();
// Invocation 2: wait completed between invocations, new step runs
- plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, startTime));
+ plugin = started(factory, new InvocationInfo("req-2", arn, false, startTime));
plugin.onOperationEnd(new OperationEndInfo(
"op-2",
"pause",
@@ -1764,7 +1789,7 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() {
var startTime = Instant.now();
// Invocation 1: step starts, attempt 1 fails, invocation suspended during retry poll
- plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, startTime));
+ var plugin = started(factory, new InvocationInfo("req-1", arn, true, startTime));
plugin.onOperationStart(
new OperationInfo("op-1", "process-payment", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onUserFunctionStart(
@@ -1806,7 +1831,7 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() {
spanExporter.reset();
// Invocation 2: step is replayed (continuation), attempt 2 executes and succeeds
- plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, startTime));
+ plugin = started(factory, new InvocationInfo("req-2", arn, false, startTime));
// isReplay=true: this operation already exists in the execution state
plugin.onOperationStart(
new OperationInfo("op-1", "process-payment", "STEP", "Step", null, Instant.now(), null, null, true));
@@ -1904,7 +1929,7 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() {
@Test
void workflowSpan_exportedOnTerminal_internal_deterministicId() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec-wf", true, InvocationStatus.SUCCEEDED, null));
var workflow = spanByName("Workflow");
@@ -1915,7 +1940,7 @@ void workflowSpan_exportedOnTerminal_internal_deterministicId() {
@Test
void workflowSpan_notExportedOnNonTerminal() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null));
assertTrue(
@@ -1927,7 +1952,7 @@ void workflowSpan_notExportedOnNonTerminal() {
@Test
void workflowSpan_notExportedOnRetrying() {
// RETRYING is non-terminal, so the deferred Workflow span is neither materialized nor abandoned.
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req-1", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient")));
@@ -1941,7 +1966,7 @@ void workflowSpan_notExportedOnRetrying() {
void deferredWorkflowSpan_whenExported_isEnded_andMatchesLinkedSpanId() {
// The Workflow span is created only at the terminal invocation, but operations that ran earlier linked to its
// deterministic context. When it is finally exported it must be ended and carry that same span ID.
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -1970,7 +1995,7 @@ void deferredWorkflowSpan_whenExported_isEnded_andMatchesLinkedSpanId() {
@Test
void operationAndAttemptSpans_linkToWorkflowSpan() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now()));
plugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onUserFunctionStart(
@@ -2017,7 +2042,7 @@ void operationAndAttemptSpans_linkToWorkflowSpan() {
void operationLinksToWorkflow_withXRayContext() {
// "Other case": invocation span is parented to the X-Ray segment, but operation spans still link to Workflow.
var exporter = InMemorySpanExporter.create();
- var xrayPlugin = new InvocationOtelPlugin(
+ var xrayPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> new ExtractedContext(
@@ -2026,7 +2051,7 @@ void operationLinksToWorkflow_withXRayContext() {
ExtractedContext.Sampling.SAMPLED))
.enableMdc(false)
.build());
- xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
xrayPlugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
xrayPlugin.onOperationEnd(new OperationEndInfo(
@@ -2062,14 +2087,14 @@ void operationLinksToWorkflow_withXRayContext() {
@Test
void workflowSpanName_isConfigurable() {
var exporter = InMemorySpanExporter.create();
- var customPlugin = new InvocationOtelPlugin(
+ var customPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(false)
.workflowSpanName("MyWorkflow")
.build());
- customPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var customPlugin = started(customPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
customPlugin.onInvocationEnd(
new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
@@ -2085,7 +2110,7 @@ void workflowSpanName_isConfigurable() {
@Test
void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() {
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req-1", "arn:exec1", true, InvocationStatus.FAILED, new RuntimeException("boom")));
@@ -2186,14 +2211,15 @@ void noRecordingSpanIsLeftOpen_onRetrying_trackedByLifecycleProcessor() {
*/
private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) {
var lifecycle = new LifecycleTrackingSpanProcessor();
- var trackingPlugin = new InvocationOtelPlugin(
+ var trackingPluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(lifecycle),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(false)
.build());
- trackingPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
+ var trackingPlugin =
+ started(trackingPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
trackingPlugin.onOperationStart(
new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false));
trackingPlugin.onUserFunctionStart(
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOutcomeBoundaryTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOutcomeBoundaryTest.java
deleted file mode 100644
index 2572d41ab0..0000000000
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOutcomeBoundaryTest.java
+++ /dev/null
@@ -1,619 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_INVOCATION_STATUS;
-
-import io.opentelemetry.api.trace.StatusCode;
-import io.opentelemetry.context.Context;
-import io.opentelemetry.context.ContextKey;
-import io.opentelemetry.context.Scope;
-import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
-import io.opentelemetry.sdk.trace.SdkTracerProvider;
-import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Proxy;
-import java.time.Duration;
-import java.util.List;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.ValueSource;
-import org.slf4j.MDC;
-import org.slf4j.spi.MDCAdapter;
-import software.amazon.awssdk.services.lambda.model.ErrorObject;
-import software.amazon.lambda.durable.DurableConfig;
-import software.amazon.lambda.durable.TypeToken;
-import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
-import software.amazon.lambda.durable.plugin.InvocationInfo;
-import software.amazon.lambda.durable.plugin.InvocationStatus;
-import software.amazon.lambda.durable.serde.JacksonSerDes;
-import software.amazon.lambda.durable.serde.SerDes;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-import software.amazon.lambda.durable.util.ExceptionHelper;
-
-class InvocationOutcomeBoundaryTest {
- @ParameterizedTest
- @CsvSource({"false,false", "false,true", "true,false", "true,true"})
- void cleanupWaitContractAlsoAppliesWithoutPlugins(boolean withPlugin, boolean retry) throws Exception {
- var entered = new CountDownLatch(1);
- var release = new CountDownLatch(1);
- var original = retryError();
- var config = DurableConfig.builder();
- if (withPlugin) config.withPlugins(new DurableExecutionPlugin() {});
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, context) -> {
- try {
- if (retry)
- context.step("retry", String.class, step -> {
- throw original;
- });
- else context.wait("pause", Duration.ofSeconds(1));
- return "done";
- } finally {
- entered.countDown();
- await(release);
- }
- },
- config.build());
- var caller = Executors.newSingleThreadExecutor();
- try {
- var response = caller.submit(() -> runner.run("input"));
- assertTrue(entered.await(5, TimeUnit.SECONDS));
- assertThrows(TimeoutException.class, () -> response.get(700, TimeUnit.MILLISECONDS));
- release.countDown();
- if (retry)
- assertSame(
- original,
- assertThrows(ExecutionException.class, () -> response.get(5, TimeUnit.SECONDS))
- .getCause());
- else
- assertEquals(
- ExecutionStatus.PENDING,
- response.get(5, TimeUnit.SECONDS).getStatus());
- } finally {
- release.countDown();
- caller.shutdownNow();
- assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
-
- @SuppressWarnings("removal")
- @ParameterizedTest
- @CsvSource({"false,false", "false,true", "true,false", "true,true"})
- void invocationEndFatalEscapesItsActualWorkerAfterSettlingCaller(boolean death, boolean earlierAssertion)
- throws Exception {
- Error fatal = death ? new ThreadDeath() : new InternalError("fatal end cleanup");
- var ownerFailure = new AtomicReference();
- var escaped = new CountDownLatch(1);
- var ends = new AtomicInteger();
- var assertion = new AssertionError("earlier nonfatal cleanup");
- var fatalPlugin = new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.incrementAndGet();
- throw fatal;
- }
- };
- var assertionPlugin = new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- throw assertion;
- }
- };
- var workers = Executors.newSingleThreadExecutor(task -> {
- var worker = new Thread(task, "end-fatal-owner");
- worker.setUncaughtExceptionHandler((thread, failure) -> {
- ownerFailure.set(failure);
- escaped.countDown();
- });
- return worker;
- });
- try {
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, ctx) -> "done",
- DurableConfig.builder()
- .withExecutorService(workers)
- .withPlugins(
- earlierAssertion
- ? new DurableExecutionPlugin[] {fatalPlugin, assertionPlugin}
- : new DurableExecutionPlugin[] {fatalPlugin})
- .build());
- assertSame(fatal, assertThrows(Error.class, () -> runner.run("input")));
- assertTrue(escaped.await(2, TimeUnit.SECONDS), "fatal end cleanup must escape its actual worker");
- assertSame(fatal, ownerFailure.get());
- assertEquals(earlierAssertion ? List.of(assertion) : List.of(), List.of(fatal.getSuppressed()));
- assertEquals(1, ends.get());
- } finally {
- workers.shutdownNow();
- assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
-
- @ParameterizedTest
- @CsvSource({"SUCCEEDED,false", "SUCCEEDED,true", "PENDING,false", "PENDING,true", "RETRYING,false", "RETRYING,true"
- })
- void inlineNonfatalMdcRestorationPreservesSelectedOutcome(String outcome, boolean ambient) throws Exception {
- runInlineMdcFailure(outcome, ambient, new IllegalStateException("restore failed"), null);
- }
-
- @ParameterizedTest
- @CsvSource({
- "SUCCEEDED,false,assertion", "SUCCEEDED,true,assertion",
- "PENDING,false,assertion", "PENDING,true,assertion",
- "RETRYING,false,assertion", "RETRYING,true,assertion",
- "SUCCEEDED,false,linkage", "SUCCEEDED,true,linkage",
- "PENDING,false,linkage", "PENDING,true,linkage",
- "RETRYING,false,linkage", "RETRYING,true,linkage"
- })
- void inlineNonfatalErrorRestorationPreservesSelectedOutcome(String outcome, boolean ambient, String kind)
- throws Exception {
- Error failure = kind.equals("linkage")
- ? new NoClassDefFoundError("restore failed")
- : new AssertionError("restore failed");
- runInlineMdcFailure(outcome, ambient, failure, null);
- }
-
- @SuppressWarnings("removal")
- @ParameterizedTest
- @CsvSource({"false,false", "false,true", "true,false", "true,true"})
- void inlineFatalRestorationStillEscapesWithOriginalIdentity(boolean death, boolean wrapped) throws Exception {
- Error fatal = death ? new ThreadDeath() : new InternalError("fatal restoration");
- runInlineMdcFailure(
- "SUCCEEDED", false, wrapped ? new CompletionException(new ExecutionException(fatal)) : fatal, fatal);
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"unreadable", "cycle", "fatal"})
- void lifecycleDiagnosticTraversalIsBoundedAndPreservesFatalIdentity(String kind) throws Exception {
- var reads = new AtomicInteger();
- var fatal = new InternalError("fatal diagnostic");
- var failure = new CompletionException("diagnostic", null) {
- @Override
- public synchronized Throwable getCause() {
- reads.incrementAndGet();
- if (kind.equals("cycle")) return this;
- if (kind.equals("fatal")) throw fatal;
- throw new IllegalStateException("unreadable diagnostic");
- }
- };
- runInlineMdcFailure("SUCCEEDED", false, failure, kind.equals("fatal") ? fatal : null);
- assertEquals(1, reads.get());
- }
-
- private static void runInlineMdcFailure(
- String outcome, boolean ambient, Throwable restoreFailure, Error expectedFatal) throws Exception {
- var adapter = MDC.getMDCAdapter();
- var before = MDC.getCopyOfContextMap();
- var ends = new CopyOnWriteArrayList();
- var injected = new AtomicBoolean();
- var original = retryError();
- var executor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new SynchronousQueue()) {
- @Override
- public void execute(Runnable task) {
- task.run();
- }
- };
- try {
- MDC.clear();
- if (ambient) MDC.put("ambient", "saved");
- replaceAdapter((MDCAdapter) Proxy.newProxyInstance(
- MDCAdapter.class.getClassLoader(), new Class[] {MDCAdapter.class}, (proxy, method, args) -> {
- if (!ends.isEmpty()
- && method.getName().equals(ambient ? "setContextMap" : "clear")
- && injected.compareAndSet(false, true)) throw restoreFailure;
- try {
- return method.invoke(adapter, args);
- } catch (InvocationTargetException failure) {
- throw failure.getCause();
- }
- }));
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, context) -> {
- if (outcome.equals("PENDING")) context.wait("pause", Duration.ofSeconds(1));
- if (outcome.equals("RETRYING"))
- context.step("retry", String.class, step -> {
- throw original;
- });
- return "done";
- },
- DurableConfig.builder()
- .withExecutorService(executor)
- .withPlugins(new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add(info);
- }
- })
- .build());
- if (expectedFatal != null) assertSame(expectedFatal, assertThrows(Error.class, () -> runner.run("input")));
- else if (outcome.equals("RETRYING"))
- assertSame(
- original,
- assertThrows(UnrecoverableDurableExecutionException.class, () -> runner.run("input")));
- else
- assertEquals(
- ExecutionStatus.valueOf(outcome), runner.run("input").getStatus());
- assertTrue(injected.get());
- assertEquals(1, ends.size());
- assertEquals(InvocationStatus.valueOf(outcome), ends.get(0).invocationStatus());
- } finally {
- executor.shutdownNow();
- replaceAdapter(adapter);
- if (before == null) MDC.clear();
- else MDC.setContextMap(before);
- }
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void deliveryFailureRemainsRetryingUntilAReplayActuallySucceeds(boolean executionView) {
- var exporter = InMemorySpanExporter.create();
- var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
- var options = OtelPluginConfig.builder()
- .contextExtractor(() -> null)
- .enableMdc(false)
- .build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(provider, options)
- : new InvocationOtelPlugin(provider, options);
- var ends = new CopyOnWriteArrayList();
- var failDelivery = new AtomicBoolean(true);
- var original = new IllegalStateException("cannot deliver result yet");
- var completedSideEffects = new AtomicInteger();
- var serDes = new SerDes() {
- private final JacksonSerDes delegate = new JacksonSerDes();
-
- @Override
- public String serialize(Object value) {
- if ("done".equals(value) && failDelivery.compareAndSet(true, false)) throw original;
- return delegate.serialize(value);
- }
-
- @Override
- public T deserialize(String value, TypeToken type) {
- return delegate.deserialize(value, type);
- }
- };
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, ctx) -> {
- ctx.step("saved", String.class, step -> {
- completedSideEffects.incrementAndGet();
- return "checkpointed";
- });
- return "done";
- },
- DurableConfig.builder()
- .withSerDes(serDes)
- .withPlugins(otel, new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add(info);
- }
- })
- .build());
- assertSame(original, assertThrows(IllegalStateException.class, () -> runner.run("input")));
- assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus());
- assertSame(original, ends.get(0).executionError());
- assertEquals(
- 0,
- exporter.getFinishedSpanItems().stream()
- .filter(s -> s.getName().equals("Workflow"))
- .count());
- assertEquals(ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus());
- assertEquals(
- List.of(InvocationStatus.RETRYING, InvocationStatus.SUCCEEDED),
- ends.stream().map(InvocationEndInfo::invocationStatus).toList());
- assertEquals(1, completedSideEffects.get());
- assertEquals(
- 1,
- exporter.getFinishedSpanItems().stream()
- .filter(s -> s.getName().equals("Workflow"))
- .count());
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void failureSerializationRemainsRetryingUntilFailureCanBeDelivered(boolean executionView) {
- var exporter = InMemorySpanExporter.create();
- var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
- var options = OtelPluginConfig.builder()
- .contextExtractor(() -> null)
- .enableMdc(false)
- .build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(provider, options)
- : new InvocationOtelPlugin(provider, options);
- var ends = new CopyOnWriteArrayList();
- var failDelivery = new AtomicBoolean(true);
- var deliveryFailure = new IllegalStateException("cannot serialize failure yet");
- var bodyFailure = new IllegalArgumentException("handler failed");
- var completedSideEffects = new AtomicInteger();
- var serDes = new SerDes() {
- private final JacksonSerDes delegate = new JacksonSerDes();
-
- @Override
- public String serialize(Object value) {
- if (value == bodyFailure && failDelivery.compareAndSet(true, false)) throw deliveryFailure;
- return delegate.serialize(value);
- }
-
- @Override
- public T deserialize(String value, TypeToken type) {
- return delegate.deserialize(value, type);
- }
- };
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, context) -> {
- context.step("saved-before-failure", String.class, step -> {
- completedSideEffects.incrementAndGet();
- return "checkpointed";
- });
- throw bodyFailure;
- },
- DurableConfig.builder()
- .withSerDes(serDes)
- .withPlugins(otel, new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add(info);
- }
- })
- .build());
- assertSame(deliveryFailure, assertThrows(IllegalStateException.class, () -> runner.run("input")));
- assertEquals(1, ends.size());
- assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus());
- assertSame(deliveryFailure, ends.get(0).executionError());
- assertEquals(
- 0,
- exporter.getFinishedSpanItems().stream()
- .filter(span -> span.getName().equals("Workflow"))
- .count());
- var firstInvocation = exporter.getFinishedSpanItems().stream()
- .filter(span -> span.getName().equals("Invocation"))
- .findFirst()
- .orElseThrow();
- assertEquals("RETRYING", firstInvocation.getAttributes().get(DURABLE_INVOCATION_STATUS));
- assertEquals(StatusCode.UNSET, firstInvocation.getStatus().getStatusCode());
-
- var result = runner.run("input");
- assertEquals(ExecutionStatus.FAILED, result.getStatus());
- assertEquals(
- bodyFailure.getClass().getName(),
- result.getError().orElseThrow().errorType());
- assertEquals(bodyFailure.getMessage(), result.getError().orElseThrow().errorMessage());
- assertEquals(
- List.of(InvocationStatus.RETRYING, InvocationStatus.FAILED),
- ends.stream().map(InvocationEndInfo::invocationStatus).toList());
- assertSame(bodyFailure, ends.get(1).executionError());
- assertEquals(1, completedSideEffects.get(), "the completed step body must not run again after delivery retry");
- assertEquals(
- 1,
- exporter.getFinishedSpanItems().stream()
- .filter(span -> span.getName().equals("Workflow"))
- .count());
- }
-
- @SuppressWarnings("removal")
- @ParameterizedTest
- @CsvSource({
- "ordinary,assertion,false,false", "ordinary,assertion,true,false",
- "vm,assertion,false,false", "vm,assertion,true,false",
- "death,assertion,false,false", "death,assertion,true,false",
- "ordinary,vm,false,false", "ordinary,vm,true,false",
- "vm,death,false,false", "vm,death,true,false",
- "wrapped,assertion,false,false", "wrapped,assertion,true,false",
- "ordinary,assertion,false,true", "ordinary,assertion,true,true",
- "vm,assertion,false,true", "vm,assertion,true,true",
- "death,assertion,false,true", "death,assertion,true,true",
- "ordinary,vm,false,true", "ordinary,vm,true,true",
- "vm,death,false,true", "vm,death,true,true",
- "wrapped,assertion,false,true", "wrapped,assertion,true,true"
- })
- void preparationAndEndFailuresRetainPrimaryAndFatalIdentity(
- String preparationKind, String endKind, boolean serializeFailure, boolean inline) throws Exception {
- Throwable original = switch (preparationKind) {
- case "vm", "wrapped" -> new InternalError("preparation fatal");
- case "death" -> new ThreadDeath();
- default -> new IllegalStateException("preparation failure");
- };
- var delivery = preparationKind.equals("wrapped") ? new CompletionException(original) : original;
- Error endFailure = switch (endKind) {
- case "vm" -> new InternalError("end fatal");
- case "death" -> new ThreadDeath();
- default -> new AssertionError("end failure");
- };
- var expected = preparationKind.equals("ordinary") && endKind.equals("vm") ? endFailure : original;
- var suppressed = expected == endFailure ? original : endFailure;
- var ends = new CopyOnWriteArrayList();
- var bodyFailure = new IllegalArgumentException("body failure");
- var escaped = new CountDownLatch(1);
- var workerFailure = new AtomicReference();
- var callers = Executors.newSingleThreadExecutor();
- var workers =
- new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new SynchronousQueue(), task -> {
- var thread = new Thread(task, "combined-failure-owner");
- thread.setUncaughtExceptionHandler((owner, failure) -> {
- workerFailure.set(failure);
- escaped.countDown();
- });
- return thread;
- }) {
- @Override
- public void execute(Runnable task) {
- if (inline) task.run();
- else super.execute(task);
- }
- };
- var serDes = new SerDes() {
- private final JacksonSerDes delegate = new JacksonSerDes();
-
- @Override
- public String serialize(Object value) {
- if (serializeFailure ? value == bodyFailure : "done".equals(value)) {
- ExceptionHelper.sneakyThrow(delivery);
- }
- return delegate.serialize(value);
- }
-
- @Override
- public T deserialize(String value, TypeToken type) {
- return delegate.deserialize(value, type);
- }
- };
- try {
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, context) -> {
- if (serializeFailure) throw bodyFailure;
- return "done";
- },
- DurableConfig.builder()
- .withExecutorService(workers)
- .withSerDes(serDes)
- .withPlugins(new DurableExecutionPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add(info);
- throw endFailure;
- }
- })
- .build());
- var response = callers.submit(() -> runner.run("input"));
- var observed = assertThrows(ExecutionException.class, () -> response.get(3, TimeUnit.SECONDS))
- .getCause();
- assertSame(expected, observed);
- assertEquals(List.of(suppressed), List.of(observed.getSuppressed()));
- assertEquals(1, ends.size());
- assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus());
- assertSame(original, ends.get(0).executionError());
- if (!inline && (expected instanceof VirtualMachineError || expected instanceof ThreadDeath)) {
- assertTrue(escaped.await(3, TimeUnit.SECONDS));
- assertSame(expected, workerFailure.get());
- }
- } finally {
- callers.shutdownNow();
- workers.shutdownNow();
- assertTrue(callers.awaitTermination(5, TimeUnit.SECONDS));
- assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
-
- @Test
- void partialStartStillUnwindsAllConfiguredEndHooksAndRestoresBothThreads() throws Exception {
- var key = ContextKey.named("partial-start-context");
- var order = new CopyOnWriteArrayList();
- var scopes = new Scope[2];
- var bodyCalls = new AtomicInteger();
- var workers = Executors.newSingleThreadExecutor(task -> new Thread(
- () -> {
- try (var ignored =
- Context.root().with(key, "worker ambient").makeCurrent()) {
- task.run();
- }
- },
- "partial-start-owner"));
- var first = new DurableExecutionPlugin() {
- @Override
- public void onInvocationStart(InvocationInfo info) {
- order.add("start:first");
- scopes[0] = Context.current().with(key, "first").makeCurrent();
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- order.add("end:first");
- scopes[0].close();
- }
- };
- var failing = new DurableExecutionPlugin() {
- @Override
- public void onInvocationStart(InvocationInfo info) {
- order.add("start:failing");
- scopes[1] = Context.current().with(key, "failing").makeCurrent();
- throw new AssertionError("start failed after acquiring context");
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- order.add("end:failing");
- scopes[1].close();
- }
- };
- var unstarted = new DurableExecutionPlugin() {
- @Override
- public void onInvocationStart(InvocationInfo info) {
- order.add("unexpected start");
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- order.add("end:unstarted");
- }
- };
- try (var ignored = Context.current().with(key, "caller ambient").makeCurrent()) {
- var before = Context.current();
- var runner = LocalDurableTestRunner.create(
- String.class,
- (input, context) -> {
- bodyCalls.incrementAndGet();
- return "done";
- },
- DurableConfig.builder()
- .withExecutorService(workers)
- .withPlugins(first, failing, unstarted)
- .build());
- assertEquals(ExecutionStatus.FAILED, runner.run("input").getStatus());
- assertEquals(List.of("start:first", "start:failing", "end:unstarted", "end:failing", "end:first"), order);
- assertEquals(0, bodyCalls.get());
- assertSame(before, Context.current());
- assertEquals(
- "worker ambient",
- workers.submit(() -> Context.current().get(key)).get(5, TimeUnit.SECONDS));
- } finally {
- workers.shutdownNow();
- assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
-
- private static UnrecoverableDurableExecutionException retryError() {
- return new UnrecoverableDurableExecutionException(
- ErrorObject.builder().errorMessage("retry").build(), true);
- }
-
- private static void await(CountDownLatch latch) {
- try {
- assertTrue(latch.await(5, TimeUnit.SECONDS));
- } catch (InterruptedException failure) {
- Thread.currentThread().interrupt();
- throw new AssertionError(failure);
- }
- }
-
- private static void replaceAdapter(MDCAdapter adapter) throws Exception {
- var setter = MDC.class.getDeclaredMethod("setMDCAdapter", MDCAdapter.class);
- setter.setAccessible(true);
- setter.invoke(null, adapter);
- }
-}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java
index 1c61311f4e..73a536a0f9 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java
@@ -99,7 +99,7 @@ public T deserialize(String value, TypeToken type) {
.withSerDes(serializer)
.withCheckpointDelay(Duration.ZERO)
.withPollingStrategy(attempt -> Duration.ofMillis(1))
- .withPlugins(new DurableExecutionPlugin() {
+ .withPlugins(ignoredInfo -> new DurableExecutionPlugin() {
public void onInvocationEnd(InvocationEndInfo info) {
ends.add(info);
}
@@ -215,7 +215,7 @@ void successfulOversizedCheckpointStillRunsAfterAdmissionCloses() throws Excepti
.withDurableExecutionClient(client)
.withExecutorService(workers)
.withCheckpointDelay(Duration.ZERO)
- .withPlugins(new DurableExecutionPlugin() {
+ .withPlugins(ignoredInfo -> new DurableExecutionPlugin() {
public void onInvocationEnd(InvocationEndInfo info) {
ends.add(info);
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java
index 9530e75bdb..2223382656 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java
@@ -78,22 +78,24 @@ protected void afterExecute(Runnable task, Throwable failure) {
.contextExtractor(() -> null)
.enableMdc(false)
.build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(builder, pluginConfig)
- : new InvocationOtelPlugin(builder, pluginConfig);
+ DurableExecutionPluginFactory otel = executionView
+ ? ExecutionOtelPlugin.factory(builder, pluginConfig)
+ : InvocationOtelPlugin.factory(builder, pluginConfig);
var config = DurableConfig.builder()
.withDurableExecutionClient(client)
.withExecutorService(workers)
.withCheckpointDelay(Duration.ZERO)
- .withPlugins(otel, new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- ended.countDown();
- }
+ .withPlugins(
+ ignoredInfo -> new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ended.countDown();
+ }
- public void onOperationEnd(OperationEndInfo info) {
- if (ended.getCount() == 0) lateOperationEnds.incrementAndGet();
- }
- })
+ public void onOperationEnd(OperationEndInfo info) {
+ if (ended.getCount() == 0) lateOperationEnds.incrementAndGet();
+ }
+ },
+ otel)
.build();
try {
var call = caller.submit(() -> DurableExecutor.execute(
@@ -255,9 +257,9 @@ public boolean awaitTermination(long duration, TimeUnit unit) throws Interrupted
.contextExtractor(() -> null)
.enableMdc(false)
.build();
- DurableExecutionPlugin otel = executionView
- ? new ExecutionOtelPlugin(builder, pluginConfig)
- : new InvocationOtelPlugin(builder, pluginConfig);
+ DurableExecutionPluginFactory otel = executionView
+ ? ExecutionOtelPlugin.factory(builder, pluginConfig)
+ : InvocationOtelPlugin.factory(builder, pluginConfig);
Thread firstWorker = null;
for (var round = 0; round < 2; round++) {
var running = new CountDownLatch(1);
@@ -269,11 +271,13 @@ public boolean awaitTermination(long duration, TimeUnit unit) throws Interrupted
.withExecutorService(dispatch)
.withDurableExecutionClient(new LocalMemoryExecutionClient())
.withCheckpointDelay(Duration.ZERO)
- .withPlugins(otel, new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- ended.countDown();
- }
- })
+ .withPlugins(
+ ignoredInfo -> new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ended.countDown();
+ }
+ },
+ otel)
.build();
try {
var call = caller.submit(() -> DurableExecutor.execute(
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java
new file mode 100644
index 0000000000..ed9a21e335
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java
@@ -0,0 +1,27 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+
+/**
+ * Test helper: builds one invocation's plugin instance the way the SDK does.
+ *
+ * A plugin instance serves exactly one invocation, so a test that drives several invocations of an execution creates
+ * one instance per invocation from the same factory — the factory being what the environment owns. The factory is
+ * called with the very {@link InvocationInfo} that {@code onInvocationStart} then receives, exactly as
+ * {@code PluginRunner} does.
+ */
+final class Invocations {
+
+ private Invocations() {}
+
+ /** One invocation's plugin instance, created from the factory and started with the same info. */
+ static DurableExecutionPlugin started(DurableExecutionPluginFactory factory, InvocationInfo info) {
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
+ return plugin;
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvokePropagationIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvokePropagationIntegrationTest.java
index 5b89d7f4b9..dadd9f2c7b 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvokePropagationIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvokePropagationIntegrationTest.java
@@ -12,6 +12,7 @@
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -51,6 +52,7 @@
import software.amazon.lambda.durable.model.DurableExecutionOutput;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.OperationInfo;
import software.amazon.lambda.durable.plugin.PropagationInput;
import software.amazon.lambda.durable.plugin.PropagationMetadata;
@@ -222,8 +224,9 @@ void batchedPublicInvokesCarryTheirOwnActualOperationSpan(boolean executionView,
"1234567890123456",
sampled ? ExtractedContext.Sampling.SAMPLED : ExtractedContext.Sampling.NOT_SAMPLED))
.build();
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
var bothInvokes = new CountDownLatch(2);
var inputs = new CopyOnWriteArrayList();
var startedOperations = new CopyOnWriteArrayList();
@@ -246,7 +249,7 @@ public PropagationMetadata providePropagationMetadata(PropagationInput input) {
return null;
}
};
- try (var fixture = new Fixture(plugin, observer)) {
+ try (var fixture = new Fixture(factory, ignored -> observer)) {
fixture.client.autoComplete = true;
var result = fixture.run((input, context) -> {
try (var parallel = context.parallel(
@@ -375,6 +378,12 @@ private static final class Fixture implements AutoCloseable {
final DurableConfig config;
Fixture(DurableExecutionPlugin... plugins) {
+ this(Arrays.stream(plugins)
+ .map(plugin -> (DurableExecutionPluginFactory) ignored -> plugin)
+ .toArray(DurableExecutionPluginFactory[]::new));
+ }
+
+ Fixture(DurableExecutionPluginFactory... plugins) {
config = DurableConfig.builder()
.withDurableExecutionClient(client)
.withExecutorService(executor)
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java
new file mode 100644
index 0000000000..95ee4c321a
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java
@@ -0,0 +1,167 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Duration;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class LateUserFunctionCleanupTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void fatalDrainWaitsForOriginalAttemptScopeWhilePeerInvocationStaysIsolated(boolean executionView)
+ throws Exception {
+ var fatal = new InternalError("peer fatal");
+ var entered = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var exited = new CountDownLatch(1);
+ var owner = new AtomicReference();
+ var before = new AtomicReference();
+ var afterTask = new AtomicReference();
+ var endedOn = new AtomicReference();
+ var endError = new AtomicReference();
+ var ends = new AtomicInteger();
+ var factories = new AtomicInteger();
+ var firstInvocationEnds = new AtomicInteger();
+ var callers = Executors.newSingleThreadExecutor();
+ var endedInstance = new AtomicInteger();
+ var exporter = InMemorySpanExporter.create();
+ var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder()
+ .enableMdc(true)
+ .contextExtractor(() -> new ExtractedContext(
+ "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ var otel = executionView
+ ? ExecutionOtelPlugin.factory(provider, config)
+ : InvocationOtelPlugin.factory(provider, config);
+ DurableExecutionPluginFactory observer = info -> {
+ var instance = factories.incrementAndGet();
+ return new DurableExecutionPlugin() {
+ @Override
+ public void onInvocationEnd(InvocationEndInfo end) {
+ if (instance == 1) firstInvocationEnds.incrementAndGet();
+ }
+
+ @Override
+ public void onUserFunctionStart(UserFunctionStartInfo start) {
+ if ("peer".equals(start.name())) {
+ owner.set(Thread.currentThread());
+ before.set(Span.current().getSpanContext());
+ }
+ }
+
+ @Override
+ public void onUserFunctionEnd(UserFunctionEndInfo end) {
+ if ("peer".equals(end.name())) {
+ endedOn.set(Thread.currentThread());
+ endError.set(end.error());
+ ends.incrementAndGet();
+ endedInstance.set(instance);
+ }
+ }
+ };
+ };
+ DurableExecutionPluginFactory fault = info -> new DurableExecutionPlugin() {
+ @Override
+ public void onUserFunctionStart(UserFunctionStartInfo start) {
+ if ("trigger".equals(start.name())) throw fatal;
+ }
+ };
+ var workers = new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>()) {
+ @Override
+ public void execute(Runnable task) {
+ super.execute(() -> {
+ try {
+ task.run();
+ } catch (InternalError expected) {
+ // Reusable custom workers make leaked thread-local scope visible after task exit.
+ } finally {
+ if (Thread.currentThread() == owner.get()) {
+ afterTask.set(Span.current().getSpanContext());
+ exited.countDown();
+ }
+ }
+ });
+ }
+ };
+ try {
+ var durable = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(observer, otel, fault)
+ .build();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ var peer = context.stepAsync("peer", String.class, step -> {
+ entered.countDown();
+ while (release.getCount() != 0) {
+ try {
+ release.await();
+ } catch (InterruptedException ignored) {
+ // Hold the actual user body past the invocation's bounded cleanup allowance.
+ }
+ }
+ return "late";
+ });
+ try {
+ assertTrue(entered.await(3, TimeUnit.SECONDS));
+ } catch (InterruptedException failure) {
+ throw new AssertionError(failure);
+ }
+ context.stepAsync("trigger", String.class, step -> "unreachable");
+ return peer.get();
+ },
+ durable);
+ var response = callers.submit(() -> runner.run("input"));
+ assertTrue(entered.await(3, TimeUnit.SECONDS));
+ assertThrows(TimeoutException.class, () -> response.get(700, TimeUnit.MILLISECONDS));
+ assertEquals(0, firstInvocationEnds.get(), "Invocation End must await the original owner");
+ assertEquals(0, ends.get(), "The peer is still held while the invocation drains");
+ var other = LocalDurableTestRunner.create(String.class, (input, context) -> "other", durable);
+ assertEquals(ExecutionStatus.SUCCEEDED, other.run("other").getStatus());
+ assertEquals(2, factories.get(), "a new invocation creates its own plugin while old cleanup is held");
+ release.countDown();
+ assertTrue(exited.await(3, TimeUnit.SECONDS));
+ assertSame(
+ fatal,
+ assertThrows(ExecutionException.class, () -> response.get(3, TimeUnit.SECONDS))
+ .getCause());
+ assertEquals(1, firstInvocationEnds.get());
+ assertAll(
+ () -> assertEquals(1, ends.get(), "the started attempt must retain its own end-hook recipients"),
+ () -> assertEquals(
+ before.get(), afterTask.get(), "the original OTel scope must close on its owner"));
+ assertSame(owner.get(), endedOn.get());
+ assertSame(fatal, endError.get());
+ assertEquals(1, endedInstance.get(), "late cleanup belongs to the original invocation's plugin");
+ assertEquals(2, factories.get(), "owner cleanup does not invoke factories again");
+ assertEquals(
+ 1,
+ exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().contains("peer")
+ && span.getName().contains(" attempt "))
+ .count());
+ } finally {
+ release.countDown();
+ callers.shutdownNow();
+ assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS));
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCompatibilityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java
similarity index 52%
rename from otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCompatibilityTest.java
rename to otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java
index 9cb19b13f0..dd74fccbbc 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCompatibilityTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java
@@ -11,10 +11,8 @@
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.LockSupport;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import software.amazon.awssdk.services.lambda.model.*;
@@ -24,20 +22,22 @@
import software.amazon.lambda.durable.model.DurableExecutionInput;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
-/** Verifies invocation-hook thread affinity, including executors that complete tasks before submission returns. */
-class InvocationEndCompatibilityTest {
+/** Identical public-API timing probe run against main and the candidate. */
+class LegacyEndThreadProbeTest {
@ParameterizedTest
@CsvSource({
"false,false,false", "true,false,false", "false,true,false", "true,true,false",
"false,false,true", "true,false,true", "false,true,true", "true,true,true"
})
- void invocationEndRunsOnHandlerThread(boolean mixed, boolean precompleted, boolean pending) throws Exception {
+ void legacyEndThread(boolean mixed, boolean precompleted, boolean pending) throws Exception {
var local = new ThreadLocal();
- var observations = new ArrayList();
+ var observations = new CopyOnWriteArrayList();
+ var endValues = new ArrayList();
var callerThread = new AtomicReference();
var caller = Executors.newSingleThreadExecutor(task -> {
var thread = daemon(task, "legacy-probe-caller");
@@ -74,8 +74,6 @@ public void execute(Runnable task) {
var startThread = new AtomicReference();
var endThread = new AtomicReference();
var previous = new AtomicReference();
- var endCalls = new AtomicInteger();
- var endOrder = new CopyOnWriteArrayList();
var legacy = new DurableExecutionPlugin() {
@Override
public void onInvocationStart(InvocationInfo info) {
@@ -86,32 +84,22 @@ public void onInvocationStart(InvocationInfo info) {
@Override
public void onInvocationEnd(InvocationEndInfo info) {
- endCalls.incrementAndGet();
- endOrder.add("legacy");
endThread.set(Thread.currentThread());
endValue.set(local.get());
local.remove();
}
};
- var plugins = new ArrayList();
- plugins.add(legacy);
+ var plugins = new ArrayList();
+ plugins.add(info -> legacy);
if (mixed)
- plugins.add(
- 0,
- new InvocationOtelPlugin(
- SdkTracerProvider.builder(),
- OtelPluginConfig.builder().enableMdc(false).build()) {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- endOrder.add("otel");
- super.onInvocationEnd(info);
- }
- });
+ plugins.add(InvocationOtelPlugin.factory(
+ SdkTracerProvider.builder(),
+ OtelPluginConfig.builder().enableMdc(false).build()));
var config = DurableConfig.builder()
.withExecutorService(workers)
.withDurableExecutionClient(new LocalMemoryExecutionClient())
.withCheckpointDelay(Duration.ZERO)
- .withPlugins(plugins.toArray(DurableExecutionPlugin[]::new))
+ .withPlugins(plugins.toArray(DurableExecutionPluginFactory[]::new))
.build();
var result = caller.submit(() -> DurableExecutor.execute(
input(marker),
@@ -134,25 +122,20 @@ public void onInvocationEnd(InvocationEndInfo info) {
pending ? ExecutionStatus.PENDING : ExecutionStatus.SUCCEEDED,
result.get(5, TimeUnit.SECONDS).status());
var after = workers.submit(local::get).get(3, TimeUnit.SECONDS);
- observations.add(new Observation(
- startThread.get(), endThread.get(), endValue.get(), after, previous.get(), endCalls.get()));
- assertEquals(
- mixed ? List.of("legacy", "otel") : List.of("legacy"),
- endOrder,
- "all end hooks unwind registration order and execute once");
+ observations.add(
+ "round=" + round + " start=" + startThread.get().getName()
+ + " end=" + endThread.get().getName() + " endValue=" + endValue.get()
+ + " workerAfter=" + after + " priorWorker=" + previous.get());
+ endValues.add(endValue.get());
} finally {
release.countDown();
}
}
- for (int index = 0; index < observations.size(); index++) {
- var observation = observations.get(index);
- assertEquals(1, observation.endCalls());
- assertSame(observation.startThread(), observation.endThread());
- assertNotSame(callerThread.get(), observation.endThread());
- assertEquals("inv" + (index + 1), observation.endValue());
- assertNull(observation.workerAfter(), "end hook clears its original worker value");
- assertNull(observation.previous(), "reused worker must not retain the prior invocation");
- }
+ System.out.println("LEGACY_END_TRACE mixed=" + mixed + " precompleted=" + precompleted + " pending="
+ + pending + " " + observations);
+ // This controlled success path attaches the legacy continuation before the handler completes.
+ // Precompleted and suspension paths are measured separately, without claiming owner-thread guarantees.
+ if (!precompleted && !pending) assertEquals(List.of("inv1", "inv2"), endValues);
} finally {
caller.shutdownNow();
workers.shutdownNow();
@@ -161,104 +144,6 @@ public void onInvocationEnd(InvocationEndInfo info) {
}
}
- @Test
- void suspendedInvocationWaitsForEndOnTheReusedWorker() throws Exception {
- var local = new ThreadLocal();
- var callerThread = new AtomicReference();
- var callers = Executors.newSingleThreadExecutor(task -> {
- var thread = daemon(task, "scoped-handoff-caller");
- callerThread.set(thread);
- return thread;
- });
- var workers = Executors.newSingleThreadExecutor(task -> daemon(task, "scoped-handoff-worker"));
- try {
- for (var round = 0; round < 3; round++) {
- var marker = "scoped-" + round;
- var scoped = new PausingOtelPlugin();
- var previous = new AtomicReference();
- var observed = new AtomicReference();
- var owner = new AtomicReference();
- var endThread = new AtomicReference();
- var legacy = new DurableExecutionPlugin() {
- @Override
- public void onInvocationStart(InvocationInfo info) {
- owner.set(Thread.currentThread());
- previous.set(local.get());
- local.set(marker);
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- endThread.set(Thread.currentThread());
- observed.set(local.get());
- local.remove();
- }
- };
- var config = DurableConfig.builder()
- .withExecutorService(workers)
- .withDurableExecutionClient(new LocalMemoryExecutionClient())
- .withCheckpointDelay(Duration.ZERO)
- .withPlugins(scoped, legacy)
- .build();
- var result = callers.submit(() -> DurableExecutor.execute(
- input(marker),
- null,
- TypeToken.get(String.class),
- (value, context) -> {
- context.wait("pause", Duration.ofSeconds(1));
- return "done";
- },
- config));
- try {
- assertTrue(scoped.closeEntered.await(3, TimeUnit.SECONDS));
- assertThrows(
- TimeoutException.class,
- () -> result.get(600, TimeUnit.MILLISECONDS),
- "a blocked end hook has no fallback to the caller thread");
- scoped.releaseClose.countDown();
- assertEquals(
- ExecutionStatus.PENDING,
- result.get(5, TimeUnit.SECONDS).status());
- assertSame(owner.get(), endThread.get());
- assertEquals(marker, observed.get());
- assertNull(previous.get(), "The prior invocation's end hook must clear the reused worker");
- assertNull(workers.submit(local::get).get(3, TimeUnit.SECONDS));
- } finally {
- scoped.releaseClose.countDown();
- }
- }
- } finally {
- callers.shutdownNow();
- workers.shutdownNow();
- assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS));
- assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
- }
- }
-
- private static final class PausingOtelPlugin extends InvocationOtelPlugin {
- private final CountDownLatch closeEntered = new CountDownLatch(1);
- private final CountDownLatch releaseClose = new CountDownLatch(1);
-
- private PausingOtelPlugin() {
- super(
- SdkTracerProvider.builder(),
- OtelPluginConfig.builder().enableMdc(false).build());
- }
-
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- try {
- super.onInvocationEnd(info);
- } finally {
- closeEntered.countDown();
- await(releaseClose);
- }
- }
- }
-
- private record Observation(
- Thread startThread, Thread endThread, String endValue, String workerAfter, String previous, int endCalls) {}
-
private static void awaitCallerJoin(Thread caller) {
var end = System.nanoTime() + TimeUnit.SECONDS.toNanos(3);
while (System.nanoTime() < end) {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassScopeCompatibilityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java
similarity index 72%
rename from otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassScopeCompatibilityTest.java
rename to otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java
index 2926a3506b..e78a66f6e7 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassScopeCompatibilityTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java
@@ -8,23 +8,18 @@
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.time.Instant;
import javax.tools.ToolProvider;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
-import software.amazon.lambda.durable.plugin.InvocationInfo;
-import software.amazon.lambda.durable.plugin.InvocationStatus;
-class LegacySubclassScopeCompatibilityTest {
+class LegacySubclassMigrationTest {
@TempDir
Path directory;
@ParameterizedTest
@CsvSource({"Invocation,false", "Invocation,true", "Execution,false", "Execution,true"})
- void preservesOldSubclassMethodsAndUnrelatedDefaults(String view, boolean ownMethod) throws Exception {
+ void requiresFactoryMigrationForOldCompiledSubclasses(String view, boolean ownMethod) throws Exception {
var baseline = Files.createDirectories(directory.resolve("baseline"));
var classes = Files.createDirectories(directory.resolve("classes"));
var baseName = view + "OtelPlugin";
@@ -64,17 +59,12 @@ public void originalCall() throws Exception { try (var scope = openHandlerScope(
source.toString()));
try (var loader = new URLClassLoader(
new URL[] {classes.toUri().toURL()}, getClass().getClassLoader())) {
- var type = loader.loadClass("LegacySubclass");
- var plugin = (DurableExecutionPlugin) type.getConstructor().newInstance();
- plugin.onInvocationStart(new InvocationInfo("req", "arn", true, Instant.now()));
- plugin.onInvocationEnd(new InvocationEndInfo("req", "arn", true, InvocationStatus.SUCCEEDED, null));
- assertEquals(0, type.getField("opened").get(null), "SDK must not invoke an unrelated subclass resource");
- type.getMethod("originalCall").invoke(plugin);
- assertEquals(
- 1, type.getField("opened").get(null), "new SDK members must not shadow an application default");
- assertEquals(1, type.getField("closed").get(null));
+ assertThrows(
+ IncompatibleClassChangeError.class,
+ () -> loader.loadClass("LegacySubclass"),
+ "3.x view implementations are final; old subclasses must migrate to factories");
}
- assertEquals(
+ assertNotEquals(
0,
compiler.run(
null,
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java
index 47daaf1820..c59bf3f698 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java
@@ -8,7 +8,9 @@
import java.lang.reflect.Proxy;
import java.lang.reflect.UndeclaredThrowableException;
import java.time.Duration;
+import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
@@ -17,15 +19,14 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.MDC;
import org.slf4j.spi.MDCAdapter;
import software.amazon.lambda.durable.DurableConfig;
@@ -35,6 +36,7 @@
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.util.ExceptionHelper;
class MdcFailureBoundaryTest {
@Test
@@ -60,7 +62,7 @@ void workerCaptureFailureDoesNotEndAnInvocationThatNeverStarted() throws Excepti
},
DurableConfig.builder()
.withExecutorService(executor)
- .withPlugins(plugin)
+ .withPlugins(info -> plugin)
.build());
var result = runner.run("input");
assertTrue(injected.get());
@@ -79,239 +81,371 @@ void workerCaptureFailureDoesNotEndAnInvocationThatNeverStarted() throws Excepti
}
}
+ @SuppressWarnings("removal")
@ParameterizedTest
- @CsvSource({
- "false,false,exception", "true,false,exception", "false,true,exception", "true,true,exception",
- "false,false,fatal", "true,false,fatal", "false,true,fatal", "true,true,fatal",
- "false,false,assertion", "true,false,assertion", "false,true,assertion", "true,true,assertion",
- "false,false,linkage", "true,false,linkage", "false,true,linkage", "true,true,linkage"
- })
- void workerMdcRestorationFailureCannotStrandTheInvocation(boolean ambientMdc, boolean suspend, String failureKind)
+ @CsvSource({"false,false", "true,false", "false,true", "true,true"})
+ void workerRestoreFatalAfterEndStillReachesTheCaller(boolean direct, boolean threadDeath) throws Exception {
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("outer worker restore");
+ var observed = runOuterFailure(direct, null, fatal);
+ assertSame(fatal, observed.failure());
+ assertEquals(1, observed.ends());
+ assertEquals(InvocationStatus.SUCCEEDED, observed.end().invocationStatus());
+ assertNull(observed.end().executionError(), "The earlier End snapshot is not rewritten by later restoration");
+ }
+
+ @ParameterizedTest
+ @CsvSource({"false,false", "true,false", "false,true", "true,true"})
+ void restorationFatalIsReportedAfterFailedBodyWithoutReplacingEarlierFatal(boolean direct, boolean bodyFatal)
+ throws Exception {
+ Throwable primary = bodyFatal ? new InternalError("primary body") : new IllegalStateException("primary body");
+ var secondary = new InternalError("restoration fatal");
+ var observed = runOuterFailure(direct, primary, secondary);
+ var selected = bodyFatal ? primary : secondary;
+ assertSame(selected, observed.failure());
+ assertEquals(1, observed.ends());
+ assertSame(primary, observed.end().executionError());
+ assertEquals(
+ bodyFatal ? InvocationStatus.RETRYING : InvocationStatus.FAILED,
+ observed.end().invocationStatus());
+ if (bodyFatal) assertTrue(Arrays.asList(selected.getSuppressed()).contains(secondary));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void ordinaryWorkerRestorationFailureRetainsSuccessfulOutcome(boolean direct) throws Exception {
+ var observed = runOuterFailure(direct, null, new IllegalStateException("adapter refused restoration"));
+ assertNull(observed.failure());
+ assertEquals(ExecutionStatus.SUCCEEDED, observed.status());
+ assertEquals(1, observed.ends());
+ assertEquals(InvocationStatus.SUCCEEDED, observed.end().invocationStatus());
+ }
+
+ private static OuterOutcome runOuterFailure(boolean direct, Throwable bodyFailure, Throwable restoreFailure)
throws Exception {
var original = MDC.getMDCAdapter();
+ var callerBefore = MDC.getCopyOfContextMap();
var injected = new AtomicBoolean();
- var plugin = new RecordingPlugin();
- Throwable failure = switch (failureKind) {
- case "fatal" -> new InternalError("worker MDC restoration failed");
- case "assertion" -> new AssertionError("worker MDC restoration failed");
- case "linkage" -> new NoClassDefFoundError("worker MDC restoration failed");
- default -> new IllegalStateException("worker MDC restoration failed");
+ var ends = new AtomicInteger();
+ var endInfo = new AtomicReference();
+ ExecutorService executor = direct ? new InlineExecutor() : new CompletingWorker();
+ DurableExecutionPlugin plugin = new DurableExecutionPlugin() {
+ @Override
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.incrementAndGet();
+ endInfo.set(info);
+ }
};
- var ownerFailure = new AtomicReference();
- var observed = new CountDownLatch(1);
- var workers = Executors.newSingleThreadExecutor(task -> {
- var thread = new Thread(
- () -> {
- if (ambientMdc) MDC.put("ambient", "saved");
- else MDC.clear();
- task.run();
+ try {
+ replaceAdapter(proxy(original, name -> {
+ if (isOuterRestore(name) && injected.compareAndSet(false, true)) throw restoreFailure;
+ return null;
+ }));
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ if (bodyFailure != null) ExceptionHelper.sneakyThrow(bodyFailure);
+ return "done";
},
- "mdc-restoration-worker");
- thread.setUncaughtExceptionHandler((owner, error) -> {
- ownerFailure.set(error);
- observed.countDown();
+ DurableConfig.builder()
+ .withExecutorService(executor)
+ .withPlugins(info -> plugin)
+ .build());
+ Error observed = null;
+ ExecutionStatus status = null;
+ String errorMessage = null;
+ try {
+ var result = runner.run("input");
+ status = result.getStatus();
+ errorMessage =
+ result.getError().map(error -> error.errorMessage()).orElse(null);
+ } catch (Error failure) {
+ observed = failure;
+ }
+ assertTrue(injected.get(), "The actual outer worker restoration must reach the injected failure");
+ return new OuterOutcome(observed, status, errorMessage, endInfo.get(), ends.get());
+ } finally {
+ try {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS));
+ } finally {
+ replaceAdapter(original);
+ if (callerBefore == null) MDC.clear();
+ else MDC.setContextMap(callerBefore);
+ }
+ }
+ }
+
+ private record OuterOutcome(
+ Error failure, ExecutionStatus status, String errorMessage, InvocationEndInfo end, int ends) {}
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false", "true,true"})
+ void postDispatchWorkerFatalEscapesOwnerWithoutRepeatingEnd(boolean threadDeath, boolean failedBody)
+ throws Exception {
+ var original = MDC.getMDCAdapter();
+ var callerBefore = MDC.getCopyOfContextMap();
+ var startEntered = new CountDownLatch(1);
+ var releaseStart = new CountDownLatch(1);
+ var restoreEntered = new CountDownLatch(1);
+ var releaseRestore = new CountDownLatch(1);
+ var ownerObserved = new CountDownLatch(1);
+ var ownerFailure = new AtomicReference();
+ var ended = new AtomicInteger();
+ var bodyFailure = new IllegalStateException("ordinary body before late restoration");
+ var injected = new AtomicBoolean();
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("late outer restore");
+ var worker = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "late-mdc-owner");
+ thread.setUncaughtExceptionHandler((owner, failure) -> {
+ ownerFailure.set(failure);
+ ownerObserved.countDown();
});
return thread;
});
+ var caller = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "late-mdc-caller");
+ return thread;
+ });
+ DurableExecutionPlugin plugin = new DurableExecutionPlugin() {
+ @Override
+ public void onInvocationStart(InvocationInfo info) {
+ startEntered.countDown();
+ awaitGate(releaseStart);
+ }
+
+ @Override
+ public void onInvocationEnd(InvocationEndInfo info) {
+ assertEquals(
+ failedBody ? InvocationStatus.FAILED : InvocationStatus.SUCCEEDED, info.invocationStatus());
+ assertSame(failedBody ? bodyFailure : null, info.executionError());
+ ended.incrementAndGet();
+ }
+ };
try {
- replaceAdapter(proxy(original, method -> {
- if (Thread.currentThread().getName().equals("mdc-restoration-worker")
- && plugin.ends.get() == 1
- && method.equals(ambientMdc ? "setContextMap" : "clear")
- && injected.compareAndSet(false, true)) throw failure;
+ replaceAdapter(proxy(original, name -> {
+ if (isOuterRestore(name) && injected.compareAndSet(false, true)) {
+ restoreEntered.countDown();
+ awaitGate(releaseRestore);
+ throw fatal;
+ }
return null;
}));
var runner = LocalDurableTestRunner.create(
String.class,
(input, context) -> {
- if (suspend) context.wait("resume", Duration.ofSeconds(1));
+ if (failedBody) throw bodyFailure;
return "done";
},
DurableConfig.builder()
- .withExecutorService(workers)
- .withPlugins(plugin)
+ .withExecutorService(worker)
+ .withPlugins(info -> plugin)
.build());
- if (failureKind.equals("fatal")) {
- assertSame(failure, assertThrows(Error.class, () -> runner.run("input")));
- } else {
- var result = runner.run("input");
- assertEquals(suspend ? ExecutionStatus.PENDING : ExecutionStatus.SUCCEEDED, result.getStatus());
- }
- assertEquals(1, plugin.starts.get());
- assertEquals(1, plugin.ends.get());
- assertTrue(injected.get());
- assertTrue(observed.await(3, TimeUnit.SECONDS));
- assertSame(failure, ownerFailure.get(), "restoration failure still escapes its worker after End");
+ var result = caller.submit(() -> runner.run("input"));
+ awaitGate(startEntered);
+ releaseStart.countDown();
+ awaitGate(restoreEntered);
+ assertThrows(TimeoutException.class, () -> result.get(150, TimeUnit.MILLISECONDS));
+ assertEquals(1, ended.get(), "The owner has delivered End, but return still awaits MDC restoration");
+ releaseRestore.countDown();
+ assertSame(
+ fatal,
+ assertThrows(ExecutionException.class, () -> result.get(2, TimeUnit.SECONDS))
+ .getCause());
+ awaitGate(ownerObserved);
+ assertSame(fatal, ownerFailure.get());
+ assertEquals(1, ended.get());
} finally {
- workers.shutdownNow();
- assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
- replaceAdapter(original);
+ releaseStart.countDown();
+ releaseRestore.countDown();
+ try {
+ caller.shutdownNow();
+ worker.shutdownNow();
+ assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS));
+ assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS));
+ } finally {
+ replaceAdapter(original);
+ if (callerBefore == null) MDC.clear();
+ else MDC.setContextMap(callerBefore);
+ }
}
}
- static Stream postEndFatalCases() {
- return Stream.of(false, true)
- .flatMap(inline -> Stream.of(false, true)
- .flatMap(suspend -> Stream.of(false, true)
- .flatMap(death -> Stream.of("direct", "completion", "execution", "reflection", "proxy")
- .map(wrapper -> Arguments.of(inline, suspend, death, wrapper)))));
+ private static boolean isOuterRestore(String method) {
+ if (!method.equals("clear") && !method.equals("setContextMap")) return false;
+ var frames = Arrays.stream(Thread.currentThread().getStackTrace()).toList();
+ return frames.stream()
+ .anyMatch(frame -> frame.getClassName().endsWith(".DurableExecutor")
+ && frame.getMethodName().startsWith("lambda$restoreMdcOnClose"))
+ && frames.stream()
+ .noneMatch(frame -> frame.getClassName().endsWith(".DurableExecutor")
+ && frame.getMethodName().equals("fireOnInvocationEnd"));
+ }
+
+ private static void awaitGate(CountDownLatch gate) {
+ try {
+ assertTrue(gate.await(5, TimeUnit.SECONDS), "Controlled MDC gate was not released");
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+
+ private static final class InlineExecutor extends AbstractExecutorService {
+ private boolean shutdown;
+
+ @Override
+ public void execute(Runnable command) {
+ command.run();
+ }
+
+ @Override
+ public void shutdown() {
+ shutdown = true;
+ }
+
+ @Override
+ public List shutdownNow() {
+ shutdown = true;
+ return List.of();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return shutdown;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return shutdown;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return shutdown;
+ }
}
- @SuppressWarnings("removal")
@ParameterizedTest
- @MethodSource("postEndFatalCases")
- void postEndFatalReachesCallerAndWorkerWithoutChangingEnd(
- boolean inline, boolean suspend, boolean death, String wrapper) throws Exception {
- Error fatal = death ? new ThreadDeath() : new InternalError("post-End restore fatal");
- Throwable failure = switch (wrapper) {
- case "completion" -> new CompletionException(fatal);
- case "execution" -> new ExecutionException(fatal);
- case "reflection" -> new InvocationTargetException(fatal);
- case "proxy" -> new UndeclaredThrowableException(fatal);
- default -> fatal;
- };
- exercisePostEndFailure(inline, suspend, failure, fatal, null);
+ @CsvSource({
+ "getCopyOfContextMap,false",
+ "clear,false",
+ "setContextMap,false",
+ "getCopyOfContextMap,true",
+ "clear,true",
+ "setContextMap,true"
+ })
+ void nonfatalMdcBoundaryFailuresPreserveTheirPhaseOutcome(String method, boolean suspend) throws Exception {
+ runEndFailure(method, suspend, new IllegalStateException("end MDC failure"));
}
@SuppressWarnings("removal")
@ParameterizedTest
@CsvSource({
- "false,false,false",
- "false,false,true",
- "false,true,false",
- "false,true,true",
- "true,false,false",
- "true,false,true",
- "true,true,false",
- "true,true,true"
+ "getCopyOfContextMap,false",
+ "clear,false",
+ "setContextMap,false",
+ "getCopyOfContextMap,true",
+ "clear,true",
+ "setContextMap,true"
})
- void fatalDiagnosticRetainsIdentityAndIsReadOnce(boolean inline, boolean suspend, boolean death) throws Exception {
- Error fatal = death ? new ThreadDeath() : new InternalError("fatal diagnostic");
+ void fatalMdcBoundaryFailuresStillEscape(String method, boolean wrapped) throws Exception {
+ Error fatal = wrapped ? new ThreadDeath() : new VirtualMachineError("fatal end MDC failure") {};
+ runEndFailure(method, false, wrapped ? new CompletionException(new ExecutionException(fatal)) : fatal, fatal);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void unreadableOrdinaryCausePreservesSuccessAndSuspension(boolean suspend) throws Exception {
var reads = new AtomicInteger();
- var wrapper = new CompletionException("diagnostic", null) {
+ var failure = new CompletionException("unreadable", null) {
@Override
public synchronized Throwable getCause() {
reads.incrementAndGet();
- throw fatal;
+ throw new IllegalStateException("diagnostic unavailable");
}
};
- exercisePostEndFailure(inline, suspend, wrapper, fatal, null);
+ runEndFailure("setContextMap", suspend, failure);
assertEquals(1, reads.get());
}
@SuppressWarnings("removal")
@ParameterizedTest
- @CsvSource({"false,false", "false,true", "true,false", "true,true"})
- void earlierEndFatalRemainsPrimaryAndRetainsRestorationDiagnostic(boolean inline, boolean death) throws Exception {
- Error primary = death ? new ThreadDeath() : new InternalError("End fatal");
- var restoration = new InternalError("later restoration fatal");
- exercisePostEndFailure(inline, false, restoration, primary, primary);
- assertArrayEquals(new Throwable[] {restoration}, primary.getSuppressed());
+ @ValueSource(booleans = {false, true})
+ void fatalCauseAccessorEscapesWithOriginalIdentity(boolean threadDeath) throws Exception {
+ var reads = new AtomicInteger();
+ Error fatal = threadDeath ? new ThreadDeath() : new VirtualMachineError("fatal accessor") {};
+ var failure = new CompletionException("unreadable", null) {
+ @Override
+ public synchronized Throwable getCause() {
+ reads.incrementAndGet();
+ throw fatal;
+ }
+ };
+ runEndFailure("setContextMap", false, failure, fatal);
+ assertEquals(1, reads.get());
}
@SuppressWarnings("removal")
@ParameterizedTest
- @CsvSource({
- "false,false,false",
- "false,false,true",
- "false,true,false",
- "false,true,true",
- "true,false,false",
- "true,false,true",
- "true,true,false",
- "true,true,true"
- })
- void sameFatalAtEndAndRestorationRetainsOwnerIdentity(boolean inline, boolean suspend, boolean death)
- throws Exception {
- Error fatal = death ? new ThreadDeath() : new InternalError("same End/restore fatal");
- exercisePostEndFailure(inline, suspend, fatal, fatal, fatal);
- assertEquals(0, fatal.getSuppressed().length, "Never suppress the fatal onto itself");
+ @CsvSource({"reflection,false", "reflection,true", "proxy,false", "proxy,true"})
+ void jdkReflectiveWrappersKeepFatalIdentity(String wrapper, boolean threadDeath) throws Exception {
+ Error fatal = threadDeath ? new ThreadDeath() : new VirtualMachineError("wrapped fatal") {};
+ Throwable failure = wrapper.equals("reflection")
+ ? new InvocationTargetException(fatal)
+ : new UndeclaredThrowableException(fatal);
+ runEndFailure("clear", false, failure, fatal);
}
- @ParameterizedTest
- @CsvSource({"false,false", "false,true", "true,false", "true,true"})
- void restorationFatalRetainsEarlierNonfatalEndFailure(boolean inline, boolean wrapped) throws Exception {
- var earlier = new AssertionError("ordinary End failure");
- var fatal = new InternalError("later restoration fatal");
- exercisePostEndFailure(inline, false, wrapped ? new CompletionException(fatal) : fatal, fatal, earlier);
- assertArrayEquals(new Throwable[] {earlier}, fatal.getSuppressed());
+ @Test
+ void cyclicCauseIsReadOnlyOnceAndPreservesOutcome() throws Exception {
+ var reads = new AtomicInteger();
+ var cycle = new CompletionException("cycle", null) {
+ @Override
+ public synchronized Throwable getCause() {
+ reads.incrementAndGet();
+ return this;
+ }
+ };
+ runEndFailure("setContextMap", false, cycle);
+ assertEquals(1, reads.get());
+ }
+
+ @Test
+ void ordinaryApplicationCauseIsNotTreatedAsATransportWrapper() throws Exception {
+ runEndFailure("clear", false, new IllegalStateException("ordinary diagnostic", new VirtualMachineError() {}));
}
- private static void exercisePostEndFailure(
- boolean inline, boolean suspend, Throwable failure, Error expectedFatal, Error endFailure)
+ private static void runEndFailure(String method, boolean suspend, Throwable failure) throws Exception {
+ runEndFailure(method, suspend, failure, null);
+ }
+
+ private static void runEndFailure(String method, boolean suspend, Throwable failure, Error expectedFatal)
throws Exception {
var original = MDC.getMDCAdapter();
+ var callerBefore = MDC.getCopyOfContextMap();
var injected = new AtomicBoolean();
- var plugin = new RecordingPlugin() {
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- super.onInvocationEnd(info);
- if (endFailure != null) throw endFailure;
- }
- };
+ var plugin = new RecordingPlugin();
var ownerFailure = new AtomicReference();
- var owner = new AtomicReference();
- var escaped = new CountDownLatch(1);
- var caller = Executors.newSingleThreadExecutor(task -> new Thread(task, "post-end-caller"));
- var delegate = Executors.newSingleThreadExecutor(task -> {
- var thread = new Thread(task, "post-end-worker");
- thread.setUncaughtExceptionHandler((worker, error) -> {
+ var ownerExited = new CountDownLatch(1);
+ var worker = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "mdc-boundary-owner");
+ thread.setUncaughtExceptionHandler((owner, error) -> {
ownerFailure.set(error);
- escaped.countDown();
+ ownerExited.countDown();
});
return thread;
});
- var executor = new AbstractExecutorService() {
- @Override
- public void execute(Runnable task) {
- Runnable owned = () -> {
- owner.set(Thread.currentThread());
- if (suspend) MDC.put("ambient", "saved");
- else MDC.clear();
- task.run();
- };
- if (inline) {
- try {
- owned.run();
- } catch (Throwable error) {
- ownerFailure.set(error);
- escaped.countDown();
- throw error;
- }
- } else delegate.execute(owned);
- }
-
- @Override
- public void shutdown() {
- delegate.shutdown();
- }
-
- @Override
- public List shutdownNow() {
- return delegate.shutdownNow();
- }
-
- @Override
- public boolean isShutdown() {
- return delegate.isShutdown();
- }
-
- @Override
- public boolean isTerminated() {
- return delegate.isTerminated();
- }
-
- @Override
- public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
- return delegate.awaitTermination(timeout, unit);
- }
- };
+ boolean capture = method.equals("getCopyOfContextMap");
try {
- replaceAdapter(proxy(original, method -> {
- if (Thread.currentThread() == owner.get()
- && plugin.ends.get() == 1
- && method.equals(suspend ? "setContextMap" : "clear")
- && injected.compareAndSet(false, true)) throw failure;
+ replaceAdapter(proxy(original, name -> {
+ var inCapture = Arrays.stream(Thread.currentThread().getStackTrace())
+ .anyMatch(frame -> frame.getClassName().endsWith(".DurableExecutor")
+ && frame.getMethodName().equals("restoreMdcOnClose"));
+ boolean boundary = capture ? inCapture : isOuterRestore(name);
+ if (boundary && name.equals(method) && injected.compareAndSet(false, true)) throw failure;
+ if (inCapture && name.equals("getCopyOfContextMap"))
+ return method.equals("setContextMap") ? Map.of("ambient", "saved") : NullSnapshot.INSTANCE;
return null;
}));
var runner = LocalDurableTestRunner.create(
@@ -321,38 +455,43 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE
return "done";
},
DurableConfig.builder()
- .withExecutorService(executor)
- .withPlugins(plugin)
+ .withExecutorService(worker)
+ .withPlugins(info -> plugin)
.build());
- var observation = caller.submit(() -> {
- try {
- return (Object) runner.run("input");
- } catch (Throwable error) {
- return error;
- }
- });
- assertSame(
- expectedFatal,
- observation.get(3, TimeUnit.SECONDS),
- "caller receives original fatal without hanging");
- assertTrue(escaped.await(3, TimeUnit.SECONDS), "fatal escapes actual worker after observation settlement");
- assertSame(expectedFatal, ownerFailure.get());
- assertTrue(injected.get());
- assertEquals(1, plugin.starts.get());
- assertEquals(1, plugin.ends.get());
- assertEquals(suspend ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED, plugin.status.get());
+ if (expectedFatal != null) {
+ assertSame(expectedFatal, assertThrows(Error.class, () -> runner.run("input")));
+ } else {
+ var result = runner.run("input");
+ assertEquals(
+ capture
+ ? ExecutionStatus.FAILED
+ : suspend ? ExecutionStatus.PENDING : ExecutionStatus.SUCCEEDED,
+ result.getStatus());
+ }
+ assertTrue(injected.get(), "The actual outer capture/restoration boundary must reach the adapter failure");
+ assertEquals(capture ? 0 : 1, plugin.starts.get());
+ assertEquals(capture ? 0 : 1, plugin.ends.get());
+ if (!capture || expectedFatal != null) {
+ assertTrue(
+ ownerExited.await(3, TimeUnit.SECONDS), "Cleanup/fatal failure must escape the actual worker");
+ assertSame(expectedFatal != null ? expectedFatal : failure, ownerFailure.get());
+ }
} finally {
- caller.shutdownNow();
- executor.shutdownNow();
try {
- assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS));
- assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS));
+ worker.shutdownNow();
+ assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS));
} finally {
replaceAdapter(original);
+ if (callerBefore == null) MDC.clear();
+ else MDC.setContextMap(callerBefore);
}
}
}
+ private enum NullSnapshot {
+ INSTANCE
+ }
+
@FunctionalInterface
private interface Fault {
Object apply(String method) throws Throwable;
@@ -362,6 +501,7 @@ private static MDCAdapter proxy(MDCAdapter delegate, Fault fault) {
return (MDCAdapter) Proxy.newProxyInstance(
MDCAdapter.class.getClassLoader(), new Class[] {MDCAdapter.class}, (proxy, method, args) -> {
var value = fault.apply(method.getName());
+ if (value == NullSnapshot.INSTANCE) return null;
if (value != null) return value;
try {
return method.invoke(delegate, args);
@@ -377,10 +517,9 @@ private static void replaceAdapter(MDCAdapter adapter) throws Exception {
setter.invoke(null, adapter);
}
- private static class RecordingPlugin implements DurableExecutionPlugin {
+ private static final class RecordingPlugin implements DurableExecutionPlugin {
final AtomicInteger starts = new AtomicInteger();
final AtomicInteger ends = new AtomicInteger();
- final AtomicReference status = new AtomicReference<>();
@Override
public void onInvocationStart(InvocationInfo info) {
@@ -389,7 +528,6 @@ public void onInvocationStart(InvocationInfo info) {
@Override
public void onInvocationEnd(InvocationEndInfo info) {
- status.set(info.invocationStatus());
ends.incrementAndGet();
}
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFatalInitializationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFatalInitializationTest.java
index 475e0f73d7..b49eb83b46 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFatalInitializationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFatalInitializationTest.java
@@ -238,7 +238,7 @@ public void onInvocationEnd(InvocationEndInfo info) {
}
};
var configuration =
- DurableConfig.builder().withExecutorService(workers).withPlugins(plugin);
+ DurableConfig.builder().withExecutorService(workers).withPlugins(invocation -> plugin);
if (overrideSerDes != null) configuration.withSerDes(overrideSerDes);
var runner = LocalDurableTestRunner.create(
String.class,
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java
index 9c4399817e..88c9a50c43 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java
@@ -3,11 +3,14 @@
package software.amazon.lambda.durable.otel;
import static org.junit.jupiter.api.Assertions.*;
+import static software.amazon.lambda.durable.otel.Invocations.started;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.SpanData;
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
import java.time.Instant;
+import java.util.List;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.slf4j.MDC;
@@ -56,14 +59,14 @@ void inject_withNoActiveSpan_doesNotSetMdcFields() {
void plugin_withMdcEnabled_setsFieldsInMdc() {
var spanExporter = InMemorySpanExporter.create();
- var plugin = new InvocationOtelPlugin(
+ var pluginFactory = InvocationOtelPlugin.factory(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
OtelPluginConfig.builder()
.contextExtractor(() -> null)
.enableMdc(true)
.build());
- plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-mdc-test", true, Instant.now()));
+ var plugin = started(pluginFactory, new InvocationInfo("req-1", "arn:exec-mdc-test", true, Instant.now()));
plugin.onUserFunctionStart(
new UserFunctionStartInfo("op-1", "step", "STEP", "Step", null, Instant.now(), false, 1));
@@ -99,4 +102,41 @@ void plugin_withMdcEnabled_setsFieldsInMdc() {
assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID));
assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED));
}
+
+ @Test
+ void logCorrelationFollowsEachInvocationsOwnInstance() {
+ // Regression guard taken from the Python port of this refactor: there a log filter installed by the first
+ // invocation's plugin outlived that plugin and kept querying the discarded instance, so log correlation
+ // silently stopped after the first invocation. Java correlates through the SLF4J MDC, written by the hooks of
+ // whichever instance is serving the invocation, so every invocation publishes its own execution trace. This
+ // test pins that down across two invocations served by two instances of one factory.
+ var spanExporter = InMemorySpanExporter.create();
+ var pluginFactory = InvocationOtelPlugin.factory(
+ SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)),
+ OtelPluginConfig.builder()
+ .contextExtractor(() -> null)
+ .enableMdc(true)
+ .build());
+
+ var first = started(pluginFactory, new InvocationInfo("req-1", "arn:exec-a", true, Instant.now()));
+ var firstTraceId = MDC.get(MdcSpanEnricher.MDC_TRACE_ID);
+ first.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec-a", true, InvocationStatus.SUCCEEDED, null));
+ assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_ID), "an invocation clears the correlation it set");
+
+ var second = started(pluginFactory, new InvocationInfo("req-2", "arn:exec-b", true, Instant.now()));
+ var secondTraceId = MDC.get(MdcSpanEnricher.MDC_TRACE_ID);
+ second.onInvocationEnd(new InvocationEndInfo("req-2", "arn:exec-b", true, InvocationStatus.SUCCEEDED, null));
+
+ assertNotNull(firstTraceId);
+ assertNotNull(secondTraceId, "log correlation must not stop after the first invocation");
+ assertNotEquals(firstTraceId, secondTraceId, "each instance publishes its own execution trace");
+ var invocationTraceIds = spanExporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("Invocation"))
+ .map(SpanData::getTraceId)
+ .toList();
+ assertEquals(
+ List.of(firstTraceId, secondTraceId),
+ invocationTraceIds,
+ "the correlated trace ID is the one on that invocation's own Invocation span");
+ }
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcWorkerCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcWorkerCleanupTest.java
new file mode 100644
index 0000000000..35696858f5
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcWorkerCleanupTest.java
@@ -0,0 +1,362 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.*;
+import org.slf4j.MDC;
+import org.slf4j.helpers.BasicMDCAdapter;
+import org.slf4j.spi.MDCAdapter;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.testing.TestResult;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+@Timeout(15)
+class MdcWorkerCleanupTest {
+ private static final String MARKER = "invocation-cleanup-marker";
+
+ static Stream cleanupFatals() {
+ return Stream.of(false, true)
+ .flatMap(direct -> Stream.of(false, true)
+ .flatMap(death -> Stream.of("direct", "completion", "execution", "reflection", "proxy")
+ .map(wrapper -> Arguments.of(direct, death, wrapper))));
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @MethodSource("cleanupFatals")
+ void ordinaryBodyFailureDoesNotHideCleanupFatal(boolean direct, boolean death, String wrapper) throws Exception {
+ var body = new IllegalStateException("ordinary handler failure");
+ Error fatal = death ? new ThreadDeath() : new InternalError("simulated restore fatal");
+ try (var fixture = new Fixture(direct, wrap(fatal, wrapper), null, body)) {
+ assertSame(fatal, assertThrows(Error.class, fixture::run));
+ fixture.assertEnd(InvocationStatus.FAILED, body);
+ // End observed the body failure before outer MDC restoration discovered this fatal.
+ fixture.assertWorkerFailure(fatal);
+ assertEquals(0, fixture.fallbackClears.get());
+ }
+ }
+
+ static Stream earlierFatals() {
+ return Stream.of(false, true)
+ .flatMap(direct -> Stream.of(false, true)
+ .flatMap(same ->
+ Stream.of("direct", "completion").map(wrapper -> Arguments.of(direct, same, wrapper))));
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @MethodSource("earlierFatals")
+ void earlierBodyFatalKeepsIdentityWhenRestorationFails(boolean direct, boolean same, String wrapper)
+ throws Exception {
+ Error first = new InternalError("first handler fatal");
+ Error cleanup = same ? first : new ThreadDeath();
+ var body = wrap(first, wrapper);
+ try (var fixture = new Fixture(direct, cleanup, null, body)) {
+ assertSame(first, assertThrows(Error.class, fixture::run));
+ fixture.assertEnd(InvocationStatus.RETRYING, first);
+ if (!same) assertTrue(Arrays.asList(first.getSuppressed()).contains(cleanup));
+ assertTrue(Arrays.stream(first.getSuppressed())
+ .noneMatch(value -> value == first || value instanceof IllegalArgumentException));
+ if (same)
+ assertFalse(
+ Arrays.asList(first.getSuppressed()).contains(body),
+ "Do not suppress the same fatal's transport wrapper");
+ fixture.assertWorkerFailure(first);
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false", "true,true"})
+ void ordinaryRestorationStillClearsAfterABodyFailure(boolean direct, boolean bodyFatal) throws Exception {
+ Throwable body =
+ bodyFatal ? new InternalError("earlier body fatal") : new IllegalStateException("ordinary body");
+ var cleanup = new IllegalStateException("ordinary restore failure");
+ try (var fixture = new Fixture(direct, cleanup, null, body)) {
+ if (bodyFatal) assertSame(body, assertThrows(Error.class, fixture::run));
+ else assertEquals(ExecutionStatus.FAILED, fixture.run().getStatus());
+ fixture.assertEnd(bodyFatal ? InvocationStatus.RETRYING : InvocationStatus.FAILED, body);
+ if (bodyFatal) assertTrue(Arrays.asList(body.getSuppressed()).contains(cleanup));
+ assertEquals(1, fixture.fallbackClears.get());
+ fixture.assertWorkerFailure(bodyFatal ? body : cleanup);
+ assertNull(fixture.markerOnNextTask());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void ordinaryRestoreFailureClearsMarkerBeforeWorkerReplacement(boolean direct) throws Exception {
+ var cleanup = new IllegalStateException("adapter refused saved map");
+ try (var fixture = new Fixture(direct, cleanup, null, null)) {
+ assertEquals(ExecutionStatus.SUCCEEDED, fixture.run().getStatus());
+ fixture.assertEnd(InvocationStatus.SUCCEEDED, null);
+ assertEquals(1, fixture.fallbackClears.get());
+ fixture.assertWorkerFailure(cleanup);
+ assertNull(fixture.markerOnNextTask(), "A replacement worker must not inherit invocation MDC");
+ if (!direct) assertEquals(2, fixture.executor.created.get(), "The failed pool worker was replaced");
+ }
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @CsvSource({"false,ordinary", "true,ordinary", "false,fatal", "true,fatal", "false,same", "true,same"})
+ void fallbackFailureIsGuardedAndRetainsFatalIdentity(boolean direct, String kind) throws Exception {
+ var restore = new IllegalStateException("restore refused");
+ Throwable clear = switch (kind) {
+ case "fatal" -> new ThreadDeath();
+ case "same" -> restore;
+ default -> new IllegalStateException("clear refused");
+ };
+ try (var fixture = new Fixture(direct, restore, clear, null)) {
+ if (clear instanceof Error fatal) {
+ assertSame(fatal, assertThrows(Error.class, fixture::run));
+ fixture.assertEnd(InvocationStatus.SUCCEEDED, null);
+ // Manager resource closure may add its own transport diagnostic; retain the exact restoration failure.
+ assertTrue(Arrays.asList(fatal.getSuppressed()).contains(restore));
+ fixture.assertWorkerFailure(fatal);
+ } else {
+ assertEquals(ExecutionStatus.SUCCEEDED, fixture.run().getStatus());
+ fixture.assertEnd(InvocationStatus.SUCCEEDED, null);
+ assertArrayEquals(
+ clear == restore ? new Throwable[0] : new Throwable[] {clear}, restore.getSuppressed());
+ fixture.assertWorkerFailure(restore);
+ }
+ assertEquals(1, fixture.fallbackClears.get(), "No repeated clear attempts against a rejecting adapter");
+ // A rejecting clear cannot guarantee a clean replacement. Do not assert quarantine or clean MDC here.
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false", "true,true"})
+ void normalRestorationPreservesBodyClassificationAndWorker(boolean direct, boolean failedBody) throws Exception {
+ var body = failedBody ? new IllegalStateException("ordinary handler failure") : null;
+ try (var fixture = new Fixture(direct, null, null, body)) {
+ var result = fixture.run();
+ assertEquals(failedBody ? ExecutionStatus.FAILED : ExecutionStatus.SUCCEEDED, result.getStatus());
+ fixture.assertEnd(failedBody ? InvocationStatus.FAILED : InvocationStatus.SUCCEEDED, body);
+ assertNull(fixture.executor.escaped.get());
+ assertEquals(0, fixture.fallbackClears.get());
+ assertEquals("ambient", fixture.valueOnNextTask("ambient"));
+ assertNull(fixture.markerOnNextTask());
+ if (!direct) assertEquals(1, fixture.executor.created.get(), "Healthy owner remains reusable");
+ }
+ }
+
+ private static Throwable wrap(Error fatal, String wrapper) {
+ return switch (wrapper) {
+ case "completion" -> new CompletionException(new ExecutionException(fatal));
+ case "execution" -> new ExecutionException(fatal);
+ case "reflection" -> new InvocationTargetException(fatal);
+ case "proxy" -> new UndeclaredThrowableException(fatal);
+ default -> fatal;
+ };
+ }
+
+ private static final class Fixture implements AutoCloseable {
+ final MDCAdapter original = MDC.getMDCAdapter();
+ // SLF4J's standard adapter inherits MDC when ThreadPoolExecutor creates a replacement worker.
+ final BasicMDCAdapter delegate = new BasicMDCAdapter();
+ final AtomicBoolean injected = new AtomicBoolean();
+ final AtomicInteger fallbackClears = new AtomicInteger();
+ final AtomicInteger starts = new AtomicInteger();
+ final AtomicInteger ends = new AtomicInteger();
+ final AtomicInteger factories = new AtomicInteger();
+ final AtomicReference end = new AtomicReference<>();
+ final OwnedExecutor executor;
+ final LocalDurableTestRunner runner;
+
+ Fixture(boolean direct, Throwable restoreFailure, Throwable clearFailure, Throwable body) throws Exception {
+ delegate.put("ambient", "ambient");
+ executor = new OwnedExecutor(direct);
+ var adapter = (MDCAdapter) Proxy.newProxyInstance(
+ MDCAdapter.class.getClassLoader(), new Class[] {MDCAdapter.class}, (proxy, method, args) -> {
+ if (Thread.currentThread() == executor.owner.get()) {
+ if (restoreFailure != null
+ && isOuterRestore(method.getName())
+ && injected.compareAndSet(false, true)) {
+ // Model an adapter that partially mutates its map then rejects restoration.
+ delegate.put(MARKER, "from-invocation");
+ throw restoreFailure;
+ }
+ if (injected.get()
+ && method.getName().equals("clear")
+ && Arrays.stream(Thread.currentThread().getStackTrace())
+ .noneMatch(frame -> frame.getClassName()
+ .endsWith(".DurableExecutor")
+ && frame.getMethodName().equals("fireOnInvocationEnd"))) {
+ fallbackClears.incrementAndGet();
+ if (clearFailure != null) throw clearFailure;
+ }
+ }
+ try {
+ return method.invoke(delegate, args);
+ } catch (InvocationTargetException failure) {
+ throw failure.getCause();
+ }
+ });
+ replaceAdapter(adapter);
+ runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ MDC.put(MARKER, "from-handler");
+ if (body != null) ExceptionHelper.sneakyThrow(body);
+ return "done";
+ },
+ DurableConfig.builder()
+ .withExecutorService(executor)
+ .withPlugins(info -> {
+ factories.incrementAndGet();
+ return new DurableExecutionPlugin() {
+ public void onInvocationStart(InvocationInfo info) {
+ starts.incrementAndGet();
+ }
+
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.incrementAndGet();
+ end.set(info);
+ }
+ };
+ })
+ .build());
+ }
+
+ TestResult run() {
+ return runner.run("input");
+ }
+
+ void assertEnd(InvocationStatus status, Throwable failure) {
+ assertEquals(1, factories.get());
+ assertEquals(1, starts.get());
+ assertEquals(1, ends.get());
+ assertEquals(status, end.get().invocationStatus());
+ assertSame(failure, end.get().executionError());
+ }
+
+ void assertWorkerFailure(Throwable expected) throws Exception {
+ if (executor.direct) {
+ assertNull(executor.escaped.get(), "Inline cleanup must preserve normal finalization dispatch");
+ } else {
+ assertTrue(
+ executor.uncaught.await(2, TimeUnit.SECONDS), "Cleanup failure must escape actual pool worker");
+ assertSame(expected, executor.escaped.get());
+ assertSame(executor.owner.get(), executor.failedWorker.get());
+ }
+ }
+
+ String markerOnNextTask() throws Exception {
+ return valueOnNextTask(MARKER);
+ }
+
+ String valueOnNextTask(String key) throws Exception {
+ if (executor.direct) return MDC.get(key);
+ return executor.delegate.submit(() -> MDC.get(key)).get(2, TimeUnit.SECONDS);
+ }
+
+ public void close() throws Exception {
+ try {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS));
+ } finally {
+ replaceAdapter(original);
+ }
+ }
+ }
+
+ // Finish the real handler task before dispatch returns; outer MDC cleanup follows the handler-owned End.
+ private static final class OwnedExecutor extends AbstractExecutorService {
+ final boolean direct;
+ final AtomicInteger created = new AtomicInteger();
+ final AtomicReference owner = new AtomicReference<>();
+ final AtomicReference failedWorker = new AtomicReference<>();
+ final AtomicReference escaped = new AtomicReference<>();
+ final CountDownLatch uncaught = new CountDownLatch(1);
+ final ThreadPoolExecutor delegate =
+ new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), task -> {
+ var thread = new Thread(task, "mdc-cleanup-owner-" + created.incrementAndGet());
+ thread.setDaemon(true);
+ thread.setUncaughtExceptionHandler((worker, failure) -> {
+ failedWorker.set(worker);
+ escaped.set(failure);
+ uncaught.countDown();
+ });
+ return thread;
+ });
+
+ OwnedExecutor(boolean direct) {
+ this.direct = direct;
+ }
+
+ public void execute(Runnable task) {
+ var done = new CountDownLatch(1);
+ Runnable observed = () -> {
+ owner.set(Thread.currentThread());
+ try {
+ task.run();
+ } finally {
+ done.countDown();
+ }
+ };
+ if (direct) observed.run();
+ else {
+ delegate.execute(observed);
+ try {
+ assertTrue(done.await(3, TimeUnit.SECONDS), "Handler cleanup did not finish");
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+ }
+
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+ }
+
+ private static boolean isOuterRestore(String method) {
+ if (!method.equals("clear") && !method.equals("setContextMap")) return false;
+ var frames = Arrays.asList(Thread.currentThread().getStackTrace());
+ return frames.stream()
+ .anyMatch(frame -> frame.getClassName().endsWith(".DurableExecutor")
+ && frame.getMethodName().startsWith("lambda$restoreMdcOnClose"))
+ && frames.stream()
+ .noneMatch(frame -> frame.getClassName().endsWith(".DurableExecutor")
+ && frame.getMethodName().equals("fireOnInvocationEnd"));
+ }
+
+ private static void replaceAdapter(MDCAdapter adapter) throws Exception {
+ var setter = MDC.class.getDeclaredMethod("setMDCAdapter", MDCAdapter.class);
+ setter.setAccessible(true);
+ setter.invoke(null, adapter);
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcWorkerReuseBoundaryTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcWorkerReuseBoundaryTest.java
deleted file mode 100644
index 7072ddfe8a..0000000000
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcWorkerReuseBoundaryTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Proxy;
-import java.time.Duration;
-import java.util.List;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-import java.util.stream.Stream;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-import org.junit.jupiter.params.provider.ValueSource;
-import org.slf4j.MDC;
-import org.slf4j.helpers.BasicMDCAdapter;
-import org.slf4j.spi.MDCAdapter;
-import software.amazon.lambda.durable.DurableConfig;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.*;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-
-class MdcWorkerReuseBoundaryTest {
- @ParameterizedTest
- @ValueSource(strings = {"start", "end"})
- void replacementWorkerDoesNotInheritInvocationMdcWhenClearSucceeds(String markerPhase) throws Exception {
- exercise(markerPhase, false, "ok");
- }
-
- static Stream fallbackCases() {
- return Stream.of(false, true)
- .flatMap(suspend -> Stream.of(
- "ordinary", "assertion", "same", "vm", "death", "wrapped-vm", "wrapped-death")
- .map(kind -> Arguments.of(suspend, kind)));
- }
-
- @ParameterizedTest
- @MethodSource("fallbackCases")
- void failedFallbackRetainsFailurePolicyAndFatalIdentity(boolean suspend, String kind) throws Exception {
- exercise("end", suspend, kind);
- }
-
- static Stream fatalEndCases() {
- return Stream.of(false, true)
- .flatMap(suspend -> Stream.of(false, true)
- .flatMap(death -> Stream.of(
- "ok",
- "ordinary",
- "assertion",
- "same",
- "vm",
- "death",
- "wrapped-vm",
- "wrapped-death")
- .map(kind -> Arguments.of(suspend, death, kind))));
- }
-
- @ParameterizedTest
- @MethodSource("fatalEndCases")
- @SuppressWarnings("removal")
- void fatalEndStillClearsOrdinaryRestoreFailureBeforeWorkerReplacement(boolean suspend, boolean death, String kind)
- throws Exception {
- exercise("end", suspend, kind, death ? new ThreadDeath() : new InternalError("original End fatal"));
- }
-
- private static void exercise(String markerPhase, boolean suspend, String kind) throws Exception {
- exercise(markerPhase, suspend, kind, null);
- }
-
- @SuppressWarnings("removal")
- private static void exercise(String markerPhase, boolean suspend, String kind, Error selectedFatal)
- throws Exception {
- var original = MDC.getMDCAdapter();
- var basic = new BasicMDCAdapter();
- var ends = new AtomicInteger();
- var endStatus = new AtomicReference();
- var failed = new AtomicBoolean();
- var clearTried = new AtomicBoolean();
- var restoreFailure = new IllegalStateException("restore failed");
- Error clearFatal = switch (kind) {
- case "vm", "wrapped-vm" -> new InternalError("fallback fatal");
- case "death", "wrapped-death" -> new ThreadDeath();
- default -> null;
- };
- Throwable clearFailure = switch (kind) {
- case "ordinary" -> new IllegalArgumentException("clear failed");
- case "assertion" -> new AssertionError("clear failed");
- case "same" -> restoreFailure;
- case "wrapped-vm", "wrapped-death" -> new CompletionException(new ExecutionException(clearFatal));
- default -> clearFatal;
- };
- Error expectedFatal = selectedFatal != null ? selectedFatal : clearFatal;
- var escaped = new CountDownLatch(1);
- var failedWorker = new AtomicReference();
- var ownerFailure = new AtomicReference();
- var dirtyAtFailure = new AtomicReference();
- var count = new AtomicInteger();
- var workers = Executors.newSingleThreadExecutor(task -> {
- var thread = new Thread(task, "mdc-inheritable-worker-" + count.incrementAndGet());
- thread.setUncaughtExceptionHandler((owner, failure) -> {
- failedWorker.set(owner);
- ownerFailure.set(failure);
- dirtyAtFailure.set(MDC.get("invocation-marker"));
- escaped.countDown();
- });
- return thread;
- });
- try {
- var adapter = (MDCAdapter) Proxy.newProxyInstance(
- MDCAdapter.class.getClassLoader(), new Class[] {MDCAdapter.class}, (proxy, method, args) -> {
- if (ends.get() == 1
- && method.getName().equals("setContextMap")
- && Thread.currentThread().getName().startsWith("mdc-inheritable-worker-")
- && failed.compareAndSet(false, true)) throw restoreFailure;
- if (failed.get()
- && method.getName().equals("clear")
- && Thread.currentThread().getName().startsWith("mdc-inheritable-worker-")
- && clearTried.compareAndSet(false, true)
- && clearFailure != null) throw clearFailure;
- try {
- return method.invoke(basic, args);
- } catch (InvocationTargetException failure) {
- throw failure.getCause();
- }
- });
- replace(adapter);
- MDC.put("ambient", "saved");
- var plugin = new DurableExecutionPlugin() {
- public void onInvocationStart(InvocationInfo info) {
- if (markerPhase.equals("start")) MDC.put("invocation-marker", "previous-invocation");
- }
-
- public void onInvocationEnd(InvocationEndInfo info) {
- if (markerPhase.equals("end")) MDC.put("invocation-marker", "previous-invocation");
- endStatus.set(info.invocationStatus());
- ends.incrementAndGet();
- if (selectedFatal != null) throw selectedFatal;
- }
- };
- var runner = LocalDurableTestRunner.create(
- String.class,
- (value, context) -> {
- if (suspend) context.wait("resume", Duration.ofSeconds(1));
- return "done";
- },
- DurableConfig.builder()
- .withExecutorService(workers)
- .withPlugins(plugin)
- .build());
- if (expectedFatal != null) assertSame(expectedFatal, assertThrows(Error.class, () -> runner.run("input")));
- else
- assertEquals(
- suspend ? ExecutionStatus.PENDING : ExecutionStatus.SUCCEEDED,
- runner.run("input").getStatus());
- assertTrue(escaped.await(3, TimeUnit.SECONDS));
- assertSame(expectedFatal != null ? expectedFatal : restoreFailure, ownerFailure.get());
-
- assertEquals(1, ends.get());
- assertEquals(suspend ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED, endStatus.get());
- if (selectedFatal != null) {
- assertEquals(
- clearFailure != null && clearFailure != restoreFailure
- ? List.of(restoreFailure, clearFailure)
- : List.of(restoreFailure),
- List.of(selectedFatal.getSuppressed()));
- } else if (expectedFatal != null)
- assertEquals(List.of(restoreFailure), List.of(expectedFatal.getSuppressed()));
- else
- assertEquals(
- clearFailure != null && clearFailure != restoreFailure ? List.of(clearFailure) : List.of(),
- List.of(restoreFailure.getSuppressed()));
- var nextThread = workers.submit(Thread::currentThread).get(3, TimeUnit.SECONDS);
- var inherited = workers.submit(() -> MDC.get("invocation-marker")).get(3, TimeUnit.SECONDS);
- assertNotSame(failedWorker.get(), nextThread);
- System.out.println("MDC_REUSE_PUBLIC phase=" + markerPhase + " kind=" + kind + " replaced=true oldMarker="
- + dirtyAtFailure.get() + " inheritedMarker=" + inherited);
- if (clearFailure == null) assertNull(inherited, "Successful fallback clears inheritable invocation state");
- assertTrue(clearTried.get());
- // An adapter whose clear also fails cannot promise clean replacement state; its failure is exposed.
- } finally {
- workers.shutdownNow();
- assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
- basic.clear();
- replace(original);
- }
- }
-
- private static void replace(MDCAdapter adapter) throws Exception {
- var setter = MDC.class.getDeclaredMethod("setMDCAdapter", MDCAdapter.class);
- setter.setAccessible(true);
- setter.invoke(null, adapter);
- }
-}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java
index 188768cf53..e5fc59f299 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java
@@ -166,8 +166,9 @@ private static List finish(Throwable error, SpanLimits limits, Sampler
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- var plugin = new ExecutionOtelPlugin(builder, config);
- plugin.onInvocationStart(new InvocationInfo("request", "arn:diagnostic", true, Instant.ofEpochSecond(1)));
+ var factory = ExecutionOtelPlugin.factory(builder, config);
+ var plugin = Invocations.started(
+ factory, new InvocationInfo("request", "arn:diagnostic", true, Instant.ofEpochSecond(1)));
plugin.onOperationEnd(new OperationEndInfo(
"op",
NAME,
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTimingTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTimingTest.java
index c3c32d0279..7497d53646 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTimingTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTimingTest.java
@@ -16,7 +16,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -51,15 +51,16 @@ public long nanoTime() {
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(builder, settings)
- : new InvocationOtelPlugin(builder, settings);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
var start = path.equals("missing-start") || path.equals("no-end") || path.equals("zero-end")
? null
: Instant.ofEpochSecond(path.equals("late-start") ? 30 : 10);
var end =
path.equals("no-end") ? null : path.equals("zero-end") ? Instant.EPOCH : Instant.ofEpochSecond(20);
- plugin.onInvocationStart(new InvocationInfo("request", "arn:timing", true, Instant.ofEpochSecond(1)));
+ var plugin = Invocations.started(
+ factory, new InvocationInfo("request", "arn:timing", true, Instant.ofEpochSecond(1)));
if (path.equals("live") || path.equals("late-start"))
plugin.onOperationStart(new OperationInfo(
"operation", "timing-target", "CALLBACK", "Callback", null, start, null, null, false));
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java
new file mode 100644
index 0000000000..23675c8d05
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java
@@ -0,0 +1,63 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Covers registration-time rejection of a null config on both plugins' factory overloads.
+ *
+ * The global-provider overload only stores the config, so a null one used to be dereferenced when an invocation's
+ * plugin instance was built. {@code PluginRunner} contains a factory failure, so the function ran without the telemetry
+ * it had asked for and reported one warning per invocation. Registration is where a caller can still act on it.
+ */
+class OtelPluginFactoryConfigTest {
+
+ @Test
+ void invocationPluginRejectsANullConfigOnTheGlobalProviderOverload() {
+ var error =
+ assertThrows(NullPointerException.class, () -> InvocationOtelPlugin.factory((OtelPluginConfig) null));
+
+ assertTrue(error.getMessage().contains("config"), error.getMessage());
+ }
+
+ @Test
+ void executionPluginRejectsANullConfigOnTheGlobalProviderOverload() {
+ var error =
+ assertThrows(NullPointerException.class, () -> ExecutionOtelPlugin.factory((OtelPluginConfig) null));
+
+ assertTrue(error.getMessage().contains("config"), error.getMessage());
+ }
+
+ @Test
+ void invocationPluginRejectsANullConfigOnTheProviderBuilderOverload() {
+ assertThrows(NullPointerException.class, () -> InvocationOtelPlugin.factory(SdkTracerProvider.builder(), null));
+ }
+
+ @Test
+ void executionPluginRejectsANullConfigOnTheProviderBuilderOverload() {
+ assertThrows(NullPointerException.class, () -> ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), null));
+ }
+
+ @Test
+ void aRejectedConfigLeavesTheBuilderUsable() {
+ // The check is the first statement of forProviderBuilder, so it precedes the ID-generator and sampler
+ // installation and the provider build. That ordering matters because a provider built and then thrown away is
+ // unreachable: its span processors and their worker threads are never shut down. The ordering itself is a
+ // property of the source rather than something this test can observe -- SdkTracerProviderBuilder exposes no
+ // getters -- so what is asserted here is the consequence a caller can see: the builder they passed still
+ // works.
+ var builder = SdkTracerProvider.builder();
+
+ assertThrows(NullPointerException.class, () -> InvocationOtelPlugin.factory(builder, null));
+
+ try (var provider = builder.build()) {
+ assertNotNull(provider.get("probe"));
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPropagationMetadataTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPropagationMetadataTest.java
index 1979b1071c..8bcf6a4f1a 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPropagationMetadataTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPropagationMetadataTest.java
@@ -22,6 +22,7 @@
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -151,9 +152,10 @@ private static Fixture fixture(boolean executionView, boolean sampled, AtomicBoo
sampled ? ExtractedContext.Sampling.SAMPLED : ExtractedContext.Sampling.NOT_SAMPLED);
})
.build();
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
- return new Fixture(new PluginRunner(List.of(new DurableExecutionPlugin() {}, plugin)), exporter);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ return new Fixture(new PluginRunner(List.of(info -> new DurableExecutionPlugin() {}, factory)), exporter);
}
private static OperationInfo operation(boolean replay) {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java
new file mode 100644
index 0000000000..3f6b574eed
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java
@@ -0,0 +1,64 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Instant;
+import java.util.List;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.plugin.PluginRunner;
+
+class PluginStartupOrderTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void earlierInitializerContextIsAvailableToOtelFactory(boolean executionView) {
+ var carrier = new ThreadLocal();
+ var expected = new ExtractedContext(
+ "6955b900123456789012345678901234", "1234567890123456", ExtractedContext.Sampling.SAMPLED);
+ DurableExecutionPluginFactory initializer = info -> new DurableExecutionPlugin() {
+ @Override
+ public void onInvocationStart(InvocationInfo info) {
+ carrier.set(expected);
+ }
+
+ @Override
+ public void onInvocationEnd(InvocationEndInfo info) {
+ carrier.remove();
+ }
+ };
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder()
+ .setSampler(Sampler.alwaysOff())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(carrier::get)
+ .build();
+ var otel = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ var runner = new PluginRunner(List.of(initializer, otel));
+ try {
+ runner.onInvocationStart(new InvocationInfo("request", "arn", true, Instant.EPOCH));
+ runner.onInvocationEnd(new InvocationEndInfo("request", "arn", true, InvocationStatus.SUCCEEDED, null));
+ var spans = exporter.getFinishedSpanItems();
+ assertEquals(2, spans.size(), "Initializer's sampled context must override the always-off fallback");
+ assertTrue(spans.stream().allMatch(span -> expected.traceId().equals(span.getTraceId())));
+ assertTrue(spans.stream().allMatch(span -> expected.parentSpanId().equals(span.getParentSpanId())));
+ } finally {
+ carrier.remove();
+ runner.releasePlugins();
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationCollectorDiagnosticsTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationCollectorDiagnosticsTest.java
index e187944006..4f2063dba4 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationCollectorDiagnosticsTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationCollectorDiagnosticsTest.java
@@ -7,10 +7,13 @@
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
+import java.time.Instant;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.slf4j.LoggerFactory;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.PluginRunner;
import software.amazon.lambda.durable.plugin.PropagationInput;
import software.amazon.lambda.durable.plugin.PropagationMetadata;
@@ -28,6 +31,7 @@ void differentValuesNameBothOwnersAndCountConflictsWithoutLoggingHeaders() {
plugin("private-first"),
plugin("private-second"),
plugin("private-third")));
+ runner.onInvocationStart(new InvocationInfo("request", "arn", true, Instant.EPOCH));
assertEquals(
"private-first",
runner.providePropagationMetadata(new PropagationInput("arn", "op", null, "target"))
@@ -49,8 +53,8 @@ void differentValuesNameBothOwnersAndCountConflictsWithoutLoggingHeaders() {
}
}
- private static DurableExecutionPlugin plugin(String header) {
- return new DurableExecutionPlugin() {
+ private static DurableExecutionPluginFactory plugin(String header) {
+ return ignored -> new DurableExecutionPlugin() {
@Override
public PropagationMetadata providePropagationMetadata(PropagationInput input) {
return new PropagationMetadata(header);
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationSamplingAuthorityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationSamplingAuthorityTest.java
index 471c565d68..862542d35b 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationSamplingAuthorityTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PropagationSamplingAuthorityTest.java
@@ -23,7 +23,7 @@
import java.util.List;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -88,9 +88,9 @@ public ContextPropagators getPropagators() {
.contextExtractor(() -> null)
.enableMdc(false)
.build();
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(config) : new InvocationOtelPlugin(config);
- var runner = new PluginRunner(List.of(plugin));
+ DurableExecutionPluginFactory factory =
+ executionView ? ExecutionOtelPlugin.factory(config) : InvocationOtelPlugin.factory(config);
+ var runner = new PluginRunner(List.of(factory));
var start = Instant.parse("2026-10-14T00:00:00Z");
var arn = "arn:aws:lambda:us-east-1:123456789012:function:parent/durable-execution/test/id";
var input = new PropagationInput(arn, "invoke-id", null, "target:live");
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java
index 0e782413e1..8a7cf03865 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java
@@ -27,7 +27,7 @@
import org.junit.jupiter.params.provider.CsvSource;
import software.amazon.lambda.durable.DurableConfig;
import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
class ReplacementSamplerPolicyTest {
@@ -96,8 +96,8 @@ public ContextPropagators getPropagators() {
return ContextPropagators.noop();
}
});
- DurableExecutionPlugin plugin =
- view.equals("invocation") ? new InvocationOtelPlugin() : new ExecutionOtelPlugin();
+ DurableExecutionPluginFactory plugin =
+ view.equals("invocation") ? InvocationOtelPlugin.factory() : ExecutionOtelPlugin.factory();
var tracer = provider.get("application");
var runner = LocalDurableTestRunner.create(
String.class,
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootFactoryFinalizationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootFactoryFinalizationTest.java
new file mode 100644
index 0000000000..2fb00e6ff4
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootFactoryFinalizationTest.java
@@ -0,0 +1,110 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.*;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Duration;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+@Timeout(15)
+class RootFactoryFinalizationTest {
+ @ParameterizedTest
+ @CsvSource({"false,none", "true,none", "false,root", "true,root", "false,invocation", "true,invocation"})
+ void freshPublicFactoryInstancesEndAnchorBeforeBodyAndFlushOnceEvenAfterOrdinaryEndFailure(
+ boolean executionView, String failurePhase) {
+ var roots = new AtomicInteger();
+ var flushes = new AtomicInteger();
+ var failOnce = new AtomicBoolean(!failurePhase.equals("none"));
+ var effects = new AtomicInteger();
+ var bodies = new AtomicInteger();
+ var instances = new CopyOnWriteArrayList();
+ try (var exporter = InMemorySpanExporter.create()) {
+ var observer = new SpanProcessor() {
+ public void onStart(Context parent, ReadWriteSpan span) {}
+
+ public boolean isStartRequired() {
+ return false;
+ }
+
+ public void onEnd(ReadableSpan span) {
+ if (span.getName().equals("DurableExecutionRoot")) roots.incrementAndGet();
+ if (span.getName().equals(failurePhase.equals("root") ? "DurableExecutionRoot" : "Invocation")
+ && failOnce.compareAndSet(true, false))
+ throw new IllegalStateException("ordinary processor failure");
+ }
+
+ public boolean isEndRequired() {
+ return true;
+ }
+
+ public CompletableResultCode forceFlush() {
+ flushes.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ var builder = SdkTracerProvider.builder()
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter))
+ .addSpanProcessor(observer);
+ var settings = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ var actual = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
+ DurableExecutionPluginFactory factory = info -> {
+ var instance = actual.createPlugin(info);
+ instances.add(instance);
+ return instance;
+ };
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ assertEquals(
+ bodies.incrementAndGet(),
+ roots.get(),
+ "Anchor ends before customer code on each invocation");
+ context.step("saved", String.class, step -> {
+ effects.incrementAndGet();
+ return "saved";
+ });
+ context.wait("pause", Duration.ofSeconds(1));
+ return "done";
+ },
+ DurableConfig.builder().withPlugins(factory).build());
+ assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
+ assertEquals(1, flushes.get());
+ runner.advanceTime();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus());
+ assertEquals(2, flushes.get());
+ assertEquals(2, instances.size());
+ assertNotSame(instances.get(0), instances.get(1));
+ assertEquals(1, effects.get());
+ var anchors = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(2, anchors.size());
+ assertEquals(anchors.get(0).getSpanContext(), anchors.get(1).getSpanContext());
+ assertTrue(anchors.stream().allMatch(span -> span.getStartEpochNanos() == span.getEndEpochNanos()));
+ instances.forEach(plugin -> plugin.onInvocationEnd(
+ new InvocationEndInfo("already-ended", "unused", false, InvocationStatus.SUCCEEDED, null)));
+ assertEquals(2, flushes.get());
+ assertEquals(2, roots.get());
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java
index 04277e3ec0..6491b3d0c0 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java
@@ -25,7 +25,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -65,8 +65,9 @@ void durableStartCallbacksRetainTheirOwnSamplingAndRandomIds(
null,
rootRecordOnly ? ExtractedContext.Sampling.UNDECIDED : ExtractedContext.Sampling.SAMPLED))
.build();
- DurableExecutionPlugin plugin =
- executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
// Rebuilding this configured builder shares the plugin-installed generator; the other branch exercises the
// same thread-scoped bridge through an independent generator instance.
try (var unrelated = sharedGenerator
@@ -79,16 +80,18 @@ void durableStartCallbacksRetainTheirOwnSamplingAndRandomIds(
observer.tracer = unrelated.get("processor");
var original = Context.current();
assertControl(observer.tracer, sampled);
- invoke(plugin, true);
- invoke(plugin, false);
+ invoke(factory, true);
+ invoke(factory, false);
assertControl(observer.tracer, sampled);
assertSame(original, Context.current());
assertObservations(observer, sampled, rootRecordOnly);
}
}
- private static void invoke(DurableExecutionPlugin plugin, boolean first) {
- plugin.onInvocationStart(new InvocationInfo("request", ARN, first, START));
+ private static void invoke(DurableExecutionPluginFactory factory, boolean first) {
+ var info = new InvocationInfo("request", ARN, first, START);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onInvocationEnd(new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null));
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java
new file mode 100644
index 0000000000..1e95284885
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java
@@ -0,0 +1,140 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Instant;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState;
+import software.amazon.awssdk.services.lambda.model.ExecutionDetails;
+import software.amazon.awssdk.services.lambda.model.Operation;
+import software.amazon.awssdk.services.lambda.model.OperationStatus;
+import software.amazon.awssdk.services.lambda.model.OperationType;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.client.DurableExecutionClient;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+
+class RuntimeHeaderFailureIntegrationTest {
+ @Test
+ void noPluginsDoesNotReadRuntimeTraceHeader() {
+ executeWithThrowingRuntimeOverride(null, new AssertionError("must not read"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "true,0,runtime",
+ "true,0,assertion",
+ "true,0,class-linkage",
+ "true,0,method-linkage",
+ "true,0,abstract-linkage",
+ "true,1,runtime",
+ "true,1,assertion",
+ "true,1,class-linkage",
+ "true,1,method-linkage",
+ "true,1,abstract-linkage",
+ "false,0,runtime",
+ "false,0,assertion",
+ "false,0,class-linkage",
+ "false,0,method-linkage",
+ "false,0,abstract-linkage",
+ "false,1,runtime",
+ "false,1,assertion",
+ "false,1,class-linkage",
+ "false,1,method-linkage",
+ "false,1,abstract-linkage"
+ })
+ void throwingRuntimeOverrideCannotBorrowStaleGlobalCarrier(
+ boolean executionView, String sampled, String failureKind) {
+ var property = "com.amazonaws.xray.traceHeader";
+ var previous = System.getProperty(property);
+ var stale = "Root=1-6955b900-aaaaaaaaaaaaaaaaaaaaaaaa;Sampled=" + sampled;
+ System.setProperty(property, stale);
+ try {
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder()
+ .setSampler(Sampler.alwaysOn())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder().enableMdc(false).build();
+ DurableExecutionPluginFactory plugin = executionView
+ ? ExecutionOtelPlugin.factory(builder, config)
+ : InvocationOtelPlugin.factory(builder, config);
+ executeWithThrowingRuntimeOverride(plugin, accessorFailure(failureKind));
+ var spans = exporter.getFinishedSpanItems();
+ assertEquals(3, spans.size(), "a failed invocation accessor must not inherit stale sampling");
+ // This factory branch already exports its synthetic ancestor in addition to Workflow and Invocation.
+ assertEquals(
+ List.of("DurableExecutionRoot", "Invocation", "Workflow"),
+ spans.stream().map(span -> span.getName()).sorted().toList());
+ assertTrue(
+ spans.stream().allMatch(span -> !"6955b900aaaaaaaaaaaaaaaaaaaaaaaa".equals(span.getTraceId())),
+ "a failed invocation accessor must not inherit another request's trace");
+ assertEquals(stale, System.getProperty(property));
+ } finally {
+ if (previous == null) System.clearProperty(property);
+ else System.setProperty(property, previous);
+ }
+ }
+
+ private static void executeWithThrowingRuntimeOverride(DurableExecutionPluginFactory plugin, Throwable failure) {
+ var context = mock(RuntimeContext.class);
+ when(context.getXrayTraceId()).thenAnswer(invocation -> {
+ throw failure;
+ });
+ when(context.getRemainingTimeInMillis()).thenReturn(30000);
+ var operation = Operation.builder()
+ .id("id")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ var input = new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id",
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operation).build());
+ var builder = DurableConfig.builder().withDurableExecutionClient(mock(DurableExecutionClient.class));
+ if (plugin != null) builder.withPlugins(plugin);
+ var config = builder.build();
+ var result =
+ DurableExecutor.execute(input, context, TypeToken.get(String.class), (value, ctx) -> value, config);
+ assertEquals(ExecutionStatus.SUCCEEDED, result.status());
+ assertEquals("\"input\"", result.result());
+ if (plugin == null) verify(context, never()).getXrayTraceId();
+ }
+
+ private static Throwable accessorFailure(String kind) {
+ return switch (kind) {
+ case "runtime" -> new SecurityException("runtime carrier access denied");
+ case "assertion" -> new AssertionError("optional runtime assertion");
+ case "class-linkage" -> new NoClassDefFoundError("optional dependency");
+ case "method-linkage" -> new NoSuchMethodError("inside available override");
+ case "abstract-linkage" -> new AbstractMethodError("inside available override");
+ default -> throw new IllegalArgumentException(kind);
+ };
+ }
+
+ private abstract static class RuntimeContext implements Context {
+ @Override
+ public String getXrayTraceId() {
+ return null;
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java
index 976711b728..39fa9686d7 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java
@@ -28,7 +28,7 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -148,10 +148,10 @@ public ContextPropagators getPropagators() {
return ContextPropagators.noop();
}
});
- var plugin = (DurableExecutionPlugin)
+ var factory = (DurableExecutionPluginFactory)
Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader)
- .getConstructor()
- .newInstance();
+ .getMethod("factory")
+ .invoke(null);
var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id";
var previousAmbient = Span.current();
var ambient = appProvider
@@ -161,7 +161,9 @@ public ContextPropagators getPropagators() {
assertFalse(ambient.isRecording(), "The app provider drops unrelated spans outside the callback");
try (var ambientScope = ambient.makeCurrent()) {
for (var first : new boolean[] {true, false}) {
- plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.EPOCH));
+ var info = new InvocationInfo("request", arn, first, Instant.EPOCH);
+ var plugin = factory.createPlugin(info);
+ plugin.onInvocationStart(info);
plugin.onInvocationEnd(
new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null));
assertSame(ambient, Span.current(), "Plugin cleanup must preserve the caller's ambient span");
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java
new file mode 100644
index 0000000000..c5bea8b306
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java
@@ -0,0 +1,96 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.HandlerScoped;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+class ScopeFatalSignalTest {
+ @ParameterizedTest
+ @CsvSource({"false,false", "true,false", "false,true", "true,true"})
+ @SuppressWarnings("removal")
+ void fatalCloseStillAwaitsTheEarlierScopeOwner(boolean threadDeath, boolean releaseInBudget) throws Exception {
+ var earlierEntered = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var closed = new AtomicBoolean();
+ var ownerFatal = new AtomicReference();
+ var ownerObserved = new CountDownLatch(1);
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("later scope close");
+ var workers = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "scope-fatal-owner");
+ thread.setUncaughtExceptionHandler((owner, failure) -> {
+ ownerFatal.set(failure);
+ ownerObserved.countDown();
+ });
+ return thread;
+ });
+ var callers = Executors.newSingleThreadExecutor();
+ try {
+ var earlier = new Scoped(() -> {
+ earlierEntered.countDown();
+ if (!release.await(5, TimeUnit.SECONDS)) throw new AssertionError("release timed out");
+ closed.set(true);
+ });
+ var later = new Scoped(() -> {
+ throw fatal;
+ });
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withPlugins(info -> earlier, info -> later)
+ .build();
+ var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config);
+ var result = callers.submit(() -> runner.run("input"));
+ assertTrue(earlierEntered.await(3, TimeUnit.SECONDS));
+ assertThrows(
+ TimeoutException.class,
+ () -> result.get(100, TimeUnit.MILLISECONDS),
+ "earlier cleanup must finish before End");
+ if (!releaseInBudget) {
+ assertThrows(TimeoutException.class, () -> result.get(600, TimeUnit.MILLISECONDS));
+ assertFalse(closed.get(), "An elapsed caller budget cannot complete the scope owner");
+ }
+ release.countDown();
+ var failure = assertThrows(
+ ExecutionException.class,
+ () -> result.get(2, TimeUnit.SECONDS),
+ "reported fatal must propagate after the owner finishes cleanup");
+ assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure));
+ assertTrue(closed.get());
+ } finally {
+ release.countDown();
+ callers.shutdown();
+ workers.shutdown();
+ assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ }
+ assertTrue(ownerObserved.await(3, TimeUnit.SECONDS));
+ assertSame(fatal, ownerFatal.get(), "the fatal still escapes its actual owner");
+ }
+
+ @HandlerScoped(Opener.class)
+ public static final class Scoped implements DurableExecutionPlugin {
+ private final AutoCloseable closer;
+
+ Scoped(AutoCloseable closer) {
+ this.closer = closer;
+ }
+ }
+
+ public static final class Opener implements Function {
+ @Override
+ public AutoCloseable apply(Object plugin) {
+ return ((Scoped) plugin).closer;
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/TerminalOperationStatusTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/TerminalOperationStatusTest.java
index dd26449322..5ec7d1d5e5 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/TerminalOperationStatusTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/TerminalOperationStatusTest.java
@@ -19,7 +19,7 @@
import software.amazon.awssdk.services.lambda.model.OperationStatus;
import software.amazon.awssdk.services.lambda.model.OperationType;
import software.amazon.lambda.durable.exception.DurableOperationException;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -70,11 +70,12 @@ private static void check(
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- DurableExecutionPlugin plugin = executionView
- ? new ExecutionOtelPlugin(builder, settings)
- : new InvocationOtelPlugin(builder, settings);
+ DurableExecutionPluginFactory factory = executionView
+ ? ExecutionOtelPlugin.factory(builder, settings)
+ : InvocationOtelPlugin.factory(builder, settings);
var start = Instant.parse("2026-10-14T00:00:00Z");
- plugin.onInvocationStart(new InvocationInfo("request", "arn:status", !matchingStart, start));
+ var plugin =
+ Invocations.started(factory, new InvocationInfo("request", "arn:status", !matchingStart, start));
if (matchingStart)
plugin.onOperationStart(new OperationInfo(
"op", "status-target", "CALLBACK", "Callback", null, start, null, null, false));
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java
new file mode 100644
index 0000000000..6a2e4d0309
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java
@@ -0,0 +1,131 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+class UserFunctionEndCleanupTest {
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false", "true,true"})
+ @SuppressWarnings("removal")
+ void earlierEndFatalCannotSkipLaterOtelOwnerCleanup(boolean executionView, boolean threadDeath) throws Exception {
+ Error fatal = threadDeath ? new ThreadDeath() : new InternalError("first user end");
+ var cleanupFatal = new InternalError("later user end");
+ var owner = new AtomicReference();
+ var before = new AtomicReference();
+ var after = new AtomicReference();
+ var afterTask = new AtomicReference();
+ var endOwner = new AtomicReference();
+ var ends = new AtomicInteger();
+ var bodies = new AtomicInteger();
+ var exited = new CountDownLatch(1);
+ var exporter = InMemorySpanExporter.create();
+ var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder()
+ .enableMdc(true)
+ .contextExtractor(() -> new ExtractedContext(
+ "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ var otel = executionView
+ ? ExecutionOtelPlugin.factory(provider, config)
+ : InvocationOtelPlugin.factory(provider, config);
+ DurableExecutionPluginFactory first = info -> new DurableExecutionPlugin() {
+ @Override
+ public void onUserFunctionStart(UserFunctionStartInfo start) {
+ if ("work".equals(start.name())) {
+ owner.set(Thread.currentThread());
+ before.set(Span.current().getSpanContext());
+ }
+ }
+
+ @Override
+ public void onUserFunctionEnd(UserFunctionEndInfo end) {
+ if ("work".equals(end.name())) throw fatal;
+ }
+ };
+ DurableExecutionPluginFactory last = info -> new DurableExecutionPlugin() {
+ @Override
+ public void onUserFunctionEnd(UserFunctionEndInfo end) {
+ if ("work".equals(end.name())) {
+ ends.incrementAndGet();
+ endOwner.set(Thread.currentThread());
+ after.set(Span.current().getSpanContext());
+ }
+ }
+ };
+ DurableExecutionPluginFactory secondFailure = info -> new DurableExecutionPlugin() {
+ @Override
+ public void onUserFunctionEnd(UserFunctionEndInfo end) {
+ if ("work".equals(end.name())) throw cleanupFatal;
+ }
+ };
+ var workers = new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>()) {
+ @Override
+ public void execute(Runnable task) {
+ super.execute(() -> {
+ try {
+ task.run();
+ } catch (VirtualMachineError | ThreadDeath expected) {
+ // Custom executors may retain workers after a task reports its fatal to the invocation.
+ } finally {
+ if (Thread.currentThread() == owner.get()) {
+ afterTask.set(Span.current().getSpanContext());
+ exited.countDown();
+ }
+ }
+ });
+ }
+ };
+ try {
+ var durable = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(first, otel, secondFailure, last)
+ .build();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> context.step("work", String.class, step -> {
+ bodies.incrementAndGet();
+ return "done";
+ }),
+ durable);
+ var thrown = assertThrows(Error.class, () -> runner.run("input"));
+ assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(thrown));
+ // Resource shutdown may additionally attach its transport failure; retain this cleanup failure once.
+ assertEquals(
+ 1,
+ Arrays.stream(fatal.getSuppressed())
+ .filter(error -> error == cleanupFatal)
+ .count());
+ assertTrue(exited.await(3, TimeUnit.SECONDS));
+ assertEquals(1, bodies.get());
+ assertEquals(1, ends.get(), "the later observer must receive the same single end event");
+ assertSame(owner.get(), endOwner.get());
+ assertEquals(
+ before.get(), after.get(), "OTel must restore its attempt scope on the owner before later hooks");
+ assertEquals(
+ before.get(), afterTask.get(), "the executor reuse boundary must not retain the attempt scope");
+ assertTrue(exporter.getFinishedSpanItems().stream()
+ .anyMatch(span -> span.getName().contains(" attempt ")));
+ } finally {
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java
new file mode 100644
index 0000000000..e8b8ead5c5
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java
@@ -0,0 +1,145 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+@Timeout(10)
+class UserFunctionStartCleanupTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void partialStartFailureRestoresContextOnReusableDirectThread(boolean executionView) throws Exception {
+ var thread = Executors.newSingleThreadExecutor(task -> daemon(task, "reused-direct-probe"));
+ var fatal = new InternalError("later start hook");
+ var starts = new AtomicInteger();
+ var ends = new AtomicInteger();
+ var settings = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> new ExtractedContext(
+ "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED))
+ .build();
+ var otel = executionView
+ ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), settings)
+ : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), settings);
+ DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() {
+ public void onUserFunctionStart(UserFunctionStartInfo start) {
+ starts.incrementAndGet();
+ }
+
+ public void onUserFunctionEnd(UserFunctionEndInfo end) {
+ ends.incrementAndGet();
+ }
+ };
+ DurableExecutionPluginFactory faulty = info -> new DurableExecutionPlugin() {
+ public void onUserFunctionStart(UserFunctionStartInfo start) {
+ throw fatal;
+ }
+ };
+ try {
+ thread.submit(() -> {
+ assertFalse(Span.current().getSpanContext().isValid());
+ assertSame(
+ fatal,
+ assertThrows(
+ InternalError.class,
+ () -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, ctx) -> {
+ ctx.stepAsync("trigger", String.class, step -> "unreachable");
+ return "done";
+ },
+ config(
+ new LocalMemoryExecutionClient(),
+ new DirectExecutor(),
+ otel,
+ observer,
+ faulty))));
+ })
+ .get(5, TimeUnit.SECONDS);
+ assertEquals(1, starts.get());
+ assertEquals(1, ends.get());
+ var reusedSpan =
+ thread.submit(() -> Span.current().getSpanContext()).get(2, TimeUnit.SECONDS);
+ assertFalse(reusedSpan.isValid(), "attempt cleanup must not restore an ended handler span");
+ } finally {
+ thread.shutdownNow();
+ thread.awaitTermination(2, TimeUnit.SECONDS);
+ }
+ }
+
+ private static DurableConfig config(
+ LocalMemoryExecutionClient client, ExecutorService workers, DurableExecutionPluginFactory... plugins) {
+ return DurableConfig.builder()
+ .withDurableExecutionClient(client)
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(plugins)
+ .build();
+ }
+
+ private static DurableExecutionInput input() {
+ var operation = Operation.builder()
+ .id("id")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id",
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operation).build());
+ }
+
+ private static Thread daemon(Runnable task, String name) {
+ var thread = new Thread(task, name);
+ thread.setDaemon(true);
+ thread.setUncaughtExceptionHandler((owner, error) -> {});
+ return thread;
+ }
+
+ private static final class DirectExecutor extends AbstractExecutorService {
+ public void execute(Runnable task) {
+ task.run();
+ }
+
+ public void shutdown() {}
+
+ public List shutdownNow() {
+ return List.of();
+ }
+
+ public boolean isShutdown() {
+ return false;
+ }
+
+ public boolean isTerminated() {
+ return false;
+ }
+
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return true;
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VirtualOperationErrorStatusTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VirtualOperationErrorStatusTest.java
index fa62205d15..e33dee0ee2 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VirtualOperationErrorStatusTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VirtualOperationErrorStatusTest.java
@@ -27,6 +27,7 @@
import software.amazon.lambda.durable.exception.DurableOperationException;
import software.amazon.lambda.durable.model.ExecutionStatus;
import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -85,7 +86,9 @@ public void onOperationEnd(OperationEndInfo info) {
context.wait("pause", Duration.ofSeconds(1));
return "handled";
},
- DurableConfig.builder().withPlugins(plugin, listener).build());
+ DurableConfig.builder()
+ .withPlugins(plugin, ignored -> listener)
+ .build());
assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
assertEquals(
ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus());
@@ -141,8 +144,9 @@ public void onOperationEnd(OperationEndInfo info) {
})
void missingStartInvocationSegmentRetainsVirtualFailure(String outcome) {
try (var exporter = InMemorySpanExporter.create()) {
- var plugin = plugin(false, exporter);
- plugin.onInvocationStart(new InvocationInfo("request", "arn:virtual", true, Instant.EPOCH));
+ var factory = plugin(false, exporter);
+ var plugin =
+ Invocations.started(factory, new InvocationInfo("request", "arn:virtual", true, Instant.EPOCH));
plugin.onOperationEnd(new OperationEndInfo(
"virtual",
NAME,
@@ -170,13 +174,15 @@ void missingStartInvocationSegmentRetainsVirtualFailure(String outcome) {
}
}
- private static DurableExecutionPlugin plugin(boolean executionView, InMemorySpanExporter exporter) {
+ private static DurableExecutionPluginFactory plugin(boolean executionView, InMemorySpanExporter exporter) {
var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
var config = OtelPluginConfig.builder()
.enableMdc(false)
.contextExtractor(() -> null)
.build();
- return executionView ? new ExecutionOtelPlugin(provider, config) : new InvocationOtelPlugin(provider, config);
+ return executionView
+ ? ExecutionOtelPlugin.factory(provider, config)
+ : InvocationOtelPlugin.factory(provider, config);
}
private static RuntimeException failure(String outcome) {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java
index d20e557028..9c5b739ac6 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java
@@ -26,7 +26,7 @@
import org.junit.jupiter.params.provider.CsvSource;
import software.amazon.lambda.durable.DurableConfig;
import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
class VisibleReplacementSdkSpanPolicyTest {
@@ -88,8 +88,8 @@ public ContextPropagators getPropagators() {
return ContextPropagators.noop();
}
});
- DurableExecutionPlugin plugin =
- view.equals("invocation") ? new InvocationOtelPlugin() : new ExecutionOtelPlugin();
+ DurableExecutionPluginFactory plugin =
+ view.equals("invocation") ? InvocationOtelPlugin.factory() : ExecutionOtelPlugin.factory();
var runner = LocalDurableTestRunner.create(
String.class,
(input, context) -> {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java
index 0a02a66e9e..e71a3b45f2 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java
@@ -4,12 +4,101 @@
import static org.junit.jupiter.api.Assertions.*;
+import java.time.Instant;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
class XRayContextExtractorTest {
+ private static final InvocationInfo INFO = new InvocationInfo("request", "arn", true, Instant.EPOCH);
+
+ @ParameterizedTest
+ @ValueSource(strings = {"0", "1"})
+ void invocationHeaderOverridesGlobalCarrier(String sampled) {
+ System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-ffffffff-ffffffffffffffffffffffff;Sampled=1");
+ try {
+ var header = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=" + sampled;
+ var extracted = new XRayContextExtractor().extract(invocation(header));
+ assertEquals("6955b900123456789012345678901234", extracted.traceId());
+ assertEquals("1234567890123456", extracted.parentSpanId());
+ assertEquals(
+ sampled.equals("1") ? ExtractedContext.Sampling.SAMPLED : ExtractedContext.Sampling.NOT_SAMPLED,
+ extracted.sampling());
+ assertNull(new XRayContextExtractor().extract(invocation("malformed")));
+ } finally {
+ System.clearProperty("com.amazonaws.xray.traceHeader");
+ }
+ }
+
+ @ParameterizedTest
+ @NullAndEmptySource
+ void unavailableAccessorFallsBackButCapturedMissingHeaderDoesNot(String header) {
+ System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234;Sampled=0");
+ try {
+ var extracted = new XRayContextExtractor().extract(invocation(header));
+ if (header == null) assertEquals(ExtractedContext.Sampling.NOT_SAMPLED, extracted.sampling());
+ else assertNull(extracted, "an available but empty runtime snapshot must not borrow the global carrier");
+ } finally {
+ System.clearProperty("com.amazonaws.xray.traceHeader");
+ }
+ }
+
+ @Test
+ void existingCustomExtractorStillReceivesOneCall() {
+ var expected =
+ new ExtractedContext("6955b900123456789012345678901234", null, ExtractedContext.Sampling.SAMPLED);
+ ContextExtractor custom = () -> expected;
+ assertSame(expected, custom.extract(invocation("ignored")));
+ }
+
+ @Test
+ void existingNoArgumentSubclassOverrideKeepsControlWithInvocationHeader() {
+ var expected =
+ new ExtractedContext("6955b900aaaaaaaaaaaaaaaaaaaaaaaa", null, ExtractedContext.Sampling.NOT_SAMPLED);
+ var extractor = new XRayContextExtractor() {
+ @Override
+ public ExtractedContext extract() {
+ return expected;
+ }
+ };
+ assertSame(
+ expected,
+ extractor.extract(
+ invocation("Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1")));
+ }
+
+ @Test
+ void delegatingSubclassUsesInvocationHeaderAndClearsItAfterExtraction() {
+ var extractor = new XRayContextExtractor() {
+ @Override
+ public ExtractedContext extract() {
+ var extracted = super.extract();
+ if (extracted != null && extracted.sampling() == ExtractedContext.Sampling.NOT_SAMPLED) {
+ throw new IllegalArgumentException("custom extraction failure");
+ }
+ return extracted;
+ }
+ };
+ var header = "Root=1-6955b900-123456789012345678901234;Sampled=";
+ assertNotNull(extractor.extract(invocation(header + "1")));
+ assertNull(extractor.extract());
+ assertThrows(IllegalArgumentException.class, () -> extractor.extract(invocation(header + "0")));
+ assertNull(extractor.extract());
+ }
+
+ private static InvocationInfo invocation(String header) {
+ return new InvocationInfo(
+ INFO.requestId(),
+ INFO.durableExecutionArn(),
+ INFO.isFirstInvocation(),
+ INFO.executionStartTime(),
+ INFO.executionInput(),
+ INFO.operations(),
+ INFO.updatedOperations(),
+ header);
+ }
@Test
void extract_withoutEnvVar_returnsNull() {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java
new file mode 100644
index 0000000000..eec55724cd
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java
@@ -0,0 +1,85 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Map;
+import javax.tools.ToolProvider;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+
+class XRayExtractorBinaryCompatibilityTest {
+ @Test
+ void subclassCompiledAgainstLegacyApiRetainsOverride(@TempDir Path directory) throws Exception {
+ var baseline = directory.resolve("baseline");
+ var consumer = directory.resolve("consumer");
+ var classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"));
+ compile(baseline, "XRayContextExtractor", """
+ package software.amazon.lambda.durable.otel;
+ public class XRayContextExtractor {
+ public ExtractedContext extract() { return null; }
+ }
+ """, classpath);
+ compile(consumer, "LegacyExtractor", """
+ import software.amazon.lambda.durable.otel.*;
+ public class LegacyExtractor extends XRayContextExtractor {
+ public int calls;
+ @Override public ExtractedContext extract() {
+ calls++;
+ return new ExtractedContext("6955b900aaaaaaaaaaaaaaaaaaaaaaaa", null,
+ ExtractedContext.Sampling.NOT_SAMPLED);
+ }
+ }
+ """, baseline + File.pathSeparator + classpath);
+ verifyConsumer(consumer);
+ }
+
+ private void verifyConsumer(Path consumer) throws Exception {
+ // Only the consumer is loaded from disk: its superclass resolves to the current SDK in the parent loader.
+ try (var loader = new URLClassLoader(
+ new URL[] {consumer.toUri().toURL()}, getClass().getClassLoader())) {
+ var type = loader.loadClass("LegacyExtractor");
+ var extractor = (XRayContextExtractor) type.getConstructor().newInstance();
+ var runtime = ("Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1");
+ var info = new InvocationInfo("request", "arn", true, Instant.EPOCH, null, Map.of(), Map.of(), runtime);
+ var extracted = extractor.extract(info);
+ assertEquals("6955b900aaaaaaaaaaaaaaaaaaaaaaaa", extracted.traceId());
+ assertEquals(ExtractedContext.Sampling.NOT_SAMPLED, extracted.sampling());
+ assertEquals(1, type.getField("calls").get(extractor));
+ }
+ }
+
+ private static void compile(Path directory, String name, String source, String classpath) throws Exception {
+ Files.createDirectories(directory);
+ var file = directory.resolve("src").resolve(name + ".java");
+ Files.createDirectories(file.getParent());
+ Files.writeString(file, source);
+ var errors = new ByteArrayOutputStream();
+ var compiler = ToolProvider.getSystemJavaCompiler();
+ assertNotNull(compiler, "Binary compatibility test requires a JDK");
+ assertEquals(
+ 0,
+ compiler.run(
+ null,
+ null,
+ errors,
+ "--release",
+ "17",
+ "-classpath",
+ classpath,
+ "-d",
+ directory.toString(),
+ file.toString()),
+ errors.toString(StandardCharsets.UTF_8));
+ }
+}
diff --git a/pom.xml b/pom.xml
index 87e21af7f2..59b6e3e46f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
software.amazon.lambda.durable
aws-durable-execution-sdk-java-parent
- 2.3.1-SNAPSHOT
+ 3.0.0-SNAPSHOT
pom
AWS Lambda Durable Execution SDK
@@ -46,6 +46,7 @@
telemetry-exporter
examples
conformance-tests
+ lmi-tests
coverage-report
diff --git a/sdk-integration-tests/pom.xml b/sdk-integration-tests/pom.xml
index 188df4ab78..d28680a796 100644
--- a/sdk-integration-tests/pom.xml
+++ b/sdk-integration-tests/pom.xml
@@ -7,7 +7,7 @@
software.amazon.lambda.durable
aws-durable-execution-sdk-java-parent
- 2.3.1-SNAPSHOT
+ 3.0.0-SNAPSHOT
aws-durable-execution-sdk-java-integration-tests
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/CheckpointTokenRevocationIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/CheckpointTokenRevocationIntegrationTest.java
index 29a487efcd..7daff793a0 100644
--- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/CheckpointTokenRevocationIntegrationTest.java
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/CheckpointTokenRevocationIntegrationTest.java
@@ -46,7 +46,7 @@ void acceptedStepOutcomeReplaysAfterResumeWithoutRepeatingUserCode(boolean fail)
void acceptedStepOutcomeIsReportedToPluginsAfterResume(boolean fail) {
var calls = new AtomicInteger();
var plugin = new RecordingPlugin();
- var config = DurableConfig.builder().withPlugins(plugin).build();
+ var config = DurableConfig.builder().withPlugins(ignoredInfo -> plugin).build();
var runner = createPausedRunner(fail, calls, config);
assertEquals(ExecutionStatus.PENDING, runner.runUntilComplete("input").getStatus());
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java
deleted file mode 100644
index de8b5c5e01..0000000000
--- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java
+++ /dev/null
@@ -1,414 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.*;
-import java.util.function.BiFunction;
-import java.util.stream.Stream;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-import org.junit.jupiter.params.provider.ValueSource;
-import software.amazon.awssdk.services.lambda.model.*;
-import software.amazon.lambda.durable.config.WaitForConditionConfig;
-import software.amazon.lambda.durable.context.BaseContextImpl;
-import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
-import software.amazon.lambda.durable.execution.DurableExecutor;
-import software.amazon.lambda.durable.execution.ExecutionManager;
-import software.amazon.lambda.durable.model.DurableExecutionInput;
-import software.amazon.lambda.durable.model.DurableExecutionOutput;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.model.WaitForConditionResult;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
-import software.amazon.lambda.durable.plugin.InvocationEndInfo;
-import software.amazon.lambda.durable.plugin.InvocationStatus;
-import software.amazon.lambda.durable.serde.JacksonSerDes;
-import software.amazon.lambda.durable.serde.SerDes;
-import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
-import software.amazon.lambda.durable.util.ExceptionHelper;
-
-class ContinuationEndCleanupOrderTest {
- static Stream cleanupCases() {
- return Stream.of("async", "inline-root", "submit-root")
- .flatMap(mode -> Stream.of("serde", "reject", "vm", "death").map(kind -> Arguments.of(mode, kind)));
- }
-
- @SuppressWarnings("removal")
- @ParameterizedTest
- @MethodSource("cleanupCases")
- void continuationFailureMustWakeBodyBeforeEndWaitsForCleanup(String mode, String kind) throws Exception {
- boolean rejection = kind.equals("reject");
- boolean withPlugin = true;
- Throwable failure = switch (kind) {
- case "reject" -> new RejectedExecutionException("resumed worker rejected");
- case "vm" -> new InternalError("resumed state fatal");
- case "death" -> new ThreadDeath();
- default -> new IllegalArgumentException("resumed state cannot deserialize");
- };
- var injectionThread = new AtomicReference();
- var handlerThread = new AtomicReference();
- var callerThread = new AtomicReference();
- var escaped = new CountDownLatch(1);
- var escapedFailure = new AtomicReference();
- var oldHandler = Thread.getDefaultUncaughtExceptionHandler();
- Thread.setDefaultUncaughtExceptionHandler((thread, error) -> {
- if (error == failure) {
- escapedFailure.set(error);
- escaped.countDown();
- } else if (oldHandler != null) oldHandler.uncaughtException(thread, error);
- });
- var managerForCleanup = new AtomicReference();
- var armed = new AtomicBoolean();
- var injected = new CountDownLatch(1);
- var injectionOwner = new AtomicReference();
- var checks = new AtomicInteger();
- var bodyFinally = new CountDownLatch(1);
- var cleanupOrder = new CopyOnWriteArrayList();
- var ends = new CopyOnWriteArrayList();
- var pollEntered = new CountDownLatch(1);
- var releaseReady = new CountDownLatch(1);
- var firstArmedPoll = new AtomicBoolean(true);
- var onCoordinator = new AtomicBoolean();
- var pendingCall = new AtomicReference>();
- var serde = new SerDes() {
- private final JacksonSerDes delegate = new JacksonSerDes();
-
- public String serialize(Object value) {
- return delegate.serialize(value);
- }
-
- public T deserialize(String value, TypeToken type) {
- if (!rejection
- && armed.get()
- && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) {
- injectionThread.set(Thread.currentThread());
- injectionOwner.set(Thread.currentThread().getName());
- onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace())
- .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation")));
- injected.countDown();
- ExceptionHelper.sneakyThrow(failure);
- }
- return delegate.deserialize(value, type);
- }
- };
- var client = new LocalMemoryExecutionClient() {
- @Override
- public CheckpointDurableExecutionResponse checkpoint(
- String arn, String token, List updates) {
- if (armed.get() && updates.isEmpty() && firstArmedPoll.compareAndSet(true, false)) {
- pollEntered.countDown();
- try {
- assertTrue(releaseReady.await(3, TimeUnit.SECONDS));
- } catch (InterruptedException interrupted) {
- throw new AssertionError(interrupted);
- }
- }
- return super.checkpoint(arn, token, updates);
- }
- };
- var workers = new RootExecutor(mode, () -> {
- if (rejection && armed.get() && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) {
- injectionThread.set(Thread.currentThread());
- injectionOwner.set(Thread.currentThread().getName());
- onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace())
- .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation")));
- injected.countDown();
- ExceptionHelper.sneakyThrow(failure);
- }
- });
- var caller = Executors.newSingleThreadExecutor(task -> {
- var thread = new Thread(task, "legacy-outcome-caller");
- callerThread.set(thread);
- return thread;
- });
- var builder = DurableConfig.builder()
- .withExecutorService(workers)
- .withDurableExecutionClient(client)
- .withCheckpointDelay(Duration.ZERO)
- .withPollingStrategy(attempt -> Duration.ZERO);
- if (withPlugin)
- builder.withPlugins(new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- ends.add(info);
- if (info.invocationStatus() == InvocationStatus.RETRYING) {
- assertSame(
- handlerThread.get(),
- Thread.currentThread(),
- "The merged lifecycle runs End on the handler owner after its finally block");
- cleanupOrder.add("end-enter");
- try {
- boolean finished = bodyFinally.await(2, TimeUnit.SECONDS);
- cleanupOrder.add(finished ? "end-complete" : "end-timed-out");
- assertTrue(finished, "End must not prevent the manager from waking handler cleanup");
- } catch (InterruptedException interrupted) {
- throw new AssertionError(interrupted);
- }
- }
- }
- });
- var config = builder.build();
- var condition = WaitForConditionConfig.builder()
- .initialState(1)
- .serDes(serde)
- .waitStrategy((state, attempt) -> Duration.ofSeconds(1))
- .build();
- BiFunction handler = (input, context) -> {
- handlerThread.set(Thread.currentThread());
- managerForCleanup.set(((BaseContextImpl) context).getExecutionManager());
- var waiting = context.waitForConditionAsync(
- "condition",
- Integer.class,
- (state, step) -> {
- checks.incrementAndGet();
- return state == 2
- ? WaitForConditionResult.stopPolling(state)
- : WaitForConditionResult.continuePolling(2);
- },
- condition);
- if (armed.get()) {
- try {
- assertTrue(pollEntered.await(3, TimeUnit.SECONDS));
- } catch (InterruptedException interrupted) {
- throw new AssertionError(interrupted);
- }
- }
- try {
- return String.valueOf(waiting.get());
- } finally {
- if (armed.get()) {
- cleanupOrder.add("body-finally");
- bodyFinally.countDown();
- }
- }
- };
- try {
- var first = caller.submit(() -> DurableExecutor.execute(
- input(List.of()), null, TypeToken.get(String.class), handler, config))
- .get(3, TimeUnit.SECONDS);
- assertEquals(ExecutionStatus.PENDING, first.status());
- var cachedPending = new ArrayList<>(client.getAllOperations());
- assertEquals(
- OperationStatus.PENDING,
- client.getOperationByName("condition").status());
- assertTrue(client.advanceTime());
- armed.set(true);
- workers.nextInvocation();
- var second = caller.submit(() ->
- DurableExecutor.execute(input(cachedPending), null, TypeToken.get(String.class), handler, config));
- pendingCall.set(second);
- assertTrue(pollEntered.await(3, TimeUnit.SECONDS));
- if (mode.equals("async")) awaitCallerOutcomeJoin(callerThread.get());
- releaseReady.countDown();
- assertTrue(injected.await(3, TimeUnit.SECONDS));
- assertTrue(onCoordinator.get(), "Fault must run in the actual WFC coordinator continuation");
- DurableExecutionOutput output = null;
- Throwable callerFailure = null;
- try {
- output = second.get(5, TimeUnit.SECONDS);
- } catch (ExecutionException observationFailure) {
- callerFailure = observationFailure.getCause();
- }
- System.out.println("CONTINUATION_ORDINARY_PUBLIC owner=" + injectionOwner.get()
- + " caller="
- + (output != null
- ? output.status()
- : callerFailure.getClass().getName())
- + " backend=" + client.getOperationByName("condition").status()
- + " checks=" + checks.get());
- assertEquals(1, checks.get(), "A failed resumed state read must not rerun the condition body");
- System.out.println("CONTINUATION_END_CLEANUP_ORDER " + cleanupOrder);
- assertFalse(cleanupOrder.contains("end-timed-out"));
- assertTrue(cleanupOrder.contains("end-complete"));
- assertTrue(cleanupOrder.indexOf("body-finally") < cleanupOrder.indexOf("end-complete"));
- if (failure instanceof Error) {
- assertTrue(escaped.await(3, TimeUnit.SECONDS));
- assertSame(failure, escapedFailure.get());
- }
- assertNull(output, "An SDK continuation failure must not produce a durable response");
- var retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, callerFailure);
- assertTrue(retry.isRetryable());
- assertSame(failure, retry.getCause());
- assertEquals(
- OperationStatus.READY,
- client.getOperationByName("condition").status());
- var activeField = ExecutionManager.class.getDeclaredField("activeThreads");
- activeField.setAccessible(true);
- assertFalse(
- ((Set) activeField.get(managerForCleanup.get()))
- .contains(client.getOperationByName("condition").id()),
- "A rejected operation worker must not leave a phantom activity registration");
- if (withPlugin)
- assertEquals(
- List.of(InvocationStatus.PENDING, InvocationStatus.RETRYING),
- ends.stream().map(InvocationEndInfo::invocationStatus).toList());
- else assertTrue(ends.isEmpty());
- armed.set(false);
- workers.nextInvocation();
- var third = caller.submit(() -> DurableExecutor.execute(
- input(client.getAllOperations()), null, TypeToken.get(String.class), handler, config));
- pendingCall.set(third);
- assertEquals(
- ExecutionStatus.SUCCEEDED, third.get(3, TimeUnit.SECONDS).status());
- assertEquals(2, checks.get(), "Only the original and successful resumed predicates run");
- } finally {
- releaseReady.countDown();
- // Cleanup-only escape hatch AFTER the real public invocation observation/timeout; never test behavior.
- var manager = managerForCleanup.get();
- if (pendingCall.get() != null
- && !pendingCall.get().isDone()
- && manager != null
- && !manager.isExecutionCompletedExceptionally()) {
- try {
- manager.terminateExecution(new UnrecoverableDurableExecutionException(
- ErrorObject.builder().errorMessage("probe cleanup").build(), true));
- } catch (UnrecoverableDurableExecutionException expected) {
- }
- }
- Thread.setDefaultUncaughtExceptionHandler(oldHandler);
- caller.shutdownNow();
- workers.shutdownNow();
- assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
- assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"async", "inline-root", "submit-root"})
- void ordinaryCompletionEndsOnItsHandlerOwner(String mode) throws Exception {
- var releaseBody = new CountDownLatch(1);
- var entered = new CountDownLatch(1);
- var callerThread = new AtomicReference();
- var bodyThread = new AtomicReference();
- var endThread = new AtomicReference();
- var workers = new RootExecutor(mode, () -> {});
- var caller = Executors.newSingleThreadExecutor(task -> {
- var thread = new Thread(task, "normal-outcome-caller");
- callerThread.set(thread);
- return thread;
- });
- var config = DurableConfig.builder()
- .withExecutorService(workers)
- .withDurableExecutionClient(new LocalMemoryExecutionClient())
- .withPlugins(new DurableExecutionPlugin() {
- public void onInvocationEnd(InvocationEndInfo info) {
- endThread.set(Thread.currentThread());
- }
- })
- .build();
- try {
- var call = caller.submit(() -> DurableExecutor.execute(
- input(List.of()),
- null,
- TypeToken.get(String.class),
- (value, context) -> {
- bodyThread.set(Thread.currentThread());
- entered.countDown();
- if (mode.equals("async")) {
- try {
- assertTrue(releaseBody.await(3, TimeUnit.SECONDS));
- } catch (InterruptedException interrupted) {
- throw new AssertionError(interrupted);
- }
- }
- return "done";
- },
- config));
- assertTrue(entered.await(3, TimeUnit.SECONDS));
- if (mode.equals("async")) awaitCallerOutcomeJoin(callerThread.get());
- releaseBody.countDown();
- assertEquals(
- ExecutionStatus.SUCCEEDED, call.get(3, TimeUnit.SECONDS).status());
- assertSame(bodyThread.get(), endThread.get());
- } finally {
- releaseBody.countDown();
- caller.shutdownNow();
- workers.shutdownNow();
- assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
- assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
- }
- }
-
- /** Wait for the actual public invocation caller to install its outcome observer and enter join, not a delay. */
- private static void awaitCallerOutcomeJoin(Thread caller) {
- var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3);
- while (System.nanoTime() < deadline) {
- if (Arrays.stream(caller.getStackTrace())
- .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName())
- && frame.getMethodName().equals("join"))) return;
- Thread.yield();
- }
- fail("Invocation caller did not reach its installed outcome observer");
- }
-
- /** Root dispatch exercises inline/submit-and-wait; operation children remain asynchronous as supported. */
- private static final class RootExecutor extends ThreadPoolExecutor {
- private final String mode;
- private final Runnable beforeDispatch;
- private final AtomicBoolean root = new AtomicBoolean(true);
-
- RootExecutor(String mode, Runnable beforeDispatch) {
- super(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<>());
- this.mode = mode;
- this.beforeDispatch = beforeDispatch;
- }
-
- void nextInvocation() {
- root.set(true);
- }
-
- @Override
- public void execute(Runnable task) {
- beforeDispatch.run();
- if (root.compareAndSet(true, false)) {
- if (mode.equals("inline-root")) {
- task.run();
- return;
- }
- if (mode.equals("submit-root")) {
- var done = new CompletableFuture();
- super.execute(() -> {
- try {
- task.run();
- } finally {
- done.complete(null);
- }
- });
- try {
- done.get(5, TimeUnit.SECONDS);
- } catch (Exception failure) {
- throw new AssertionError(failure);
- }
- return;
- }
- }
- super.execute(task);
- }
- }
-
- private static DurableExecutionInput input(List operations) {
- var execution = Operation.builder()
- .id("execution")
- .type(OperationType.EXECUTION)
- .status(OperationStatus.STARTED)
- .startTimestamp(Instant.EPOCH)
- .executionDetails(
- ExecutionDetails.builder().inputPayload("\"input\"").build())
- .build();
- var history = new ArrayList();
- history.add(execution);
- history.addAll(operations);
- return new DurableExecutionInput(
- "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fatal/execution",
- "token",
- CheckpointUpdatedExecutionState.builder().operations(history).build());
- }
-}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java
index 7837527b50..3f10533b7c 100644
--- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java
@@ -109,7 +109,7 @@ public void execute(Runnable task) {
.withCheckpointDelay(Duration.ZERO)
.withPollingStrategy(attempt -> Duration.ZERO);
if (withPlugin)
- builder.withPlugins(new DurableExecutionPlugin() {
+ builder.withPlugins(invocation -> new DurableExecutionPlugin() {
public void onInvocationEnd(InvocationEndInfo info) {
ends.add(info);
}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java
index 43e76b8cb3..3b49869932 100644
--- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java
@@ -115,7 +115,7 @@ public CheckpointDurableExecutionResponse checkpoint(
.withCheckpointDelay(Duration.ZERO)
.withPollingStrategy(attempt -> Duration.ZERO);
if (withPlugin)
- builder.withPlugins(new DurableExecutionPlugin() {
+ builder.withPlugins(invocation -> new DurableExecutionPlugin() {
public void onInvocationEnd(InvocationEndInfo info) {
ends.add(info);
if (info.invocationStatus() == InvocationStatus.RETRYING && endFailure != null) throw endFailure;
@@ -188,12 +188,17 @@ public void onInvocationEnd(InvocationEndInfo info) {
assertSame(endFailure, callerFailure, "Existing Root End JVM-fatal precedence remains");
assertTrue(endEscaped.await(3, TimeUnit.SECONDS));
assertSame(endFailure, endOwnerFailure.get());
- assertEquals(1, endFailure.getSuppressed().length);
- retry = assertInstanceOf(
- UnrecoverableDurableExecutionException.class, endFailure.getSuppressed()[0]);
+ // Major resource closure may additionally retain its own CompletionException diagnostic.
+ retry = Arrays.stream(endFailure.getSuppressed())
+ .filter(UnrecoverableDurableExecutionException.class::isInstance)
+ .map(UnrecoverableDurableExecutionException.class::cast)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Missing original retry control"));
} else {
retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, callerFailure);
- if (endFailure != null) assertArrayEquals(new Throwable[] {endFailure}, retry.getSuppressed());
+ // Factory API 2 contains ordinary End Errors; they do not replace or attach to retry control.
+ if (endFailure != null)
+ assertFalse(Arrays.asList(retry.getSuppressed()).contains(endFailure));
}
assertTrue(retry.isRetryable());
assertSame(fatal, retry.getCause());
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java
new file mode 100644
index 0000000000..104414f34c
--- /dev/null
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java
@@ -0,0 +1,138 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.config.StepConfig;
+import software.amazon.lambda.durable.config.StepSemantics;
+import software.amazon.lambda.durable.context.BaseContextImpl;
+import software.amazon.lambda.durable.context.DurableContextImpl;
+import software.amazon.lambda.durable.execution.ExecutionManager;
+import software.amazon.lambda.durable.execution.ThreadContext;
+import software.amazon.lambda.durable.execution.ThreadType;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+@Timeout(10)
+class DelayedCheckpointFatalIntegrationTest {
+ @SuppressWarnings("removal")
+ static Stream failures() {
+ return Stream.of(false, true)
+ .flatMap(wrapped -> Stream.of(new InternalError("external plugin fatal"), new ThreadDeath())
+ .map(fatal -> Arguments.of(wrapped, fatal)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("failures")
+ void externalFatalSettlesDelayedAtMostOnceStartBeforeItsTimer(boolean wrapped, Error fatal) throws Exception {
+ var startQueued = new CountDownLatch(1);
+ var startFuture = new AtomicReference>();
+ var bodies = new AtomicInteger();
+ var backendCalls = new AtomicInteger();
+ var workers = Executors.newCachedThreadPool(task -> {
+ var thread = new Thread(task, "delayed-start-owner");
+ thread.setDaemon(true);
+ thread.setUncaughtExceptionHandler((owner, failure) -> {});
+ return thread;
+ });
+ DurableExecutionPluginFactory fault = ignored -> new DurableExecutionPlugin() {
+ public void onOperationStart(OperationInfo info) {
+ if (!"trigger".equals(info.name())) return;
+ if (wrapped) throw new CompletionException(new ExecutionException(fatal));
+ throw fatal;
+ }
+ };
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ofMinutes(1))
+ .withDurableExecutionClient(new LocalMemoryExecutionClient() {
+ public CheckpointDurableExecutionResponse checkpoint(
+ String arn, String token, List batch) {
+ backendCalls.incrementAndGet();
+ return super.checkpoint(arn, token, batch);
+ }
+ })
+ .withPlugins(fault)
+ .build();
+ var input = input();
+ var manager = new ExecutionManager(input, config, null) {
+ public CompletableFuture sendOperationUpdate(OperationUpdate update) {
+ var result = super.sendOperationUpdate(update);
+ if ("blocked".equals(update.name()) && update.action() == OperationAction.START) {
+ startFuture.set(result);
+ startQueued.countDown();
+ }
+ return result;
+ }
+ };
+ try {
+ manager.registerActiveThread(null);
+ manager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT));
+ manager.getPluginRunner()
+ .onInvocationStart(new InvocationInfo("request", input.durableExecutionArn(), true, Instant.EPOCH));
+ var context = DurableContextImpl.createRootContext(manager, config, null);
+ BaseContextImpl.setCurrentContext(context);
+ context.stepAsync(
+ "blocked",
+ String.class,
+ step -> {
+ bodies.incrementAndGet();
+ return "unexpected";
+ },
+ StepConfig.builder()
+ .semanticsPerRetry(StepSemantics.AT_MOST_ONCE_PER_RETRY)
+ .build());
+ assertTrue(startQueued.await(3, TimeUnit.SECONDS));
+ assertSame(
+ fatal,
+ assertThrows(Error.class, () -> context.stepAsync("trigger", String.class, step -> "unreachable")));
+ assertSame(
+ fatal,
+ assertThrows(
+ ExecutionException.class,
+ () -> startFuture.get().get(500, TimeUnit.MILLISECONDS))
+ .getCause());
+ assertEquals(0, bodies.get());
+ assertEquals(0, backendCalls.get());
+ } finally {
+ // Keep the negative control bounded even before the production abort path exists.
+ if (startFuture.get() != null) startFuture.get().completeExceptionally(fatal);
+ workers.shutdown();
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ try {
+ manager.close();
+ } catch (CompletionException failure) {
+ assertSame(fatal, failure.getCause());
+ }
+ BaseContextImpl.setCurrentContext(null);
+ }
+ }
+
+ private static DurableExecutionInput input() {
+ var operation = Operation.builder()
+ .id("id")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id",
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operation).build());
+ }
+}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FailureDeliveryFinalizationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FailureDeliveryFinalizationTest.java
new file mode 100644
index 0000000000..f1b5962f78
--- /dev/null
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FailureDeliveryFinalizationTest.java
@@ -0,0 +1,119 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.slf4j.LoggerFactory;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.serde.JacksonSerDes;
+import software.amazon.lambda.durable.serde.SerDes;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class FailureDeliveryFinalizationTest {
+ @Test
+ @Timeout(10)
+ void throwingDiagnosticStillFinalizesOnceAndReplaysSavedWork() {
+ assertFalse(LoggerFactory.getLogger(DurableExecutor.class).isDebugEnabled());
+ var rejectMessage = new AtomicBoolean(true);
+ var effects = new AtomicInteger();
+ var ends = new CopyOnWriteArrayList();
+ var diagnostic = new IllegalStateException("diagnostic getter failed");
+ var failure = new IllegalArgumentException("handler failure") {
+ public String getMessage() {
+ if (rejectMessage.get()) throw diagnostic;
+ return super.getMessage();
+ }
+
+ public String toString() {
+ return "application exception with a safe toString";
+ }
+ };
+ var config = DurableConfig.builder()
+ .withPlugins(info -> new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo end) {
+ ends.add(end);
+ }
+ })
+ .build();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ context.step("saved", String.class, step -> {
+ effects.incrementAndGet();
+ return "saved";
+ });
+ throw failure;
+ },
+ config);
+ assertSame(diagnostic, assertThrows(IllegalStateException.class, () -> runner.run("input")));
+ assertEquals(1, ends.size());
+ assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus());
+ assertSame(diagnostic, ends.get(0).executionError());
+ rejectMessage.set(false);
+ assertEquals(ExecutionStatus.FAILED, runner.run("input").getStatus());
+ assertEquals(2, ends.size());
+ assertEquals(InvocationStatus.FAILED, ends.get(1).invocationStatus());
+ assertEquals(1, effects.get());
+ }
+
+ @Test
+ @Timeout(10)
+ void failedResponseMustBePreparedBeforeTerminalEndAndReplaysSavedWork() {
+ var rejectError = new AtomicBoolean(true);
+ var effects = new AtomicInteger();
+ var factories = new AtomicInteger();
+ var ends = new CopyOnWriteArrayList();
+ var preparation = new IllegalStateException("error payload could not be encoded");
+ SerDes serializer = new SerDes() {
+ final JacksonSerDes delegate = new JacksonSerDes();
+
+ public String serialize(Object value) {
+ if (value instanceof Throwable && rejectError.get()) throw preparation;
+ return delegate.serialize(value);
+ }
+
+ public T deserialize(String value, TypeToken type) {
+ return delegate.deserialize(value, type);
+ }
+ };
+ var config = DurableConfig.builder()
+ .withSerDes(serializer)
+ .withPlugins(info -> {
+ factories.incrementAndGet();
+ return new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo end) {
+ ends.add(end);
+ }
+ };
+ })
+ .build();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ context.step("saved", String.class, step -> {
+ effects.incrementAndGet();
+ return "saved";
+ });
+ throw new IllegalArgumentException("ordinary handler failure");
+ },
+ config);
+ assertSame(preparation, assertThrows(IllegalStateException.class, () -> runner.run("input")));
+ assertEquals(1, ends.size());
+ assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus());
+ assertSame(preparation, ends.get(0).executionError());
+ rejectError.set(false);
+ assertEquals(ExecutionStatus.FAILED, runner.run("input").getStatus());
+ assertEquals(2, factories.get());
+ assertEquals(2, ends.size());
+ assertEquals(InvocationStatus.FAILED, ends.get(1).invocationStatus());
+ assertEquals(1, effects.get());
+ }
+}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FatalCheckpointResponseLifetimeTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FatalCheckpointResponseLifetimeTest.java
new file mode 100644
index 0000000000..1875b473be
--- /dev/null
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FatalCheckpointResponseLifetimeTest.java
@@ -0,0 +1,294 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.function.BiFunction;
+import java.util.stream.*;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.*;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.context.BaseContextImpl;
+import software.amazon.lambda.durable.execution.*;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+@Timeout(15)
+class FatalCheckpointResponseLifetimeTest {
+ private static final String ARN =
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/lifetime/execution";
+
+ static Stream fatalCases() {
+ return Stream.of("checkpoint", "page")
+ .flatMap(stage -> Stream.of(false, true)
+ .flatMap(holdEnd -> Stream.of(false, true).map(death -> Arguments.of(stage, holdEnd, death))));
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @MethodSource("fatalCases")
+ void fatalDrainDiscardsAbortedResponseBeforeEndAndKeepsCommittedReplay(String stage, boolean holdEnd, boolean death)
+ throws Exception {
+ Error fatal = death ? new ThreadDeath() : new InternalError("simulated peer fatal");
+ try (var run = new Run(stage, holdEnd, fatal)) {
+ var response = run.caller.submit(() -> DurableExecutor.execute(
+ input(List.of()), null, TypeToken.get(String.class), run.handler, run.config));
+ await(run.client.ready);
+ assertThrows(TimeoutException.class, () -> response.get(700, TimeUnit.MILLISECONDS));
+ assertEquals(1, run.endEntered.getCount(), "End cannot precede admitted transport settlement");
+ var before = snapshot(run.manager.get());
+ assertEquals(
+ OperationStatus.SUCCEEDED,
+ run.client.getOperationByName("saved").status(),
+ "The real local backend persisted the successful side effect before the response gate");
+ assertSame(
+ fatal,
+ ExceptionHelper.unwrapAsyncFailure(assertThrows(
+ CompletionException.class, () -> run.poller.get().join())));
+ run.client.release.countDown();
+ await(run.client.transportExited);
+ await(run.endEntered);
+ assertSame(fatal, run.client.transportFailure.get());
+ assertEquals(before, snapshot(run.manager.get()), "An aborted invocation must not publish a late response");
+ assertEquals(0, run.hooksAfterEnd.get(), "No new checkpoint lifecycle hook may run after End starts");
+ assertEquals(0, run.pollerSuccess.get());
+ assertEquals(0, run.client.pageTwoCalls.get(), "A fatal page return must not fetch another page");
+ run.releaseEnd.countDown();
+ assertSame(
+ fatal,
+ ExceptionHelper.unwrapAsyncFailure(
+ assertThrows(ExecutionException.class, () -> response.get(2, TimeUnit.SECONDS))));
+ assertEquals(1, run.ends.get());
+ assertEquals(1, run.effects.get());
+ run.manager.get().getPluginRunner().onInvocationEnd(null);
+ assertEquals(1, run.ends.get(), "Released plugins cannot receive another End");
+ // The next public invocation consumes persisted state; the completed side effect is not repeated.
+ run.fault.set(false);
+ var replay = DurableExecutor.execute(
+ input(run.client.getAllOperations()), null, TypeToken.get(String.class), run.handler, run.config);
+ assertEquals(ExecutionStatus.SUCCEEDED, replay.status());
+ assertEquals("\"saved-value\"", replay.result());
+ assertEquals(1, run.effects.get());
+ assertEquals(1, run.triggerBodies.get());
+ assertEquals(2, run.ends.get());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"checkpoint", "page"})
+ void ordinaryInFlightResponseStillPublishesAndCompletesPollers(String stage) throws Exception {
+ try (var run = new Run(stage, false, null)) {
+ var response = run.caller.submit(() -> DurableExecutor.execute(
+ input(List.of()), null, TypeToken.get(String.class), run.handler, run.config));
+ await(run.client.ready);
+ assertThrows(TimeoutException.class, () -> response.get(100, TimeUnit.MILLISECONDS));
+ assertEquals(0, run.ends.get());
+ run.client.release.countDown();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, response.get(3, TimeUnit.SECONDS).status());
+ assertEquals(OperationStatus.SUCCEEDED, run.poller.get().join().status());
+ assertEquals(1, run.pollerSuccess.get());
+ assertEquals(stage.equals("page") ? 1 : 0, run.client.pageTwoCalls.get());
+ assertEquals(1, run.effects.get());
+ assertEquals(1, run.ends.get());
+ assertEquals(
+ OperationStatus.SUCCEEDED,
+ snapshot(run.manager.get()).values().stream()
+ .filter(op -> "saved".equals(op.name()))
+ .findFirst()
+ .orElseThrow()
+ .status());
+ }
+ }
+
+ private static Map snapshot(ExecutionManager manager) {
+ return manager.getOperationsSnapshot().stream().collect(Collectors.toMap(Operation::id, op -> op));
+ }
+
+ private static DurableExecutionInput input(List persisted) {
+ var operations = new ArrayList();
+ operations.add(Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build());
+ operations.addAll(persisted);
+ return new DurableExecutionInput(
+ ARN,
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operations).build());
+ }
+
+ private static final class Run implements AutoCloseable {
+ final AtomicBoolean fault;
+ final AtomicReference manager = new AtomicReference<>();
+ final AtomicReference> poller = new AtomicReference<>();
+ final AtomicInteger pollerSuccess = new AtomicInteger();
+ final AtomicInteger effects = new AtomicInteger();
+ final AtomicInteger triggerBodies = new AtomicInteger();
+ final AtomicInteger ends = new AtomicInteger();
+ final AtomicBoolean endStarted = new AtomicBoolean();
+ final AtomicInteger hooksAfterEnd = new AtomicInteger();
+ final CountDownLatch endEntered = new CountDownLatch(1);
+ final CountDownLatch releaseEnd = new CountDownLatch(1);
+ final ExecutorService workers = Executors.newCachedThreadPool(task -> {
+ var thread = new Thread(task, "publication-user-owner");
+ thread.setDaemon(true);
+ thread.setUncaughtExceptionHandler((owner, failure) -> {});
+ return thread;
+ });
+ final ExecutorService caller = Executors.newSingleThreadExecutor();
+ final GatedClient client;
+ final DurableConfig config;
+ final BiFunction handler;
+
+ Run(String stage, boolean holdEnd, Error fatal) {
+ fault = new AtomicBoolean(fatal != null);
+ client = new GatedClient(stage, fatal);
+ config = DurableConfig.builder()
+ .withDurableExecutionClient(client)
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPollingStrategy(attempt -> Duration.ZERO)
+ .withPlugins(info -> {
+ endStarted.set(false);
+ return new DurableExecutionPlugin() {
+ public void onUserFunctionStart(UserFunctionStartInfo start) {
+ if (fault.get() && "trigger".equals(start.name())) throw fatal;
+ }
+
+ public void onOperationChange(OperationChangeInfo change) {
+ if (endStarted.get()) hooksAfterEnd.incrementAndGet();
+ }
+
+ public void onOperationEnd(OperationEndInfo end) {
+ if (endStarted.get()) hooksAfterEnd.incrementAndGet();
+ }
+
+ public void onInvocationEnd(InvocationEndInfo end) {
+ endStarted.set(true);
+ ends.incrementAndGet();
+ endEntered.countDown();
+ if (holdEnd && fault.get()) await(releaseEnd);
+ }
+ };
+ })
+ .build();
+ handler = (input, context) -> {
+ var actual = ((BaseContextImpl) context).getExecutionManager();
+ manager.compareAndSet(null, actual);
+ var saved = context.stepAsync("saved", String.class, step -> {
+ effects.incrementAndGet();
+ return "saved-value";
+ });
+ await(client.ready);
+ if (poller.get() == null) {
+ var waiting = actual.pollForOperationUpdates(
+ client.getOperationByName("saved").id());
+ waiting.thenAccept(ignored -> pollerSuccess.incrementAndGet());
+ poller.set(waiting);
+ }
+ context.step("trigger", String.class, step -> {
+ triggerBodies.incrementAndGet();
+ return "allowed";
+ });
+ return saved.get();
+ };
+ }
+
+ public void close() throws Exception {
+ client.release.countDown();
+ releaseEnd.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS));
+ }
+ }
+
+ private static final class GatedClient extends LocalMemoryExecutionClient {
+ final String stage;
+ final Error fatal;
+ final AtomicBoolean armed = new AtomicBoolean(true);
+ final CountDownLatch ready = new CountDownLatch(1);
+ final CountDownLatch release = new CountDownLatch(1);
+ final CountDownLatch transportExited = new CountDownLatch(1);
+ final AtomicReference transportFailure = new AtomicReference<>();
+ final AtomicInteger pageTwoCalls = new AtomicInteger();
+ List page;
+
+ GatedClient(String stage, Error fatal) {
+ this.stage = stage;
+ this.fatal = fatal;
+ }
+
+ public CheckpointDurableExecutionResponse checkpoint(String arn, String token, List updates) {
+ var response = super.checkpoint(arn, token, updates);
+ if (updates.stream().anyMatch(u -> "saved".equals(u.name()) && u.action() == OperationAction.SUCCEED)
+ && armed.compareAndSet(true, false)) {
+ if (stage.equals("checkpoint")) gate();
+ else {
+ page = response.newExecutionState().operations();
+ return response.toBuilder()
+ .newExecutionState(CheckpointUpdatedExecutionState.builder()
+ .operations(List.of())
+ .nextMarker("first-page")
+ .build())
+ .build();
+ }
+ }
+ return response;
+ }
+
+ public GetDurableExecutionStateResponse getExecutionState(String arn, String token, String marker) {
+ if (marker.equals("first-page")) {
+ gate();
+ return GetDurableExecutionStateResponse.builder()
+ .operations(page)
+ .nextMarker("second-page")
+ .build();
+ }
+ assertEquals("second-page", marker);
+ pageTwoCalls.incrementAndGet();
+ return GetDurableExecutionStateResponse.builder()
+ .operations(List.of())
+ .build();
+ }
+
+ private void gate() {
+ if (fatal != null) {
+ var owner = Thread.currentThread();
+ var original = owner.getUncaughtExceptionHandler();
+ owner.setUncaughtExceptionHandler((thread, failure) -> {
+ transportFailure.set(failure);
+ thread.setUncaughtExceptionHandler(original);
+ transportExited.countDown();
+ });
+ }
+ ready.countDown();
+ await(release);
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ assertTrue(latch.await(5, TimeUnit.SECONDS), "controlled response gate not released");
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerOutcomeObservationIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerOutcomeObservationIntegrationTest.java
new file mode 100644
index 0000000000..873bd6038a
--- /dev/null
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerOutcomeObservationIntegrationTest.java
@@ -0,0 +1,240 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.services.lambda.model.ErrorObject;
+import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.HandlerScoped;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class HandlerOutcomeObservationIntegrationTest {
+ static Stream outcomes() {
+ return Stream.of("inline", "submit-wait")
+ .flatMap(mode -> Stream.of(false, true)
+ .flatMap(retry -> Stream.of(false, true)
+ .flatMap(returnFinally -> Stream.of("none", "plain", "scoped")
+ .map(plugin -> Arguments.of(mode, retry, returnFinally, plugin)))));
+ }
+
+ static Stream successes() {
+ return Stream.of("inline", "submit-wait")
+ .flatMap(mode -> Stream.of("none", "plain", "scoped").map(plugin -> Arguments.of(mode, plugin)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("outcomes")
+ @Timeout(15)
+ void naturalManagerOutcomePrecedesFastHandlerFinally(
+ String mode, boolean retry, boolean returnFinally, String pluginKind) throws Exception {
+ var workers = new FirstDispatchExecutor(mode);
+ var hooks = new CopyOnWriteArrayList();
+ var effects = new AtomicInteger();
+ var attempts = new AtomicInteger();
+ var overrideFinally = new AtomicBoolean(true);
+ var original = new UnrecoverableDurableExecutionException(
+ ErrorObject.builder().errorMessage("retry invocation").build(), true);
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ context.step("saved", String.class, step -> {
+ effects.incrementAndGet();
+ return "saved";
+ });
+ try {
+ if (retry) {
+ context.step("retry-target", String.class, step -> {
+ if (attempts.incrementAndGet() == 1) throw original;
+ return "target";
+ });
+ } else context.wait("pause", Duration.ofSeconds(1));
+ return "done";
+ } finally {
+ if (overrideFinally.get()) {
+ if (returnFinally) return "cleanup result";
+ throw new IllegalStateException("fast handler finally");
+ }
+ }
+ },
+ config(workers, pluginKind, hooks));
+ try {
+ if (retry) {
+ assertSame(
+ original,
+ assertThrows(UnrecoverableDurableExecutionException.class, () -> runner.run("input")));
+ } else assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
+ assertHooks(hooks, pluginKind, retry ? InvocationStatus.RETRYING : InvocationStatus.PENDING, original);
+ overrideFinally.set(false);
+ var resumed = runner.runUntilComplete("input");
+ assertEquals(ExecutionStatus.SUCCEEDED, resumed.getStatus());
+ assertEquals("done", resumed.getResult(String.class));
+ assertEquals(1, effects.get(), "Completed step must replay without another side effect");
+ if (retry) assertEquals(2, attempts.get());
+ if (!pluginKind.equals("none")) {
+ assertTrue(hooks.size() >= 2, "Factories create instances for initial invocation and replay");
+ for (var plugin : hooks) {
+ assertEquals(1, plugin.starts);
+ assertEquals(1, plugin.ends.size());
+ }
+ assertEquals(
+ InvocationStatus.SUCCEEDED,
+ hooks.get(hooks.size() - 1).ends.get(0).invocationStatus());
+ }
+ } finally {
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("successes")
+ @Timeout(15)
+ void fastOrdinarySuccessKeepsItsResult(String mode, String pluginKind) throws Exception {
+ var workers = new FirstDispatchExecutor(mode);
+ var hooks = new CopyOnWriteArrayList();
+ var runner = LocalDurableTestRunner.create(
+ String.class, (input, context) -> "body result", config(workers, pluginKind, hooks));
+ try {
+ var result = runner.run("input");
+ assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
+ assertEquals("body result", result.getResult(String.class));
+ assertHooks(hooks, pluginKind, InvocationStatus.SUCCEEDED, null);
+ } finally {
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ private static DurableConfig config(ExecutorService workers, String pluginKind, List hooks) {
+ var builder = DurableConfig.builder().withExecutorService(workers);
+ if (!pluginKind.equals("none"))
+ builder.withPlugins(info -> {
+ var plugin = pluginKind.equals("scoped") ? new ScopedPlugin() : new RecordingPlugin();
+ hooks.add(plugin);
+ return plugin;
+ });
+ return builder.build();
+ }
+
+ private static void assertHooks(
+ List