Repository navigation
Expand file tree
/
Copy pathDurableFunctionsIntegrationTest.cs
More file actions
196 lines (178 loc) · 14.3 KB
/
Copy pathDurableFunctionsIntegrationTest.cs
File metadata and controls
196 lines (178 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
using Aspire.Hosting.Testing;
using NUnit.Framework;
using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using UnitTestEx.Aspire;
using UnitTestEx.DurableFunction;
namespace UnitTestEx.Azure.Functions.Durable.Integration.Test;
/// <summary>Exercises the real Functions worker and Azure Storage Durable provider against disposable Azurite storage.</summary>
/// <remarks>Requires Core Tools and a container runtime. No in-memory SDK test host, mock activity or virtual clock is used.</remarks>
[TestFixture]
[NonParallelizable]
[Category("DurableIntegration")]
public sealed class DurableFunctionsIntegrationTest
{
/// <summary>Provides case-insensitive JSON conversion for Functions management payloads and workflow models.</summary>
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
/// <summary>Reads the finalization stage independently of the worker's JSON property casing.</summary>
private sealed record FinalizationStatus(string Stage);
/// <summary>Executes the real batch, including the instance child, injected journal and worker-supplied activity context.</summary>
[Test]
public async Task BatchExecutesRealActivitiesAndChild()
{
await using var tester = AspireTester.Create<Projects.UnitTestEx_Aspire_Durable_AppHost>().EnableHostDiagnostics();
await tester.WaitForFunctionsAsync("durable-functions", ["ProcessBatch", "NormalizeReference", "ProcessItem", "TotalItems", "FinalizeBatch", "RecordWorkflow", "PublishWorkflow"]);
using var client = (await tester.GetDistributedApplicationAsync()).CreateHttpClient("durable-functions");
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(90));
var request = new BatchRequest(" batch-7 ", [new("A", 10m), new("B", 20m), new("C", 30m)]);
var management = await StartAsync(client, nameof(BatchOrchestration.ProcessBatch), request, deadline.Token);
var status = await WaitForTerminalAsync(tester, client, management, deadline.Token);
Assert.That(status.GetProperty("runtimeStatus").GetString(), Is.EqualTo("Completed"), status.ToString());
var storedInput = status.GetProperty("input").Deserialize<BatchRequest>(JsonOptions) ?? throw new InvalidOperationException("Stored batch input was null.");
Assert.That(storedInput.Reference, Is.EqualTo(request.Reference));
Assert.That(storedInput.Items, Is.EqualTo(request.Items));
var output = status.GetProperty("output").Deserialize<BatchResult>(JsonOptions) ?? throw new InvalidOperationException("Completed batch output was null.");
Assert.That(output.Reference, Is.EqualTo("BATCH-7"));
Assert.That(output.Total, Is.EqualTo(120m));
Assert.That(output.Items, Is.EqualTo(new[] { new ItemResult("A", 20m), new ItemResult("B", 40m), new ItemResult("C", 60m) }));
Assert.That(status.GetProperty("customStatus").Deserialize<FinalizationStatus>(JsonOptions)?.Stage, Is.EqualTo("Finalizing"));
// Resource logs are forwarded asynchronously; wait for publication before checking the complete application path.
await tester.EventuallyAsync(_ =>
{
try
{
tester.AssertLogContains("Published workflow BATCH-7 with total 120.", "durable-functions");
return Task.FromResult(true);
}
catch (AssertionException)
{
return Task.FromResult(false);
}
}, observed => observed, cancellationToken: deadline.Token);
tester.Checkpoint("Real Durable batch completed.");
tester.AssertLogContains("Processing item A.", "durable-functions");
tester.AssertLogContains("Processing item B.", "durable-functions");
tester.AssertLogContains("Processing item C.", "durable-functions");
tester.AssertLogContains("Recorded workflow BATCH-7.", "durable-functions");
tester.AssertLogContains("Published workflow BATCH-7 with total 120.", "durable-functions");
await PurgeAsync(client, management, deadline.Token);
}
/// <summary>Verifies that a real failed activity crosses the Functions extension boundary and is caught by the orchestration.</summary>
[Test]
public async Task ActivityFailureIsRecoveredByRealOrchestration()
{
await using var tester = AspireTester.Create<Projects.UnitTestEx_Aspire_Durable_AppHost>().EnableHostDiagnostics().ErrorWhenLogContains(Microsoft.Extensions.Logging.LogLevel.None);
await tester.WaitForFunctionsAsync("durable-functions", ["RecoverWorkflow", "RejectWorkflow"]);
using var client = (await tester.GetDistributedApplicationAsync()).CreateHttpClient("durable-functions");
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(90));
var management = await StartAsync(client, nameof(FailureOrchestration.RecoverWorkflow), "ORDER-REJECT", deadline.Token);
var status = await WaitForTerminalAsync(tester, client, management, deadline.Token);
Assert.That(status.GetProperty("runtimeStatus").GetString(), Is.EqualTo("Completed"), status.ToString());
Assert.That(status.GetProperty("output").GetString(), Is.EqualTo("Declined"));
Assert.That(status.GetProperty("customStatus").GetString(), Is.EqualTo("Declined"));
await PurgeAsync(client, management, deadline.Token);
}
/// <summary>Delivers an event through the real management API without shortening the application's 24-hour timeout.</summary>
[Test]
public async Task ApprovalEventWinsProductionLengthTimer()
{
await using var tester = AspireTester.Create<Projects.UnitTestEx_Aspire_Durable_AppHost>().EnableHostDiagnostics();
await tester.WaitForFunctionsAsync("durable-functions", ["AwaitApproval"]);
using var client = (await tester.GetDistributedApplicationAsync()).CreateHttpClient("durable-functions");
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(90));
var management = await StartAsync(client, nameof(ApprovalOrchestration.AwaitApproval), new ApprovalRequest(TimeSpan.FromHours(24)), deadline.Token);
await tester.EventuallyAsync(token => ReadStatusAsync(client, management, token), status => status?.GetProperty("runtimeStatus").GetString() == "Running",
timeout: TimeSpan.FromSeconds(60), cancellationToken: deadline.Token);
await SendEventAsync(client, management, "Approved", new ApprovalDecision(true), deadline.Token);
var status = await WaitForTerminalAsync(tester, client, management, deadline.Token);
Assert.That(status.GetProperty("runtimeStatus").GetString(), Is.EqualTo("Completed"), status.ToString());
Assert.That(status.GetProperty("output").Deserialize<ApprovalDecision>(JsonOptions), Is.EqualTo(new ApprovalDecision(true)));
await PurgeAsync(client, management, deadline.Token);
}
/// <summary>Checks buffered-event preservation and discard through the real Functions extension and storage provider.</summary>
/// <param name="preserveEvents">Whether the first generation preserves its buffered payload.</param>
[TestCase(true)]
[TestCase(false)]
public async Task ContinuationControlsBufferedEventsInRealWorker(bool preserveEvents)
{
await using var tester = AspireTester.Create<Projects.UnitTestEx_Aspire_Durable_AppHost>().EnableHostDiagnostics();
await tester.WaitForFunctionsAsync("durable-functions", [nameof(ContinuationOrchestration.ContinueWithEvents)]);
using var client = (await tester.GetDistributedApplicationAsync()).CreateHttpClient("durable-functions");
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(90));
var management = await StartAsync(client, nameof(ContinuationOrchestration.ContinueWithEvents), new ContinuationRequest(0, preserveEvents), deadline.Token);
await WaitForCustomStatusAsync(tester, client, management, "initial", deadline.Token);
await SendEventAsync(client, management, "Payload", "buffered payload", deadline.Token);
await SendEventAsync(client, management, "Barrier", "payload sent", deadline.Token);
await WaitForCustomStatusAsync(tester, client, management, "buffered", deadline.Token);
await SendEventAsync(client, management, "Continue", "restart", deadline.Token);
await WaitForCustomStatusAsync(tester, client, management, "continued", deadline.Token);
if (!preserveEvents)
await SendEventAsync(client, management, "Payload", "fresh payload", deadline.Token);
var status = await WaitForTerminalAsync(tester, client, management, deadline.Token);
Assert.That(status.GetProperty("runtimeStatus").GetString(), Is.EqualTo("Completed"), status.ToString());
Assert.That(status.GetProperty("output").GetString(), Is.EqualTo(preserveEvents ? "buffered payload" : "fresh payload"));
Assert.That(status.GetProperty("input").Deserialize<ContinuationRequest>(JsonOptions)?.Generation, Is.EqualTo(1));
await PurgeAsync(client, management, deadline.Token);
}
/// <summary>Waits for a persisted orchestration stage before sending the next coordinating event.</summary>
private static Task<JsonElement?> WaitForCustomStatusAsync(AspireTesterBase tester, HttpClient client, JsonElement management, string stage, CancellationToken cancellationToken)
=> tester.EventuallyAsync(token => ReadStatusAsync(client, management, token),
value => value is { } status && status.TryGetProperty("customStatus", out var customStatus) && customStatus.ValueKind == JsonValueKind.String && customStatus.GetString() == stage,
timeout: TimeSpan.FromSeconds(60), cancellationToken: cancellationToken);
/// <summary>Sends a typed event through the management URL returned by the extension.</summary>
private static async Task SendEventAsync<T>(HttpClient client, JsonElement management, string eventName, T payload, CancellationToken cancellationToken)
{
var eventUrl = management.GetProperty("sendEventPostUri").GetString()!.Replace("{eventName}", Uri.EscapeDataString(eventName), StringComparison.Ordinal);
using var response = await client.PostAsJsonAsync(LocalUri(client, eventUrl), payload, cancellationToken);
await RequireStatusAsync(response, HttpStatusCode.Accepted, cancellationToken);
}
/// <summary>Schedules an instance through the extension's documented HTTP API and retains its returned management URLs.</summary>
private static async Task<JsonElement> StartAsync<T>(HttpClient client, string name, T input, CancellationToken cancellationToken)
{
using var response = await client.PostAsJsonAsync($"runtime/webhooks/durabletask/orchestrators/{Uri.EscapeDataString(name)}/{Guid.NewGuid():N}", input, cancellationToken);
await RequireStatusAsync(response, HttpStatusCode.Accepted, cancellationToken);
return await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken);
}
/// <summary>Polls persisted management state and returns any terminal result so failures are asserted rather than mistaken for timeouts.</summary>
private static async Task<JsonElement> WaitForTerminalAsync(AspireTesterBase tester, HttpClient client, JsonElement management, CancellationToken cancellationToken)
{
var status = await tester.EventuallyAsync(token => ReadStatusAsync(client, management, token),
value => value?.GetProperty("runtimeStatus").GetString() is "Completed" or "Failed" or "Terminated" or "Canceled",
timeout: TimeSpan.FromSeconds(60), cancellationToken: cancellationToken);
return status ?? throw new InvalidOperationException("Durable terminal status was missing.");
}
/// <summary>Reads the instance's management status; only documented successful pending/completed responses are accepted.</summary>
private static async Task<JsonElement?> ReadStatusAsync(HttpClient client, JsonElement management, CancellationToken cancellationToken)
{
using var response = await client.GetAsync(LocalUri(client, management.GetProperty("statusQueryGetUri").GetString()!), cancellationToken);
// The extension can acknowledge scheduling before the instance is visible in Azure Storage.
if (response.StatusCode == HttpStatusCode.NotFound)
return null;
if (response.StatusCode is not (HttpStatusCode.OK or HttpStatusCode.Accepted))
throw new HttpRequestException($"Durable status returned {(int)response.StatusCode}: {await response.Content.ReadAsStringAsync(cancellationToken)}", null, response.StatusCode);
return await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken);
}
/// <summary>Purges the completed parent history; AppHost disposal removes the nonpersistent container, including child history and pending timers.</summary>
private static async Task PurgeAsync(HttpClient client, JsonElement management, CancellationToken cancellationToken)
{
using var response = await client.DeleteAsync(LocalUri(client, management.GetProperty("purgeHistoryDeleteUri").GetString()!), cancellationToken);
await RequireStatusAsync(response, HttpStatusCode.OK, cancellationToken);
var result = await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken);
Assert.That(result.GetProperty("instancesDeleted").GetInt32(), Is.EqualTo(1));
}
/// <summary>Restricts returned management URLs to the test-owned Functions host.</summary>
private static Uri LocalUri(HttpClient client, string value)
{
var uri = new Uri(value, UriKind.Absolute);
if (client.BaseAddress is null || uri.GetLeftPart(UriPartial.Authority) != client.BaseAddress.GetLeftPart(UriPartial.Authority))
throw new InvalidOperationException("Durable management URL does not belong to the test Functions host.");
return uri;
}
/// <summary>Reports nonexpected HTTP responses with their bodies rather than losing extension diagnostics.</summary>
private static async Task RequireStatusAsync(HttpResponseMessage response, HttpStatusCode expected, CancellationToken cancellationToken)
{
if (response.StatusCode != expected)
throw new HttpRequestException($"Expected {(int)expected}, received {(int)response.StatusCode}: {await response.Content.ReadAsStringAsync(cancellationToken)}", null, response.StatusCode);
}
}