diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index 08781ce209..7ae6375121 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -78,17 +78,23 @@ public async Task Should_Create_A_Session_With_Replaced_SystemMessage_Config() SystemMessage = new SystemMessageConfig { Mode = SystemMessageMode.Replace, Content = testSystemMessage } }); - await session.SendAsync(new MessageOptions { Prompt = "What is your full name?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); + await AssertReplacedSystemMessageResponseAsync(session, TimeSpan.FromSeconds(120)); + + var traffic = await Ctx.GetExchangesAsync(); + Assert.NotEmpty(traffic); + Assert.Equal(testSystemMessage, GetSystemMessage(traffic[0])); + } + + internal static async Task AssertReplacedSystemMessageResponseAsync(CopilotSession session, TimeSpan timeout) + { + // Subscribe before sending: the ephemeral idle event cannot be recovered from history. + var assistantMessage = await session.SendAndWaitAsync( + new MessageOptions { Prompt = "What is your full name?" }, timeout); Assert.NotNull(assistantMessage); var content = assistantMessage!.Data.Content ?? string.Empty; Assert.DoesNotContain("GitHub", content); Assert.Contains("Testy", content); - - var traffic = await Ctx.GetExchangesAsync(); - Assert.NotEmpty(traffic); - Assert.Equal(testSystemMessage, GetSystemMessage(traffic[0])); } [Fact] diff --git a/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs b/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs index 41c46d3b9d..1334ef96bb 100644 --- a/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs +++ b/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs @@ -61,8 +61,13 @@ public async Task Should_Use_Replaced_Preamble_Section_In_Response() } }); - await session.SendAsync(new MessageOptions { Prompt = "Who are you?" }); - var response = await TestHelper.GetFinalAssistantMessageAsync(session); + await AssertReplacedPreambleResponseAsync(session, TimeSpan.FromSeconds(120)); + } + + internal static async Task AssertReplacedPreambleResponseAsync(CopilotSession session, TimeSpan timeout) + { + // Subscribe before sending: the ephemeral idle event cannot be recovered from history. + var response = await session.SendAndWaitAsync(new MessageOptions { Prompt = "Who are you?" }, timeout); Assert.NotNull(response); var content = response.Data.Content.ToLowerInvariant(); diff --git a/dotnet/test/Unit/ClientSessionLifetimeTests.cs b/dotnet/test/Unit/ClientSessionLifetimeTests.cs index dd7fdc2bbb..1addda6c30 100644 --- a/dotnet/test/Unit/ClientSessionLifetimeTests.cs +++ b/dotnet/test/Unit/ClientSessionLifetimeTests.cs @@ -1717,6 +1717,148 @@ private static void AssertMessageSource(JsonElement request, string? source) Assert.False(request.TryGetProperty("wait", out _)); } + [Fact] + public async Task Replaced_System_Message_Observes_Idle_Before_Send_Reply() + { + await using var server = await FakeCopilotServer.StartAsync(); + await using var client = new CopilotClient(new CopilotClientOptions { Connection = RuntimeConnection.ForUri(server.Url) }); + await using var session = await client.CreateSessionAsync(new SessionConfig()); + var timeout = TimeSpan.FromSeconds(5); + const string finalContent = "My full name is Testy McTestface."; + var drained = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var subscription = session.On(_ => drained.TrySetResult()); + var idleBeforeReply = false; + server.BeforeResponseAsync = async (request, cancellationToken) => + { + if (request.Method != "session.send") + { + return; + } + + await server.SendSessionEventAsync(session.SessionId, "user.message", new() + { + ["content"] = request.Params.GetProperty("prompt").GetString() + }); + await server.SendSessionEventAsync(session.SessionId, "assistant.message", new() + { + ["messageId"] = "intermediate-message", + ["content"] = "I will check before answering." + }); + await server.SendSessionEventAsync(session.SessionId, "tool.execution_start", new() + { + ["toolCallId"] = "check-name", + ["toolName"] = "shell" + }); + await server.SendSessionEventAsync(session.SessionId, "tool.execution_complete", new() + { + ["toolCallId"] = "check-name", + ["success"] = true + }); + await server.SendSessionEventAsync(session.SessionId, "assistant.message", new() + { + ["messageId"] = "final-message", + ["content"] = finalContent + }); + await server.SendSessionEventAsync(session.SessionId, "session.idle", new()); + // Drain a later event before replying, so idle cannot remain queued for a late subscriber. + await server.SendSessionEventAsync(session.SessionId, "session.title_changed", new() { ["title"] = "fence" }); + await drained.Task.WaitAsync(timeout, cancellationToken); + idleBeforeReply = true; + }; + + try + { + await E2E.SessionE2ETests.AssertReplacedSystemMessageResponseAsync(session, timeout); + } + catch (TimeoutException error) + { + var history = await session.GetEventsAsync(); + throw new TimeoutException( + $"{error.Message}; idle drained before send reply: {idleBeforeReply}; " + + $"durable assistant messages: {history.OfType().Count()}; " + + $"durable idle events: {history.OfType().Count()}", error); + } + + Assert.True(idleBeforeReply); + var request = Assert.Single(server.Requests, request => request.Method == "session.send"); + Assert.Equal("What is your full name?", request.Params.GetProperty("prompt").GetString()); + var events = await session.GetEventsAsync(); + Assert.DoesNotContain(events, evt => evt is SessionIdleEvent); + Assert.Equal(2, events.OfType().Count()); + Assert.Equal(finalContent, events.OfType().Last().Data.Content); + } + + [Fact] + public async Task Replaced_Preamble_Response_Observes_Idle_Before_Send_Reply() + { + await using var server = await FakeCopilotServer.StartAsync(); + await using var client = new CopilotClient(new CopilotClientOptions { Connection = RuntimeConnection.ForUri(server.Url) }); + await using var session = await client.CreateSessionAsync(new SessionConfig()); + var timeout = TimeSpan.FromSeconds(5); + const string finalContent = "I am Botanica, your gardening assistant."; + var drained = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var subscription = session.On(_ => drained.TrySetResult()); + var idleBeforeReply = false; + server.BeforeResponseAsync = async (request, cancellationToken) => + { + if (request.Method != "session.send") + { + return; + } + + await server.SendSessionEventAsync(session.SessionId, "user.message", new() + { + ["content"] = request.Params.GetProperty("prompt").GetString() + }); + await server.SendSessionEventAsync(session.SessionId, "assistant.message", new() + { + ["messageId"] = "intermediate-message", + ["content"] = "I will check before answering." + }); + await server.SendSessionEventAsync(session.SessionId, "tool.execution_start", new() + { + ["toolCallId"] = "check-section", + ["toolName"] = "shell" + }); + await server.SendSessionEventAsync(session.SessionId, "tool.execution_complete", new() + { + ["toolCallId"] = "check-section", + ["success"] = true + }); + await server.SendSessionEventAsync(session.SessionId, "assistant.message", new() + { + ["messageId"] = "final-message", + ["content"] = finalContent + }); + await server.SendSessionEventAsync(session.SessionId, "session.idle", new()); + // Drain a later event before replying, so the idle cannot remain queued for a late subscriber. + await server.SendSessionEventAsync(session.SessionId, "session.title_changed", new() { ["title"] = "fence" }); + await drained.Task.WaitAsync(timeout, cancellationToken); + idleBeforeReply = true; + }; + + try + { + await E2E.SystemMessageSectionsE2ETests.AssertReplacedPreambleResponseAsync(session, timeout); + } + catch (TimeoutException error) + { + var history = await session.GetEventsAsync(); + throw new TimeoutException( + $"{error.Message}; idle drained before send reply: {idleBeforeReply}; " + + $"durable assistant messages: {history.OfType().Count()}; " + + $"durable idle events: {history.OfType().Count()}", error); + } + + Assert.True(idleBeforeReply); + var request = Assert.Single(server.Requests, request => request.Method == "session.send"); + Assert.Equal("Who are you?", request.Params.GetProperty("prompt").GetString()); + var events = await session.GetEventsAsync(); + Assert.DoesNotContain(events, evt => evt is SessionIdleEvent); + Assert.Equal(2, events.OfType().Count()); + Assert.Equal(finalContent, events.OfType().Last().Data.Content); + } + [Theory] [InlineData(true)] [InlineData(false)]