feat(mcp): log the failures the MCP server hides from its clients

docs/mcp-reference.md

@@ -515,6 +515,27 @@ Command-backed resources expose the rendered handler return value as the
 resource body. Low-level writes to `IReplIoContext.Output` are treated as
 side-channel command output and are not included in `resources/read` bodies.
 
+## Operator diagnostics
+
+Some failures are hidden from the MCP client on purpose. `Repl.Mcp` records them for the operator
+through `Microsoft.Extensions.Logging`, under the category `Repl.Mcp`:
+
+| Event | Level | What the client saw instead |
+|---|---|---|
+| 2001 | Error | A command failed with a detail that is not meant for a remote caller, such as an unhandled exception's message. The client gets `Command failed with exit code N.`; the log gets the exception and the rendered detail. |
+| 2002 | Warning | A catalog rebuild failed on an initialize-era connection. That connection keeps its previous catalog until a rebuild succeeds or a visibility retraction withdraws it. |
+| 2003 | Warning | Priming the client's roots failed. The command still ran. |
+| 2004 | Debug | 2002 again for the same routing version. |
+| 2005 | Information | A rebuild succeeded after a 2002. |
+| 2006 | Debug | 2003 again before any prime succeeded. |
+
+A failure that can repeat on every request warns once per episode and logs its repeats at Debug, so a
+client that keeps asking cannot flood the log.
+
+`ReplApp` registers logging by default, but it stays silent until the app adds a provider. Under
+`mcp serve` on stdio, standard output carries the protocol, so send logs to standard error or a file.
+For the console provider, set `LogToStandardErrorThreshold = LogLevel.Trace`.
+
 ## Client compatibility
 
 Feature support varies across agents. Check [mcp-availability.com](https://mcp-availability.com/) for current data.

src/Repl.Mcp/McpClientRootsService.cs

@@ -18,6 +18,9 @@ internal sealed class McpClientRootsService : IMcpClientRoots
 	private readonly ICoreReplApp _app;
 	private readonly McpRequestServerAccessor _servers;
 	private readonly McpRootsScope _scope;
+	// 1 while a failure to prime is being reported at Debug instead of Warning; reset by a successful
+	// prime. Interlocked, since concurrent tool calls on one connection prime concurrently.
+	private int _primeFailureReported;
 	private readonly Lock _syncRoot = new();
 	// Bounds the one outbound call this type makes. Request scope pays it per request rather than once
 	// per connection, so a client that never answers roots/list would otherwise hold every tool call
@@ -119,11 +122,15 @@ public IReadOnlyList<McpClientRoot> Current
 	/// <see cref="Current"/> is documented as only what this request already resolved, and an eager fetch
 	/// would add a round-trip to every request rather than to every connection.
 	/// </remarks>
-	internal async ValueTask PrimeCurrentAsync(CancellationToken cancellationToken)
+	/// <returns>
+	/// <see langword="true"/> only when roots were actually fetched; <see langword="false"/> when the
+	/// prime did not ask at all — outside connection scope, without client support, or standing down.
+	/// </returns>
+	internal async ValueTask<bool> PrimeCurrentAsync(CancellationToken cancellationToken)
 	{
 		if (_scope is not McpRootsScope.Connection || !IsSupported)
 		{
-			return;
+			return false;
 		}
 
 		lock (_syncRoot)
@@ -134,7 +141,7 @@ internal async ValueTask PrimeCurrentAsync(CancellationToken cancellationToken)
 				&& _primeFailedAt is { } failedAt
 				&& Stopwatch.GetElapsedTime(failedAt) < PrimeRetryCooldown)
 			{
-				return;
+				return false;
 			}
 		}
 
@@ -175,6 +182,8 @@ internal async ValueTask PrimeCurrentAsync(CancellationToken cancellationToken)
 		{
 			_primeFailedAt = null;
 		}
+
+		return true;
 	}
 
 	public async ValueTask<IReadOnlyList<McpClientRoot>> GetAsync(CancellationToken cancellationToken = default)
@@ -384,7 +393,8 @@ private async Task<IReadOnlyList<McpClientRoot>> FetchHardRootsOnceAsync(McpServ
 	}
 
 	/// <summary>
-	/// Primes the connection's native roots from <paramref name="services"/>, swallowing a failure.
+	/// Primes the connection's native roots from <paramref name="services"/>, logging and swallowing a
+	/// failure.
 	/// </summary>
 	/// <remarks>
 	/// Called from every execution entry point. A handler that never reads roots must not fail because
@@ -402,15 +412,31 @@ internal static async ValueTask PrimeFromServicesAsync(
 
 		try
 		{
-			await roots.PrimeCurrentAsync(cancellationToken).ConfigureAwait(false);
+			// Only a prime that actually asked ends the episode: one skipped while standing down says
+			// nothing about whether the client can answer yet.
+			if (await roots.PrimeCurrentAsync(cancellationToken).ConfigureAwait(false))
+			{
+				Interlocked.Exchange(ref roots._primeFailureReported, 0);
+			}
 		}
 		catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
 		{
 			throw;
 		}
-		catch (Exception)
+		catch (Exception ex)
 		{
-			// Swallowed: see the remarks above.
+			// Swallowed for the command (see the remarks above), not for the operator. Warned once per
+			// failure episode: a fast failure is retried on every tool call, where the prime cooldown only
+			// covers one that exhausted its budget. Built only here, so a successful prime pays nothing.
+			var diagnostics = new McpLoggerDiagnostics(services);
+			if (Interlocked.Exchange(ref roots._primeFailureReported, 1) == 0)
+			{
+				diagnostics.RootsPrimeFailed(ex);
+			}
+			else
+			{
+				diagnostics.RootsPrimeStillFailing(ex);
+			}
 		}
 	}
 

src/Repl.Mcp/McpLoggerDiagnostics.cs

@@ -0,0 +1,46 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Repl.Mcp;
+
+/// <summary>
+/// Operator-facing record of the failures this package absorbs on purpose. Each one is withheld from,
+/// or hidden behind a fallback for, the remote MCP client by design; this is what keeps it from also
+/// disappearing for the operator.
+/// </summary>
+/// <remarks>
+/// Event ids 2000-2999 belong to this package; <c>Repl.Logging</c> uses the 1000 range. A failure that
+/// can repeat on every request warns once per episode and logs its repeats at Debug, so a client that
+/// keeps asking cannot flood the operator's log with the same exception.
+/// </remarks>
+internal sealed partial class McpLoggerDiagnostics
+{
+	private const string LoggerCategory = "Repl.Mcp";
+	private readonly ILogger _logger;
+
+	public McpLoggerDiagnostics(IServiceProvider services)
+	{
+		ArgumentNullException.ThrowIfNull(services);
+		var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory
+			?? NullLoggerFactory.Instance;
+		_logger = loggerFactory.CreateLogger(LoggerCategory);
+	}
+
+	[LoggerMessage(EventId = 2001, Level = LogLevel.Error, Message = "An MCP command failed with exit code {ExitCode}; the client received a generic message instead of this detail: {Detail}")]
+	public partial void ToolFailureWithheld(Exception? exception, int exitCode, string detail);
+
+	[LoggerMessage(EventId = 2002, Level = LogLevel.Warning, Message = "Building the MCP catalog at routing version {FailingVersion} failed; the connection keeps its previous catalog (version {ServedVersion}) until a rebuild succeeds or a visibility retraction withdraws it. Repeats are logged at Debug.")]
+	public partial void StaleCatalogServed(Exception exception, long failingVersion, long servedVersion);
+
+	[LoggerMessage(EventId = 2003, Level = LogLevel.Warning, Message = "Priming the MCP client's roots failed; commands still run, and a handler that reads roots asks the client again. Repeats are logged at Debug until a prime succeeds.")]
+	public partial void RootsPrimeFailed(Exception exception);
+
+	[LoggerMessage(EventId = 2004, Level = LogLevel.Debug, Message = "Building the MCP catalog at routing version {FailingVersion} failed again; the previous catalog is still served.")]
+	public partial void StaleCatalogStillServed(Exception exception, long failingVersion);
+
+	[LoggerMessage(EventId = 2005, Level = LogLevel.Information, Message = "The MCP catalog was rebuilt at routing version {Version} after a failure; the connection serves the current catalog again.")]
+	public partial void CatalogRecovered(long version);
+
+	[LoggerMessage(EventId = 2006, Level = LogLevel.Debug, Message = "Priming the MCP client's roots failed again.")]
+	public partial void RootsPrimeStillFailing(Exception exception);
+}

src/Repl.Mcp/McpServerHandler.cs

@@ -33,6 +33,7 @@ internal sealed class McpServerHandler
 	private readonly McpSamplingService _sampling;
 	private readonly McpElicitationService _elicitation;
 	private readonly McpFeedbackService _feedback;
+	private readonly McpLoggerDiagnostics _diagnostics;
 	// Context for work that belongs to no MCP session: eager fail-fast validation, the pre-built
 	// catalog behind BuildMcpServerOptions, and the snapshot test seams. One per handler, sharing its
 	// lifetime, so nothing disposes it — and CreateSessionContext deliberately opens it no DI scope to
@@ -101,6 +102,7 @@ public McpServerHandler(
 		_sampling = new McpSamplingService(_requestServers);
 		_elicitation = new McpElicitationService(_requestServers);
 		_feedback = new McpFeedbackService(_requestServers);
+		_diagnostics = new McpLoggerDiagnostics(services);
 		_catalogContext = CreateSessionContext(McpRootsScope.Request);
 	}
 
@@ -500,6 +502,14 @@ private async ValueTask<McpGeneratedSnapshot> BuildOrServePreviousAsync(
 		{
 			var built = await BuildCurrentSnapshotAsync(context, snapshotVersion, sessionless, cancellationToken)
 				.ConfigureAwait(false);
+			// Read back what was just published rather than trusting snapshotVersion: a build that raced a
+			// routing change is republished stale, and one that raced a retraction is rebuilt at a newer
+			// version. Only a current entry ends the episode, and it is logged at the version it was built.
+			if (context.SnapshotCache is { IsStale: false } published && context.TryClearCatalogFailure())
+			{
+				_diagnostics.CatalogRecovered(published.Version);
+			}
+
 			return built;
 		}
 		// Filtered on the caller's own token, like every other cancellation catch here: a projection
@@ -515,7 +525,7 @@ private async ValueTask<McpGeneratedSnapshot> BuildOrServePreviousAsync(
 			ThrowSanitizedIfAClientAlreadyHasASchema(previousSnapshot);
 			throw;
 		}
-		catch (Exception) when (IsFallbackEligible(context, sessionless))
+		catch (Exception ex) when (IsFallbackEligible(context, sessionless))
 		{
 			// Preserve availability for transient projection failures, but republish as stale so the
 			// next request retries without requiring another routing mutation. The entry keeps the
@@ -530,6 +540,19 @@ private async ValueTask<McpGeneratedSnapshot> BuildOrServePreviousAsync(
 				throw;
 			}
 
+			// The client cannot tell a stale catalog from a current one, so the operator log is the only
+			// sign the failure is going on. Warned once per failing version: the fallback is republished
+			// stale, so every request retries the build — tools/call and prompts/get included, on every
+			// connection — and would otherwise repeat the same exception each time.
+			if (context.TryMarkCatalogFailure(snapshotVersion))
+			{
+				_diagnostics.StaleCatalogServed(ex, snapshotVersion, fallback.Version);
+			}
+			else
+			{
+				_diagnostics.StaleCatalogStillServed(ex, snapshotVersion);
+			}
+
 			context.PublishStaleSnapshot(fallback.Snapshot, fallback.Version, fallback.Sessionless);
 			return fallback.Snapshot;
 		}

src/Repl.Mcp/McpSessionContext.cs

@@ -22,6 +22,9 @@ internal sealed class McpSessionContext : IAsyncDisposable
 {
 	private SnapshotCacheEntry? _snapshotCache;
 	private int _compatibilityIntroServed;
+	// The routing version whose build last failed onto the availability fallback, or 0 for none —
+	// versions start at 1. Read and written only under SnapshotGate, like the cache it describes.
+	private long _catalogFailureVersion;
 
 	private readonly AsyncServiceScope? _scope;
 
@@ -67,6 +70,37 @@ public void PublishStaleSnapshot(
 		bool sessionless) =>
 		Volatile.Write(ref _snapshotCache, new SnapshotCacheEntry(snapshot, version, IsStale: true, sessionless));
 
+	/// <summary>
+	/// Records that the build for <paramref name="failingVersion"/> failed onto the fallback;
+	/// <see langword="true"/> the first time for that version, so the failure warns once rather than
+	/// on every request that retries it. Call only under <see cref="SnapshotGate"/>.
+	/// </summary>
+	public bool TryMarkCatalogFailure(long failingVersion)
+	{
+		if (_catalogFailureVersion == failingVersion)
+		{
+			return false;
+		}
+
+		_catalogFailureVersion = failingVersion;
+		return true;
+	}
+
+	/// <summary>
+	/// Ends a failure episode after a successful build; <see langword="true"/> if one was in progress.
+	/// Call only under <see cref="SnapshotGate"/>.
+	/// </summary>
+	public bool TryClearCatalogFailure()
+	{
+		if (_catalogFailureVersion == 0)
+		{
+			return false;
+		}
+
+		_catalogFailureVersion = 0;
+		return true;
+	}
+
 	/// <summary>
 	/// Claims this session's one-time compatibility-shim intro; <see langword="true"/> for the first
 	/// caller only.

src/Repl.Mcp/McpToolAdapter.cs

@@ -24,6 +24,7 @@ internal sealed partial class McpToolAdapter
 	private readonly ReplMcpServerOptions _options;
 	private readonly IServiceProvider _services;
 	private readonly McpRequestServerAccessor _requestServers;
+	private readonly McpLoggerDiagnostics _diagnostics;
 	// Whether the catalog this adapter serves was built from the frozen discovery view. A property of
 	// the catalog, not of the request: a reusable BuildMcpServerOptions() result is frozen once and then
 	// serves clients of either era, so asking the request would leave an initialize-era caller unable to
@@ -44,6 +45,7 @@ public McpToolAdapter(
 		_services = services;
 		_requestServers = requestServers;
 		_catalogIsFrozen = catalogIsFrozen;
+		_diagnostics = new McpLoggerDiagnostics(services);
 	}
 
 	/// <summary>
@@ -323,11 +325,18 @@ public static CommandOutputCapture Create(bool captureCommandOutput)
 	/// unable to say what it needs.
 	/// </para>
 	/// </remarks>
-	private static string DescribeFailure(in McpPipelineInvocation invocation)
+	private string DescribeFailure(in McpPipelineInvocation invocation)
 	{
 		var withheld = $"Command failed with exit code {invocation.ExitCode}.";
 		if (WithholdsFailureText(invocation))
 		{
+			// Withheld from the client, not discarded: the rendered detail and the exception behind it
+			// are exactly what the operator needs, and the log is the one reader that should get them.
+			var detail = string.IsNullOrWhiteSpace(invocation.Output) ? invocation.Error : invocation.Output;
+			_diagnostics.ToolFailureWithheld(
+				invocation.Failure,
+				invocation.ExitCode,
+				string.IsNullOrWhiteSpace(detail) ? "(nothing was rendered)" : detail);
 			return withheld;
 		}