diff --git a/Directory.Packages.props b/Directory.Packages.props index 7baaa1562b..eef4ec509d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -62,10 +62,11 @@ - - - - + + diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClient.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClient.cs index ae70269ff2..495d717d0e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClient.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClient.cs @@ -24,7 +24,30 @@ public AppInsightTelemetryClient(string? currentSessionId, string osVersion) } public void TrackEvent(string eventName, Dictionary properties, Dictionary metrics) - => _telemetryClient.TrackEvent(eventName, properties, metrics); + { + // Microsoft.ApplicationInsights 3.x (an OpenTelemetry shim) removed the metrics parameter + // from TrackEvent. Tracking metrics separately via TrackMetric() would emit uncorrelated + // metric instruments that are neither enriched with this client's context (session/OS) nor + // tied back to the event, and would additionally require every metric key to satisfy the + // OpenTelemetry instrument-name syntax (no spaces, must start with a letter). + // + // Instead we fold the numeric measurements into the event's properties as invariant-culture + // strings. This keeps a single correlated, context-enriched customEvent and imposes no naming + // restrictions on the keys. Consumers read these values back with todouble(customDimensions[...]). + if (metrics.Count == 0) + { + _telemetryClient.TrackEvent(eventName, properties); + return; + } + + var combinedProperties = new Dictionary(properties); + foreach (KeyValuePair metric in metrics) + { + combinedProperties[metric.Key] = metric.Value.ToString("R", CultureInfo.InvariantCulture); + } + + _telemetryClient.TrackEvent(eventName, combinedProperties); + } public void Flush() => _telemetryClient.Flush(); diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClientFactory.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClientFactory.cs index 642da76ed9..1a212b47c9 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClientFactory.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClientFactory.cs @@ -1,10 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Testing.Platform; + namespace Microsoft.Testing.Extensions.Telemetry; internal sealed class AppInsightTelemetryClientFactory : ITelemetryClientFactory { + private readonly string? _localExportFilePath; + + public AppInsightTelemetryClientFactory(string? localExportFilePath = null) + => _localExportFilePath = localExportFilePath; + public ITelemetryClient Create(string? currentSessionId, string osVersion) - => new AppInsightTelemetryClient(currentSessionId, osVersion); + => RoslynString.IsNullOrWhiteSpace(_localExportFilePath) + ? new AppInsightTelemetryClient(currentSessionId, osVersion) + : new LocalFileTelemetryClient(_localExportFilePath, currentSessionId, osVersion); } diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs index 3e309a92b6..49cc238f0e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs @@ -32,6 +32,11 @@ internal sealed partial class AppInsightsProvider : // Note: We're currently using the same environment variable as dotnet CLI. public static readonly string SessionIdEnvVar = "TESTINGPLATFORM_APPINSIGHTS_SESSIONID"; + // When set to a file path, telemetry is written locally to that file (as JSON Lines) via + // LocalFileTelemetryClient instead of being shipped to Application Insights. This is the + // "local exporter" hook used to verify what telemetry would be collected, without the network. + public static readonly string LocalExportPathEnvVar = "TESTINGPLATFORM_TELEMETRY_LOCALEXPORTPATH"; + // Allows us to correlate events produced from the same process. // Not calling this ProcessId, because it has a different meaning. private static readonly string CurrentReporterId = Guid.NewGuid().ToString(); diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsTelemetryProviderExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsTelemetryProviderExtensions.cs index 4998ce6538..a1cc617ad3 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsTelemetryProviderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsTelemetryProviderExtensions.cs @@ -44,6 +44,10 @@ public static void AddAppInsightsTelemetryProvider(this ITestApplicationBuilder // We want to flow down the processes the same session id for correlation purposes. environment.SetEnvironmentVariable(AppInsightsProvider.SessionIdEnvVar, sessionId); + + // Opt-in local export: when set, telemetry is written to this file instead of AppInsights. + string? localExportPath = environment.GetEnvironmentVariable(AppInsightsProvider.LocalExportPathEnvVar); + return new AppInsightsProvider( services.GetRequiredService(), services.GetTestApplicationCancellationTokenSource(), @@ -52,7 +56,7 @@ public static void AddAppInsightsTelemetryProvider(this ITestApplicationBuilder services.GetClock(), services.GetConfiguration(), services.GetRequiredService(), - new AppInsightTelemetryClientFactory(), + new AppInsightTelemetryClientFactory(localExportPath), sessionId); }); #pragma warning restore IDE0022 // Use expression body for method diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/LocalFileTelemetryClient.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/LocalFileTelemetryClient.cs new file mode 100644 index 0000000000..3236375230 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/LocalFileTelemetryClient.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; + +namespace Microsoft.Testing.Extensions.Telemetry; + +/// +/// A local, network-free that appends every telemetry event to a +/// file as a single JSON line (JSON Lines / NDJSON). It is the "local exporter" equivalent used to +/// verify what telemetry would be collected — without shipping anything to Application Insights. +/// It is selected instead of when the +/// environment variable points at a file. +/// +internal sealed class LocalFileTelemetryClient : ITelemetryClient +{ + private readonly string _filePath; + private readonly string? _sessionId; + private readonly string _osVersion; + + public LocalFileTelemetryClient(string filePath, string? currentSessionId, string osVersion) + { + _filePath = filePath; + _sessionId = currentSessionId; + _osVersion = osVersion; + + string? directory = Path.GetDirectoryName(_filePath); + if (!RoslynString.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + } + + public void TrackEvent(string eventName, Dictionary properties, Dictionary metrics) + { + var builder = new StringBuilder(); + builder.Append('{'); + AppendJsonString(builder, "eventName", eventName); + builder.Append(','); + AppendJsonString(builder, "sessionId", _sessionId ?? string.Empty); + builder.Append(','); + AppendJsonString(builder, "osVersion", _osVersion); + + builder.Append(",\"properties\":{"); + bool first = true; + foreach (KeyValuePair property in properties) + { + if (!first) + { + builder.Append(','); + } + + AppendJsonString(builder, property.Key, property.Value); + first = false; + } + + builder.Append("},\"metrics\":{"); + first = true; + foreach (KeyValuePair metric in metrics) + { + if (!first) + { + builder.Append(','); + } + + AppendJsonKey(builder, metric.Key); + builder.Append(metric.Value.ToString("R", CultureInfo.InvariantCulture)); + first = false; + } + + builder.Append("}}"); + + // TrackEvent is only ever invoked from the provider's single-consumer ingest loop, so no + // synchronization is needed here (mirroring AppInsightTelemetryClient). + File.AppendAllText(_filePath, builder.ToString() + Environment.NewLine); + } + + // No-op: writes are flushed to disk synchronously as each event is tracked. + public void Flush() + { + } + + private static void AppendJsonString(StringBuilder builder, string key, string value) + { + AppendJsonKey(builder, key); + AppendEscaped(builder, value); + } + + private static void AppendJsonKey(StringBuilder builder, string key) + { + AppendEscaped(builder, key); + builder.Append(':'); + } + + private static void AppendEscaped(StringBuilder builder, string value) + { + builder.Append('"'); + foreach (char c in value) + { + switch (c) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append("\\\\"); + break; + case '\b': + builder.Append("\\b"); + break; + case '\f': + builder.Append("\\f"); + break; + case '\n': + builder.Append("\\n"); + break; + case '\r': + builder.Append("\\r"); + break; + case '\t': + builder.Append("\\t"); + break; + default: + if (c < ' ') + { + builder.Append("\\u"); + builder.Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + builder.Append(c); + } + + break; + } + } + + builder.Append('"'); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/Microsoft.Testing.Extensions.Telemetry.csproj b/src/Platform/Microsoft.Testing.Extensions.Telemetry/Microsoft.Testing.Extensions.Telemetry.csproj index 1c9ee840cf..8d4cda86ff 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/Microsoft.Testing.Extensions.Telemetry.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/Microsoft.Testing.Extensions.Telemetry.csproj @@ -50,10 +50,6 @@ This package provides telemetry for the platform.]]> - - - - diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.Telemetry/PACKAGE.md index c9016f7874..87a677ee7e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/PACKAGE.md @@ -16,6 +16,7 @@ This package extends Microsoft.Testing.Platform with: - **Usage telemetry**: collects usage data to help understand product usage and prioritize improvements - **Opt-out support**: telemetry can be disabled via the `TESTINGPLATFORM_TELEMETRY_OPTOUT` or `DOTNET_CLI_TELEMETRY_OPTOUT` environment variables +- **Local export (verification)**: set `TESTINGPLATFORM_TELEMETRY_LOCALEXPORTPATH` to a file path to write the collected telemetry events locally (as JSON Lines) instead of sending them to Application Insights. This is useful to inspect exactly what would be collected, without any network traffic. - **Disclosure**: telemetry information is shown on first run, with opt-out guidance This package is an optional, opt-in extension. To enable telemetry when using Microsoft.Testing.Platform (including when running tests with [MSTest](https://www.nuget.org/packages/MSTest)), you must explicitly reference the `Microsoft.Testing.Extensions.Telemetry` package from your test project or from your own test framework or tooling package. diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryProperties.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryProperties.cs index 523040b09b..13f9fcced1 100644 --- a/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryProperties.cs +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryProperties.cs @@ -10,7 +10,11 @@ internal static class TelemetryProperties public const string ReporterIdPropertyName = "reporter id"; public const string IsCIPropertyName = "is ci"; - public const string VersionValue = "20"; + // Bump this whenever the telemetry schema changes so dashboards can branch old vs new. + // 21: Microsoft.ApplicationInsights 3.x (OpenTelemetry shim) removed TrackEvent's metrics + // parameter, so numeric measurements are now folded into event properties + // (customDimensions) instead of customMeasurements. See #7465. + public const string VersionValue = "21"; public const string True = "true"; public const string False = "false"; diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/LocalFileTelemetryClientTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/LocalFileTelemetryClientTests.cs new file mode 100644 index 0000000000..28262aaedc --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/LocalFileTelemetryClientTests.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.Telemetry; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class LocalFileTelemetryClientTests +{ + [TestMethod] + public void TrackEvent_WritesJsonLine_WithEventPropertiesAndMetrics() + { + string filePath = Path.Combine(Path.GetTempPath(), $"mtp-telemetry-{Guid.NewGuid():N}.jsonl"); + try + { + var client = new LocalFileTelemetryClient(filePath, "session-42", "Windows 11"); + client.TrackEvent( + "dotnet/testingplatform/host/consoletesthostexit", + new Dictionary { ["is ci"] = "true" }, + new Dictionary { ["run start"] = 1234.5 }); + + string[] lines = File.ReadAllLines(filePath); + Assert.HasCount(1, lines); + + string line = lines[0]; + Assert.Contains("\"eventName\":\"dotnet/testingplatform/host/consoletesthostexit\"", line); + Assert.Contains("\"sessionId\":\"session-42\"", line); + Assert.Contains("\"osVersion\":\"Windows 11\"", line); + Assert.Contains("\"is ci\":\"true\"", line); + Assert.Contains("\"run start\":1234.5", line); + } + finally + { + File.Delete(filePath); + } + } + + [TestMethod] + public void TrackEvent_AppendsOneLinePerEvent() + { + string filePath = Path.Combine(Path.GetTempPath(), $"mtp-telemetry-{Guid.NewGuid():N}.jsonl"); + try + { + var client = new LocalFileTelemetryClient(filePath, "s", "os"); + client.TrackEvent("first", [], []); + client.TrackEvent("second", [], []); + + string[] lines = File.ReadAllLines(filePath); + Assert.HasCount(2, lines); + Assert.Contains("\"eventName\":\"first\"", lines[0]); + Assert.Contains("\"eventName\":\"second\"", lines[1]); + } + finally + { + File.Delete(filePath); + } + } + + [TestMethod] + public void TrackEvent_EscapesSpecialCharacters() + { + string filePath = Path.Combine(Path.GetTempPath(), $"mtp-telemetry-{Guid.NewGuid():N}.jsonl"); + try + { + var client = new LocalFileTelemetryClient(filePath, null, "os"); + client.TrackEvent( + "evt", + new Dictionary { ["quote\"backslash\\"] = "line\nbreak" }, + []); + + string content = File.ReadAllText(filePath); + Assert.Contains("quote\\\"backslash\\\\", content); + Assert.Contains("line\\nbreak", content); + + // The raw newline must be escaped so the record stays on a single line. + Assert.HasCount(1, File.ReadAllLines(filePath)); + } + finally + { + File.Delete(filePath); + } + } + + [TestMethod] + public void Factory_WithLocalExportPath_CreatesLocalFileClient() + { + string filePath = Path.Combine(Path.GetTempPath(), $"mtp-telemetry-{Guid.NewGuid():N}.jsonl"); + var factory = new AppInsightTelemetryClientFactory(filePath); + + ITelemetryClient client = factory.Create("session", "os"); + + Assert.IsInstanceOfType(client); + } +}