From ab64225cee17ccd029632e00353841e7604e75a5 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:19:47 +0000 Subject: [PATCH 1/6] Apply sample rate to all profiler subsystem telemetry to bypass ingestion sampling Make SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING public and set it on the ServiceProfilerIndex event, the StartProfiler/StopProfiler messages, and GcEvent telemetry. These events are emitted directly via TelemetryClient (not through the trace sampler), so previously they had no sample rate set, leaving them eligible for backend ingestion sampling. Using 99.99 keeps the item count at 1 while ensuring the events are not dropped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../triggers/AlertingSubsystemInit.java | 3 + .../profiler/triggers/GcEventInit.java | 2 + .../agent/internal/sampling/SamplerUtil.java | 2 +- .../profiler/triggers/GcEventInitTest.java | 96 +++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java index b7c5b3a5b85..d8e2d633694 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java @@ -16,6 +16,7 @@ import com.microsoft.applicationinsights.agent.internal.profiler.Profiler; import com.microsoft.applicationinsights.agent.internal.profiler.ProfilerControl; import com.microsoft.applicationinsights.agent.internal.profiler.upload.ServiceProfilerIndex; +import com.microsoft.applicationinsights.agent.internal.sampling.SamplerUtil; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; import com.microsoft.applicationinsights.alerting.AlertingSubsystem; @@ -149,6 +150,7 @@ private static void sendServiceProfilerIndex( EventTelemetryBuilder telemetryBuilder = telemetryClient.newEventTelemetryBuilder(); telemetryBuilder.setName("ServiceProfilerIndex"); + telemetryBuilder.setSampleRate((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); for (Map.Entry entry : serviceProfilerIndex.getProperties().entrySet()) { telemetryBuilder.addProperty(entry.getKey(), entry.getValue()); @@ -170,6 +172,7 @@ private static void sendMessageTelemetry(TelemetryClient telemetryClient, String MessageTelemetryBuilder telemetryBuilder = telemetryClient.newMessageTelemetryBuilder(); telemetryBuilder.setMessage(message); + telemetryBuilder.setSampleRate((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); telemetryBuilder.setTime(FormattedTime.offSetDateTimeFromNow()); telemetryClient.trackAsync(telemetryBuilder.build()); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInit.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInit.java index 8f58ccfa30d..71cac16643a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInit.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInit.java @@ -7,6 +7,7 @@ import com.azure.monitor.opentelemetry.autoconfigure.implementation.utils.FormattedTime; import com.microsoft.applicationinsights.agent.internal.configuration.GcReportingLevel; import com.microsoft.applicationinsights.agent.internal.profiler.util.ServiceLoaderUtil; +import com.microsoft.applicationinsights.agent.internal.sampling.SamplerUtil; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.alerting.AlertingSubsystem; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; @@ -125,6 +126,7 @@ private static void emitGcEvent( EventTelemetryBuilder telemetryBuilder = telemetryClient.newEventTelemetryBuilder(); telemetryBuilder.setName("GcEvent"); + telemetryBuilder.setSampleRate((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); telemetryBuilder.addProperty("collector", event.getCollector().getName()); telemetryBuilder.addProperty("type", event.getGcCause()); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java index fd810248a09..ad16cae0ec1 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplerUtil.java @@ -12,7 +12,7 @@ public class SamplerUtil { - private static final double SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING = 99.99; + public static final double SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING = 99.99; private static final Cache recordAndSampleWithSampleRateMap = Cache.bounded(100); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index 5b1257d5f16..481a2d3e877 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -5,9 +5,14 @@ import static org.assertj.core.api.Assertions.assertThat; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.TelemetryEventData; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.TelemetryItem; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.statsbeat.StatsbeatModule; import com.microsoft.applicationinsights.agent.internal.configuration.GcReportingLevel; import com.microsoft.applicationinsights.agent.internal.profiler.testutil.TestTimeSource; +import com.microsoft.applicationinsights.agent.internal.sampling.SamplerUtil; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; import com.microsoft.applicationinsights.alerting.AlertingSubsystem; import com.microsoft.applicationinsights.alerting.alert.AlertBreach; import com.microsoft.applicationinsights.alerting.analysis.TimeSource; @@ -19,7 +24,10 @@ import com.microsoft.gcmonitor.MemoryManagement; import com.microsoft.gcmonitor.garbagecollectors.GarbageCollector; import com.microsoft.gcmonitor.memorypools.MemoryPool; +import io.opentelemetry.sdk.resources.Resource; import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -27,12 +35,79 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import javax.management.MBeanServerConnection; import org.junit.jupiter.api.Test; import org.mockito.Mockito; class GcEventInitTest { + private static final String FAKE_CONNECTION_STRING = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://fake-ingestion-endpoint.example.com/"; + + @Test + void gcEventIsEmittedWithSampleRateThatBypassesIngestionSampling() + throws ExecutionException, InterruptedException, TimeoutException { + + CompletableFuture gcEventFuture = new CompletableFuture<>(); + Consumer observer = + telemetryItem -> { + if (telemetryItem.getData().getBaseData() instanceof TelemetryEventData + && "GcEvent" + .equals(((TelemetryEventData) telemetryItem.getData().getBaseData()).getName())) { + gcEventFuture.complete(telemetryItem); + } + }; + TelemetryObservers.INSTANCE.getObservers().add(observer); + + try { + CompletableFuture alertFuture = new CompletableFuture<>(); + TestTimeSource timeSource = new TestTimeSource(); + AlertingSubsystem alertingSubsystem = getAlertingSubsystem(alertFuture, timeSource); + + GcMonitorFactory factory = + new GcMonitorFactory() { + @Override + public MemoryManagement monitorSelf( + ExecutorService executorService, GcEventConsumer consumer) { + consumer.accept(fullyMockedGcEvent()); + return null; + } + + @Override + public MemoryManagement monitor( + MBeanServerConnection connection, + ExecutorService executorService, + GcEventConsumer consumer) { + return null; + } + }; + + TelemetryClient telemetryClient = + TelemetryClient.builder() + .setCustomDimensions(new HashMap<>()) + .setMetricFilters(new ArrayList<>()) + .setStatsbeatModule(new StatsbeatModule(response -> {})) + .setConnectionStrings(FAKE_CONNECTION_STRING) + .build(); + telemetryClient.setOtelResource(Resource.empty()); + + GcEventInit.init( + alertingSubsystem, + telemetryClient, + Executors.newSingleThreadExecutor(), + new GcEventInit.GcEventMonitorConfiguration(GcReportingLevel.ALL), + factory); + + TelemetryItem gcEvent = gcEventFuture.get(10, TimeUnit.SECONDS); + + assertThat(gcEvent.getSampleRate()) + .isEqualTo((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); + } finally { + TelemetryObservers.INSTANCE.getObservers().remove(observer); + } + } + @Test void endToEndAlertIsTriggered() throws ExecutionException, InterruptedException, TimeoutException { @@ -102,4 +177,25 @@ private static GcCollectionEvent mockGcEvent() { .thenReturn(new MemoryUsage(1, 9, 10, 10)); return event; } + + private static GcCollectionEvent fullyMockedGcEvent() { + GcCollectionEvent event = Mockito.mock(GcCollectionEvent.class); + GarbageCollector collector = Mockito.mock(GarbageCollector.class); + MemoryPool tenuredPool = Mockito.mock(MemoryPool.class); + MemoryUsage memoryUsage = new MemoryUsage(1, 9, 10, 10); + + Mockito.when(collector.isTenuredCollector()).thenReturn(true); + Mockito.when(collector.getName()).thenReturn("test-collector"); + Mockito.when(event.getCollector()).thenReturn(collector); + Mockito.when(event.getGcCause()).thenReturn("test-cause"); + Mockito.when(event.getGcAction()).thenReturn("test-action"); + Mockito.when(event.getTenuredPool()).thenReturn(Optional.of(tenuredPool)); + Mockito.when(event.getMemoryUsageBeforeGc(Mockito.any(MemoryPool.class))) + .thenReturn(memoryUsage); + Mockito.when(event.getMemoryUsageAfterGc(Mockito.any(MemoryPool.class))) + .thenReturn(memoryUsage); + Mockito.when(event.getMemoryUsageBeforeGc(Mockito.anyList())).thenReturn(memoryUsage); + Mockito.when(event.getMemoryUsageAfterGc(Mockito.anyList())).thenReturn(memoryUsage); + return event; + } } From edf93b04ded3ecf791ecb37a5a046f37860fd710 Mon Sep 17 00:00:00 2001 From: John <1615532+johnoliver@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:25:55 +0100 Subject: [PATCH 2/6] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../profiler/triggers/GcEventInitTest.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index 481a2d3e877..855e18c6b7a 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -42,8 +42,6 @@ class GcEventInitTest { - private static final String FAKE_CONNECTION_STRING = - "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://fake-ingestion-endpoint.example.com/"; @Test void gcEventIsEmittedWithSampleRateThatBypassesIngestionSampling() @@ -88,21 +86,25 @@ public MemoryManagement monitor( .setCustomDimensions(new HashMap<>()) .setMetricFilters(new ArrayList<>()) .setStatsbeatModule(new StatsbeatModule(response -> {})) - .setConnectionStrings(FAKE_CONNECTION_STRING) .build(); telemetryClient.setOtelResource(Resource.empty()); - GcEventInit.init( - alertingSubsystem, - telemetryClient, - Executors.newSingleThreadExecutor(), - new GcEventInit.GcEventMonitorConfiguration(GcReportingLevel.ALL), - factory); - - TelemetryItem gcEvent = gcEventFuture.get(10, TimeUnit.SECONDS); - - assertThat(gcEvent.getSampleRate()) - .isEqualTo((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + try { + GcEventInit.init( + alertingSubsystem, + telemetryClient, + executorService, + new GcEventInit.GcEventMonitorConfiguration(GcReportingLevel.ALL), + factory); + + TelemetryItem gcEvent = gcEventFuture.get(10, TimeUnit.SECONDS); + + assertThat(gcEvent.getSampleRate()) + .isEqualTo((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); + } finally { + executorService.shutdownNow(); + } } finally { TelemetryObservers.INSTANCE.getObservers().remove(observer); } From e5ff84ed5aa7f727474e1ad9da2a9f64f6784247 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:30:13 +0000 Subject: [PATCH 3/6] Apply spotless formatting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent/internal/profiler/triggers/GcEventInitTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index 855e18c6b7a..f3ab3d77f21 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -42,7 +42,6 @@ class GcEventInitTest { - @Test void gcEventIsEmittedWithSampleRateThatBypassesIngestionSampling() throws ExecutionException, InterruptedException, TimeoutException { From 195de0d90b47894c24ab2d04c985ef042e836762 Mon Sep 17 00:00:00 2001 From: John <1615532+johnoliver@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:40:27 +0100 Subject: [PATCH 4/6] Stabilize GcEventInitTest by increasing async wait timeout to avoid CI flake --- .../agent/internal/profiler/triggers/GcEventInitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index f3ab3d77f21..fbde971d228 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -97,7 +97,7 @@ public MemoryManagement monitor( new GcEventInit.GcEventMonitorConfiguration(GcReportingLevel.ALL), factory); - TelemetryItem gcEvent = gcEventFuture.get(10, TimeUnit.SECONDS); + TelemetryItem gcEvent = gcEventFuture.get(30, TimeUnit.SECONDS); assertThat(gcEvent.getSampleRate()) .isEqualTo((float) SamplerUtil.SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING); From e3c73e25520879363673e0d76fc5ed6ad6525c24 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:08:31 +0000 Subject: [PATCH 5/6] Initial plan From 6a216903e19c1fad0608930db35400773cbf2307 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:18:29 +0000 Subject: [PATCH 6/6] Fix GcEventInitTest: add connection string so trackAsync notifies observers Root cause: TelemetryClient.trackAsync() returns early when connectionString is null, before notifying TelemetryObservers. The test built a TelemetryClient without a connection string, so the observer CompletableFuture was never completed, causing TimeoutException. Fix: supply a dummy connection string in the test builder so trackAsync() proceeds past the null check and correctly fires the registered observer. Test-only change; production code is unmodified. --- .../agent/internal/profiler/triggers/GcEventInitTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index fbde971d228..da62321a329 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -85,6 +85,8 @@ public MemoryManagement monitor( .setCustomDimensions(new HashMap<>()) .setMetricFilters(new ArrayList<>()) .setStatsbeatModule(new StatsbeatModule(response -> {})) + .setConnectionStrings( + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://test.in.applicationinsights.azure.com/") .build(); telemetryClient.setOtelResource(Resource.empty());