Skip to content

Add pagination metrics for known tests fetch #11925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
calvinbayer wants to merge 1 commit into
base: master
Choose a base branch
from
+44 −2
Draft

Add pagination metrics for known tests fetch #11925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions ...t-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ public Map<String, Collection<TestFQN>> getKnownTestsByModule(TracerEnvironment
Map<String, Map<String, List<String>>> aggregateTests = new HashMap<>();
String pageState = null;
int pageNumber = 0;
long startMs = System.currentTimeMillis();
int totalRequestMs = 0;

do {
pageNumber += 1;
Expand All @@ -233,6 +235,7 @@ public Map<String, Collection<TestFQN>> getKnownTestsByModule(TracerEnvironment
new Envelope<>(new Data<>(uuid, "ci_app_libraries_tests_request", requestDto));
String json = knownTestsRequestAdapter.toJson(request);
RequestBody requestBody = RequestBody.create(JSON, json);
long pageStartMs = System.currentTimeMillis();
KnownTestsResponse knownTests =
backendApi.post(
KNOWN_TESTS_URI,
Expand All @@ -243,6 +246,8 @@ public Map<String, Collection<TestFQN>> getKnownTestsByModule(TracerEnvironment
.attributes,
telemetryListener,
false);
long pageEndMs = System.currentTimeMillis();
totalRequestMs += (int) (pageEndMs - pageStartMs);

mergeKnownTests(aggregateTests, knownTests.tests);

Expand All @@ -266,6 +271,12 @@ public Map<String, Collection<TestFQN>> getKnownTestsByModule(TracerEnvironment
: 0;
LOGGER.debug("Received {} known tests in total", knownTestsCount);
metricCollector.add(CiVisibilityDistributionMetric.KNOWN_TESTS_RESPONSE_TESTS, knownTestsCount);
long totalFetchMs = System.currentTimeMillis() - startMs;
metricCollector.add(CiVisibilityDistributionMetric.KNOWN_TESTS_PAGES_FETCHED, pageNumber);
metricCollector.add(
CiVisibilityDistributionMetric.KNOWN_TESTS_TOTAL_FETCH_MS, (int) totalFetchMs);
metricCollector.add(
CiVisibilityDistributionMetric.KNOWN_TESTS_TOTAL_REQUEST_MS, totalRequestMs);
// returning null disables features that rely on known tests; this is intentional on the very
// first execution for a repository, when we want to seed the backend with the initial set.
return testsByModule;
Expand Down
29 changes: 27 additions & 2 deletions ...-visibility/src/test/java/datadog/trace/civisibility/config/ConfigurationApiImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import datadog.communication.BackendApi;
Expand All @@ -13,6 +15,8 @@
import datadog.communication.http.OkHttpUtils;
import datadog.trace.agent.test.server.http.JavaTestHttpServer;
import datadog.trace.api.civisibility.config.TestFQN;
import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric;
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector;
import datadog.trace.api.intake.Intake;
import datadog.trace.civisibility.config.api.dto.request.TracerEnvironment;
Expand Down Expand Up @@ -214,7 +218,9 @@ void testKnownTestsRequestWithPagination() throws IOException {
sendResponseBody(api, responses.get(current - 1).getBytes());
})));
servers.add(intakeServer);
ConfigurationApi configurationApi = givenConfigurationApi(intakeServer, true, true);
CiVisibilityMetricCollector metricCollector = mock(CiVisibilityMetricCollector.class);
ConfigurationApi configurationApi =
givenConfigurationApi(intakeServer, true, true, metricCollector);

Map<String, Collection<TestFQN>> knownTests =
configurationApi.getKnownTestsByModule(tracerEnvironment);
Expand All @@ -233,15 +239,34 @@ void testKnownTestsRequestWithPagination() throws IOException {
"module-c",
Arrays.asList(new TestFQN("suite-c", "test-5"), new TestFQN("suite-c", "test-6")));
assertEquals(expected, knownTests);

// Verify pagination metrics
verify(metricCollector)
.add(eq(CiVisibilityDistributionMetric.KNOWN_TESTS_RESPONSE_TESTS), eq(6));
verify(metricCollector)
.add(eq(CiVisibilityDistributionMetric.KNOWN_TESTS_PAGES_FETCHED), eq(3));
verify(metricCollector)
.add(eq(CiVisibilityDistributionMetric.KNOWN_TESTS_TOTAL_FETCH_MS), any(Integer.class));
verify(metricCollector)
.add(eq(CiVisibilityDistributionMetric.KNOWN_TESTS_TOTAL_REQUEST_MS), any(Integer.class));
}

private ConfigurationApi givenConfigurationApi(
JavaTestHttpServer intakeServer, boolean agentless, boolean compression) {
return givenConfigurationApi(
intakeServer, agentless, compression, NoOpMetricCollector.INSTANCE);
}

private ConfigurationApi givenConfigurationApi(
JavaTestHttpServer intakeServer,
boolean agentless,
boolean compression,
CiVisibilityMetricCollector metricCollector) {
BackendApi api =
agentless
? givenIntakeApi(intakeServer.getAddress(), compression)
: givenEvpProxy(intakeServer.getAddress(), compression);
return new ConfigurationApiImpl(api, NoOpMetricCollector.INSTANCE, () -> REQUEST_UID);
return new ConfigurationApiImpl(api, metricCollector, () -> REQUEST_UID);
}

private JavaTestHttpServer givenBackendEndpoint(
Expand Down
6 changes: 6 additions & 0 deletions ...rc/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ public enum CiVisibilityDistributionMetric {
KNOWN_TESTS_RESPONSE_BYTES("known_tests.response_bytes", ResponseCompressed.class),
/** The number of tests received by the known tests endpoint */
KNOWN_TESTS_RESPONSE_TESTS("known_tests.response_tests"),
/** The number of pages fetched during known tests pagination */
KNOWN_TESTS_PAGES_FETCHED("known_tests.pages_fetched"),
/** The total wall-clock time to fetch all pages of known tests in ms */
KNOWN_TESTS_TOTAL_FETCH_MS("known_tests.total_fetch_ms"),
/** The sum of individual per-page request durations in ms */
KNOWN_TESTS_TOTAL_REQUEST_MS("known_tests.total_request_ms"),
/** The time it takes to get the response of the flaky tests endpoint request in ms */
FLAKY_TESTS_REQUEST_MS("flaky_tests.request_ms"),
/** The number of bytes received by the flaky tests endpoint */
Expand Down