From a553e2038bc4588fb12fef5b48bc98469e38cdd6 Mon Sep 17 00:00:00 2001 From: deardeng Date: Tue, 4 Aug 2026 21:35:03 +0800 Subject: [PATCH 1/2] [improvement](fe) Release cloud tablet scheduling indexes after balance ### What problem does this PR solve? Issue Number: None Related PR: #66378, #66389 Problem Summary: Cloud tablet route rebuilding retains the current and future table-level and partition-level scheduling indexes for the entire sleep interval after each balancing round, although later status checks and external readers only need the global indexes. At 4 million tablets across 4 clusters, a single-threaded JDK 17 path-level model estimates that releasing these four nested graphs reduces approximate post-full-GC retained route-index heap from 4.66 GiB to 1.35 GiB, a 3.31 GiB or 70.96% reduction. Cumulative route-index construction allocation remains 7.36 GiB, so this change reduces between-round retention rather than allocation volume or in-round peak memory. Replace the four top-level maps in a finally block after balancing so all exit paths release the old graphs without an O(N) clear traversal, while preserving current and future global routes. The next route-statistics pass rebuilds the scheduling indexes before they are used again. These numbers are model estimates, not production RSS measurements. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.cloud.catalog.CloudTabletRebalancerTest (16 tests passed) - mvn checkstyle:check -pl fe-core (0 violations) - Behavior changed: No - Does this need documentation: No --- .../cloud/catalog/CloudTabletRebalancer.java | 15 ++++- .../catalog/CloudTabletRebalancerTest.java | 60 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index fde9c0d4850741..9a2c41622cfdf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -523,7 +523,11 @@ protected void runAfterCatalogReady() { indexBalanced = true; tableBalanced = true; - performBalancing(); + try { + performBalancing(); + } finally { + releaseSchedulingIndexes(); + } checkDecommissionState(clusterToBes); inited = true; @@ -642,6 +646,15 @@ private void performBalancing() { } } + private void releaseSchedulingIndexes() { + // These indexes are no longer used after balancing. Replace their top-level maps instead of clearing + // every entry so the complete tablet membership graphs can become collectible without an O(N) traversal. + partitionToTablets = new ConcurrentHashMap<>(); + futurePartitionToTablets = new ConcurrentHashMap<>(); + beToTabletsInTable = new ConcurrentHashMap<>(); + futureBeToTabletsInTable = new ConcurrentHashMap<>(); + } + private boolean shouldForceInactivePhase(boolean activeBalanced) { if (activeBalanced) { consecutiveActiveUnbalancedRounds = 0; diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java index 520932b5fb6283..9a41d2734baaa1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java @@ -349,6 +349,66 @@ public void testWarmupRollbackReusesInflightBoxedTabletIdAfterRouteRebuild() thr } } + @Test + public void testReleaseSchedulingIndexesKeepsGlobalRoutesAndAllowsNextRebuild() throws Exception { + TestRebalancer rebalancer = new TestRebalancer(); + Long srcBe = 10_001L; + Long dbId = 15_001L; + Long tableId = 20_001L; + Long partitionId = 30_001L; + Long indexId = 40_001L; + Long tabletId = 50_001L; + String clusterId = "cluster-a"; + RouteMaps current = new RouteMaps(); + RouteMaps future = new RouteMaps(); + initializeRouteMaps(rebalancer, current, future, srcBe, tableId, partitionId, indexId, tabletId); + + invokePrivate(rebalancer, "releaseSchedulingIndexes", new Class[] {}, new Object[] {}); + + ConcurrentHashMap> currentGlobal = getField(rebalancer, "beToTabletsGlobal"); + ConcurrentHashMap> futureGlobal = getField(rebalancer, "futureBeToTabletsGlobal"); + ConcurrentHashMap>> releasedCurrentByTable = + getField(rebalancer, "beToTabletsInTable"); + ConcurrentHashMap>> releasedFutureByTable = + getField(rebalancer, "futureBeToTabletsInTable"); + ConcurrentHashMap>>> + releasedCurrentByPartition = getField(rebalancer, "partitionToTablets"); + ConcurrentHashMap>>> + releasedFutureByPartition = getField(rebalancer, "futurePartitionToTablets"); + Assertions.assertSame(current.global, currentGlobal); + Assertions.assertSame(future.global, futureGlobal); + Assertions.assertNotSame(current.byTable, releasedCurrentByTable); + Assertions.assertNotSame(future.byTable, releasedFutureByTable); + Assertions.assertNotSame(current.byPartition, releasedCurrentByPartition); + Assertions.assertNotSame(future.byPartition, releasedFutureByPartition); + Assertions.assertTrue(releasedCurrentByTable.isEmpty()); + Assertions.assertTrue(releasedFutureByTable.isEmpty()); + Assertions.assertTrue(releasedCurrentByPartition.isEmpty()); + Assertions.assertTrue(releasedFutureByPartition.isEmpty()); + + setField(rebalancer, "clusterToBes", Collections.singletonMap(clusterId, List.of(srcBe))); + setField(rebalancer, "allBes", Set.of(srcBe)); + try (MockedStatic ignored = mockRouteEnvironment( + dbId, tableId, partitionId, indexId, tabletId, clusterId, srcBe)) { + rebalancer.statRouteInfo(); + } + + ConcurrentHashMap>> rebuiltCurrentByTable = + getField(rebalancer, "beToTabletsInTable"); + ConcurrentHashMap>> rebuiltFutureByTable = + getField(rebalancer, "futureBeToTabletsInTable"); + ConcurrentHashMap>>> + rebuiltCurrentByPartition = getField(rebalancer, "partitionToTablets"); + ConcurrentHashMap>>> + rebuiltFutureByPartition = getField(rebalancer, "futurePartitionToTablets"); + Assertions.assertEquals(Set.of(tabletId), rebuiltCurrentByTable.get(tableId).get(srcBe)); + Assertions.assertEquals(Set.of(tabletId), rebuiltFutureByTable.get(tableId).get(srcBe)); + Assertions.assertEquals(Set.of(tabletId), + rebuiltCurrentByPartition.get(partitionId).get(indexId).get(srcBe)); + Assertions.assertEquals(Set.of(tabletId), + rebuiltFutureByPartition.get(partitionId).get(indexId).get(srcBe)); + } + private static void initializeRouteMaps(TestRebalancer rebalancer, RouteMaps current, RouteMaps future, Long srcBe, Long tableId, Long partitionId, Long indexId, Long tabletId) throws Exception { rebalancer.fillBeToTablets(srcBe, tableId, partitionId, indexId, tabletId, From df70ffb0f9137578ed1179e4ea5ee17870a54eed Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 5 Aug 2026 11:23:33 +0800 Subject: [PATCH 2/2] [fix](fe) Release cloud tablet scheduling indexes on migration failures ### What problem does this PR solve? Issue Number: None Related PR: #66451, #66378, #66389 Problem Summary: Cloud tablet route indexes are rebuilt before smooth-upgrade migration, but the scheduling-index cleanup fence previously covered only performBalancing(). If migration or the optional second route rebuild threw after the first rebuild, the daemon propagated the failure without replacing the four current/future table-level and partition-level indexes, retaining their tablet membership graphs until a later cycle. Move the existing try/finally boundary before the first statRouteInfo() call so route rebuild, smooth-upgrade migration, the optional second rebuild, and balancing all release the scheduling-only indexes on every exit path. Global route indexes and exception propagation remain unchanged. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.cloud.catalog.CloudTabletRebalancerTest#testRunAfterCatalogReadyReleasesSchedulingIndexesWhenMigrationFails - ./run-fe-ut.sh --run org.apache.doris.cloud.catalog.CloudTabletRebalancerTest (17 tests passed) - mvn checkstyle:check -pl fe-core (0 violations) - Behavior changed: No - Does this need documentation: No --- .../cloud/catalog/CloudTabletRebalancer.java | 15 +++--- .../catalog/CloudTabletRebalancerTest.java | 48 ++++++++++++++++++- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index 9a2c41622cfdf0..d33448e0a9b10b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -514,16 +514,15 @@ protected void runAfterCatalogReady() { return; } - statRouteInfo(); - boolean migrated = migrateTabletsForSmoothUpgrade(); - if (migrated) { + try { statRouteInfo(); - } - - indexBalanced = true; - tableBalanced = true; + boolean migrated = migrateTabletsForSmoothUpgrade(); + if (migrated) { + statRouteInfo(); + } - try { + indexBalanced = true; + tableBalanced = true; performBalancing(); } finally { releaseSchedulingIndexes(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java index 9a41d2734baaa1..a2b415dd162e19 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java @@ -83,7 +83,11 @@ private static class TestRebalancer extends CloudTabletRebalancer { private final Set internalDbIds = new HashSet<>(); TestRebalancer() { - super(null); + this(null); + } + + TestRebalancer(CloudSystemInfoService cloudSystemInfoService) { + super(cloudSystemInfoService); } void setInternalDbIds(Set ids) { @@ -409,6 +413,47 @@ public void testReleaseSchedulingIndexesKeepsGlobalRoutesAndAllowsNextRebuild() rebuiltFutureByPartition.get(partitionId).get(indexId).get(srcBe)); } + @Test + public void testRunAfterCatalogReadyReleasesSchedulingIndexesWhenMigrationFails() throws Exception { + Long srcBe = 10_001L; + Long destBe = 10_002L; + Long dbId = 15_001L; + Long tableId = 20_001L; + Long partitionId = 30_001L; + Long indexId = 40_001L; + Long tabletId = 50_001L; + String clusterId = "cluster-a"; + CloudSystemInfoService systemInfoService = Mockito.mock(CloudSystemInfoService.class); + Backend srcBackend = Mockito.mock(Backend.class); + Mockito.when(systemInfoService.getAllBackendIds()).thenReturn(List.of(srcBe)); + Mockito.when(systemInfoService.getBackend(srcBe)).thenReturn(srcBackend); + Mockito.when(srcBackend.getCloudClusterId()).thenReturn(clusterId); + TestRebalancer rebalancer = new TestRebalancer(systemInfoService); + rebalancer.addTabletMigrationTask(srcBe, destBe); + + boolean oldEnableCloudMultiReplica = Config.enable_cloud_multi_replica; + Config.enable_cloud_multi_replica = false; + try (MockedStatic ignored = mockRouteEnvironment( + dbId, tableId, partitionId, indexId, tabletId, clusterId, srcBe)) { + TabletInvertedIndex invertedIndex = Env.getCurrentEnv().getTabletInvertedIndex(); + Mockito.when(invertedIndex.getTabletMeta(tabletId)) + .thenThrow(new RuntimeException("injected migration failure")); + + RuntimeException exception = Assertions.assertThrows( + RuntimeException.class, rebalancer::runAfterCatalogReady); + + Assertions.assertEquals("injected migration failure", exception.getMessage()); + ConcurrentHashMap> currentGlobal = getField(rebalancer, "beToTabletsGlobal"); + Assertions.assertEquals(Set.of(tabletId), currentGlobal.get(srcBe)); + Assertions.assertTrue(((Map) getField(rebalancer, "beToTabletsInTable")).isEmpty()); + Assertions.assertTrue(((Map) getField(rebalancer, "futureBeToTabletsInTable")).isEmpty()); + Assertions.assertTrue(((Map) getField(rebalancer, "partitionToTablets")).isEmpty()); + Assertions.assertTrue(((Map) getField(rebalancer, "futurePartitionToTablets")).isEmpty()); + } finally { + Config.enable_cloud_multi_replica = oldEnableCloudMultiReplica; + } + } + private static void initializeRouteMaps(TestRebalancer rebalancer, RouteMaps current, RouteMaps future, Long srcBe, Long tableId, Long partitionId, Long indexId, Long tabletId) throws Exception { rebalancer.fillBeToTablets(srcBe, tableId, partitionId, indexId, tabletId, @@ -475,6 +520,7 @@ private static MockedStatic mockRouteEnvironment(Long dbId, Long tableId, L Mockito.when(tablet.getReplicas()).thenReturn(Collections.singletonList(replica)); Mockito.when(replica.getPrimaryBackend(clusterId, false)).thenReturn(primaryBackend); Mockito.when(primaryBackend.getId()).thenReturn(srcBe); + Mockito.when(primaryBackend.isQueryAvailable()).thenReturn(true); MockedStatic mockedEnv = Mockito.mockStatic(Env.class); mockedEnv.when(Env::getCurrentEnv).thenReturn(env);