From 4f35f7d6b5b32375f8432a4356db46cab4d3bc3f Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 10 Jul 2026 18:30:44 -0700 Subject: [PATCH 1/4] HDDS-15828. Fix flaky TestDirectoryDeletingServiceWithFSO snapshot cleanup. Wait for SnapshotDeletingService to purge deleted snapshots before snapshot tests return on the shared mini-cluster, so later tests do not chain off stale global snapshot entries. Co-authored-by: Cursor Change-Id: I57ec596cd5870b68f21ff9d9a89850ea9832f56e --- .../TestDirectoryDeletingServiceWithFSO.java | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java index e64a3f09e216..6b99731479bf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java @@ -571,7 +571,7 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() Table snapshotInfoTable = omMetadataManager.getSnapshotInfoTable(); Table deletedDirTable = omMetadataManager.getDeletedDirTable(); Table renameTable = omMetadataManager.getSnapshotRenamedTable(); - cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService().shutdown(); + cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService().suspend(); DirectoryDeletingService dirDeletingService = cluster.getOzoneManager().getKeyManager().getDirDeletingService(); // Suspend KeyDeletingService dirDeletingService.suspend(); @@ -659,23 +659,7 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() service.runPeriodicalTaskNow(); store.deleteSnapshot(testVolumeName, testBucketName, snap1); cluster.getOzoneManager().awaitDoubleBufferFlush(); - cluster.restartOzoneManager(); - cluster.waitForClusterToBeReady(); - cluster.getOzoneManager().awaitDoubleBufferFlush(); - SnapshotDeletingService snapshotDeletingService = - cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService(); - GenericTestUtils.waitFor(() -> { - try { - snapshotDeletingService.runPeriodicalTaskNow(); - cluster.getOzoneManager().awaitDoubleBufferFlush(); - long currentSnapshotCount = cluster.getOzoneManager().getMetadataManager() - .countRowsInTable(cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable()); - return currentSnapshotCount <= initialSnapshotCount; - } catch (Exception e) { - throw new RuntimeException(e); - } - }, 100, 10000); - assertTableRowCount(cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable(), initialSnapshotCount); + waitForSnapshotsPurged(snapshotInfoTable, initialSnapshotCount); dirDeletingService.resume(); } @@ -789,6 +773,8 @@ public void testDirDeletedTableCleanUpForSnapshot() throws Exception { // Manual cleanup deletedDirTable for next tests client.getObjectStore().deleteSnapshot(volumeName, bucketName, "snap1"); + cluster.getOzoneManager().awaitDoubleBufferFlush(); + waitForSnapshotsPurged(snapshotInfoTable, 0); cleanupTables(); } @@ -823,6 +809,28 @@ static void assertSubPathsCount(LongSupplier pathCount, long expectedCount) 1000, 120000); } + /** + * Wait until SnapshotDeletingService has purged snapshots and snapshotInfoTable + * reaches the expected row count. + */ + private void waitForSnapshotsPurged(Table snapshotInfoTable, + int expectedCount) throws Exception { + SnapshotDeletingService snapshotDeletingService = + cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService(); + snapshotDeletingService.resume(); + GenericTestUtils.waitFor(() -> { + try { + snapshotDeletingService.runPeriodicalTaskNow(); + cluster.getOzoneManager().awaitDoubleBufferFlush(); + return cluster.getOzoneManager().getMetadataManager() + .countRowsInTable(snapshotInfoTable) == expectedCount; + } catch (Exception e) { + return false; + } + }, 1000, 120000); + assertTableRowCount(snapshotInfoTable, expectedCount); + } + private void assertTableRowCount(Table table, int count) throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000, From e24f4f0527f5b8e05321bf73baafea8e829cfc9a Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 13 Jul 2026 11:38:52 -0700 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../om/service/TestDirectoryDeletingServiceWithFSO.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java index 6b99731479bf..d4bba6364dc0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java @@ -824,9 +824,9 @@ private void waitForSnapshotsPurged(Table snapshotInfoTabl cluster.getOzoneManager().awaitDoubleBufferFlush(); return cluster.getOzoneManager().getMetadataManager() .countRowsInTable(snapshotInfoTable) == expectedCount; - } catch (Exception e) { - return false; - } +} catch (Exception e) { + throw new RuntimeException("Failed to run SnapshotDeletingService purge task", e); +} }, 1000, 120000); assertTableRowCount(snapshotInfoTable, expectedCount); } From 45abf8f7c61e7aac32dc186c507f4fa6872e4455 Mon Sep 17 00:00:00 2001 From: Arun Sarin Date: Sun, 6 Sep 2026 02:49:35 +0530 Subject: [PATCH 3/4] HDDS-15828. Address review comments: move cleanup to @AfterEach, fix waitForSnapshotsPurged --- .../TestDirectoryDeletingServiceWithFSO.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java index d4bba6364dc0..6d0fae92936b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java @@ -107,6 +107,11 @@ public class TestDirectoryDeletingServiceWithFSO { private static OzoneClient client; private static DeletingServiceMetrics metrics; + // Per-test state for @AfterEach cleanup: tracked when a test suspends services or needs table cleanup. + private int snapshotCountAfterTest = -1; + private DirectoryDeletingService pendingResumeDds = null; + private boolean needsTableCleanup = false; + @BeforeAll public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); @@ -149,7 +154,21 @@ public static void teardown() { } @AfterEach - public void cleanup() throws InterruptedException, TimeoutException { + public void cleanup() throws Exception { + if (snapshotCountAfterTest >= 0) { + Table snapshotInfoTable = + cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable(); + waitForSnapshotsPurged(snapshotInfoTable, snapshotCountAfterTest); + snapshotCountAfterTest = -1; + } + if (pendingResumeDds != null) { + pendingResumeDds.resume(); + pendingResumeDds = null; + } + if (needsTableCleanup) { + cleanupTables(); + needsTableCleanup = false; + } assertDoesNotThrow(() -> { Path root = new Path("/"); FileStatus[] fileStatuses = fs.listStatus(root); @@ -575,6 +594,7 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() DirectoryDeletingService dirDeletingService = cluster.getOzoneManager().getKeyManager().getDirDeletingService(); // Suspend KeyDeletingService dirDeletingService.suspend(); + pendingResumeDds = dirDeletingService; Random random = new Random(); final String testVolumeName = "volume" + random.nextInt(); final String testBucketName = "bucket" + random.nextInt(); @@ -594,6 +614,7 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() service.shutdown(); final int initialSnapshotCount = (int) cluster.getOzoneManager().getMetadataManager().countRowsInTable(snapshotInfoTable); + snapshotCountAfterTest = initialSnapshotCount; final int initialDeletedCount = (int) omMetadataManager.countRowsInTable(deletedDirTable); final int initialRenameCount = (int) omMetadataManager.countRowsInTable(renameTable); String snap1 = "snap1"; @@ -659,8 +680,6 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() service.runPeriodicalTaskNow(); store.deleteSnapshot(testVolumeName, testBucketName, snap1); cluster.getOzoneManager().awaitDoubleBufferFlush(); - waitForSnapshotsPurged(snapshotInfoTable, initialSnapshotCount); - dirDeletingService.resume(); } @Test @@ -771,11 +790,10 @@ public void testDirDeletedTableCleanUpForSnapshot() throws Exception { assertSubPathsCount(dirDeletingService::getMovedDirsCount, 4); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0); - // Manual cleanup deletedDirTable for next tests + // Manual cleanup deletedDirTable for next tests - completed in @AfterEach cleanup() client.getObjectStore().deleteSnapshot(volumeName, bucketName, "snap1"); - cluster.getOzoneManager().awaitDoubleBufferFlush(); - waitForSnapshotsPurged(snapshotInfoTable, 0); - cleanupTables(); + snapshotCountAfterTest = 0; + needsTableCleanup = true; } private void cleanupTables() throws IOException { @@ -824,11 +842,10 @@ private void waitForSnapshotsPurged(Table snapshotInfoTabl cluster.getOzoneManager().awaitDoubleBufferFlush(); return cluster.getOzoneManager().getMetadataManager() .countRowsInTable(snapshotInfoTable) == expectedCount; -} catch (Exception e) { - throw new RuntimeException("Failed to run SnapshotDeletingService purge task", e); -} + } catch (Exception e) { + throw new RuntimeException("Failed to run SnapshotDeletingService purge task", e); + } }, 1000, 120000); - assertTableRowCount(snapshotInfoTable, expectedCount); } private void assertTableRowCount(Table table, int count) From 9a67d4e406cd06528fc5b1ce5712c31ad36088b8 Mon Sep 17 00:00:00 2001 From: Arun Sarin Date: Thu, 10 Sep 2026 00:52:32 +0530 Subject: [PATCH 4/4] HDDS-15828. Addressed review comments --- .../Ozone - Container Balancer Metrics.json | 1379 ----------------- .../TestDirectoryDeletingServiceWithFSO.java | 71 +- 2 files changed, 39 insertions(+), 1411 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Container Balancer Metrics.json diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Container Balancer Metrics.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Container Balancer Metrics.json deleted file mode 100644 index bc8360c312b2..000000000000 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Container Balancer Metrics.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "annotations": [ - { - "kind": "AnnotationQuery", - "spec": { - "builtIn": true, - "enable": true, - "hide": true, - "iconColor": "", - "name": "Annotations & Alerts", - "query": { - "group": "grafana", - "kind": "DataQuery", - "spec": {}, - "version": "v0" - } - } - } - ], - "cursorSync": "Crosshair", - "description": "Comprehensive tracking of Ozone cluster balancing operations. Monitors real-time Datanode capacity convergence, current iteration health (Scheduled vs Completed), and lifetime data movement metrics.", - "editable": true, - "elements": { - "panel-1": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_datanodes_unbalanced)", - "legendFormat": "Unbalanced Datanodes", - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Tracks the total number of Datanodes whose capacity usage falls outside the configured cluster balance threshold. A healthy, fully balanced cluster should ideally maintain a value of 0.", - "id": 1, - "links": [], - "title": "Unbalanced Datanodes", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-2": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_data_size_unbalanced_gb * 1024 * 1024 * 1024)", - "legendFormat": "Total Unbalanced Data Size", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Represents the total volume of data in gigabytes currently residing on over-utilized nodes that must be shifted to under-utilized nodes to satisfy your configured container balancing thresholds.", - "id": 2, - "links": [], - "title": "Cluster Unbalanced Data Size Over Time", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-3": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_data_size_moved_gb_in_latest_iteration)", - "legendFormat": "Moved Data Size (GB)", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Measures the total volume of data in gigabytes successfully transferred between source and target Datanodes during the most recently executed balancer iteration loop.", - "id": 3, - "links": [], - "title": "Size Moved (Latest)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "decgbytes" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-4": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_datanodes_involved_in_latest_iteration)", - "legendFormat": "Datanodes Involved", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "The count of unique Datanode hosts that actively participated as either a source (sender) or target (receiver) of data blocks in the latest iteration.", - "id": 4, - "links": [], - "title": "Datanodes Involved (Latest)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-5": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_scheduled_in_latest_iteration)", - "legendFormat": "Scheduled", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_completed_in_latest_iteration)", - "legendFormat": "Completed", - "range": true - }, - "version": "v0" - }, - "refId": "B" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_failed_in_latest_iteration)", - "legendFormat": "Failed", - "range": true - }, - "version": "v0" - }, - "refId": "C" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_timeout_in_latest_iteration)", - "legendFormat": "Timeout", - "range": true - }, - "version": "v0" - }, - "refId": "D" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "A real-time status breakdown of individual container transfers during the current or latest iteration. Displays the exact counts of Scheduled, Completed, Failed, and Timeout movements.", - "id": 5, - "links": [], - "title": "Container Move Operations Breakdown (Latest Iteration)", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-6": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "container_balancer_metrics_data_size_moved_gb * 1024 * 1024 * 1024", - "legendFormat": "Total Data Moved" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "An accumulating historical counter showing the total volume of data moved across the cluster since tracking began. This acts as a lifetime indicator of balancer workload.", - "id": 6, - "links": [], - "title": "Cumulative Data Volume Moved", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-7": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_completed)", - "legendFormat": "Completed", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_failed)", - "instant": false, - "legendFormat": "Failed", - "range": true - }, - "version": "v0" - }, - "refId": "B" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_scheduled)", - "instant": false, - "legendFormat": "Scheduled", - "range": true - }, - "version": "v0" - }, - "refId": "C" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(container_balancer_metrics_num_container_moves_timeout)", - "instant": false, - "legendFormat": "Timeout", - "range": true - }, - "version": "v0" - }, - "refId": "D" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Cluster-wide historical aggregation of all attempted container migrations since inception. Compares total Scheduled vs Completed moves alongside long-term Failed and Timeout counts to assess network and disk reliability.", - "id": 7, - "links": [], - "title": "Cumulative Executed Container Moves", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Completed" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-8": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "container_balancer_metrics_num_iterations", - "legendFormat": "Completed Iterations" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "The lifetime count of fully executed, successful balancing loops completed by the Storage Container Manager (SCM). If the balancer exits during initialization due to an already balanced cluster, this counter does not increment.", - "id": 8, - "links": [], - "title": "Total Balancer Iterations Completed", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Completed Iterations" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.0.1+security-01" - } - } - }, - "panel-9": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "${datasource}" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "volume_info_metrics_used", - "legendFormat": "{{hostname}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Tracks the raw physical bytes consumed across individual Datanode storage volumes over time. This panel visualizes how storage distribution scales and shifts across nodes during active cluster balancing.", - "id": 9, - "links": [], - "title": "Datanode Disk Usage (Convergence)", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.0.1+security-01" - } - } - } - }, - "layout": { - "kind": "RowsLayout", - "spec": { - "rows": [ - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-1" - }, - "height": 4, - "width": 10, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-9" - }, - "height": 4, - "width": 14, - "x": 10, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-2" - }, - "height": 4, - "width": 24, - "x": 0, - "y": 4 - } - } - ] - } - }, - "title": "Cluster Imbalance Status" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-3" - }, - "height": 4, - "width": 8, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-4" - }, - "height": 4, - "width": 8, - "x": 8, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-5" - }, - "height": 4, - "width": 8, - "x": 16, - "y": 0 - } - } - ] - } - }, - "title": "Latest Iteration Metrics" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-6" - }, - "height": 5, - "width": 8, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-7" - }, - "height": 5, - "width": 8, - "x": 8, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-8" - }, - "height": 5, - "width": 8, - "x": 16, - "y": 0 - } - } - ] - } - }, - "title": "Lifetime Metrics" - } - } - ] - } - }, - "links": [], - "liveNow": false, - "preload": false, - "tags": [ - "Ozone", - "SCM" - ], - "timeSettings": { - "autoRefresh": "5s", - "autoRefreshIntervals": [ - "5s", - "10s", - "30s" - ], - "fiscalYearStartMonth": 0, - "from": "now-24h", - "hideTimepicker": false, - "timezone": "browser", - "to": "now" - }, - "title": "Ozone - Container Balancer", - "variables": [ - { - "kind": "DatasourceVariable", - "spec": { - "allowCustomValue": true, - "current": { - "text": "default", - "value": "default" - }, - "hide": "dontHide", - "includeAll": false, - "label": "Datasource", - "multi": false, - "name": "datasource", - "options": [], - "pluginId": "prometheus", - "refresh": "onDashboardLoad", - "regex": "", - "skipUrlSync": false - } - } - ] -} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java index 6d0fae92936b..a3683bc2537a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java @@ -109,8 +109,10 @@ public class TestDirectoryDeletingServiceWithFSO { // Per-test state for @AfterEach cleanup: tracked when a test suspends services or needs table cleanup. private int snapshotCountAfterTest = -1; + private boolean pendingSdsResume = false; private DirectoryDeletingService pendingResumeDds = null; private boolean needsTableCleanup = false; + private String snapshotToDeleteInCleanup = null; @BeforeAll public static void init() throws Exception { @@ -155,19 +157,34 @@ public static void teardown() { @AfterEach public void cleanup() throws Exception { - if (snapshotCountAfterTest >= 0) { - Table snapshotInfoTable = - cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable(); - waitForSnapshotsPurged(snapshotInfoTable, snapshotCountAfterTest); - snapshotCountAfterTest = -1; - } - if (pendingResumeDds != null) { - pendingResumeDds.resume(); - pendingResumeDds = null; - } - if (needsTableCleanup) { - cleanupTables(); - needsTableCleanup = false; + try { + if (pendingSdsResume) { + cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService().resume(); + pendingSdsResume = false; + } + if (snapshotToDeleteInCleanup != null) { + try { + client.getObjectStore().deleteSnapshot(volumeName, bucketName, snapshotToDeleteInCleanup); + } catch (Exception ignored) { + // snapshot may have already been deleted by the test body + } + snapshotToDeleteInCleanup = null; + } + if (snapshotCountAfterTest >= 0) { + Table snapshotInfoTable = + cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable(); + waitForSnapshotsPurged(snapshotInfoTable, snapshotCountAfterTest); + snapshotCountAfterTest = -1; + } + } finally { + if (pendingResumeDds != null) { + pendingResumeDds.resume(); + pendingResumeDds = null; + } + if (needsTableCleanup) { + cleanupTables(); + needsTableCleanup = false; + } } assertDoesNotThrow(() -> { Path root = new Path("/"); @@ -591,6 +608,7 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() Table deletedDirTable = omMetadataManager.getDeletedDirTable(); Table renameTable = omMetadataManager.getSnapshotRenamedTable(); cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService().suspend(); + pendingSdsResume = true; DirectoryDeletingService dirDeletingService = cluster.getOzoneManager().getKeyManager().getDirDeletingService(); // Suspend KeyDeletingService dirDeletingService.suspend(); @@ -733,8 +751,11 @@ public void testDirDeletedTableCleanUpForSnapshot() throws Exception { assertTableRowCount(keyTable, 8); assertTableRowCount(dirTable, 5); - // Create snapshot + // Create snapshot; register for cleanup in @AfterEach in case the test fails before deleting it. client.getObjectStore().createSnapshot(volumeName, bucketName, "snap1"); + snapshotToDeleteInCleanup = "snap1"; + snapshotCountAfterTest = 0; + needsTableCleanup = true; assertTableRowCount(snapshotInfoTable, 1); // Case-1) Delete 3 Files directly. @@ -790,10 +811,8 @@ public void testDirDeletedTableCleanUpForSnapshot() throws Exception { assertSubPathsCount(dirDeletingService::getMovedDirsCount, 4); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0); - // Manual cleanup deletedDirTable for next tests - completed in @AfterEach cleanup() + // Snapshot deletion and table cleanup are handled in @AfterEach cleanup(). client.getObjectStore().deleteSnapshot(volumeName, bucketName, "snap1"); - snapshotCountAfterTest = 0; - needsTableCleanup = true; } private void cleanupTables() throws IOException { @@ -829,23 +848,11 @@ static void assertSubPathsCount(LongSupplier pathCount, long expectedCount) /** * Wait until SnapshotDeletingService has purged snapshots and snapshotInfoTable - * reaches the expected row count. + * reaches the expected row count. SDS must already be resumed before calling this. */ private void waitForSnapshotsPurged(Table snapshotInfoTable, - int expectedCount) throws Exception { - SnapshotDeletingService snapshotDeletingService = - cluster.getOzoneManager().getKeyManager().getSnapshotDeletingService(); - snapshotDeletingService.resume(); - GenericTestUtils.waitFor(() -> { - try { - snapshotDeletingService.runPeriodicalTaskNow(); - cluster.getOzoneManager().awaitDoubleBufferFlush(); - return cluster.getOzoneManager().getMetadataManager() - .countRowsInTable(snapshotInfoTable) == expectedCount; - } catch (Exception e) { - throw new RuntimeException("Failed to run SnapshotDeletingService purge task", e); - } - }, 1000, 120000); + int expectedCount) throws TimeoutException, InterruptedException { + assertTableRowCount(snapshotInfoTable, expectedCount); } private void assertTableRowCount(Table table, int count)