diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index e921fd499427..4066a56f9635 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -664,6 +664,14 @@ possible configurations would overwhelm and obscure the important.
true
Whether to merge a region as part of normalization.
+
+ hbase.normalizer.skip.broken.chain
+ true
+ Whether the normalizer should skip a table whose region chain is broken
+ (contains a hole, an overlap, or a degenerate region). Normalizing (in particular
+ splitting) such a table can increase the number of holes/overlaps and complicate the
+ eventual repair by the CatalogJanitor/HBCK. Set to false to normalize regardless.
+
hbase.normalizer.merge.min.region.count
3
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 934b8de1b94c..e673f675e085 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.assignment.RegionStates;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -68,6 +69,8 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
static final boolean DEFAULT_SPLIT_ENABLED = true;
static final String MERGE_ENABLED_KEY = "hbase.normalizer.merge.enabled";
static final boolean DEFAULT_MERGE_ENABLED = true;
+ static final String SKIP_BROKEN_CHAIN_KEY = "hbase.normalizer.skip.broken.chain";
+ static final boolean DEFAULT_SKIP_BROKEN_CHAIN = true;
/**
* @deprecated since 2.5.0 and will be removed in 4.0.0. Use
* {@link SimpleRegionNormalizer#MERGE_MIN_REGION_COUNT_KEY} instead.
@@ -178,6 +181,13 @@ public boolean isMergeEnabled() {
return normalizerConfiguration.isMergeEnabled();
}
+ /**
+ * Return this instance's configured value for {@value #SKIP_BROKEN_CHAIN_KEY}.
+ */
+ public boolean isSkipBrokenChain() {
+ return normalizerConfiguration.isSkipBrokenChain();
+ }
+
/**
* Return this instance's configured value for {@value #MERGE_MIN_REGION_COUNT_KEY}.
*/
@@ -231,6 +241,23 @@ public List computePlansForTable(final TableDescriptor tableD
return Collections.emptyList();
}
+ if (normalizerConfiguration.isSkipBrokenChain()) {
+ final String brokenChainReason = findBrokenRegionChain(ctx.getTableRegions());
+ if (brokenChainReason != null) {
+ // Normalizing (in particular splitting) a table whose region chain already has holes or
+ // overlaps can increase the number of holes/overlaps and complicate the eventual repair by
+ // the CatalogJanitor/HBCK. Stand down until the chain is intact. Note the region list comes
+ // from the in-memory RegionStates, which can briefly lag hbase:meta while regions are in
+ // transition, so this may be transient and self-correct on the next run.
+ LOG.warn(
+ "Skipping normalization of table {} because its region chain appears to be broken: {}."
+ + " If this persists, run the CatalogJanitor/HBCK to fix holes and overlaps. Set {}="
+ + "false to normalize regardless.",
+ table, brokenChainReason, SKIP_BROKEN_CHAIN_KEY);
+ return Collections.emptyList();
+ }
+ }
+
LOG.debug("Computing normalization plan for table: {}, number of regions: {}", table,
ctx.getTableRegions().size());
@@ -261,6 +288,79 @@ public List computePlansForTable(final TableDescriptor tableD
return plans;
}
+ /**
+ * Walk the region chain of a table (already sorted by {@link RegionInfo#COMPARATOR}) and detect
+ * the first structural inconsistency: a degenerate region (startKey > endKey), an overlap, or a
+ * hole between adjacent regions. Non-default replicas and split-parent regions are ignored, as
+ * they legitimately do not participate in the table's linear key-space chain. Boundary
+ * (first/last) completeness is intentionally not checked here; only gaps/overlaps between the
+ * regions that are present are considered.
+ *
+ * The input is derived from the in-memory {@link RegionStates}, which can briefly lag
+ * {@code hbase:meta} while regions are in transition, so a positive result may be transient.
+ * @param tableRegions regions of a single table, sorted by {@link RegionInfo#COMPARATOR}
+ * @return a human-readable description of the first problem found, or {@code null} if the chain is
+ * intact.
+ */
+ private static String findBrokenRegionChain(final List tableRegions) {
+ RegionInfo previous = null;
+ RegionInfo highestEndKeyRegionInfo = null;
+ for (final RegionInfo current : tableRegions) {
+ if (current.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID || current.isSplitParent()) {
+ continue;
+ }
+ if (current.isDegenerate()) {
+ return "degenerate region " + current.getShortNameToLog() + " (startKey > endKey)";
+ }
+ if (previous != null) {
+ if (!previous.isNext(current)) {
+ if (previous.isOverlap(current)) {
+ return "overlap between " + previous.getShortNameToLog() + " and "
+ + current.getShortNameToLog();
+ } else if (current.isOverlap(highestEndKeyRegionInfo)) {
+ // A region seen a few rows back may still overlap the current region.
+ return "overlap between " + highestEndKeyRegionInfo.getShortNameToLog() + " and "
+ + current.getShortNameToLog();
+ } else if (!highestEndKeyRegionInfo.isNext(current)) {
+ return "hole between " + highestEndKeyRegionInfo.getShortNameToLog() + " and "
+ + current.getShortNameToLog();
+ }
+ } else if (current.isOverlap(highestEndKeyRegionInfo)) {
+ // Adjacent to the immediate predecessor, but overlaps an earlier, wider region.
+ return "overlap between " + highestEndKeyRegionInfo.getShortNameToLog() + " and "
+ + current.getShortNameToLog();
+ }
+ }
+ previous = current;
+ highestEndKeyRegionInfo = getRegionInfoWithLargestEndKey(highestEndKeyRegionInfo, current);
+ }
+ return null;
+ }
+
+ /**
+ * Returns whichever of {@code a} or {@code b} has the larger end key, treating an empty end key
+ * (the last region of a table) as the largest. Mirrors the running "highest end key seen" tracking
+ * used by the CatalogJanitor consistency check.
+ */
+ private static RegionInfo getRegionInfoWithLargestEndKey(RegionInfo a, RegionInfo b) {
+ if (a == null) {
+ return b;
+ }
+ if (b == null) {
+ return a;
+ }
+ if (!a.getTable().equals(b.getTable())) {
+ return b;
+ }
+ if (a.isLast()) {
+ return a;
+ }
+ if (b.isLast()) {
+ return b;
+ }
+ return Bytes.compareTo(a.getEndKey(), b.getEndKey()) >= 0 ? a : b;
+ }
+
/** Returns size of region in MB and if region is not found than -1 */
private long getRegionSizeMB(RegionInfo hri) {
ServerName sn =
@@ -518,6 +618,7 @@ private static final class NormalizerConfiguration {
private final Configuration conf;
private final boolean splitEnabled;
private final boolean mergeEnabled;
+ private final boolean skipBrokenChain;
private final int mergeMinRegionCount;
private final Period mergeMinRegionAge;
private final long mergeMinRegionSizeMb;
@@ -528,6 +629,7 @@ private NormalizerConfiguration() {
conf = null;
splitEnabled = DEFAULT_SPLIT_ENABLED;
mergeEnabled = DEFAULT_MERGE_ENABLED;
+ skipBrokenChain = DEFAULT_SKIP_BROKEN_CHAIN;
mergeMinRegionCount = DEFAULT_MERGE_MIN_REGION_COUNT;
mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS);
mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB;
@@ -540,6 +642,7 @@ private NormalizerConfiguration(final Configuration conf,
this.conf = conf;
splitEnabled = conf.getBoolean(SPLIT_ENABLED_KEY, DEFAULT_SPLIT_ENABLED);
mergeEnabled = conf.getBoolean(MERGE_ENABLED_KEY, DEFAULT_MERGE_ENABLED);
+ skipBrokenChain = conf.getBoolean(SKIP_BROKEN_CHAIN_KEY, DEFAULT_SKIP_BROKEN_CHAIN);
mergeMinRegionCount = parseMergeMinRegionCount(conf);
mergeMinRegionAge = parseMergeMinRegionAge(conf);
mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf);
@@ -550,6 +653,8 @@ private NormalizerConfiguration(final Configuration conf,
splitEnabled);
logConfigurationUpdated(MERGE_ENABLED_KEY, currentConfiguration.isMergeEnabled(),
mergeEnabled);
+ logConfigurationUpdated(SKIP_BROKEN_CHAIN_KEY, currentConfiguration.isSkipBrokenChain(),
+ skipBrokenChain);
logConfigurationUpdated(MERGE_MIN_REGION_COUNT_KEY,
currentConfiguration.getMergeMinRegionCount(), mergeMinRegionCount);
logConfigurationUpdated(MERGE_MIN_REGION_AGE_DAYS_KEY,
@@ -573,6 +678,10 @@ public boolean isMergeEnabled() {
return mergeEnabled;
}
+ public boolean isSkipBrokenChain() {
+ return skipBrokenChain;
+ }
+
public int getMergeMinRegionCount() {
return mergeMinRegionCount;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java
index 82d35e208d07..2443a78e3dc6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java
@@ -78,18 +78,21 @@ public void before() {
public void test() {
assertTrue(normalizer.isMergeEnabled());
assertEquals(3, normalizer.getMergeMinRegionCount());
+ assertTrue(normalizer.isSkipBrokenChain());
assertEquals(1_000_000L, parseConfiguredRateLimit(worker.getRateLimiter()));
final Configuration newConf = new Configuration(conf);
// configs on SimpleRegionNormalizer
newConf.setBoolean("hbase.normalizer.merge.enabled", false);
newConf.setInt("hbase.normalizer.merge.min.region.count", 100);
+ newConf.setBoolean("hbase.normalizer.skip.broken.chain", false);
// config on RegionNormalizerWorker
newConf.set("hbase.normalizer.throughput.max_bytes_per_sec", "12g");
configurationManager.notifyAllObservers(newConf);
assertFalse(normalizer.isMergeEnabled());
assertEquals(100, normalizer.getMergeMinRegionCount());
+ assertFalse(normalizer.isSkipBrokenChain());
assertEquals(12_884L, parseConfiguredRateLimit(worker.getRateLimiter()));
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index f474d795aa8b..390ee99cd2da 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -26,12 +26,14 @@
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
+import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SKIP_BROKEN_CHAIN_KEY;
import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
@@ -55,6 +57,7 @@
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.RegionMetrics;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Size;
@@ -222,6 +225,53 @@ public void testSplitOfLargeRegion() {
contains(new SplitNormalizationPlan(regionInfos.get(3), 30)));
}
+ @Test
+ public void testNoNormalizationWhenChainHasHole() {
+ final TableName tableName = this.tableName;
+ // Region chain with a hole between 'c' and 'd': [, b) [b, c) [d, )
+ final List regionInfos = new ArrayList<>();
+ regionInfos.add(createRegionInfo(tableName, HConstants.EMPTY_START_ROW, Bytes.toBytes("b")));
+ regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("b"), Bytes.toBytes("c")));
+ regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("d"), HConstants.EMPTY_END_ROW));
+ final Map regionSizes = createRegionSizesMap(regionInfos, 1, 1, 30);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+
+ // A hole in the region chain must cause the whole table to be skipped.
+ assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
+ }
+
+ @Test
+ public void testNoNormalizationWhenChainHasOverlap() {
+ final TableName tableName = this.tableName;
+ // Region chain with an overlap: [, d) overlaps [b, f), then [f, )
+ final List regionInfos = new ArrayList<>();
+ regionInfos.add(createRegionInfo(tableName, HConstants.EMPTY_START_ROW, Bytes.toBytes("d")));
+ regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("b"), Bytes.toBytes("f")));
+ regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("f"), HConstants.EMPTY_END_ROW));
+ final Map regionSizes = createRegionSizesMap(regionInfos, 5, 30, 5);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+
+ // An overlap in the region chain must cause the whole table to be skipped.
+ assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
+ }
+
+ @Test
+ public void testBrokenChainGuardCanBeDisabled() {
+ conf.setBoolean(SKIP_BROKEN_CHAIN_KEY, false);
+ final TableName tableName = this.tableName;
+ // Same hole as testNoNormalizationWhenChainHasHole.
+ final List regionInfos = new ArrayList<>();
+ regionInfos.add(createRegionInfo(tableName, HConstants.EMPTY_START_ROW, Bytes.toBytes("b")));
+ regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("b"), Bytes.toBytes("c")));
+ regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("d"), HConstants.EMPTY_END_ROW));
+ final Map regionSizes = createRegionSizesMap(regionInfos, 1, 1, 30);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+
+ // With the guard disabled, the oversized region is still split despite the hole.
+ assertThat(normalizer.computePlansForTable(tableDescriptor),
+ hasItem(instanceOf(SplitNormalizationPlan.class)));
+ }
+
@Test
public void testWithTargetRegionSize() throws Exception {
final TableName tableName = this.tableName;