diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index e921fd499427..4066a56f9635 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -664,6 +664,14 @@ possible configurations would overwhelm and obscure the important. true Whether to merge a region as part of normalization. + + hbase.normalizer.skip.broken.chain + true + Whether the normalizer should skip a table whose region chain is broken + (contains a hole, an overlap, or a degenerate region). Normalizing (in particular + splitting) such a table can increase the number of holes/overlaps and complicate the + eventual repair by the CatalogJanitor/HBCK. Set to false to normalize regardless. + hbase.normalizer.merge.min.region.count 3 diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index 934b8de1b94c..e673f675e085 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.assignment.RegionStates; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -68,6 +69,8 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver static final boolean DEFAULT_SPLIT_ENABLED = true; static final String MERGE_ENABLED_KEY = "hbase.normalizer.merge.enabled"; static final boolean DEFAULT_MERGE_ENABLED = true; + static final String SKIP_BROKEN_CHAIN_KEY = "hbase.normalizer.skip.broken.chain"; + static final boolean DEFAULT_SKIP_BROKEN_CHAIN = true; /** * @deprecated since 2.5.0 and will be removed in 4.0.0. Use * {@link SimpleRegionNormalizer#MERGE_MIN_REGION_COUNT_KEY} instead. @@ -178,6 +181,13 @@ public boolean isMergeEnabled() { return normalizerConfiguration.isMergeEnabled(); } + /** + * Return this instance's configured value for {@value #SKIP_BROKEN_CHAIN_KEY}. + */ + public boolean isSkipBrokenChain() { + return normalizerConfiguration.isSkipBrokenChain(); + } + /** * Return this instance's configured value for {@value #MERGE_MIN_REGION_COUNT_KEY}. */ @@ -231,6 +241,23 @@ public List computePlansForTable(final TableDescriptor tableD return Collections.emptyList(); } + if (normalizerConfiguration.isSkipBrokenChain()) { + final String brokenChainReason = findBrokenRegionChain(ctx.getTableRegions()); + if (brokenChainReason != null) { + // Normalizing (in particular splitting) a table whose region chain already has holes or + // overlaps can increase the number of holes/overlaps and complicate the eventual repair by + // the CatalogJanitor/HBCK. Stand down until the chain is intact. Note the region list comes + // from the in-memory RegionStates, which can briefly lag hbase:meta while regions are in + // transition, so this may be transient and self-correct on the next run. + LOG.warn( + "Skipping normalization of table {} because its region chain appears to be broken: {}." + + " If this persists, run the CatalogJanitor/HBCK to fix holes and overlaps. Set {}=" + + "false to normalize regardless.", + table, brokenChainReason, SKIP_BROKEN_CHAIN_KEY); + return Collections.emptyList(); + } + } + LOG.debug("Computing normalization plan for table: {}, number of regions: {}", table, ctx.getTableRegions().size()); @@ -261,6 +288,79 @@ public List computePlansForTable(final TableDescriptor tableD return plans; } + /** + * Walk the region chain of a table (already sorted by {@link RegionInfo#COMPARATOR}) and detect + * the first structural inconsistency: a degenerate region (startKey > endKey), an overlap, or a + * hole between adjacent regions. Non-default replicas and split-parent regions are ignored, as + * they legitimately do not participate in the table's linear key-space chain. Boundary + * (first/last) completeness is intentionally not checked here; only gaps/overlaps between the + * regions that are present are considered. + *

+ * The input is derived from the in-memory {@link RegionStates}, which can briefly lag + * {@code hbase:meta} while regions are in transition, so a positive result may be transient. + * @param tableRegions regions of a single table, sorted by {@link RegionInfo#COMPARATOR} + * @return a human-readable description of the first problem found, or {@code null} if the chain is + * intact. + */ + private static String findBrokenRegionChain(final List tableRegions) { + RegionInfo previous = null; + RegionInfo highestEndKeyRegionInfo = null; + for (final RegionInfo current : tableRegions) { + if (current.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID || current.isSplitParent()) { + continue; + } + if (current.isDegenerate()) { + return "degenerate region " + current.getShortNameToLog() + " (startKey > endKey)"; + } + if (previous != null) { + if (!previous.isNext(current)) { + if (previous.isOverlap(current)) { + return "overlap between " + previous.getShortNameToLog() + " and " + + current.getShortNameToLog(); + } else if (current.isOverlap(highestEndKeyRegionInfo)) { + // A region seen a few rows back may still overlap the current region. + return "overlap between " + highestEndKeyRegionInfo.getShortNameToLog() + " and " + + current.getShortNameToLog(); + } else if (!highestEndKeyRegionInfo.isNext(current)) { + return "hole between " + highestEndKeyRegionInfo.getShortNameToLog() + " and " + + current.getShortNameToLog(); + } + } else if (current.isOverlap(highestEndKeyRegionInfo)) { + // Adjacent to the immediate predecessor, but overlaps an earlier, wider region. + return "overlap between " + highestEndKeyRegionInfo.getShortNameToLog() + " and " + + current.getShortNameToLog(); + } + } + previous = current; + highestEndKeyRegionInfo = getRegionInfoWithLargestEndKey(highestEndKeyRegionInfo, current); + } + return null; + } + + /** + * Returns whichever of {@code a} or {@code b} has the larger end key, treating an empty end key + * (the last region of a table) as the largest. Mirrors the running "highest end key seen" tracking + * used by the CatalogJanitor consistency check. + */ + private static RegionInfo getRegionInfoWithLargestEndKey(RegionInfo a, RegionInfo b) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + if (!a.getTable().equals(b.getTable())) { + return b; + } + if (a.isLast()) { + return a; + } + if (b.isLast()) { + return b; + } + return Bytes.compareTo(a.getEndKey(), b.getEndKey()) >= 0 ? a : b; + } + /** Returns size of region in MB and if region is not found than -1 */ private long getRegionSizeMB(RegionInfo hri) { ServerName sn = @@ -518,6 +618,7 @@ private static final class NormalizerConfiguration { private final Configuration conf; private final boolean splitEnabled; private final boolean mergeEnabled; + private final boolean skipBrokenChain; private final int mergeMinRegionCount; private final Period mergeMinRegionAge; private final long mergeMinRegionSizeMb; @@ -528,6 +629,7 @@ private NormalizerConfiguration() { conf = null; splitEnabled = DEFAULT_SPLIT_ENABLED; mergeEnabled = DEFAULT_MERGE_ENABLED; + skipBrokenChain = DEFAULT_SKIP_BROKEN_CHAIN; mergeMinRegionCount = DEFAULT_MERGE_MIN_REGION_COUNT; mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS); mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB; @@ -540,6 +642,7 @@ private NormalizerConfiguration(final Configuration conf, this.conf = conf; splitEnabled = conf.getBoolean(SPLIT_ENABLED_KEY, DEFAULT_SPLIT_ENABLED); mergeEnabled = conf.getBoolean(MERGE_ENABLED_KEY, DEFAULT_MERGE_ENABLED); + skipBrokenChain = conf.getBoolean(SKIP_BROKEN_CHAIN_KEY, DEFAULT_SKIP_BROKEN_CHAIN); mergeMinRegionCount = parseMergeMinRegionCount(conf); mergeMinRegionAge = parseMergeMinRegionAge(conf); mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf); @@ -550,6 +653,8 @@ private NormalizerConfiguration(final Configuration conf, splitEnabled); logConfigurationUpdated(MERGE_ENABLED_KEY, currentConfiguration.isMergeEnabled(), mergeEnabled); + logConfigurationUpdated(SKIP_BROKEN_CHAIN_KEY, currentConfiguration.isSkipBrokenChain(), + skipBrokenChain); logConfigurationUpdated(MERGE_MIN_REGION_COUNT_KEY, currentConfiguration.getMergeMinRegionCount(), mergeMinRegionCount); logConfigurationUpdated(MERGE_MIN_REGION_AGE_DAYS_KEY, @@ -573,6 +678,10 @@ public boolean isMergeEnabled() { return mergeEnabled; } + public boolean isSkipBrokenChain() { + return skipBrokenChain; + } + public int getMergeMinRegionCount() { return mergeMinRegionCount; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java index 82d35e208d07..2443a78e3dc6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerManagerConfigurationObserver.java @@ -78,18 +78,21 @@ public void before() { public void test() { assertTrue(normalizer.isMergeEnabled()); assertEquals(3, normalizer.getMergeMinRegionCount()); + assertTrue(normalizer.isSkipBrokenChain()); assertEquals(1_000_000L, parseConfiguredRateLimit(worker.getRateLimiter())); final Configuration newConf = new Configuration(conf); // configs on SimpleRegionNormalizer newConf.setBoolean("hbase.normalizer.merge.enabled", false); newConf.setInt("hbase.normalizer.merge.min.region.count", 100); + newConf.setBoolean("hbase.normalizer.skip.broken.chain", false); // config on RegionNormalizerWorker newConf.set("hbase.normalizer.throughput.max_bytes_per_sec", "12g"); configurationManager.notifyAllObservers(newConf); assertFalse(normalizer.isMergeEnabled()); assertEquals(100, normalizer.getMergeMinRegionCount()); + assertFalse(normalizer.isSkipBrokenChain()); assertEquals(12_884L, parseConfiguredRateLimit(worker.getRateLimiter())); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index f474d795aa8b..390ee99cd2da 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -26,12 +26,14 @@ import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY; +import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SKIP_BROKEN_CHAIN_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -55,6 +57,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Size; @@ -222,6 +225,53 @@ public void testSplitOfLargeRegion() { contains(new SplitNormalizationPlan(regionInfos.get(3), 30))); } + @Test + public void testNoNormalizationWhenChainHasHole() { + final TableName tableName = this.tableName; + // Region chain with a hole between 'c' and 'd': [, b) [b, c) [d, ) + final List regionInfos = new ArrayList<>(); + regionInfos.add(createRegionInfo(tableName, HConstants.EMPTY_START_ROW, Bytes.toBytes("b"))); + regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("b"), Bytes.toBytes("c"))); + regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("d"), HConstants.EMPTY_END_ROW)); + final Map regionSizes = createRegionSizesMap(regionInfos, 1, 1, 30); + setupMocksForNormalizer(regionSizes, regionInfos); + + // A hole in the region chain must cause the whole table to be skipped. + assertThat(normalizer.computePlansForTable(tableDescriptor), empty()); + } + + @Test + public void testNoNormalizationWhenChainHasOverlap() { + final TableName tableName = this.tableName; + // Region chain with an overlap: [, d) overlaps [b, f), then [f, ) + final List regionInfos = new ArrayList<>(); + regionInfos.add(createRegionInfo(tableName, HConstants.EMPTY_START_ROW, Bytes.toBytes("d"))); + regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("b"), Bytes.toBytes("f"))); + regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("f"), HConstants.EMPTY_END_ROW)); + final Map regionSizes = createRegionSizesMap(regionInfos, 5, 30, 5); + setupMocksForNormalizer(regionSizes, regionInfos); + + // An overlap in the region chain must cause the whole table to be skipped. + assertThat(normalizer.computePlansForTable(tableDescriptor), empty()); + } + + @Test + public void testBrokenChainGuardCanBeDisabled() { + conf.setBoolean(SKIP_BROKEN_CHAIN_KEY, false); + final TableName tableName = this.tableName; + // Same hole as testNoNormalizationWhenChainHasHole. + final List regionInfos = new ArrayList<>(); + regionInfos.add(createRegionInfo(tableName, HConstants.EMPTY_START_ROW, Bytes.toBytes("b"))); + regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("b"), Bytes.toBytes("c"))); + regionInfos.add(createRegionInfo(tableName, Bytes.toBytes("d"), HConstants.EMPTY_END_ROW)); + final Map regionSizes = createRegionSizesMap(regionInfos, 1, 1, 30); + setupMocksForNormalizer(regionSizes, regionInfos); + + // With the guard disabled, the oversized region is still split despite the hole. + assertThat(normalizer.computePlansForTable(tableDescriptor), + hasItem(instanceOf(SplitNormalizationPlan.class))); + } + @Test public void testWithTargetRegionSize() throws Exception { final TableName tableName = this.tableName;