From e1e2e89aa9e79112d3d300518644845d57441fe6 Mon Sep 17 00:00:00 2001 From: "linguantian.lgt" Date: Wed, 5 Aug 2026 15:48:17 +0800 Subject: [PATCH 1/4] perf: reduce HBase read-path CPU overhead Optimize point Get and closestRowBefore, consume compact KQTV batches, avoid TableGroup intermediate copies, and use lightweight result cells for Get and Scan. --- pom.xml | 2 +- .../com/alipay/oceanbase/hbase/OHTable.java | 305 ++++++++--- .../hbase/constants/OHConstants.java | 16 + .../hbase/result/ClientStreamScanner.java | 103 +++- .../hbase/result/OHBaseResultCell.java | 219 ++++++++ .../oceanbase/hbase/util/OHBaseFuncUtils.java | 97 +++- .../hbase/OHTableGetMaxRowResultTest.java | 475 ++++++++++++++++++ ...HTableLightweightResultCellConfigTest.java | 91 ++++ .../ClientStreamScannerCompactResultTest.java | 148 ++++++ .../hbase/result/OHBaseResultCellTest.java | 127 +++++ 10 files changed, 1471 insertions(+), 112 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/result/OHBaseResultCell.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/result/OHBaseResultCellTest.java diff --git a/pom.xml b/pom.xml index b9b722d1..9ec39ef8 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 2.4.0 + 2.4.1-SNAPSHOT diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index dcb18e25..018b89a7 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -24,6 +24,7 @@ import com.alipay.oceanbase.hbase.metrics.MetricsImporter; import com.alipay.oceanbase.hbase.metrics.OHMetrics; import com.alipay.oceanbase.hbase.result.ClientStreamScanner; +import com.alipay.oceanbase.hbase.result.OHBaseResultCell; import com.alipay.oceanbase.hbase.util.*; import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.ObTableClient; @@ -203,6 +204,16 @@ public class OHTable implements HTableInterface { */ private final boolean hotKeyGetOptimizeEnableGlobal; + /** + * whether point read results use field-backed lightweight cells. + */ + private final boolean getLightweightResultCellEnabled; + + /** + * whether Scan results use field-backed lightweight cells. + */ + private final boolean scanLightweightResultCellEnabled; + /** * whether test load is enabled. * Cached at construction time to avoid repeated configuration lookups. @@ -253,6 +264,12 @@ public OHTable(Configuration configuration, String tableName) throws IOException } this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); + this.getLightweightResultCellEnabled = configuration.getBoolean( + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED, + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + this.scanLightweightResultCellEnabled = configuration.getBoolean( + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED, + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -315,6 +332,12 @@ public OHTable(Configuration configuration, final byte[] tableName, } this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); + this.getLightweightResultCellEnabled = configuration.getBoolean( + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED, + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + this.scanLightweightResultCellEnabled = configuration.getBoolean( + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED, + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -337,6 +360,19 @@ public OHTable(Configuration configuration, final byte[] tableName, @InterfaceAudience.Private public OHTable(final byte[] tableName, final ObTableClient obTableClient, final ExecutorService executePool) { + this(tableName, obTableClient, executePool, + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + } + + OHTable(final byte[] tableName, final ObTableClient obTableClient, + final ExecutorService executePool, boolean getLightweightResultCellEnabled) { + this(tableName, obTableClient, executePool, getLightweightResultCellEnabled, + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + } + + OHTable(final byte[] tableName, final ObTableClient obTableClient, + final ExecutorService executePool, boolean getLightweightResultCellEnabled, + boolean scanLightweightResultCellEnabled) { checkArgument(tableName != null, "tableNameString is blank."); checkArgument(executePool != null && !executePool.isShutdown(), "executePool is null or executePool is shutdown"); @@ -350,6 +386,8 @@ public OHTable(final byte[] tableName, final ObTableClient obTableClient, this.metrics = null; this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); + this.getLightweightResultCellEnabled = getLightweightResultCellEnabled; + this.scanLightweightResultCellEnabled = scanLightweightResultCellEnabled; this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -400,6 +438,12 @@ public OHTable(TableName tableName, Connection connection, } this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); + this.getLightweightResultCellEnabled = configuration.getBoolean( + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED, + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + this.scanLightweightResultCellEnabled = configuration.getBoolean( + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED, + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -865,21 +909,16 @@ private List generateGetResult(ObTableSingleOpResult getResult) throws IOE while (valueIdx < propertiesValues.size()) { // values in propertiesValues like: [ K, Q, T, V, K, Q, T, V ... ] // we need to retrieve K Q T V and construct them to cells: [ cell_0, cell_1, ... ] - byte[][] familyAndQualifier = new byte[2][]; try { - // split family and qualifier - familyAndQualifier = OHBaseFuncUtils - .extractFamilyFromQualifier((byte[]) propertiesValues.get(valueIdx + 1).getValue()); + byte[] rowKey = (byte[]) propertiesValues.get(valueIdx).getValue(); + byte[] familyQualifier = (byte[]) propertiesValues.get(valueIdx + 1).getValue(); + long timestamp = (Long) propertiesValues.get(valueIdx + 2).getValue(); + byte[] value = (byte[]) propertiesValues.get(valueIdx + 3).getValue(); + addResultCell(cells, rowKey, familyQualifier, timestamp, value, true, + HConstants.EMPTY_BYTE_ARRAY); } catch (Exception e) { throw new IOException(e); } - KeyValue kv = new KeyValue((byte[]) propertiesValues.get(valueIdx).getValue(),//K - familyAndQualifier[0], // family - familyAndQualifier[1], // qualifiermat - (Long) propertiesValues.get(valueIdx + 2).getValue(), // T - (byte[]) propertiesValues.get(valueIdx + 3).getValue()// V - ); - cells.add(kv); valueIdx += 4; } return cells; @@ -968,41 +1007,171 @@ public static int compareByteArray(byte[] bt1, byte[] bt2) { return bt1.length - bt2.length; } - private void getMaxRowFromResult(AbstractQueryStreamResult clientQueryStreamResult, - List keyValueList, boolean isTableGroup, + private void addRowToResultCells(List cells, List row, byte[] rowKey, + boolean isTableGroup, byte[] family) throws Exception { + byte[] qualifier = (byte[]) row.get(1).getValue(); + long timestamp = (Long) row.get(2).getValue(); + byte[] value = (byte[]) row.get(3).getValue(); + addResultCell(cells, rowKey, qualifier, timestamp, value, isTableGroup, family); + } + + private void addResultCell(List cells, byte[] rowKey, byte[] qualifier, long timestamp, + byte[] value, boolean isTableGroup, byte[] family) throws Exception { + if (getLightweightResultCellEnabled) { + if (isTableGroup) { + cells.add(OHBaseResultCell.createTableGroup(rowKey, qualifier, timestamp, value)); + } else { + cells.add(OHBaseResultCell.create(rowKey, family, qualifier, timestamp, value)); + } + return; + } + if (isTableGroup) { + cells + .add(OHBaseFuncUtils.createTableGroupKeyValue(rowKey, qualifier, timestamp, value)); + } else { + cells.add(new KeyValue(rowKey, family, qualifier, timestamp, value)); + } + } + + /** + * A point Get uses an exact row range, so all returned cells must belong to the requested row. + * Validate the invariant once and reuse the returned rowkey while assembling result cells. + */ + private boolean fillPointGetFromResult(AbstractQueryStreamResult clientQueryStreamResult, + List cells, boolean isTableGroup, byte[] family, + byte[] expectedRowKey, boolean checkExistenceOnly) + throws Exception { + byte[] canonicalRowKey = null; + while (clientQueryStreamResult.next()) { + if (checkExistenceOnly) { + // An existence-only response contains an empty row when the requested row exists. + return true; + } + + if (clientQueryStreamResult.isCurrentHBaseCell()) { + ObHBaseCellRow hbaseRow = clientQueryStreamResult.drainCurrentHBaseRow(); + byte[] actualRowKey = hbaseRow.getRowKey(); + if (canonicalRowKey == null) { + if (!Bytes.equals(expectedRowKey, actualRowKey)) { + throw new ObTableUnexpectedException( + "point Get returned an unexpected rowkey, expected length=" + + expectedRowKey.length + ", actual length=" + + actualRowKey.length); + } + canonicalRowKey = actualRowKey; + } + addCompactRowToResultCells(cells, hbaseRow, canonicalRowKey, isTableGroup, + family); + continue; + } + + byte[] actualRowKey; + byte[] qualifier; + long timestamp; + byte[] value; + List row = clientQueryStreamResult.getRow(); + actualRowKey = (byte[]) row.get(0).getValue(); + qualifier = (byte[]) row.get(1).getValue(); + timestamp = (Long) row.get(2).getValue(); + value = (byte[]) row.get(3).getValue(); + if (canonicalRowKey == null) { + if (!Bytes.equals(expectedRowKey, actualRowKey)) { + throw new ObTableUnexpectedException( + "point Get returned an unexpected rowkey, expected length=" + + expectedRowKey.length + ", actual length=" + + actualRowKey.length); + } + canonicalRowKey = actualRowKey; + } + addResultCell(cells, canonicalRowKey, qualifier, timestamp, value, isTableGroup, family); + } + return canonicalRowKey != null; + } + + private void addCompactRowToResultCells(List cells, ObHBaseCellRow hbaseRow, + byte[] canonicalRowKey, boolean isTableGroup, + byte[] family) throws Exception { + for (int sliceIndex = 0; sliceIndex < hbaseRow.getSliceCount(); sliceIndex++) { + ObHBaseCellBatch batch = hbaseRow.getBatch(sliceIndex); + int toIndex = hbaseRow.getToIndex(sliceIndex); + for (int index = hbaseRow.getFromIndex(sliceIndex); index < toIndex; index++) { + addResultCell(cells, canonicalRowKey, batch.getQualifier(index), + batch.getTimestamp(index), batch.getValue(index), isTableGroup, family); + } + } + } + + /** + * closestRowBefore may collect candidates from multiple partitions. Keep only cells belonging + * to the greatest returned rowkey without cloning the rowkey from a temporary result cell. + */ + private boolean getMaxRowFromResult(AbstractQueryStreamResult clientQueryStreamResult, + List cells, boolean isTableGroup, byte[] family, boolean checkExistenceOnly) throws Exception { - byte[][] familyAndQualifier = new byte[2][]; - KeyValue kv = null; + byte[] currentMaxRowKey = null; while (clientQueryStreamResult.next()) { if (checkExistenceOnly) { - // Currently, checkExistOnly is set, and if the row exists, it returns an empty row. - keyValueList.add(new KeyValue()); - return; + // An existence-only response contains an empty row when a candidate row exists. + return true; + } + + byte[] rowKey; + byte[] qualifier; + long timestamp; + byte[] value; + if (clientQueryStreamResult.isCurrentHBaseCell()) { + ObHBaseCellBatch batch = clientQueryStreamResult.getCurrentHBaseCellBatch(); + int index = clientQueryStreamResult.getCurrentHBaseCellIndex(); + rowKey = batch.getRowKey(index); + qualifier = batch.getQualifier(index); + timestamp = batch.getTimestamp(index); + value = batch.getValue(index); } else { List row = clientQueryStreamResult.getRow(); - if (kv == null - || compareByteArray(kv.getRow(), (byte[]) row.get(0).getValue()) <= 0) { - if (kv != null - && compareByteArray(kv.getRow(), (byte[]) row.get(0).getValue()) != 0) { - keyValueList.clear(); - } - if (isTableGroup) { - // split family and qualifier - familyAndQualifier = OHBaseFuncUtils - .extractFamilyFromQualifier((byte[]) row.get(1).getValue()); - } else { - familyAndQualifier[0] = family; - familyAndQualifier[1] = (byte[]) row.get(1).getValue(); - } - kv = new KeyValue((byte[]) row.get(0).getValue(),//K - familyAndQualifier[0], // family - familyAndQualifier[1], // qualifiermat - (Long) row.get(2).getValue(), // T - (byte[]) row.get(3).getValue() // V - ); - keyValueList.add(kv); + rowKey = (byte[]) row.get(0).getValue(); + qualifier = (byte[]) row.get(1).getValue(); + timestamp = (Long) row.get(2).getValue(); + value = (byte[]) row.get(3).getValue(); + } + if (currentMaxRowKey != null) { + int rowComparison = compareByteArray(currentMaxRowKey, rowKey); + if (rowComparison > 0) { + continue; + } + if (rowComparison < 0) { + cells.clear(); + currentMaxRowKey = rowKey; } + } else { + currentMaxRowKey = rowKey; + } + addResultCell(cells, rowKey, qualifier, timestamp, value, isTableGroup, family); + } + return currentMaxRowKey != null; + } + + @SuppressWarnings("unchecked") + private Result createGetResult(List cells) { + if (getLightweightResultCellEnabled) { + return Result.create(cells); + } + return new Result((List) (List) cells); + } + + private void addQueryResultToKeyValueList(ObTableQueryResult queryResult, + List keyValues, byte[] family) + throws Exception { + ObHBaseCellBatch batch = queryResult.getHBaseCellBatch(); + if (batch != null) { + for (int i = 0; i < batch.size(); i++) { + keyValues.add(new KeyValue(batch.getRowKey(i), family, batch.getQualifier(i), batch + .getTimestamp(i), batch.getValue(i))); } + return; + } + for (List row : queryResult.getPropertiesRows()) { + keyValues.add(new KeyValue((byte[]) row.get(0).getValue(), family, (byte[]) row.get(1) + .getValue(), (Long) row.get(2).getValue(), (byte[]) row.get(3).getValue())); } } @@ -1073,8 +1242,9 @@ private Result innerGetImpl(final Get get, OHOperationType opType) throws IOExce ServerCallable serverCallable = new ServerCallable(configuration, obTableClient, tableNameString, get.getRow(), get.getRow(), operationTimeout) { public Result call() throws IOException { - List keyValueList = new ArrayList<>(); + List cells = new ArrayList<>(); byte[] family = new byte[] {}; + boolean exists = false; ObTableQuery obTableQuery; try { if (get.getFamilyMap().keySet().isEmpty() @@ -1093,7 +1263,12 @@ public Result call() throws IOException { ObTableClientQueryAsyncStreamResult clientQueryStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); - getMaxRowFromResult(clientQueryStreamResult, keyValueList, true, family, get.isCheckExistenceOnly()); + if (get.isClosestRowBefore()) { + exists = getMaxRowFromResult(clientQueryStreamResult, cells, true, family, get.isCheckExistenceOnly()); + } else { + exists = fillPointGetFromResult(clientQueryStreamResult, cells, + true, family, get.getRow(), get.isCheckExistenceOnly()); + } } else { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { @@ -1114,8 +1289,14 @@ public Result call() throws IOException { getTargetTableName(tableNameString, Bytes.toString(family)), opType, isWeakRead(get)); ObTableClientQueryStreamResult clientQueryStreamResult = (ObTableClientQueryStreamResult) obTableClient .execute(request); - getMaxRowFromResult(clientQueryStreamResult, keyValueList, false, + if (get.isClosestRowBefore()) { + exists = getMaxRowFromResult(clientQueryStreamResult, cells, false, family, get.isCheckExistenceOnly()); + } else { + exists = fillPointGetFromResult(clientQueryStreamResult, cells, + false, family, get.getRow(), + get.isCheckExistenceOnly()); + } } } } catch (Exception e) { @@ -1123,11 +1304,11 @@ public Result call() throws IOException { + Bytes.toString(family) + " error.", e); } if (get.isCheckExistenceOnly()) { - return Result.create(null, !keyValueList.isEmpty()); + return Result.create(null, exists); } - // sort keyValues - OHBaseFuncUtils.sortHBaseResult(keyValueList); - return new Result(keyValueList); + // sort result cells + OHBaseFuncUtils.sortHBaseResult(cells); + return createGetResult(cells); } }; return executeServerCallable(serverCallable); @@ -1217,7 +1398,8 @@ public ResultScanner call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); return new ClientStreamScanner(clientQueryAsyncStreamResult, - tableNameString, family, true, metrics); + tableNameString, family, true, metrics, + scanLightweightResultCellEnabled); } else { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { @@ -1243,7 +1425,8 @@ public ResultScanner call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); return new ClientStreamScanner(clientQueryAsyncStreamResult, - tableNameString, family, false, metrics); + tableNameString, family, false, metrics, + scanLightweightResultCellEnabled); } } } catch (Exception e) { @@ -1303,7 +1486,8 @@ public List call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); ClientStreamScanner clientScanner = new ClientStreamScanner( - clientQueryAsyncStreamResult, tableNameString, family, true, metrics); + clientQueryAsyncStreamResult, tableNameString, family, true, metrics, + scanLightweightResultCellEnabled); resultScanners.add(clientScanner); } return resultScanners; @@ -1328,7 +1512,7 @@ public List call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); ClientStreamScanner clientScanner = new ClientStreamScanner( - clientQueryAsyncStreamResult, tableNameString, family, false, metrics); + clientQueryAsyncStreamResult, tableNameString, family, false, metrics, scanLightweightResultCellEnabled); resultScanners.add(clientScanner); } return resultScanners; @@ -1627,15 +1811,7 @@ Result execute() throws IOException { } ObTableQueryResult queryResult = result.getAffectedEntity(); List keyValues = new ArrayList(); - for (List row : queryResult.getPropertiesRows()) { - byte[] k = (byte[]) row.get(0).getValue(); - byte[] q = (byte[]) row.get(1).getValue(); - long t = (Long) row.get(2).getValue(); - byte[] v = (byte[]) row.get(3).getValue(); - KeyValue kv = new KeyValue(k, f, q, t, v); - - keyValues.add(kv); - } + addQueryResultToKeyValueList(queryResult, keyValues, f); return new Result(keyValues); } catch (Exception e) { throw new IOException("append table " + tableNameString + " error.", e); @@ -1684,14 +1860,7 @@ Result execute() throws IOException { } ObTableQueryResult queryResult = result.getAffectedEntity(); List keyValues = new ArrayList(); - for (List row : queryResult.getPropertiesRows()) { - byte[] k = (byte[]) row.get(0).getValue(); - byte[] q = (byte[]) row.get(1).getValue(); - long t = (Long) row.get(2).getValue(); - byte[] v = (byte[]) row.get(3).getValue(); - KeyValue kv = new KeyValue(k, f, q, t, v); - keyValues.add(kv); - } + addQueryResultToKeyValueList(queryResult, keyValues, f); return new Result(keyValues); } catch (Exception e) { throw new IOException("increment table " + tableNameString + " error.", e); @@ -2893,4 +3062,4 @@ public OHMetrics getMetrics() { return metrics; } -} \ No newline at end of file +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index 72fb5b8b..47bc6c60 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -167,6 +167,18 @@ public final class OHConstants { */ public static final String HBASE_HTABLE_USE_PUT_OPTIMIZATION = "hbase.htable.use.put.optimization"; + /** + * use to specify whether point read results use field-backed lightweight cells. + * Default is true (enabled). + */ + public static final String HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED = "hbase.htable.get.lightweight.result.cell.enabled"; + + /** + * use to specify whether Scan results use field-backed lightweight cells. + * Default is true (enabled). + */ + public static final String HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED = "hbase.htable.scan.lightweight.result.cell.enabled"; + /*-------------------------------------------------------------------------------------------------------------*/ /** @@ -196,4 +208,8 @@ public final class OHConstants { public static final boolean HBASE_HTABLE_USE_PUT_OPTIMIZATION_DEFAULT = true; + public static final boolean HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT = true; + + public static final boolean HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT = true; + } diff --git a/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java b/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java index 51e6e882..def6c644 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java +++ b/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java @@ -26,9 +26,12 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.OHOperationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellBatch; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellRow; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.AbstractClientScanner; import org.apache.hadoop.hbase.client.Result; @@ -37,6 +40,7 @@ import java.io.IOException; import java.util.*; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.LCD; @InterfaceAudience.Private @@ -57,24 +61,42 @@ public class ClientStreamScanner extends AbstractClientScanner { private boolean isTableGroup = false; + private final boolean lightweightResultCellEnabled; + private OHMetrics metrics; public ClientStreamScanner(ObTableClientQueryStreamResult streamResult, String tableName, byte[] family, boolean isTableGroup, OHMetrics metrics) { + this(streamResult, tableName, family, isTableGroup, metrics, + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + } + + public ClientStreamScanner(ObTableClientQueryStreamResult streamResult, String tableName, + byte[] family, boolean isTableGroup, OHMetrics metrics, + boolean lightweightResultCellEnabled) { this.streamResult = streamResult; this.tableName = tableName; this.family = family; this.isTableGroup = isTableGroup; this.metrics = metrics; + this.lightweightResultCellEnabled = lightweightResultCellEnabled; } public ClientStreamScanner(ObTableClientQueryAsyncStreamResult streamResult, String tableName, byte[] family, boolean isTableGroup, OHMetrics metrics) { + this(streamResult, tableName, family, isTableGroup, metrics, + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + } + + public ClientStreamScanner(ObTableClientQueryAsyncStreamResult streamResult, String tableName, + byte[] family, boolean isTableGroup, OHMetrics metrics, + boolean lightweightResultCellEnabled) { this.streamResult = streamResult; this.tableName = tableName; this.family = family; this.isTableGroup = isTableGroup; this.metrics = metrics; + this.lightweightResultCellEnabled = lightweightResultCellEnabled; } @Override @@ -83,14 +105,70 @@ public Result next() throws IOException { MetricsImporter importer = metrics == null ? null : new MetricsImporter(); try { checkStatus(); - List startRow; - if (streamResult.next()) { - startRow = streamResult.getRow(); - } else { + if (!streamResult.next()) { return null; } + if (streamResult.isCurrentHBaseCell()) { + return buildCompactResult(streamResult.drainCurrentHBaseRow()); + } + return buildLegacyResult(streamResult.getRow()); + } catch (Exception e) { + throw new IOException(String.format("get table %s stream next result error ", + streamResult.getTableName()), e); + } finally { + if (metrics != null) { + long duration = System.currentTimeMillis() - startTimeMs; + importer.setDuration(duration); + importer.setBatchSize(1); + metrics.update(new ObPair(OHOperationType.SCAN, + importer)); + } + } + } - byte[][] familyAndQualifier = new byte[2][]; + private Result buildCompactResult(ObHBaseCellRow hbaseRow) { + List cells = new ArrayList(hbaseRow.getCellCount()); + byte[] rowKey = hbaseRow.getRowKey(); + for (int sliceIndex = 0; sliceIndex < hbaseRow.getSliceCount(); sliceIndex++) { + ObHBaseCellBatch batch = hbaseRow.getBatch(sliceIndex); + int toIndex = hbaseRow.getToIndex(sliceIndex); + for (int index = hbaseRow.getFromIndex(sliceIndex); index < toIndex; index++) { + addCompactResultCell(cells, rowKey, batch.getQualifier(index), + batch.getTimestamp(index), batch.getValue(index)); + } + } + OHBaseFuncUtils.sortHBaseResult(cells); + return createCompactResult(cells); + } + + private void addCompactResultCell(List cells, byte[] rowKey, byte[] qualifier, + long timestamp, byte[] value) { + if (lightweightResultCellEnabled) { + if (isTableGroup) { + cells.add(OHBaseResultCell.createTableGroup(rowKey, qualifier, timestamp, value)); + } else { + cells.add(OHBaseResultCell.create(rowKey, family, qualifier, timestamp, value)); + } + return; + } + if (isTableGroup) { + cells + .add(OHBaseFuncUtils.createTableGroupKeyValue(rowKey, qualifier, timestamp, value)); + } else { + cells.add(new KeyValue(rowKey, family, qualifier, timestamp, value)); + } + } + + @SuppressWarnings("unchecked") + private Result createCompactResult(List cells) { + if (lightweightResultCellEnabled) { + return Result.create(cells); + } + return new Result((List) (List) cells); + } + + private Result buildLegacyResult(List startRow) throws Exception { + byte[][] familyAndQualifier = new byte[2][]; if (this.isTableGroup) { // split family and qualifier familyAndQualifier = OHBaseFuncUtils.extractFamilyFromQualifier((byte[]) startRow @@ -130,21 +208,8 @@ public Result next() throws IOException { break; } } - // sort keyValues - OHBaseFuncUtils.sortHBaseResult(keyValues); + OHBaseFuncUtils.sortHBaseResult(keyValues); return new Result(keyValues); - } catch (Exception e) { - throw new IOException(String.format("get table %s stream next result error ", - streamResult.getTableName()), e); - } finally { - if (metrics != null) { - long duration = System.currentTimeMillis() - startTimeMs; - importer.setDuration(duration); - importer.setBatchSize(1); - metrics.update(new ObPair(OHOperationType.SCAN, - importer)); - } - } } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/result/OHBaseResultCell.java b/src/main/java/com/alipay/oceanbase/hbase/result/OHBaseResultCell.java new file mode 100644 index 00000000..d4c29f57 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/result/OHBaseResultCell.java @@ -0,0 +1,219 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.result; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.util.Objects; + +/** + * Immutable read-result cell backed by the decoded HBase field arrays. + */ +@InterfaceAudience.Private +public final class OHBaseResultCell implements Cell { + + private static final byte PUT_TYPE = KeyValue.Type.Put.getCode(); + private static final byte[] EMPTY = HConstants.EMPTY_BYTE_ARRAY; + + private final byte[] row; + private final byte[] familyArray; + private final int familyOffset; + private final int familyLength; + private final byte[] qualifierArray; + private final int qualifierOffset; + private final int qualifierLength; + private final long timestamp; + private final byte[] value; + + public static OHBaseResultCell create(byte[] row, byte[] family, byte[] qualifier, + long timestamp, byte[] value) { + return new OHBaseResultCell(row, family, 0, length(family), qualifier, 0, + length(qualifier), timestamp, value); + } + + public static OHBaseResultCell createTableGroup(byte[] row, byte[] familyQualifier, + long timestamp, byte[] value) { + Objects.requireNonNull(familyQualifier, "familyQualifier is null"); + int familyLength = findFamilyDelimiter(familyQualifier); + int qualifierOffset = familyLength + 1; + return new OHBaseResultCell(row, familyQualifier, 0, familyLength, familyQualifier, + qualifierOffset, familyQualifier.length - qualifierOffset, timestamp, value); + } + + private OHBaseResultCell(byte[] row, byte[] familyArray, int familyOffset, int familyLength, + byte[] qualifierArray, int qualifierOffset, int qualifierLength, + long timestamp, byte[] value) { + this.row = Objects.requireNonNull(row, "row is null"); + this.familyArray = Objects.requireNonNull(familyArray, "family is null"); + this.qualifierArray = Objects.requireNonNull(qualifierArray, "qualifier is null"); + this.value = value == null ? EMPTY : value; + checkRange(familyArray, familyOffset, familyLength, "family"); + checkRange(qualifierArray, qualifierOffset, qualifierLength, "qualifier"); + if (row.length > Short.MAX_VALUE) { + throw new IllegalArgumentException("row length " + row.length + " exceeds " + + Short.MAX_VALUE); + } + if (familyLength > Byte.MAX_VALUE) { + throw new IllegalArgumentException("family length " + familyLength + " exceeds " + + Byte.MAX_VALUE); + } + this.familyOffset = familyOffset; + this.familyLength = familyLength; + this.qualifierOffset = qualifierOffset; + this.qualifierLength = qualifierLength; + this.timestamp = timestamp; + } + + private static int length(byte[] value) { + return value == null ? 0 : value.length; + } + + private static void checkRange(byte[] array, int offset, int length, String field) { + if (offset < 0 || length < 0 || offset > array.length - length) { + throw new IndexOutOfBoundsException(field + " range is out of bounds"); + } + } + + private static int findFamilyDelimiter(byte[] familyQualifier) { + for (int i = 0; i < familyQualifier.length; i++) { + if (familyQualifier[i] == '\0') { + return i; + } + } + throw new RuntimeException("Cannot get family name"); + } + + @Override + public byte[] getRowArray() { + return row; + } + + @Override + public int getRowOffset() { + return 0; + } + + @Override + public short getRowLength() { + return (short) row.length; + } + + @Override + public byte[] getFamilyArray() { + return familyArray; + } + + @Override + public int getFamilyOffset() { + return familyOffset; + } + + @Override + public byte getFamilyLength() { + return (byte) familyLength; + } + + @Override + public byte[] getQualifierArray() { + return qualifierArray; + } + + @Override + public int getQualifierOffset() { + return qualifierOffset; + } + + @Override + public int getQualifierLength() { + return qualifierLength; + } + + @Override + public long getTimestamp() { + return timestamp; + } + + @Override + public byte getTypeByte() { + return PUT_TYPE; + } + + @Override + public long getMvccVersion() { + return 0L; + } + + @Override + public long getSequenceId() { + return 0L; + } + + @Override + public byte[] getValueArray() { + return value; + } + + @Override + public int getValueOffset() { + return 0; + } + + @Override + public int getValueLength() { + return value.length; + } + + @Override + public byte[] getTagsArray() { + return EMPTY; + } + + @Override + public int getTagsOffset() { + return 0; + } + + @Override + public int getTagsLength() { + return 0; + } + + @Override + public byte[] getValue() { + return Bytes.copy(value, 0, value.length); + } + + @Override + public byte[] getFamily() { + return Bytes.copy(familyArray, familyOffset, familyLength); + } + + @Override + public byte[] getQualifier() { + return Bytes.copy(qualifierArray, qualifierOffset, qualifierLength); + } + + @Override + public byte[] getRow() { + return Bytes.copy(row, 0, row.length); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseFuncUtils.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseFuncUtils.java index 734b226f..63fa5f79 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseFuncUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseFuncUtils.java @@ -21,6 +21,7 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.OHOperationType; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Row; @@ -32,6 +33,76 @@ @InterfaceAudience.Private public class OHBaseFuncUtils { + private static final Comparator HBASE_CELL_COMPARATOR = new Comparator() { + @Override + public int compare(Cell cell1, + Cell cell2) { + int familyComparison = Bytes + .compareTo( + cell1 + .getFamilyArray(), + cell1 + .getFamilyOffset(), + cell1 + .getFamilyLength(), + cell2 + .getFamilyArray(), + cell2 + .getFamilyOffset(), + cell2 + .getFamilyLength()); + if (familyComparison != 0) { + return familyComparison; + } + + int qualifierComparison = Bytes + .compareTo( + cell1 + .getQualifierArray(), + cell1 + .getQualifierOffset(), + cell1 + .getQualifierLength(), + cell2 + .getQualifierArray(), + cell2 + .getQualifierOffset(), + cell2 + .getQualifierLength()); + if (qualifierComparison != 0) { + return qualifierComparison; + } + + return Long.compare( + cell2.getTimestamp(), + cell1.getTimestamp()); + } + }; + + /** + * Build a TableGroup KeyValue directly from the protocol's {@code family\0qualifier} column. + * The offset constructor copies both ranges into the final KeyValue backing array and avoids + * allocating temporary family and qualifier arrays. + */ + public static KeyValue createTableGroupKeyValue(byte[] row, byte[] familyQualifier, + long timestamp, byte[] value) { + int familyLength = findFamilyDelimiter(familyQualifier); + int qualifierOffset = familyLength + 1; + return new KeyValue(row, 0, row == null ? 0 : row.length, familyQualifier, 0, familyLength, + familyQualifier, qualifierOffset, familyQualifier.length - qualifierOffset, timestamp, + KeyValue.Type.Put, value, 0, value == null ? 0 : value.length); + } + + private static int findFamilyDelimiter(byte[] familyQualifier) { + for (int i = 0; i < familyQualifier.length; i++) { + if (familyQualifier[i] == '\0') { + return i; + } + } + // Keep the failure contract of extractFamilyFromQualifier for malformed responses. + throw new RuntimeException("Cannot get family name"); + } + public static byte[][] extractFamilyFromQualifier(byte[] qualifier) throws Exception { int familyLen = -1; for (int i = 0; i < qualifier.length; i++) { @@ -80,30 +151,8 @@ public static boolean isAllPut(OHOperationType opType, List actio } } - public static void sortHBaseResult(List cells) { - cells.sort(new Comparator() { - @Override - public int compare(KeyValue kv1, KeyValue kv2) { - // 1. sort family in lexicographical order - int familyComparison = Bytes.compareTo(kv1.getFamilyArray(), kv1.getFamilyOffset(), - kv1.getFamilyLength(), kv2.getFamilyArray(), kv2.getFamilyOffset(), - kv2.getFamilyLength()); - if (familyComparison != 0) { - return familyComparison; - } - - // 2: sort qualifier in lexicographical order - int qualifierComparison = Bytes.compareTo(kv1.getQualifierArray(), - kv1.getQualifierOffset(), kv1.getQualifierLength(), kv2.getQualifierArray(), - kv2.getQualifierOffset(), kv2.getQualifierLength()); - if (qualifierComparison != 0) { - return qualifierComparison; - } - - // 3: sort timestamp in descend order - return Long.compare(kv2.getTimestamp(), kv1.getTimestamp()); - } - }); + public static void sortHBaseResult(List cells) { + cells.sort(HBASE_CELL_COMPARATOR); } public static boolean serverCanRetry(ObTableClient tableClient) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java new file mode 100644 index 00000000..6fe03417 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java @@ -0,0 +1,475 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2026 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationLevel; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjMeta; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpEntity; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellBatch; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellRow; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; +import com.alipay.oceanbase.hbase.result.OHBaseResultCell; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +public class OHTableGetMaxRowResultTest { + + private OHTable table; + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newSingleThreadExecutor(); + table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executorService, true); + } + + @After + public void tearDown() { + executorService.shutdownNow(); + } + + @Test + public void testPointGetValidatesOnlyFirstCellAndUsesExpectedRowKey() throws Exception { + byte[] expectedRowKey = Bytes.toBytes("row-1"); + AbstractQueryStreamResult streamResult = stream(row("row-1", "q1", 3L, "v1"), + row("unexpected-later-row", "q2", 2L, "v2")); + List keyValues = new ArrayList<>(); + + boolean found = invokeFillPointGet(streamResult, keyValues, false, Bytes.toBytes("f"), + expectedRowKey, false); + + assertTrue(found); + assertEquals(2, keyValues.size()); + assertArrayEquals(expectedRowKey, keyValues.get(0).getRow()); + assertArrayEquals(expectedRowKey, keyValues.get(1).getRow()); + assertTrue(keyValues.get(0) instanceof OHBaseResultCell); + verify(streamResult, times(2)).getRow(); + } + + @Test + public void testPointGetRejectsUnexpectedFirstRowKey() throws Exception { + AbstractQueryStreamResult streamResult = stream(row("actual", "q1", 1L, "v1")); + + try { + invokeFillPointGet(streamResult, new ArrayList(), false, Bytes.toBytes("f"), + Bytes.toBytes("expected"), false); + fail("unexpected first rowkey must fail the point Get"); + } catch (InvocationTargetException e) { + assertTrue(e.getCause() instanceof ObTableUnexpectedException); + } + } + + @Test + public void testPointGetConsumesCompactBatchWithoutMaterializingRows() throws Exception { + byte[] expectedRowKey = Bytes.toBytes("row-1"); + ObHBaseCellBatch firstBatch = compactBatch(row("row-1", "q1", 3L, "v1")); + ObHBaseCellBatch secondBatch = compactBatch(row("unexpected-later-row", "q2", 2L, + "v2")); + AbstractQueryStreamResult streamResult = compactPointGetStream(compactRow(firstBatch), + compactRow(secondBatch)); + List keyValues = new ArrayList(); + + boolean found = invokeFillPointGet(streamResult, keyValues, false, Bytes.toBytes("f"), + expectedRowKey, false); + + assertTrue(found); + assertEquals(2, keyValues.size()); + assertArrayEquals(expectedRowKey, keyValues.get(0).getRow()); + assertArrayEquals(expectedRowKey, keyValues.get(1).getRow()); + assertArrayEquals(Bytes.toBytes("q2"), keyValues.get(1).getQualifier()); + verify(streamResult, times(3)).next(); + verify(streamResult, times(2)).drainCurrentHBaseRow(); + verify(streamResult, never()).getRow(); + verify(streamResult, never()).getCurrentHBaseCellBatch(); + verify(streamResult, never()).getCurrentHBaseCellIndex(); + } + + @Test + public void testPointGetDrainsSameRowAcrossCachedBatchesOnce() throws Exception { + byte[] expectedRowKey = Bytes.toBytes("row-1"); + ObHBaseCellBatch firstBatch = compactBatch(row("row-1", "q1", 4L, "v1"), + row("row-1", "q2", 3L, "v2")); + ObHBaseCellBatch secondBatch = compactBatch(row("row-1", "q3", 2L, "v3"), + row("row-1", "q4", 1L, "v4")); + AbstractQueryStreamResult streamResult = compactPointGetStream(compactRow(firstBatch, + secondBatch)); + List cells = new ArrayList(); + + boolean found = invokeFillPointGet(streamResult, cells, false, Bytes.toBytes("f"), + expectedRowKey, false); + + assertTrue(found); + assertEquals(4, cells.size()); + assertArrayEquals(Bytes.toBytes("q1"), cells.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("q4"), cells.get(3).getQualifier()); + verify(streamResult, times(2)).next(); + verify(streamResult, times(1)).drainCurrentHBaseRow(); + verify(streamResult, never()).getRow(); + } + + @Test + public void testDisabledLightweightCellUsesKeyValue() throws Exception { + OHTable fallbackTable = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, false); + AbstractQueryStreamResult streamResult = stream(row("row-1", "q1", 1L, "v1")); + List cells = new ArrayList(); + + boolean found = invokeFillPointGet(fallbackTable, streamResult, cells, false, + Bytes.toBytes("f"), Bytes.toBytes("row-1"), false); + + assertTrue(found); + assertEquals(1, cells.size()); + assertTrue(cells.get(0) instanceof KeyValue); + } + + @Test + public void testPointGetExistenceOnlyDoesNotCreateKeyValue() throws Exception { + AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); + when(streamResult.next()).thenReturn(true); + List keyValues = new ArrayList<>(); + + boolean found = invokeFillPointGet(streamResult, keyValues, false, Bytes.toBytes("f"), + Bytes.toBytes("row-1"), true); + + assertTrue(found); + assertTrue(keyValues.isEmpty()); + verify(streamResult, never()).drainCurrentHBaseRow(); + verify(streamResult, never()).getRow(); + } + + @Test + public void testPointGetReturnsFalseForEmptyResult() throws Exception { + AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); + when(streamResult.next()).thenReturn(false); + + boolean found = invokeFillPointGet(streamResult, new ArrayList(), false, + Bytes.toBytes("f"), Bytes.toBytes("row-1"), false); + + assertFalse(found); + } + + @Test + public void testClosestRowBeforeKeepsOnlyCurrentMaxRow() throws Exception { + AbstractQueryStreamResult streamResult = stream(row("row-1", "q1", 4L, "v1"), + row("row-3", "q1", 3L, "v2"), row("row-3", "q2", 2L, "v3"), + row("row-2", "q1", 1L, "v4")); + List keyValues = new ArrayList<>(); + + boolean found = invokeGetMaxRow(streamResult, keyValues, false, Bytes.toBytes("f"), false); + + assertTrue(found); + assertEquals(2, keyValues.size()); + assertArrayEquals(Bytes.toBytes("row-3"), keyValues.get(0).getRow()); + assertArrayEquals(Bytes.toBytes("row-3"), keyValues.get(1).getRow()); + assertArrayEquals(Bytes.toBytes("q1"), keyValues.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("q2"), keyValues.get(1).getQualifier()); + } + + @Test + public void testClosestRowBeforeExistenceOnlyDoesNotCreateKeyValue() throws Exception { + AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); + when(streamResult.next()).thenReturn(true); + List keyValues = new ArrayList<>(); + + boolean found = invokeGetMaxRow(streamResult, keyValues, false, Bytes.toBytes("f"), true); + + assertTrue(found); + assertTrue(keyValues.isEmpty()); + verify(streamResult, never()).getRow(); + } + + @Test + public void testClosestRowBeforeConsumesCompactBatch() throws Exception { + AbstractQueryStreamResult streamResult = compactStream(compactBatch( + row("row-1", "q1", 4L, "v1"), row("row-3", "q1", 3L, "v2"), + row("row-3", "q2", 2L, "v3"), row("row-2", "q1", 1L, "v4"))); + List keyValues = new ArrayList(); + + boolean found = invokeGetMaxRow(streamResult, keyValues, false, Bytes.toBytes("f"), false); + + assertTrue(found); + assertEquals(2, keyValues.size()); + assertArrayEquals(Bytes.toBytes("row-3"), keyValues.get(0).getRow()); + assertArrayEquals(Bytes.toBytes("q2"), keyValues.get(1).getQualifier()); + verify(streamResult, never()).drainCurrentHBaseRow(); + verify(streamResult, never()).getRow(); + } + + @Test + public void testTableGroupSplitsFamilyAndQualifier() throws Exception { + byte[] familyAndQualifier = Bytes.add(Bytes.toBytes("family"), new byte[] { 0 }, + Bytes.toBytes("qualifier")); + AbstractQueryStreamResult streamResult = stream(row(Bytes.toBytes("row-1"), + familyAndQualifier, 1L, Bytes.toBytes("value"))); + List keyValues = new ArrayList<>(); + + boolean found = invokeFillPointGet(streamResult, keyValues, true, new byte[0], + Bytes.toBytes("row-1"), false); + + assertTrue(found); + assertEquals(1, keyValues.size()); + assertArrayEquals(Bytes.toBytes("family"), keyValues.get(0).getFamily()); + assertArrayEquals(Bytes.toBytes("qualifier"), keyValues.get(0).getQualifier()); + assertSame(keyValues.get(0).getFamilyArray(), keyValues.get(0).getQualifierArray()); + } + + @Test + public void testTableGroupSupportsEmptyQualifier() throws Exception { + byte[] familyAndQualifier = Bytes.add(Bytes.toBytes("family"), new byte[] { 0 }); + AbstractQueryStreamResult streamResult = stream(row(Bytes.toBytes("row-1"), + familyAndQualifier, 1L, Bytes.toBytes("value"))); + List keyValues = new ArrayList<>(); + + boolean found = invokeFillPointGet(streamResult, keyValues, true, new byte[0], + Bytes.toBytes("row-1"), false); + + assertTrue(found); + assertEquals(1, keyValues.size()); + assertArrayEquals(Bytes.toBytes("family"), keyValues.get(0).getFamily()); + assertArrayEquals(new byte[0], keyValues.get(0).getQualifier()); + } + + @Test + public void testTableGroupSupportsMaximumFamilyLength() throws Exception { + byte[] family = new byte[Byte.MAX_VALUE]; + Arrays.fill(family, (byte) 'f'); + byte[] qualifier = new byte[] { 0, (byte) 0xff, 1 }; + byte[] familyAndQualifier = Bytes.add(family, new byte[] { 0 }, qualifier); + AbstractQueryStreamResult streamResult = stream(row(new byte[] { 0, (byte) 0xff }, + familyAndQualifier, 1L, Bytes.toBytes("value"))); + List keyValues = new ArrayList<>(); + + boolean found = invokeGetMaxRow(streamResult, keyValues, true, new byte[0], false); + + assertTrue(found); + assertEquals(1, keyValues.size()); + assertArrayEquals(family, keyValues.get(0).getFamily()); + assertArrayEquals(qualifier, keyValues.get(0).getQualifier()); + } + + @Test + public void testTableGroupRejectsMissingFamilyDelimiter() throws Exception { + AbstractQueryStreamResult streamResult = stream(row(Bytes.toBytes("row-1"), + Bytes.toBytes("family-without-delimiter"), 1L, Bytes.toBytes("value"))); + + try { + invokeFillPointGet(streamResult, new ArrayList(), true, new byte[0], + Bytes.toBytes("row-1"), false); + fail("missing family delimiter must fail the TableGroup Get"); + } catch (InvocationTargetException e) { + assertTrue(e.getCause() instanceof RuntimeException); + assertEquals("Cannot get family name", e.getCause().getMessage()); + } + } + + @Test + public void testBatchGetBuildsTableGroupKeyValuesFromCompositeQualifier() throws Exception { + byte[] firstFamilyQualifier = Bytes.add(Bytes.toBytes("f1"), new byte[] { 0 }, + Bytes.toBytes("q1")); + byte[] secondFamilyQualifier = Bytes.add(Bytes.toBytes("f2"), new byte[] { 0 }, + Bytes.toBytes("q2")); + ObTableSingleOpEntity entity = mock(ObTableSingleOpEntity.class); + when(entity.getPropertiesValues()).thenReturn( + Arrays.asList(ObObj.getInstance(Bytes.toBytes("row-1")), + ObObj.getInstance(firstFamilyQualifier), ObObj.getInstance(2L), + ObObj.getInstance(Bytes.toBytes("v1")), ObObj.getInstance(Bytes.toBytes("row-1")), + ObObj.getInstance(secondFamilyQualifier), ObObj.getInstance(1L), + ObObj.getInstance(Bytes.toBytes("v2")))); + ObTableSingleOpResult result = mock(ObTableSingleOpResult.class); + when(result.getEntity()).thenReturn(entity); + + List cells = invokeGenerateGetResult(result); + + assertEquals(2, cells.size()); + assertTrue(cells.get(0) instanceof OHBaseResultCell); + assertArrayEquals(Bytes.toBytes("f1"), cells.get(0).getFamily()); + assertArrayEquals(Bytes.toBytes("q1"), cells.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("f2"), cells.get(1).getFamily()); + assertArrayEquals(Bytes.toBytes("q2"), cells.get(1).getQualifier()); + } + + @Test + public void testQueryAndMutateResultConsumesCompactBatch() throws Exception { + ObTableQueryResult queryResult = compactQueryResult(row("row-1", "q1", 2L, "v1"), + row("row-1", "q2", 1L, "v2")); + List keyValues = new ArrayList(); + + Method method = OHTable.class.getDeclaredMethod("addQueryResultToKeyValueList", + ObTableQueryResult.class, List.class, byte[].class); + method.setAccessible(true); + method.invoke(table, queryResult, keyValues, Bytes.toBytes("f")); + + assertEquals(2, keyValues.size()); + assertArrayEquals(Bytes.toBytes("q1"), keyValues.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("v2"), keyValues.get(1).getValue()); + assertTrue(queryResult.hasHBaseCellBatch()); + } + + @SuppressWarnings("unchecked") + private List invokeGenerateGetResult(ObTableSingleOpResult result) throws Exception { + Method method = OHTable.class.getDeclaredMethod("generateGetResult", + ObTableSingleOpResult.class); + method.setAccessible(true); + return (List) method.invoke(table, result); + } + + private boolean invokeFillPointGet(AbstractQueryStreamResult streamResult, + List keyValues, boolean isTableGroup, byte[] family, + byte[] expectedRowKey, boolean checkExistenceOnly) + throws Exception { + return invokeFillPointGet(table, streamResult, keyValues, isTableGroup, family, + expectedRowKey, checkExistenceOnly); + } + + private boolean invokeFillPointGet(OHTable targetTable, AbstractQueryStreamResult streamResult, + List keyValues, boolean isTableGroup, byte[] family, + byte[] expectedRowKey, boolean checkExistenceOnly) + throws Exception { + Method method = OHTable.class.getDeclaredMethod("fillPointGetFromResult", + AbstractQueryStreamResult.class, List.class, boolean.class, byte[].class, byte[].class, + boolean.class); + method.setAccessible(true); + return (Boolean) method.invoke(targetTable, streamResult, keyValues, isTableGroup, family, + expectedRowKey, checkExistenceOnly); + } + + private boolean invokeGetMaxRow(AbstractQueryStreamResult streamResult, List keyValues, + boolean isTableGroup, byte[] family, boolean checkExistenceOnly) + throws Exception { + Method method = OHTable.class + .getDeclaredMethod("getMaxRowFromResult", AbstractQueryStreamResult.class, List.class, + boolean.class, byte[].class, boolean.class); + method.setAccessible(true); + return (Boolean) method.invoke(table, streamResult, keyValues, isTableGroup, family, + checkExistenceOnly); + } + + private static AbstractQueryStreamResult stream(List... rows) throws Exception { + AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); + Boolean[] remaining = new Boolean[Math.max(0, rows.length - 1)]; + Arrays.fill(remaining, true); + when(streamResult.next()).thenReturn(true, remaining).thenReturn(false); + when(streamResult.getRow()).thenReturn(rows[0], Arrays.copyOfRange(rows, 1, rows.length)); + return streamResult; + } + + private static AbstractQueryStreamResult compactStream(ObHBaseCellBatch batch) + throws Exception { + AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); + AtomicInteger index = new AtomicInteger(-1); + when(streamResult.next()).thenAnswer(invocation -> index.incrementAndGet() < batch.size()); + when(streamResult.isCurrentHBaseCell()).thenReturn(true); + when(streamResult.getCurrentHBaseCellBatch()).thenReturn(batch); + when(streamResult.getCurrentHBaseCellIndex()).thenAnswer(invocation -> index.get()); + return streamResult; + } + + private static AbstractQueryStreamResult compactPointGetStream(ObHBaseCellRow... rows) + throws Exception { + AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); + Boolean[] remaining = new Boolean[Math.max(0, rows.length - 1)]; + Arrays.fill(remaining, true); + when(streamResult.next()).thenReturn(true, remaining).thenReturn(false); + when(streamResult.isCurrentHBaseCell()).thenReturn(true); + when(streamResult.drainCurrentHBaseRow()).thenReturn(rows[0], Arrays.copyOfRange(rows, 1, + rows.length)); + return streamResult; + } + + private static ObHBaseCellRow compactRow(ObHBaseCellBatch... batches) throws Exception { + assertTrue(batches.length > 0); + Constructor constructor = ObHBaseCellRow.class + .getDeclaredConstructor(byte[].class); + constructor.setAccessible(true); + ObHBaseCellRow row = constructor.newInstance(batches[0].getRowKey(0)); + Method addSlice = ObHBaseCellRow.class.getDeclaredMethod("addSlice", + ObHBaseCellBatch.class, int.class, int.class); + addSlice.setAccessible(true); + for (ObHBaseCellBatch batch : batches) { + addSlice.invoke(row, batch, 0, batch.size()); + } + return row; + } + + @SafeVarargs + private static ObHBaseCellBatch compactBatch(List... rows) { + return compactQueryResult(rows).getHBaseCellBatch(); + } + + @SafeVarargs + private static ObTableQueryResult compactQueryResult(List... rows) { + ObTableQueryResult encodedResult = new ObTableQueryResult(); + encodedResult.addPropertiesName("K"); + encodedResult.addPropertiesName("Q"); + encodedResult.addPropertiesName("T"); + encodedResult.addPropertiesName("V"); + encodedResult.addAllPropertiesRows(Arrays.asList(rows)); + encodedResult.setRowCount(rows.length); + + ByteBuf buf = Unpooled.wrappedBuffer(encodedResult.encode()); + try { + ObTableQueryResult decodedResult = new ObTableQueryResult(); + decodedResult.decode(buf); + assertTrue(decodedResult.hasHBaseCellBatch()); + return decodedResult; + } finally { + buf.release(); + } + } + + private static List row(String rowKey, String qualifier, long timestamp, String value) { + return row(Bytes.toBytes(rowKey), Bytes.toBytes(qualifier), timestamp, Bytes.toBytes(value)); + } + + private static List row(byte[] rowKey, byte[] qualifier, long timestamp, byte[] value) { + return Arrays.asList(binaryObj(rowKey), binaryObj(qualifier), new ObObj(new ObObjMeta( + ObObjType.ObInt64Type, ObCollationLevel.CS_LEVEL_NUMERIC, + ObCollationType.CS_TYPE_BINARY, (byte) 0), timestamp), binaryObj(value)); + } + + private static ObObj binaryObj(byte[] value) { + return new ObObj(new ObObjMeta(ObObjType.ObVarcharType, ObCollationLevel.CS_LEVEL_EXPLICIT, + ObCollationType.CS_TYPE_BINARY, (byte) 0), value); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java new file mode 100644 index 00000000..1c656b44 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java @@ -0,0 +1,91 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.rpc.ObTableClient; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +public class OHTableLightweightResultCellConfigTest { + + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newSingleThreadExecutor(); + } + + @After + public void tearDown() { + executorService.shutdownNow(); + } + + @Test + public void testPointAndScanLightweightCellSwitchesAreIndependent() throws Exception { + OHTable pointOnly = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, true, false); + assertTrue(getBooleanField(pointOnly, "getLightweightResultCellEnabled")); + assertFalse(getBooleanField(pointOnly, "scanLightweightResultCellEnabled")); + + OHTable scanOnly = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, false, true); + assertFalse(getBooleanField(scanOnly, "getLightweightResultCellEnabled")); + assertTrue(getBooleanField(scanOnly, "scanLightweightResultCellEnabled")); + } + + @Test + public void testInternalConstructorUsesScanLightweightCellDefault() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, true); + + assertTrue(getBooleanField(table, "getLightweightResultCellEnabled")); + assertTrue(getBooleanField(table, "scanLightweightResultCellEnabled")); + } + + @Test + public void testLightweightCellConfigurationNamesAndDefaults() { + assertEquals("hbase.htable.get.lightweight.result.cell.enabled", + HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED); + assertEquals("hbase.htable.scan.lightweight.result.cell.enabled", + HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED); + assertTrue(HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + assertTrue(HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); + } + + private static boolean getBooleanField(OHTable table, String fieldName) throws Exception { + Field field = OHTable.class.getDeclaredField(fieldName); + field.setAccessible(true); + return field.getBoolean(table); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java b/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java new file mode 100644 index 00000000..42a15347 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java @@ -0,0 +1,148 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.result; + +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellBatch; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellRow; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ClientStreamScannerCompactResultTest { + + @Test + public void testCompactResultUsesKeyValueWhenLightweightCellIsDisabled() throws Exception { + ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( + new String[] { "q-2", "q-1" }, new long[] { 101L, 102L })); + ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", bytes("f"), + false, null, false); + + Result result = scanner.next(); + + assertEquals(2, result.size()); + assertTrue(result.rawCells()[0] instanceof KeyValue); + assertArrayEquals(bytes("q-1"), CellUtil.cloneQualifier(result.rawCells()[0])); + assertArrayEquals(bytes("q-2"), CellUtil.cloneQualifier(result.rawCells()[1])); + verify(streamResult, never()).getRow(); + verify(streamResult, never()).getCacheRows(); + } + + @Test + public void testCompactResultUsesLightweightCellByDefault() throws Exception { + ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( + new String[] { "q-1" }, new long[] { 102L })); + ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", bytes("f"), + false, null); + + Result result = scanner.next(); + + assertEquals(1, result.size()); + assertTrue(result.rawCells()[0] instanceof OHBaseResultCell); + } + + @Test + public void testCompactResultUsesLightweightCellWhenEnabled() throws Exception { + ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( + new String[] { "q-1", "q-2" }, new long[] { 102L, 101L })); + ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", bytes("f"), + false, null, true); + + Result result = scanner.next(); + + assertEquals(2, result.size()); + for (Cell cell : result.rawCells()) { + assertTrue(cell instanceof OHBaseResultCell); + assertArrayEquals(bytes("row-1"), CellUtil.cloneRow(cell)); + assertArrayEquals(bytes("f"), CellUtil.cloneFamily(cell)); + } + verify(streamResult, never()).getRow(); + verify(streamResult, never()).getCacheRows(); + } + + @Test + public void testCompactTableGroupResultUsesQualifierOffsets() throws Exception { + ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( + new String[] { "f1\0q-1", "f2\0q-2" }, new long[] { 102L, 101L })); + ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", new byte[0], + true, null, true); + + Result result = scanner.next(); + + assertEquals(2, result.size()); + assertArrayEquals(bytes("f1"), CellUtil.cloneFamily(result.rawCells()[0])); + assertArrayEquals(bytes("q-1"), CellUtil.cloneQualifier(result.rawCells()[0])); + assertArrayEquals(bytes("f2"), CellUtil.cloneFamily(result.rawCells()[1])); + assertArrayEquals(bytes("q-2"), CellUtil.cloneQualifier(result.rawCells()[1])); + } + + private static ObTableClientQueryAsyncStreamResult compactStreamResult(ObHBaseCellRow row) + throws Exception { + ObTableClientQueryAsyncStreamResult streamResult = mock(ObTableClientQueryAsyncStreamResult.class); + when(streamResult.next()).thenReturn(true); + when(streamResult.isCurrentHBaseCell()).thenReturn(true); + when(streamResult.drainCurrentHBaseRow()).thenReturn(row); + when(streamResult.getTableName()).thenReturn("test"); + return streamResult; + } + + private static ObHBaseCellRow compactRow(String[] qualifiers, long[] timestamps) + throws Exception { + assertEquals(qualifiers.length, timestamps.length); + Constructor batchConstructor = ObHBaseCellBatch.class + .getDeclaredConstructor(int.class); + batchConstructor.setAccessible(true); + ObHBaseCellBatch batch = batchConstructor.newInstance(qualifiers.length); + Method setCell = ObHBaseCellBatch.class.getDeclaredMethod("setCell", int.class, + byte[].class, byte[].class, long.class, byte[].class); + setCell.setAccessible(true); + byte[] rowKey = bytes("row-1"); + for (int i = 0; i < qualifiers.length; i++) { + setCell.invoke(batch, i, rowKey, bytes(qualifiers[i]), timestamps[i], bytes("value-" + + i)); + } + + Constructor rowConstructor = ObHBaseCellRow.class + .getDeclaredConstructor(byte[].class); + rowConstructor.setAccessible(true); + ObHBaseCellRow row = rowConstructor.newInstance(rowKey); + Method addSlice = ObHBaseCellRow.class.getDeclaredMethod("addSlice", + ObHBaseCellBatch.class, int.class, int.class); + addSlice.setAccessible(true); + addSlice.invoke(row, batch, 0, qualifiers.length); + return row; + } + + private static byte[] bytes(String value) { + return Bytes.toBytes(value); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/result/OHBaseResultCellTest.java b/src/test/java/com/alipay/oceanbase/hbase/result/OHBaseResultCellTest.java new file mode 100644 index 00000000..7f4d3a50 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/result/OHBaseResultCellTest.java @@ -0,0 +1,127 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.result; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.Collections; + +import static org.junit.Assert.*; + +public class OHBaseResultCellTest { + + @Test + public void testFieldBackedCellAccessors() { + byte[] row = bytes("row-1"); + byte[] family = bytes("f"); + byte[] qualifier = bytes("q1"); + byte[] value = bytes("value-1"); + OHBaseResultCell cell = OHBaseResultCell.create(row, family, qualifier, 123L, value); + + assertSame(row, cell.getRowArray()); + assertSame(family, cell.getFamilyArray()); + assertSame(qualifier, cell.getQualifierArray()); + assertSame(value, cell.getValueArray()); + assertEquals(0, cell.getRowOffset()); + assertEquals(row.length, cell.getRowLength()); + assertEquals(0, cell.getFamilyOffset()); + assertEquals(family.length, cell.getFamilyLength()); + assertEquals(0, cell.getQualifierOffset()); + assertEquals(qualifier.length, cell.getQualifierLength()); + assertEquals(0, cell.getValueOffset()); + assertEquals(value.length, cell.getValueLength()); + assertEquals(123L, cell.getTimestamp()); + assertEquals(KeyValue.Type.Put.getCode(), cell.getTypeByte()); + assertEquals(0L, cell.getMvccVersion()); + assertEquals(0L, cell.getSequenceId()); + assertEquals(0, cell.getTagsLength()); + + assertArrayEquals(row, cell.getRow()); + assertArrayEquals(family, cell.getFamily()); + assertArrayEquals(qualifier, cell.getQualifier()); + assertArrayEquals(value, cell.getValue()); + assertNotSame(row, cell.getRow()); + assertNotSame(value, cell.getValue()); + } + + @Test + public void testTableGroupCellUsesSharedArrayRanges() { + byte[] familyQualifier = new byte[] { 'f', '1', 0, 'q', '1' }; + OHBaseResultCell cell = OHBaseResultCell.createTableGroup(bytes("row-1"), familyQualifier, + 99L, bytes("v")); + + assertSame(familyQualifier, cell.getFamilyArray()); + assertSame(familyQualifier, cell.getQualifierArray()); + assertEquals(0, cell.getFamilyOffset()); + assertEquals(2, cell.getFamilyLength()); + assertEquals(3, cell.getQualifierOffset()); + assertEquals(2, cell.getQualifierLength()); + assertArrayEquals(bytes("f1"), CellUtil.cloneFamily(cell)); + assertArrayEquals(bytes("q1"), CellUtil.cloneQualifier(cell)); + } + + @Test + public void testResultAndLegacyKeyValueApisRemainCompatible() { + byte[] family = bytes("f"); + byte[] qualifier = bytes("q"); + byte[] value = bytes("value"); + Cell cell = OHBaseResultCell.create(bytes("row"), family, qualifier, 7L, value); + Result result = Result.create(Collections.singletonList(cell)); + + assertSame(cell, result.rawCells()[0]); + assertSame(cell, result.listCells().get(0)); + assertArrayEquals(value, result.getValue(family, qualifier)); + ByteBuffer valueBuffer = result.getValueAsByteBuffer(family, qualifier); + assertArrayEquals(value, Bytes.toBytes(valueBuffer)); + assertSame(cell, result.getColumnLatestCell(family, qualifier)); + assertEquals(1, result.getColumnCells(family, qualifier).size()); + assertArrayEquals(value, result.getFamilyMap(family).get(qualifier)); + + KeyValue converted = KeyValueUtil.ensureKeyValue(cell); + assertArrayEquals(bytes("row"), converted.getRow()); + assertArrayEquals(family, converted.getFamily()); + assertArrayEquals(qualifier, converted.getQualifier()); + assertArrayEquals(value, converted.getValue()); + assertEquals(7L, converted.getTimestamp()); + assertTrue(result.raw()[0] instanceof KeyValue); + assertTrue(result.list().get(0) instanceof KeyValue); + assertTrue(result.getColumnLatest(family, qualifier) instanceof KeyValue); + } + + @Test(expected = RuntimeException.class) + public void testTableGroupCellRejectsMissingDelimiter() { + OHBaseResultCell.createTableGroup(bytes("row"), bytes("family-qualifier"), 1L, bytes("v")); + } + + @Test(expected = IllegalArgumentException.class) + public void testCellRejectsOversizedFamily() { + OHBaseResultCell.create(bytes("row"), new byte[Byte.MAX_VALUE + 1], bytes("q"), 1L, + bytes("v")); + } + + private static byte[] bytes(String value) { + return Bytes.toBytes(value); + } +} From 1f68e1e89201286acac0c9fe66c48f4295144471 Mon Sep 17 00:00:00 2001 From: "linguantian.lgt" Date: Mon, 10 Aug 2026 12:00:01 +0800 Subject: [PATCH 2/4] perf: optimize HBase Put write path Reduce HBase client CPU usage by simplifying synchronous Put execution and integrating compact Put request encoding. --- .../com/alipay/oceanbase/hbase/OHTable.java | 309 ++++++++++++++---- .../hbase/constants/OHConstants.java | 25 ++ .../hbase/util/OHBufferedMutatorImpl.java | 18 +- .../hbase/OHTableCompactPutCellTest.java | 155 +++++++++ .../hbase/OHTablePutDirectAutoFlushTest.java | 248 ++++++++++++++ .../hbase/OHTablePutSkipCellCloneTest.java | 243 ++++++++++++++ .../hbase/OHTablePutValidationTest.java | 144 ++++++++ 7 files changed, 1067 insertions(+), 75 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 018b89a7..484e9958 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -28,6 +28,7 @@ import com.alipay.oceanbase.hbase.util.*; import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.util.ObBytesString; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; @@ -168,6 +169,23 @@ public class OHTable implements HTableInterface { */ private boolean enablePutOptimization; + /** + * When autoFlush is on, bypass BufferedMutator queue/flush and call innerBatchImpl + * directly. Default true; set hbase.htable.put.direct.autoflush.enabled=false to + * restore the legacy path. + */ + private boolean enablePutDirectAutoFlush; + + /** + * On sync-complete Put V2 paths (autoFlush), skip CellUtil.clone* for contiguous + * Q/V arrays. Default true; set hbase.htable.put.skip.cell.clone.enabled=false to + * always clone. + */ + private boolean enablePutSkipCellClone; + + /** Use compact Q/T/V/(TTL) arrays for Put V2 request encoding. */ + private boolean enablePutCompactCell; + // i.e., doPut checks the writebuffer every X Puts. /** @@ -503,6 +521,12 @@ private void finishSetUp() { WRITE_BUFFER_SIZE_DEFAULT); this.enablePutOptimization = this.configuration.getBoolean(HBASE_HTABLE_USE_PUT_OPTIMIZATION, HBASE_HTABLE_USE_PUT_OPTIMIZATION_DEFAULT); + this.enablePutDirectAutoFlush = this.configuration.getBoolean( + HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED, HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT); + this.enablePutSkipCellClone = this.configuration.getBoolean( + HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED, HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT); + this.enablePutCompactCell = this.configuration.getBoolean( + HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED, HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT); } public static OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, @@ -829,16 +853,20 @@ public void innerBatchImpl(final List actions, final Object[] res throw new AssertionError("results.length"); } } - BatchError batchError = new BatchError(); obTableClient.setRuntimeBatchExecutor(executePool); - List resultMapSingleOp = new LinkedList<>(); if (!ObGlobal.isHBaseBatchSupport()) { + BatchError batchError = new BatchError(); try { compatOldServerBatch(actions, results, batchError); } catch (Exception e) { throw new IOException(tableNameString + " table occurred unexpected error." , e); } - } else if (OHBaseFuncUtils.isAllPut(opType, actions) && OHBaseFuncUtils.isHBasePutPefSupport(obTableClient, enablePutOptimization)) { + if (batchError.hasErrors()) { + throw batchError.makeException(); + } + return; + } + if (OHBaseFuncUtils.isAllPut(opType, actions) && OHBaseFuncUtils.isHBasePutPefSupport(obTableClient, enablePutOptimization)) { // only support Put now ObHbaseRequest request = buildHbaseRequest(actions, opType); try { @@ -851,48 +879,50 @@ public void innerBatchImpl(final List actions, final Object[] res } catch (Exception e) { throw new IOException(tableNameString + " table occurred unexpected error." , e); } - } else { - String realTableName = getTargetTableName(actions); - BatchOperation batch = buildBatchOperation(realTableName, actions, - tableNameString.equals(realTableName), resultMapSingleOp); - batch.setHbaseOpType(opType); - BatchOperationResult tmpResults; - try { - tmpResults = batch.execute(); - } catch (Exception e) { - throw new IOException(tableNameString + " table occurred unexpected error.", e); - } - int index = 0; - for (int i = 0; i != actions.size(); ++i) { - if (tmpResults.getResults().get(index) instanceof ObTableException) { - if (results != null) { - results[i] = tmpResults.getResults().get(index); - } - batchError.add((ObTableException) tmpResults.getResults().get(index), actions.get(i), null); - } else if (actions.get(i) instanceof Get) { - if (results != null) { - // get results have been wrapped in MutationResult, need to fetch it - if (tmpResults.getResults().get(index) instanceof MutationResult) { - MutationResult mutationResult = (MutationResult) tmpResults.getResults().get(index); - ObPayload innerResult = mutationResult.getResult(); - if (innerResult instanceof ObTableSingleOpResult) { - ObTableSingleOpResult singleOpResult = (ObTableSingleOpResult) innerResult; - List cells = generateGetResult(singleOpResult); - results[i] = Result.create(cells); - } else { - throw new ObTableUnexpectedException("Unexpected type of result in MutationResult"); - } + return; + } + BatchError batchError = new BatchError(); + List resultMapSingleOp = new LinkedList<>(); + String realTableName = getTargetTableName(actions); + BatchOperation batch = buildBatchOperation(realTableName, actions, + tableNameString.equals(realTableName), resultMapSingleOp); + batch.setHbaseOpType(opType); + BatchOperationResult tmpResults; + try { + tmpResults = batch.execute(); + } catch (Exception e) { + throw new IOException(tableNameString + " table occurred unexpected error.", e); + } + int index = 0; + for (int i = 0; i != actions.size(); ++i) { + if (tmpResults.getResults().get(index) instanceof ObTableException) { + if (results != null) { + results[i] = tmpResults.getResults().get(index); + } + batchError.add((ObTableException) tmpResults.getResults().get(index), actions.get(i), null); + } else if (actions.get(i) instanceof Get) { + if (results != null) { + // get results have been wrapped in MutationResult, need to fetch it + if (tmpResults.getResults().get(index) instanceof MutationResult) { + MutationResult mutationResult = (MutationResult) tmpResults.getResults().get(index); + ObPayload innerResult = mutationResult.getResult(); + if (innerResult instanceof ObTableSingleOpResult) { + ObTableSingleOpResult singleOpResult = (ObTableSingleOpResult) innerResult; + List cells = generateGetResult(singleOpResult); + results[i] = Result.create(cells); } else { - throw new ObTableUnexpectedException("Unexpected type of result in batch"); + throw new ObTableUnexpectedException("Unexpected type of result in MutationResult"); } - } - } else { - if (results != null) { - results[i] = new Result(); + } else { + throw new ObTableUnexpectedException("Unexpected type of result in batch"); } } - index += resultMapSingleOp.get(i); + } else { + if (results != null) { + results[i] = new Result(); + } } + index += resultMapSingleOp.get(i); } if (batchError.hasErrors()) { throw batchError.makeException(); @@ -1549,6 +1579,9 @@ public void put(Put put) throws IOException { execute(new OperationExecuteCallback(opType, 1 /* batchSize */) { @Override public Void execute() throws IOException { + if (tryDirectAutoFlushPuts(Collections.singletonList(put), opType)) { + return null; + } ((OHBufferedMutatorImpl) getBufferedMutator()).setOpType(opType); getBufferedMutator().mutate(put); if (autoFlush) { @@ -1565,6 +1598,12 @@ public void put(List puts) throws IOException { execute(new OperationExecuteCallback(opType, puts.size() /* batchSize */) { @Override public Void execute() throws IOException { + if (puts.isEmpty()) { + return null; + } + if (tryDirectAutoFlushPuts(puts, opType)) { + return null; + } ((OHBufferedMutatorImpl) getBufferedMutator()).setOpType(opType); getBufferedMutator().mutate(puts); if (autoFlush) { @@ -1575,21 +1614,98 @@ public Void execute() throws IOException { }); } + /** + * AutoFlush fast path: validate then innerBatchImpl, skipping BufferedMutator + * queue / heapSize / LinkedList / flush. If a pending buffer exists, flush it + * first so buffered Puts stay ordered ahead of the direct Put. + * + * @return true if the direct path handled the Puts; false to use legacy mutate+flush + */ + private boolean tryDirectAutoFlushPuts(List puts, OHOperationType opType) + throws IOException { + if (!enablePutDirectAutoFlush || !autoFlush) { + return false; + } + for (Row row : puts) { + if (!(row instanceof Put)) { + return false; + } + validatePutMutation((Put) row); + } + if (!isWriteBufferEmpty()) { + flushCommits(); + if (!isWriteBufferEmpty()) { + // Listener may have swallowed errors while leaving residual state; + // fall back to the legacy path rather than reorder writes. + return false; + } + } + Object[] results = null; + innerBatchImpl(puts, results, opType); + return true; + } + + @VisibleForTesting + public boolean isWriteBufferEmpty() { + return mutator == null || mutator.isBufferEmpty(); + } + + @VisibleForTesting + public boolean isPutDirectAutoFlushEnabled() { + return enablePutDirectAutoFlush; + } + + @VisibleForTesting + public boolean isPutSkipCellCloneEnabled() { + return enablePutSkipCellClone; + } + + /** + * Put validation shared by the autoFlush direct path and BufferedMutator. + */ + public void validatePutMutation(Put put) { + NavigableMap> familyCellMap = put.getFamilyCellMap(); + validatePut(put, familyCellMap, maxKeyValueSize); + if (isMultiFamilyWriteSupport()) { + checkFamilyViolation(familyCellMap.keySet(), true); + } else { + checkFamilyViolationForOneFamily(familyCellMap.keySet()); + } + } + + /** + * Same multi-CF capability gate as OHBufferedMutatorImpl historically used. + */ + public static boolean isMultiFamilyWriteSupport() { + long multiCfSince425Bp1 = ObGlobal.calcVersion(4, (short) 2, (byte) 5, (byte) 1); + long before430 = ObGlobal.calcVersion(4, (short) 3, (byte) 0, (byte) 0); + long multiCfSince434 = ObGlobal.calcVersion(4, (short) 3, (byte) 4, (byte) 0); + return (ObGlobal.OB_VERSION >= multiCfSince425Bp1 && ObGlobal.OB_VERSION < before430) + || (ObGlobal.OB_VERSION >= multiCfSince434); + } + /** * 校验 put 里的参数是否合法，需要传入 family ，并且 keyvalue 的 size 不能太大 * @param put the put */ public static void validatePut(Put put, int maxKeyValueSize) { + validatePut(put, put.getFamilyCellMap(), maxKeyValueSize); + } + + private static void validatePut(Put put, NavigableMap> familyCellMap, + int maxKeyValueSize) { if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { - for (Map.Entry> entry : put.getFamilyMap().entrySet()) { + for (Map.Entry> entry : familyCellMap.entrySet()) { if (entry.getKey() == null || entry.getKey().length == 0) { throw new IllegalArgumentException("family is empty"); } - for (KeyValue kv : entry.getValue()) { - if (kv.getLength() > maxKeyValueSize) { + for (Cell cell : entry.getValue()) { + int cellLength = cell instanceof KeyValue ? ((KeyValue) cell).getLength() + : KeyValueUtil.length(cell); + if (cellLength > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } @@ -2546,10 +2662,11 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, switch (kvType) { case Put: String[] property_columns = V_COLUMNS; - Object[] property = new Object[] { CellUtil.cloneValue(kv) }; + Object value = CellUtil.cloneValue(kv); + Object[] property = new Object[] { value }; if (TTL != Long.MAX_VALUE) { property_columns = PROPERTY_COLUMNS; - property = new Object[] { CellUtil.cloneValue(kv), TTL }; + property = new Object[] { value, TTL }; } return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, ROW_KEY_COLUMNS, @@ -2769,13 +2886,14 @@ private BatchOperation buildBatchOperation(String tableName, List return batch; } - private ObHbaseRequest buildHbaseRequest(List actions, OHOperationType hbaseOpType) - throws FeatureNotSupportedException, - IllegalArgumentException, - IOException { + @VisibleForTesting + ObHbaseRequest buildHbaseRequest(List actions, OHOperationType hbaseOpType) + throws FeatureNotSupportedException, + IllegalArgumentException, + IOException { ObHbaseRequest request = new ObHbaseRequest(); ObTableOperationType opType = null; - List keys = new ArrayList<>(); + List keys = new ArrayList<>(actions.size()); List cfRowsArray = new ArrayList<>(); Map cfRowsMap = new HashMap<>(); int keyIndex = 0; @@ -2791,30 +2909,45 @@ private ObHbaseRequest buildHbaseRequest(List actions, OHOperatio if (ttl != Long.MAX_VALUE) { isCellTTL = true; } - keys.add(ObObj.getInstance(put.getRow())); + ObObj ttlObj = isCellTTL && !enablePutCompactCell ? ObObj.hbasePutInt64(ttl) + : null; + keys.add(ObObj.hbasePutVarchar(put.getRow())); + boolean shareCellBytes = enablePutSkipCellClone && autoFlush; for (Map.Entry> entry : put.getFamilyCellMap().entrySet()) { String family = Bytes.toString(entry.getKey()); ObHbaseCfRows sameCfRows = cfRowsMap.get(family); if (sameCfRows == null) { sameCfRows = new ObHbaseCfRows(); + sameCfRows.reserveKeyRuns(actions.size()); String realTableName = getTargetTableName(tableNameString, family); sameCfRows.setRealTableName(realTableName); cfRowsMap.put(family, sameCfRows); cfRowsArray.add(sameCfRows); } List keyValueList = entry.getValue(); - List cells = new ArrayList<>(); - for (Cell kv : keyValueList) { - ObHbaseCell cell = new ObHbaseCell(isCellTTL); - cell.setQ(ObObj.getInstance(CellUtil.cloneQualifier(kv))); - cell.setT(ObObj.getInstance(-getEffectiveTimestampForWrite(kv.getTimestamp()))); // set timestamp as negative - cell.setV(ObObj.getInstance(CellUtil.cloneValue(kv))); - if (isCellTTL) { - cell.setTTL(ObObj.getInstance(ttl)); + if (enablePutCompactCell) { + sameCfRows.reserveAdditionalCompactCells(keyValueList.size()); + sameCfRows.beginCompactKeyCells(keyIndex, keyValueList.size(), ttl); + for (Cell kv : keyValueList) { + appendCompactPutCell(sameCfRows, kv, shareCellBytes); + } + } else { + sameCfRows.reserveAdditionalCells(keyValueList.size()); + sameCfRows.beginKeyCells(keyIndex, keyValueList.size()); + for (Cell kv : keyValueList) { + ObHbaseCell cell = new ObHbaseCell(isCellTTL); + cell.setQ(ObObj.hbasePutVarchar(bytesForPutCell(kv, shareCellBytes, + true))); + cell.setT(ObObj.hbasePutInt64(-getEffectiveTimestampForWrite(kv + .getTimestamp()))); + cell.setV(ObObj.hbasePutVarchar(bytesForPutCell(kv, shareCellBytes, + false))); + if (isCellTTL) { + cell.setTTL(ttlObj); + } + sameCfRows.appendCell(cell); } - cells.add(cell); } - sameCfRows.add(keyIndex, cells.size(), cells); } } else { throw new FeatureNotSupportedException( @@ -2831,15 +2964,65 @@ private ObHbaseRequest buildHbaseRequest(List actions, OHOperatio return request; } + /** + * Put V2 Q/V payload for ObObj. When {@code share} is true, reference the Cell + * region without clone: contiguous arrays are returned as {@code byte[]}; sliced + * regions use an {@link ObBytesString} view. Encode copies once into the RPC buffer. + * Delayed-flush / non-autoFlush paths must pass share=false (clone to byte[]). + */ + @VisibleForTesting + static Object bytesForPutCell(Cell cell, boolean share, boolean qualifier) { + if (share) { + byte[] array = qualifier ? cell.getQualifierArray() : cell.getValueArray(); + int offset = qualifier ? cell.getQualifierOffset() : cell.getValueOffset(); + int length = qualifier ? cell.getQualifierLength() : cell.getValueLength(); + if (array == null) { + return new byte[0]; + } + if (offset == 0 && length == array.length) { + return array; + } + return new ObBytesString(array, offset, length); + } + return qualifier ? CellUtil.cloneQualifier(cell) : CellUtil.cloneValue(cell); + } + + private void appendCompactPutCell(ObHbaseCfRows cfRows, Cell cell, boolean shareCellBytes) { + byte[] qualifier; + int qualifierOffset; + int qualifierLength; + byte[] value; + int valueOffset; + int valueLength; + if (shareCellBytes) { + qualifier = cell.getQualifierArray(); + qualifierOffset = cell.getQualifierOffset(); + qualifierLength = cell.getQualifierLength(); + value = cell.getValueArray(); + valueOffset = cell.getValueOffset(); + valueLength = cell.getValueLength(); + } else { + qualifier = CellUtil.cloneQualifier(cell); + qualifierOffset = 0; + qualifierLength = qualifier.length; + value = CellUtil.cloneValue(cell); + valueOffset = 0; + valueLength = value.length; + } + cfRows.appendCompactCell(qualifier, qualifierOffset, qualifierLength, + -getEffectiveTimestampForWrite(cell.getTimestamp()), value, valueOffset, valueLength); + } + public ObTableOperation buildObTableOperation(KeyValue kv, ObTableOperationType operationType, Long TTL) { KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); String[] property_columns = V_COLUMNS; - Object[] property = new Object[] { CellUtil.cloneValue(kv) }; + Object value = CellUtil.cloneValue(kv); + Object[] property = new Object[] { value }; if (TTL != Long.MAX_VALUE) { property_columns = PROPERTY_COLUMNS; - property = new Object[] { CellUtil.cloneValue(kv), TTL }; + property = new Object[] { value, TTL }; } switch (kvType) { case Put: diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index 47bc6c60..df9ad650 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -179,6 +179,25 @@ public final class OHConstants { */ public static final String HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED = "hbase.htable.scan.lightweight.result.cell.enabled"; + /** + * When autoFlush is enabled, put(Put)/put(List) bypass BufferedMutator and call + * innerBatchImpl directly. Default is true (enabled). + */ + public static final String HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED = "hbase.htable.put.direct.autoflush.enabled"; + + /** + * When building Put V2 requests on a sync-complete path (autoFlush / direct put), + * skip CellUtil.clone* for contiguous qualifier/value byte arrays and share the + * Cell backing array until encode. Default is true (enabled). + */ + public static final String HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED = "hbase.htable.put.skip.cell.clone.enabled"; + + /** + * Store Put V2 cells in compact parallel arrays and encode Q/T/V/(TTL) directly. + * Default is true (enabled). + */ + public static final String HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED = "hbase.htable.put.compact.cell.enabled"; + /*-------------------------------------------------------------------------------------------------------------*/ /** @@ -212,4 +231,10 @@ public final class OHConstants { public static final boolean HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT = true; + public static final boolean HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT = true; + + public static final boolean HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT = true; + + public static final boolean HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT = true; + } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index c3879147..5ab9a0c5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -33,8 +33,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static com.alipay.oceanbase.rpc.ObGlobal.*; - @InterfaceAudience.Private public class OHBufferedMutatorImpl implements BufferedMutator { private static final Logger LOGGER = TableHBaseLoggerFactory @@ -56,8 +54,6 @@ public class OHBufferedMutatorImpl implements BufferedMutator { private int rpcTimeout; private int operationTimeout; private OHOperationType opType = OHOperationType.INVALID; - private static final long OB_VERSION_4_2_5_1 = calcVersion(4, (short) 2, - (byte) 5, (byte) 1); public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params, OHTable ohTable) throws IOException { @@ -174,12 +170,7 @@ private void validateOperation(Mutation mt) throws IllegalArgumentException { } if (mt instanceof Put) { // family empty check is in validatePut - OHTable.validatePut((Put) mt, maxKeyValueSize); - if (isMultiFamilySupport()) { - OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), true); - } else { - OHTable.checkFamilyViolationForOneFamily(mt.getFamilyMap().keySet()); - } + ohTable.validatePutMutation((Put) mt); } else { if (isMultiFamilySupport()) { OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), false); @@ -304,8 +295,7 @@ public void setWriteBufferSize(long writeBufferSize) throws IOException { * Only 4_2_5 BP1 - 4_3_0 and after 4_3_4 support multi-cf * */ boolean isMultiFamilySupport() { - return (OB_VERSION >= OB_VERSION_4_2_5_1 && OB_VERSION < OB_VERSION_4_3_0_0) - || (OB_VERSION >= OB_VERSION_4_3_4_0); + return OHTable.isMultiFamilyWriteSupport(); } /** @@ -336,6 +326,10 @@ public long getCurrentBufferSize() { return currentAsyncBufferSize.get(); } + public boolean isBufferEmpty() { + return asyncWriteBuffer.isEmpty() && currentAsyncBufferSize.get() == 0L; + } + @Deprecated public List getWriteBuffer() { return Arrays.asList(asyncWriteBuffer.toArray(new Row[0])); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java new file mode 100644 index 00000000..68fc1a94 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java @@ -0,0 +1,155 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.OHOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObHbaseRequest; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +public class OHTableCompactPutCellTest { + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newSingleThreadExecutor(); + } + + @After + public void tearDown() { + executorService.shutdownNow(); + } + + @Test + public void testConfigNameAndDefault() throws Exception { + assertEquals("hbase.htable.put.compact.cell.enabled", HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED); + assertTrue(HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT); + OHTable table = newTable(); + assertTrue(getCompactEnabled(table)); + } + + @Test + public void testCompactRequestMatchesLegacyWithoutTtl() throws Exception { + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), 1001L, Bytes.toBytes("value1")); + put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("qualifier-2"), 1002L, + Bytes.toBytes("value-2")); + assertLegacyAndCompactEqual(put, OHOperationType.PUT); + } + + @Test + public void testCompactRequestMatchesLegacyWithTtlAndMultipleRows() throws Exception { + Put first = new Put(Bytes.toBytes("row-1")); + first.setTTL(60000L); + first.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), 2001L, Bytes.toBytes("value-1")); + first.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q2"), 2002L, Bytes.toBytes("value-2")); + first.addColumn(Bytes.toBytes("cf2"), Bytes.toBytes("q3"), 2003L, Bytes.toBytes("value-3")); + Put second = new Put(Bytes.toBytes("row-2")); + second.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q4"), 3001L, Bytes.toBytes("value-4")); + + OHTable legacyTable = newTable(); + OHTable compactTable = newTable(); + setCompactEnabled(legacyTable, false); + setCompactEnabled(compactTable, true); + ObHbaseRequest legacy = legacyTable.buildHbaseRequest(Arrays.asList(first, second), + OHOperationType.PUT_LIST); + ObHbaseRequest compact = compactTable.buildHbaseRequest(Arrays.asList(first, second), + OHOperationType.PUT_LIST); + + assertEquals(2, legacy.getCfRows().size()); + assertEquals(2, compact.getCfRows().size()); + for (int i = 0; i < compact.getCfRows().size(); i++) { + assertFalse(legacy.getCfRows().get(i).hasCompactCells()); + assertTrue(compact.getCfRows().get(i).hasCompactCells()); + } + assertArrayEquals(legacy.encode(), compact.encode()); + } + + @Test + public void testBufferedCompactRequestOwnsQualifierAndValueBytes() throws Exception { + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("qualifier"), 4001L, + Bytes.toBytes("original-value")); + Cell source = put.getFamilyCellMap().get(Bytes.toBytes("cf")).get(0); + + OHTable compactTable = newTable(); + compactTable.setAutoFlush(false); + setCompactEnabled(compactTable, true); + ObHbaseRequest compact = compactTable.buildHbaseRequest(Collections.singletonList(put), + OHOperationType.PUT); + byte[] encodedBeforeMutation = compact.encode(); + + Arrays.fill(source.getQualifierArray(), source.getQualifierOffset(), + source.getQualifierOffset() + source.getQualifierLength(), (byte) 'x'); + Arrays.fill(source.getValueArray(), source.getValueOffset(), source.getValueOffset() + + source.getValueLength(), + (byte) 'y'); + + assertArrayEquals(encodedBeforeMutation, compact.encode()); + } + + private void assertLegacyAndCompactEqual(Put put, OHOperationType operationType) + throws Exception { + OHTable legacyTable = newTable(); + OHTable compactTable = newTable(); + setCompactEnabled(legacyTable, false); + setCompactEnabled(compactTable, true); + ObHbaseRequest legacy = legacyTable.buildHbaseRequest(Collections.singletonList(put), + operationType); + ObHbaseRequest compact = compactTable.buildHbaseRequest(Collections.singletonList(put), + operationType); + assertFalse(legacy.getCfRows().get(0).hasCompactCells()); + assertTrue(compact.getCfRows().get(0).hasCompactCells()); + assertArrayEquals(legacy.encode(), compact.encode()); + } + + private OHTable newTable() { + return new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executorService, true); + } + + private static boolean getCompactEnabled(OHTable table) throws Exception { + Field field = OHTable.class.getDeclaredField("enablePutCompactCell"); + field.setAccessible(true); + return field.getBoolean(table); + } + + private static void setCompactEnabled(OHTable table, boolean enabled) throws Exception { + Field field = OHTable.class.getDeclaredField("enablePutCompactCell"); + field.setAccessible(true); + field.setBoolean(table, enabled); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java new file mode 100644 index 00000000..26e5c49d --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java @@ -0,0 +1,248 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.OHOperationType; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class OHTablePutDirectAutoFlushTest { + + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newSingleThreadExecutor(); + } + + @After + public void tearDown() { + executorService.shutdownNow(); + } + + @Test + public void testConfigNameAndDefault() { + assertEquals("hbase.htable.put.direct.autoflush.enabled", + HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED); + assertTrue(HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT); + } + + @Test + public void testDefaultEnablesDirectAutoFlush() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, true); + assertTrue(table.isPutDirectAutoFlushEnabled()); + assertTrue(table.isWriteBufferEmpty()); + } + + @Test + public void testConfigCanDisableDirectAutoFlush() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, true); + Field enabled = OHTable.class.getDeclaredField("enablePutDirectAutoFlush"); + enabled.setAccessible(true); + enabled.setBoolean(table, false); + assertFalse(table.isPutDirectAutoFlushEnabled()); + } + + @Test + public void testAutoFlushSinglePutBypassesMutator() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + Put put = newPut("row1", "cf", "q", "v"); + + table.put(put); + + assertEquals(1, table.directBatchCalls.get()); + assertEquals(0, table.legacyMutateFlushes.get()); + assertEquals(1, table.lastActions.size()); + assertEquals(OHOperationType.PUT, table.lastOpType); + assertTrue(table.isWriteBufferEmpty()); + assertNull(getMutator(table)); + } + + @Test + public void testAutoFlushPutListBypassesMutator() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + List puts = Arrays + .asList(newPut("r1", "cf", "q", "v1"), newPut("r2", "cf", "q", "v2")); + + table.put(puts); + + assertEquals(1, table.directBatchCalls.get()); + assertEquals(0, table.legacyMutateFlushes.get()); + assertEquals(2, table.lastActions.size()); + assertEquals(OHOperationType.PUT_LIST, table.lastOpType); + assertNull(getMutator(table)); + } + + @Test + public void testAutoFlushFalseUsesBufferedMutator() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + table.setAutoFlush(false); + Put put = newPut("row1", "cf", "q", "v"); + + table.put(put); + + assertEquals(0, table.directBatchCalls.get()); + assertFalse(table.isWriteBufferEmpty()); + assertTrue(getMutator(table) != null); + assertEquals(1, getMutator(table).getCurrentBufferSize() > 0 ? 1 : 0); + + table.flushCommits(); + assertEquals(1, table.directBatchCalls.get()); // flush -> innerBatchImpl + assertTrue(table.isWriteBufferEmpty()); + } + + @Test + public void testPendingBufferFlushedBeforeDirectPut() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + table.setAutoFlush(false); + Put buffered = newPut("old", "cf", "q", "v0"); + table.put(buffered); + assertFalse(table.isWriteBufferEmpty()); + + table.setAutoFlush(true); + Put direct = newPut("new", "cf", "q", "v1"); + table.put(direct); + + // First call from flush of pending, second from direct put. + assertEquals(2, table.directBatchCalls.get()); + assertEquals(OHOperationType.PUT, table.lastOpType); + assertEquals(1, table.lastActions.size()); + assertTrue(Bytes.equals(((Put) table.lastActions.get(0)).getRow(), Bytes.toBytes("new"))); + assertTrue(table.isWriteBufferEmpty()); + } + + @Test + public void testEmptyPutRejectedOnDirectPath() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + try { + table.put(new Put(Bytes.toBytes("row"))); + fail("empty put should fail"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("No columns")); + } + assertEquals(0, table.directBatchCalls.get()); + assertNull(getMutator(table)); + } + + @Test + public void testInvalidDirectPutDoesNotFlushPendingBuffer() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + table.setAutoFlush(false); + table.put(newPut("old", "cf", "q", "v")); + assertFalse(table.isWriteBufferEmpty()); + + table.setAutoFlush(true); + try { + table.put(new Put(Bytes.toBytes("invalid"))); + fail("empty put should fail"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("No columns")); + } + + assertEquals(0, table.directBatchCalls.get()); + assertFalse(table.isWriteBufferEmpty()); + } + + @Test + public void testDirectDisabledFallsBackToMutator() throws Exception { + CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), + mock(ObTableClient.class), executorService); + Field enabled = OHTable.class.getDeclaredField("enablePutDirectAutoFlush"); + enabled.setAccessible(true); + enabled.setBoolean(table, false); + + table.put(newPut("row1", "cf", "q", "v")); + + assertEquals(1, table.directBatchCalls.get()); // via flush path after mutate + assertTrue(getMutator(table) != null); + } + + private static Put newPut(String row, String family, String qualifier, String value) { + Put put = new Put(Bytes.toBytes(row)); + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); + return put; + } + + private static com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl getMutator(OHTable table) + throws Exception { + Field field = OHTable.class.getDeclaredField("mutator"); + field.setAccessible(true); + return (com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl) field.get(table); + } + + /** + * Captures innerBatchImpl invocations without talking to OceanBase. + * Legacy mutate+flush still ends in innerBatchImpl, so call counts distinguish + * "mutator never created" vs "buffer then flush". + */ + private static final class CapturingOHTable extends OHTable { + final AtomicInteger directBatchCalls = new AtomicInteger(); + final AtomicInteger legacyMutateFlushes = new AtomicInteger(); + volatile List lastActions = Collections.emptyList(); + volatile OHOperationType lastOpType; + + CapturingOHTable(byte[] tableName, ObTableClient client, ExecutorService pool) { + super(tableName, client, pool, true); + } + + @Override + public void innerBatchImpl(final List actions, final Object[] results, + final OHOperationType opType) throws IOException { + directBatchCalls.incrementAndGet(); + lastActions = new ArrayList(actions); + lastOpType = opType; + if (results != null) { + for (int i = 0; i < results.length; i++) { + results[i] = org.apache.hadoop.hbase.client.Result.EMPTY_RESULT; + } + } + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java new file mode 100644 index 00000000..15ce7a4d --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java @@ -0,0 +1,243 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.OHOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObHbaseCell; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObHbaseRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.util.ObBytesString; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OHTablePutSkipCellCloneTest { + + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newSingleThreadExecutor(); + } + + @After + public void tearDown() { + executorService.shutdownNow(); + } + + @Test + public void testConfigNameAndDefault() { + assertEquals("hbase.htable.put.skip.cell.clone.enabled", + HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED); + assertTrue(HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT); + } + + @Test + public void testDefaultEnablesSkipCellClone() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, true); + assertTrue(table.isPutSkipCellCloneEnabled()); + } + + @Test + public void testShareContiguousReturnsByteArray() { + byte[] q = Bytes.toBytes("qual"); + byte[] v = Bytes.toBytes("val"); + Cell bare = mock(Cell.class); + when(bare.getQualifierArray()).thenReturn(q); + when(bare.getQualifierOffset()).thenReturn(0); + when(bare.getQualifierLength()).thenReturn(q.length); + when(bare.getValueArray()).thenReturn(v); + when(bare.getValueOffset()).thenReturn(0); + when(bare.getValueLength()).thenReturn(v.length); + Object qObj = OHTable.bytesForPutCell(bare, true, true); + Object vObj = OHTable.bytesForPutCell(bare, true, false); + assertTrue(qObj instanceof byte[]); + assertTrue(vObj instanceof byte[]); + assertSame(q, qObj); + assertSame(v, vObj); + } + + @Test + public void testShareSliceReturnsObBytesStringView() { + byte[] qBuf = new byte[16]; + byte[] q = Bytes.toBytes("qual"); + System.arraycopy(q, 0, qBuf, 3, q.length); + byte[] vBuf = new byte[16]; + byte[] v = Bytes.toBytes("val"); + System.arraycopy(v, 0, vBuf, 2, v.length); + Cell sliced = new KeyValue(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("cf"), 0, 2, qBuf, 3, + q.length, System.currentTimeMillis(), KeyValue.Type.Put, vBuf, 2, v.length); + ObBytesString qView = (ObBytesString) OHTable.bytesForPutCell(sliced, true, true); + ObBytesString vView = (ObBytesString) OHTable.bytesForPutCell(sliced, true, false); + assertSame(sliced.getQualifierArray(), qView.bytes); + assertSame(sliced.getValueArray(), vView.bytes); + assertEquals(sliced.getQualifierOffset(), qView.offset); + assertEquals(sliced.getValueOffset(), vView.offset); + assertEquals(q.length, qView.length()); + assertEquals(v.length, vView.length()); + } + + @Test + public void testShareDisabledAlwaysClones() { + byte[] q = Bytes.toBytes("qual"); + byte[] v = Bytes.toBytes("val"); + Cell contiguous = contiguousCell(Bytes.toBytes("row"), Bytes.toBytes("cf"), q, v); + Object qObj = OHTable.bytesForPutCell(contiguous, false, true); + Object vObj = OHTable.bytesForPutCell(contiguous, false, false); + assertTrue(qObj instanceof byte[]); + assertTrue(vObj instanceof byte[]); + assertNotSame(q, qObj); + assertNotSame(v, vObj); + assertTrue(Bytes.equals(q, (byte[]) qObj)); + assertTrue(Bytes.equals(v, (byte[]) vObj)); + } + + @Test + public void testBuildHbaseRequestSharesViewOnAutoFlush() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, + true); + assertTrue(table.isAutoFlush()); + byte[] q = Bytes.toBytes("q1"); + byte[] v = Bytes.toBytes("v1"); + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(Bytes.toBytes("cf"), q, v); + Cell src = put.getFamilyCellMap().get(Bytes.toBytes("cf")).get(0); + + ObHbaseRequest request = table.buildHbaseRequest(Collections.singletonList(put), + OHOperationType.PUT); + ObHbaseCell cell = request.getCfRows().get(0).getCells().get(0); + Object qVal = cell.getQ().getValue(); + Object vVal = cell.getV().getValue(); + // addColumn may pack KeyValue; assert we share the Cell backing region. + if (qVal instanceof byte[]) { + assertSame(src.getQualifierArray(), qVal); + } else { + ObBytesString qView = (ObBytesString) qVal; + assertSame(src.getQualifierArray(), qView.bytes); + assertEquals(src.getQualifierOffset(), qView.offset); + assertEquals(src.getQualifierLength(), qView.length()); + } + if (vVal instanceof byte[]) { + assertSame(src.getValueArray(), vVal); + } else { + ObBytesString vView = (ObBytesString) vVal; + assertSame(src.getValueArray(), vView.bytes); + assertEquals(src.getValueOffset(), vView.offset); + assertEquals(src.getValueLength(), vView.length()); + } + assertTrue(Bytes.equals( + q, + qVal instanceof byte[] ? (byte[]) qVal : Bytes.copy(((ObBytesString) qVal).bytes, + ((ObBytesString) qVal).offset, ((ObBytesString) qVal).length()))); + assertTrue(Bytes.equals( + v, + vVal instanceof byte[] ? (byte[]) vVal : Bytes.copy(((ObBytesString) vVal).bytes, + ((ObBytesString) vVal).offset, ((ObBytesString) vVal).length()))); + } + + @Test + public void testBuildHbaseRequestClonesWhenAutoFlushOff() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, + true); + table.setAutoFlush(false); + byte[] q = Bytes.toBytes("q1"); + byte[] v = Bytes.toBytes("v1"); + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(Bytes.toBytes("cf"), q, v); + Cell src = put.getFamilyCellMap().get(Bytes.toBytes("cf")).get(0); + + ObHbaseRequest request = table.buildHbaseRequest(Collections.singletonList(put), + OHOperationType.PUT); + ObHbaseCell cell = request.getCfRows().get(0).getCells().get(0); + assertTrue(cell.getQ().getValue() instanceof byte[]); + assertTrue(cell.getV().getValue() instanceof byte[]); + assertNotSame(src.getQualifierArray(), cell.getQ().getValue()); + assertNotSame(src.getValueArray(), cell.getV().getValue()); + assertTrue(Bytes.equals(q, (byte[]) cell.getQ().getValue())); + assertTrue(Bytes.equals(v, (byte[]) cell.getV().getValue())); + } + + @Test + public void testConfigOffForcesCloneEvenWithAutoFlush() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, + true); + Field enabled = OHTable.class.getDeclaredField("enablePutSkipCellClone"); + enabled.setAccessible(true); + enabled.setBoolean(table, false); + + byte[] q = Bytes.toBytes("q1"); + byte[] v = Bytes.toBytes("v1"); + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(Bytes.toBytes("cf"), q, v); + Cell src = put.getFamilyCellMap().get(Bytes.toBytes("cf")).get(0); + + ObHbaseRequest request = table.buildHbaseRequest(Collections.singletonList(put), + OHOperationType.PUT); + ObHbaseCell cell = request.getCfRows().get(0).getCells().get(0); + assertTrue(cell.getQ().getValue() instanceof byte[]); + assertTrue(cell.getV().getValue() instanceof byte[]); + assertNotSame(src.getQualifierArray(), cell.getQ().getValue()); + assertNotSame(src.getValueArray(), cell.getV().getValue()); + } + + @Test + public void testLegacyTtlReusesSingleValueClone() throws Exception { + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, + true); + byte[] value = Bytes.toBytes("payload"); + KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("q"), + value); + long ttl = 60_000L; + ObTableOperation op = table.buildObTableOperation(kv, ObTableOperationType.INSERT_OR_UPDATE, + ttl); + assertTrue(op != null); + java.lang.reflect.Method buildMutation = OHTable.class.getDeclaredMethod("buildMutation", + KeyValue.class, ObTableOperationType.class, boolean.class, Long.class); + buildMutation.setAccessible(true); + Object mutation = buildMutation.invoke(table, kv, ObTableOperationType.INSERT_OR_UPDATE, + false, ttl); + assertTrue(mutation != null); + } + + private static Cell contiguousCell(byte[] row, byte[] family, byte[] qualifier, byte[] value) { + return new KeyValue(row, 0, row.length, family, 0, family.length, qualifier, 0, + qualifier.length, System.currentTimeMillis(), KeyValue.Type.Put, value, 0, value.length); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java new file mode 100644 index 00000000..0cc732cb --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java @@ -0,0 +1,144 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.rpc.ObTableClient; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.NavigableMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OHTablePutValidationTest { + + private static final byte[] ROW = Bytes.toBytes("row"); + private static final byte[] FAMILY = Bytes.toBytes("cf"); + + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newSingleThreadExecutor(); + } + + @After + public void tearDown() { + executorService.shutdownNow(); + } + + @Test + public void testStaticValidationDoesNotMaterializeDeprecatedFamilyMap() { + Put put = newPut(FAMILY, "q", "value"); + Cell cell = put.getFamilyCellMap().get(FAMILY).get(0); + + OHTable.validatePut(put, KeyValueUtil.length(cell)); + } + + @Test + public void testTableValidationDoesNotMaterializeDeprecatedFamilyMap() { + OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), + executorService, true); + + table.validatePutMutation(newPut(FAMILY, "q", "value")); + } + + @Test + public void testCellAtMaximumSizeIsAccepted() { + Put put = newPut(FAMILY, "q", "value"); + Cell cell = put.getFamilyCellMap().get(FAMILY).get(0); + + OHTable.validatePut(put, KeyValueUtil.length(cell)); + } + + @Test + public void testCellOverMaximumSizeIsRejected() { + Put put = newPut(FAMILY, "q", "value"); + Cell cell = put.getFamilyCellMap().get(FAMILY).get(0); + + try { + OHTable.validatePut(put, KeyValueUtil.length(cell) - 1); + fail("oversized cell should fail"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("KeyValue size too large")); + } + } + + @Test + public void testEmptyPutIsRejected() { + try { + OHTable.validatePut(new NoDeprecatedFamilyMapPut(ROW), -1); + fail("empty put should fail"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("No columns to insert")); + } + } + + @Test + public void testMultipleFamiliesUseOriginalCellMap() { + Put put = newPut(FAMILY, "q1", "value1"); + put.addColumn(Bytes.toBytes("cf2"), Bytes.toBytes("q2"), Bytes.toBytes("value2")); + + OHTable.validatePut(put, Integer.MAX_VALUE); + } + + @Test + public void testNonKeyValueCellUsesCalculatedLengthWithoutConversion() { + Put put = newPut(FAMILY, "q", "value"); + Cell cell = mock(Cell.class); + when(cell.getRowLength()).thenReturn((short) 3); + when(cell.getFamilyLength()).thenReturn((byte) 2); + when(cell.getQualifierLength()).thenReturn(1); + when(cell.getValueLength()).thenReturn(5); + when(cell.getTagsLength()).thenReturn(0); + put.getFamilyCellMap().get(FAMILY).set(0, cell); + + OHTable.validatePut(put, KeyValueUtil.length(cell)); + } + + private static Put newPut(byte[] family, String qualifier, String value) { + Put put = new NoDeprecatedFamilyMapPut(ROW); + put.addColumn(family, Bytes.toBytes(qualifier), Bytes.toBytes(value)); + return put; + } + + private static final class NoDeprecatedFamilyMapPut extends Put { + + NoDeprecatedFamilyMapPut(byte[] row) { + super(row); + } + + @Override + @Deprecated + public NavigableMap> getFamilyMap() { + throw new AssertionError("Put validation must not materialize getFamilyMap()"); + } + } +} From 1b89c631684f9d8a7fabd5b56efcf890810c911f Mon Sep 17 00:00:00 2001 From: "linguantian.lgt" Date: Wed, 12 Aug 2026 16:51:40 +0800 Subject: [PATCH 3/4] perf: optimize HBase batch get result handling Use a dedicated pure-Get batch result loop and preallocate result containers. Consume compact K/Q/T/V batches directly without rebuilding results from per-field ObObj values. Enable the compact Batch Get decoder by default while retaining an explicit configuration fallback. --- .../com/alipay/oceanbase/hbase/OHTable.java | 119 ++++++++++++--- .../hbase/constants/OHConstants.java | 5 + .../hbase/OHTableBatchGetResultTest.java | 141 ++++++++++++++++++ 3 files changed, 245 insertions(+), 20 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 484e9958..3ee53d9d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -186,6 +186,9 @@ public class OHTable implements HTableInterface { /** Use compact Q/T/V/(TTL) arrays for Put V2 request encoding. */ private boolean enablePutCompactCell; + /** Decode Batch Get K/Q/T/V results into compact arrays instead of per-field ObObj. */ + private boolean enableBatchGetCompactDecoder; + // i.e., doPut checks the writebuffer every X Puts. /** @@ -527,6 +530,9 @@ private void finishSetUp() { HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED, HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT); this.enablePutCompactCell = this.configuration.getBoolean( HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED, HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT); + this.enableBatchGetCompactDecoder = this.configuration.getBoolean( + HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_ENABLED, + HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT); } public static OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, @@ -656,7 +662,7 @@ public boolean[] existsAll(List gets) throws IOException { @Override boolean[] execute() throws IOException { boolean[] ret = new boolean[gets.size()]; - List newGets = new ArrayList<>(); + List newGets = new ArrayList<>(gets.size()); // if just checkExistOnly, batch get will not return any result or row count // therefore we have to set checkExistOnly as false and so the result can be returned for (Get get : gets) { @@ -882,7 +888,15 @@ public void innerBatchImpl(final List actions, final Object[] res return; } BatchError batchError = new BatchError(); - List resultMapSingleOp = new LinkedList<>(); + boolean pureGetBatch = true; + for (Row action : actions) { + if (!(action instanceof Get)) { + pureGetBatch = false; + break; + } + } + List resultMapSingleOp = pureGetBatch ? null + : new ArrayList<>(actions.size()); String realTableName = getTargetTableName(actions); BatchOperation batch = buildBatchOperation(realTableName, actions, tableNameString.equals(realTableName), resultMapSingleOp); @@ -893,18 +907,25 @@ public void innerBatchImpl(final List actions, final Object[] res } catch (Exception e) { throw new IOException(tableNameString + " table occurred unexpected error.", e); } + List batchResults = tmpResults.getResults(); + if (pureGetBatch) { + consumePureGetBatchResults(actions, results, batchResults, batchError); + if (batchError.hasErrors()) { + throw batchError.makeException(); + } + return; + } int index = 0; for (int i = 0; i != actions.size(); ++i) { - if (tmpResults.getResults().get(index) instanceof ObTableException) { + if (batchResults.get(index) instanceof ObTableException) { if (results != null) { - results[i] = tmpResults.getResults().get(index); + results[i] = batchResults.get(index); } - batchError.add((ObTableException) tmpResults.getResults().get(index), actions.get(i), null); + batchError.add((ObTableException) batchResults.get(index), actions.get(i), null); } else if (actions.get(i) instanceof Get) { if (results != null) { - // get results have been wrapped in MutationResult, need to fetch it - if (tmpResults.getResults().get(index) instanceof MutationResult) { - MutationResult mutationResult = (MutationResult) tmpResults.getResults().get(index); + if (batchResults.get(index) instanceof MutationResult) { + MutationResult mutationResult = (MutationResult) batchResults.get(index); ObPayload innerResult = mutationResult.getResult(); if (innerResult instanceof ObTableSingleOpResult) { ObTableSingleOpResult singleOpResult = (ObTableSingleOpResult) innerResult; @@ -929,27 +950,82 @@ public void innerBatchImpl(final List actions, final Object[] res } } - private List generateGetResult(ObTableSingleOpResult getResult) throws IOException { - List cells = new ArrayList<>(); + @VisibleForTesting + void consumePureGetBatchResults(List actions, Object[] results, + List batchResults, BatchError batchError) + throws IOException { + if (batchResults.isEmpty() && actions.size() == 1) { + if (results != null) { + results[0] = Result.create(Collections. emptyList()); + } + return; + } + if (batchResults.size() != actions.size()) { + throw new ObTableUnexpectedException("Unexpected pure Get batch result count, expected=" + + actions.size() + ", actual=" + + batchResults.size()); + } + for (int i = 0; i < actions.size(); i++) { + Object batchResult = batchResults.get(i); + if (batchResult instanceof ObTableException) { + if (results != null) { + results[i] = batchResult; + } + batchError.add((ObTableException) batchResult, actions.get(i), null); + continue; + } + if (results == null) { + continue; + } + if (!(batchResult instanceof MutationResult)) { + throw new ObTableUnexpectedException("Unexpected type of result in pure Get batch"); + } + ObPayload innerResult = ((MutationResult) batchResult).getResult(); + if (!(innerResult instanceof ObTableSingleOpResult)) { + throw new ObTableUnexpectedException( + "Unexpected type of inner result in pure Get batch"); + } + results[i] = Result.create(generateGetResult((ObTableSingleOpResult) innerResult)); + } + } + + @VisibleForTesting + List generateGetResult(ObTableSingleOpResult getResult) throws IOException { ObTableSingleOpEntity singleOpEntity = getResult.getEntity(); + ObHBaseCellBatch compactBatch = singleOpEntity.getHBaseCellBatch(); + if (compactBatch != null) { + List cells = new ArrayList<>(compactBatch.size()); + try { + for (int cellIndex = 0; cellIndex < compactBatch.size(); cellIndex++) { + addResultCell(cells, compactBatch.getRowKey(cellIndex), + compactBatch.getQualifier(cellIndex), compactBatch.getTimestamp(cellIndex), + compactBatch.getValue(cellIndex), true, HConstants.EMPTY_BYTE_ARRAY); + } + } catch (Exception e) { + throw new IOException(e); + } + return cells; + } // all values queried by this get are contained in properties // qualifier in batch get result is always appended after family List propertiesValues = singleOpEntity.getPropertiesValues(); - int valueIdx = 0; - while (valueIdx < propertiesValues.size()) { - // values in propertiesValues like: [ K, Q, T, V, K, Q, T, V ... ] - // we need to retrieve K Q T V and construct them to cells: [ cell_0, cell_1, ... ] - try { + int propertyCount = propertiesValues.size(); + if ((propertyCount & 3) != 0) { + throw new IOException("Malformed Batch Get K/Q/T/V result, property count=" + + propertyCount); + } + List cells = new ArrayList<>(propertyCount / 4); + try { + for (int valueIdx = 0; valueIdx < propertyCount; valueIdx += 4) { byte[] rowKey = (byte[]) propertiesValues.get(valueIdx).getValue(); byte[] familyQualifier = (byte[]) propertiesValues.get(valueIdx + 1).getValue(); long timestamp = (Long) propertiesValues.get(valueIdx + 2).getValue(); byte[] value = (byte[]) propertiesValues.get(valueIdx + 3).getValue(); addResultCell(cells, rowKey, familyQualifier, timestamp, value, true, HConstants.EMPTY_BYTE_ARRAY); - } catch (Exception e) { - throw new IOException(e); } - valueIdx += 4; + } catch (Exception e) { + throw new IOException(e); } return cells; } @@ -1354,7 +1430,7 @@ Result[] execute() throws IOException { if (ObGlobal.isHBaseBatchGetSupport()) { // get only supported in BatchSupport version innerBatchImpl(gets, results, opType); } else { - List> futures = new LinkedList<>(); + List> futures = new ArrayList<>(gets.size()); for (int i = 0; i < gets.size(); i++) { int index = i; Future future = executePool.submit(() -> innerGetImpl(gets.get(index), opType)); // still use list type even executing gets one by one in loop @@ -2874,13 +2950,16 @@ private BatchOperation buildBatchOperation(String tableName, List throw new FeatureNotSupportedException( "not supported other type in batch yet,only support get, put and delete"); } - resultMapSingleOp.add(singleOpResultNum); + if (resultMapSingleOp != null) { + resultMapSingleOp.add(singleOpResultNum); + } } // only set weak read consistency when all operations are Get and all Get operations are weak read if (getOperationNum == actions.size() && allGetIsWeakRead) { batch.setReadConsistency(ObReadConsistency.WEAK); } batch.setEntityType(ObTableEntityType.HKV); + batch.setHBaseBatchGetCompactDecoderEnabled(enableBatchGetCompactDecoder); batch.setServerCanRetry(OHBaseFuncUtils.serverCanRetry(obTableClient)); batch.setNeedTabletId(OHBaseFuncUtils.needTabletId(obTableClient)); return batch; diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index df9ad650..cd32ad21 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -179,6 +179,9 @@ public final class OHConstants { */ public static final String HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED = "hbase.htable.scan.lightweight.result.cell.enabled"; + /** Decode LS Batch Get K/Q/T/V results directly into a compact cell batch. */ + public static final String HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_ENABLED = "hbase.htable.batch.get.compact.decoder.enabled"; + /** * When autoFlush is enabled, put(Put)/put(List) bypass BufferedMutator and call * innerBatchImpl directly. Default is true (enabled). @@ -231,6 +234,8 @@ public final class OHConstants { public static final boolean HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT = true; + public static final boolean HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT = true; + public static final boolean HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT = true; public static final boolean HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT = true; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java new file mode 100644 index 00000000..68f18905 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java @@ -0,0 +1,141 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2026 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.BatchError; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpEntity; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellBatch; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +public class OHTableBatchGetResultTest { + private ExecutorService executor; + private OHTable table; + + @Before + public void setUp() { + executor = Executors.newSingleThreadExecutor(); + table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executor, true); + } + + @After + public void tearDown() { + executor.shutdownNow(); + } + + @Test + public void convertsKqtvAndPreservesRequestOrder() throws Exception { + List actions = Arrays. asList(new Get(Bytes.toBytes("r1")), + new Get(Bytes.toBytes("r2"))); + List raw = Arrays. asList(wrappedResult("r1", "cf\0q1", 2), + wrappedResult("r2", "cf\0q2", 1)); + Object[] results = new Object[2]; + + table.consumePureGetBatchResults(actions, results, raw, new BatchError()); + + assertEquals(2, ((Result) results[0]).size()); + assertEquals(1, ((Result) results[1]).size()); + assertEquals("r1", Bytes.toString(((Result) results[0]).getRow())); + assertEquals("r2", Bytes.toString(((Result) results[1]).getRow())); + } + + @Test + public void mapsSingleMissingResultToEmptyResult() throws Exception { + Object[] results = new Object[1]; + table.consumePureGetBatchResults( + Collections. singletonList(new Get(Bytes.toBytes("missing"))), results, + Collections.emptyList(), new BatchError()); + assertTrue(((Result) results[0]).isEmpty()); + } + + @Test(expected = IOException.class) + public void rejectsMalformedKqtvResult() throws Exception { + ObTableSingleOpResult result = new ObTableSingleOpResult(); + result.setEntity(ObTableSingleOpEntity.getInstance(null, null, + new String[] { "K", "Q", "T" }, + new Object[] { Bytes.toBytes("r"), Bytes.toBytes("cf\0q"), 1L })); + table.generateGetResult(result); + } + + @Test + public void consumesCompactKqtvBatch() throws Exception { + assertTrue(HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT); + ObHBaseCellBatch batch = new ObHBaseCellBatch(2); + batch.setCell(0, Bytes.toBytes("r1"), Bytes.toBytes("cf\0q1"), 100L, Bytes.toBytes("v0")); + batch.setCell(1, Bytes.toBytes("r1"), Bytes.toBytes("cf\0q1"), 99L, Bytes.toBytes("v1")); + ObTableSingleOpEntity entity = new ObTableSingleOpEntity(); + setCompactBatch(entity, batch); + ObTableSingleOpResult result = new ObTableSingleOpResult(); + result.setEntity(entity); + + List cells = table.generateGetResult(result); + + assertEquals(2, cells.size()); + assertEquals("r1", Bytes.toString(cells.get(0).getRowArray(), cells.get(0).getRowOffset(), + cells.get(0).getRowLength())); + assertEquals(99L, cells.get(1).getTimestamp()); + } + + private static MutationResult wrappedResult(String row, String qualifier, int versions) { + String[] names = new String[versions * 4]; + Object[] values = new Object[versions * 4]; + for (int i = 0; i < versions; i++) { + int offset = i * 4; + names[offset] = "K"; + names[offset + 1] = "Q"; + names[offset + 2] = "T"; + names[offset + 3] = "V"; + values[offset] = Bytes.toBytes(row); + values[offset + 1] = Bytes.toBytes(qualifier); + values[offset + 2] = 100L - i; + values[offset + 3] = Bytes.toBytes("v" + i); + } + ObTableSingleOpResult result = new ObTableSingleOpResult(); + result.setEntity(ObTableSingleOpEntity.getInstance(null, null, names, values)); + return new MutationResult(result); + } + + private static void setCompactBatch(ObTableSingleOpEntity entity, ObHBaseCellBatch batch) + throws Exception { + java.lang.reflect.Field field = ObTableSingleOpEntity.class + .getDeclaredField("hbaseCellBatch"); + field.setAccessible(true); + field.set(entity, batch); + } +} From 5804732ebb8a0222b8dfc2ceb638c6e6002bdf99 Mon Sep 17 00:00:00 2001 From: "linguantian.lgt" Date: Mon, 17 Aug 2026 21:01:56 +0800 Subject: [PATCH 4/4] Apply HBase request-path optimizations without public switches --- .../com/alipay/oceanbase/hbase/OHTable.java | 199 +++--------------- .../hbase/constants/OHConstants.java | 46 ---- .../hbase/result/ClientStreamScanner.java | 35 +-- .../hbase/OHTableBatchGetResultTest.java | 7 +- .../hbase/OHTableCompactPutCellTest.java | 66 ++---- .../hbase/OHTableGetMaxRowResultTest.java | 82 +------- ...HTableLightweightResultCellConfigTest.java | 91 -------- .../hbase/OHTablePutDirectAutoFlushTest.java | 43 +--- .../hbase/OHTablePutSkipCellCloneTest.java | 103 ++------- .../hbase/OHTablePutValidationTest.java | 2 +- .../ClientStreamScannerCompactResultTest.java | 40 +--- 11 files changed, 79 insertions(+), 635 deletions(-) delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 3ee53d9d..77a34449 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -169,26 +169,6 @@ public class OHTable implements HTableInterface { */ private boolean enablePutOptimization; - /** - * When autoFlush is on, bypass BufferedMutator queue/flush and call innerBatchImpl - * directly. Default true; set hbase.htable.put.direct.autoflush.enabled=false to - * restore the legacy path. - */ - private boolean enablePutDirectAutoFlush; - - /** - * On sync-complete Put V2 paths (autoFlush), skip CellUtil.clone* for contiguous - * Q/V arrays. Default true; set hbase.htable.put.skip.cell.clone.enabled=false to - * always clone. - */ - private boolean enablePutSkipCellClone; - - /** Use compact Q/T/V/(TTL) arrays for Put V2 request encoding. */ - private boolean enablePutCompactCell; - - /** Decode Batch Get K/Q/T/V results into compact arrays instead of per-field ObObj. */ - private boolean enableBatchGetCompactDecoder; - // i.e., doPut checks the writebuffer every X Puts. /** @@ -225,16 +205,6 @@ public class OHTable implements HTableInterface { */ private final boolean hotKeyGetOptimizeEnableGlobal; - /** - * whether point read results use field-backed lightweight cells. - */ - private final boolean getLightweightResultCellEnabled; - - /** - * whether Scan results use field-backed lightweight cells. - */ - private final boolean scanLightweightResultCellEnabled; - /** * whether test load is enabled. * Cached at construction time to avoid repeated configuration lookups. @@ -285,12 +255,6 @@ public OHTable(Configuration configuration, String tableName) throws IOException } this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); - this.getLightweightResultCellEnabled = configuration.getBoolean( - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED, - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - this.scanLightweightResultCellEnabled = configuration.getBoolean( - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED, - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -353,12 +317,6 @@ public OHTable(Configuration configuration, final byte[] tableName, } this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); - this.getLightweightResultCellEnabled = configuration.getBoolean( - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED, - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - this.scanLightweightResultCellEnabled = configuration.getBoolean( - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED, - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -381,19 +339,6 @@ public OHTable(Configuration configuration, final byte[] tableName, @InterfaceAudience.Private public OHTable(final byte[] tableName, final ObTableClient obTableClient, final ExecutorService executePool) { - this(tableName, obTableClient, executePool, - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - } - - OHTable(final byte[] tableName, final ObTableClient obTableClient, - final ExecutorService executePool, boolean getLightweightResultCellEnabled) { - this(tableName, obTableClient, executePool, getLightweightResultCellEnabled, - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - } - - OHTable(final byte[] tableName, final ObTableClient obTableClient, - final ExecutorService executePool, boolean getLightweightResultCellEnabled, - boolean scanLightweightResultCellEnabled) { checkArgument(tableName != null, "tableNameString is blank."); checkArgument(executePool != null && !executePool.isShutdown(), "executePool is null or executePool is shutdown"); @@ -407,8 +352,6 @@ public OHTable(final byte[] tableName, final ObTableClient obTableClient, this.metrics = null; this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); - this.getLightweightResultCellEnabled = getLightweightResultCellEnabled; - this.scanLightweightResultCellEnabled = scanLightweightResultCellEnabled; this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -459,12 +402,6 @@ public OHTable(TableName tableName, Connection connection, } this.fillTimestampInClient = configuration.getBoolean(HBASE_HTABLE_AUTO_FILL_TIMESTAMP_IN_CLIENT, false); this.hotKeyGetOptimizeEnableGlobal = configuration.getBoolean(HBASE_HTABLE_HOTKEY_GET_OPTIMIZE_ENABLE_GLOBAL, false); - this.getLightweightResultCellEnabled = configuration.getBoolean( - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED, - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - this.scanLightweightResultCellEnabled = configuration.getBoolean( - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED, - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); this.testLoadEnable = configuration.getBoolean(HBASE_HTABLE_TEST_LOAD_ENABLE, false); this.testLoadSuffix = testLoadEnable ? configuration.get(HBASE_HTABLE_TEST_LOAD_SUFFIX, DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX) @@ -524,15 +461,6 @@ private void finishSetUp() { WRITE_BUFFER_SIZE_DEFAULT); this.enablePutOptimization = this.configuration.getBoolean(HBASE_HTABLE_USE_PUT_OPTIMIZATION, HBASE_HTABLE_USE_PUT_OPTIMIZATION_DEFAULT); - this.enablePutDirectAutoFlush = this.configuration.getBoolean( - HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED, HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT); - this.enablePutSkipCellClone = this.configuration.getBoolean( - HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED, HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT); - this.enablePutCompactCell = this.configuration.getBoolean( - HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED, HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT); - this.enableBatchGetCompactDecoder = this.configuration.getBoolean( - HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_ENABLED, - HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT); } public static OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, @@ -1123,19 +1051,10 @@ private void addRowToResultCells(List cells, List row, byte[] rowKe private void addResultCell(List cells, byte[] rowKey, byte[] qualifier, long timestamp, byte[] value, boolean isTableGroup, byte[] family) throws Exception { - if (getLightweightResultCellEnabled) { - if (isTableGroup) { - cells.add(OHBaseResultCell.createTableGroup(rowKey, qualifier, timestamp, value)); - } else { - cells.add(OHBaseResultCell.create(rowKey, family, qualifier, timestamp, value)); - } - return; - } if (isTableGroup) { - cells - .add(OHBaseFuncUtils.createTableGroupKeyValue(rowKey, qualifier, timestamp, value)); + cells.add(OHBaseResultCell.createTableGroup(rowKey, qualifier, timestamp, value)); } else { - cells.add(new KeyValue(rowKey, family, qualifier, timestamp, value)); + cells.add(OHBaseResultCell.create(rowKey, family, qualifier, timestamp, value)); } } @@ -1154,32 +1073,24 @@ private boolean fillPointGetFromResult(AbstractQueryStreamResult clientQueryStre return true; } - if (clientQueryStreamResult.isCurrentHBaseCell()) { - ObHBaseCellRow hbaseRow = clientQueryStreamResult.drainCurrentHBaseRow(); - byte[] actualRowKey = hbaseRow.getRowKey(); - if (canonicalRowKey == null) { - if (!Bytes.equals(expectedRowKey, actualRowKey)) { - throw new ObTableUnexpectedException( - "point Get returned an unexpected rowkey, expected length=" - + expectedRowKey.length + ", actual length=" - + actualRowKey.length); - } - canonicalRowKey = actualRowKey; - } - addCompactRowToResultCells(cells, hbaseRow, canonicalRowKey, isTableGroup, - family); - continue; - } - byte[] actualRowKey; byte[] qualifier; long timestamp; byte[] value; - List row = clientQueryStreamResult.getRow(); - actualRowKey = (byte[]) row.get(0).getValue(); - qualifier = (byte[]) row.get(1).getValue(); - timestamp = (Long) row.get(2).getValue(); - value = (byte[]) row.get(3).getValue(); + if (clientQueryStreamResult.isCurrentHBaseCell()) { + ObHBaseCellBatch batch = clientQueryStreamResult.getCurrentHBaseCellBatch(); + int index = clientQueryStreamResult.getCurrentHBaseCellIndex(); + actualRowKey = batch.getRowKey(index); + qualifier = batch.getQualifier(index); + timestamp = batch.getTimestamp(index); + value = batch.getValue(index); + } else { + List row = clientQueryStreamResult.getRow(); + actualRowKey = (byte[]) row.get(0).getValue(); + qualifier = (byte[]) row.get(1).getValue(); + timestamp = (Long) row.get(2).getValue(); + value = (byte[]) row.get(3).getValue(); + } if (canonicalRowKey == null) { if (!Bytes.equals(expectedRowKey, actualRowKey)) { throw new ObTableUnexpectedException( @@ -1194,19 +1105,6 @@ private boolean fillPointGetFromResult(AbstractQueryStreamResult clientQueryStre return canonicalRowKey != null; } - private void addCompactRowToResultCells(List cells, ObHBaseCellRow hbaseRow, - byte[] canonicalRowKey, boolean isTableGroup, - byte[] family) throws Exception { - for (int sliceIndex = 0; sliceIndex < hbaseRow.getSliceCount(); sliceIndex++) { - ObHBaseCellBatch batch = hbaseRow.getBatch(sliceIndex); - int toIndex = hbaseRow.getToIndex(sliceIndex); - for (int index = hbaseRow.getFromIndex(sliceIndex); index < toIndex; index++) { - addResultCell(cells, canonicalRowKey, batch.getQualifier(index), - batch.getTimestamp(index), batch.getValue(index), isTableGroup, family); - } - } - } - /** * closestRowBefore may collect candidates from multiple partitions. Keep only cells belonging * to the greatest returned rowkey without cloning the rowkey from a temporary result cell. @@ -1256,12 +1154,8 @@ private boolean getMaxRowFromResult(AbstractQueryStreamResult clientQueryStreamR return currentMaxRowKey != null; } - @SuppressWarnings("unchecked") private Result createGetResult(List cells) { - if (getLightweightResultCellEnabled) { - return Result.create(cells); - } - return new Result((List) (List) cells); + return Result.create(cells); } private void addQueryResultToKeyValueList(ObTableQueryResult queryResult, @@ -1504,8 +1398,7 @@ public ResultScanner call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); return new ClientStreamScanner(clientQueryAsyncStreamResult, - tableNameString, family, true, metrics, - scanLightweightResultCellEnabled); + tableNameString, family, true, metrics); } else { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { @@ -1531,8 +1424,7 @@ public ResultScanner call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); return new ClientStreamScanner(clientQueryAsyncStreamResult, - tableNameString, family, false, metrics, - scanLightweightResultCellEnabled); + tableNameString, family, false, metrics); } } } catch (Exception e) { @@ -1592,8 +1484,7 @@ public List call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); ClientStreamScanner clientScanner = new ClientStreamScanner( - clientQueryAsyncStreamResult, tableNameString, family, true, metrics, - scanLightweightResultCellEnabled); + clientQueryAsyncStreamResult, tableNameString, family, true, metrics); resultScanners.add(clientScanner); } return resultScanners; @@ -1618,7 +1509,8 @@ public List call() throws IOException { clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); ClientStreamScanner clientScanner = new ClientStreamScanner( - clientQueryAsyncStreamResult, tableNameString, family, false, metrics, scanLightweightResultCellEnabled); + clientQueryAsyncStreamResult, tableNameString, family, false, + metrics); resultScanners.add(clientScanner); } return resultScanners; @@ -1699,7 +1591,7 @@ public Void execute() throws IOException { */ private boolean tryDirectAutoFlushPuts(List puts, OHOperationType opType) throws IOException { - if (!enablePutDirectAutoFlush || !autoFlush) { + if (!autoFlush) { return false; } for (Row row : puts) { @@ -1726,16 +1618,6 @@ public boolean isWriteBufferEmpty() { return mutator == null || mutator.isBufferEmpty(); } - @VisibleForTesting - public boolean isPutDirectAutoFlushEnabled() { - return enablePutDirectAutoFlush; - } - - @VisibleForTesting - public boolean isPutSkipCellCloneEnabled() { - return enablePutSkipCellClone; - } - /** * Put validation shared by the autoFlush direct path and BufferedMutator. */ @@ -2959,7 +2841,6 @@ private BatchOperation buildBatchOperation(String tableName, List batch.setReadConsistency(ObReadConsistency.WEAK); } batch.setEntityType(ObTableEntityType.HKV); - batch.setHBaseBatchGetCompactDecoderEnabled(enableBatchGetCompactDecoder); batch.setServerCanRetry(OHBaseFuncUtils.serverCanRetry(obTableClient)); batch.setNeedTabletId(OHBaseFuncUtils.needTabletId(obTableClient)); return batch; @@ -2983,15 +2864,9 @@ ObHbaseRequest buildHbaseRequest(List actions, OHOperationType hb if (put.isEmpty()) { throw new IllegalArgumentException("No columns to put for item"); } - boolean isCellTTL = false; long ttl = put.getTTL(); - if (ttl != Long.MAX_VALUE) { - isCellTTL = true; - } - ObObj ttlObj = isCellTTL && !enablePutCompactCell ? ObObj.hbasePutInt64(ttl) - : null; keys.add(ObObj.hbasePutVarchar(put.getRow())); - boolean shareCellBytes = enablePutSkipCellClone && autoFlush; + boolean shareCellBytes = autoFlush; for (Map.Entry> entry : put.getFamilyCellMap().entrySet()) { String family = Bytes.toString(entry.getKey()); ObHbaseCfRows sameCfRows = cfRowsMap.get(family); @@ -3004,28 +2879,10 @@ ObHbaseRequest buildHbaseRequest(List actions, OHOperationType hb cfRowsArray.add(sameCfRows); } List keyValueList = entry.getValue(); - if (enablePutCompactCell) { - sameCfRows.reserveAdditionalCompactCells(keyValueList.size()); - sameCfRows.beginCompactKeyCells(keyIndex, keyValueList.size(), ttl); - for (Cell kv : keyValueList) { - appendCompactPutCell(sameCfRows, kv, shareCellBytes); - } - } else { - sameCfRows.reserveAdditionalCells(keyValueList.size()); - sameCfRows.beginKeyCells(keyIndex, keyValueList.size()); - for (Cell kv : keyValueList) { - ObHbaseCell cell = new ObHbaseCell(isCellTTL); - cell.setQ(ObObj.hbasePutVarchar(bytesForPutCell(kv, shareCellBytes, - true))); - cell.setT(ObObj.hbasePutInt64(-getEffectiveTimestampForWrite(kv - .getTimestamp()))); - cell.setV(ObObj.hbasePutVarchar(bytesForPutCell(kv, shareCellBytes, - false))); - if (isCellTTL) { - cell.setTTL(ttlObj); - } - sameCfRows.appendCell(cell); - } + sameCfRows.reserveAdditionalCompactCells(keyValueList.size()); + sameCfRows.beginCompactKeyCells(keyIndex, keyValueList.size(), ttl); + for (Cell kv : keyValueList) { + appendCompactPutCell(sameCfRows, kv, shareCellBytes); } } } else { diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index cd32ad21..72fb5b8b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -167,40 +167,6 @@ public final class OHConstants { */ public static final String HBASE_HTABLE_USE_PUT_OPTIMIZATION = "hbase.htable.use.put.optimization"; - /** - * use to specify whether point read results use field-backed lightweight cells. - * Default is true (enabled). - */ - public static final String HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED = "hbase.htable.get.lightweight.result.cell.enabled"; - - /** - * use to specify whether Scan results use field-backed lightweight cells. - * Default is true (enabled). - */ - public static final String HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED = "hbase.htable.scan.lightweight.result.cell.enabled"; - - /** Decode LS Batch Get K/Q/T/V results directly into a compact cell batch. */ - public static final String HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_ENABLED = "hbase.htable.batch.get.compact.decoder.enabled"; - - /** - * When autoFlush is enabled, put(Put)/put(List) bypass BufferedMutator and call - * innerBatchImpl directly. Default is true (enabled). - */ - public static final String HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED = "hbase.htable.put.direct.autoflush.enabled"; - - /** - * When building Put V2 requests on a sync-complete path (autoFlush / direct put), - * skip CellUtil.clone* for contiguous qualifier/value byte arrays and share the - * Cell backing array until encode. Default is true (enabled). - */ - public static final String HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED = "hbase.htable.put.skip.cell.clone.enabled"; - - /** - * Store Put V2 cells in compact parallel arrays and encode Q/T/V/(TTL) directly. - * Default is true (enabled). - */ - public static final String HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED = "hbase.htable.put.compact.cell.enabled"; - /*-------------------------------------------------------------------------------------------------------------*/ /** @@ -230,16 +196,4 @@ public final class OHConstants { public static final boolean HBASE_HTABLE_USE_PUT_OPTIMIZATION_DEFAULT = true; - public static final boolean HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT = true; - - public static final boolean HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT = true; - - public static final boolean HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT = true; - - public static final boolean HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT = true; - - public static final boolean HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT = true; - - public static final boolean HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT = true; - } diff --git a/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java b/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java index def6c644..aee59af3 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java +++ b/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.*; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.LCD; @InterfaceAudience.Private @@ -61,42 +60,24 @@ public class ClientStreamScanner extends AbstractClientScanner { private boolean isTableGroup = false; - private final boolean lightweightResultCellEnabled; - private OHMetrics metrics; public ClientStreamScanner(ObTableClientQueryStreamResult streamResult, String tableName, byte[] family, boolean isTableGroup, OHMetrics metrics) { - this(streamResult, tableName, family, isTableGroup, metrics, - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - } - - public ClientStreamScanner(ObTableClientQueryStreamResult streamResult, String tableName, - byte[] family, boolean isTableGroup, OHMetrics metrics, - boolean lightweightResultCellEnabled) { this.streamResult = streamResult; this.tableName = tableName; this.family = family; this.isTableGroup = isTableGroup; this.metrics = metrics; - this.lightweightResultCellEnabled = lightweightResultCellEnabled; } public ClientStreamScanner(ObTableClientQueryAsyncStreamResult streamResult, String tableName, byte[] family, boolean isTableGroup, OHMetrics metrics) { - this(streamResult, tableName, family, isTableGroup, metrics, - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - } - - public ClientStreamScanner(ObTableClientQueryAsyncStreamResult streamResult, String tableName, - byte[] family, boolean isTableGroup, OHMetrics metrics, - boolean lightweightResultCellEnabled) { this.streamResult = streamResult; this.tableName = tableName; this.family = family; this.isTableGroup = isTableGroup; this.metrics = metrics; - this.lightweightResultCellEnabled = lightweightResultCellEnabled; } @Override @@ -143,14 +124,6 @@ private Result buildCompactResult(ObHBaseCellRow hbaseRow) { private void addCompactResultCell(List cells, byte[] rowKey, byte[] qualifier, long timestamp, byte[] value) { - if (lightweightResultCellEnabled) { - if (isTableGroup) { - cells.add(OHBaseResultCell.createTableGroup(rowKey, qualifier, timestamp, value)); - } else { - cells.add(OHBaseResultCell.create(rowKey, family, qualifier, timestamp, value)); - } - return; - } if (isTableGroup) { cells .add(OHBaseFuncUtils.createTableGroupKeyValue(rowKey, qualifier, timestamp, value)); @@ -159,12 +132,10 @@ private void addCompactResultCell(List cells, byte[] rowKey, byte[] qualif } } - @SuppressWarnings("unchecked") private Result createCompactResult(List cells) { - if (lightweightResultCellEnabled) { - return Result.create(cells); - } - return new Result((List) (List) cells); + @SuppressWarnings("unchecked") + List keyValues = (List) (List) cells; + return new Result(keyValues); } private Result buildLegacyResult(List startRow) throws Exception { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java index 68f18905..39901a93 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableBatchGetResultTest.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.BatchError; +import com.alipay.oceanbase.hbase.result.OHBaseResultCell; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpEntity; @@ -39,7 +40,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -51,7 +51,7 @@ public class OHTableBatchGetResultTest { @Before public void setUp() { executor = Executors.newSingleThreadExecutor(); - table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executor, true); + table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executor); } @After @@ -73,6 +73,7 @@ public void convertsKqtvAndPreservesRequestOrder() throws Exception { assertEquals(1, ((Result) results[1]).size()); assertEquals("r1", Bytes.toString(((Result) results[0]).getRow())); assertEquals("r2", Bytes.toString(((Result) results[1]).getRow())); + assertTrue(((Result) results[0]).rawCells()[0] instanceof OHBaseResultCell); } @Test @@ -95,7 +96,6 @@ public void rejectsMalformedKqtvResult() throws Exception { @Test public void consumesCompactKqtvBatch() throws Exception { - assertTrue(HBASE_HTABLE_BATCH_GET_COMPACT_DECODER_DEFAULT); ObHBaseCellBatch batch = new ObHBaseCellBatch(2); batch.setCell(0, Bytes.toBytes("r1"), Bytes.toBytes("cf\0q1"), 100L, Bytes.toBytes("v0")); batch.setCell(1, Bytes.toBytes("r1"), Bytes.toBytes("cf\0q1"), 99L, Bytes.toBytes("v1")); @@ -110,6 +110,7 @@ public void consumesCompactKqtvBatch() throws Exception { assertEquals("r1", Bytes.toString(cells.get(0).getRowArray(), cells.get(0).getRowOffset(), cells.get(0).getRowLength())); assertEquals(99L, cells.get(1).getTimestamp()); + assertTrue(cells.get(0) instanceof OHBaseResultCell); } private static MutationResult wrappedResult(String row, String qualifier, int versions) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java index 68fc1a94..08d90494 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableCompactPutCellTest.java @@ -27,17 +27,13 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collections; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -55,24 +51,22 @@ public void tearDown() { } @Test - public void testConfigNameAndDefault() throws Exception { - assertEquals("hbase.htable.put.compact.cell.enabled", HBASE_HTABLE_PUT_COMPACT_CELL_ENABLED); - assertTrue(HBASE_HTABLE_PUT_COMPACT_CELL_DEFAULT); - OHTable table = newTable(); - assertTrue(getCompactEnabled(table)); - } - - @Test - public void testCompactRequestMatchesLegacyWithoutTtl() throws Exception { + public void testPutAlwaysUsesCompactCellsWithoutTtl() throws Exception { Put put = new Put(Bytes.toBytes("row")); put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), 1001L, Bytes.toBytes("value1")); put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("qualifier-2"), 1002L, Bytes.toBytes("value-2")); - assertLegacyAndCompactEqual(put, OHOperationType.PUT); + + ObHbaseRequest request = newTable().buildHbaseRequest(Collections.singletonList(put), + OHOperationType.PUT); + + assertEquals(1, request.getCfRows().size()); + assertTrue(request.getCfRows().get(0).hasCompactCells()); + assertTrue(request.encode().length > 0); } @Test - public void testCompactRequestMatchesLegacyWithTtlAndMultipleRows() throws Exception { + public void testPutAlwaysUsesCompactCellsWithTtlAndMultipleRows() throws Exception { Put first = new Put(Bytes.toBytes("row-1")); first.setTTL(60000L); first.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q1"), 2001L, Bytes.toBytes("value-1")); @@ -81,22 +75,14 @@ public void testCompactRequestMatchesLegacyWithTtlAndMultipleRows() throws Excep Put second = new Put(Bytes.toBytes("row-2")); second.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q4"), 3001L, Bytes.toBytes("value-4")); - OHTable legacyTable = newTable(); - OHTable compactTable = newTable(); - setCompactEnabled(legacyTable, false); - setCompactEnabled(compactTable, true); - ObHbaseRequest legacy = legacyTable.buildHbaseRequest(Arrays.asList(first, second), - OHOperationType.PUT_LIST); - ObHbaseRequest compact = compactTable.buildHbaseRequest(Arrays.asList(first, second), + ObHbaseRequest compact = newTable().buildHbaseRequest(Arrays.asList(first, second), OHOperationType.PUT_LIST); - assertEquals(2, legacy.getCfRows().size()); assertEquals(2, compact.getCfRows().size()); for (int i = 0; i < compact.getCfRows().size(); i++) { - assertFalse(legacy.getCfRows().get(i).hasCompactCells()); assertTrue(compact.getCfRows().get(i).hasCompactCells()); } - assertArrayEquals(legacy.encode(), compact.encode()); + assertTrue(compact.encode().length > 0); } @Test @@ -108,7 +94,6 @@ public void testBufferedCompactRequestOwnsQualifierAndValueBytes() throws Except OHTable compactTable = newTable(); compactTable.setAutoFlush(false); - setCompactEnabled(compactTable, true); ObHbaseRequest compact = compactTable.buildHbaseRequest(Collections.singletonList(put), OHOperationType.PUT); byte[] encodedBeforeMutation = compact.encode(); @@ -122,34 +107,7 @@ public void testBufferedCompactRequestOwnsQualifierAndValueBytes() throws Except assertArrayEquals(encodedBeforeMutation, compact.encode()); } - private void assertLegacyAndCompactEqual(Put put, OHOperationType operationType) - throws Exception { - OHTable legacyTable = newTable(); - OHTable compactTable = newTable(); - setCompactEnabled(legacyTable, false); - setCompactEnabled(compactTable, true); - ObHbaseRequest legacy = legacyTable.buildHbaseRequest(Collections.singletonList(put), - operationType); - ObHbaseRequest compact = compactTable.buildHbaseRequest(Collections.singletonList(put), - operationType); - assertFalse(legacy.getCfRows().get(0).hasCompactCells()); - assertTrue(compact.getCfRows().get(0).hasCompactCells()); - assertArrayEquals(legacy.encode(), compact.encode()); - } - private OHTable newTable() { - return new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executorService, true); - } - - private static boolean getCompactEnabled(OHTable table) throws Exception { - Field field = OHTable.class.getDeclaredField("enablePutCompactCell"); - field.setAccessible(true); - return field.getBoolean(table); - } - - private static void setCompactEnabled(OHTable table, boolean enabled) throws Exception { - Field field = OHTable.class.getDeclaredField("enablePutCompactCell"); - field.setAccessible(true); - field.setBoolean(table, enabled); + return new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executorService); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java index 6fe03417..f09a420d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableGetMaxRowResultTest.java @@ -28,7 +28,6 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellBatch; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellRow; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; import com.alipay.oceanbase.hbase.result.OHBaseResultCell; import io.netty.buffer.ByteBuf; @@ -40,7 +39,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -61,7 +59,7 @@ public class OHTableGetMaxRowResultTest { @Before public void setUp() { executorService = Executors.newSingleThreadExecutor(); - table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executorService, true); + table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), executorService); } @After @@ -103,11 +101,8 @@ public void testPointGetRejectsUnexpectedFirstRowKey() throws Exception { @Test public void testPointGetConsumesCompactBatchWithoutMaterializingRows() throws Exception { byte[] expectedRowKey = Bytes.toBytes("row-1"); - ObHBaseCellBatch firstBatch = compactBatch(row("row-1", "q1", 3L, "v1")); - ObHBaseCellBatch secondBatch = compactBatch(row("unexpected-later-row", "q2", 2L, - "v2")); - AbstractQueryStreamResult streamResult = compactPointGetStream(compactRow(firstBatch), - compactRow(secondBatch)); + AbstractQueryStreamResult streamResult = compactStream(compactBatch( + row("row-1", "q1", 3L, "v1"), row("unexpected-later-row", "q2", 2L, "v2"))); List keyValues = new ArrayList(); boolean found = invokeFillPointGet(streamResult, keyValues, false, Bytes.toBytes("f"), @@ -118,49 +113,7 @@ public void testPointGetConsumesCompactBatchWithoutMaterializingRows() throws Ex assertArrayEquals(expectedRowKey, keyValues.get(0).getRow()); assertArrayEquals(expectedRowKey, keyValues.get(1).getRow()); assertArrayEquals(Bytes.toBytes("q2"), keyValues.get(1).getQualifier()); - verify(streamResult, times(3)).next(); - verify(streamResult, times(2)).drainCurrentHBaseRow(); verify(streamResult, never()).getRow(); - verify(streamResult, never()).getCurrentHBaseCellBatch(); - verify(streamResult, never()).getCurrentHBaseCellIndex(); - } - - @Test - public void testPointGetDrainsSameRowAcrossCachedBatchesOnce() throws Exception { - byte[] expectedRowKey = Bytes.toBytes("row-1"); - ObHBaseCellBatch firstBatch = compactBatch(row("row-1", "q1", 4L, "v1"), - row("row-1", "q2", 3L, "v2")); - ObHBaseCellBatch secondBatch = compactBatch(row("row-1", "q3", 2L, "v3"), - row("row-1", "q4", 1L, "v4")); - AbstractQueryStreamResult streamResult = compactPointGetStream(compactRow(firstBatch, - secondBatch)); - List cells = new ArrayList(); - - boolean found = invokeFillPointGet(streamResult, cells, false, Bytes.toBytes("f"), - expectedRowKey, false); - - assertTrue(found); - assertEquals(4, cells.size()); - assertArrayEquals(Bytes.toBytes("q1"), cells.get(0).getQualifier()); - assertArrayEquals(Bytes.toBytes("q4"), cells.get(3).getQualifier()); - verify(streamResult, times(2)).next(); - verify(streamResult, times(1)).drainCurrentHBaseRow(); - verify(streamResult, never()).getRow(); - } - - @Test - public void testDisabledLightweightCellUsesKeyValue() throws Exception { - OHTable fallbackTable = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, false); - AbstractQueryStreamResult streamResult = stream(row("row-1", "q1", 1L, "v1")); - List cells = new ArrayList(); - - boolean found = invokeFillPointGet(fallbackTable, streamResult, cells, false, - Bytes.toBytes("f"), Bytes.toBytes("row-1"), false); - - assertTrue(found); - assertEquals(1, cells.size()); - assertTrue(cells.get(0) instanceof KeyValue); } @Test @@ -174,7 +127,6 @@ public void testPointGetExistenceOnlyDoesNotCreateKeyValue() throws Exception { assertTrue(found); assertTrue(keyValues.isEmpty()); - verify(streamResult, never()).drainCurrentHBaseRow(); verify(streamResult, never()).getRow(); } @@ -232,7 +184,6 @@ public void testClosestRowBeforeConsumesCompactBatch() throws Exception { assertEquals(2, keyValues.size()); assertArrayEquals(Bytes.toBytes("row-3"), keyValues.get(0).getRow()); assertArrayEquals(Bytes.toBytes("q2"), keyValues.get(1).getQualifier()); - verify(streamResult, never()).drainCurrentHBaseRow(); verify(streamResult, never()).getRow(); } @@ -405,33 +356,6 @@ private static AbstractQueryStreamResult compactStream(ObHBaseCellBatch batch) return streamResult; } - private static AbstractQueryStreamResult compactPointGetStream(ObHBaseCellRow... rows) - throws Exception { - AbstractQueryStreamResult streamResult = mock(AbstractQueryStreamResult.class); - Boolean[] remaining = new Boolean[Math.max(0, rows.length - 1)]; - Arrays.fill(remaining, true); - when(streamResult.next()).thenReturn(true, remaining).thenReturn(false); - when(streamResult.isCurrentHBaseCell()).thenReturn(true); - when(streamResult.drainCurrentHBaseRow()).thenReturn(rows[0], Arrays.copyOfRange(rows, 1, - rows.length)); - return streamResult; - } - - private static ObHBaseCellRow compactRow(ObHBaseCellBatch... batches) throws Exception { - assertTrue(batches.length > 0); - Constructor constructor = ObHBaseCellRow.class - .getDeclaredConstructor(byte[].class); - constructor.setAccessible(true); - ObHBaseCellRow row = constructor.newInstance(batches[0].getRowKey(0)); - Method addSlice = ObHBaseCellRow.class.getDeclaredMethod("addSlice", - ObHBaseCellBatch.class, int.class, int.class); - addSlice.setAccessible(true); - for (ObHBaseCellBatch batch : batches) { - addSlice.invoke(row, batch, 0, batch.size()); - } - return row; - } - @SafeVarargs private static ObHBaseCellBatch compactBatch(List... rows) { return compactQueryResult(rows).getHBaseCellBatch(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java deleted file mode 100644 index 1c656b44..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableLightweightResultCellConfigTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2022 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import com.alipay.oceanbase.rpc.ObTableClient; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -public class OHTableLightweightResultCellConfigTest { - - private ExecutorService executorService; - - @Before - public void setUp() { - executorService = Executors.newSingleThreadExecutor(); - } - - @After - public void tearDown() { - executorService.shutdownNow(); - } - - @Test - public void testPointAndScanLightweightCellSwitchesAreIndependent() throws Exception { - OHTable pointOnly = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, true, false); - assertTrue(getBooleanField(pointOnly, "getLightweightResultCellEnabled")); - assertFalse(getBooleanField(pointOnly, "scanLightweightResultCellEnabled")); - - OHTable scanOnly = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, false, true); - assertFalse(getBooleanField(scanOnly, "getLightweightResultCellEnabled")); - assertTrue(getBooleanField(scanOnly, "scanLightweightResultCellEnabled")); - } - - @Test - public void testInternalConstructorUsesScanLightweightCellDefault() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, true); - - assertTrue(getBooleanField(table, "getLightweightResultCellEnabled")); - assertTrue(getBooleanField(table, "scanLightweightResultCellEnabled")); - } - - @Test - public void testLightweightCellConfigurationNamesAndDefaults() { - assertEquals("hbase.htable.get.lightweight.result.cell.enabled", - HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_ENABLED); - assertEquals("hbase.htable.scan.lightweight.result.cell.enabled", - HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_ENABLED); - assertTrue(HBASE_HTABLE_GET_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - assertTrue(HBASE_HTABLE_SCAN_LIGHTWEIGHT_RESULT_CELL_DEFAULT); - } - - private static boolean getBooleanField(OHTable table, String fieldName) throws Exception { - Field field = OHTable.class.getDeclaredField(fieldName); - field.setAccessible(true); - return field.getBoolean(table); - } - -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java index 26e5c49d..97c6e854 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutDirectAutoFlushTest.java @@ -36,8 +36,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -59,31 +57,6 @@ public void tearDown() { executorService.shutdownNow(); } - @Test - public void testConfigNameAndDefault() { - assertEquals("hbase.htable.put.direct.autoflush.enabled", - HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_ENABLED); - assertTrue(HBASE_HTABLE_PUT_DIRECT_AUTOFLUSH_DEFAULT); - } - - @Test - public void testDefaultEnablesDirectAutoFlush() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, true); - assertTrue(table.isPutDirectAutoFlushEnabled()); - assertTrue(table.isWriteBufferEmpty()); - } - - @Test - public void testConfigCanDisableDirectAutoFlush() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, true); - Field enabled = OHTable.class.getDeclaredField("enablePutDirectAutoFlush"); - enabled.setAccessible(true); - enabled.setBoolean(table, false); - assertFalse(table.isPutDirectAutoFlushEnabled()); - } - @Test public void testAutoFlushSinglePutBypassesMutator() throws Exception { CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), @@ -190,20 +163,6 @@ public void testInvalidDirectPutDoesNotFlushPendingBuffer() throws Exception { assertFalse(table.isWriteBufferEmpty()); } - @Test - public void testDirectDisabledFallsBackToMutator() throws Exception { - CapturingOHTable table = new CapturingOHTable(Bytes.toBytes("t"), - mock(ObTableClient.class), executorService); - Field enabled = OHTable.class.getDeclaredField("enablePutDirectAutoFlush"); - enabled.setAccessible(true); - enabled.setBoolean(table, false); - - table.put(newPut("row1", "cf", "q", "v")); - - assertEquals(1, table.directBatchCalls.get()); // via flush path after mutate - assertTrue(getMutator(table) != null); - } - private static Put newPut(String row, String family, String qualifier, String value) { Put put = new Put(Bytes.toBytes(row)); put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); @@ -229,7 +188,7 @@ private static final class CapturingOHTable extends OHTable { volatile OHOperationType lastOpType; CapturingOHTable(byte[] tableName, ObTableClient client, ExecutorService pool) { - super(tableName, client, pool, true); + super(tableName, client, pool); } @Override diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java index 15ce7a4d..9c38676a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutSkipCellCloneTest.java @@ -20,6 +20,7 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.OHOperationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObHbaseCell; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObHbaseCfRows; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObHbaseRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; @@ -37,8 +38,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT; -import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; @@ -61,20 +60,6 @@ public void tearDown() { executorService.shutdownNow(); } - @Test - public void testConfigNameAndDefault() { - assertEquals("hbase.htable.put.skip.cell.clone.enabled", - HBASE_HTABLE_PUT_SKIP_CELL_CLONE_ENABLED); - assertTrue(HBASE_HTABLE_PUT_SKIP_CELL_CLONE_DEFAULT); - } - - @Test - public void testDefaultEnablesSkipCellClone() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, true); - assertTrue(table.isPutSkipCellCloneEnabled()); - } - @Test public void testShareContiguousReturnsByteArray() { byte[] q = Bytes.toBytes("qual"); @@ -131,8 +116,7 @@ public void testShareDisabledAlwaysClones() { @Test public void testBuildHbaseRequestSharesViewOnAutoFlush() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, - true); + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService); assertTrue(table.isAutoFlush()); byte[] q = Bytes.toBytes("q1"); byte[] v = Bytes.toBytes("v1"); @@ -142,40 +126,15 @@ public void testBuildHbaseRequestSharesViewOnAutoFlush() throws Exception { ObHbaseRequest request = table.buildHbaseRequest(Collections.singletonList(put), OHOperationType.PUT); - ObHbaseCell cell = request.getCfRows().get(0).getCells().get(0); - Object qVal = cell.getQ().getValue(); - Object vVal = cell.getV().getValue(); - // addColumn may pack KeyValue; assert we share the Cell backing region. - if (qVal instanceof byte[]) { - assertSame(src.getQualifierArray(), qVal); - } else { - ObBytesString qView = (ObBytesString) qVal; - assertSame(src.getQualifierArray(), qView.bytes); - assertEquals(src.getQualifierOffset(), qView.offset); - assertEquals(src.getQualifierLength(), qView.length()); - } - if (vVal instanceof byte[]) { - assertSame(src.getValueArray(), vVal); - } else { - ObBytesString vView = (ObBytesString) vVal; - assertSame(src.getValueArray(), vView.bytes); - assertEquals(src.getValueOffset(), vView.offset); - assertEquals(src.getValueLength(), vView.length()); - } - assertTrue(Bytes.equals( - q, - qVal instanceof byte[] ? (byte[]) qVal : Bytes.copy(((ObBytesString) qVal).bytes, - ((ObBytesString) qVal).offset, ((ObBytesString) qVal).length()))); - assertTrue(Bytes.equals( - v, - vVal instanceof byte[] ? (byte[]) vVal : Bytes.copy(((ObBytesString) vVal).bytes, - ((ObBytesString) vVal).offset, ((ObBytesString) vVal).length()))); + ObHbaseCfRows cfRows = request.getCfRows().get(0); + assertTrue(cfRows.hasCompactCells()); + assertSame(src.getQualifierArray(), compactByteArrays(cfRows, "compactQualifierArrays")[0]); + assertSame(src.getValueArray(), compactByteArrays(cfRows, "compactValueArrays")[0]); } @Test public void testBuildHbaseRequestClonesWhenAutoFlushOff() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, - true); + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService); table.setAutoFlush(false); byte[] q = Bytes.toBytes("q1"); byte[] v = Bytes.toBytes("v1"); @@ -185,42 +144,19 @@ public void testBuildHbaseRequestClonesWhenAutoFlushOff() throws Exception { ObHbaseRequest request = table.buildHbaseRequest(Collections.singletonList(put), OHOperationType.PUT); - ObHbaseCell cell = request.getCfRows().get(0).getCells().get(0); - assertTrue(cell.getQ().getValue() instanceof byte[]); - assertTrue(cell.getV().getValue() instanceof byte[]); - assertNotSame(src.getQualifierArray(), cell.getQ().getValue()); - assertNotSame(src.getValueArray(), cell.getV().getValue()); - assertTrue(Bytes.equals(q, (byte[]) cell.getQ().getValue())); - assertTrue(Bytes.equals(v, (byte[]) cell.getV().getValue())); - } - - @Test - public void testConfigOffForcesCloneEvenWithAutoFlush() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, - true); - Field enabled = OHTable.class.getDeclaredField("enablePutSkipCellClone"); - enabled.setAccessible(true); - enabled.setBoolean(table, false); - - byte[] q = Bytes.toBytes("q1"); - byte[] v = Bytes.toBytes("v1"); - Put put = new Put(Bytes.toBytes("row")); - put.addColumn(Bytes.toBytes("cf"), q, v); - Cell src = put.getFamilyCellMap().get(Bytes.toBytes("cf")).get(0); - - ObHbaseRequest request = table.buildHbaseRequest(Collections.singletonList(put), - OHOperationType.PUT); - ObHbaseCell cell = request.getCfRows().get(0).getCells().get(0); - assertTrue(cell.getQ().getValue() instanceof byte[]); - assertTrue(cell.getV().getValue() instanceof byte[]); - assertNotSame(src.getQualifierArray(), cell.getQ().getValue()); - assertNotSame(src.getValueArray(), cell.getV().getValue()); + ObHbaseCfRows cfRows = request.getCfRows().get(0); + assertTrue(cfRows.hasCompactCells()); + byte[] qualifier = compactByteArrays(cfRows, "compactQualifierArrays")[0]; + byte[] value = compactByteArrays(cfRows, "compactValueArrays")[0]; + assertNotSame(src.getQualifierArray(), qualifier); + assertNotSame(src.getValueArray(), value); + assertTrue(Bytes.equals(q, qualifier)); + assertTrue(Bytes.equals(v, value)); } @Test public void testLegacyTtlReusesSingleValueClone() throws Exception { - OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService, - true); + OHTable table = new OHTable(Bytes.toBytes("t"), mock(ObTableClient.class), executorService); byte[] value = Bytes.toBytes("payload"); KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("q"), value); @@ -240,4 +176,11 @@ private static Cell contiguousCell(byte[] row, byte[] family, byte[] qualifier, return new KeyValue(row, 0, row.length, family, 0, family.length, qualifier, 0, qualifier.length, System.currentTimeMillis(), KeyValue.Type.Put, value, 0, value.length); } + + private static byte[][] compactByteArrays(ObHbaseCfRows cfRows, String fieldName) + throws Exception { + Field field = ObHbaseCfRows.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (byte[][]) field.get(cfRows); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java index 0cc732cb..723e5321 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePutValidationTest.java @@ -65,7 +65,7 @@ public void testStaticValidationDoesNotMaterializeDeprecatedFamilyMap() { @Test public void testTableValidationDoesNotMaterializeDeprecatedFamilyMap() { OHTable table = new OHTable(Bytes.toBytes("test"), mock(ObTableClient.class), - executorService, true); + executorService); table.validatePutMutation(newPut(FAMILY, "q", "value")); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java b/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java index 42a15347..8ad6abe7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/result/ClientStreamScannerCompactResultTest.java @@ -20,7 +20,6 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellBatch; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHBaseCellRow; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; @@ -41,11 +40,11 @@ public class ClientStreamScannerCompactResultTest { @Test - public void testCompactResultUsesKeyValueWhenLightweightCellIsDisabled() throws Exception { + public void testCompactResultUsesKeyValueAndDirectConsume() throws Exception { ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( new String[] { "q-2", "q-1" }, new long[] { 101L, 102L })); ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", bytes("f"), - false, null, false); + false, null); Result result = scanner.next(); @@ -57,48 +56,17 @@ public void testCompactResultUsesKeyValueWhenLightweightCellIsDisabled() throws verify(streamResult, never()).getCacheRows(); } - @Test - public void testCompactResultUsesLightweightCellByDefault() throws Exception { - ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( - new String[] { "q-1" }, new long[] { 102L })); - ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", bytes("f"), - false, null); - - Result result = scanner.next(); - - assertEquals(1, result.size()); - assertTrue(result.rawCells()[0] instanceof OHBaseResultCell); - } - - @Test - public void testCompactResultUsesLightweightCellWhenEnabled() throws Exception { - ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( - new String[] { "q-1", "q-2" }, new long[] { 102L, 101L })); - ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", bytes("f"), - false, null, true); - - Result result = scanner.next(); - - assertEquals(2, result.size()); - for (Cell cell : result.rawCells()) { - assertTrue(cell instanceof OHBaseResultCell); - assertArrayEquals(bytes("row-1"), CellUtil.cloneRow(cell)); - assertArrayEquals(bytes("f"), CellUtil.cloneFamily(cell)); - } - verify(streamResult, never()).getRow(); - verify(streamResult, never()).getCacheRows(); - } - @Test public void testCompactTableGroupResultUsesQualifierOffsets() throws Exception { ObTableClientQueryAsyncStreamResult streamResult = compactStreamResult(compactRow( new String[] { "f1\0q-1", "f2\0q-2" }, new long[] { 102L, 101L })); ClientStreamScanner scanner = new ClientStreamScanner(streamResult, "test", new byte[0], - true, null, true); + true, null); Result result = scanner.next(); assertEquals(2, result.size()); + assertTrue(result.rawCells()[0] instanceof KeyValue); assertArrayEquals(bytes("f1"), CellUtil.cloneFamily(result.rawCells()[0])); assertArrayEquals(bytes("q-1"), CellUtil.cloneQualifier(result.rawCells()[0])); assertArrayEquals(bytes("f2"), CellUtil.cloneFamily(result.rawCells()[1]));