HIVE-29365: Range predicate within the same histogram bucket leads to an estimate rowcount=1

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java

@@ -937,12 +937,16 @@ private static Range<Float> convertRangeToClosedOpen(Range<Float> boundaries) {
    * @return the selectivity of "val1 &lt;= column &lt; val2"
    */
   static double rangedSelectivity(KllFloatsSketch kll, float val1, float val2) {
-    if (val1 >= val2) {
+    if (val1 >= val2 || val2 <= kll.getMinItem() || val1 > kll.getMaxItem()) {
       return 0;
     }
     float[] splitPoints = new float[] { val1, val2 };
     double[] boundaries = kll.getCDF(splitPoints, QuantileSearchCriteria.EXCLUSIVE);
-    return boundaries[1] - boundaries[0];
+    // due to the way the KLL sketch is constructed,
+    // it is not possible to differentiate selectivity values below the error
+    // (e.g., if the error is 2%, a real selectivity of 1.5% might be estimated as 0% by KLL)
+    double normalizedRankError = kll.getNormalizedRankError(false);
+    return Math.max(boundaries[1] - boundaries[0], normalizedRankError);
   }
 
   /**
@@ -1015,8 +1019,8 @@ public static double lessThanSelectivity(KllFloatsSketch kll, float value) {
    * @throws IllegalArgumentException if leftValue is equal to rightValue
    */
   public static double betweenSelectivity(KllFloatsSketch kll, float leftValue, float rightValue) {
-    // column >= leftValue AND column <= rightValue
-    if (rightValue < leftValue) {
+    // is it possible to fulfill the BETWEEN?
+    if (rightValue < leftValue || rightValue < kll.getMinItem() || leftValue > kll.getMaxItem()) {
       return 0;
     }
     if (Double.compare(leftValue, rightValue) == 0) {

ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java

@@ -130,6 +130,7 @@ public class TestFilterSelectivityEstimator {
   private static final KllFloatsSketch KLL3 = StatisticsTestUtils.createKll(VALUES3);
   private static final KllFloatsSketch KLL_TIME = StatisticsTestUtils.createKll(VALUES_TIME);
   private static final float DELTA = 1e-7f;
+  private static final double MIN_KLL_SELECTIVITY = 0.013294757464848584;
   private static final RexBuilder REX_BUILDER = new RexBuilder(new JavaTypeFactoryImpl(new HiveTypeSystemImpl()));
   private static final RelDataTypeFactory TYPE_FACTORY = REX_BUILDER.getTypeFactory();
 
@@ -272,6 +273,25 @@ public void testIsHistogramAvailableWhenEmptyArray() {
     Assert.assertFalse(isHistogramAvailable(colStatistics));
   }
 
+  /**
+   * Check the KLL resolution.
+   * <p>
+   * The resolution may not be lower than the minimum rank error [HIVE-29365].
+   * </p> */
+  @Test
+  public void testKllResolution() {
+    Assert.assertEquals(MIN_KLL_SELECTIVITY, betweenSelectivity(KLL, 1.20f, 1.21f), DELTA);
+
+    doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+    RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.AND,
+        REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN, inputRef0, literalFloat(1.20f)),
+        REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN, inputRef0, literalFloat(1.21f)));
+    filter = simplify(filter);
+    Assert.assertEquals(SqlKind.SEARCH, filter.getKind());
+    FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+    Assert.assertEquals(MIN_KLL_SELECTIVITY, estimator.estimateSelectivity(filter), DELTA);
+  }
+
   @Test
   public void testLessThanSelectivity() {
     Assert.assertEquals(0.6153846153846154, lessThanSelectivity(KLL, 3), DELTA);