From 84c54b5a909ceee14fd41c5d771332269ba56ff8 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Mon, 7 Sep 2026 15:18:37 +0800 Subject: [PATCH] [common] Fix invalid DecimalType when extracting a variant decimal VariantGet took the precision and scale straight off the BigDecimal that getDecimal() returns. precision() counts the digits of the unscaled value, so it is below the scale for anything under 0.1, and the trailing zero stripping in getDecimal() turns 100.00 into 1E+2, a negative scale. DecimalType rejects both, so extracting such a value threw "Decimal scale must be between 0 and the precision 1". Rescale a negative scale to zero and widen the precision to the scale, which is bit for bit what Spark's Decimal.set(BigDecimal) does. The reader caps scale and precision at 38 in GenericVariantUtil.checkDecimal, so the widened precision stays inside DecimalType. --- .../paimon/data/variant/VariantGet.java | 10 +++++- .../data/variant/GenericVariantTest.java | 32 +++++++++++++++++++ .../variant/PaimonShreddingUtilsTest.java | 21 ++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java index e053f330f48f..37aa20b4af19 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java @@ -138,8 +138,16 @@ public static Object cast(GenericVariant v, DataType dataType, VariantCastArgs c break; case DECIMAL: BigDecimal decimal = v.getDecimal(); - int precision = decimal.precision(); + if (decimal.scale() < 0) { + // stripTrailingZeros folds trailing zeros into a negative exponent, + // and a negative scale is not a Paimon decimal + decimal = decimal.setScale(0); + } int scale = decimal.scale(); + // precision() counts the digits of the unscaled value, so it is smaller than + // the scale for a value below 0.1, which DecimalType rejects. The variant + // writer caps both at MAX_DECIMAL16_PRECISION, so this stays in range. + int precision = Math.max(decimal.precision(), scale); input = Decimal.fromBigDecimal(decimal, precision, scale); inputType = DataTypes.DECIMAL(precision, scale); break; diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java index be8051cc7257..e5e507c455eb 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java @@ -276,6 +276,38 @@ public void testVariantGet() { assertThat(variant.variantGet("$.nullField", DataTypes.BOOLEAN(), castArgs)).isNull(); } + @Test + public void testVariantGetDecimalWithScaleAbovePrecision() { + // precision() counts the digits of the unscaled value, so it is below the scale for any + // value under 0.1. The scale 38 case is the widest the reader admits, and it is the + // first one here that needs a non compact Decimal. + String tiny = "0.00000000000000000000000000000000000001"; + Variant variant = GenericVariant.fromJson("{\"small\": 0.05, \"tiny\": " + tiny + "}"); + VariantCastArgs castArgs = new VariantCastArgs(false, ZoneOffset.UTC); + + assertThat(variant.variantGet("$.small", DataTypes.STRING(), castArgs)) + .isEqualTo(BinaryString.fromString("0.05")); + assertThat(variant.variantGet("$.small", DataTypes.DECIMAL(5, 3), castArgs)) + .isEqualTo(Decimal.fromBigDecimal(new BigDecimal("0.050"), 5, 3)); + assertThat(variant.variantGet("$.tiny", DataTypes.STRING(), castArgs)) + .isEqualTo(BinaryString.fromString(tiny)); + assertThat(variant.variantGet("$.tiny", DataTypes.DECIMAL(38, 38), castArgs)) + .isEqualTo(Decimal.fromBigDecimal(new BigDecimal(tiny), 38, 38)); + } + + @Test + public void testVariantGetDecimalWithNegativeScale() { + // getDecimal() strips trailing zeros, which turns 100.00 into 1E+2, a negative scale + Variant variant = GenericVariant.fromJson("{\"round\": 100.00}"); + VariantCastArgs castArgs = new VariantCastArgs(false, ZoneOffset.UTC); + + // rescaling rather than un-stripping keeps this in step with toJson + assertThat(variant.variantGet("$.round", DataTypes.STRING(), castArgs)) + .isEqualTo(BinaryString.fromString("100")); + assertThat(variant.variantGet("$.round", DataTypes.DECIMAL(5, 1), castArgs)) + .isEqualTo(Decimal.fromBigDecimal(new BigDecimal("100.0"), 5, 1)); + } + @Test public void testObjectFieldOrderingCompatibility() { String bmpKey = "\uE000"; diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java index 19ebca4fbf32..b1952357d774 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java @@ -291,6 +291,27 @@ void testAssembleAllTypes() { }))); } + @Test + public void testAssembleDecimalWithScaleAbovePrecision() { + // the unshredded leg extracts through VariantGet, which used to build an invalid + // DecimalType for a value below 0.1 or one whose trailing zeros were stripped off + GenericVariant v = GenericVariant.fromJson("{\"round\": 100.00, \"small\": 0.05}"); + VariantCastArgs castArgs = new VariantCastArgs(true, ZoneOffset.UTC); + + VariantSchema variantSchema = buildVariantSchema(variantShreddingSchema(RowType.of())); + FieldToExtract[] fieldsToExtract = { + buildFieldsToExtract(DataTypes.STRING(), "$.round", castArgs, variantSchema), + buildFieldsToExtract(DataTypes.STRING(), "$.small", castArgs, variantSchema) + }; + + assertThat( + assembleVariantStruct( + castShredded(v, variantSchema), variantSchema, fieldsToExtract)) + .isEqualTo( + GenericRow.of( + BinaryString.fromString("100"), BinaryString.fromString("0.05"))); + } + private static void assertVariantStructEquals( RowType shreddedType, RowType allTypes, GenericVariant v, GenericRow expected) { VariantCastArgs castArgs = new VariantCastArgs(true, ZoneOffset.UTC);