diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java index 807f3a901c45..2e3265389aef 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java @@ -154,12 +154,8 @@ public static BaseVariantReader create( } else if (targetType instanceof ArrayType) { return new ArrayReader(schema, (ArrayType) targetType, castArgs); } else if (targetType instanceof MapType - && ((MapType) targetType).getKeyType().equals(DataTypes.STRING())) { - if (((MapType) targetType).getKeyType().equals(DataTypes.STRING())) { - return new MapReader(schema, (MapType) targetType, castArgs); - } else { - throw new UnsupportedOperationException(); - } + && ((MapType) targetType).getKeyType().equalsIgnoreNullable(DataTypes.STRING())) { + return new MapReader(schema, (MapType) targetType, castArgs); } else if (targetType instanceof VariantType) { return new VariantReader( schema, (VariantType) targetType, castArgs, isTopLevelUnshredded); diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java index cf81143cc862..04b6cb62cd2d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java @@ -88,7 +88,7 @@ public static Object cast(GenericVariant v, DataType dataType, VariantCastArgs c } else if (dataType instanceof MapType) { MapType mapType = (MapType) dataType; DataType valueType = mapType.getValueType(); - if (mapType.getKeyType().equals(DataTypes.STRING())) { + if (mapType.getKeyType().equalsIgnoreNullable(DataTypes.STRING())) { if (variantType == Type.OBJECT) { int size = v.objectSize(); HashMap map = new HashMap<>(); diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/BaseVariantReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/BaseVariantReaderTest.java index e3f0b558bf0e..de387ca20d1a 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/BaseVariantReaderTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/BaseVariantReaderTest.java @@ -19,15 +19,19 @@ package org.apache.paimon.data.variant; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.junit.jupiter.api.Test; import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; import static org.apache.paimon.data.variant.PaimonShreddingUtils.buildVariantSchema; import static org.apache.paimon.data.variant.PaimonShreddingUtils.castShredded; @@ -63,4 +67,27 @@ void testRowReaderWithUnshreddedTargetField() { assertThat(reader.read(shredded, shredded.getBinary(schema.topLevelMetadataIdx))) .isEqualTo(GenericRow.of(27, null)); } + + @Test + void testMapReaderWithNotNullStringKey() { + // The file shreds the object, so the map has to be read from the typed fields. + RowType shreddedType = + RowType.of( + new DataType[] {DataTypes.INT(), DataTypes.INT()}, new String[] {"a", "b"}); + VariantSchema schema = buildVariantSchema(variantShreddingSchema(shreddedType)); + MapType targetType = new MapType(DataTypes.STRING().notNull(), DataTypes.INT()); + InternalRow shredded = + castShredded(GenericVariant.fromJson("{\"a\": 1, \"b\": 2}"), schema); + + Map expected = new HashMap<>(); + expected.put(BinaryString.fromString("a"), 1); + expected.put(BinaryString.fromString("b"), 2); + for (boolean failOnError : new boolean[] {false, true}) { + VariantCastArgs castArgs = new VariantCastArgs(failOnError, ZoneOffset.UTC); + BaseVariantReader reader = + BaseVariantReader.create(schema, targetType, castArgs, false); + assertThat(reader.read(shredded, shredded.getBinary(schema.topLevelMetadataIdx))) + .isEqualTo(new GenericMap(expected)); + } + } } diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java index 12c54e54f8ce..9b47ab1e216f 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java @@ -21,11 +21,13 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InstantiationUtil; @@ -282,6 +284,37 @@ public void testVariantGet() { assertThat(variant.variantGet("$.nullField", DataTypes.BOOLEAN(), castArgs)).isNull(); } + @Test + public void testVariantGetMapWithNotNullStringKey() { + // Spark hands over map key types as NOT NULL, so a STRING key must match whatever its + // nullability is. + Variant variant = + GenericVariant.fromJson("{\"m\": {\"a\": 1, \"b\": 2}, \"arr\": [{\"a\": 1}]}"); + VariantCastArgs tryCast = new VariantCastArgs(false, ZoneOffset.UTC); + VariantCastArgs strict = new VariantCastArgs(true, ZoneOffset.UTC); + MapType notNullKey = new MapType(DataTypes.STRING().notNull(), DataTypes.INT()); + + Map expected = new HashMap<>(); + expected.put(BinaryString.fromString("a"), 1); + expected.put(BinaryString.fromString("b"), 2); + assertThat(variant.variantGet("$.m", notNullKey, tryCast)) + .isEqualTo(new GenericMap(expected)); + assertThat(variant.variantGet("$.m", notNullKey, strict)) + .isEqualTo(new GenericMap(expected)); + + Map element = new HashMap<>(); + element.put(BinaryString.fromString("a"), 1); + GenericArray expectedArray = new GenericArray(new Object[] {new GenericMap(element)}); + assertThat(variant.variantGet("$.arr", DataTypes.ARRAY(notNullKey), tryCast)) + .isEqualTo(expectedArray); + assertThat(variant.variantGet("$.arr", DataTypes.ARRAY(notNullKey), strict)) + .isEqualTo(expectedArray); + + // a non string key is still an invalid cast + MapType intKey = new MapType(DataTypes.INT().notNull(), DataTypes.INT()); + assertThat(variant.variantGet("$.m", intKey, tryCast)).isNull(); + } + @Test public void testVariantGetDecimalWithScaleAbovePrecision() { // precision() counts the digits of the unscaled value, so it is below the scale for any diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java index 01915ed422a0..0556b88187d9 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java @@ -291,6 +291,19 @@ void testAssembleAllTypes() { BinaryString.fromString("Apache Paimon")); } }))); + + // cast struct to map whose key is NOT NULL + fields[0] = + buildFieldsToExtract( + new MapType(DataTypes.STRING().notNull(), DataTypes.STRING()), + "$.object", + castArgs, + variantSchema); + Map expectedMap = new HashMap<>(); + expectedMap.put(BinaryString.fromString("age"), BinaryString.fromString("3")); + expectedMap.put(BinaryString.fromString("name"), BinaryString.fromString("Apache Paimon")); + assertThat(assembleVariantStruct(castShredded(v, variantSchema), variantSchema, fields)) + .isEqualTo(GenericRow.of(new GenericMap(expectedMap))); } @Test