From 798bb46df5042b7300b9dfb6112927d65c059b56 Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Tue, 3 Nov 2026 02:50:04 +1100 Subject: [PATCH 1/2] Write each row once when an identity partition has both -0.0 and 0.0 _determine_partitions groups rows by partition value and then selects each group with an equality filter. group_by keeps -0.0 and 0.0 apart, but -0.0 == 0.0, so both partitions selected the rows of both, and every such row was written twice. Match the sign of zero as well, the same way NaN is already selected with is_nan. --- pyiceberg/io/pyarrow.py | 5 +++++ tests/catalog/test_catalog_behaviors.py | 24 ++++++++++++++++++++++++ tests/io/test_pyarrow.py | 21 +++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..7f7b28c9dc 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3218,6 +3218,11 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express return pc.field(partition_field_name).is_null() elif isinstance(value, float) and math.isnan(value): return pc.is_nan(pc.field(partition_field_name)) + elif isinstance(value, float) and value == 0.0: + # -0.0 and 0.0 are separate partitions but compare equal, so also match the sign (1 / -0.0 is -inf) + negative_sign = pc.divide(pc.scalar(1.0), pc.field(partition_field_name)) < 0 + same_sign = negative_sign if math.copysign(1.0, value) < 0 else ~negative_sign + return (pc.field(partition_field_name) == value) & same_sign return pc.field(partition_field_name) == value diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 1148ca8527..9ea50532ac 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1345,6 +1345,30 @@ def test_append_nan_to_identity_partitioned_table(catalog: Catalog) -> None: assert sorted(tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist()) == [2, 4] +def test_append_signed_zero_to_identity_partitioned_table(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.append_signed_zero_identity_partition_{catalog.name}" + iceberg_schema = Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "value", DoubleType(), required=False), + ) + partition_spec = PartitionSpec( + PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value"), + ) + tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema, partition_spec=partition_spec) + + tbl.append( + pa.Table.from_pydict( + {"id": [1, 2, 3], "value": [-0.0, 0.0, 1.0]}, + schema=schema_to_pyarrow(iceberg_schema), + ) + ) + + # Each row is written once, although -0.0 and 0.0 compare equal + assert sorted(tbl.scan().to_arrow()["id"].to_pylist()) == [1, 2, 3] + assert sorted(tbl.scan(row_filter="value = 0").to_arrow()["id"].to_pylist()) == [1, 2] + + def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.scan_integer_decimal_literal_{catalog.name}" diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index abb7d384d5..a1afcced61 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3287,6 +3287,27 @@ def test_determine_partitions_identity_nan() -> None: assert rows_by_partition == {"1.0": [1], "nan": [2, 4], "None": [3]} +def test_determine_partitions_identity_signed_zero() -> None: + for field_type in (DoubleType(), FloatType()): + schema = Schema( + NestedField(id=1, name="id", field_type=IntegerType(), required=False), + NestedField(id=2, name="value", field_type=field_type, required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value")) + arrow_table = pa.Table.from_pydict( + {"id": [1, 2, 3, 4], "value": [-0.0, 0.0, 1.0, -0.0]}, + schema=schema.as_arrow(), + ) + + partitions = list(_determine_partitions(spec, schema, arrow_table)) + + # -0.0 == 0.0, but they are separate partitions, so each row must land in exactly one of them + rows_by_partition = { + repr(p.partition_key.partition[0]): sorted(p.arrow_table_partition["id"].to_pylist()) for p in partitions + } + assert rows_by_partition == {"-0.0": [1, 4], "0.0": [2], "1.0": [3]}, field_type + + @pytest.mark.parametrize("transform", [IdentityTransform(), BucketTransform(4), TruncateTransform(1)]) def test_determine_partitions_with_view_types(transform: Transform[Any, Any]) -> None: schema = Schema( From efb2848954f5ecf192b93446e784e93176ea64ee Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Wed, 4 Nov 2026 12:33:48 +1100 Subject: [PATCH 2/2] Match the sign of zero with a single comparison --- pyiceberg/io/pyarrow.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 7f7b28c9dc..3468ec9f44 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3220,9 +3220,8 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express return pc.is_nan(pc.field(partition_field_name)) elif isinstance(value, float) and value == 0.0: # -0.0 and 0.0 are separate partitions but compare equal, so also match the sign (1 / -0.0 is -inf) - negative_sign = pc.divide(pc.scalar(1.0), pc.field(partition_field_name)) < 0 - same_sign = negative_sign if math.copysign(1.0, value) < 0 else ~negative_sign - return (pc.field(partition_field_name) == value) & same_sign + field = pc.field(partition_field_name) + return (field == value) & (pc.divide(pc.scalar(1.0), field) == math.copysign(math.inf, value)) return pc.field(partition_field_name) == value