diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 0a2a72b6f0..6faf98874c 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -311,8 +311,8 @@ Next, write the data to the table. Both `append` and `overwrite` produce the sam -!!! note inline end "Fast append" - PyIceberg defaults to the [fast append](https://iceberg.apache.org/spec/#snapshots) to minimize the amount of data written. This enables fast commit operations, reducing the possibility of conflicts. The downside of the fast append is that it creates more metadata than a merge commit. [Compaction is planned](https://github.com/apache/iceberg-python/issues/270) and will automatically rewrite all the metadata when a threshold is hit, to maintain performant reads. +!!! note inline end "Merge and fast append" + PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached to keep metadata compact and reads performant. Fast append remains available by setting the table property `commit.manifest-merge.enabled` to `False`. Fast append minimizes the metadata written during a commit, enabling faster commits and reducing the possibility of conflicts, but accumulates more manifest metadata over time. diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 44b5e395a9..98b3bf856c 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -99,12 +99,12 @@ Iceberg tables support table properties to configure table behavior. | ------------------------------------ | ------------------- | ------------- | ----------------------------------------------------------- | | `commit.manifest.target-size-bytes` | Size in bytes | 8388608 (8MB) | Target size when merging manifest files | | `commit.manifest.min-count-to-merge` | Number of manifests | 100 | Minimum number of manifests to accumulate before merging | -| `commit.manifest-merge.enabled` | Boolean | False | Controls whether to automatically merge manifests on writes | +| `commit.manifest-merge.enabled` | Boolean | True | Controls whether to automatically merge manifests on writes | -!!! note "Fast append" - Unlike Java implementation, PyIceberg default to the [fast append](api.md#write-to-a-table) and thus `commit.manifest-merge.enabled` is set to `False` by default. +!!! note "Append modes" + PyIceberg defaults to [merge append](api.md#write-to-a-table) to keep manifest metadata compact. Set `commit.manifest-merge.enabled` to `False` on a table to use fast append, which writes less metadata during each commit but accumulates more manifests over time. diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index c0adce84dc..0e344c22c7 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -198,7 +198,7 @@ class TableProperties: MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100 MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled" - MANIFEST_MERGE_ENABLED_DEFAULT = False + MANIFEST_MERGE_ENABLED_DEFAULT = True METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max" METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100 diff --git a/tests/integration/test_writes/test_partitioned_writes.py b/tests/integration/test_writes/test_partitioned_writes.py index 1d1488255f..040c0cfeea 100644 --- a/tests/integration/test_writes/test_partitioned_writes.py +++ b/tests/integration/test_writes/test_partitioned_writes.py @@ -748,8 +748,11 @@ def test_dynamic_partition_overwrite_evolve_partition(spark: SparkSession, sessi tbl.dynamic_partition_overwrite(arrow_table) result = tbl.scan().to_arrow() - assert result["place"].to_pylist() == ["Groningen", "Amsterdam", "Drachten"] - assert result["inhabitants"].to_pylist() == [238147, 921402, 44940] + assert sorted(zip(result["place"].to_pylist(), result["inhabitants"].to_pylist(), strict=True)) == [ + ("Amsterdam", 921402), + ("Drachten", 44940), + ("Groningen", 238147), + ] @pytest.mark.integration diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 30fdd76ab7..553a9ac574 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1571,7 +1571,7 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_a = _create_table( session_catalog, "default.merge_manifest_a", - {"commit.manifest-merge.enabled": "true", "commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, [], ) tbl_b = _create_table( @@ -1588,7 +1588,11 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_c = _create_table( session_catalog, "default.merge_manifest_c", - {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + { + "commit.manifest-merge.enabled": "false", + "commit.manifest.min-count-to-merge": "1", + "format-version": format_version, + }, [], ) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 5f1680ed59..d813665e2d 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -649,3 +649,22 @@ def summary_calls(n_files: int) -> int: f"_MergeAppendFiles.__init__ made {merge_init - fast_init} extra update_table_metadata " "calls over its superclass; expected 1 (hoisted)" ) + + +def test_append_snapshot_producer_defaults_to_merge_append(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _MergeAppendFiles + + append = table_v2.transaction()._append_snapshot_producer({}) + + assert type(append) is _MergeAppendFiles + + +def test_append_snapshot_producer_uses_fast_append_when_manifest_merge_disabled(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _FastAppendFiles + + transaction = table_v2.transaction() + transaction.set_properties({"commit.manifest-merge.enabled": "false"}) + + append = transaction._append_snapshot_producer({}) + + assert type(append) is _FastAppendFiles