From 1f1278fb1335f332cae761ac3144d938e7fd75bb Mon Sep 17 00:00:00 2001 From: Suraj Suthar Date: Tue, 3 Nov 2026 12:54:59 +0530 Subject: [PATCH] feat: support ALP parquet encoding in writer options Add ALP to DataFusion's parquet encoding configuration, conversion, documentation, protobuf round-trip tests, and SQL writer coverage. --- datafusion/common/src/config.rs | 4 ++-- datafusion/common/src/parquet_config.rs | 8 ++++++- datafusion/proto-common/src/from_proto/mod.rs | 6 +++++- datafusion/sqllogictest/test_files/copy.slt | 21 +++++++++++++++++++ .../test_files/information_schema.slt | 2 +- .../sqllogictest/test_files/set_variable.slt | 12 +++++++++-- docs/source/user-guide/configs.md | 2 +- docs/source/user-guide/sql/format_options.md | 2 +- 8 files changed, 48 insertions(+), 9 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index c50d061977b77..3253ce404d322 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1512,7 +1512,7 @@ config_namespace! { /// (writing) Sets default encoding for any column. /// Valid values are: plain, plain_dictionary, rle, /// bit_packed, delta_binary_packed, delta_length_byte_array, - /// delta_byte_array, rle_dictionary, and byte_stream_split. + /// delta_byte_array, rle_dictionary, byte_stream_split, and alp. /// These values are not case sensitive. If NULL, uses /// default parquet writer setting pub encoding: Option, default = None @@ -3509,7 +3509,7 @@ config_namespace_with_hashmap! { /// Sets encoding for the column path. /// Valid values are: plain, plain_dictionary, rle, /// bit_packed, delta_binary_packed, delta_length_byte_array, - /// delta_byte_array, rle_dictionary, and byte_stream_split. + /// delta_byte_array, rle_dictionary, byte_stream_split, and alp. /// These values are not case-sensitive. If NULL, uses /// default parquet options pub encoding: Option, default = None diff --git a/datafusion/common/src/parquet_config.rs b/datafusion/common/src/parquet_config.rs index d518a2924c175..827d2ed71cd4e 100644 --- a/datafusion/common/src/parquet_config.rs +++ b/datafusion/common/src/parquet_config.rs @@ -285,6 +285,8 @@ pub enum DFParquetEncoding { RleDictionary, /// Byte stream split encoding ByteStreamSplit, + /// Adaptive Lossless Floating-Point Encoding + Alp, } impl FromStr for DFParquetEncoding { @@ -301,11 +303,12 @@ impl FromStr for DFParquetEncoding { "delta_byte_array" => Ok(Self::DeltaByteArray), "rle_dictionary" => Ok(Self::RleDictionary), "byte_stream_split" => Ok(Self::ByteStreamSplit), + "alp" => Ok(Self::Alp), _ => Err(DataFusionError::Configuration(format!( "Unknown or unsupported parquet encoding: \ {s}. Valid values are: plain, plain_dictionary, rle, \ bit_packed, delta_binary_packed, delta_length_byte_array, \ - delta_byte_array, rle_dictionary, and byte_stream_split." + delta_byte_array, rle_dictionary, byte_stream_split, and alp." ))), } } @@ -323,6 +326,7 @@ impl Display for DFParquetEncoding { Self::DeltaByteArray => "delta_byte_array", Self::RleDictionary => "rle_dictionary", Self::ByteStreamSplit => "byte_stream_split", + Self::Alp => "alp", }) } } @@ -374,6 +378,7 @@ impl From for parquet::basic::Encoding { DFParquetEncoding::DeltaByteArray => Self::DELTA_BYTE_ARRAY, DFParquetEncoding::RleDictionary => Self::RLE_DICTIONARY, DFParquetEncoding::ByteStreamSplit => Self::BYTE_STREAM_SPLIT, + DFParquetEncoding::Alp => Self::ALP, } } } @@ -737,6 +742,7 @@ mod tests { DFParquetEncoding::ByteStreamSplit, Encoding::BYTE_STREAM_SPLIT, ), + ("alp", DFParquetEncoding::Alp, Encoding::ALP), ] { let parsed: DFParquetEncoding = input.parse().unwrap(); assert_eq!(parsed, expected, "input: {input}"); diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index 919431303587f..fd94fc38d5901 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -1723,7 +1723,11 @@ mod tests { #[test] fn test_parquet_encoding_round_trip() { - for encoding in [None, Some(DFParquetEncoding::ByteStreamSplit)] { + for encoding in [ + None, + Some(DFParquetEncoding::ByteStreamSplit), + Some(DFParquetEncoding::Alp), + ] { let opts = ParquetOptions { encoding, ..ParquetOptions::default() diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt index 5a44d92e9ff12..3a959a7c50265 100644 --- a/datafusion/sqllogictest/test_files/copy.slt +++ b/datafusion/sqllogictest/test_files/copy.slt @@ -387,6 +387,27 @@ OPTIONS ( ---- 2 +# ALP encoding is accepted for floating-point columns. +query I +COPY ( + VALUES (CAST(1.25 AS DOUBLE)), (CAST(2.5 AS DOUBLE)), (CAST(3.75 AS DOUBLE)) +) +TO 'test_files/scratch/copy/alp/' STORED AS PARQUET +OPTIONS ('format.encoding' 'alp', 'format.dictionary_enabled' false); +---- +3 + +statement ok +CREATE EXTERNAL TABLE validate_alp +STORED AS PARQUET LOCATION 'test_files/scratch/copy/alp/'; + +query R rowsort +SELECT column1 FROM validate_alp; +---- +1.25 +2.5 +3.75 + # valid vs invalid metadata # accepts map with a single entry diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index f9531f20e85a8..e7b0e08d5975c 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -410,7 +410,7 @@ datafusion.execution.parquet.dictionary_enabled true (writing) Sets if dictionar datafusion.execution.parquet.dictionary_page_size_limit 1048576 (writing) Sets best effort maximum dictionary page size, in bytes datafusion.execution.parquet.enable_page_index true (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. datafusion.execution.parquet.enable_rle_to_dictionary false (reading) If true, top-level string and binary Parquet columns with dictionary pages are inferred and scanned as `Dictionary` / `Dictionary` instead of their plain value type. This applies only when DataFusion infers the table schema. Tables with a user-supplied schema are not promoted because the Parquet footer is not read at DDL time, so dictionary pages cannot be detected per column. See -datafusion.execution.parquet.encoding NULL (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting +datafusion.execution.parquet.encoding NULL (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, byte_stream_split, and alp. These values are not case sensitive. If NULL, uses default parquet writer setting datafusion.execution.parquet.force_filter_selections false (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. datafusion.execution.parquet.max_in_list_size 20 Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. datafusion.execution.parquet.max_predicate_cache_size NULL (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt index 1b2e0feee395e..cd2457090d94a 100644 --- a/datafusion/sqllogictest/test_files/set_variable.slt +++ b/datafusion/sqllogictest/test_files/set_variable.slt @@ -754,18 +754,26 @@ SHOW datafusion.execution.parquet.encoding ---- datafusion.execution.parquet.encoding plain +statement ok +SET datafusion.execution.parquet.encoding = 'ALP' + +query TT +SHOW datafusion.execution.parquet.encoding +---- +datafusion.execution.parquet.encoding alp + statement error SET datafusion.execution.parquet.encoding = 'invalid' ---- DataFusion error: Error setting config datafusion.execution.parquet.encoding caused by -Invalid or Unsupported Configuration: Unknown or unsupported parquet encoding: invalid. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. +Invalid or Unsupported Configuration: Unknown or unsupported parquet encoding: invalid. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, byte_stream_split, and alp. query TT SHOW datafusion.execution.parquet.encoding ---- -datafusion.execution.parquet.encoding plain +datafusion.execution.parquet.encoding alp statement ok RESET datafusion.execution.parquet.encoding diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 1ec4fc1aef7af..40a8b806f3847 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -111,7 +111,7 @@ The following configuration settings are available: | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | -| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, byte_stream_split, and alp. These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | | datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | diff --git a/docs/source/user-guide/sql/format_options.md b/docs/source/user-guide/sql/format_options.md index 738ed8b5de638..3d188c771d312 100644 --- a/docs/source/user-guide/sql/format_options.md +++ b/docs/source/user-guide/sql/format_options.md @@ -154,7 +154,7 @@ The following options are available when reading or writing Parquet files. If an | Option | Can be Column Specific? | Description | OPTIONS Key | Default Value | | ------------------------------------------ | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- | ------------------------ | | COMPRESSION | Yes | Sets the internal Parquet **compression codec** for data pages, optionally including the compression level. Applies globally if set without `::col`, or specifically to a column if set using `'compression::column_name'`. Valid values: `uncompressed`, `snappy`, `gzip(level)`, `brotli(level)`, `lz4`, `zstd(level)`, `lz4_raw`. | `'compression'` or `'compression::col'` | zstd(3) | -| ENCODING | Yes | Sets the **encoding** scheme for data pages. Valid values: `plain`, `plain_dictionary`, `rle`, `bit_packed`, `delta_binary_packed`, `delta_length_byte_array`, `delta_byte_array`, `rle_dictionary`, `byte_stream_split`. Use key `'encoding'` or `'encoding::col'` in OPTIONS. | `'encoding'` or `'encoding::col'` | None | +| ENCODING | Yes | Sets the **encoding** scheme for data pages. Valid values: `plain`, `plain_dictionary`, `rle`, `bit_packed`, `delta_binary_packed`, `delta_length_byte_array`, `delta_byte_array`, `rle_dictionary`, `byte_stream_split`, `alp`. Use key `'encoding'` or `'encoding::col'` in OPTIONS. | `'encoding'` or `'encoding::col'` | None | | DICTIONARY_ENABLED | Yes | Sets whether dictionary encoding should be enabled globally or for a specific column. | `'dictionary_enabled'` or `'dictionary_enabled::col'` | true | | STATISTICS_ENABLED | Yes | Sets the level of statistics to write (`none`, `chunk`, `page`). | `'statistics_enabled'` or `'statistics_enabled::col'` | page | | BLOOM_FILTER_ENABLED | Yes | Sets whether a bloom filter should be written for a specific column. | `'bloom_filter_enabled::column_name'` | None |