From f1e9c3711ad412c74a09f38893bbd98dd0f199bd Mon Sep 17 00:00:00 2001 From: SWARAJ SINGH Date: Fri, 11 Sep 2026 20:05:40 +0530 Subject: [PATCH 1/3] fix: make csv-read-options.py example self-contained with temporary fixtures (#1728) --- examples/csv-read-options.py | 157 ++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 68 deletions(-) diff --git a/examples/csv-read-options.py b/examples/csv-read-options.py index a5952d950..f540b6525 100644 --- a/examples/csv-read-options.py +++ b/examples/csv-read-options.py @@ -17,80 +17,101 @@ """Example demonstrating CsvReadOptions usage.""" +import gzip +import tempfile +from pathlib import Path + from datafusion import CsvReadOptions, SessionContext -# Create a SessionContext -ctx = SessionContext() +# Create sample data files in a temporary directory to make the example self-contained +with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + csv_file = tmp_path / "data.csv" + gz_file = tmp_path / "data.csv.gz" + + sample_csv = "id,name,value\n1,alice,100\n2,bob,200\n3,charlie,null\n" + csv_file.write_text(sample_csv) + + with gzip.open(gz_file, "wt") as f: + f.write(sample_csv) + + # Create a SessionContext + ctx = SessionContext() -# Example 1: Using CsvReadOptions with default values -print("Example 1: Default CsvReadOptions") -options = CsvReadOptions() -df = ctx.read_csv("data.csv", options=options) + # Example 1: Using CsvReadOptions with default values + print("Example 1: Default CsvReadOptions") + options = CsvReadOptions() + df = ctx.read_csv(str(csv_file), options=options) + df.show() -# Example 2: Using CsvReadOptions with custom parameters -print("\nExample 2: Custom CsvReadOptions") -options = CsvReadOptions( - has_header=True, - delimiter=",", - quote='"', - schema_infer_max_records=1000, - file_extension=".csv", -) -df = ctx.read_csv("data.csv", options=options) + # Example 2: Using CsvReadOptions with custom parameters + print("\nExample 2: Custom CsvReadOptions") + options = CsvReadOptions( + has_header=True, + delimiter=",", + quote='"', + schema_infer_max_records=1000, + file_extension=".csv", + ) + df = ctx.read_csv(str(csv_file), options=options) + df.show() -# Example 3: Using the builder pattern (recommended for readability) -print("\nExample 3: Builder pattern") -options = ( - CsvReadOptions() - .with_has_header(True) # noqa: FBT003 - .with_delimiter("|") - .with_quote("'") - .with_schema_infer_max_records(500) - .with_truncated_rows(False) # noqa: FBT003 - .with_newlines_in_values(True) # noqa: FBT003 -) -df = ctx.read_csv("data.csv", options=options) + # Example 3: Using the builder pattern (recommended for readability) + print("\nExample 3: Builder pattern") + options = ( + CsvReadOptions() + .with_has_header(True) # noqa: FBT003 + .with_delimiter("|") + .with_quote("'") + .with_schema_infer_max_records(500) + .with_truncated_rows(False) # noqa: FBT003 + .with_newlines_in_values(True) # noqa: FBT003 + ) + df = ctx.read_csv(str(csv_file), options=options) -# Example 4: Advanced options -print("\nExample 4: Advanced options") -options = ( - CsvReadOptions() - .with_has_header(True) # noqa: FBT003 - .with_delimiter(",") - .with_comment("#") # Skip lines starting with # - .with_escape("\\") # Escape character - .with_null_regex(r"^(null|NULL|N/A)$") # Treat these as NULL - .with_truncated_rows(True) # noqa: FBT003 - .with_file_compression_type("gzip") # Read gzipped CSV - .with_file_extension(".gz") -) -df = ctx.read_csv("data.csv.gz", options=options) + # Example 4: Advanced options + print("\nExample 4: Advanced options") + options = ( + CsvReadOptions() + .with_has_header(True) # noqa: FBT003 + .with_delimiter(",") + .with_comment("#") # Skip lines starting with # + .with_escape("\\") # Escape character + .with_null_regex(r"^(null|NULL|N/A)$") # Treat these as NULL + .with_truncated_rows(True) # noqa: FBT003 + .with_file_compression_type("gzip") # Read gzipped CSV + .with_file_extension(".gz") + ) + df = ctx.read_csv(str(gz_file), options=options) + df.show() -# Example 5: Register CSV table with options -print("\nExample 5: Register CSV table") -options = CsvReadOptions().with_has_header(True).with_delimiter(",") # noqa: FBT003 -ctx.register_csv("my_table", "data.csv", options=options) -df = ctx.sql("SELECT * FROM my_table") + # Example 5: Register CSV table with options + print("\nExample 5: Register CSV table") + options = CsvReadOptions().with_has_header(True).with_delimiter(",") # noqa: FBT003 + ctx.register_csv("my_table", str(csv_file), options=options) + df = ctx.sql("SELECT * FROM my_table") + df.show() -# Example 6: Backward compatibility (without options) -print("\nExample 6: Backward compatibility") -# Still works the old way! -df = ctx.read_csv("data.csv", has_header=True, delimiter=",") + # Example 6: Backward compatibility (without options) + print("\nExample 6: Backward compatibility") + # Still works the old way! + df = ctx.read_csv(str(csv_file), has_header=True, delimiter=",") + df.show() -print("\nAll examples completed!") -print("\nFor all available options, see the CsvReadOptions documentation:") -print(" - has_header: bool") -print(" - delimiter: str") -print(" - quote: str") -print(" - terminator: str | None") -print(" - escape: str | None") -print(" - comment: str | None") -print(" - newlines_in_values: bool") -print(" - schema: pa.Schema | None") -print(" - schema_infer_max_records: int") -print(" - file_extension: str") -print(" - table_partition_cols: list[tuple[str, pa.DataType]]") -print(" - file_compression_type: str") -print(" - file_sort_order: list[list[SortExpr]]") -print(" - null_regex: str | None") -print(" - truncated_rows: bool") + print("\nAll examples completed!") + print("\nFor all available options, see the CsvReadOptions documentation:") + print(" - has_header: bool") + print(" - delimiter: str") + print(" - quote: str") + print(" - terminator: str | None") + print(" - escape: str | None") + print(" - comment: str | None") + print(" - newlines_in_values: bool") + print(" - schema: pa.Schema | None") + print(" - schema_infer_max_records: int") + print(" - file_extension: str") + print(" - table_partition_cols: list[tuple[str, pa.DataType]]") + print(" - file_compression_type: str") + print(" - file_sort_order: list[list[SortExpr]]") + print(" - null_regex: str | None") + print(" - truncated_rows: bool") From 20ab8fa5f544b538cd8a36dc27688b48aaf7c00c Mon Sep 17 00:00:00 2001 From: dedsec-terminal Date: Fri, 11 Sep 2026 20:36:04 +0530 Subject: [PATCH 2/3] chore(examples): add df.show() to execute Example 3 in csv-read-options.py --- examples/csv-read-options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/csv-read-options.py b/examples/csv-read-options.py index f540b6525..ead84d1d8 100644 --- a/examples/csv-read-options.py +++ b/examples/csv-read-options.py @@ -68,6 +68,7 @@ .with_newlines_in_values(True) # noqa: FBT003 ) df = ctx.read_csv(str(csv_file), options=options) + df.show() # Example 4: Advanced options print("\nExample 4: Advanced options") From c0452ed15201b87f204dcee013808cfc157ff211 Mon Sep 17 00:00:00 2001 From: dedsec-terminal Date: Sat, 12 Sep 2026 01:09:29 +0530 Subject: [PATCH 3/3] fix(examples): match builder fixture delimiter Signed-off-by: dedsec-terminal --- examples/csv-read-options.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/csv-read-options.py b/examples/csv-read-options.py index ead84d1d8..e3f333979 100644 --- a/examples/csv-read-options.py +++ b/examples/csv-read-options.py @@ -28,10 +28,14 @@ tmp_path = Path(tmp_dir) csv_file = tmp_path / "data.csv" gz_file = tmp_path / "data.csv.gz" + builder_csv_file = tmp_path / "builder-data.csv" sample_csv = "id,name,value\n1,alice,100\n2,bob,200\n3,charlie,null\n" csv_file.write_text(sample_csv) + builder_csv = "id|name|value\n1|'alice'|100\n2|'bob'|200\n3|'charlie'|null\n" + builder_csv_file.write_text(builder_csv) + with gzip.open(gz_file, "wt") as f: f.write(sample_csv) @@ -67,7 +71,7 @@ .with_truncated_rows(False) # noqa: FBT003 .with_newlines_in_values(True) # noqa: FBT003 ) - df = ctx.read_csv(str(csv_file), options=options) + df = ctx.read_csv(str(builder_csv_file), options=options) df.show() # Example 4: Advanced options