From 67db06d996c2f6c9725ff1d18b57144330404eed Mon Sep 17 00:00:00 2001 From: Salih Date: Sun, 13 Sep 2026 17:34:08 +0300 Subject: [PATCH] Run examples/*.py in CI, fix the broken example, and give silent ones output Nothing in CI ran the top-level examples/*.py scripts, so they drifted out of sync with the API. examples/csv-read-options.py crashed because it reads data.csv and data.csv.gz that do not exist in the repository, and nine other examples end in asserts without printing anything, so a reader cannot tell a working example from a no-op. - Make csv-read-options.py self-contained: it now writes its own small CSV and gzipped CSV into a temporary directory. - Add a terminal call to the examples that printed nothing so each one shows its result. - Add a CI step, gated to the 3.12 abi3 entry, that runs every examples/*.py script against the built wheel with an explicit skip list for examples that need network/credentials, hand-downloaded data, generated TPC-H data, or the optional Ray dependency. Closes #1728 --- .github/workflows/test.yml | 41 +++++++++++++++++++++++++++++++ examples/csv-read-options.py | 33 ++++++++++++++++++++----- examples/export.py | 2 ++ examples/import.py | 2 ++ examples/python-udaf.py | 2 ++ examples/python-udf.py | 2 ++ examples/query-pyarrow-data.py | 2 ++ examples/sql-to-pandas.py | 2 ++ examples/sql-using-python-udaf.py | 2 ++ examples/sql-using-python-udf.py | 2 ++ examples/substrait.py | 2 ++ 11 files changed, 86 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 881548073..6741d221c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,6 +119,47 @@ jobs: # free-threaded build and re-pick the system 3.12 (see install step). uv run --python "$PWD/.venv/bin/python" --no-project pytest -v --import-mode=importlib + # Run the top-level examples once. They exercise the public Python API + # that the test suite does not: several are plain scripts with no pytest + # coverage, so nothing caught a broken example before this job existed. + - name: Run example scripts + if: matrix.python-version == '3.12' + run: | + set -euo pipefail + + # Extra runtime dependencies used by some examples but not part of + # the dev group: pandas and polars for import.py/export.py, and + # matplotlib for sql-to-pandas.py. + uv pip install --python "$PWD/.venv/bin/python" pandas polars matplotlib + + # Examples that cannot run here: + # - sql-parquet-s3.py: needs network access and AWS credentials + # - sql-parquet.py, dataframe-parquet.py, sql-to-pandas.py: need the + # NYC taxi parquet file documented in examples/README.md + # - python-udf-comparisons.py: needs the TPC-H dataset generated + # later by the tpchgen-cli step + # - ray_pickle_expr.py: needs the optional (heavy) `ray` dependency + skip="sql-parquet-s3.py sql-parquet.py dataframe-parquet.py sql-to-pandas.py python-udf-comparisons.py ray_pickle_expr.py" + + failed="" + for script in examples/*.py; do + name="$(basename "$script")" + if [[ " $skip " == *" $name "* ]]; then + echo "Skipping $script" + continue + fi + echo "::group::Running $script" + if ! uv run --python "$PWD/.venv/bin/python" --no-project python "$script"; then + failed="$failed $script" + fi + echo "::endgroup::" + done + + if [[ -n "$failed" ]]; then + echo "Example scripts failed:$failed" + exit 1 + fi + # FFI + TPC-H examples only need to run once; gate to abi3 entries. - name: FFI unit tests if: matrix.wheel-tag == 'abi3' diff --git a/examples/csv-read-options.py b/examples/csv-read-options.py index a5952d950..49d5483e2 100644 --- a/examples/csv-read-options.py +++ b/examples/csv-read-options.py @@ -17,15 +17,31 @@ """Example demonstrating CsvReadOptions usage.""" +import gzip +import tempfile +from pathlib import Path + from datafusion import CsvReadOptions, SessionContext +# Write the CSV files used below into a temporary directory so the example is +# self-contained and runnable without any external data. +_tmpdir = tempfile.TemporaryDirectory() +_data_dir = Path(_tmpdir.name) +_csv_path = _data_dir / "data.csv" +_gzip_path = _data_dir / "data.csv.gz" + +_csv_path.write_text("a,b,c\n1,4,foo\n2,5,bar\n3,,baz\n") +with gzip.open(_gzip_path, "wt") as _f: + _f.write("a,b,c\n1,4,foo\n2,5,N/A\n3,6,baz\n") + # Create a SessionContext ctx = SessionContext() # Example 1: Using CsvReadOptions with default values print("Example 1: Default CsvReadOptions") options = CsvReadOptions() -df = ctx.read_csv("data.csv", options=options) +df = ctx.read_csv(str(_csv_path), options=options) +df.show() # Example 2: Using CsvReadOptions with custom parameters print("\nExample 2: Custom CsvReadOptions") @@ -36,7 +52,8 @@ schema_infer_max_records=1000, file_extension=".csv", ) -df = ctx.read_csv("data.csv", options=options) +df = ctx.read_csv(str(_csv_path), options=options) +df.show() # Example 3: Using the builder pattern (recommended for readability) print("\nExample 3: Builder pattern") @@ -49,7 +66,8 @@ .with_truncated_rows(False) # noqa: FBT003 .with_newlines_in_values(True) # noqa: FBT003 ) -df = ctx.read_csv("data.csv", options=options) +df = ctx.read_csv(str(_csv_path), options=options) +df.show() # Example 4: Advanced options print("\nExample 4: Advanced options") @@ -64,18 +82,21 @@ .with_file_compression_type("gzip") # Read gzipped CSV .with_file_extension(".gz") ) -df = ctx.read_csv("data.csv.gz", options=options) +df = ctx.read_csv(str(_gzip_path), options=options) +df.show() # Example 5: Register CSV table with options print("\nExample 5: Register CSV table") options = CsvReadOptions().with_has_header(True).with_delimiter(",") # noqa: FBT003 -ctx.register_csv("my_table", "data.csv", options=options) +ctx.register_csv("my_table", str(_csv_path), options=options) df = ctx.sql("SELECT * FROM my_table") +df.show() # Example 6: Backward compatibility (without options) print("\nExample 6: Backward compatibility") # Still works the old way! -df = ctx.read_csv("data.csv", has_header=True, delimiter=",") +df = ctx.read_csv(str(_csv_path), has_header=True, delimiter=",") +df.show() print("\nAll examples completed!") print("\nFor all available options, see the CsvReadOptions documentation:") diff --git a/examples/export.py b/examples/export.py index c7a387bcb..348c3290c 100644 --- a/examples/export.py +++ b/examples/export.py @@ -50,3 +50,5 @@ # export to Python dictionary of columns pydict = df.to_pydict() assert pydict == {"a": [1, 2, 3], "b": [4, 5, 6]} + +df.show() diff --git a/examples/import.py b/examples/import.py index 7b5ab5082..4fec1dee9 100644 --- a/examples/import.py +++ b/examples/import.py @@ -55,3 +55,5 @@ arrow_table = pa.Table.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]}) df = ctx.from_arrow(arrow_table) assert type(df) is datafusion.DataFrame + +df.show() diff --git a/examples/python-udaf.py b/examples/python-udaf.py index 6655edb0a..3a8f24d57 100644 --- a/examples/python-udaf.py +++ b/examples/python-udaf.py @@ -67,3 +67,5 @@ def evaluate(self) -> pa.Scalar: result = df.collect()[0] assert result.column(0) == pa.array([6.0]) + +df.show() diff --git a/examples/python-udf.py b/examples/python-udf.py index 1c08acd1a..ffeaa4f7a 100644 --- a/examples/python-udf.py +++ b/examples/python-udf.py @@ -41,3 +41,5 @@ def is_null(array: pa.Array) -> pa.Array: result = df.collect()[0] assert result.column(0) == pa.array([False] * 3) + +df.show() diff --git a/examples/query-pyarrow-data.py b/examples/query-pyarrow-data.py index 9cfe8a62b..1b338f1de 100644 --- a/examples/query-pyarrow-data.py +++ b/examples/query-pyarrow-data.py @@ -40,3 +40,5 @@ assert result.column(0) == pa.array([5, 7, 9]) assert result.column(1) == pa.array([-3, -3, -3]) + +df.show() diff --git a/examples/sql-to-pandas.py b/examples/sql-to-pandas.py index 34f7bde1b..9326078b2 100644 --- a/examples/sql-to-pandas.py +++ b/examples/sql-to-pandas.py @@ -40,3 +40,5 @@ kind="bar", title="Trip Count by Number of Passengers" ).get_figure() fig.savefig("chart.png") + +print(pandas_df) diff --git a/examples/sql-using-python-udaf.py b/examples/sql-using-python-udaf.py index f42bbdc23..a9a32c207 100644 --- a/examples/sql-using-python-udaf.py +++ b/examples/sql-using-python-udaf.py @@ -84,3 +84,5 @@ def evaluate(self) -> pa.Scalar: # +---+--------------+ assert result_df.to_pydict()["a"] == [1, 3] assert result_df.to_pydict()["b_aggregated"] == [9, 6] + +result_df.show() diff --git a/examples/sql-using-python-udf.py b/examples/sql-using-python-udf.py index 2f0a0b67d..28d4f05e3 100644 --- a/examples/sql-using-python-udf.py +++ b/examples/sql-using-python-udf.py @@ -62,3 +62,5 @@ def is_null(array: pa.Array) -> pa.Array: # | 3 | false | # +---+-----------+ assert result_df.to_pydict()["b_is_null"] == [False, True, False] + +result_df.show() diff --git a/examples/substrait.py b/examples/substrait.py index fa6f77912..b1a313587 100644 --- a/examples/substrait.py +++ b/examples/substrait.py @@ -47,3 +47,5 @@ # Back to Substrait Plan just for demonstration purposes # type(substrait_plan) -> substrait_plan = ss.Producer.to_substrait_plan(df_logical_plan, ctx) + +print(f"Serialized and deserialized a Substrait plan ({len(substrait_bytes)} bytes).")