Run examples/*.py in CI, fix the broken example, and give silent ones output

.github/workflows/test.yml

@@ -119,6 +119,47 @@ jobs:
           # free-threaded build and re-pick the system 3.12 (see install step).
           uv run --python "$PWD/.venv/bin/python" --no-project pytest -v --import-mode=importlib
 
+      # Run the top-level examples once. They exercise the public Python API
+      # that the test suite does not: several are plain scripts with no pytest
+      # coverage, so nothing caught a broken example before this job existed.
+      - name: Run example scripts
+        if: matrix.python-version == '3.12'
+        run: |
+          set -euo pipefail
+
+          # Extra runtime dependencies used by some examples but not part of
+          # the dev group: pandas and polars for import.py/export.py, and
+          # matplotlib for sql-to-pandas.py.
+          uv pip install --python "$PWD/.venv/bin/python" pandas polars matplotlib
+
+          # Examples that cannot run here:
+          #   - sql-parquet-s3.py: needs network access and AWS credentials
+          #   - sql-parquet.py, dataframe-parquet.py, sql-to-pandas.py: need the
+          #     NYC taxi parquet file documented in examples/README.md
+          #   - python-udf-comparisons.py: needs the TPC-H dataset generated
+          #     later by the tpchgen-cli step
+          #   - ray_pickle_expr.py: needs the optional (heavy) `ray` dependency
+          skip="sql-parquet-s3.py sql-parquet.py dataframe-parquet.py sql-to-pandas.py python-udf-comparisons.py ray_pickle_expr.py"
+
+          failed=""
+          for script in examples/*.py; do
+            name="$(basename "$script")"
+            if [[ " $skip " == *" $name "* ]]; then
+              echo "Skipping $script"
+              continue
+            fi
+            echo "::group::Running $script"
+            if ! uv run --python "$PWD/.venv/bin/python" --no-project python "$script"; then
+              failed="$failed $script"
+            fi
+            echo "::endgroup::"
+          done
+
+          if [[ -n "$failed" ]]; then
+            echo "Example scripts failed:$failed"
+            exit 1
+          fi
+
       # FFI + TPC-H examples only need to run once; gate to abi3 entries.
       - name: FFI unit tests
         if: matrix.wheel-tag == 'abi3'

examples/csv-read-options.py

@@ -17,15 +17,31 @@
 
 """Example demonstrating CsvReadOptions usage."""
 
+import gzip
+import tempfile
+from pathlib import Path
+
 from datafusion import CsvReadOptions, SessionContext
 
+# Write the CSV files used below into a temporary directory so the example is
+# self-contained and runnable without any external data.
+_tmpdir = tempfile.TemporaryDirectory()
+_data_dir = Path(_tmpdir.name)
+_csv_path = _data_dir / "data.csv"
+_gzip_path = _data_dir / "data.csv.gz"
+
+_csv_path.write_text("a,b,c\n1,4,foo\n2,5,bar\n3,,baz\n")
+with gzip.open(_gzip_path, "wt") as _f:
+    _f.write("a,b,c\n1,4,foo\n2,5,N/A\n3,6,baz\n")
+
 # Create a SessionContext
 ctx = SessionContext()
 
 # Example 1: Using CsvReadOptions with default values
 print("Example 1: Default CsvReadOptions")
 options = CsvReadOptions()
-df = ctx.read_csv("data.csv", options=options)
+df = ctx.read_csv(str(_csv_path), options=options)
+df.show()
 
 # Example 2: Using CsvReadOptions with custom parameters
 print("\nExample 2: Custom CsvReadOptions")
@@ -36,7 +52,8 @@
     schema_infer_max_records=1000,
     file_extension=".csv",
 )
-df = ctx.read_csv("data.csv", options=options)
+df = ctx.read_csv(str(_csv_path), options=options)
+df.show()
 
 # Example 3: Using the builder pattern (recommended for readability)
 print("\nExample 3: Builder pattern")
@@ -49,7 +66,8 @@
     .with_truncated_rows(False)  # noqa: FBT003
     .with_newlines_in_values(True)  # noqa: FBT003
 )
-df = ctx.read_csv("data.csv", options=options)
+df = ctx.read_csv(str(_csv_path), options=options)
+df.show()
 
 # Example 4: Advanced options
 print("\nExample 4: Advanced options")
@@ -64,18 +82,21 @@
     .with_file_compression_type("gzip")  # Read gzipped CSV
     .with_file_extension(".gz")
 )
-df = ctx.read_csv("data.csv.gz", options=options)
+df = ctx.read_csv(str(_gzip_path), options=options)
+df.show()
 
 # Example 5: Register CSV table with options
 print("\nExample 5: Register CSV table")
 options = CsvReadOptions().with_has_header(True).with_delimiter(",")  # noqa: FBT003
-ctx.register_csv("my_table", "data.csv", options=options)
+ctx.register_csv("my_table", str(_csv_path), options=options)
 df = ctx.sql("SELECT * FROM my_table")
+df.show()
 
 # Example 6: Backward compatibility (without options)
 print("\nExample 6: Backward compatibility")
 # Still works the old way!
-df = ctx.read_csv("data.csv", has_header=True, delimiter=",")
+df = ctx.read_csv(str(_csv_path), has_header=True, delimiter=",")
+df.show()
 
 print("\nAll examples completed!")
 print("\nFor all available options, see the CsvReadOptions documentation:")

examples/export.py

@@ -50,3 +50,5 @@
 # export to Python dictionary of columns
 pydict = df.to_pydict()
 assert pydict == {"a": [1, 2, 3], "b": [4, 5, 6]}
+
+df.show()

examples/import.py

@@ -55,3 +55,5 @@
 arrow_table = pa.Table.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]})
 df = ctx.from_arrow(arrow_table)
 assert type(df) is datafusion.DataFrame
+
+df.show()

examples/python-udaf.py

@@ -67,3 +67,5 @@ def evaluate(self) -> pa.Scalar:
 result = df.collect()[0]
 
 assert result.column(0) == pa.array([6.0])
+
+df.show()

examples/python-udf.py

@@ -41,3 +41,5 @@ def is_null(array: pa.Array) -> pa.Array:
 result = df.collect()[0]
 
 assert result.column(0) == pa.array([False] * 3)
+
+df.show()

examples/query-pyarrow-data.py

@@ -40,3 +40,5 @@
 
 assert result.column(0) == pa.array([5, 7, 9])
 assert result.column(1) == pa.array([-3, -3, -3])
+
+df.show()

examples/sql-to-pandas.py

@@ -40,3 +40,5 @@
     kind="bar", title="Trip Count by Number of Passengers"
 ).get_figure()
 fig.savefig("chart.png")
+
+print(pandas_df)

examples/sql-using-python-udaf.py

@@ -84,3 +84,5 @@ def evaluate(self) -> pa.Scalar:
 # +---+--------------+
 assert result_df.to_pydict()["a"] == [1, 3]
 assert result_df.to_pydict()["b_aggregated"] == [9, 6]
+
+result_df.show()

examples/sql-using-python-udf.py

@@ -62,3 +62,5 @@ def is_null(array: pa.Array) -> pa.Array:
 # | 3 | false     |
 # +---+-----------+
 assert result_df.to_pydict()["b_is_null"] == [False, True, False]
+
+result_df.show()

examples/substrait.py

@@ -47,3 +47,5 @@
 # Back to Substrait Plan just for demonstration purposes
 # type(substrait_plan) -> <class 'datafusion.substrait.plan'>
 substrait_plan = ss.Producer.to_substrait_plan(df_logical_plan, ctx)
+
+print(f"Serialized and deserialized a Substrait plan ({len(substrait_bytes)} bytes).")