From e1cbb4ced2b91db11c228906075e7e4a63f66e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Tue, 26 May 2026 21:23:40 +0200 Subject: [PATCH] Use nextest for CI test jobs --- .asf.yaml | 1 - .github/workflows/extended.yml | 28 +++++-- .github/workflows/rust.yml | 46 +++++++++--- datafusion/sqllogictest/bin/sqllogictests.rs | 77 +++++++++++++++----- datafusion/sqllogictest/src/filters.rs | 13 ++++ 5 files changed, 127 insertions(+), 38 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 7317c9cbaed02..ce93dfd97f7da 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -114,4 +114,3 @@ github: # https://datafusion.apache.org/ publish: whoami: asf-site - diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index a143cb49fd35b..65c421add6180 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -59,7 +59,7 @@ permissions: jobs: # Run extended tests (with feature 'extended_tests') linux-test-extended: - name: cargo test 'extended_tests' (amd64) + name: cargo nextest 'extended_tests' (amd64) runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: @@ -80,16 +80,20 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest # For debugging, test binaries can be large. - name: Show available disk space run: | df -h - - name: Run tests (excluding doctests) + - name: Run tests with nextest (excluding doctests) env: RUST_BACKTRACE: 1 run: | - cargo test \ - --profile ci \ + cargo nextest run \ + --cargo-profile ci \ --exclude datafusion-examples \ --exclude datafusion-benchmarks \ --exclude datafusion-cli \ @@ -105,7 +109,7 @@ jobs: # Check answers are correct when hash values collide hash-collisions: - name: cargo test hash collisions (amd64) + name: cargo nextest hash collisions (amd64) runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -120,10 +124,14 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest - name: Run tests run: | cd datafusion - cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --exclude datafusion-cli --workspace --lib --tests --features=force_hash_collisions,avro + cargo nextest run --cargo-profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --exclude datafusion-cli --workspace --lib --tests --features=force_hash_collisions,avro cargo clean sqllogictest-sqlite: @@ -142,6 +150,12 @@ jobs: - name: Install protobuf compiler run: | apt-get update && apt-get install -y protobuf-compiler + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest - name: Run sqllogictest + env: + INCLUDE_SQLITE: true run: | - cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite \ No newline at end of file + cargo nextest run --features backtrace,parquet_encryption --cargo-profile ci-optimized --test sqllogictests diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bb2075ee018c3..afd49a70cd9ca 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,12 +290,16 @@ jobs: with: save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci" - - name: Run tests (excluding doctests and datafusion-cli) + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest + - name: Run tests with nextest (excluding doctests and datafusion-cli) env: RUST_BACKTRACE: 1 run: | - cargo test \ - --profile ci \ + cargo nextest run \ + --cargo-profile ci \ --exclude datafusion-examples \ --exclude ffi_example_table_provider \ --exclude datafusion-cli \ @@ -334,7 +338,11 @@ jobs: with: save-if: false # set in linux-test shared-key: "amd-ci" - - name: Run tests (excluding doctests) + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest + - name: Run tests with nextest (excluding doctests) env: RUST_BACKTRACE: 1 AWS_ENDPOINT: http://127.0.0.1:9000 @@ -342,7 +350,7 @@ jobs: AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword TEST_STORAGE_INTEGRATION: 1 AWS_ALLOW_HTTP: true - run: cargo test --features backtrace --profile ci -p datafusion-cli --lib --tests --bins + run: cargo nextest run --features backtrace --cargo-profile ci -p datafusion-cli --lib --tests --bins - name: Verify Working Directory Clean run: git diff --exit-code @@ -457,6 +465,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest - name: Generate benchmark data and expected query results run: | mkdir -p datafusion/sqllogictest/test_files/tpch/data @@ -470,8 +482,8 @@ jobs: # increase stack size to fix stack overflow export RUST_MIN_STACK=20971520 export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests + cargo nextest run --package datafusion-benchmarks --cargo-profile ci --features=ci --test-threads=1 plan_q + INCLUDE_TPCH=true cargo nextest run --features backtrace,parquet_encryption,substrait --cargo-profile ci --package datafusion-sqllogictest --test sqllogictests - name: Verify Working Directory Clean run: git diff --exit-code @@ -505,10 +517,14 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest - name: Run sqllogictest run: | cd datafusion/sqllogictest - PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests + PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo nextest run --features backtrace --cargo-profile ci --features=postgres --test sqllogictests env: # use postgres for the host here because we have specified a container for the job POSTGRES_HOST: postgres @@ -530,12 +546,18 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest - name: Run sqllogictest # TODO: Right now several tests are failing in Substrait round-trip mode, so this # command cannot be run for all the .slt files. Run it for just one that works (limit.slt) # until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed # and this command can be run without filters. - run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt + env: + SUBSTRAIT_ROUND_TRIP: true + run: cargo nextest run -p datafusion-sqllogictest --test sqllogictests --features substrait limit.slt # Temporarily commenting out the Windows flow, the reason is enormously slow running build # Waiting for new Windows 2025 github runner @@ -571,9 +593,13 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-macos-aarch64-builder + - name: Install nextest + uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d # v2.79.2 + with: + tool: nextest - name: Run datafusion-ffi tests shell: bash - run: cargo test --profile ci -p datafusion-ffi --lib --tests --features integration-tests + run: cargo nextest run --cargo-profile ci -p datafusion-ffi --lib --tests --features integration-tests vendor: name: Verify Vendored Code diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 69ae3a2fa7dd3..307d1f8f4f979 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -84,10 +84,18 @@ fn config_change_result( } pub fn main() -> Result<()> { + // Enable logging (e.g. set RUST_LOG=debug to see debug logs) + env_logger::init(); + + let options: Options = Parser::parse(); + if options.list { + return list_tests(&options); + } + tokio::runtime::Builder::new_multi_thread() .enable_all() .build()? - .block_on(run_tests()) + .block_on(run_tests(options)) } fn sqlite_value_validator( @@ -119,21 +127,8 @@ fn sqlite_value_validator( normalized_actual == normalized_expected } -async fn run_tests() -> Result<()> { - // Enable logging (e.g. set RUST_LOG=debug to see debug logs) - env_logger::init(); - - let options: Options = Parser::parse(); +async fn run_tests(options: Options) -> Result<()> { let timing_debug_slow_files = is_env_truthy(TIMING_DEBUG_SLOW_FILES_ENV); - if options.list { - // nextest parses stdout, so print messages to stderr - eprintln!("NOTICE: --list option unsupported, quitting"); - // return Ok, not error so that tools like nextest which are listing all - // workspace tests (by running `cargo test ... --list --format terse`) - // do not fail when they encounter this binary. Instead, print nothing - // to stdout and return OK so they can continue listing other tests. - return Ok(()); - } options.warn_on_ignored(); @@ -428,6 +423,18 @@ fn is_env_truthy(name: &str) -> bool { }) } +fn list_tests(options: &Options) -> Result<()> { + if options.ignored { + return Ok(()); + } + + for test_file in read_test_files(options)? { + println!("{}: test", test_file.relative_path.display()); + } + + Ok(()) +} + #[cfg(feature = "substrait")] async fn run_test_file_substrait_round_trip( test_file: TestFile, @@ -801,7 +808,7 @@ fn read_test_files(options: &Options) -> Result> { .into_iter() .flatten() .map(|p| TestFile::new(p, prefixes)) - .filter(|f| options.check_test_file(&f.path)) + .filter(|f| options.check_test_file(f)) .filter(|f| f.is_slt_file()) .filter(|f| !f.relative_path_starts_with(TPCH_PREFIX) || options.include_tpch) .filter(|f| !f.relative_path_starts_with(SQLITE_PREFIX) || options.include_sqlite) @@ -833,6 +840,7 @@ struct Options { #[clap( long, + env = "SUBSTRAIT_ROUND_TRIP", conflicts_with = "complete", conflicts_with = "postgres_runner", help = "Before executing each query, convert its logical plan to Substrait and from Substrait back to its logical plan" @@ -882,6 +890,25 @@ struct Options { )] ignored: bool, + #[clap( + long, + help = "IGNORED (for compatibility with built-in rust test runner)" + )] + include_ignored: bool, + + #[clap( + long, + help = "IGNORED (for compatibility with built-in rust test runner)" + )] + exact: bool, + + #[clap( + long, + value_name = "PATTERN", + help = "IGNORED (for compatibility with built-in rust test runner)" + )] + skip: Vec, + #[clap( long, help = "IGNORED (for compatibility with built-in rust test runner)" @@ -922,10 +949,20 @@ impl Options { /// Will end up passing `foo` as a command line argument. /// /// To be compatible with this, treat the command line arguments as a - /// filter and that does a substring match on each input. returns - /// true f this path should be run - fn check_test_file(&self, path: &Path) -> bool { - !should_skip_file(path, &self.filters) + /// filter that does a substring match on each input. Returns + /// true if this path should be run. + fn check_test_file(&self, test_file: &TestFile) -> bool { + if self.filters.is_empty() { + return true; + } + + if self.exact && self.filters.iter().all(Filter::is_file_filter) { + return self.filters.iter().any(|filter| { + filter.matches_exact_relative_path(&test_file.relative_path) + }); + } + + !should_skip_file(&test_file.path, &self.filters) } /// Postgres runner executes only tests in files with specific names or in diff --git a/datafusion/sqllogictest/src/filters.rs b/datafusion/sqllogictest/src/filters.rs index 568fa3f66676e..03452d99fde68 100644 --- a/datafusion/sqllogictest/src/filters.rs +++ b/datafusion/sqllogictest/src/filters.rs @@ -62,6 +62,19 @@ impl FromStr for Filter { } } +impl Filter { + /// Returns true if this filter selects a file without narrowing to a line. + pub fn is_file_filter(&self) -> bool { + self.line_number.is_none() + } + + /// Returns true if this filter exactly matches the test file's relative path. + pub fn matches_exact_relative_path(&self, relative_path: &Path) -> bool { + self.line_number.is_none() + && relative_path.to_string_lossy() == self.file_substring + } +} + /// Given a list of [`Filter`]s, determines if the whole file in the provided /// path can be skipped. ///