From bc1d67c86b816205119b548f682875f7fb1d57db Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Thu, 5 Nov 2026 00:13:01 -0800 Subject: [PATCH 1/3] Benchmark two CI shards and add a required aggregate check --- .github/workflows/lint.yml | 30 ------------------------ .github/workflows/tests.yml | 46 +++++++++++++++++++++++++++++++++---- CONTRIBUTING.md | 9 +++++--- script/parallel_rspec.rb | 13 ++++++++--- spec/spec_shard_spec.rb | 42 +++++++++++++++++++++++++++++++++ spec/support/spec_shard.rb | 23 +++++++++++++++++++ 6 files changed, 123 insertions(+), 40 deletions(-) delete mode 100644 .github/workflows/lint.yml create mode 100644 spec/spec_shard_spec.rb create mode 100644 spec/support/spec_shard.rb diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index c53b7193..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Lint -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - overcommit: - timeout-minutes: 10 - runs-on: ubuntu-26.04 - - steps: - - uses: actions/checkout@v4 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: 3.3 - bundler-cache: true - - - name: Prepare environment - run: | - git config --global user.email "gh-actions@example.com" - git config --global user.name "GitHub Actions" - bundle exec overcommit --sign - bundle exec overcommit --sign pre-commit - - - name: Run pre-commit checks - run: bundle exec overcommit --run diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c01908d6..f6826b08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,18 +1,19 @@ name: Tests on: push: - branches: [main] + branches: [main, ci/shard-tests] pull_request: branches: [main] jobs: rspec: - name: rspec (${{ matrix.ruby-version }}, ${{ matrix.os }}) + name: rspec (${{ matrix.ruby-version }}, ${{ matrix.os }}, shard ${{ matrix.shard }}) timeout-minutes: ${{ matrix.os == 'windows' && 30 || 15 }} runs-on: ${{ format('{0}-latest', matrix.os) }} env: BUNDLE_WITHOUT: lint SPEC_WORKERS: ${{ matrix.os == 'windows' && 4 || 8 }} + SPEC_SHARD: ${{ matrix.shard }} strategy: fail-fast: false @@ -24,6 +25,7 @@ jobs: os: - ubuntu - windows + shard: ["1/2", "2/2"] steps: - uses: actions/checkout@v4 @@ -54,12 +56,12 @@ jobs: uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.github_token }} - flag-name: ruby${{ matrix.ruby-version }}-${{ matrix.os }} + flag-name: ruby${{ matrix.ruby-version }}-${{ matrix.os }}-shard${{ strategy.job-index }} parallel: true finish: needs: rspec - runs-on: ubuntu-26.04 + runs-on: ubuntu-latest steps: - name: Finalize code coverage report @@ -67,3 +69,39 @@ jobs: with: github-token: ${{ secrets.github_token }} parallel-finished: true + + overcommit: + timeout-minutes: 10 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.3 + bundler-cache: true + + - name: Prepare environment + run: | + git config --global user.email "gh-actions@example.com" + git config --global user.name "GitHub Actions" + bundle exec overcommit --sign + bundle exec overcommit --sign pre-commit + + - name: Run pre-commit checks + run: bundle exec overcommit --run + + ci: + name: CI + if: ${{ always() }} + needs: [rspec, overcommit, finish] + runs-on: ubuntu-latest + timeout-minutes: 2 + + steps: + - name: Require all jobs to pass + env: + JOB_RESULTS: ${{ toJSON(needs) }} + run: echo "$JOB_RESULTS" | jq -e 'all(.[]; .result == "success")' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b17ff1c2..82f9cf65 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,13 +45,16 @@ The best way to see a feature added, however, is to submit a pull request. bundle exec ruby script/parallel_rspec.rb # Optionally choose the number of worker processes (default: up to 8 CPUs): SPEC_WORKERS=4 bundle exec ruby script/parallel_rspec.rb + # Run one runner's share of the suite (default: 1/1, the whole suite): + SPEC_SHARD=1/3 bundle exec ruby script/parallel_rspec.rb ``` Workers run in separate processes because specs change the working directory and environment. The runner checks that every discovered example ran exactly - once and merges coverage from all workers. You can also pass spec paths to - run a subset. The `lint` dependency group is only needed for lint checks; - test-only CI jobs install with `BUNDLE_WITHOUT=lint`. + once within its shard and merges coverage from all workers. Shards partition + examples in a deterministic order; run every shard to cover the complete suite. + You can also pass spec paths to run a subset. The `lint` dependency group is + only needed for lint checks; test-only CI jobs install with `BUNDLE_WITHOUT=lint`. * Submit your pull request! diff --git a/script/parallel_rspec.rb b/script/parallel_rspec.rb index 0d134a60..08a70c58 100644 --- a/script/parallel_rspec.rb +++ b/script/parallel_rspec.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true require 'etc' +require 'digest' require 'json' require 'rbconfig' require 'tmpdir' +require_relative '../spec/support/spec_shard' if ARGV.first == '--worker' require 'rspec/core' @@ -15,6 +17,7 @@ # spec files don't leave one worker busy after the others have finished. workers = Integer(ENV.fetch('SPEC_WORKERS') { [Etc.nprocessors, 8].min.to_s }) abort 'SPEC_WORKERS must be positive' unless workers > 0 +shard = SpecShard.new(ENV.fetch('SPEC_SHARD', '1/1')) Dir.chdir(File.expand_path('..', __dir__)) rspec = [RbConfig.ruby, Gem.bin_path('rspec-core', 'rspec')] @@ -27,12 +30,16 @@ '--dry-run', '--format', 'json', '--out', discovery) abort 'RSpec discovery failed' end - ids = JSON.parse(File.read(discovery)).fetch('examples').map { |example| example.fetch('id') } - abort 'RSpec discovery found no examples' if ids.empty? + all_ids = JSON.parse(File.read(discovery)).fetch('examples').map { |example| example.fetch('id') } + abort 'RSpec discovery found no examples' if all_ids.empty? + + ids = shard.select(all_ids) + abort "Shard #{shard} has no examples" if ids.empty? + puts "Discovered #{all_ids.size} examples (SHA256: #{Digest::SHA256.hexdigest(all_ids.join("\n"))})" groups = Array.new([workers, ids.size].min) { [] } ids.each_with_index { |id, index| groups[index % groups.size] << id } - puts "Running all #{ids.size} examples in #{groups.size} processes" + puts "Running shard #{shard}: #{ids.size}/#{all_ids.size} examples in #{groups.size} processes" processes = groups.each_with_index.map do |group, index| prefix = File.join(tmpdir, "worker-#{index}") diff --git a/spec/spec_shard_spec.rb b/spec/spec_shard_spec.rb new file mode 100644 index 00000000..258fc606 --- /dev/null +++ b/spec/spec_shard_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe SpecShard do + let(:examples) { (1..17).map { |index| "example-#{index}" } } + + it 'runs the complete suite when using a single shard' do + described_class.new('1/1').select(examples).should == examples + end + + [2, 3].each do |total| + context "with #{total} shards" do + let(:shards) do + (1..total).map { |index| described_class.new("#{index}/#{total}").select(examples) } + end + + it 'assigns every example to exactly one shard' do + shards.flatten.sort.should == examples.sort + end + + it 'balances example counts within one example' do + counts = shards.map(&:size) + (counts.max - counts.min).should <= 1 + end + + it 'assigns examples deterministically in their original order' do + shards.first.should == examples.each_slice(total).map(&:first) + end + end + end + + it 'handles more shards than examples without duplicating examples' do + described_class.new('3/3').select(['only-example']).should == [] + end + + ['', '1', '0/2', '1/0', '3/2', '-1/2', '1/2/3', '1/2junk'].each do |value| + it "rejects the invalid shard #{value.inspect}" do + expect { described_class.new(value) }.to raise_error(ArgumentError, /SPEC_SHARD/) + end + end +end diff --git a/spec/support/spec_shard.rb b/spec/support/spec_shard.rb new file mode 100644 index 00000000..a77c6387 --- /dev/null +++ b/spec/support/spec_shard.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Deterministically partitions examples across CI runners before each runner +# distributes its share among local worker processes. +class SpecShard + attr_reader :index, :total + + def initialize(value) + match = %r{\A([1-9]\d*)/([1-9]\d*)\z}.match(value) + raise ArgumentError, 'SPEC_SHARD must be INDEX/TOTAL (for example, 1/3)' unless match + + @index, @total = match.captures.map(&:to_i) + raise ArgumentError, 'SPEC_SHARD index must not exceed total' if index > total + end + + def select(examples) + examples.each_with_index.select { |_, offset| offset % total == index - 1 }.map(&:first) + end + + def to_s + "#{index}/#{total}" + end +end From e372511e405713eded7a6181163b99f5aaa14372 Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Thu, 5 Nov 2026 00:16:24 -0800 Subject: [PATCH 2/3] Benchmark three CI shards --- .github/workflows/tests.yml | 2 +- CONTRIBUTING.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6826b08..390078f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,7 +25,7 @@ jobs: os: - ubuntu - windows - shard: ["1/2", "2/2"] + shard: ["1/3", "2/3", "3/3"] steps: - uses: actions/checkout@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82f9cf65..be9a4121 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,6 +56,9 @@ The best way to see a feature added, however, is to submit a pull request. You can also pass spec paths to run a subset. The `lint` dependency group is only needed for lint checks; test-only CI jobs install with `BUNDLE_WITHOUT=lint`. + The final `CI` check depends on all test shards, lint, and coverage reporting. + Repository rulesets can require this single check instead of individual matrix jobs. + * Submit your pull request! All pull requests will be tested against [Travis CI][3], where the following From ce3e5087eee0e70167242204012ba36cd89f526a Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Thu, 5 Nov 2026 00:20:20 -0800 Subject: [PATCH 3/3] Finalize three-shard CI with one aggregate required check --- .github/workflows/tests.yml | 21 ++++++++------------- CONTRIBUTING.md | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 390078f3..964c2103 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ name: Tests on: push: - branches: [main, ci/shard-tests] + branches: [main] pull_request: branches: [main] @@ -59,17 +59,6 @@ jobs: flag-name: ruby${{ matrix.ruby-version }}-${{ matrix.os }}-shard${{ strategy.job-index }} parallel: true - finish: - needs: rspec - runs-on: ubuntu-latest - - steps: - - name: Finalize code coverage report - uses: coverallsapp/github-action@v2 - with: - github-token: ${{ secrets.github_token }} - parallel-finished: true - overcommit: timeout-minutes: 10 runs-on: ubuntu-latest @@ -96,7 +85,7 @@ jobs: ci: name: CI if: ${{ always() }} - needs: [rspec, overcommit, finish] + needs: [rspec, overcommit] runs-on: ubuntu-latest timeout-minutes: 2 @@ -105,3 +94,9 @@ jobs: env: JOB_RESULTS: ${{ toJSON(needs) }} run: echo "$JOB_RESULTS" | jq -e 'all(.[]; .result == "success")' + + - name: Finalize code coverage report + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.github_token }} + parallel-finished: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index be9a4121..83dfc146 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ The best way to see a feature added, however, is to submit a pull request. You can also pass spec paths to run a subset. The `lint` dependency group is only needed for lint checks; test-only CI jobs install with `BUNDLE_WITHOUT=lint`. - The final `CI` check depends on all test shards, lint, and coverage reporting. + The final `CI` check depends on all test shards and lint, and finalizes coverage reporting. Repository rulesets can require this single check instead of individual matrix jobs. * Submit your pull request!