Skip to content

GH-50046: [CI][C++] Improve caching with apache/infrastructure-actions/stash and more general cache keys #50047

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rok wants to merge 6 commits into
base: main
Choose a base branch
from
+110 −40
Open

GH-50046: [CI][C++] Improve caching with apache/infrastructure-actions/stash and more general cache keys #50047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a893c1
store to ccache only from main
rok May 26, 2026
afb1a1f
save from main if not cancelled and before test
rok May 27, 2026
3b73f71
Cache steps now continue-on-error: true
rok May 27, 2026
6f83ce4
switch to apache/infrastructure-actions/stash
rok Jun 1, 2026
cf15c34
stash on non-main branches
rok Jun 2, 2026
f465a72
Apply suggestions from code review
rok Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 32 additions & 12 deletions .github/workflows/cpp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ concurrency:
cancel-in-progress: true

permissions:
actions: read
contents: read
Comment thread
rok marked this conversation as resolved.

env:
Expand Down Expand Up @@ -119,12 +120,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
submodules: recursive
- name: Cache Docker Volumes
uses: actions/cache@v5
- name: Restore Docker Volumes
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
restore-keys: ${{ matrix.image }}-
key: cpp-${{ matrix.image }}
- name: Setup Python on hosted runner
Comment thread
rok marked this conversation as resolved.
if: |
matrix.runs-on == 'ubuntu-latest'
Expand All @@ -149,6 +149,14 @@ jobs:
sudo sysctl -w vm.mmap_rnd_bits=28
source ci/scripts/util_enable_core_dumps.sh
archery docker run ${{ matrix.image }}
- name: Save Docker Volumes
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: cpp-${{ matrix.image }}
include-hidden-files: true
- name: Docker Push
if: >-
success() &&
Expand Down Expand Up @@ -254,12 +262,11 @@ jobs:
id: ccache-info
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v5
- name: Restore ccache
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-ccache-macos-${{ matrix.macos-version }}-
key: cpp-ccache-macos-${{ matrix.macos-version }}
- name: Build
run: |
if [ "${{ matrix.macos-version }}" = "15-intel" ]; then
Expand All @@ -276,6 +283,13 @@ jobs:
export BUILD_WARNING_LEVEL=PRODUCTION
fi
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- name: Save ccache
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-macos-${{ matrix.macos-version }}
- name: Test
shell: bash
run: |
Expand Down Expand Up @@ -361,17 +375,23 @@ jobs:
- name: Setup MSYS2
shell: msys2 {0}
run: ci/scripts/msys2_setup.sh cpp
- name: Cache ccache
uses: actions/cache@v5
- name: Restore ccache
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ccache
key: cpp-ccache-${{ matrix.msystem_lower}}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-ccache-${{ matrix.msystem_lower}}-
key: cpp-ccache-${{ matrix.msystem_lower}}
- name: Build
shell: msys2 {0}
run: |
export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
- name: Save ccache
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ccache
key: cpp-ccache-${{ matrix.msystem_lower}}
- name: Download Timezone Database
if: matrix.msystem_upper == 'CLANG64'
shell: bash
Expand Down
91 changes: 67 additions & 24 deletions .github/workflows/cpp_extra.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,12 +141,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
submodules: recursive
- name: Cache Docker Volumes
uses: actions/cache@v5
- name: Restore Docker Volumes
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }}
restore-keys: extra-${{ matrix.image }}-
key: cpp-extra-${{ matrix.image }}
- name: Setup Python
uses: actions/setup-python@v6
with:
Expand All @@ -169,6 +168,14 @@ jobs:
done
fi
archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }}
- name: Save Docker Volumes
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: cpp-extra-${{ matrix.image }}
include-hidden-files: true
Comment thread
rok marked this conversation as resolved.
- name: Docker Push
if: >-
success() &&
Expand Down Expand Up @@ -204,6 +211,8 @@ jobs:
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
timeout-minutes: 240
permissions:
actions: read
contents: read
# This is for using GitHub Packages for vcpkg cache
packages: write
Comment thread
rok marked this conversation as resolved.
strategy:
Expand All @@ -227,12 +236,11 @@ jobs:
- name: Free up disk space
run: |
ci/scripts/util_free_space.sh
- name: Cache Docker Volumes
uses: actions/cache@v5
- name: Restore Docker Volumes
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }}
restore-keys: jni-${{ matrix.platform.runs-on }}-
key: jni-${{ matrix.platform.runs-on }}
- name: Setup Python
uses: actions/setup-python@v6
with:
Expand All @@ -248,6 +256,14 @@ jobs:
run: |
source ci/scripts/util_enable_core_dumps.sh
archery docker run cpp-jni
- name: Save Docker Volumes
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: jni-${{ matrix.platform.runs-on }}
include-hidden-files: true
- name: Docker Push
if: >-
success() &&
Expand Down Expand Up @@ -300,12 +316,11 @@ jobs:
- name: Prepare ccache
run: |
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
- name: Cache ccache
uses: actions/cache@v5
- name: Restore ccache
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ccache
key: jni-macos-${{ hashFiles('cpp/**') }}
restore-keys: jni-macos-
key: jni-macos
- name: CMake
run: |
cmake \
Expand Down Expand Up @@ -347,6 +362,13 @@ jobs:
cmake --build cpp/examples/minimal_build.build
cd cpp/examples/minimal_build
../minimal_build.build/arrow-example
- name: Save ccache
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ccache
key: jni-macos

odbc-linux:
needs: check-labels
Expand All @@ -372,12 +394,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
submodules: recursive
- name: Cache Docker Volumes
uses: actions/cache@v5
- name: Restore Docker Volumes
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: ubuntu-cpp-odbc-${{ hashFiles('cpp/**') }}
restore-keys: ubuntu-cpp-odbc-
key: ubuntu-cpp-odbc
- name: Setup Python on hosted runner
uses: actions/setup-python@v6
with:
Expand All @@ -393,6 +414,14 @@ jobs:
sudo sysctl -w vm.mmap_rnd_bits=28
source ci/scripts/util_enable_core_dumps.sh
archery docker run ubuntu-cpp-odbc
- name: Save Docker Volumes
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
key: ubuntu-cpp-odbc
include-hidden-files: true
- name: Docker Push
if: >-
success() &&
Expand Down Expand Up @@ -464,12 +493,11 @@ jobs:
id: ccache-info
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v5.0.2
- name: Restore ccache
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{ matrix.build-type }}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{ matrix.build-type }}-
key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{ matrix.build-type }}
- name: Build
run: |
# Homebrew uses /usr/local as prefix. So packages
Expand All @@ -486,6 +514,13 @@ jobs:
export ARROW_CMAKE_ARGS="-DODBC_INCLUDE_DIR=$ODBC_INCLUDE_DIR"
export CXXFLAGS="$CXXFLAGS -I$ODBC_INCLUDE_DIR"
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- name: Save ccache
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{ matrix.build-type }}
- name: Setup Python
uses: actions/setup-python@v6
with:
Expand Down Expand Up @@ -541,6 +576,8 @@ jobs:
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
timeout-minutes: 240
permissions:
actions: read
contents: read
packages: write
Comment thread
rok marked this conversation as resolved.
env:
ARROW_BUILD_SHARED: ON
Expand Down Expand Up @@ -589,12 +626,11 @@ jobs:
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v5
- name: Restore ccache
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }}
restore-keys: cpp-odbc-ccache-windows-x64-
key: cpp-odbc-ccache-windows-x64
- name: Checkout vcpkg
uses: actions/checkout@v6
with:
Expand Down Expand Up @@ -630,6 +666,13 @@ jobs:
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
set VCPKG_ROOT=%VCPKG_ROOT_KEEP%
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
- name: Save ccache
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-odbc-ccache-windows-x64
- name: Register Flight SQL ODBC Driver
shell: cmd
run: |
Expand Down
15 changes: 11 additions & 4 deletions .github/workflows/cpp_windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ on:
type: string

permissions:
actions: read
contents: read
Comment thread
rok marked this conversation as resolved.

jobs:
Expand Down Expand Up @@ -107,19 +108,25 @@ jobs:
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v5
- name: Restore ccache
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-windows-${{ inputs.arch }}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-ccache-windows-${{ inputs.arch }}-
key: cpp-ccache-windows-${{ inputs.arch }}
- name: Build
shell: cmd
env:
VCVARS_ARCH: ${{ inputs.arch }}
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" %VCVARS_ARCH%
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
- name: Save ccache
if: ${{ !cancelled() }}
continue-on-error: true
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-windows-${{ inputs.arch }}
- name: Test
shell: cmd
env:
Expand Down
Loading