From 663e4a61b675b1a84c61b60b8cd117c08eaa7a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 11:32:49 +0200 Subject: [PATCH 01/15] GH-49676: [Python][Packaging] Avoid rebuilding all base images by not pulling bases --- dev/tasks/python-wheels/github.windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 77e2a04e3a0a..763116dc4e74 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -70,7 +70,7 @@ jobs: echo on archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 if errorlevel 1 ( - archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 + archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 ) archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 @@ -85,7 +85,7 @@ jobs: cd arrow archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-test if errorlevel 1 ( - archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 + archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 ) archery docker run %TEST_IMAGE_PREFIX%-wheel-windows-test From a0edd40b1a2d03cb17dc9c6d1b9379500ca35531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 11:48:41 +0200 Subject: [PATCH 02/15] Revert --no-pull and split vcpkg installation in two steps to try and reduce layer size --- ...ython-wheel-windows-vs2022-base.dockerfile | 22 ++++++++++++------- dev/tasks/python-wheels/github.windows.yml | 4 ++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index f1bc314d013a..6bf2bd419d6a 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -128,20 +128,26 @@ ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` VCPKG_FEATURE_FLAGS="manifests" COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ -# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains -# ssl related fixes as well as we can patch the vcpkg portfile to support -# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186 -# but we cannot patch those portfiles since vcpkg-tool handles the checkout of -# previous versions => use bundled S3 build + +# We split the dependencies installation into two steps to reduce the size +# if the intermediate image layers. +# Install FS dependencies first to avoid hitting the image size limit of +# the Windows container. GH-49676 RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` --x-manifest-root=arrow/ci/vcpkg ` --x-feature=azure ` - --x-feature=flight ` --x-feature=gcs ` + --x-feature=s3 + +# Install the rest of the dependencies. +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=flight ` --x-feature=json ` --x-feature=opentelemetry ` --x-feature=orc ` - --x-feature=parquet ` - --x-feature=s3 + --x-feature=parquet diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 763116dc4e74..77e2a04e3a0a 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -70,7 +70,7 @@ jobs: echo on archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 if errorlevel 1 ( - archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 + archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 ) archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 @@ -85,7 +85,7 @@ jobs: cd arrow archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-test if errorlevel 1 ( - archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 + archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 ) archery docker run %TEST_IMAGE_PREFIX%-wheel-windows-test From b45c2874d6ce646d654b2f4c04494c01593303dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 14:53:36 +0200 Subject: [PATCH 03/15] Add opentelemetry to its own Docker image layer --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 6bf2bd419d6a..9671b046696c 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -148,6 +148,11 @@ RUN vcpkg install ` --x-manifest-root=arrow/ci/vcpkg ` --x-feature=flight ` --x-feature=json ` - --x-feature=opentelemetry ` --x-feature=orc ` --x-feature=parquet + +# Install Opentelemetry alone. +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-feature=opentelemetry ` \ No newline at end of file From 4b9fd9ccb1f175025797743abb1e7d0ef34dccc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 15:10:56 +0200 Subject: [PATCH 04/15] Investigate whether the problem is around path length instead of layer size --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 11 +++++++---- ci/scripts/python_wheel_windows_build.bat | 4 ++-- ...indows-static-md-debug.cmake => x64-win-dbg.cmake} | 0 ...dows-static-md-release.cmake => x64-win-rel.cmake} | 0 4 files changed, 9 insertions(+), 6 deletions(-) rename ci/vcpkg/{amd64-windows-static-md-debug.cmake => x64-win-dbg.cmake} (100%) rename ci/vcpkg/{amd64-windows-static-md-release.cmake => x64-win-rel.cmake} (100%) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 9671b046696c..713b393934ca 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,11 +110,11 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin" # started to ship precompiled binaries for the vcpkg-tool. ARG vcpkg COPY ci/vcpkg/*.patch ` - ci/vcpkg/*windows*.cmake ` + ci/vcpkg/*win*.cmake ` arrow/ci/vcpkg/ COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/ -ENV VCPKG_ROOT=C:\\vcpkg -RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` +ENV VCPKG_ROOT=C:\\v +RUN bash arrow/ci/scripts/install_vcpkg.sh /c/v %vcpkg% && ` setx PATH "%PATH%;%VCPKG_ROOT%" # Configure vcpkg and install dependencies @@ -125,8 +125,11 @@ RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` ARG build_type=release ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg ` - VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` VCPKG_FEATURE_FLAGS="manifests" +# Derive a short triplet name to keep paths short and avoid hitting the +# hcsshim::ImportLayer MAX_PATH limit on Windows containers. GH-49676 +RUN if "%build_type%"=="release" (setx VCPKG_DEFAULT_TRIPLET x64-win-rel) ` + else if "%build_type%"=="debug" (setx VCPKG_DEFAULT_TRIPLET x64-win-dbg) COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ # We split the dependencies installation into two steps to reduce the size diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index e094d82861df..e38cdc6eb1c6 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -61,9 +61,9 @@ set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 17 2022 set CMAKE_PLATFORM=x64 -set VCPKG_ROOT=C:\vcpkg +set VCPKG_ROOT=C:\v set VCPKG_FEATURE_FLAGS=-manifests -set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% +if "%CMAKE_BUILD_TYPE%"=="release" (set VCPKG_TARGET_TRIPLET=x64-win-rel) else (set VCPKG_TARGET_TRIPLET=x64-win-dbg) mkdir C:\arrow-build pushd C:\arrow-build diff --git a/ci/vcpkg/amd64-windows-static-md-debug.cmake b/ci/vcpkg/x64-win-dbg.cmake similarity index 100% rename from ci/vcpkg/amd64-windows-static-md-debug.cmake rename to ci/vcpkg/x64-win-dbg.cmake diff --git a/ci/vcpkg/amd64-windows-static-md-release.cmake b/ci/vcpkg/x64-win-rel.cmake similarity index 100% rename from ci/vcpkg/amd64-windows-static-md-release.cmake rename to ci/vcpkg/x64-win-rel.cmake From 90df6a81a75a46eaab7b2d3ebab9aeaaa258ed38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 19:28:06 +0200 Subject: [PATCH 05/15] Revert "Investigate whether the problem is around path length instead of layer size" This reverts commit 4b9fd9ccb1f175025797743abb1e7d0ef34dccc2. --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 11 ++++------- ci/scripts/python_wheel_windows_build.bat | 4 ++-- ...-dbg.cmake => amd64-windows-static-md-debug.cmake} | 0 ...el.cmake => amd64-windows-static-md-release.cmake} | 0 4 files changed, 6 insertions(+), 9 deletions(-) rename ci/vcpkg/{x64-win-dbg.cmake => amd64-windows-static-md-debug.cmake} (100%) rename ci/vcpkg/{x64-win-rel.cmake => amd64-windows-static-md-release.cmake} (100%) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 713b393934ca..9671b046696c 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,11 +110,11 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin" # started to ship precompiled binaries for the vcpkg-tool. ARG vcpkg COPY ci/vcpkg/*.patch ` - ci/vcpkg/*win*.cmake ` + ci/vcpkg/*windows*.cmake ` arrow/ci/vcpkg/ COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/ -ENV VCPKG_ROOT=C:\\v -RUN bash arrow/ci/scripts/install_vcpkg.sh /c/v %vcpkg% && ` +ENV VCPKG_ROOT=C:\\vcpkg +RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` setx PATH "%PATH%;%VCPKG_ROOT%" # Configure vcpkg and install dependencies @@ -125,11 +125,8 @@ RUN bash arrow/ci/scripts/install_vcpkg.sh /c/v %vcpkg% && ` ARG build_type=release ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg ` + VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` VCPKG_FEATURE_FLAGS="manifests" -# Derive a short triplet name to keep paths short and avoid hitting the -# hcsshim::ImportLayer MAX_PATH limit on Windows containers. GH-49676 -RUN if "%build_type%"=="release" (setx VCPKG_DEFAULT_TRIPLET x64-win-rel) ` - else if "%build_type%"=="debug" (setx VCPKG_DEFAULT_TRIPLET x64-win-dbg) COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ # We split the dependencies installation into two steps to reduce the size diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index e38cdc6eb1c6..e094d82861df 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -61,9 +61,9 @@ set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 17 2022 set CMAKE_PLATFORM=x64 -set VCPKG_ROOT=C:\v +set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests -if "%CMAKE_BUILD_TYPE%"=="release" (set VCPKG_TARGET_TRIPLET=x64-win-rel) else (set VCPKG_TARGET_TRIPLET=x64-win-dbg) +set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% mkdir C:\arrow-build pushd C:\arrow-build diff --git a/ci/vcpkg/x64-win-dbg.cmake b/ci/vcpkg/amd64-windows-static-md-debug.cmake similarity index 100% rename from ci/vcpkg/x64-win-dbg.cmake rename to ci/vcpkg/amd64-windows-static-md-debug.cmake diff --git a/ci/vcpkg/x64-win-rel.cmake b/ci/vcpkg/amd64-windows-static-md-release.cmake similarity index 100% rename from ci/vcpkg/x64-win-rel.cmake rename to ci/vcpkg/amd64-windows-static-md-release.cmake From 5479a8d10e0cd3e5015a91ce34f071a91bc47624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 19:29:26 +0200 Subject: [PATCH 06/15] Remove opentelemetry from Windows wheels dependencies base image --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 9671b046696c..5aba02ed9fd7 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -150,9 +150,3 @@ RUN vcpkg install ` --x-feature=json ` --x-feature=orc ` --x-feature=parquet - -# Install Opentelemetry alone. -RUN vcpkg install ` - --clean-after-build ` - --x-install-root=%VCPKG_ROOT%\installed ` - --x-feature=opentelemetry ` \ No newline at end of file From a7c0613e95ff206781cf48e931749b63c90212d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 09:45:01 +0200 Subject: [PATCH 07/15] Move flight to its own layer to see if that helps --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 5aba02ed9fd7..f6f09c3e8fe4 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -141,12 +141,19 @@ RUN vcpkg install ` --x-feature=gcs ` --x-feature=s3 -# Install the rest of the dependencies. +# Install Flight separately as it pulls in grpc, protobuf, abseil which +# together produce the largest layer. +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=flight + +# Install other dependencies. RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` --x-manifest-root=arrow/ci/vcpkg ` - --x-feature=flight ` --x-feature=json ` --x-feature=orc ` --x-feature=parquet From 14c89a46e8f4eab988c536888a41b3d61f4c9f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 13:05:20 +0200 Subject: [PATCH 08/15] Move gRPC to its own feature to validate whether the problem is installing gRPC with vcpkg --- ...ython-wheel-windows-vs2022-base.dockerfile | 19 +++++++++++++------ ci/vcpkg/vcpkg.json | 6 ++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index f6f09c3e8fe4..271534d76b73 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -129,10 +129,18 @@ ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_FEATURE_FLAGS="manifests" COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ -# We split the dependencies installation into two steps to reduce the size -# if the intermediate image layers. -# Install FS dependencies first to avoid hitting the image size limit of -# the Windows container. GH-49676 +# Install gRPC first in isolation to validate it's the problematic layer. +# gRPC pulls in protobuf, abseil, c-ares, openssl and produces the largest +# single dependency. GH-49676 +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=grpc + +# We split the dependencies installation into multiple steps to reduce the +# size of the intermediate image layers. +# Install FS dependencies. GH-49676 RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` @@ -141,8 +149,7 @@ RUN vcpkg install ` --x-feature=gcs ` --x-feature=s3 -# Install Flight separately as it pulls in grpc, protobuf, abseil which -# together produce the largest layer. +# Install Flight (reuses already-installed grpc). RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json index 75f2b25cc0bf..017623189483 100644 --- a/ci/vcpkg/vcpkg.json +++ b/ci/vcpkg/vcpkg.json @@ -44,6 +44,12 @@ "gtest" ] }, + "grpc": { + "description": "gRPC (isolated for Windows layer sizing)", + "dependencies": [ + "grpc" + ] + }, "flight": { "description": "Flight RPC support", "dependencies": [ From 7f24ad8f36241f5114499e66679cf62003cfdcea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 13:08:21 +0200 Subject: [PATCH 09/15] Try increasing storage size for image on docker from original 20GB to 80GB --- dev/tasks/python-wheels/github.windows.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 77e2a04e3a0a..669471945216 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -44,6 +44,17 @@ jobs: {{ macros.github_login_ghcr()|indent }} {{ macros.github_install_archery()|indent }} + - name: Increase Docker storage limit + shell: pwsh + run: | + $configPath = "$env:ProgramData\docker\config\daemon.json" + $config = if (Test-Path $configPath) { Get-Content $configPath | ConvertFrom-Json } else { @{} } + $config | Add-Member -Force -Name "storage-opts" -Value @("size=80GB") -MemberType NoteProperty + $config | ConvertTo-Json -Depth 10 | Set-Content $configPath + Restart-Service docker + Start-Sleep -Seconds 5 + docker info + - name: Prepare shell: bash run: | From 6bd37139a76366d13f60145fc5b8668c9c0e2da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 13:11:20 +0200 Subject: [PATCH 10/15] What about enabling DOCKER_BUILDKIT? --- dev/tasks/python-wheels/github.windows.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 669471945216..72cb25d06e4b 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -34,10 +34,9 @@ jobs: # note that we don't run docker build since there wouldn't be a cache hit # and rebuilding the dependencies takes a fair amount of time REPO: ghcr.io/ursacomputing/arrow - # BuildKit isn't really supported on Windows for now. - # NuGet + GitHub Packages based vcpkg cache is also disabled for now. - # Because secret mount requires BuildKit. - DOCKER_BUILDKIT: 0 + # Try BuildKit - was previously disabled due to lack of Windows support, + # but Docker 29.x may handle it better. Testing as part of GH-49676. + DOCKER_BUILDKIT: 1 steps: {{ macros.github_checkout_arrow()|indent }} From 79da88c219963a8ce8857879330f109e2c756f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 15:17:14 +0200 Subject: [PATCH 11/15] Revert DOCKER_BUILDKIT attempt and apply Kou's patch for vcpkg removing /Z7 flag from build --- ci/vcpkg/vcpkg.patch | 15 +++++++++++++++ dev/tasks/python-wheels/github.windows.yml | 7 ++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ci/vcpkg/vcpkg.patch b/ci/vcpkg/vcpkg.patch index a4c8d5209785..2627f8c562c8 100644 --- a/ci/vcpkg/vcpkg.patch +++ b/ci/vcpkg/vcpkg.patch @@ -1,3 +1,18 @@ +diff --git a/scripts/toolchains/windows.cmake b/scripts/toolchains/windows.cmake +index 3cc90cc886..36af495687 100644 +--- a/scripts/toolchains/windows.cmake ++++ b/scripts/toolchains/windows.cmake +@@ -86,8 +86,8 @@ if(NOT _VCPKG_WINDOWS_TOOLCHAIN) + + set(CMAKE_CXX_FLAGS_DEBUG "${VCPKG_CRT_LINK_FLAG_PREFIX}d /Z7 /Ob0 /Od /RTC1 ${VCPKG_CXX_FLAGS_DEBUG}" CACHE STRING "") + set(CMAKE_C_FLAGS_DEBUG "${VCPKG_CRT_LINK_FLAG_PREFIX}d /Z7 /Ob0 /Od /RTC1 ${VCPKG_C_FLAGS_DEBUG}" CACHE STRING "") +- set(CMAKE_CXX_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG /Z7 ${VCPKG_CXX_FLAGS_RELEASE}" CACHE STRING "") +- set(CMAKE_C_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG /Z7 ${VCPKG_C_FLAGS_RELEASE}" CACHE STRING "") ++ set(CMAKE_CXX_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG ${VCPKG_CXX_FLAGS_RELEASE}" CACHE STRING "") ++ set(CMAKE_C_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG ${VCPKG_C_FLAGS_RELEASE}" CACHE STRING "") + + string(APPEND CMAKE_STATIC_LINKER_FLAGS_RELEASE_INIT " /nologo ") + set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "/nologo /DEBUG /INCREMENTAL:NO /OPT:REF /OPT:ICF ${VCPKG_LINKER_FLAGS} ${VCPKG_LINKER_FLAGS_RELEASE}" CACHE STRING "") diff --git a/scripts/cmake/vcpkg_execute_build_process.cmake b/scripts/cmake/vcpkg_execute_build_process.cmake index 60fd5b587a..c8dc021af8 100644 --- a/scripts/cmake/vcpkg_execute_build_process.cmake diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 72cb25d06e4b..669471945216 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -34,9 +34,10 @@ jobs: # note that we don't run docker build since there wouldn't be a cache hit # and rebuilding the dependencies takes a fair amount of time REPO: ghcr.io/ursacomputing/arrow - # Try BuildKit - was previously disabled due to lack of Windows support, - # but Docker 29.x may handle it better. Testing as part of GH-49676. - DOCKER_BUILDKIT: 1 + # BuildKit isn't really supported on Windows for now. + # NuGet + GitHub Packages based vcpkg cache is also disabled for now. + # Because secret mount requires BuildKit. + DOCKER_BUILDKIT: 0 steps: {{ macros.github_checkout_arrow()|indent }} From 96931be2ad1511bb413aa98f798e6352415f1799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 15:50:05 +0200 Subject: [PATCH 12/15] Reduce patch context so Windows doesn't trip over empty lines --- ci/vcpkg/vcpkg.patch | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ci/vcpkg/vcpkg.patch b/ci/vcpkg/vcpkg.patch index 2627f8c562c8..5a7d7b279554 100644 --- a/ci/vcpkg/vcpkg.patch +++ b/ci/vcpkg/vcpkg.patch @@ -1,18 +1,14 @@ diff --git a/scripts/toolchains/windows.cmake b/scripts/toolchains/windows.cmake -index 3cc90cc886..36af495687 100644 +index 3cc90cc..36af495 100644 --- a/scripts/toolchains/windows.cmake +++ b/scripts/toolchains/windows.cmake -@@ -86,8 +86,8 @@ if(NOT _VCPKG_WINDOWS_TOOLCHAIN) - - set(CMAKE_CXX_FLAGS_DEBUG "${VCPKG_CRT_LINK_FLAG_PREFIX}d /Z7 /Ob0 /Od /RTC1 ${VCPKG_CXX_FLAGS_DEBUG}" CACHE STRING "") +@@ -88,4 +88,4 @@ if(NOT _VCPKG_WINDOWS_TOOLCHAIN) set(CMAKE_C_FLAGS_DEBUG "${VCPKG_CRT_LINK_FLAG_PREFIX}d /Z7 /Ob0 /Od /RTC1 ${VCPKG_C_FLAGS_DEBUG}" CACHE STRING "") - set(CMAKE_CXX_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG /Z7 ${VCPKG_CXX_FLAGS_RELEASE}" CACHE STRING "") - set(CMAKE_C_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG /Z7 ${VCPKG_C_FLAGS_RELEASE}" CACHE STRING "") + set(CMAKE_CXX_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG ${VCPKG_CXX_FLAGS_RELEASE}" CACHE STRING "") + set(CMAKE_C_FLAGS_RELEASE "${VCPKG_CRT_LINK_FLAG_PREFIX} /O2 /Oi /Gy /DNDEBUG ${VCPKG_C_FLAGS_RELEASE}" CACHE STRING "") - - string(APPEND CMAKE_STATIC_LINKER_FLAGS_RELEASE_INIT " /nologo ") - set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "/nologo /DEBUG /INCREMENTAL:NO /OPT:REF /OPT:ICF ${VCPKG_LINKER_FLAGS} ${VCPKG_LINKER_FLAGS_RELEASE}" CACHE STRING "") + diff --git a/scripts/cmake/vcpkg_execute_build_process.cmake b/scripts/cmake/vcpkg_execute_build_process.cmake index 60fd5b587a..c8dc021af8 100644 --- a/scripts/cmake/vcpkg_execute_build_process.cmake From a0332583bf142b766ce5950f5e6d06a6d5fa6b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 17:14:03 +0200 Subject: [PATCH 13/15] Try splitting grpc-deps from grpc --- .../python-wheel-windows-vs2022-base.dockerfile | 13 ++++++++++--- ci/vcpkg/vcpkg.json | 10 ++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 271534d76b73..f8df7d64abe6 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -129,9 +129,16 @@ ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_FEATURE_FLAGS="manifests" COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ -# Install gRPC first in isolation to validate it's the problematic layer. -# gRPC pulls in protobuf, abseil, c-ares, openssl and produces the largest -# single dependency. GH-49676 +# Install gRPC dependencies first in their own layer, then gRPC itself +# in a separate layer, to keep each layer small enough for Windows +# Docker's hcsshim. GH-49676 +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=grpc-deps + +# Install gRPC (reuses already-installed deps). RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json index 017623189483..9a00ec1062cd 100644 --- a/ci/vcpkg/vcpkg.json +++ b/ci/vcpkg/vcpkg.json @@ -44,6 +44,16 @@ "gtest" ] }, + "grpc-deps": { + "description": "gRPC dependencies (isolated for Windows layer sizing)", + "dependencies": [ + "abseil", + "c-ares", + "openssl", + "protobuf", + "re2" + ] + }, "grpc": { "description": "gRPC (isolated for Windows layer sizing)", "dependencies": [ From 14f1b2736f2056cc5d599a0c645fb676f844f00b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 19:17:52 +0200 Subject: [PATCH 14/15] Revert back unnecessary changes --- ...ython-wheel-windows-vs2022-base.dockerfile | 43 ++++--------------- ci/vcpkg/vcpkg.json | 16 ------- dev/tasks/python-wheels/github.windows.yml | 11 ----- 3 files changed, 9 insertions(+), 61 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index f8df7d64abe6..56177c65eab5 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -129,45 +129,20 @@ ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_FEATURE_FLAGS="manifests" COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ -# Install gRPC dependencies first in their own layer, then gRPC itself -# in a separate layer, to keep each layer small enough for Windows -# Docker's hcsshim. GH-49676 -RUN vcpkg install ` - --clean-after-build ` - --x-install-root=%VCPKG_ROOT%\installed ` - --x-manifest-root=arrow/ci/vcpkg ` - --x-feature=grpc-deps - -# Install gRPC (reuses already-installed deps). -RUN vcpkg install ` - --clean-after-build ` - --x-install-root=%VCPKG_ROOT%\installed ` - --x-manifest-root=arrow/ci/vcpkg ` - --x-feature=grpc - -# We split the dependencies installation into multiple steps to reduce the -# size of the intermediate image layers. -# Install FS dependencies. GH-49676 +# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains +# ssl related fixes as well as we can patch the vcpkg portfile to support +# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186 +# but we cannot patch those portfiles since vcpkg-tool handles the checkout of +# previous versions => use bundled S3 build RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` --x-manifest-root=arrow/ci/vcpkg ` --x-feature=azure ` + --x-feature=flight ` --x-feature=gcs ` - --x-feature=s3 - -# Install Flight (reuses already-installed grpc). -RUN vcpkg install ` - --clean-after-build ` - --x-install-root=%VCPKG_ROOT%\installed ` - --x-manifest-root=arrow/ci/vcpkg ` - --x-feature=flight - -# Install other dependencies. -RUN vcpkg install ` - --clean-after-build ` - --x-install-root=%VCPKG_ROOT%\installed ` - --x-manifest-root=arrow/ci/vcpkg ` --x-feature=json ` + --x-feature=opentelemetry ` --x-feature=orc ` - --x-feature=parquet + --x-feature=parquet ` + --x-feature=s3 diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json index 9a00ec1062cd..75f2b25cc0bf 100644 --- a/ci/vcpkg/vcpkg.json +++ b/ci/vcpkg/vcpkg.json @@ -44,22 +44,6 @@ "gtest" ] }, - "grpc-deps": { - "description": "gRPC dependencies (isolated for Windows layer sizing)", - "dependencies": [ - "abseil", - "c-ares", - "openssl", - "protobuf", - "re2" - ] - }, - "grpc": { - "description": "gRPC (isolated for Windows layer sizing)", - "dependencies": [ - "grpc" - ] - }, "flight": { "description": "Flight RPC support", "dependencies": [ diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 669471945216..77e2a04e3a0a 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -44,17 +44,6 @@ jobs: {{ macros.github_login_ghcr()|indent }} {{ macros.github_install_archery()|indent }} - - name: Increase Docker storage limit - shell: pwsh - run: | - $configPath = "$env:ProgramData\docker\config\daemon.json" - $config = if (Test-Path $configPath) { Get-Content $configPath | ConvertFrom-Json } else { @{} } - $config | Add-Member -Force -Name "storage-opts" -Value @("size=80GB") -MemberType NoteProperty - $config | ConvertTo-Json -Depth 10 | Set-Content $configPath - Restart-Service docker - Start-Sleep -Seconds 5 - docker info - - name: Prepare shell: bash run: | From a38282c165bcda13531a4db0790be47018b6bae9 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 8 Apr 2026 08:41:40 -0700 Subject: [PATCH 15/15] Update github.windows.yml --- dev/tasks/python-wheels/github.windows.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 77e2a04e3a0a..96605614e321 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -57,6 +57,28 @@ jobs: esac echo "TEST_IMAGE_PREFIX=${test_image_prefix}" >> ${GITHUB_ENV} + - name: Configure Docker data-root + shell: powershell + run: | + # The D: drive on windows-2022 GH Actions runners has ~44GB free vs ~14GB on C:. + # Moving Docker's data-root to D: prevents hcsshim::ImportLayer failures when + # building large Windows container layers (e.g. the vcpkg install layer). GH-49676 + Stop-Service docker + $daemonJson = "C:\ProgramData\Docker\config\daemon.json" + New-Item -ItemType Directory -Force -Path (Split-Path $daemonJson) | Out-Null + if (Test-Path $daemonJson) { + $json = Get-Content $daemonJson | ConvertFrom-Json + $json | Add-Member -Force -NotePropertyName "data-root" -NotePropertyValue "D:\docker" + $json | ConvertTo-Json -Depth 10 | Set-Content $daemonJson + } else { + Set-Content $daemonJson -Value '{"data-root":"D:\docker"}' + } + Start-Service docker + Write-Host "=== daemon.json ===" + Get-Content $daemonJson + Write-Host "=== docker info ===" + docker info + - name: Build wheel shell: cmd run: |