GH-47195: [Python][CI] Add support for building PyArrow library on Windows ARM64

ci/scripts/python_wheel_windows_build.bat

@@ -24,8 +24,31 @@ py -0p
 
 %PYTHON_CMD% -m sysconfig || exit /B 1
 
-call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
-@echo on
+@REM Detect architecture if not set
+if "%arch%"=="" set arch=x64
+
+@REM Set Architecture-specific CMAKE Options
+if "%arch%"=="ARM64" (
+    set CMAKE_PLATFORM=ARM64
+    set VCVARS_BAT=C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat
+    set VCPKG_TARGET_TRIPLET=arm64-windows-static-md
+    set ARROW_SRC=%GITHUB_WORKSPACE%\arrow
+    set ARROW_DIST=%GITHUB_WORKSPACE%\arrow-dist
+    set CMAKE_TOOLCHAIN=-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
+    set UTF8PROC_SOURCE=-Dutf8proc_SOURCE=BUNDLED
+    set ARROW_SIMD_OPTIONS=-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE -DARROW_USE_XSIMD=OFF -DARROW_WITH_UTF8PROC=ON
+    set XSIMD_SOURCE=
+) else (
+    set CMAKE_PLATFORM=x64
+    set VCVARS_BAT=C:\Program Files ^(x86^)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat
+    set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
+    set ARROW_SRC=C:\arrow
+    set ARROW_DIST=C:\arrow-dist
+    set CMAKE_TOOLCHAIN=
+    set UTF8PROC_SOURCE=
+    set ARROW_SIMD_OPTIONS=
+    set XSIMD_SOURCE=-Dxsimd_SOURCE=BUNDLED
+)
 
 echo "=== (%PYTHON%) Clear output directories and leftovers ==="
 del /s /q C:\arrow-build
@@ -35,7 +58,12 @@ del /s /q C:\arrow\python\build
 del /s /q C:\arrow\python\pyarrow\*.so
 del /s /q C:\arrow\python\pyarrow\*.so.*
 
+call "%VCVARS_BAT%"
+@echo on
+
 echo "=== (%PYTHON%) Building Arrow C++ libraries ==="
+
+@REM Common Arrow feature flags
 set ARROW_ACERO=ON
 set ARROW_AZURE=ON
 set ARROW_DATASET=ON
@@ -47,8 +75,11 @@ set ARROW_ORC=ON
 set ARROW_PARQUET=ON
 set PARQUET_REQUIRE_ENCRYPTION=ON
 set ARROW_MIMALLOC=ON
-set ARROW_SUBSTRAIT=ON
 set ARROW_S3=ON
+set CMAKE_GENERATOR=Visual Studio 17 2022
+set VCPKG_ROOT=C:\vcpkg
+set VCPKG_FEATURE_FLAGS=-manifests
+set ARROW_SUBSTRAIT=ON
 set ARROW_TENSORFLOW=ON
 set ARROW_WITH_BROTLI=ON
 set ARROW_WITH_BZ2=ON
@@ -59,16 +90,20 @@ set ARROW_WITH_ZLIB=ON
 set ARROW_WITH_ZSTD=ON
 set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON
 set CMAKE_UNITY_BUILD=ON
-set CMAKE_GENERATOR=Visual Studio 17 2022
-set CMAKE_PLATFORM=x64
-set VCPKG_ROOT=C:\vcpkg
-set VCPKG_FEATURE_FLAGS=-manifests
-set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
 mkdir C:\arrow-build
 pushd C:\arrow-build
+
+@REM Configure the CMake build
 cmake ^
     -DARROW_ACERO=%ARROW_ACERO% ^
+     %CMAKE_TOOLCHAIN% ^
+    -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+    -DCMAKE_INSTALL_PREFIX=%ARROW_DIST% ^
+    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDULAR_OPTIMIZATION% ^
+    -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
+     %UTF8PROC_SOURCE% ^
+     %ARROW_SIMD_OPTIONS% ^
     -DARROW_AZURE=%ARROW_AZURE% ^
     -DARROW_BUILD_SHARED=ON ^
     -DARROW_BUILD_STATIC=OFF ^
@@ -99,18 +134,15 @@ cmake ^
     -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
     -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
     -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
-    -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
-    -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
-    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION% ^
-    -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
     -DMSVC_LINK_VERBOSE=ON ^
     -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^
     -DVCPKG_MANIFEST_MODE=OFF ^
     -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^
-    -Dxsimd_SOURCE=BUNDLED ^
     -G "%CMAKE_GENERATOR%" ^
-    -A "%CMAKE_PLATFORM%" ^
-    C:\arrow\cpp || exit /B 1
+    -A %CMAKE_PLATFORM% ^
+     %XSIMD_SOURCE% ^
+     "%ARROW_SRC%\cpp" || exit /B 1
+
 cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1
 popd
 
@@ -129,10 +161,10 @@ set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
 set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
 set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
 set PYARROW_WITH_S3=%ARROW_S3%
-set ARROW_HOME=C:\arrow-dist
-set CMAKE_PREFIX_PATH=C:\arrow-dist
+set ARROW_HOME=%ARROW_DIST%
+set CMAKE_PREFIX_PATH=%ARROW_DIST%
 
-pushd C:\arrow\python
+pushd %ARROW_SRC%\python
 
 @REM Build wheel
 %PYTHON_CMD% -m build --sdist --wheel . --no-isolation -vv ^

ci/scripts/python_wheel_windows_test.bat

@@ -17,6 +17,24 @@
 
 @echo on
 
+@REM Detect architecture if not set
+if "%arch%"=="" set arch=x64
+
+@REM Set Architecture-specific paths
+if "%arch%"=="ARM64" (
+    set ARROW_SRC=%GITHUB_WORKSPACE%\arrow
+    set ARROW_TEST_DATA=%GITHUB_WORKSPACE%\arrow\testing\data
+    set PARQUET_TEST_DATA=%GITHUB_WORKSPACE%\arrow\cpp\submodules\parquet-testing\data
+    set WHEEL_PATH=%GITHUB_WORKSPACE%\arrow\python\repaired_wheels
+    set VALIDATION_SCRIPT=%GITHUB_WORKSPACE%\arrow\ci\scripts\python_wheel_validate_contents.py
+) else (
+    set ARROW_SRC=C:\arrow
+    set ARROW_TEST_DATA=C:\arrow\testing\data
+    set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
+    set WHEEL_PATH=C:\arrow\python\repaired_wheels
+    set VALIDATION_SCRIPT=C:\arrow\ci\scripts\python_wheel_validate_contents.py
+)
+
 set PYARROW_TEST_ACERO=ON
 set PYARROW_TEST_AZURE=ON
 set PYARROW_TEST_CYTHON=ON
@@ -33,14 +51,11 @@ set PYARROW_TEST_SUBSTRAIT=ON
 set PYARROW_TEST_S3=ON
 set PYARROW_TEST_TENSORFLOW=ON
 
-set ARROW_TEST_DATA=C:\arrow\testing\data
-set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
-
 @REM List installed Pythons
 py -0p
 
 @REM Install the built wheels
-%PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\repaired_wheels pyarrow || exit /B 1
+%PYTHON_CMD% -m pip install --no-index --find-links=%WHEEL_PATH% pyarrow || exit /B 1
 
 @REM Test that the modules are importable
 %PYTHON_CMD% -c "import pyarrow" || exit /B 1
@@ -58,7 +73,7 @@ py -0p
 %PYTHON_CMD% -c "import pyarrow.substrait" || exit /B 1
 
 @REM Validate wheel contents
-%PYTHON_CMD% C:\arrow\ci\scripts\python_wheel_validate_contents.py --path C:\arrow\python\repaired_wheels || exit /B 1
+%PYTHON_CMD% %VALIDATION_SCRIPT% --path %WHEEL_PATH% || exit /B 1
 
 @REM Execute unittest
 %PYTHON_CMD% -m pytest -r s --pyargs pyarrow || exit /B 1

ci/vcpkg/vcpkg.json

@@ -3,7 +3,7 @@
   "version-string": "",
   "description": "Cross-language development platform for in-memory analytics",
   "homepage": "https://arrow.apache.org",
-  "supports": "x64 | (arm64 & !windows)",
+  "supports": "x64 | arm64",
   "dependencies": [
     {
       "name": "boost-multiprecision",
@@ -19,7 +19,10 @@
     "re2",
     "snappy",
     "utf8proc",
-    "xsimd",
+    {
+      "name": "xsimd",
+      "platform": "!(arm64 & windows)"
+    },
     "zlib",
     "zstd",
     {

dev/archery/setup.py

@@ -29,9 +29,9 @@
 
 extras = {
     'benchmark': ['pandas'],
-    'crossbow': ['github3.py', jinja_req, 'pygit2>=1.14.0', 'requests',
+    'crossbow': ['cryptography==46.0.3','github3.py', jinja_req, 'pygit2>=1.14.0', 'requests',
                  'ruamel.yaml', 'setuptools_scm>=8.0.0'],
-    'crossbow-upload': ['github3.py', jinja_req, 'ruamel.yaml',
+    'crossbow-upload': ['cryptography==46.0.3','github3.py', jinja_req, 'ruamel.yaml',
                         'setuptools_scm'],
     'docker': ['ruamel.yaml', 'python-dotenv'],
     'integration': ['cffi', 'numpy'],

dev/tasks/python-wheels/github.windows.arm64.yml

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+jobs:
+  build:
+    name: "Build wheel for Windows ARM64"
+    runs-on: "{{ runs_on }}"
+    timeout-minutes: 300
+    env:
+      PYTHON_CMD: "py -{{ python_version }}{% if python_abi_tag.endswith('t') %}t{% endif %}-arm64"
+      CMAKE_BUILD_TYPE: release
+    steps:
+      {{ macros.github_checkout_arrow()|indent }}
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "{{ python_version }}{% if python_abi_tag.endswith('t') %}t{% endif %}"
+          architecture: "arm64"
+
+      - name: Install Python build dependencies
+        shell: cmd
+        run: |
+          %PYTHON_CMD% -m pip install --upgrade pip
+          %PYTHON_CMD% -m pip install -r arrow/python/requirements-wheel-build.txt
+          %PYTHON_CMD% -m pip install pandas pytest hypothesis pytz
+
+      - name: Cache vcpkg dependencies
+        uses: actions/cache@v4
+        with:
+          path: C:\vcpkg\installed
+          key: "{% raw %}vcpkg-installed-windows-arm64-${{ hashFiles('arrow/ci/vcpkg/vcpkg.json') }}{% endraw %}"      
+          restore-keys: vcpkg-installed-windows-arm64-
+
+      - name: Install vcpkg dependencies
+        shell: cmd
+        run: |
+          cd arrow\ci\vcpkg
+          C:\vcpkg\vcpkg.exe install ^
+            --triplet arm64-windows-static-md ^
+            --binarysource="clear;x-gha,readwrite" ^
+            --x-install-root=C:\vcpkg\installed ^
+            --x-manifest-root=. ^
+            --x-feature=azure ^
+            --x-feature=flight ^
+            --x-feature=json ^
+            --x-feature=gcs ^
+            --x-feature=opentelemetry ^
+            --x-feature=orc ^
+            --x-feature=parquet ^
+            --x-feature=s3
+
+      - name: Build Arrow C++ and PyArrow wheel
+        shell: cmd
+        env:
+          ARCH: "ARM64"
+          SETUPTOOLS_SCM_PRETEND_VERSION: "{{ arrow.no_rc_version }}"
+        run: |
+          call "arrow\ci\scripts\python_wheel_windows_build.bat"
+
+      - name: Test wheel
+        shell: cmd
+        env:
+          ARCH: "ARM64"
+        run: |
+          call "arrow\ci\scripts\python_wheel_windows_test.bat"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel
+          path: arrow/python/repaired_wheels/*.whl
+      {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
+      {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }}

dev/tasks/tasks.yml

@@ -240,6 +240,18 @@ tasks:
     artifacts:
       - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl
 
+{% if python_version != "3.10" %}
+  wheel-windows-{{ python_tag }}-{{ abi_tag }}-arm64:
+    ci: github
+    template: python-wheels/github.windows.arm64.yml
+    params:
+      python_version: "{{ python_version }}"
+      python_abi_tag: "{{ abi_tag }}"
+      runs_on: "windows-11-arm"
+    artifacts:
+      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_arm64.whl
+{% endif %}
+
 {% endfor %}
 
 {############################ Python sdist ####################################}