diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml new file mode 100644 index 000000000..5bf2f5fbc --- /dev/null +++ b/.github/workflows/native-packages.yml @@ -0,0 +1,2331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build native release artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + prepare: + name: Prepare package versions + runs-on: ubuntu-24.04 + outputs: + base_version: ${{ steps.versions.outputs.base_version }} + python_version: ${{ steps.versions.outputs.python_version }} + logical_version: ${{ steps.versions.outputs.logical_version }} + deb_version: ${{ steps.versions.outputs.deb_version }} + rpm_version: ${{ steps.versions.outputs.rpm_version }} + rpm_release: ${{ steps.versions.outputs.rpm_release }} + archive_version: ${{ steps.versions.outputs.archive_version }} + homebrew_version: ${{ steps.versions.outputs.homebrew_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Check packaging helpers and workflow contracts + run: | + python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v + ruby packaging/tests/check_native_workflow.rb + + - name: Upload CLI data from the source checkout + uses: actions/upload-artifact@v7 + with: + name: native-cli-test-data + path: python/tests/resources/*.tsfile + if-no-files-found: error + retention-days: 14 + + - name: Generate package versions + id: versions + run: | + python3 packaging/scripts/native_package_versions.py \ + --cmake-file cpp/CMakeLists.txt \ + --json-out versions.json \ + --github-output "$GITHUB_OUTPUT" + + - name: Upload package versions + uses: actions/upload-artifact@v7 + with: + name: native-package-versions + path: versions.json + if-no-files-found: error + retention-days: 14 + + build-deb: + name: Build Ubuntu 22.04 C++ SDK and DEB packages + needs: prepare + runs-on: ubuntu-22.04 + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Java 17 + uses: actions/setup-java@v6.0.0 + with: + distribution: temurin + java-version: "17" + + - name: Install DEB build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + dpkg-dev \ + uuid-dev + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + ./mvnw -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.dependency.source=BUNDLED \ + -Dtsfile.version.sync.skip=true \ + -Denable.cpack=ON \ + -Dtsfile.archive.version="$ARCHIVE_VERSION" \ + -Dtsfile.debian.package.version="$DEBIAN_VERSION" \ + -Dtsfile.rpm.package.version="$RPM_VERSION" \ + -Dtsfile.rpm.package.release="$RPM_RELEASE" + test -f cpp/target/build/CPackConfig.cmake + + - name: Stage and verify the Ubuntu SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: bash + run: | + sdk_name="tsfile-sdk-ubuntu22.04-amd64-$ARCHIVE_VERSION" + sdk_root="$PWD/sdk/$sdk_name" + cmake --install cpp/target/build --prefix "$sdk_root" + test -f "$sdk_root/bin/tsfile-cli" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/deb \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/deb --parallel + ./consumer/deb/tsfile_installed_consumer + tar -C sdk -czf "$sdk_root.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Build and verify DEB packages + env: + EXPECTED_DEB_VERSION: ${{ needs.prepare.outputs.deb_version }} + shell: bash + run: | + mkdir -p packages + cpack -G DEB --config cpp/target/build/CPackConfig.cmake -B packages + + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.deb' -print | sort) + if [[ ${#packages[@]} -ne 3 ]]; then + printf 'Expected exactly 3 DEB packages, found %s\n' "${#packages[@]}" + exit 1 + fi + + actual_names="$({ + for package in "${packages[@]}"; do + dpkg-deb --field "$package" Package + done + } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-dev tsfile-tools | sort)" + if [[ "$actual_names" != "$expected_names" ]]; then + printf 'Unexpected DEB package names:\n%s\n' "$actual_names" + exit 1 + fi + + for package in "${packages[@]}"; do + package_version="$(dpkg-deb --field "$package" Version)" + if [[ "$package_version" != "$EXPECTED_DEB_VERSION" ]]; then + printf '%s has version %s, expected %s\n' \ + "$package" "$package_version" "$EXPECTED_DEB_VERSION" + exit 1 + fi + done + + - name: Upload Ubuntu SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-ubuntu22.04-amd64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload DEB packages + uses: actions/upload-artifact@v7 + with: + name: native-deb-ubuntu22.04-amd64 + path: packages/*.deb + if-no-files-found: error + retention-days: 14 + + test-deb: + name: Test DEB packages on ${{ matrix.container }} + needs: build-deb + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + container: + - ubuntu:22.04 + - ubuntu:24.04 + container: ${{ matrix.container }} + timeout-minutes: 45 + steps: + - name: Checkout installation test sources + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + cpp/test + cpp/cmake/tests/projects/InstalledConsumer + go + sparse-checkout-cone-mode: false + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Download DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-amd64 + path: packages + + - name: Install DEB packages + shell: bash + run: | + apt-get update + apt-get install -y ./packages/*.deb cmake build-essential python3 pkg-config git curl ca-certificates + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Verify installed DEB packages + shell: bash + run: | + cli_path="$(command -v tsfile-cli)" + cli_path="$(readlink -f -- "$cli_path")" + cli_owner="$(dpkg-query -S "$cli_path")" + cli_owner="${cli_owner%%:*}" + if [[ "$cli_owner" != "tsfile-tools" ]]; then + printf '%s is owned by %s, expected tsfile-tools\n' "$cli_path" "$cli_owner" + exit 1 + fi + tsfile-cli --version + dpkg-query -W tsfile tsfile-dev tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/tsfile_installed_consumer + + - name: Test the installed CLI with Python test data + shell: bash + run: | + python3 packaging/tests/cli_package_smoke.py \ + --cli "$(command -v tsfile-cli)" --resources cli-test-data + + - name: Compile and run C++ regressions against the installed SDK + shell: bash + run: | + python3 packaging/tests/cpp_installed_tests.py \ + --prefix /usr --build-dir installed-tests + + - name: Run Go tests against the installed SDK + shell: bash + run: | + python3 packaging/tests/go_installed_tests.py \ + --prefix /usr --goarch amd64 + + - name: Download the standalone SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-ubuntu22.04-amd64 + path: sdk-download + + - name: Test the downloaded and relocated SDK archive + shell: bash + run: | + python3 packaging/tests/sdk_archive_tests.py \ + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests \ + --resources cli-test-data --goarch amd64 \ + --gtest-prefix installed-tests/dependencies + + - name: Upload C++ installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-cpp-tests-${{ github.job }}-${{ runner.arch }}-${{ strategy.job-index }} + path: | + installed-tests/cpp-results.xml + installed-tests/tests/Testing/Temporary/LastTest.log + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + build-deb-arm64: + name: Build Ubuntu 22.04 ARM64 C++ SDK and DEB packages + needs: prepare + runs-on: ubuntu-22.04-arm + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Java 17 + uses: actions/setup-java@v6.0.0 + with: + distribution: temurin + java-version: "17" + + - name: Install DEB build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + dpkg-dev \ + uuid-dev + + - name: Verify native ARM64 runner + run: test "$(dpkg --print-architecture)" = arm64 + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + ./mvnw -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.dependency.source=BUNDLED \ + -Dtsfile.version.sync.skip=true \ + -Denable.cpack=ON \ + -Dtsfile.archive.version="$ARCHIVE_VERSION" \ + -Dtsfile.debian.package.version="$DEBIAN_VERSION" \ + -Dtsfile.rpm.package.version="$RPM_VERSION" \ + -Dtsfile.rpm.package.release="$RPM_RELEASE" + test -f cpp/target/build/CPackConfig.cmake + + - name: Stage and verify the Ubuntu ARM64 SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: bash + run: | + sdk_name="tsfile-sdk-ubuntu22.04-arm64-$ARCHIVE_VERSION" + sdk_root="$PWD/sdk/$sdk_name" + cmake --install cpp/target/build --prefix "$sdk_root" + test -f "$sdk_root/bin/tsfile-cli" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/deb-arm64 \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/deb-arm64 --parallel + ./consumer/deb-arm64/tsfile_installed_consumer + tar -C sdk -czf "$sdk_root.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Build and verify ARM64 DEB packages + env: + EXPECTED_DEB_VERSION: ${{ needs.prepare.outputs.deb_version }} + shell: bash + run: | + mkdir -p packages + cpack -G DEB --config cpp/target/build/CPackConfig.cmake -B packages + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.deb' -print | sort) + [[ ${#packages[@]} -eq 3 ]] + actual_names="$({ for package in "${packages[@]}"; do dpkg-deb --field "$package" Package; done; } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-dev tsfile-tools | sort)" + [[ "$actual_names" == "$expected_names" ]] + for package in "${packages[@]}"; do + [[ "$(dpkg-deb --field "$package" Architecture)" == arm64 ]] + [[ "$(dpkg-deb --field "$package" Version)" == "$EXPECTED_DEB_VERSION" ]] + done + + - name: Upload Ubuntu ARM64 SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-ubuntu22.04-arm64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload ARM64 DEB packages + uses: actions/upload-artifact@v7 + with: + name: native-deb-ubuntu22.04-arm64 + path: packages/*.deb + if-no-files-found: error + retention-days: 14 + + test-deb-arm64: + name: Test ARM64 DEB packages on ${{ matrix.container }} + needs: build-deb-arm64 + runs-on: ubuntu-24.04-arm + strategy: + fail-fast: false + matrix: + container: + - ubuntu:22.04 + - ubuntu:24.04 + container: ${{ matrix.container }} + timeout-minutes: 45 + steps: + - name: Checkout installation test sources + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + cpp/test + cpp/cmake/tests/projects/InstalledConsumer + go + sparse-checkout-cone-mode: false + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Download ARM64 DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-arm64 + path: packages + + - name: Install ARM64 DEB packages + shell: bash + run: | + test "$(dpkg --print-architecture)" = arm64 + apt-get update + apt-get install -y ./packages/*.deb cmake build-essential python3 pkg-config git curl ca-certificates + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Verify installed ARM64 DEB packages + shell: bash + run: | + cli_path="$(readlink -f -- "$(command -v tsfile-cli)")" + cli_owner="$(dpkg-query -S "$cli_path")" + cli_owner="${cli_owner%%:*}" + [[ "$cli_owner" == tsfile-tools ]] + tsfile-cli --version + dpkg-query -W tsfile tsfile-dev tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/tsfile_installed_consumer + + - name: Test the installed CLI with Python test data + shell: bash + run: | + python3 packaging/tests/cli_package_smoke.py \ + --cli "$(command -v tsfile-cli)" --resources cli-test-data + + - name: Compile and run C++ regressions against the installed SDK + shell: bash + run: | + python3 packaging/tests/cpp_installed_tests.py \ + --prefix /usr --build-dir installed-tests + + - name: Run Go tests against the installed SDK + shell: bash + run: | + python3 packaging/tests/go_installed_tests.py \ + --prefix /usr --goarch arm64 + + - name: Download the standalone SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-ubuntu22.04-arm64 + path: sdk-download + + - name: Test the downloaded and relocated SDK archive + shell: bash + run: | + python3 packaging/tests/sdk_archive_tests.py \ + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests \ + --resources cli-test-data --goarch arm64 \ + --gtest-prefix installed-tests/dependencies + + - name: Upload C++ installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-cpp-tests-${{ github.job }}-${{ runner.arch }}-${{ strategy.job-index }} + path: | + installed-tests/cpp-results.xml + installed-tests/tests/Testing/Temporary/LastTest.log + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + build-rpm: + name: Build AlmaLinux 9 C++ SDK and RPM packages + needs: prepare + runs-on: ubuntu-24.04 + container: almalinux:9 + timeout-minutes: 45 + steps: + - name: Install RPM build prerequisites + run: | + dnf install -y \ + java-17-openjdk-devel \ + cmake \ + gcc-c++ \ + make \ + pkgconf-pkg-config \ + rpm-build \ + git \ + curl-minimal \ + tar \ + gzip \ + libuuid-devel + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + ./mvnw -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.dependency.source=BUNDLED \ + -Dtsfile.version.sync.skip=true \ + -Denable.cpack=ON \ + -Dtsfile.archive.version="$ARCHIVE_VERSION" \ + -Dtsfile.debian.package.version="$DEBIAN_VERSION" \ + -Dtsfile.rpm.package.version="$RPM_VERSION" \ + -Dtsfile.rpm.package.release="$RPM_RELEASE" + test -f cpp/target/build/CPackConfig.cmake + + - name: Stage and verify the AlmaLinux SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: bash + run: | + sdk_name="tsfile-sdk-almalinux9-x86_64-$ARCHIVE_VERSION" + sdk_root="$PWD/sdk/$sdk_name" + cmake --install cpp/target/build --prefix "$sdk_root" + test -f "$sdk_root/bin/tsfile-cli" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/rpm \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/rpm --parallel + ./consumer/rpm/tsfile_installed_consumer + tar -C sdk -czf "$sdk_root.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Build and verify RPM packages + env: + EXPECTED_RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + EXPECTED_RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + mkdir -p packages + cpack -G RPM --config cpp/target/build/CPackConfig.cmake -B packages + + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.rpm' -print | sort) + if [[ ${#packages[@]} -ne 3 ]]; then + printf 'Expected exactly 3 RPM packages, found %s\n' "${#packages[@]}" + exit 1 + fi + + actual_names="$({ + for package in "${packages[@]}"; do + rpm -qp --queryformat '%{NAME}\n' "$package" + done + } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-devel tsfile-tools | sort)" + if [[ "$actual_names" != "$expected_names" ]]; then + printf 'Unexpected RPM package names:\n%s\n' "$actual_names" + exit 1 + fi + + for package in "${packages[@]}"; do + package_version="$(rpm -qp --queryformat '%{VERSION}' "$package")" + package_release="$(rpm -qp --queryformat '%{RELEASE}' "$package")" + if [[ "$package_version" != "$EXPECTED_RPM_VERSION" ]]; then + printf '%s has version %s, expected %s\n' \ + "$package" "$package_version" "$EXPECTED_RPM_VERSION" + exit 1 + fi + if [[ "$package_release" != "$EXPECTED_RPM_RELEASE" ]]; then + printf '%s has release %s, expected %s\n' \ + "$package" "$package_release" "$EXPECTED_RPM_RELEASE" + exit 1 + fi + done + + - name: Upload AlmaLinux SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-almalinux9-x86_64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload RPM packages + uses: actions/upload-artifact@v7 + with: + name: native-rpm-almalinux9-x86_64 + path: packages/*.rpm + if-no-files-found: error + retention-days: 14 + + test-rpm: + name: Test RPM packages on AlmaLinux 9 + needs: build-rpm + runs-on: ubuntu-24.04 + container: almalinux:9 + timeout-minutes: 45 + steps: + - name: Checkout installation test sources + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + cpp/test + cpp/cmake/tests/projects/InstalledConsumer + go + sparse-checkout-cone-mode: false + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Download RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-x86_64 + path: packages + + - name: Install RPM packages + run: dnf install -y packages/*.rpm cmake gcc-c++ python3 pkgconf-pkg-config git curl-minimal tar gzip + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Verify installed RPM packages + shell: bash + run: | + cli_path="$(command -v tsfile-cli)" + cli_path="$(readlink -f -- "$cli_path")" + cli_owner="$(rpm -qf --queryformat '%{NAME}' "$cli_path")" + if [[ "$cli_owner" != "tsfile-tools" ]]; then + printf '%s is owned by %s, expected tsfile-tools\n' "$cli_path" "$cli_owner" + exit 1 + fi + tsfile-cli --version + rpm -q tsfile tsfile-devel tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/tsfile_installed_consumer + + - name: Test the installed CLI with Python test data + shell: bash + run: | + python3 packaging/tests/cli_package_smoke.py \ + --cli "$(command -v tsfile-cli)" --resources cli-test-data + + - name: Compile and run C++ regressions against the installed SDK + shell: bash + run: | + python3 packaging/tests/cpp_installed_tests.py \ + --prefix /usr --build-dir installed-tests + + - name: Run Go tests against the installed SDK + shell: bash + run: | + python3 packaging/tests/go_installed_tests.py \ + --prefix /usr --goarch amd64 + + - name: Download the standalone SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-almalinux9-x86_64 + path: sdk-download + + - name: Test the downloaded and relocated SDK archive + shell: bash + run: | + python3 packaging/tests/sdk_archive_tests.py \ + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests \ + --resources cli-test-data --goarch amd64 \ + --gtest-prefix installed-tests/dependencies + + - name: Upload C++ installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-cpp-tests-${{ github.job }}-${{ runner.arch }}-${{ strategy.job-index }} + path: | + installed-tests/cpp-results.xml + installed-tests/tests/Testing/Temporary/LastTest.log + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + build-rpm-aarch64: + name: Build AlmaLinux 9 aarch64 C++ SDK and RPM packages + needs: prepare + runs-on: ubuntu-24.04-arm + container: almalinux:9 + timeout-minutes: 45 + steps: + - name: Install RPM build prerequisites + run: | + test "$(uname -m)" = aarch64 + dnf install -y \ + java-17-openjdk-devel \ + cmake \ + gcc-c++ \ + make \ + pkgconf-pkg-config \ + rpm-build \ + git \ + curl-minimal \ + tar \ + gzip \ + libuuid-devel + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + ./mvnw -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.dependency.source=BUNDLED \ + -Dtsfile.version.sync.skip=true \ + -Denable.cpack=ON \ + -Dtsfile.archive.version="$ARCHIVE_VERSION" \ + -Dtsfile.debian.package.version="$DEBIAN_VERSION" \ + -Dtsfile.rpm.package.version="$RPM_VERSION" \ + -Dtsfile.rpm.package.release="$RPM_RELEASE" + test -f cpp/target/build/CPackConfig.cmake + + - name: Stage and verify the AlmaLinux aarch64 SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: bash + run: | + sdk_name="tsfile-sdk-almalinux9-aarch64-$ARCHIVE_VERSION" + sdk_root="$PWD/sdk/$sdk_name" + cmake --install cpp/target/build --prefix "$sdk_root" + test -f "$sdk_root/bin/tsfile-cli" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/rpm-aarch64 \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/rpm-aarch64 --parallel + ./consumer/rpm-aarch64/tsfile_installed_consumer + tar -C sdk -czf "$sdk_root.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Build and verify aarch64 RPM packages + env: + EXPECTED_RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + EXPECTED_RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + mkdir -p packages + cpack -G RPM --config cpp/target/build/CPackConfig.cmake -B packages + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.rpm' -print | sort) + [[ ${#packages[@]} -eq 3 ]] + actual_names="$({ for package in "${packages[@]}"; do rpm -qp --queryformat '%{NAME}\n' "$package"; done; } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-devel tsfile-tools | sort)" + [[ "$actual_names" == "$expected_names" ]] + for package in "${packages[@]}"; do + [[ "$(rpm -qp --queryformat '%{ARCH}' "$package")" == aarch64 ]] + [[ "$(rpm -qp --queryformat '%{VERSION}' "$package")" == "$EXPECTED_RPM_VERSION" ]] + [[ "$(rpm -qp --queryformat '%{RELEASE}' "$package")" == "$EXPECTED_RPM_RELEASE" ]] + done + + - name: Upload AlmaLinux aarch64 SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-almalinux9-aarch64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload aarch64 RPM packages + uses: actions/upload-artifact@v7 + with: + name: native-rpm-almalinux9-aarch64 + path: packages/*.rpm + if-no-files-found: error + retention-days: 14 + + test-rpm-aarch64: + name: Test RPM packages on AlmaLinux 9 aarch64 + needs: build-rpm-aarch64 + runs-on: ubuntu-24.04-arm + container: almalinux:9 + timeout-minutes: 45 + steps: + - name: Checkout installation test sources + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + cpp/test + cpp/cmake/tests/projects/InstalledConsumer + go + sparse-checkout-cone-mode: false + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Download aarch64 RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-aarch64 + path: packages + + - name: Install aarch64 RPM packages + run: | + test "$(uname -m)" = aarch64 + dnf install -y packages/*.rpm cmake gcc-c++ python3 pkgconf-pkg-config git curl-minimal tar gzip + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Verify installed aarch64 RPM packages + shell: bash + run: | + cli_path="$(readlink -f -- "$(command -v tsfile-cli)")" + cli_owner="$(rpm -qf --queryformat '%{NAME}' "$cli_path")" + [[ "$cli_owner" == tsfile-tools ]] + tsfile-cli --version + rpm -q tsfile tsfile-devel tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/tsfile_installed_consumer + + - name: Test the installed CLI with Python test data + shell: bash + run: | + python3 packaging/tests/cli_package_smoke.py \ + --cli "$(command -v tsfile-cli)" --resources cli-test-data + + - name: Compile and run C++ regressions against the installed SDK + shell: bash + run: | + python3 packaging/tests/cpp_installed_tests.py \ + --prefix /usr --build-dir installed-tests + + - name: Run Go tests against the installed SDK + shell: bash + run: | + python3 packaging/tests/go_installed_tests.py \ + --prefix /usr --goarch arm64 + + - name: Download the standalone SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-almalinux9-aarch64 + path: sdk-download + + - name: Test the downloaded and relocated SDK archive + shell: bash + run: | + python3 packaging/tests/sdk_archive_tests.py \ + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests \ + --resources cli-test-data --goarch arm64 \ + --gtest-prefix installed-tests/dependencies + + - name: Upload C++ installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-cpp-tests-${{ github.job }}-${{ runner.arch }}-${{ strategy.job-index }} + path: | + installed-tests/cpp-results.xml + installed-tests/tests/Testing/Temporary/LastTest.log + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + build-windows: + name: Build Windows MSVC C++ SDK and ZIP + needs: prepare + runs-on: windows-2022 + timeout-minutes: 50 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Java 17 + uses: actions/setup-java@v6.0.0 + with: + distribution: temurin + java-version: "17" + + - name: Configure the MSVC environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $projectInclude = (Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path + .\mvnw.cmd -Pwith-cpp package ` + "-Dcpp.toolchain=msvc" ` + "-Dbuild.type=Release" ` + "-Dbuild.test=OFF" ` + "-DskipTests" ` + "-Dspotless.skip=true" ` + "-Dtsfile.dependency.source=BUNDLED" ` + "-Dtsfile.msvc.static.runtime=ON" ` + "-Denable.lzma2=ON" ` + "-Dtsfile.project.include=$projectInclude" ` + "-Dtsfile.version.sync.skip=true" ` + "-Denable.cpack=ON" ` + "-Dtsfile.archive.version=$env:ARCHIVE_VERSION" ` + "-Dtsfile.debian.package.version=$env:DEBIAN_VERSION" ` + "-Dtsfile.rpm.package.version=$env:RPM_VERSION" ` + "-Dtsfile.rpm.package.release=$env:RPM_RELEASE" + if (-not (Test-Path -LiteralPath cpp/target/build/CPackConfig.cmake -PathType Leaf)) { + throw 'Maven did not generate cpp/target/build/CPackConfig.cmake' + } + + - name: Stage and verify the Windows SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $sdkName = "tsfile-sdk-windows-msvc-x86_64-$env:ARCHIVE_VERSION" + $sdkRoot = Join-Path $PWD "sdk/$sdkName" + cmake --install cpp/target/build --config Release --prefix $sdkRoot + $requiredPaths = @( + "$sdkRoot/bin/tsfile-cli.exe" + "$sdkRoot/bin/tsfile.dll" + "$sdkRoot/lib/tsfile.lib" + "$sdkRoot/include/tsfile/cwrapper/tsfile_cwrapper.h" + "$sdkRoot/lib/cmake/TsFile/TsFileConfig.cmake" + "$sdkRoot/share/doc/tsfile/LICENSE" + "$sdkRoot/share/doc/tsfile/NOTICE" + ) + foreach ($path in $requiredPaths) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required SDK file is absent: $path" + } + } + python packaging/scripts/verify_windows_runtime.py $sdkRoot + $env:PATH = "$sdkRoot/bin;$env:PATH" + & "$sdkRoot/bin/tsfile-cli.exe" --version + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/windows ` + -G "Visual Studio 17 2022" -A x64 ` + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ` + "-DCMAKE_PREFIX_PATH=$sdkRoot" + cmake --build consumer/windows --config Release --parallel + & consumer/windows/Release/tsfile_installed_consumer.exe + tar.exe -czf "$sdkRoot.tar.gz" -C sdk $sdkName + Remove-Item -LiteralPath $sdkRoot -Recurse -Force + + - name: Build and inspect combined ZIP + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $cpackConfig = Get-Content cpp/target/build/CPackConfig.cmake -Raw + if ($cpackConfig -notmatch 'set\(CPACK_ARCHIVE_COMPONENT_INSTALL "OFF"\)') { + throw 'CPACK_ARCHIVE_COMPONENT_INSTALL must remain disabled' + } + + New-Item -ItemType Directory -Path packages -Force | Out-Null + cpack -G ZIP -C Release --config cpp/target/build/CPackConfig.cmake -B packages ` + -D "CPACK_PACKAGE_VERSION=$env:ARCHIVE_VERSION" + + $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) + if ($archives.Count -ne 1) { + throw "Expected exactly one combined ZIP, found $($archives.Count)" + } + $expectedName = "tsfile-$env:ARCHIVE_VERSION-windows-x86_64.zip" + if ($archives[0].Name -cne $expectedName) { + throw "Unexpected ZIP name '$($archives[0].Name)'; expected '$expectedName'" + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead($archives[0].FullName) + try { + $entryPaths = @($archive.Entries | ForEach-Object { + $_.FullName.Replace('\', '/') + }) + $requiredSuffixes = @( + 'bin/tsfile-cli.exe' + 'bin/tsfile.dll' + 'lib/tsfile.lib' + 'include/tsfile/cwrapper/tsfile_cwrapper.h' + ) + foreach ($suffix in $requiredSuffixes) { + $match = $entryPaths | Where-Object { + $_.EndsWith($suffix, [System.StringComparison]::Ordinal) + } + if (-not $match) { + throw "ZIP is missing a path ending in '$suffix'" + } + } + } finally { + $archive.Dispose() + } + + Expand-Archive -LiteralPath $archives[0].FullName -DestinationPath extracted/windows + python packaging/scripts/verify_windows_runtime.py extracted/windows + $zipCli = @(Get-ChildItem extracted/windows -Recurse -Filter tsfile-cli.exe -File) + if ($zipCli.Count -ne 1) { throw 'Expected exactly one extracted CLI' } + $env:PATH = "$($zipCli[0].DirectoryName);$env:SystemRoot/System32" + & $zipCli[0].FullName --version + + - name: Upload Windows SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-windows-msvc-x86_64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload Windows ZIP + uses: actions/upload-artifact@v7 + with: + name: native-windows-msvc-x86_64 + path: packages/*.zip + if-no-files-found: error + retention-days: 14 + + build-windows-arm64: + name: Build Windows MSVC ARM64 C++ SDK and ZIP + needs: prepare + runs-on: windows-11-vs2026-arm + timeout-minutes: 50 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Select the native ARM64 Java runtime + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + if (-not $env:JAVA_HOME_21_AARCH64) { + throw 'The native ARM64 Java runtime is missing from the runner image' + } + "JAVA_HOME=$env:JAVA_HOME_21_AARCH64" >> $env:GITHUB_ENV + "$env:JAVA_HOME_21_AARCH64\bin" >> $env:GITHUB_PATH + java -version + + - name: Configure the native ARM64 MSVC environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: arm64 + + - name: Verify native ARM64 MSVC compiler + shell: pwsh + run: | + $compiler = (Get-Command cl.exe).Source + Write-Host "Native compiler: $compiler" + if ($compiler -notmatch 'HostARM64\\ARM64\\cl\.exe$') { + throw "Expected the ARM64-hosted ARM64 compiler, got $compiler" + } + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $projectInclude = (Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path + .\mvnw.cmd -Pwith-cpp package ` + "-Dcpp.toolchain=msvc-arm64-native" ` + "-Dbuild.type=Release" ` + "-Dbuild.test=OFF" ` + "-DskipTests" ` + "-Dspotless.skip=true" ` + "-Dtsfile.dependency.source=BUNDLED" ` + "-Dtsfile.msvc.static.runtime=ON" ` + "-Denable.lzma2=ON" ` + "-Dtsfile.project.include=$projectInclude" ` + "-Dtsfile.version.sync.skip=true" ` + "-Denable.cpack=ON" ` + "-Dtsfile.archive.version=$env:ARCHIVE_VERSION" ` + "-Dtsfile.debian.package.version=$env:DEBIAN_VERSION" ` + "-Dtsfile.rpm.package.version=$env:RPM_VERSION" ` + "-Dtsfile.rpm.package.release=$env:RPM_RELEASE" + if (-not (Test-Path -LiteralPath cpp/target/build/CPackConfig.cmake -PathType Leaf)) { + throw 'Maven did not generate cpp/target/build/CPackConfig.cmake' + } + + - name: Stage and verify the Windows ARM64 SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $sdkName = "tsfile-sdk-windows-msvc-arm64-$env:ARCHIVE_VERSION" + $sdkRoot = Join-Path $PWD "sdk/$sdkName" + cmake --install cpp/target/build --config Release --prefix $sdkRoot + $requiredPaths = @( + "$sdkRoot/bin/tsfile-cli.exe" + "$sdkRoot/bin/tsfile.dll" + "$sdkRoot/lib/tsfile.lib" + "$sdkRoot/include/tsfile/cwrapper/tsfile_cwrapper.h" + "$sdkRoot/lib/cmake/TsFile/TsFileConfig.cmake" + "$sdkRoot/share/doc/tsfile/LICENSE" + "$sdkRoot/share/doc/tsfile/NOTICE" + ) + foreach ($path in $requiredPaths) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required ARM64 SDK file is absent: $path" + } + } + python packaging/scripts/verify_windows_runtime.py $sdkRoot + $env:PATH = "$sdkRoot/bin;$env:PATH" + & "$sdkRoot/bin/tsfile-cli.exe" --version + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/windows-arm64 ` + -G "Ninja Multi-Config" ` + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ` + "-DCMAKE_PREFIX_PATH=$sdkRoot" + cmake --build consumer/windows-arm64 --config Release --parallel + & consumer/windows-arm64/Release/tsfile_installed_consumer.exe + tar.exe -czf "$sdkRoot.tar.gz" -C sdk $sdkName + Remove-Item -LiteralPath $sdkRoot -Recurse -Force + + - name: Build and inspect the Windows ARM64 ZIP + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $cpackConfig = Get-Content cpp/target/build/CPackConfig.cmake -Raw + if ($cpackConfig -notmatch 'set\(CPACK_ARCHIVE_COMPONENT_INSTALL "OFF"\)') { + throw 'CPACK_ARCHIVE_COMPONENT_INSTALL must remain disabled' + } + New-Item -ItemType Directory -Path packages -Force | Out-Null + cpack -G ZIP -C Release --config cpp/target/build/CPackConfig.cmake -B packages ` + -D "CPACK_PACKAGE_VERSION=$env:ARCHIVE_VERSION" ` + -D "CPACK_PACKAGE_FILE_NAME=tsfile-$env:ARCHIVE_VERSION-windows-arm64" + $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) + if ($archives.Count -ne 1) { throw "Expected exactly one ARM64 ZIP, found $($archives.Count)" } + $expectedName = "tsfile-$env:ARCHIVE_VERSION-windows-arm64.zip" + if ($archives[0].Name -cne $expectedName) { + throw "Unexpected ZIP name '$($archives[0].Name)'; expected '$expectedName'" + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead($archives[0].FullName) + try { + $entryPaths = @($archive.Entries | ForEach-Object { $_.FullName.Replace('\', '/') }) + foreach ($suffix in @( + 'bin/tsfile-cli.exe' + 'bin/tsfile.dll' + 'lib/tsfile.lib' + 'include/tsfile/cwrapper/tsfile_cwrapper.h' + )) { + if (-not ($entryPaths | Where-Object { $_.EndsWith($suffix, [System.StringComparison]::Ordinal) })) { + throw "ARM64 ZIP is missing a path ending in '$suffix'" + } + } + } finally { $archive.Dispose() } + Expand-Archive -LiteralPath $archives[0].FullName -DestinationPath extracted/windows-arm64 + python packaging/scripts/verify_windows_runtime.py extracted/windows-arm64 + $zipCli = @(Get-ChildItem extracted/windows-arm64 -Recurse -Filter tsfile-cli.exe -File) + if ($zipCli.Count -ne 1) { throw 'Expected exactly one ARM64 CLI' } + & $zipCli[0].FullName --version + + - name: Upload Windows ARM64 SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-windows-msvc-arm64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload Windows ARM64 ZIP + uses: actions/upload-artifact@v7 + with: + name: native-windows-msvc-arm64 + path: packages/*.zip + if-no-files-found: error + retention-days: 14 + + build-python-unix: + name: Build Python wheels on ${{ matrix.name }} + needs: prepare + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - name: manylinux_2_28-x86_64 + os: ubuntu-22.04 + arch: x86_64 + - name: manylinux_2_28-aarch64 + os: ubuntu-22.04-arm + arch: aarch64 + - name: macos12-x86_64 + os: macos-15-intel + arch: x86_64 + - name: macos12-arm64 + os: macos-latest + arch: arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Set up Java 17 for macOS + if: runner.os == 'macOS' + uses: actions/setup-java@v6.0.0 + with: + distribution: temurin + java-version: "17" + + - name: Install macOS build prerequisites + if: runner.os == 'macOS' + run: | + command -v cmake > /dev/null || brew install cmake + command -v pkg-config > /dev/null || brew install pkgconf + + - name: Install wheel build tools + run: python -m pip install cibuildwheel==3.4.1 + + - name: Build wheels from one SDK per compatibility baseline + env: + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_SKIP: "pp* *-musllinux*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL_LINUX: >- + bash packaging/scripts/build_python_sdk.sh + ${{ matrix.name }} ${{ needs.prepare.outputs.archive_version }} + CIBW_BEFORE_ALL_MACOS: >- + bash packaging/scripts/build_python_sdk.sh + ${{ matrix.name }} ${{ needs.prepare.outputs.archive_version }} + CIBW_ENVIRONMENT_LINUX: >- + TSFILE_CPP_BUILD=/project/python-sdk + TSFILE_PYTHON_SDK_EXPORT="/host${{ github.workspace }}/python-sdk-output" + CIBW_ENVIRONMENT_MACOS: >- + TSFILE_CPP_BUILD="${{ github.workspace }}/python-sdk" + MACOSX_DEPLOYMENT_TARGET=12.0 + MACOSX_DEPLOYMENT_TARGET: "12.0" + CIBW_TEST_SKIP: "*" + CIBW_BUILD_VERBOSITY: "1" + run: python -m cibuildwheel --output-dir wheelhouse python + + - name: Upload the Python compatibility SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-${{ matrix.name }} + path: python-sdk-output/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Check runtime-only wheel contents + run: python packaging/scripts/check_python_wheel.py --wheelhouse wheelhouse + + - name: Upload the Python wheels + uses: actions/upload-artifact@v7 + with: + name: native-python-wheel-${{ matrix.name }} + path: wheelhouse/*.whl + if-no-files-found: error + retention-days: 14 + + build-python-windows: + name: Build Python wheels from the Windows MSVC SDK + needs: build-windows + runs-on: windows-2022 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Configure the MSVC environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Download the Windows SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-windows-msvc-x86_64 + path: sdk-download + + - name: Prepare the SDK for the Python build + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $archives = @(Get-ChildItem sdk-download -Filter *.tar.gz -File) + if ($archives.Count -ne 1) { throw 'Expected one Windows SDK archive' } + New-Item -ItemType Directory -Path sdk | Out-Null + tar.exe -xzf $archives[0].FullName -C sdk + $roots = @(Get-ChildItem sdk -Directory) + if ($roots.Count -ne 1) { throw 'Expected one Windows SDK prefix' } + $sdkRoot = $roots[0].FullName + # setup.py consumes libraries from lib/; the installed SDK stores DLLs in bin/. + Copy-Item "$sdkRoot/bin/*.dll" "$sdkRoot/lib/" + if (-not (Test-Path "$sdkRoot/lib/tsfile.lib")) { throw 'Missing MSVC import library' } + if (-not (Test-Path "$sdkRoot/lib/tsfile.dll")) { throw 'Missing TsFile DLL' } + "TSFILE_CPP_BUILD=$sdkRoot" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Install wheel build tools + run: python -m pip install cibuildwheel==3.4.1 + + - name: Build wheels using the installed SDK + env: + CIBW_ARCHS_WINDOWS: AMD64 + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_TEST_SKIP: "*" + CIBW_BUILD_VERBOSITY: "1" + run: python -m cibuildwheel --output-dir wheelhouse python + + - name: Check runtime-only wheel contents + run: python packaging/scripts/check_python_wheel.py --wheelhouse wheelhouse + + - name: Upload the Python wheels + uses: actions/upload-artifact@v7 + with: + name: native-python-wheel-windows-msvc-x86_64 + path: wheelhouse/*.whl + if-no-files-found: error + retention-days: 14 + + build-homebrew: + name: Build Homebrew bottle on ${{ matrix.name }} + needs: prepare + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: macos-arm64 + os: macos-latest + - name: macos-x86_64 + os: macos-15-intel + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_CLEANUP: "1" + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} + SOURCE_REPOSITORY: ${{ github.repository }} + SOURCE_GIT_SHA: ${{ github.sha }} + BOTTLE_ROOT_URL: https://packages.apache.org/artifactory/tsfile/homebrew/release/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Render immutable source Formula in a temporary local tap + shell: bash + run: | + mkdir -p homebrew/Formula homebrew/bottles + curl --fail --location --retry 3 \ + "https://github.com/$SOURCE_REPOSITORY/archive/$SOURCE_GIT_SHA.tar.gz" \ + --output "$RUNNER_TEMP/tsfile-source.tar.gz" + source_sha256="$(shasum -a 256 "$RUNNER_TEMP/tsfile-source.tar.gz" | awk '{print $1}')" + python3 packaging/scripts/render_homebrew_formula.py \ + --template packaging/homebrew/tsfile.rb.in \ + --repository "$SOURCE_REPOSITORY" \ + --git-sha "$SOURCE_GIT_SHA" \ + --version "$TSFILE_HOMEBREW_VERSION" \ + --source-sha256 "$source_sha256" \ + --output homebrew/Formula/tsfile.rb + ruby -c homebrew/Formula/tsfile.rb + brew tap-new apache/tsfile + tap_path="$(brew --repository apache/tsfile)" + cp homebrew/Formula/tsfile.rb "$tap_path/Formula/tsfile.rb" + + - name: Build, test, and bottle the release Formula + shell: bash + run: | + brew install --build-bottle apache/tsfile/tsfile + python3 packaging/scripts/verify_macos_runtime.py "$(brew --prefix apache/tsfile/tsfile | tail -n 1)" + brew test apache/tsfile/tsfile + cd homebrew/bottles + brew bottle --json --root-url "$BOTTLE_ROOT_URL" apache/tsfile/tsfile + shopt -s nullglob + bottles=(*.bottle*.tar.gz) + metadata=(*.bottle.json) + [[ ${#bottles[@]} -eq 1 && ${#metadata[@]} -eq 1 ]] + + - name: Stage and verify the macOS SDK + shell: bash + run: | + # The runner's /bin/bash is 3.2, where "set -u" rejects empty arrays. + set -eo pipefail + fail() { printf '::error::%s\n' "$1"; exit 1; } + sdk_name="tsfile-sdk-${{ matrix.name }}-$ARCHIVE_VERSION" + # Homebrew may print advisory lines before the path, so keep the last line. + formula_prefix="$(brew --prefix apache/tsfile/tsfile | tail -n 1)" + [ -n "$formula_prefix" ] || fail "brew --prefix returned no formula path" + printf 'formula prefix: <%s>\n' "$formula_prefix" + keg_root="$(cd "$formula_prefix" && pwd -P)" + printf 'resolved keg: <%s>\n' "$keg_root" + command -v pkg-config > /dev/null 2>&1 || brew install pkgconf + command -v pkg-config > /dev/null 2>&1 || fail "pkg-config is unavailable on the runner" + shopt -s nullglob + dylibs=("$keg_root"/lib/libtsfile*.dylib) + pkgconfigs=("$keg_root"/lib/pkgconfig/libtsfile.pc) + [ -d "$keg_root/include" ] || fail "missing include/ under $keg_root" + [ ${#dylibs[@]} -gt 0 ] || fail "missing libtsfile*.dylib under $keg_root/lib" + [ -x "$keg_root/bin/tsfile-cli" ] || fail "missing bin/tsfile-cli under $keg_root" + [ -f "$keg_root/lib/cmake/TsFile/TsFileConfig.cmake" ] || + fail "missing TsFileConfig.cmake under $keg_root/lib/cmake/TsFile" + [ ${#pkgconfigs[@]} -gt 0 ] || fail "missing libtsfile.pc under $keg_root/lib/pkgconfig" + + # Stage a real tree: the keg lives at a symlinked opt path and must not + # be archived as a single link. + sdk_root="$PWD/sdk/$sdk_name" + rm -rf "$sdk_root" + mkdir -p "$sdk_root" + ditto "$keg_root" "$sdk_root" + + # Drop Homebrew keg bookkeeping so the archive holds the same SDK + # layout as the Linux and Windows archives. + rm -rf "$sdk_root/.brew" "$sdk_root/INSTALL_RECEIPT.json" "$sdk_root/sbom.spdx.json" + + # A keg records its own absolute path as the dylib identity, which + # would pin every downstream consumer to this runner's keg. + real_dylib="" + for candidate in "$sdk_root"/lib/libtsfile*.dylib; do + [ -L "$candidate" ] || real_dylib="$candidate" + done + [ -n "$real_dylib" ] || fail "missing real libtsfile dylib under $sdk_root/lib" + dylib_id="$(basename "$(otool -D "$real_dylib" | sed -n '2p')")" + [ -n "$dylib_id" ] || fail "cannot read the dylib identity from $real_dylib" + install_name_tool -id "@rpath/$dylib_id" "$real_dylib" + codesign --force --sign - "$real_dylib" + [ "$(otool -D "$real_dylib" | sed -n '2p')" = "@rpath/$dylib_id" ] || + fail "dylib identity was not normalized to @rpath" + + python3 packaging/scripts/verify_macos_runtime.py "$sdk_root" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/macos \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/macos --parallel + ./consumer/macos/tsfile_installed_consumer + tar -C sdk -czf "sdk/$sdk_name.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Upload platform Homebrew bottle and source Formula + uses: actions/upload-artifact@v7 + with: + name: native-homebrew-${{ matrix.name }} + path: homebrew/ + if-no-files-found: error + retention-days: 14 + + - name: Upload macOS SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-${{ matrix.name }} + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + merge-homebrew: + name: Merge Homebrew release bottles + needs: [prepare, build-homebrew] + runs-on: macos-latest + timeout-minutes: 15 + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} + BOTTLE_ROOT_URL: https://packages.apache.org/artifactory/tsfile/homebrew/release/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles + steps: + - name: Download both platform Homebrew artifacts separately + uses: actions/download-artifact@v8 + with: + pattern: native-homebrew-macos-* + path: homebrew-intermediate + merge-multiple: false + + - name: Recreate the same pre-bottle Formula and collect metadata + shell: bash + run: | + arm=homebrew-intermediate/native-homebrew-macos-arm64 + intel=homebrew-intermediate/native-homebrew-macos-x86_64 + cmp "$arm/Formula/tsfile.rb" "$intel/Formula/tsfile.rb" + brew tap-new apache/tsfile + tap_path="$(brew --repository apache/tsfile)" + cp "$arm/Formula/tsfile.rb" "$tap_path/Formula/tsfile.rb" + mkdir -p homebrew/Formula homebrew/bottles + python3 - <<'PY' + import hashlib + import json + import os + import shutil + from pathlib import Path + + destination = Path("homebrew/bottles") + tags = set() + for platform in ("macos-arm64", "macos-x86_64"): + directory = Path("homebrew-intermediate") / f"native-homebrew-{platform}" / "bottles" + metadata = list(directory.glob("*.bottle.json")) + assert len(metadata) == 1, f"Expected one JSON for {platform}" + data = json.loads(metadata[0].read_text()) + assert list(data) == ["apache/tsfile/tsfile"] + entry = data["apache/tsfile/tsfile"] + assert entry["formula"]["pkg_version"] == os.environ["TSFILE_HOMEBREW_VERSION"] + assert entry["formula"]["path"] == "Library/Taps/apache/homebrew-tsfile/Formula/tsfile.rb" + assert entry["bottle"]["root_url"] == os.environ["BOTTLE_ROOT_URL"] + assert len(entry["bottle"]["tags"]) == 1 + tag, bottle = next(iter(entry["bottle"]["tags"].items())) + assert (platform == "macos-arm64") == tag.startswith("arm64_") + assert tag not in tags, f"Duplicate bottle tag: {tag}" + tags.add(tag) + local_name = bottle["local_filename"] + assert Path(local_name).name == local_name + source = directory / local_name + assert hashlib.sha256(source.read_bytes()).hexdigest() == bottle["sha256"] + for source in (source, metadata[0]): + target = destination / source.name + assert not target.exists(), f"Duplicate artifact: {target}" + shutil.copy2(source, target) + assert len(tags) == 2 + PY + + - name: Merge both bottle tags into the Formula + shell: bash + run: | + brew trust apache/tsfile + brew bottle --merge --write --no-commit homebrew/bottles/*.json + tap_path="$(brew --repository apache/tsfile)" + cp "$tap_path/Formula/tsfile.rb" homebrew/Formula/tsfile.rb + ruby -c homebrew/Formula/tsfile.rb + + - name: Verify merged Formula and normalize HTTP bottle filenames + shell: bash + run: | + python3 - <<'PY' + import hashlib + import json + import os + import re + from pathlib import Path + from urllib.parse import unquote + + formula = Path("homebrew/Formula/tsfile.rb").read_text() + assert not re.search(r"@[A-Z0-9_]+@", formula), "Unresolved Formula token" + assert len(re.findall(r"^\s*bottle\s+do\b", formula, re.M)) == 1 + assert f'root_url "{os.environ["BOTTLE_ROOT_URL"]}"' in formula + directory = Path("homebrew/bottles") + metadata = list(directory.glob("*.bottle.json")) + assert len(metadata) == 2 + tags = set() + for path in metadata: + data = json.loads(path.read_text()) + entry = data["apache/tsfile/tsfile"] + for tag, bottle in entry["bottle"]["tags"].items(): + assert tag not in tags, f"Duplicate bottle tag: {tag}" + tags.add(tag) + assert re.search(rf'\b{re.escape(tag)}:\s*"{bottle["sha256"]}"', formula) + # Homebrew emits a double-dash local name but serves a single-dash URL. + filename = unquote(bottle["filename"]) + assert Path(filename).name == filename and filename.endswith(".tar.gz") + source = directory / bottle["local_filename"] + target = directory / filename + assert hashlib.sha256(source.read_bytes()).hexdigest() == bottle["sha256"] + if source != target: + assert not target.exists(), f"Duplicate HTTP bottle name: {filename}" + source.rename(target) + bottle["local_filename"] = filename + path.write_text(json.dumps(data, indent=2) + "\n") + assert len(tags) == 2 + assert len(list(directory.glob("*.tar.gz"))) == 2 + PY + + - name: Upload merged Homebrew Formula and bottles + uses: actions/upload-artifact@v7 + with: + name: native-homebrew + path: homebrew/ + if-no-files-found: error + retention-days: 14 + + test-windows: + name: Test installed Windows C++, CLI and Go on ${{ matrix.name }} + needs: [build-windows, build-windows-arm64] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: windows-msvc-x86_64 + os: windows-2022 + msvc-arch: x64 + goarch: amd64 + llvm-arch: x86_64 + llvm-sha256: 317492c456aa27ee607a5919f1d2d38dcdc1112516a24d0bf4b00d078f52d17a + - name: windows-msvc-arm64 + os: windows-11-vs2026-arm + msvc-arch: arm64 + goarch: arm64 + llvm-arch: aarch64 + llvm-sha256: 9a835d5179c9f3a5a783c11a7f14062a249b4765dd11add64a66786864e82ab2 + steps: + - name: Checkout installation test sources + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + cpp/test + go + sparse-checkout-cone-mode: false + + - name: Set up native MSVC for C++ tests + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.msvc-arch }} + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Download Windows ZIP + uses: actions/download-artifact@v8 + with: + name: native-${{ matrix.name }} + path: packages + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Install ZIP and test the CLI with Python test data + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $archives = @(Get-ChildItem packages -Filter *.zip -File) + if ($archives.Count -ne 1) { throw 'Expected one Windows ZIP' } + Expand-Archive -LiteralPath $archives[0].FullName -DestinationPath installed + $executables = @(Get-ChildItem installed -Recurse -Filter tsfile-cli.exe -File) + if ($executables.Count -ne 1) { throw 'Expected one installed CLI' } + $prefix = $executables[0].Directory.Parent.FullName + "TSFILE_INSTALLED_PREFIX=$prefix" >> $env:GITHUB_ENV + $python = (Get-Command python).Source + $env:PATH = "$($executables[0].DirectoryName);$env:SystemRoot/System32;$env:SystemRoot" + & $python packaging/tests/cli_package_smoke.py ` + --cli $executables[0].FullName --resources cli-test-data + + - name: Compile and run C++ regressions against the installed SDK + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + python packaging/tests/cpp_installed_tests.py ` + --prefix "$env:TSFILE_INSTALLED_PREFIX" --build-dir installed-tests + + - name: Install pinned LLVM-MinGW for Go cgo + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $archive = Join-Path $env:RUNNER_TEMP 'llvm-mingw.zip' + $url = 'https://github.com/mstorsjo/llvm-mingw/releases/download/20261006/llvm-mingw-20261006-ucrt-${{ matrix.llvm-arch }}.zip' + curl.exe --fail --location --retry 3 --output $archive $url + $hash = (Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() + if ($hash -ne '${{ matrix.llvm-sha256 }}') { throw 'LLVM-MinGW checksum mismatch' } + $destination = Join-Path $env:RUNNER_TEMP 'llvm-mingw' + Expand-Archive -LiteralPath $archive -DestinationPath $destination + $tools = Join-Path $destination 'llvm-mingw-20261006-ucrt-${{ matrix.llvm-arch }}/bin' + if (-not (Test-Path "$tools/llvm-dlltool.exe")) { throw 'LLVM-MinGW tools missing' } + "TSFILE_LLVM_BIN=$tools" >> $env:GITHUB_ENV + + - name: Run Go tests against the installed MSVC DLL + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + python packaging/tests/go_installed_tests.py ` + --prefix "$env:TSFILE_INSTALLED_PREFIX" --goarch ${{ matrix.goarch }} ` + --llvm-bin "$env:TSFILE_LLVM_BIN" + + - name: Download the standalone SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-${{ matrix.name }} + path: sdk-download + + - name: Test the downloaded and relocated SDK archive + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + python packaging/tests/sdk_archive_tests.py ` + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests ` + --resources cli-test-data --goarch ${{ matrix.goarch }} ` + --gtest-prefix installed-tests/dependencies --llvm-bin "$env:TSFILE_LLVM_BIN" + + - name: Upload C++ installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-cpp-tests-${{ matrix.name }} + path: | + installed-tests/cpp-results.xml + installed-tests/tests/Testing/Temporary/LastTest.log + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + test-homebrew: + name: Test installed Homebrew C++, CLI and Go on ${{ matrix.name }} + needs: build-homebrew + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: macos-arm64 + os: macos-latest + goarch: arm64 + - name: macos-x86_64 + os: macos-15-intel + goarch: amd64 + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_CLEANUP: "1" + steps: + - name: Checkout installation test sources + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + packaging/scripts/verify_macos_runtime.py + cpp/test + cpp/cmake/tests/projects/InstalledConsumer + go + sparse-checkout-cone-mode: false + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Install test build tools + run: | + command -v cmake >/dev/null || brew install cmake + command -v pkg-config >/dev/null || brew install pkgconf + + - name: Download Homebrew bottle and Formula + uses: actions/download-artifact@v8 + with: + name: native-homebrew-${{ matrix.name }} + path: homebrew + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Install the local bottle and test the CLI with Python test data + shell: bash + run: | + brew tap-new apache/tsfile + tap_dir="$(brew --repository apache/tsfile)" + cp homebrew/Formula/tsfile.rb "$tap_dir/Formula/tsfile.rb" + brew trust apache/tsfile + shopt -s nullglob + bottles=("$PWD"/homebrew/bottles/*.bottle*.tar.gz) + [[ ${#bottles[@]} -eq 1 ]] + # Load bottle checksums into the trusted tap, then seed the exact + # download cache path. Install by formula name: current Homebrew + # rejects direct package paths outside developer mode. + brew bottle --merge --write --no-commit homebrew/bottles/*.json + cached_bottle="$(brew --cache --force-bottle apache/tsfile/tsfile | tail -n 1)" + [[ "$cached_bottle" == *.bottle*.tar.gz ]] + mkdir -p "$(dirname "$cached_bottle")" + cp "${bottles[0]}" "$cached_bottle" + brew install --force-bottle apache/tsfile/tsfile + formula_prefix="$(brew --prefix apache/tsfile/tsfile | tail -n 1)" + echo "TSFILE_INSTALLED_PREFIX=$formula_prefix" >> "$GITHUB_ENV" + python3 - "$formula_prefix" <<'PY' + import json + import sys + from pathlib import Path + + receipt = json.loads((Path(sys.argv[1]) / "INSTALL_RECEIPT.json").read_text()) + if receipt.get("poured_from_bottle") is not True: + raise SystemExit("CLI acceptance must install the built bottle") + PY + python3 packaging/scripts/verify_macos_runtime.py "$formula_prefix" + python3 packaging/tests/cli_package_smoke.py \ + --cli "$formula_prefix/bin/tsfile-cli" --resources cli-test-data + + - name: Compile and run C++ regressions against the installed SDK + shell: bash + run: | + python3 packaging/tests/cpp_installed_tests.py \ + --prefix "$TSFILE_INSTALLED_PREFIX" --build-dir installed-tests + + - name: Run Go tests against the installed SDK + shell: bash + run: | + python3 packaging/tests/go_installed_tests.py \ + --prefix "$TSFILE_INSTALLED_PREFIX" --goarch ${{ matrix.goarch }} + + - name: Download the standalone SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-${{ matrix.name }} + path: sdk-download + + - name: Test the downloaded and relocated SDK archive + shell: bash + run: | + python3 packaging/tests/sdk_archive_tests.py \ + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests \ + --resources cli-test-data --goarch ${{ matrix.goarch }} \ + --gtest-prefix installed-tests/dependencies + + - name: Upload C++ installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-cpp-tests-${{ github.job }}-${{ runner.arch }}-${{ strategy.job-index }} + path: | + installed-tests/cpp-results.xml + installed-tests/tests/Testing/Temporary/LastTest.log + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + test-compatibility-sdk: + name: Test downloaded compatibility SDK on ${{ matrix.name }} + needs: [prepare, build-python-unix] + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - name: manylinux_2_28-x86_64 + os: ubuntu-22.04 + goarch: amd64 + - name: manylinux_2_28-aarch64 + os: ubuntu-22.04-arm + goarch: arm64 + - name: macos12-x86_64 + os: macos-15-intel + goarch: amd64 + - name: macos12-arm64 + os: macos-latest + goarch: arm64 + steps: + - name: Checkout SDK consumer tests only + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests + packaging/cmake/InstalledGTest + packaging/scripts/verify_macos_runtime.py + cpp/test + go + sparse-checkout-cone-mode: false + + - name: Set up Go for installed-package tests + uses: actions/setup-go@v7 + with: + go-version: "1.26.x" + cache-dependency-path: go/go.sum + + - name: Install Linux consumer prerequisites + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y cmake build-essential pkg-config python3 libuuid1 + + - name: Install macOS consumer prerequisites + if: runner.os == 'macOS' + run: | + command -v cmake >/dev/null || brew install cmake + command -v pkg-config >/dev/null || brew install pkgconf + + - name: Download the compatibility SDK archive + uses: actions/download-artifact@v8 + with: + name: native-sdk-${{ matrix.name }} + path: sdk-download + + - name: Download CLI test data + uses: actions/download-artifact@v8 + with: + name: native-cli-test-data + path: cli-test-data + + - name: Test the downloaded and relocated SDK archive + run: | + python3 packaging/tests/sdk_archive_tests.py \ + --archives sdk-download --destination downloaded-sdk --build-dir sdk-tests \ + --resources cli-test-data --goarch ${{ matrix.goarch }} + + - name: Upload SDK archive test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-sdk-tests-${{ matrix.name }} + path: | + sdk-tests/cpp-results.xml + sdk-tests/tests/Testing/Temporary/LastTest.log + if-no-files-found: ignore + retention-days: 14 + + test-python-unix: + name: Test installed Python ${{ matrix.python-version }} wheel on ${{ matrix.platform.name }} + needs: [prepare, build-python-unix] + runs-on: ${{ matrix.platform.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + platform: + - name: manylinux_2_28-x86_64 + os: ubuntu-22.04 + - name: manylinux_2_28-aarch64 + os: ubuntu-22.04-arm + - name: macos12-x86_64 + os: macos-15-intel + - name: macos12-arm64 + os: macos-latest + steps: + - name: Checkout Python tests only + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests/python_installed_tests.py + packaging/scripts/check_python_wheel.py + python/tests + sparse-checkout-cone-mode: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Download the platform wheels + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-${{ matrix.platform.name }} + path: wheelhouse + + - name: Install wheel and run the existing pytest suite + run: | + python packaging/tests/python_installed_tests.py \ + --wheelhouse wheelhouse --version "${{ needs.prepare.outputs.python_version }}" \ + --results python-test-results.xml + + - name: Upload Python installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-pytest-${{ matrix.platform.name }}-py${{ matrix.python-version }} + path: python-test-results.xml + if-no-files-found: error + retention-days: 14 + + test-python-windows: + name: Test installed Windows Python ${{ matrix.python-version }} wheel + needs: [prepare, build-python-windows] + runs-on: windows-2022 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - name: Checkout Python tests only + uses: actions/checkout@v7 + with: + sparse-checkout: | + packaging/tests/python_installed_tests.py + packaging/scripts/check_python_wheel.py + python/tests + sparse-checkout-cone-mode: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Download the Windows wheels + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-windows-msvc-x86_64 + path: wheelhouse + + - name: Install wheel and run the existing pytest suite + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + python packaging/tests/python_installed_tests.py ` + --wheelhouse wheelhouse --version "${{ needs.prepare.outputs.python_version }}" ` + --results python-test-results.xml + + - name: Upload Python installation test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: native-pytest-windows-msvc-x86_64-py${{ matrix.python-version }} + path: python-test-results.xml + if-no-files-found: error + retention-days: 14 + + assemble: + name: Assemble checksummed native package bundle + needs: + - prepare + - test-deb + - test-deb-arm64 + - test-rpm + - test-rpm-aarch64 + - test-windows + - test-homebrew + - test-compatibility-sdk + - test-python-unix + - test-python-windows + - build-python-unix + - build-python-windows + - merge-homebrew + - build-windows + - build-windows-arm64 + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Download package versions + uses: actions/download-artifact@v8 + with: + name: native-package-versions + path: versions + + - name: Download DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-amd64 + path: input/deb/amd64 + + - name: Download ARM64 DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-arm64 + path: input/deb/arm64 + + - name: Download RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-x86_64 + path: input/rpm/x86_64 + + - name: Download aarch64 RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-aarch64 + path: input/rpm/aarch64 + + - name: Download Homebrew Formula and bottles + uses: actions/download-artifact@v8 + with: + name: native-homebrew + path: input/homebrew + + - name: Download Windows ZIP + uses: actions/download-artifact@v8 + with: + name: native-windows-msvc-x86_64 + path: input/windows/x86_64 + + - name: Download Windows ARM64 ZIP + uses: actions/download-artifact@v8 + with: + name: native-windows-msvc-arm64 + path: input/windows/arm64 + + - name: Download Ubuntu SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-ubuntu22.04-amd64 + path: input/sdk/ubuntu22.04-amd64 + + - name: Download Ubuntu ARM64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-ubuntu22.04-arm64 + path: input/sdk/ubuntu22.04-arm64 + + - name: Download AlmaLinux SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-almalinux9-x86_64 + path: input/sdk/almalinux9-x86_64 + + - name: Download AlmaLinux aarch64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-almalinux9-aarch64 + path: input/sdk/almalinux9-aarch64 + + - name: Download Windows SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-windows-msvc-x86_64 + path: input/sdk/windows-msvc-x86_64 + + - name: Download Windows ARM64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-windows-msvc-arm64 + path: input/sdk/windows-msvc-arm64 + + - name: Download macOS arm64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-macos-arm64 + path: input/sdk/macos-arm64 + + - name: Download macOS x86_64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-macos-x86_64 + path: input/sdk/macos-x86_64 + + - name: Download the Python SDK for manylinux_2_28-x86_64 + uses: actions/download-artifact@v8 + with: + name: native-sdk-manylinux_2_28-x86_64 + path: input/sdk/manylinux_2_28-x86_64 + + - name: Download Python wheels for manylinux_2_28-x86_64 + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-manylinux_2_28-x86_64 + path: input/python/manylinux_2_28-x86_64 + + - name: Download the Python SDK for manylinux_2_28-aarch64 + uses: actions/download-artifact@v8 + with: + name: native-sdk-manylinux_2_28-aarch64 + path: input/sdk/manylinux_2_28-aarch64 + + - name: Download Python wheels for manylinux_2_28-aarch64 + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-manylinux_2_28-aarch64 + path: input/python/manylinux_2_28-aarch64 + + - name: Download the Python SDK for macos12-x86_64 + uses: actions/download-artifact@v8 + with: + name: native-sdk-macos12-x86_64 + path: input/sdk/macos12-x86_64 + + - name: Download Python wheels for macos12-x86_64 + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-macos12-x86_64 + path: input/python/macos12-x86_64 + + - name: Download the Python SDK for macos12-arm64 + uses: actions/download-artifact@v8 + with: + name: native-sdk-macos12-arm64 + path: input/sdk/macos12-arm64 + + - name: Download Python wheels for macos12-arm64 + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-macos12-arm64 + path: input/python/macos12-arm64 + + - name: Download Python wheels for windows-msvc-x86_64 + uses: actions/download-artifact@v8 + with: + name: native-python-wheel-windows-msvc-x86_64 + path: input/python/windows-msvc-x86_64 + + - name: Assemble the publishable bundle + env: + LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} + shell: bash + run: | + python3 packaging/scripts/assemble_native_packages.py \ + --input-dir input \ + --output-dir "tsfile-native-packages-$LOGICAL_VERSION" \ + --versions-json versions/versions.json \ + --source-repository "${{ github.repository }}" \ + --source-commit "${{ github.sha }}" + + - name: Verify assembled manifest checksums + env: + LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} + shell: bash + run: | + python3 packaging/scripts/assemble_native_packages.py \ + --verify-bundle "tsfile-native-packages-$LOGICAL_VERSION" + + - name: Upload checksummed native package bundle + uses: actions/upload-artifact@v7 + with: + name: tsfile-native-packages-${{ needs.prepare.outputs.archive_version }} + path: tsfile-native-packages-${{ needs.prepare.outputs.logical_version }}/ + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml deleted file mode 100644 index c0ca24ec2..000000000 --- a/.github/workflows/wheels.yml +++ /dev/null @@ -1,314 +0,0 @@ -name: Build TsFile wheels(multi-platform) - -on: - push: - branches: - - "rc/**" - workflow_dispatch: - -jobs: - build: - name: Build wheels on ${{ matrix.name }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - name: linux-x86_64 - os: ubuntu-22.04 - platform: linux - cibw_archs_linux: "x86_64" - - - name: linux-aarch64 - os: ubuntu-22.04-arm - platform: linux - cibw_archs_linux: "aarch64" - - - name: macos-x86_64 - os: macos-15-intel - platform: macos - cibw_archs_macos: "x86_64" - - - name: macos-arm64 - os: macos-latest - platform: macos - cibw_archs_macos: "arm64" -# Windows is handled by the build-windows job below - - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - submodules: false - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.11" - - - name: Set up Java 17 - uses: actions/setup-java@v6.0.1 - with: - distribution: temurin - java-version: "17" - - - name: Install system deps (macOS) - if: matrix.platform == 'macos' - run: | - set -eux - brew update - brew install pkg-config || true - - - name: Install build tools - run: | - python -m pip install -U pip wheel - python -m pip install cibuildwheel==3.4.1 - - - name: Pre-download virtualenv for cibuildwheel - run: | - if [ "$(uname)" = "Darwin" ]; then - CACHE_DIR="$HOME/Library/Caches/cibuildwheel" - else - CACHE_DIR="$HOME/.cache/cibuildwheel" - fi - mkdir -p "$CACHE_DIR" - TARGET="$CACHE_DIR/virtualenv-20.26.6.pyz" - if [ ! -f "$TARGET" ]; then - curl -sSL --retry 5 --retry-delay 15 --retry-all-errors \ - -o "$TARGET" \ - "https://github.com/pypa/get-virtualenv/raw/20.26.6/public/virtualenv.pyz" - fi - ls -la "$TARGET" - - - - name: Build C++ core via Maven (macOS) - if: matrix.platform == 'macos' - shell: bash - env: - MACOSX_DEPLOYMENT_TARGET: "12.0" - CFLAGS: "-mmacosx-version-min=12.0" - CXXFLAGS: "-mmacosx-version-min=12.0" - LDFLAGS: "-mmacosx-version-min=12.0" - run: | - set -euxo pipefail - chmod +x mvnw || true - ./mvnw -Pwith-cpp clean package \ - -DskipTests -Dspotless.check.skip=true -Dspotless.apply.skip=true \ - -Dbuild.test=OFF \ - -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ - -Dcmake.args="-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0" - otool -l cpp/target/build/lib/libtsfile*.dylib | grep -A2 LC_VERSION_MIN_MACOSX || true - - - name: Build wheels via cibuildwheel - if: matrix.platform != 'macos' - env: - CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} -# CIBW_ARCHS_WINDOWS: ${{ matrix.cibw_archs_windows }} - - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" - CIBW_SKIP: "pp* *-musllinux*" - - CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" - CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28" - - MACOSX_DEPLOYMENT_TARGET: "12.0" - - CIBW_BEFORE_ALL_LINUX: | - set -euxo pipefail - if command -v yum >/dev/null 2>&1; then - yum install -y wget tar gzip pkgconfig libuuid-devel libblkid-devel - else - echo "Not a yum-based image?" ; exit 1 - fi - ARCH="$(uname -m)" - mkdir -p /opt/java - if [ "$ARCH" = "x86_64" ]; then - JDK_URL="https://download.oracle.com/java/17/archive/jdk-17.0.12_linux-x64_bin.tar.gz" - else - # aarch64 - JDK_URL="https://download.oracle.com/java/17/archive/jdk-17.0.12_linux-aarch64_bin.tar.gz" - fi - curl -L -o /tmp/jdk17.tar.gz "$JDK_URL" - tar -xzf /tmp/jdk17.tar.gz -C /opt/java - export JAVA_HOME=$(echo /opt/java/jdk-17.0.12*) - export PATH="$JAVA_HOME/bin:$PATH" - java -version - - chmod +x mvnw || true - ./mvnw -Pwith-cpp clean package \ - -DskipTests -Dbuild.test=OFF \ - -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ - -Dspotless.check.skip=true -Dspotless.apply.skip=true - test -d cpp/target/build/lib && test -d cpp/target/build/include - - CIBW_TEST_COMMAND: > - python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok:')" - CIBW_BUILD_VERBOSITY: "1" - run: cibuildwheel --output-dir wheelhouse python - - - name: Build wheels via cibuildwheel (macOS) - if: matrix.platform == 'macos' - env: - CIBW_ARCHS_MACOS: ${{ matrix.cibw_archs_macos }} - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" -# CIBW_BUILD: "cp313-*" - CIBW_SKIP: "pp*" - CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=12.0" - MACOSX_DEPLOYMENT_TARGET: "12.0" - CIBW_TEST_COMMAND: > - python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok:')" - CIBW_BUILD_VERBOSITY: "1" - run: cibuildwheel --output-dir wheelhouse python - - - name: Upload wheels as artifact - uses: actions/upload-artifact@v7 - with: - name: tsfile-wheels-${{ matrix.name }} - path: wheelhouse/*.whl - - # ── Windows: build C++ once, then build wheels for each Python version ── - build-windows-cpp: - name: Build C++ core (Windows) - runs-on: windows-2022 - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - submodules: false - fetch-depth: 0 - - - name: Set up Java 17 - uses: actions/setup-java@v6.0.1 - with: - distribution: temurin - java-version: "17" - - - name: Set up MSYS2 / MinGW - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-cmake - mingw-w64-x86_64-make - make - - - name: Build C++ core via Maven - shell: msys2 {0} - run: | - set -euxo pipefail - export JAVA_HOME="$(cygpath "$JAVA_HOME")" - export PATH="$JAVA_HOME/bin:$PATH" - java -version - chmod +x mvnw || true - ./mvnw -Pwith-cpp clean package \ - -DskipTests -Dbuild.test=OFF \ - -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ - -Dspotless.check.skip=true -Dspotless.apply.skip=true - test -d cpp/target/build/lib - test -d cpp/target/build/include - cp /mingw64/bin/libstdc++-6.dll cpp/target/build/lib/ - cp /mingw64/bin/libgcc_s_seh-1.dll cpp/target/build/lib/ - cp /mingw64/bin/libwinpthread-1.dll cpp/target/build/lib/ - - - name: Upload C++ build output - uses: actions/upload-artifact@v7 - with: - name: tsfile-cpp-windows - path: | - cpp/target/build/lib/ - cpp/target/build/include/ - - build-windows-wheels: - name: Build wheel (Windows, Python ${{ matrix.python-version }}) - needs: build-windows-cpp - runs-on: windows-2022 - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - submodules: false - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Java 17 - uses: actions/setup-java@v6.0.1 - with: - distribution: temurin - java-version: "17" - - - name: Set up MSYS2 / MinGW - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: false - install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-make - make - - - name: Download C++ build output - uses: actions/download-artifact@v8 - with: - name: tsfile-cpp-windows - path: cpp/target/build/ - - - name: Verify C++ artifacts exist - shell: msys2 {0} - run: | - set -euxo pipefail - ls -laR cpp/target/build/lib/ - ls -laR cpp/target/build/include/ - test -f cpp/target/build/lib/libtsfile.dll - test -f cpp/target/build/lib/libtsfile.dll.a - test -f cpp/target/build/lib/libstdc++-6.dll - test -f cpp/target/build/lib/libgcc_s_seh-1.dll - test -f cpp/target/build/lib/libwinpthread-1.dll - - - name: Build wheel - shell: msys2 {0} - run: | - set -euxo pipefail - export JAVA_HOME="$(cygpath "$JAVA_HOME")" - export PYTHON_HOME="$(cygpath "$pythonLocation")" - export PATH="/mingw64/bin:$PYTHON_HOME:$PYTHON_HOME/Scripts:$JAVA_HOME/bin:$PATH" - - # Build wheel via Maven (no clean — keep C++ artifacts from previous job) - chmod +x mvnw || true - cd python - BLACK_VER_PROP="" - case "${{ matrix.python-version }}" in - 3.9) BLACK_VER_PROP="-Dblack.version=25.11.0" ;; - esac - ../mvnw package -DskipTests -Dmaven.test.skip=true \ - -Dspotless.check.skip=true -Dspotless.apply.skip=true \ - ${BLACK_VER_PROP} - ls -la dist/ - - - name: Verify wheel - shell: msys2 {0} - run: | - set -euxo pipefail - export PYTHON_HOME="$(cygpath "$pythonLocation")" - export PATH="$PYTHON_HOME:$PYTHON_HOME/Scripts:$PATH" - python -m pip install python/dist/*.whl - python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok')" - - - name: Upload wheel - uses: actions/upload-artifact@v7 - with: - name: tsfile-wheels-windows-py${{ matrix.python-version }} - path: python/dist/*.whl diff --git a/.gitignore b/.gitignore index cf943dae7..dc16352bd 100644 --- a/.gitignore +++ b/.gitignore @@ -40,8 +40,11 @@ python/tsfile/**/*so* python/data python/venv/* python/tests/__pycache__/* +packaging/**/__pycache__/ +packaging/**/*.py[cod] python/tests/*.tsfile python/tsfile/include +python/tsfile/licenses/ cpp/cmake-build-debug-mingw/ cpp/third_party/googletest-release-1.12.1.zip diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c39bcbc28..ebfd40f4d 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -17,6 +17,21 @@ specific language governing permissions and limitations under the License. ]] cmake_minimum_required(VERSION 3.11) +option(TSFILE_MSVC_STATIC_RUNTIME + "Use the static MSVC runtime throughout a portable bundled build" OFF) +if (POLICY CMP0091) + # Use the runtime property for both /MD and /MT builds so exported SDK + # metadata describes the actual compiler selection. Select this before + # project() and inherit it in dependencies with older policy versions. + cmake_policy(SET CMP0091 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0091 NEW) +endif () +if (TSFILE_MSVC_STATIC_RUNTIME) + if (NOT POLICY CMP0091) + message(FATAL_ERROR "TSFILE_MSVC_STATIC_RUNTIME requires CMake 3.15+") + endif () + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +endif () project(TsFile_CPP) if (DEFINED ToolChain) @@ -32,9 +47,42 @@ endif () if (POLICY CMP0074) cmake_policy(SET CMP0074 NEW) endif () -set(TsFile_CPP_VERSION 2.3.2.dev) +set(TsFile_CPP_VERSION 2.5.0) + +include(GNUInstallDirs) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFilePublicHeaders.cmake) + +# The package version identifies the source release. Development suffixes are +# intentionally removed from generated package metadata and never enter the +# SONAME. +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" TSFILE_PACKAGE_VERSION + "${TsFile_CPP_VERSION}") +if ("${TSFILE_PACKAGE_VERSION}" STREQUAL "") + message(FATAL_ERROR + "TsFile_CPP_VERSION must start with a semantic version: " + "${TsFile_CPP_VERSION}") +endif () +set(TSFILE_ARCHIVE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "Version used in portable archive names") +set(TSFILE_DEBIAN_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "Complete Debian package version") +set(TSFILE_RPM_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "RPM Version value") +set(TSFILE_RPM_PACKAGE_RELEASE "1" CACHE STRING + "RPM Release value") +set(TSFILE_ABI_VERSION "1" CACHE STRING + "TsFile shared-library ABI epoch (changes only on ABI breaks)") +if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") + message(FATAL_ERROR "TSFILE_ABI_VERSION must be a positive integer") +endif () include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/DependencySource.cmake) +if (TSFILE_MSVC_STATIC_RUNTIME AND + NOT TSFILE_DEPENDENCY_SOURCE STREQUAL "BUNDLED") + message(FATAL_ERROR + "TSFILE_MSVC_STATIC_RUNTIME requires TSFILE_DEPENDENCY_SOURCE=BUNDLED " + "so prebuilt dependencies cannot introduce a different MSVC runtime") +endif () list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") option(TSFILE_BUILD_SHARED "Build libtsfile as a shared library" ON) @@ -64,10 +112,18 @@ endif () message("cmake using: USE_CPP11=${USE_CPP11}") # MSVC has no /std:c++11; CMake maps this to the closest supported standard # (C++14 default on MSVC), which compiles the C++11 codebase fine. -set(CMAKE_CXX_STANDARD 11) +set(TSFILE_CXX_STANDARD "11" CACHE STRING + "C++ language standard used to build TsFile (11, 14, or 17)") +set_property(CACHE TSFILE_CXX_STANDARD PROPERTY STRINGS 11 14 17) +if (NOT TSFILE_CXX_STANDARD MATCHES "^(11|14|17)$") + message(FATAL_ERROR + "TSFILE_CXX_STANDARD must be one of 11, 14, or 17") +endif () +set(CMAKE_CXX_STANDARD ${TSFILE_CXX_STANDARD}) set(CMAKE_CXX_STANDARD_REQUIRED OFF) if (NOT MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -std=c++${TSFILE_CXX_STANDARD}") endif () if (DEFINED ENV{CXX}) @@ -336,7 +392,8 @@ if (MSVC) # C++17 extensions), so we pin it explicitly for reproducibility. set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14") else () - set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11") + set(CMAKE_CXX_FLAGS + "$ENV{CXXFLAGS} -Wall -std=c++${TSFILE_CXX_STANDARD}") endif () add_subdirectory(third_party) @@ -418,3 +475,160 @@ endif () unset(_TSFILE_PROJECT_DEPENDENCIES) add_subdirectory(examples) + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFileBuildConfig.cmake) + +# Install the compatibility header closure from the generated staging tree. The +# staging tree is populated by the existing copy_* targets and contains only +# headers selected by the reviewed closure manifest. The closure is retained +# for source compatibility; it is not the final public API boundary. +foreach (header_dir IN LISTS TSFILE_PUBLIC_HEADER_CLOSURE) + install(DIRECTORY "${LIBRARY_INCLUDE_DIR}/${header_dir}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tsfile/${header_dir}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endforeach() +if (ENABLE_SIMD AND TSFILE_SIMDE_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_SIMDE_INCLUDE_ROOT}/simde/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/simde" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_ANTLR4_INCLUDE_ROOT}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () +if (ENABLE_ANTLR4 AND TSFILE_UTF8CPP_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_UTF8CPP_INCLUDE_ROOT}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () + +set(TSFILE_LICENSE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE") +set(TSFILE_NOTICE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../NOTICE") +if (EXISTS "${TSFILE_LICENSE_FILE}") + install(FILES "${TSFILE_LICENSE_FILE}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/tsfile" + COMPONENT runtime) +endif() +if (EXISTS "${TSFILE_NOTICE_FILE}") + install(FILES "${TSFILE_NOTICE_FILE}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/tsfile" + COMPONENT runtime) +endif() + +include(CMakePackageConfigHelpers) +set(TSFILE_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile") +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFileConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + INSTALL_DESTINATION "${TSFILE_CMAKE_INSTALL_DIR}") +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfigVersion.cmake" + VERSION "${TSFILE_PACKAGE_VERSION}" + COMPATIBILITY SameMajorVersion) +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfigVersion.cmake" + DESTINATION "${TSFILE_CMAKE_INSTALL_DIR}" + COMPONENT development) + +set(TSFILE_PKGCONFIG_CFLAGS "") +if (NOT TSFILE_BUILD_SHARED) + string(APPEND TSFILE_PKGCONFIG_CFLAGS " -DTSFILE_STATIC") + if (ENABLE_ANTLR4 AND (TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED" OR + TSFILE_ANTLR4_SYSTEM_STATIC)) + string(APPEND TSFILE_PKGCONFIG_CFLAGS " -DANTLR4CPP_STATIC") + endif () +endif () +foreach (_TSFILE_PUBLIC_FEATURE + ENABLE_ANTLR4 + ENABLE_MEM_STAT + ENABLE_SNAPPY + ENABLE_LZ4 + ENABLE_LZOKAY + ENABLE_ZLIB + ENABLE_GZIP + ENABLE_ZSTD + ENABLE_LZMA2 + ENABLE_THREADS + ENABLE_SIMD) + if (${_TSFILE_PUBLIC_FEATURE}) + string(APPEND TSFILE_PKGCONFIG_CFLAGS + " -D${_TSFILE_PUBLIC_FEATURE}") + endif () +endforeach () +file(RELATIVE_PATH TSFILE_PKGCONFIG_PREFIX_RELATIVE + "${CMAKE_INSTALL_FULL_LIBDIR}/pkgconfig" "${CMAKE_INSTALL_PREFIX}") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libtsfile.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" + @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" + COMPONENT development) +unset(_TSFILE_PUBLIC_FEATURE) + +# CPack is an opt-in local/package-builder integration. Native release jobs +# use BUNDLED dependencies so packages do not depend on third-party shared +# libraries installed on the build host. +option(TSFILE_ENABLE_CPACK + "Enable CPack native package metadata generation" OFF) +if (TSFILE_ENABLE_CPACK) + set(CPACK_PACKAGE_NAME "tsfile") + set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Apache TsFile C++ library") + set(CPACK_PACKAGE_DESCRIPTION + "Columnar storage library and command-line tools for time series data") + set(CPACK_PACKAGE_HOMEPAGE_URL "https://tsfile.apache.org/") + set(CPACK_PACKAGE_CONTACT "dev@tsfile.apache.org") + set(CPACK_RESOURCE_FILE_LICENSE "${TSFILE_LICENSE_FILE}") + set(CPACK_PACKAGE_VERSION "${TSFILE_ARCHIVE_VERSION}") + string(REPLACE "." ";" _TSFILE_PACKAGE_VERSION_PARTS + "${TSFILE_PACKAGE_VERSION}") + list(GET _TSFILE_PACKAGE_VERSION_PARTS 0 CPACK_PACKAGE_VERSION_MAJOR) + list(GET _TSFILE_PACKAGE_VERSION_PARTS 1 CPACK_PACKAGE_VERSION_MINOR) + list(GET _TSFILE_PACKAGE_VERSION_PARTS 2 CPACK_PACKAGE_VERSION_PATCH) + + set(CPACK_COMPONENTS_ALL runtime development tools) + set(CPACK_COMPONENT_DEVELOPMENT_DEPENDS runtime) + set(CPACK_COMPONENT_TOOLS_DEPENDS runtime) + # Portable archives contain the complete runtime, SDK, and tools install + # tree in one file. Native DEB and RPM generators remain componentized. + set(CPACK_ARCHIVE_COMPONENT_INSTALL OFF) + if (WIN32) + set(CPACK_PACKAGE_FILE_NAME + "tsfile-${TSFILE_ARCHIVE_VERSION}-windows-x86_64") + endif () + set(CPACK_DEB_COMPONENT_INSTALL ON) + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) + set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) + set(CPACK_DEBIAN_PACKAGE_VERSION "${TSFILE_DEBIAN_PACKAGE_VERSION}") + set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "tsfile") + set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_NAME "tsfile-dev") + set(CPACK_DEBIAN_TOOLS_PACKAGE_NAME "tsfile-tools") + set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_DEPENDS + "tsfile (= ${TSFILE_DEBIAN_PACKAGE_VERSION})") + set(CPACK_DEBIAN_TOOLS_PACKAGE_DEPENDS + "tsfile (= ${TSFILE_DEBIAN_PACKAGE_VERSION})") + + set(CPACK_RPM_COMPONENT_INSTALL ON) + set(CPACK_RPM_FILE_NAME RPM-DEFAULT) + set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") + set(CPACK_RPM_PACKAGE_VERSION "${TSFILE_RPM_PACKAGE_VERSION}") + set(CPACK_RPM_PACKAGE_RELEASE "${TSFILE_RPM_PACKAGE_RELEASE}") + set(CPACK_RPM_PACKAGE_AUTOREQPROV ON) + set(CPACK_RPM_RUNTIME_PACKAGE_NAME "tsfile") + set(CPACK_RPM_DEVELOPMENT_PACKAGE_NAME "tsfile-devel") + set(CPACK_RPM_TOOLS_PACKAGE_NAME "tsfile-tools") + set(CPACK_RPM_DEVELOPMENT_PACKAGE_REQUIRES + "tsfile = ${TSFILE_RPM_PACKAGE_VERSION}-${TSFILE_RPM_PACKAGE_RELEASE}") + set(CPACK_RPM_TOOLS_PACKAGE_REQUIRES + "tsfile = ${TSFILE_RPM_PACKAGE_VERSION}-${TSFILE_RPM_PACKAGE_RELEASE}") + + include(CPack) + unset(_TSFILE_PACKAGE_VERSION_PARTS) +endif () diff --git a/cpp/README.md b/cpp/README.md index b1a69b0a7..8089ac1cd 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -161,12 +161,13 @@ cmake --build cpp/build/static --target tsfile The global `TSFILE_DEPENDENCY_SOURCE` CMake option defines how migrated C++ dependencies are resolved: -- `AUTO` (default): prefer a compatible system package and fall back to the +- `AUTO`: prefer a compatible system package and fall back to the verified source archive managed by the build. - `SYSTEM`: require compatible system packages and fail configuration with a clear error when one is unavailable. -- `BUNDLED`: download and build pinned dependency source archives managed by - the TsFile build. +- `BUNDLED` (default): download and build pinned dependency source archives + managed by the TsFile build. Native release packages use this mode to avoid + runtime dependencies on third-party libraries installed on the build host. ANTLR4, Snappy, LZ4, lzokay, SIMDe, zlib, Zstandard, and liblzma are currently resolved through this policy. A compatible system ANTLR4 must be version 4.9.3 diff --git a/cpp/VersionUpdater.groovy b/cpp/VersionUpdater.groovy index 49cbc6504..cde2f109b 100644 --- a/cpp/VersionUpdater.groovy +++ b/cpp/VersionUpdater.groovy @@ -19,6 +19,17 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Synchronize the version in CMakeLists.txt and the one used in the maven pom. + +// Native package jobs derive their identity from native_package_versions.py. +// Keep that identity stable by leaving source versions untouched in those jobs. +def skipVersionSync = project.properties.getProperty("tsfile.version.sync.skip") +if (skipVersionSync == null) { + skipVersionSync = System.getProperty("tsfile.version.sync.skip") +} +if (skipVersionSync != null && skipVersionSync.equalsIgnoreCase("true")) { + println "Skipping source version synchronization for native packaging" + return +} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// def cppProjectFile = new File(project.basedir, "CMakeLists.txt") diff --git a/cpp/cmake/ANTLR4Source.cmake b/cpp/cmake/ANTLR4Source.cmake index 9094ab09f..81d1a595f 100644 --- a/cpp/cmake/ANTLR4Source.cmake +++ b/cpp/cmake/ANTLR4Source.cmake @@ -17,6 +17,8 @@ specific language governing permissions and limitations under the License. ]] +include("${CMAKE_CURRENT_LIST_DIR}/DownloadWithRetry.cmake") + set(_TSFILE_ANTLR4_ARCHIVE_NAME "antlr4-${TSFILE_ANTLR4_BUNDLED_VERSION}.tar.gz") set(_TSFILE_ANTLR4_URL @@ -89,13 +91,8 @@ function(_tsfile_prepare_antlr4_archive set(_TSFILE_PARTIAL_ARCHIVE "${_TSFILE_ARCHIVE}.part") file(REMOVE "${_TSFILE_PARTIAL_ARCHIVE}") message(STATUS "Downloading ${DEPENDENCY}") - file(DOWNLOAD - "${URL}" - "${_TSFILE_PARTIAL_ARCHIVE}" - STATUS _TSFILE_DOWNLOAD_STATUS - TLS_VERIFY ON - TIMEOUT 120 - INACTIVITY_TIMEOUT 30) + tsfile_download_with_retry( + "${URL}" "${_TSFILE_PARTIAL_ARCHIVE}" _TSFILE_DOWNLOAD_STATUS) list(GET _TSFILE_DOWNLOAD_STATUS 0 _TSFILE_DOWNLOAD_RESULT) list(GET _TSFILE_DOWNLOAD_STATUS 1 _TSFILE_DOWNLOAD_MESSAGE) if (NOT _TSFILE_DOWNLOAD_RESULT EQUAL 0) diff --git a/cpp/cmake/DependencySource.cmake b/cpp/cmake/DependencySource.cmake index 5f4bc25a6..7573e41f7 100644 --- a/cpp/cmake/DependencySource.cmake +++ b/cpp/cmake/DependencySource.cmake @@ -26,7 +26,7 @@ set(TSFILE_DEPENDENCY_OFFLINE OFF CACHE BOOL "Disallow network downloads and require cached dependency archives") -set(TSFILE_DEPENDENCY_SOURCE "AUTO" +set(TSFILE_DEPENDENCY_SOURCE "BUNDLED" CACHE STRING "Dependency source policy: AUTO, SYSTEM, or BUNDLED") set_property(CACHE TSFILE_DEPENDENCY_SOURCE diff --git a/cpp/cmake/DownloadWithRetry.cmake b/cpp/cmake/DownloadWithRetry.cmake new file mode 100644 index 000000000..632e64b72 --- /dev/null +++ b/cpp/cmake/DownloadWithRetry.cmake @@ -0,0 +1,63 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +include_guard(GLOBAL) + +# Retry only transient transport/server failures. Archive identity is still +# checked by the caller before the downloaded file enters the dependency cache. +function(tsfile_download_with_retry URL DESTINATION OUTPUT_STATUS) + set(_TSFILE_TRANSIENT_CURL_CODES 6 7 18 28 35 52 55 56) + set(_TSFILE_TRANSIENT_HTTP_CODES 408 429 500 502 503 504) + foreach (_TSFILE_ATTEMPT RANGE 1 3) + file(REMOVE "${DESTINATION}") + file(DOWNLOAD "${URL}" "${DESTINATION}" + STATUS _TSFILE_STATUS + LOG _TSFILE_LOG + TLS_VERIFY ON + TIMEOUT 120 + INACTIVITY_TIMEOUT 30) + list(GET _TSFILE_STATUS 0 _TSFILE_RESULT) + if (_TSFILE_RESULT EQUAL 0) + break() + endif () + file(REMOVE "${DESTINATION}") + list(FIND _TSFILE_TRANSIENT_CURL_CODES "${_TSFILE_RESULT}" + _TSFILE_TRANSIENT_INDEX) + if (_TSFILE_RESULT EQUAL 22) + # Use the final HTTP response after redirects/proxy negotiation. + string(REGEX MATCHALL "HTTP/[0-9.]+ [0-9][0-9][0-9]" + _TSFILE_RESPONSES "${_TSFILE_LOG}") + if (_TSFILE_RESPONSES) + list(GET _TSFILE_RESPONSES -1 _TSFILE_RESPONSE) + string(REGEX REPLACE "^HTTP/[0-9.]+ " "" + _TSFILE_HTTP_CODE "${_TSFILE_RESPONSE}") + list(FIND _TSFILE_TRANSIENT_HTTP_CODES "${_TSFILE_HTTP_CODE}" + _TSFILE_TRANSIENT_INDEX) + endif () + endif () + if (_TSFILE_TRANSIENT_INDEX EQUAL -1 OR _TSFILE_ATTEMPT EQUAL 3) + break() + endif () + list(GET _TSFILE_STATUS 1 _TSFILE_MESSAGE) + message(STATUS "Dependency download attempt ${_TSFILE_ATTEMPT}/3 failed: " + "${_TSFILE_MESSAGE}; retrying ${URL}") + execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep "${_TSFILE_ATTEMPT}") + endforeach () + set(${OUTPUT_STATUS} "${_TSFILE_STATUS}" PARENT_SCOPE) +endfunction() diff --git a/cpp/cmake/TsFileBuildConfig.cmake b/cpp/cmake/TsFileBuildConfig.cmake new file mode 100644 index 000000000..bbd215935 --- /dev/null +++ b/cpp/cmake/TsFileBuildConfig.cmake @@ -0,0 +1,45 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Record the resolved runtime on both the build tree and the installed SDK. +# file(GENERATE) evaluates Debug suffixes for each configuration separately. +set(_TSFILE_MSVC_RUNTIME "") +if (MSVC) + get_target_property(_TSFILE_MSVC_RUNTIME tsfile MSVC_RUNTIME_LIBRARY) + if (NOT _TSFILE_MSVC_RUNTIME) + set(_TSFILE_MSVC_RUNTIME "MultiThreaded$<$:Debug>DLL") + endif () +endif () +set(_TSFILE_RUNTIME_LIBRARY "") +if (TSFILE_BUILD_SHARED) + if (WIN32) + set(_TSFILE_RUNTIME_LIBRARY "$") + else () + set(_TSFILE_RUNTIME_LIBRARY "$") + endif () +endif () +set(_TSFILE_BUILD_CONFIG "$/tsfile-build-config.json") +file(GENERATE OUTPUT "${_TSFILE_BUILD_CONFIG}" CONTENT +"{\n \"schema_version\": 1,\n \"version\": \"${TSFILE_PACKAGE_VERSION}\",\n \"compiler\": \"${CMAKE_CXX_COMPILER_ID}\",\n \"configuration\": \"$\",\n \"msvc_runtime\": \"${_TSFILE_MSVC_RUNTIME}\",\n \"runtime_library\": \"${_TSFILE_RUNTIME_LIBRARY}\"\n}\n") +install(FILES "${_TSFILE_BUILD_CONFIG}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/tsfile" + COMPONENT development) +unset(_TSFILE_BUILD_CONFIG) +unset(_TSFILE_RUNTIME_LIBRARY) +unset(_TSFILE_MSVC_RUNTIME) diff --git a/cpp/cmake/TsFileConfig.cmake.in b/cpp/cmake/TsFileConfig.cmake.in new file mode 100644 index 000000000..1357f4898 --- /dev/null +++ b/cpp/cmake/TsFileConfig.cmake.in @@ -0,0 +1,43 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +if (@ENABLE_THREADS@) + find_dependency(Threads) +endif() + +if (NOT @TSFILE_BUILD_SHARED@) + set(_TSFILE_SAVED_MODULE_PATH "${CMAKE_MODULE_PATH}") + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + file(GLOB _TSFILE_STATIC_CONFIGS + "${CMAKE_CURRENT_LIST_DIR}/TsFileStaticDependencies-*.cmake") + foreach (_TSFILE_STATIC_CONFIG IN LISTS _TSFILE_STATIC_CONFIGS) + include("${_TSFILE_STATIC_CONFIG}") + endforeach () + set(CMAKE_MODULE_PATH "${_TSFILE_SAVED_MODULE_PATH}") + unset(_TSFILE_STATIC_CONFIG) + unset(_TSFILE_STATIC_CONFIGS) + unset(_TSFILE_SAVED_MODULE_PATH) +endif () + +include("${CMAKE_CURRENT_LIST_DIR}/TsFileTargets.cmake") + +check_required_components(TsFile) diff --git a/cpp/cmake/TsFilePublicHeaders.cmake b/cpp/cmake/TsFilePublicHeaders.cmake new file mode 100644 index 000000000..a517d3a9c --- /dev/null +++ b/cpp/cmake/TsFilePublicHeaders.cmake @@ -0,0 +1,47 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Public entry points are intentionally listed separately from the transitive +# header closure. The current public headers include implementation-level +# declarations, so the closure is installed until those dependencies can be +# hidden behind a stable facade. Adding a new public entry point requires an +# explicit review of this list. +set(TSFILE_PUBLIC_ENTRYPOINT_HEADERS + cwrapper/tsfile_cwrapper.h + cwrapper/tsfile_cwrapper_expression.h + reader/tsfile_reader.h + writer/tsfile_writer.h + writer/tsfile_table_writer.h + writer/tsfile_tree_writer.h) + +# These directories form the reviewed transitive closure of the entry points +# above. This is an installation compatibility closure, not a promise that +# every header is a stable public API. Compression, encoding, parser, and +# utility headers are implementation dependencies today, but are required for +# consumers to parse the installed public declarations. +set(TSFILE_PUBLIC_HEADER_CLOSURE + common + compress + cwrapper + encoding + file + parser + reader + utils + writer) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake b/cpp/cmake/TsFileStaticDependencies.cmake new file mode 100644 index 000000000..8233487f3 --- /dev/null +++ b/cpp/cmake/TsFileStaticDependencies.cmake @@ -0,0 +1,128 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Static archives do not absorb their link dependencies. Keep the private +# build wrappers out of the export: install bundled archives and describe +# imported link-only targets, or rediscover system packages on the consumer. +# install(FILES) also works on CMake 3.11 for targets in another directory. +function(tsfile_install_static_dependency NAME SOURCE SYSTEM_TARGET FIND_CODE) + if (NOT TARGET TsFile::${NAME}) + return() + endif () + target_link_libraries(tsfile INTERFACE + $>) + set(_CODE "${TSFILE_STATIC_DEPENDENCY_CODE}") + if (SOURCE STREQUAL "BUNDLED") + get_target_property(_TARGET TsFile::${NAME} INTERFACE_LINK_LIBRARIES) + install(FILES $ + DESTINATION "${CMAKE_INSTALL_LIBDIR}/tsfile/$" + COMPONENT development) + string(APPEND _CODE + "if (NOT TARGET TsFile::Static_${NAME})\n" + " add_library(TsFile::Static_${NAME} STATIC IMPORTED)\n" + "endif ()\n" + "set_property(TARGET TsFile::Static_${NAME} APPEND PROPERTY IMPORTED_CONFIGURATIONS $>)\n" + "set_target_properties(TsFile::Static_${NAME} PROPERTIES IMPORTED_LOCATION_$> \"\${CMAKE_CURRENT_LIST_DIR}/../../tsfile/$/$\")\n") + else () + string(APPEND _CODE "${FIND_CODE}\n" + "if (NOT TARGET TsFile::Static_${NAME})\n" + " add_library(TsFile::Static_${NAME} INTERFACE IMPORTED)\n" + " set_target_properties(TsFile::Static_${NAME} PROPERTIES INTERFACE_LINK_LIBRARIES ${SYSTEM_TARGET})\n" + "endif ()\n") + endif () + set(TSFILE_STATIC_DEPENDENCY_CODE "${_CODE}" PARENT_SCOPE) +endfunction() + +set(TSFILE_STATIC_DEPENDENCY_CODE "") +# Keep the build-side compatibility interval when a static consumer rediscovers +# ANTLR4. Explicit comparisons also support CMake 3.11 and packages that report +# ANTLR_VERSION instead of antlr4-runtime_VERSION. +string(CONFIGURE [=[ +find_package(utf8cpp CONFIG QUIET) +find_package(antlr4-runtime CONFIG QUIET) +set(_TSFILE_ANTLR4_VERSION "${antlr4-runtime_VERSION}") +if ("${_TSFILE_ANTLR4_VERSION}" STREQUAL "" AND DEFINED ANTLR_VERSION) + set(_TSFILE_ANTLR4_VERSION "${ANTLR_VERSION}") +endif () +if (NOT antlr4-runtime_FOUND OR + "${_TSFILE_ANTLR4_VERSION}" STREQUAL "" OR + _TSFILE_ANTLR4_VERSION VERSION_LESS "@TSFILE_ANTLR4_MIN_VERSION@" OR + NOT _TSFILE_ANTLR4_VERSION VERSION_LESS "@TSFILE_ANTLR4_NEXT_INCOMPATIBLE_VERSION@" OR + NOT TARGET @TSFILE_ANTLR4_SYSTEM_TARGET@) + set(TsFile_FOUND FALSE) + set(TsFile_NOT_FOUND_MESSAGE + "Static TsFile requires a compatible system ANTLR4 >=@TSFILE_ANTLR4_MIN_VERSION@ and <@TSFILE_ANTLR4_NEXT_INCOMPATIBLE_VERSION@ with target @TSFILE_ANTLR4_SYSTEM_TARGET@; reported version '${_TSFILE_ANTLR4_VERSION}'.") + unset(_TSFILE_ANTLR4_VERSION) + return() +endif () +unset(_TSFILE_ANTLR4_VERSION) +]=] _TSFILE_FIND_STATIC_ANTLR4 @ONLY) +tsfile_install_static_dependency(ANTLR4 "${TSFILE_ANTLR4_SOURCE}" + "${TSFILE_ANTLR4_SYSTEM_TARGET}" + "${_TSFILE_FIND_STATIC_ANTLR4}") +unset(_TSFILE_FIND_STATIC_ANTLR4) +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED") + if (WIN32) + string(APPEND TSFILE_STATIC_DEPENDENCY_CODE + "set_property(TARGET TsFile::Static_ANTLR4 PROPERTY INTERFACE_LINK_LIBRARIES ole32)\n") + else () + if (APPLE) + set(_PLATFORM_LIBRARY CoreFoundation) + else () + set(_PLATFORM_LIBRARY uuid) + endif () + string(APPEND TSFILE_STATIC_DEPENDENCY_CODE + "find_library(_TSFILE_ANTLR4_PLATFORM_LIBRARY ${_PLATFORM_LIBRARY})\n" + "if (NOT _TSFILE_ANTLR4_PLATFORM_LIBRARY)\n" + " set(TsFile_FOUND FALSE)\n" + " set(TsFile_NOT_FOUND_MESSAGE \"Static TsFile needs ${_PLATFORM_LIBRARY}\")\n" + " return()\n" + "endif ()\n" + "set_property(TARGET TsFile::Static_ANTLR4 PROPERTY INTERFACE_LINK_LIBRARIES \"\${_TSFILE_ANTLR4_PLATFORM_LIBRARY}\")\n") + unset(_PLATFORM_LIBRARY) + endif () +endif () +tsfile_install_static_dependency(Snappy "${TSFILE_SNAPPY_SOURCE}" Snappy::snappy + "find_dependency(Snappy ${TSFILE_SNAPPY_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(LZ4 "${TSFILE_LZ4_SOURCE}" LZ4::LZ4 + "find_dependency(LZ4 ${TSFILE_LZ4_MIN_VERSION})") +if (ENABLE_LZ4 AND TSFILE_LZ4_SOURCE STREQUAL "SYSTEM") + install(FILES "${CMAKE_SOURCE_DIR}/cmake/FindLZ4.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile" + COMPONENT development) +endif () +tsfile_install_static_dependency(LZOKAY "${TSFILE_LZOKAY_SOURCE}" lzokay::lzokay + "find_dependency(lzokay ${TSFILE_LZOKAY_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(ZLIB "${TSFILE_ZLIB_SOURCE}" ZLIB::ZLIB + "find_dependency(ZLIB ${TSFILE_ZLIB_MIN_VERSION})") +tsfile_install_static_dependency(ZSTD "${TSFILE_ZSTD_SOURCE}" + "${TSFILE_ZSTD_SYSTEM_TARGET}" + "find_dependency(zstd ${TSFILE_ZSTD_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(LibLZMA "${TSFILE_LIBLZMA_SOURCE}" LibLZMA::LibLZMA + "find_dependency(LibLZMA ${TSFILE_LIBLZMA_MIN_VERSION})\nif (NOT TARGET LibLZMA::LibLZMA)\n add_library(LibLZMA::LibLZMA INTERFACE IMPORTED)\n set_target_properties(LibLZMA::LibLZMA PROPERTIES INTERFACE_LINK_LIBRARIES \"\${LIBLZMA_LIBRARIES}\")\nendif ()") + +configure_file("${CMAKE_SOURCE_DIR}/cmake/TsFileStaticDependencies.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies.cmake.in" @ONLY) +file(GENERATE + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies-$.cmake" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies.cmake.in") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies-$.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile" + COMPONENT development) +unset(TSFILE_STATIC_DEPENDENCY_CODE) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake.in b/cpp/cmake/TsFileStaticDependencies.cmake.in new file mode 100644 index 000000000..877bc21df --- /dev/null +++ b/cpp/cmake/TsFileStaticDependencies.cmake.in @@ -0,0 +1,20 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +@TSFILE_STATIC_DEPENDENCY_CODE@ diff --git a/cpp/cmake/libtsfile.pc.in b/cpp/cmake/libtsfile.pc.in new file mode 100644 index 000000000..f7b3db9bf --- /dev/null +++ b/cpp/cmake/libtsfile.pc.in @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The .pc file is installed below ${prefix}/@CMAKE_INSTALL_LIBDIR@/pkgconfig. +# Deriving the prefix from its own location keeps DESTDIR and relocations +# working without embedding the build-time install prefix. +prefix=${pcfiledir}/@TSFILE_PKGCONFIG_PREFIX_RELATIVE@ +exec_prefix=${prefix} +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libtsfile +Description: Apache TsFile C++ library +Version: @TSFILE_PACKAGE_VERSION@ +URL: https://tsfile.apache.org/ +Libs: -L${libdir} -ltsfile +Cflags: -I${includedir} -I${includedir}/tsfile@TSFILE_PKGCONFIG_CFLAGS@ diff --git a/cpp/cmake/tests/ANTLR4DependencyTest.cmake b/cpp/cmake/tests/ANTLR4DependencyTest.cmake index f3594929b..b30df26b7 100644 --- a/cpp/cmake/tests/ANTLR4DependencyTest.cmake +++ b/cpp/cmake/tests/ANTLR4DependencyTest.cmake @@ -22,6 +22,14 @@ if (NOT TEST_BINARY_ROOT) endif () include("${CMAKE_CURRENT_LIST_DIR}/TestGeneratorArguments.cmake") +# Package lookup is intentionally isolated below. Keep compiler discovery out +# of that isolation by reusing the compiler selected by the outer build. +set(_TSFILE_ANTLR4_COMPILER_ARGUMENTS "") +if (TEST_CMAKE_CXX_COMPILER) + list(APPEND _TSFILE_ANTLR4_COMPILER_ARGUMENTS + "-DCMAKE_CXX_COMPILER=${TEST_CMAKE_CXX_COMPILER}") +endif () + set(_TSFILE_FIXTURE_SOURCE "${CMAKE_CURRENT_LIST_DIR}/projects/ANTLR4Dependency") set(_TSFILE_TEST_ROOT "${TEST_BINARY_ROOT}/antlr4-dependency") @@ -54,6 +62,7 @@ function(_tsfile_run_antlr4_case NAME POLICY EXPECTED_SOURCE EXPECT_SUCCESS ROOT execute_process( COMMAND "${CMAKE_COMMAND}" ${_TSFILE_TEST_GENERATOR_ARGUMENTS} + ${_TSFILE_ANTLR4_COMPILER_ARGUMENTS} "-DTSFILE_DEPENDENCY_SOURCE=${POLICY}" "-DEXPECTED_ANTLR4_SOURCE=${EXPECTED_SOURCE}" "-DCMAKE_PREFIX_PATH=${ROOT}" diff --git a/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake b/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake new file mode 100644 index 000000000..a30ddfcab --- /dev/null +++ b/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake @@ -0,0 +1,38 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# A configure-time contract test also runnable on non-MSVC hosts. Runtime +# properties initialize on every platform; Windows CI checks actual PE imports. +function(tsfile_check_static_runtime DIRECTORY) + get_property(_TARGETS DIRECTORY "${DIRECTORY}" PROPERTY BUILDSYSTEM_TARGETS) + foreach (_TARGET IN LISTS _TARGETS) + get_target_property(_TYPE ${_TARGET} TYPE) + if (_TYPE MATCHES "^(STATIC_LIBRARY|SHARED_LIBRARY|OBJECT_LIBRARY|EXECUTABLE)$") + get_target_property(_RUNTIME ${_TARGET} MSVC_RUNTIME_LIBRARY) + if (NOT _RUNTIME STREQUAL "MultiThreaded$<$:Debug>") + message(FATAL_ERROR "${_TARGET} has inconsistent MSVC runtime: ${_RUNTIME}") + endif () + endif () + endforeach () + get_property(_SUBDIRECTORIES DIRECTORY "${DIRECTORY}" PROPERTY SUBDIRECTORIES) + foreach (_SUBDIRECTORY IN LISTS _SUBDIRECTORIES) + tsfile_check_static_runtime("${_SUBDIRECTORY}") + endforeach () +endfunction() +cmake_language(DEFER CALL tsfile_check_static_runtime "${CMAKE_SOURCE_DIR}") diff --git a/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt b/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt index 61901e38a..551373afe 100644 --- a/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt +++ b/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt @@ -18,7 +18,7 @@ under the License. ]] cmake_minimum_required(VERSION 3.11) -project(TsFileANTLR4DependencyTest NONE) +project(TsFileANTLR4DependencyTest LANGUAGES CXX) get_filename_component(_TSFILE_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt new file mode 100644 index 000000000..40ea9481d --- /dev/null +++ b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt @@ -0,0 +1,45 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +cmake_minimum_required(VERSION 3.11) +if (POLICY CMP0091) + cmake_policy(SET CMP0091 NEW) +endif () +project(TsFileInstalledConsumer LANGUAGES CXX) + +find_package(TsFile CONFIG REQUIRED) + +get_target_property(_TSFILE_INCLUDE_DIRS TsFile::tsfile + INTERFACE_INCLUDE_DIRECTORIES) +if (NOT _TSFILE_INCLUDE_DIRS MATCHES "/include") + message(FATAL_ERROR "TsFile target does not expose the installed include root") +endif () +if (_TSFILE_INCLUDE_DIRS MATCHES "cpp/src|build/") + message(FATAL_ERROR "TsFile target leaks a source or build include path: ${_TSFILE_INCLUDE_DIRS}") +endif () + +get_target_property(_TSFILE_LINK_LIBRARIES TsFile::tsfile + INTERFACE_LINK_LIBRARIES) +if (_TSFILE_LINK_LIBRARIES MATCHES "(^|;)[^;]*_obj($|;)|cpp/src|build/") + message(FATAL_ERROR + "TsFile target leaks private build link requirements: " + "${_TSFILE_LINK_LIBRARIES}") +endif () + +add_executable(tsfile_installed_consumer main.cc) +target_link_libraries(tsfile_installed_consumer PRIVATE TsFile::tsfile) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/main.cc b/cpp/cmake/tests/projects/InstalledConsumer/main.cc new file mode 100644 index 000000000..2d265d429 --- /dev/null +++ b/cpp/cmake/tests/projects/InstalledConsumer/main.cc @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include + +int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt new file mode 100644 index 000000000..1aaabc396 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt @@ -0,0 +1,32 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +cmake_minimum_required(VERSION 3.11) +project(TsFilePkgConfigConsumer LANGUAGES C CXX) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(TsFile REQUIRED IMPORTED_TARGET libtsfile) +if (NOT TsFile_INCLUDEDIR MATCHES "/include$") + message(FATAL_ERROR "pkg-config did not report the installed include root: ${TsFile_INCLUDEDIR}") +endif () + +add_executable(tsfile_pkgconfig_consumer main.c) +target_link_libraries(tsfile_pkgconfig_consumer PRIVATE PkgConfig::TsFile) + +add_executable(tsfile_pkgconfig_cpp_consumer main.cc) +target_link_libraries(tsfile_pkgconfig_cpp_consumer PRIVATE PkgConfig::TsFile) diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c new file mode 100644 index 000000000..05a1566f7 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include + +int main(void) { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc new file mode 100644 index 000000000..daa807236 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include + +int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/pom.xml b/cpp/pom.xml index b3cbcb7f3..12e6fe337 100644 --- a/cpp/pom.xml +++ b/cpp/pom.xml @@ -36,7 +36,7 @@ OFF ON ON - AUTO + BUNDLED OFF ${project.build.directory}/dependency-cache ON @@ -52,6 +52,24 @@ OFF ON ON + + OFF + + OFF + + ${project.basedir} @@ -105,6 +123,9 @@ + + + + + native-package-versions + + + tsfile.archive.version + + + + + + com.googlecode.cmake-maven-project + cmake-maven-plugin + + + cmake-generate-test-compile + + + + + + + + + + + + + + with-code-coverage diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 2ab2dd5d6..96c7d9f47 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -79,6 +79,18 @@ add_subdirectory(reader) add_subdirectory(utils) add_subdirectory(writer) +# Object libraries need the directory-level project dependencies on older +# CMake versions, but those dependencies are implementation details of the +# shared library and must not become exported consumer requirements. Keep the +# platform thread target in the directory scope for its public link interface, +# then link codecs explicitly as PRIVATE below. +get_property(_TSFILE_SRC_LINK_LIBRARIES DIRECTORY PROPERTY LINK_LIBRARIES) +if (ENABLE_THREADS) + set_property(DIRECTORY PROPERTY LINK_LIBRARIES Threads::Threads) +else () + set_property(DIRECTORY PROPERTY LINK_LIBRARIES "") +endif () + set(_TSFILE_OBJECT_TARGETS common_obj compress_obj @@ -114,6 +126,10 @@ else() # Consumers of the CMake target must see TSFILE_API without DLL import # decoration when linking the static library on MSVC. target_compile_definitions(tsfile INTERFACE TSFILE_STATIC) + if (ENABLE_ANTLR4 AND (TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED" OR + TSFILE_ANTLR4_SYSTEM_STATIC)) + target_compile_definitions(tsfile INTERFACE ANTLR4CPP_STATIC) + endif () endif() if (${COV_ENABLED}) @@ -125,35 +141,118 @@ if (${COV_ENABLED}) else() set(COV_LINK_LIB -lgcov) endif() - if (CMAKE_VERSION VERSION_LESS "3.12") - target_link_libraries(tsfile ${COV_LINK_LIB}) - else() - target_link_libraries(tsfile ${_TSFILE_OBJECT_TARGETS} ${COV_LINK_LIB}) - endif() + target_link_libraries(tsfile PRIVATE ${COV_LINK_LIB}) else() message("Disable code cov...") - if (NOT CMAKE_VERSION VERSION_LESS "3.12") - target_link_libraries(tsfile ${_TSFILE_OBJECT_TARGETS}) - endif() endif() +# Linking object libraries directly makes static-library exports depend on +# private, non-exported object targets. Add their object files as sources +# instead; this works on the CMake 3.11 baseline and keeps the install target +# self-contained for both shared and static builds. +foreach (_TSFILE_OBJECT_TARGET IN LISTS _TSFILE_OBJECT_TARGETS) + target_sources(tsfile PRIVATE + $) +endforeach () + +foreach (_TSFILE_DEPENDENCY IN LISTS _TSFILE_PROJECT_DEPENDENCIES) + target_link_libraries(tsfile PRIVATE $) +endforeach () +unset(_TSFILE_DEPENDENCY) + unset(_TSFILE_OBJECT_SOURCES) unset(_TSFILE_OBJECT_TARGET) unset(_TSFILE_OBJECT_TARGETS) add_dependencies(tsfile utils_obj encoding_obj) +set_property(DIRECTORY PROPERTY LINK_LIBRARIES + "${_TSFILE_SRC_LINK_LIBRARIES}") +unset(_TSFILE_SRC_LINK_LIBRARIES) + if (TSFILE_BUILD_SHARED) - set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION}) - set(LIBTSFILE_SO_VERSION ${TsFile_CPP_VERSION}) + # Keep development suffixes out of the install name and SONAME. The ABI + # epoch changes only for incompatible binary interface changes. + set(LIBTSFILE_PROJECT_VERSION ${TSFILE_PACKAGE_VERSION}) + set(LIBTSFILE_SO_VERSION ${TSFILE_ABI_VERSION}) set_target_properties(tsfile PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION}) set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION}) endif() +add_library(TsFile::tsfile ALIAS tsfile) + +target_include_directories(tsfile PUBLIC + $ + $ + $ + $) +if (ENABLE_SIMD AND TSFILE_SIMDE_INCLUDE_ROOT) + # Public TsFile headers include SIMDe directly. Copy the resolved header + # tree into the install prefix so consumers do not need the build host's + # system or dependency-cache paths. + target_include_directories(tsfile PUBLIC + $ + $) +endif () +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_INCLUDE_ROOT) + target_include_directories(tsfile PUBLIC + $ + $) +endif () +if (ENABLE_ANTLR4 AND TSFILE_UTF8CPP_INCLUDE_ROOT) + target_include_directories(tsfile PUBLIC + $ + $) +endif () +target_compile_features(tsfile PUBLIC cxx_std_11) +set(_TSFILE_PUBLIC_FEATURE_DEFINITIONS "") +foreach (_TSFILE_PUBLIC_FEATURE + ENABLE_ANTLR4 + ENABLE_MEM_STAT + ENABLE_SNAPPY + ENABLE_LZ4 + ENABLE_LZOKAY + ENABLE_ZLIB + ENABLE_GZIP + ENABLE_ZSTD + ENABLE_LZMA2 + ENABLE_THREADS + ENABLE_SIMD) + if (${_TSFILE_PUBLIC_FEATURE}) + list(APPEND _TSFILE_PUBLIC_FEATURE_DEFINITIONS + ${_TSFILE_PUBLIC_FEATURE}) + endif () +endforeach () +if (NOT "${_TSFILE_PUBLIC_FEATURE_DEFINITIONS}" STREQUAL "") + target_compile_definitions(tsfile INTERFACE + ${_TSFILE_PUBLIC_FEATURE_DEFINITIONS}) +endif () +unset(_TSFILE_PUBLIC_FEATURE) +unset(_TSFILE_PUBLIC_FEATURE_DEFINITIONS) + +if (TSFILE_BUILD_SHARED) + # A shared library already resolves its private codec and object-library + # dependencies at link time. Do not export those build-only targets as + # consumer link requirements; they are not installed with the package. + set_property(TARGET tsfile PROPERTY INTERFACE_LINK_LIBRARIES "") +else () + include(${CMAKE_SOURCE_DIR}/cmake/TsFileStaticDependencies.cmake) +endif () + # A shared library is a RUNTIME plus an import ARCHIVE on Windows and a LIBRARY # on Unix. A static library is an ARCHIVE on every platform. Cover all three so # the install step works for either library type. install(TARGETS tsfile - RUNTIME DESTINATION ${LIBRARY_OUTPUT_PATH} - LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH} - ARCHIVE DESTINATION ${LIBRARY_OUTPUT_PATH}) + EXPORT TsFileTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT runtime + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT runtime + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT development) + +install(EXPORT TsFileTargets + FILE TsFileTargets.cmake + NAMESPACE TsFile:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/TsFile + COMPONENT development) diff --git a/cpp/src/utils/util_define.h b/cpp/src/utils/util_define.h index 564581a11..33784e152 100644 --- a/cpp/src/utils/util_define.h +++ b/cpp/src/utils/util_define.h @@ -120,14 +120,14 @@ typedef int mode_t; #endif // __GNUC__ >= 4 /* ======== nullptr ======== */ -#if __cplusplus < 201103L +#if __cplusplus < 201103L && !defined(_MSC_VER) #ifndef nullptr #define nullptr NULL #endif #define OVERRIDE #else #define OVERRIDE override -#endif // __cplusplus < 201103L +#endif // __cplusplus < 201103L && !defined(_MSC_VER) /* ======== cache line ======== */ #ifndef CACHE_LINE_SIZE @@ -147,13 +147,13 @@ typedef int mode_t; * @msg should be a single word (use -/_ to concat) * such as This_should_be_TRUE */ -#if __cplusplus < 201103L +#if __cplusplus < 201103L && !defined(_MSC_VER) // TODO only define this when DEBUG #define STATIC_ASSERT(cond, msg) \ typedef char static_assertion_##msg[(cond) ? 1 : -1] __attribute__((unused)) #else #define STATIC_ASSERT(cond, msg) static_assert((cond), #msg) -#endif // __cplusplus < 201103L +#endif // __cplusplus < 201103L && !defined(_MSC_VER) /* ======== atomic operation ======== * diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 3c3da441b..fa160bc61 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -19,6 +19,13 @@ under the License. cmake_minimum_required(VERSION 3.11) project(TsFile_CPP_TEST) +# Configuring cpp/test directly validates an already installed SDK. The normal +# add_subdirectory(test) path continues to test the source-built library. +if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + include("${CMAKE_CURRENT_SOURCE_DIR}/InstalledSDKTests.cmake") + return() +endif () + set(CMAKE_VERBOSE_MAKEFILE ON) set(GTEST_URL_LIST @@ -215,6 +222,7 @@ add_test( NAME ANTLR4DependencyCMakeTest COMMAND ${CMAKE_COMMAND} ${_TSFILE_DEPENDENCY_TEST_CMAKE_ARGUMENTS} + "-DTEST_CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" -DTEST_BINARY_ROOT=${CMAKE_CURRENT_BINARY_DIR}/cmake-tests -DTEST_ANTLR4_ARCHIVE=${TSFILE_ANTLR4_ARCHIVE_RESOLVED} -P ${CMAKE_SOURCE_DIR}/cmake/tests/ANTLR4DependencyTest.cmake) diff --git a/cpp/test/InstalledSDKTests.cmake b/cpp/test/InstalledSDKTests.cmake new file mode 100644 index 000000000..facd3ae7e --- /dev/null +++ b/cpp/test/InstalledSDKTests.cmake @@ -0,0 +1,63 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +# This entry point compiles test sources only. All TsFile headers, feature +# definitions and libraries must come from the installed CMake package. +find_package(TsFile CONFIG REQUIRED) +find_package(GTest CONFIG REQUIRED) +find_package(Threads REQUIRED) + +enable_testing() + +# Reuse existing API regression tests without linking CLI implementation objects, +# codec implementation targets, or a library compiled with ENABLE_TEST hooks. +add_executable(TsFile_Installed_Test + cwrapper/cwrapper_test.cc + cwrapper/cwrapper_public_writer_test.cc + cwrapper/cwrapper_metadata_test.cc + cwrapper/cwrapper_properties_test.cc + cwrapper/c_release_test.cc + cwrapper/query_by_row_cwrapper_test.cc + reader/table_view/tsfile_reader_table_test.cc + reader/table_view/tsfile_reader_table_batch_test.cc + reader/table_view/tsfile_table_query_by_row_test.cc + reader/tree_view/tsfile_reader_tree_test.cc + reader/tree_view/tsfile_tree_query_by_row_test.cc + writer/table_view/tsfile_writer_table_test.cc + writer/table_view/tsfile_parallel_tablet_write_regression_test.cc + writer/tsfile_writer_test.cc + writer/tsfile_properties_test.cc) +target_compile_features(TsFile_Installed_Test PRIVATE cxx_std_14) +target_link_libraries(TsFile_Installed_Test PRIVATE + TsFile::tsfile GTest::gtest_main Threads::Threads) + +# Windows resolves DLLs beside the executable before searching PATH. Copy the +# imported SDK library, rather than relying on a build machine's library path. +get_target_property(_TSFILE_INSTALLED_LIBRARY_TYPE TsFile::tsfile TYPE) +if (WIN32 AND _TSFILE_INSTALLED_LIBRARY_TYPE STREQUAL "SHARED_LIBRARY") + add_custom_command(TARGET TsFile_Installed_Test POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + VERBATIM) +endif () +unset(_TSFILE_INSTALLED_LIBRARY_TYPE) + +include(GoogleTest) +gtest_discover_tests(TsFile_Installed_Test DISCOVERY_TIMEOUT 120 + PROPERTIES TIMEOUT 120) diff --git a/cpp/third_party/CMakeLists.txt b/cpp/third_party/CMakeLists.txt index f7a8851b5..203b93c34 100755 --- a/cpp/third_party/CMakeLists.txt +++ b/cpp/third_party/CMakeLists.txt @@ -35,6 +35,12 @@ if (ENABLE_ANTLR4) "${TSFILE_UTF8CPP_SOURCE_DIR}/source") add_library(tsfile_antlr4_bundled STATIC ${_TSFILE_ANTLR4_SOURCES}) + set(TSFILE_ANTLR4_INCLUDE_ROOT + "${TSFILE_ANTLR4_SOURCE_DIR}/runtime/Cpp/runtime/src" + CACHE INTERNAL "Resolved ANTLR4 include root" FORCE) + set(TSFILE_UTF8CPP_INCLUDE_ROOT + "${TSFILE_UTF8CPP_SOURCE_DIR}/source" + CACHE INTERNAL "Resolved utf8cpp include root" FORCE) set_target_properties(tsfile_antlr4_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_STANDARD 11) @@ -93,6 +99,17 @@ if (ENABLE_ANTLR4) target_link_libraries(tsfile_antlr4 INTERFACE tsfile_antlr4_bundled) else () + set(TSFILE_ANTLR4_INCLUDE_ROOT + "${TSFILE_ANTLR4_SYSTEM_INCLUDE_DIR}" + CACHE INTERNAL "Resolved ANTLR4 include root" FORCE) + if (TARGET utf8cpp) + get_target_property(_TSFILE_UTF8CPP_INCLUDE_ROOT utf8cpp + INTERFACE_INCLUDE_DIRECTORIES) + set(TSFILE_UTF8CPP_INCLUDE_ROOT + "${_TSFILE_UTF8CPP_INCLUDE_ROOT}" + CACHE INTERNAL "Resolved utf8cpp include root" FORCE) + unset(_TSFILE_UTF8CPP_INCLUDE_ROOT) + endif () target_link_libraries(tsfile_antlr4 INTERFACE ${TSFILE_ANTLR4_SYSTEM_TARGET}) if (TSFILE_ANTLR4_SYSTEM_INCLUDE_DIR) @@ -449,9 +466,15 @@ if (ENABLE_SIMD) add_library(tsfile_simde INTERFACE) if (TSFILE_SIMDE_SOURCE STREQUAL "BUNDLED") + set(TSFILE_SIMDE_INCLUDE_ROOT "${TSFILE_SIMDE_SOURCE_DIR}" + CACHE INTERNAL "Resolved SIMDe include root" FORCE) target_include_directories(tsfile_simde INTERFACE "${TSFILE_SIMDE_SOURCE_DIR}") else () + get_target_property(TSFILE_SIMDE_INCLUDE_ROOT simde::simde + INTERFACE_INCLUDE_DIRECTORIES) + set(TSFILE_SIMDE_INCLUDE_ROOT "${TSFILE_SIMDE_INCLUDE_ROOT}" + CACHE INTERNAL "Resolved SIMDe include root" FORCE) target_link_libraries(tsfile_simde INTERFACE simde::simde) endif () add_library(TsFile::SIMDe ALIAS tsfile_simde) diff --git a/cpp/third_party/README.md b/cpp/third_party/README.md index 74ae8c65e..f1bf7fd44 100644 --- a/cpp/third_party/README.md +++ b/cpp/third_party/README.md @@ -31,7 +31,9 @@ must be updated together. The build provides explicit `SYSTEM`, `BUNDLED`, and `AUTO` dependency source modes. ANTLR4, Snappy, LZ4, lzokay, SIMDe, zlib, Zstandard, and liblzma have migrated to these modes. In `BUNDLED` mode, source is downloaded into the build -cache rather than kept in this repository. +cache rather than kept in this repository. `BUNDLED` is the default for both +CMake and Maven and is used by every native release package, including Homebrew. +`AUTO` and `SYSTEM` remain explicit options for custom source builds. LZMA2 support is disabled by default so this optional codec does not raise the CMake 3.11 baseline for long-lived industrial and legacy toolchains. Set diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index 00bc20ea0..b78d06442 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -59,4 +59,19 @@ set_target_properties(tsfile_cli PROPERTIES OUTPUT_NAME tsfile-cli RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin) -install(TARGETS tsfile_cli RUNTIME DESTINATION bin) +# Keep the installed CLI self-contained with the sibling lib directory. This +# is relative to the executable location and therefore remains valid when the +# installation prefix is relocated. +if (APPLE) + set_target_properties(tsfile_cli PROPERTIES + INSTALL_RPATH "@loader_path/../${CMAKE_INSTALL_LIBDIR}" + BUILD_WITH_INSTALL_RPATH TRUE) +elseif (UNIX) + set_target_properties(tsfile_cli PROPERTIES + INSTALL_RPATH "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" + BUILD_WITH_INSTALL_RPATH TRUE) +endif () + +install(TARGETS tsfile_cli + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT tools) diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 000000000..f6282d491 --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,428 @@ + + +# TsFile C++ Packages + +The C++ installation contract is shared by all package formats. It installs +the library, the current compatibility header closure, `TsFileConfig.cmake`, +pkg-config metadata, the CLI, and Apache license files below one prefix. + +The header closure mirrors the include relationships used by the current C++ +implementation. It is intentionally broader than the long-term stable public +API; a separate API cleanup will narrow it in a future version. + +## Manual artifact workflow + +`Build native release artifacts` (`.github/workflows/native-packages.yml`) is +the authoritative native packaging workflow. In the Apache TsFile repository's GitHub Actions tab, +select this workflow, choose **Run workflow**, select the branch containing the +commit to build, and dispatch it manually. It runs only on `workflow_dispatch`, +with read-only repository permissions; pushes and pull requests do not trigger +native packaging. + +A successful run builds the C++ core through Maven, stages one SDK per +platform and compatibility baseline, and then produces Ubuntu amd64 and arm64 DEBs, AlmaLinux +x86_64 and aarch64 RPMs, ARM64 and Intel Homebrew release bottles with a +merged Formula, Windows x86_64 and ARM64 SDK/CLI ZIPs, and Python wheels for Linux and macOS +on both architectures plus Windows x86_64. Every wheel platform covers CPython +3.9 through 3.14. The Go consumer and Python extensions reuse staged SDKs; C++ +is built once per compatibility baseline. The job combines these into +`tsfile-native-packages-` with `manifest.json` and +`SHA256SUMS`. Download this final artifact from the workflow +run; both intermediate and final artifacts are retained for 14 days. + +Publishing is a separate, manual step. This workflow only builds, tests, and +uploads GitHub Actions artifacts. It has no publishing credentials and does +not upload to JFrog, sign packages, update `latest`, create tags or releases, or +promote a build to an official release. + +For a source version of `2.5.0`, the workflow generates release-format +metadata: SDK archives and Python wheels use `2.5.0`, DEBs use `2.5.0-1`, +and RPMs use `2.5.0-1.el9`. This workflow accepts only `MAJOR.MINOR.PATCH` +source versions; development and prerelease versions fail during preparation. +The bundle records the exact source repository and commit in `manifest.json`. +Final bundle assembly rejects wheels whose version differs from the release +version. C++ publication targets always use the `release` directory or Debian +component. The DEB package name `tsfile-dev` denotes SDK headers and is retained +for release packages. + +## Portable archive + +Build a relocatable SDK and portable archive through the Maven entry point: + +```bash +./mvnw -Pwith-cpp package \ + -Dbuild.test=OFF \ + -Dtsfile.dependency.source=BUNDLED \ + -Denable.cpack=ON +cmake --install cpp/target/build --prefix cpp/build/sdk-root +cpack --config cpp/target/build/CPackConfig.cmake -G TGZ -B cpp/build/packages +``` + +The staged `cpp/build/sdk-root` contains a standard prefix layout and can be +unpacked at `/usr/local`, a user directory, or a relocated application prefix. + +## Native Linux packages + +DEB and RPM packages must be built in the target distribution environment so +CPack can run the native dependency scanner (`dpkg-shlibdeps` or +`rpmbuild`). Build the same Maven-driven tree once, then select the native +generator in the same job: + +```bash +./mvnw -Pwith-cpp package \ + -Dbuild.test=OFF \ + -Dtsfile.dependency.source=BUNDLED \ + -Denable.cpack=ON +cpack -G DEB --config cpp/target/build/CPackConfig.cmake -B cpp/build/packages +# or: cpack -G RPM --config cpp/target/build/CPackConfig.cmake -B cpp/build/packages +``` + +All release packages use `BUNDLED`, including DEB, RPM and Homebrew builds. +Pinned, SHA-256-verified third-party sources are compiled into the native library, +so installing TsFile does not require separate Snappy, LZ4 or other codec libraries. +Operating-system runtimes remain dynamic dependencies. `AUTO` and `SYSTEM` are +optional custom source-build policies, not release packaging policies. + +The manual `Build native release artifacts` workflow is the authoritative +native package build. Its Linux jobs create `tsfile`, `tsfile-dev`, and +`tsfile-tools` DEBs on native Ubuntu 22.04 amd64 and arm64 runners, and +`tsfile`, `tsfile-devel`, and `tsfile-tools` RPMs on native AlmaLinux 9 +x86_64 and aarch64 runners. Fresh Ubuntu 22.04, Ubuntu 24.04, and AlmaLinux 9 +containers install the matching packages and verify both `tsfile-cli` and an +external CMake consumer. The workflow uploads intermediate artifacts for 14 +days and does not publish packages. + +## Test an installed C/C++ SDK + +Configure `cpp/test` directly to compile existing C/C++ API regression tests +against an installed package or extracted SDK. This standalone entry point uses +`find_package(TsFile CONFIG REQUIRED)` and links `TsFile::tsfile`; it does not +configure or compile the TsFile library or CLI. Install a CMake-configured +GoogleTest package for the same architecture and compatible compiler runtime. + +```bash +cmake -S cpp/test -B installed-sdk-tests \ + -DCMAKE_PREFIX_PATH=/absolute/path/to/sdk +cmake --build installed-sdk-tests --config Release --parallel +ctest --test-dir installed-sdk-tests -C Release --output-on-failure +``` + +For a system-installed DEB or RPM development package, omit +`CMAKE_PREFIX_PATH`. If GoogleTest is installed under a separate prefix, pass +its config directory with `-DGTest_DIR=/absolute/path/to/lib/cmake/GTest`. +Only the contents of `cpp/test` are needed on the test machine; neither +`cpp/src` nor `cpp/tools` is required. Run CTest sequentially because some +reused tests write fixed filenames in their working directory. + +The standalone suite reuses 15 existing test files covering C bindings, +table/tree reads and writes, row queries, Arrow batches, properties, recovery, +and parallel writes. Existing disabled tests remain disabled. The complete +source-built suite continues to run through `-Pwith-cpp verify`; internal +allocator/fault-injection tests, codec implementation tests and CLI object +unit tests remain there. Every native installation job now runs this suite +through `packaging/tests/cpp_installed_tests.py`, in addition to the small +consumer check. The runner builds a SHA-256-verified GoogleTest 1.12.1 dependency +and links the test executable to the installed `TsFile::tsfile` target. Windows +tests and GoogleTest use the same static MSVC runtime as the packaged DLL. + +## CLI installation acceptance + +`packaging/tests/cli_package_smoke.py` executes an installed CLI against the +four resources in the current source checkout's `python/tests/resources`: + +| Resource | Model | Expected rows | +| --- | --- | ---: | +| `simple_tree.tsfile` | Tree | 105 | +| `simple_table_t1.tsfile` | Table | 60 | +| `simple_table_t2.tsfile` | Table | 40 | +| `table_with_time_column.tsfile` | Table with a named time column | 25 | + +The workflow takes these files from the same checked-out commit used for the +build and distributes one `native-cli-test-data` artifact. The script verifies each file's SHA-256 +before using it. It checks successful exit status, schema/types, metadata, +row/null counts, selected data sums and BOOLEAN values, head/offset/limit, +exported CSV versus `cat`, and sketch availability. Expected data facts come +from `python/tests/test_load_tsfile_from_iotdb.py`. + +```bash +python3 packaging/tests/cli_package_smoke.py \ + --cli /absolute/path/to/installed/bin/tsfile-cli \ + --resources /absolute/path/to/source/python/tests/resources +``` + +Only Python's standard library is required. Every CLI invocation has a +30-second timeout; generated exports live in a temporary directory. The script +compares data semantics and requires no captured output baselines or CLI test +implementation objects. + +Fresh Ubuntu 22.04/24.04 and AlmaLinux 9 containers install DEB/RPM packages +before running the checks. Separate Windows x86_64/ARM64 jobs unpack the built +ZIPs and run with the package's binary directory and Windows system paths. +Separate macOS Intel/ARM64 jobs merge the downloaded bottle metadata into a +trusted temporary tap, seed Homebrew's verified bottle cache, and install by +formula name. This supports Homebrew's package-path restrictions while keeping +SHA-256 verification and the `poured_from_bottle` receipt check. Acceptance jobs check out only +the acceptance scripts, existing C++ tests, and Go module. They compile the +C++ test executable and Go consumers against the installed package; they do not +compile TsFile or the CLI. Final assembly waits for all installation checks. + +## macOS + +Homebrew is the native macOS distribution path. The workflow renders the +release Formula `tsfile` from `packaging/homebrew/tsfile.rb.in`, using the exact +source repository and full workflow commit. It hashes that source archive, +uses the C++ release version, and builds and tests the installed C++ consumer +and CLI before bottling. Its bundled build has no third-party runtime Formula +dependencies. The generated Formula is shipped as `homebrew/Formula/tsfile.rb` +in the final artifact; no source URL or checksum needs to be edited after download. + +The Formula is keg-only because the SDK ships its dependency header closure; +linking that closure into Homebrew's shared prefix would collide with headers +owned by dependencies such as `simde`. Invoke the CLI as +`$(brew --prefix tsfile)/bin/tsfile-cli`, or add the Formula's `bin` directory +to `PATH`. The temporary `apache/tsfile` tap used by CI is local to each job; +a public Git tap is a separate distribution step. + +The ARM64 job runs on `macos-latest` and the Intel job on `macos-15-intel`. +Their `native-homebrew-macos-arm64` and `native-homebrew-macos-x86_64` +intermediate artifacts remain separate until merge. Each job also archives its +installed Formula prefix as a standalone SDK artifact, +`native-sdk-macos-arm64` or `native-sdk-macos-x86_64`, so macOS C++ consumers +do not have to depend on Homebrew. The SDK archive is checked with the CLI, +pkg-config, and an installed CMake consumer before upload. +`verify_macos_runtime.py` inspects Mach-O load commands in the installed bottle +and staged SDK, allowing only macOS system libraries and the packaged TsFile +library. Downloaded SDK tests repeat this check before executing any binaries, +so preinstalled Homebrew dependencies cannot conceal an external reference. +The merge job checks that +both source Formula files match, merges both platform JSON files with Homebrew, +and checks both generated tags and checksums in the resulting Formula. + +The `native-homebrew` artifact contains `Formula/tsfile.rb` and `bottles/` +with both bottle tarballs and both JSON metadata files. When downloaded into +`homebrew/`, this gives the final `homebrew/Formula/` and `homebrew/bottles/` +layout. The configured future bottle root is +`https://packages.apache.org/artifactory/tsfile/homebrew/release/versions//bottles`. +Homebrew generates local tarballs with a double dash before the version but +requests a single dash in HTTP URLs. After merge, the workflow renames each +tarball to the JSON `filename` (URL-decoded) and updates its `local_filename` +to match, preserving its checksum. The workflow only uploads GitHub Actions +artifacts for 14 days; it does not publish to that root or update `latest`. + +## Windows + +The manual workflow builds native x86_64 and ARM64 Release packages with +Visual Studio 2022 and bundled dependencies. Each job stages and tests the CLI +plus an external CMake SDK consumer before producing +`tsfile--windows-x86_64.zip` or +`tsfile--windows-arm64.zip`. Each archive is one combined ZIP +rather than one archive per CPack component, and contains the runtime, +development files, and tools under a relocatable prefix. The MSVC outputs are +installed as `bin/tsfile.dll`, `lib/tsfile.lib`, and `bin/tsfile-cli.exe`. + +The portable ZIP targets Windows 10 / Windows Server 2022 or newer and uses +the static MSVC runtime (`/MT`) in Release. `TSFILE_MSVC_STATIC_RUNTIME=ON` +requires CMake 3.15+ and `TSFILE_DEPENDENCY_SOURCE=BUNDLED`: all codec archives, +TsFile, and the CLI are compiled with the same runtime selection. The workflow +checks the runtime property on every native target and inspects every shipped +EXE/DLL with `dumpbin`, rejecting dynamic Visual C++ runtime imports and missing +non-system DLLs in both the staged tree and the extracted ZIP. Running the CLI +does not require a separate Visual C++ Redistributable installation. Microsoft +redistributable DLLs are not copied from the runner or included in the ZIP; +runtime security updates require rebuilding the static-runtime package. + +C++ SDK consumers should use an MSVC Release toolset compatible with the +compiler used to build that architecture and `/MT` +(`CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded`, with CMake policy CMP0091 set to +NEW before `project()`). Keep allocation and release paired through the public +APIs; CRT-owned objects such as `FILE*` must not cross DLL boundaries. The +installed-consumer fixture demonstrates a public setting/getter round trip +that requires real symbols from `tsfile.dll`. + +The workflow uploads the ZIPs as `native-windows-msvc-x86_64` and +`native-windows-msvc-arm64` for 14 days. Like the Linux jobs, it validates the +license files, CMake package config, staged executable, library, import library, +and public headers without publishing the artifacts. Windows ARM uses the +native ARM64 compiler and produces unsigned binary archives; no MSIX/AppX +signing step is part of this workflow. + +## Static SDK and install regressions + +`TSFILE_BUILD_SHARED=OFF` remains installable. Its CMake package rediscovers +system codec packages on the consuming machine and installs bundled codec +archives under `/tsfile/`. Link with `TsFile::tsfile` +to receive the complete static dependency list. Bundled archives and CMake +metadata move with the install prefix; system dependency development packages +must be available on the consuming machine. The shared package retains its +existing runtime/development/tools layout. + +The pkg-config prefix is derived from the configured pkg-config installation +directory, including multiarch library directories. The optional Unix native +integration suite builds, installs, relocates, and links CMake and pkg-config +consumers. It requires CMake 3.19+ for the runtime-property test, pkg-config, +system LZ4, and a verified dependency cache containing ANTLR4, utf8cpp, zlib, +and LZOKAY archives: + +```bash +TSFILE_RUN_INSTALL_TESTS=1 \ +TSFILE_TEST_DEPENDENCY_CACHE=/path/to/dependency-cache \ +python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v +``` + +## Go SDK verification + +`packaging/tests/go_installed_tests.py` runs the Go module's own `go vet`, +`go test`, and `go test -race` after native package installation on Linux, +macOS and Windows, on both architectures. The jobs use Go 1.26. Windows ARM64 +runs `vet` and normal tests; Go does not support its race detector on that +platform, so the runner explicitly reports that one check as not applicable. + +On Unix, the runner obtains both installed include roots, feature definitions, +and the library search path from the package's relocatable `libtsfile.pc`. +On Windows, a SHA-256-verified LLVM-MinGW 20261006 toolchain compiles cgo's C +bridge. The runner reads the installed MSVC DLL's C exports and generates a +MinGW import library, without rebuilding or replacing the DLL. This exercises +the same manually installed native package that Go users consume. + +## Python wheels + +The native workflow is the single packaging entry point. It replaces the former +`.github/workflows/wheels.yml` workflow; pushing an `rc/**` branch no longer +starts a separate wheel build. Dispatch the native workflow to build the full +package bundle, including all 30 Python wheels: + +| Wheel platform | CPython | C++ SDK source | +| --- | --- | --- | +| manylinux_2_28 x86_64 | 3.9–3.14 | One bundled build inside the manylinux container | +| manylinux_2_28 aarch64 | 3.9–3.14 | One bundled build inside the native ARM container | +| macOS 12 x86_64 | 3.9–3.14 | One bundled build with deployment target 12.0 | +| macOS 12 arm64 | 3.9–3.14 | One bundled build with deployment target 12.0 | +| Windows x86_64 | 3.9–3.14 | The existing native MSVC SDK | + +Each SDK installs `share/tsfile/tsfile-build-config.json`, recording its compiler, +configuration, resolved MSVC runtime and runtime library filename. The same +file is generated beside the library in a CMake build tree. `setup.py` reads +this metadata to select `/MT`, `/MD` or their Debug variants automatically; +no Python build environment variable duplicates the SDK's CRT selection. +Rebuild the native SDK if this metadata is missing. + +Wheels contain one TsFile runtime library under its runtime filename, the +Python extensions, Python code and license/notice texts. SDK headers, Cython build declarations, +import libraries and static archives remain build inputs and are excluded from +the wheel. Release wheel builds strip Unix debug/local symbols before +`auditwheel` or `delocate` changes library load commands, and macOS binaries +are signed again after stripping. SDKs and in-place development builds retain +their symbols. `packaging/scripts/check_python_wheel.py` rejects duplicate +native libraries, build-only files and ELF debug sections in every built and +downloaded wheel; fresh installation jobs then run the complete pytest suite. + +`cibuildwheel==3.4.1` builds the Python extensions and handles platform wheel +repair. Its `before-all` hook invokes `packaging/scripts/build_python_sdk.sh` +once per Unix compatibility baseline. Every Python ABI then links against that +SDK through `TSFILE_CPP_BUILD`. The manylinux hook exports the SDK archive +through cibuildwheel's `/host` mount so the subsequent upload runs on the host. +The Windows job downloads the existing MSVC SDK +and copies its DLLs from `bin/` into a Python build view under `lib/`; it does +not compile another C++ core. The native Windows SDK enables LZMA2, matching +the wheel compatibility SDKs. + +The wheel SDKs remain distinct from the Ubuntu/AlmaLinux SDKs and Homebrew +SDKs: an Ubuntu SDK does not by itself establish manylinux_2_28 compatibility, +and a Homebrew SDK may require newer macOS. All of these SDKs bundle their +third-party dependencies. The four +Unix wheel SDKs use bundled dependencies and are included in the final bundle +as `native-sdk-manylinux_2_28-{x86_64,aarch64}` and +`native-sdk-macos12-{x86_64,arm64}`. This also makes them available for direct +download and other consumers of the same compatibility baseline. + +Wheel jobs build, repair and upload artifacts. Separate `test-python-unix` +and `test-python-windows` jobs download those artifacts on fresh runners, one +job for each of the 30 platform/CPython combinations. They check out only +`python/tests` and the installation test runner; neither the Python package +implementation nor C++ sources are present. + +`packaging/tests/python_installed_tests.py` selects exactly one downloaded +wheel matching the expected package version and interpreter ABI, installs it +with binary-only dependencies, and runs the existing full `pytest` suite. +Imports use Python's isolated mode from a temporary working directory, and the +runner checks the installed module path and distribution version. Individual +tests have a 120-second timeout with stack traces; import and whole-suite +timeouts prevent native hangs from consuming an entire job. Each combination +uploads a JUnit result artifact. Existing test-suite platform skips remain +visible in the report. + +Final assembly rejects missing Python ABIs, duplicate ABIs, incompatible +platform tags, and inconsistent wheel versions. Wheel manifest entries include +the Python ABI, wheel version, and the SDK platform used to build the extension. + +## Installed package test matrix + +| Installed package | Architectures | Separate installation tests | +| --- | --- | --- | +| DEB on Ubuntu 22.04 and 24.04 | x86_64, ARM64 | C++ API regressions, CLI resource checks, Go vet/test/race | +| RPM on AlmaLinux 9 | x86_64, ARM64 | C++ API regressions, CLI resource checks, Go vet/test/race | +| Homebrew bottle on macOS | x86_64, ARM64 | C++ API regressions, CLI resource checks, Go vet/test/race | +| Windows ZIP | x86_64, ARM64 | C++ API regressions, CLI resource checks, Go vet/test; race on x86_64 | +| Python wheel | Linux/macOS x86_64/ARM64, Windows x86_64 | Full installed-wheel pytest for every CPython 3.9–3.14 ABI | +| Standalone SDK archives | All 12 distribution and compatibility SDK variants | Download, extract to a new prefix, then C++ API regressions, CLI resource checks and Go tests | + +All native tests consume downloaded, installed packages. CLI acceptance uses +the source checkout's Python resources and does not use CLI golden/E2E files. +The C++ regression suite compiles existing API tests through `find_package`; +internal tests requiring a special test build remain in source CI. The release +bundle is gated on every installation-test matrix completing successfully. + +Each native installation job also downloads its separate SDK archive and runs +`packaging/tests/sdk_archive_tests.py` against an extracted prefix. Its C++ test +executable links that prefix's `TsFile::tsfile`; CLI and Go consume the same +extracted files. The existing GoogleTest dependency is reused, while the SDK's +tests are rebuilt and rerun. Separate `test-compatibility-sdk` jobs cover the +four bundled manylinux/macOS 12 SDK archives produced for wheel builds. These +jobs check out consumer tests only and also gate final assembly. SDK archive +test reports are retained alongside native-package test reports. + +## Final native package bundle + +Only after the Ubuntu amd64 and arm64 DEB installation tests, AlmaLinux x86_64 +and aarch64 RPM installation tests, macOS and Windows C++/CLI/Go installation +tests, all standalone and compatibility SDK archive tests, all 30 installed-wheel +pytest jobs, macOS SDK and Homebrew bottle merge, +and Windows x86_64 and ARM64 SDK/CLI builds +all succeed, the workflow assembles +`tsfile-native-packages-`. The final +GitHub Actions artifact retains the platform SDKs, including both Linux ARM +archives, both macOS archives, the Windows ARM64 archive, the Python +wheels and their compatibility SDKs, the DEBs, RPMs, merged Formula and bottles, +and Windows ZIPs in their package-family layouts for 14 days. It also contains a +sorted `SHA256SUMS` and `manifest.json` with the source identity, generated +versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target +path, and properties required for later manual publication. + +The final job has no publishing credentials and does not upload to JFrog. A +maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the +recorded Debian coordinates, RPMs to `tsfile-rpm/release/el9/`, +Homebrew formula/bottles to `tsfile/homebrew/release/versions/`, SDKs to +`tsfile/sdk/release/versions/`, Windows ZIPs to +`tsfile/windows/release/versions/`. DEBs use the `release` component +and `pool/release/` paths. Python wheels retain their platform, +version, size and checksum metadata, but have no Apache repository or target +path: they are published separately through the project's PyPI process. diff --git a/packaging/cmake/InstalledGTest/CMakeLists.txt b/packaging/cmake/InstalledGTest/CMakeLists.txt new file mode 100644 index 000000000..3f310ace3 --- /dev/null +++ b/packaging/cmake/InstalledGTest/CMakeLists.txt @@ -0,0 +1,36 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +cmake_minimum_required(VERSION 3.15) +if (POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif () +project(TsFileInstalledTestDependencies LANGUAGES CXX) + +# Build the test framework only; the TsFile implementation comes from the +# downloaded and installed package. Match the Windows SDK's static CRT. +set(BUILD_GMOCK OFF CACHE BOOL "" FORCE) +set(INSTALL_GTEST ON CACHE BOOL "" FORCE) +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) +include(FetchContent) +FetchContent_Declare(googletest + URL "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.zip" + URL_HASH SHA256=24564e3b712d3eb30ac9a85d92f7d720f60cc0173730ac166f27dda7fed76cb2) +FetchContent_MakeAvailable(googletest) diff --git a/packaging/cmake/PythonSDK.cmake b/packaging/cmake/PythonSDK.cmake new file mode 100644 index 000000000..872d30803 --- /dev/null +++ b/packaging/cmake/PythonSDK.cmake @@ -0,0 +1,24 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# setup.py consumes an SDK with include/ and lib/ on every platform. +set(CMAKE_INSTALL_LIBDIR "lib" CACHE PATH "Python SDK library directory" FORCE) +if (APPLE) + set(CMAKE_OSX_DEPLOYMENT_TARGET "12.0" CACHE STRING "Wheel deployment target" FORCE) +endif () diff --git a/packaging/homebrew/tsfile.rb.in b/packaging/homebrew/tsfile.rb.in new file mode 100644 index 000000000..ad4c3aa29 --- /dev/null +++ b/packaging/homebrew/tsfile.rb.in @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Tsfile < Formula + desc "Columnar storage library for time series data" + homepage "https://tsfile.apache.org/" + url "https://github.com/@SOURCE_REPOSITORY@/archive/@GIT_SHA@.tar.gz" + version "@HOMEBREW_VERSION@" + sha256 "@SOURCE_SHA256@" + license "Apache-2.0" + keg_only "the SDK installs its dependency header closure" + +@BOTTLE_BLOCK@ + + depends_on "cmake" => :build + + def install + args = %W[ + -DBUILD_TEST=OFF + -DBUILD_TOOLS=ON + -DENABLE_LZMA2=OFF + -DTSFILE_DEPENDENCY_SOURCE=BUNDLED + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + -DCMAKE_INSTALL_RPATH=#{rpath} + ] + + system "cmake", "-S", "cpp", "-B", "build", *args, *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + (testpath / "test.cpp").write <<~CPP + #include + + int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; + } + CPP + + system ENV.cxx, "-std=c++11", "test.cpp", "-I#{include}", "-L#{lib}", + "-Wl,-rpath,#{lib}", "-ltsfile", "-o", "test" + system "./test" + system bin / "tsfile-cli", "--version" + end +end diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py new file mode 100644 index 000000000..c18a3f079 --- /dev/null +++ b/packaging/scripts/assemble_native_packages.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Assemble native package artifacts into a checksummed publishable bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +from pathlib import Path +from typing import Sequence + +DEB_PROPERTIES = { + "deb.distribution": ["jammy", "noble"], + "deb.component": ["release"], +} +DEB_PLATFORMS = {"ubuntu22.04-amd64", "ubuntu22.04-arm64"} +RPM_PLATFORMS = {"almalinux9-x86_64", "almalinux9-aarch64"} +WINDOWS_PLATFORMS = {"windows-msvc-x86_64", "windows-msvc-arm64"} +REQUIRED_VERSIONS = {"archive_version", "homebrew_version"} +REQUIRED_SOURCE = {"commit", "repository"} +SDK_PLATFORMS = { + "ubuntu22.04-amd64", + "ubuntu22.04-arm64", + "almalinux9-x86_64", + "almalinux9-aarch64", + "macos-arm64", + "macos-x86_64", + "windows-msvc-x86_64", + "windows-msvc-arm64", + "manylinux_2_28-x86_64", + "manylinux_2_28-aarch64", + "macos12-x86_64", + "macos12-arm64", +} +PYTHON_PLATFORMS = { + "manylinux_2_28-x86_64", + "manylinux_2_28-aarch64", + "macos12-x86_64", + "macos12-arm64", + "windows-msvc-x86_64", +} +PYTHON_TAGS = {"cp39", "cp310", "cp311", "cp312", "cp313", "cp314"} + + +def _wheel_metadata(filename: str, platform: str) -> dict[str, str]: + match = re.fullmatch(r"tsfile-([^-]+)-(cp\d+)-(cp\d+)-([^-]+)\.whl", filename) + if match is None: + raise ValueError(f"unsupported Python wheel filename: {filename}") + version, python_tag, abi_tag, platform_tags = match.groups() + if python_tag not in PYTHON_TAGS or abi_tag != python_tag: + raise ValueError(f"unsupported Python wheel ABI: {filename}") + tags = platform_tags.split(".") + if platform.startswith("manylinux_2_28-"): + architecture = platform.removeprefix("manylinux_2_28-") + valid = f"manylinux_2_28_{architecture}" in tags and all( + re.fullmatch(rf"manylinux(?:_\d+_\d+|\d+)_{architecture}", tag) + for tag in tags + ) + elif platform.startswith("macos12-"): + architecture = platform.removeprefix("macos12-") + matches = [ + re.fullmatch(rf"macosx_(\d+)_(\d+)_{architecture}", tag) for tag in tags + ] + valid = all( + item is not None and tuple(map(int, item.groups())) <= (12, 0) + for item in matches + ) + else: + valid = tags == ["win_amd64"] + if not valid: + raise ValueError(f"incompatible Python wheel tags for {platform}: {filename}") + return { + "pythonTag": python_tag, + "abiTag": abi_tag, + "wheelVersion": version, + "sdkPlatform": platform, + } + + +def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> None: + if not isinstance(values, dict) or not required.issubset(values): + raise ValueError(f"{name} must contain {sorted(required)}") + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in values.items() + ): + raise ValueError(f"{name} must map strings to strings") + + +def _validate_versions(versions: dict[str, str]) -> None: + _validate_metadata(versions, REQUIRED_VERSIONS, "versions") + for name in REQUIRED_VERSIONS: + value = versions[name] + if ( + not value.strip() + or value in {".", ".."} + or "/" in value + or "\\" in value + or "\x00" in value + ): + raise ValueError(f"version {name} must be a safe path component") + if not re.fullmatch( + r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)", value + ): + raise ValueError( + f"version {name} must be a release version MAJOR.MINOR.PATCH" + ) + release_version = versions["archive_version"] + for name in ( + "homebrew_version", + "base_version", + "logical_version", + "python_version", + ): + if name in versions and versions[name] != release_version: + raise ValueError( + f"version {name} must match release version {release_version}" + ) + + +def _validate_source(source: dict[str, str]) -> None: + _validate_metadata(source, REQUIRED_SOURCE, "source") + for name in REQUIRED_SOURCE: + if not source[name].strip(): + raise ValueError(f"source {name} must be nonempty") + + +def _input_files(input_dir: Path) -> list[Path]: + if not input_dir.is_dir() or input_dir.is_symlink(): + raise ValueError(f"input directory is not a real directory: {input_dir}") + files: list[Path] = [] + for directory, directories, names in os.walk(input_dir, followlinks=False): + directory_path = Path(directory) + for child in directories: + path = directory_path / child + if path.is_symlink(): + raise ValueError(f"symbolic link input is not allowed: {path}") + for name in names: + path = directory_path / name + if path.is_symlink(): + raise ValueError(f"symbolic link input is not allowed: {path}") + if not path.is_file(): + raise ValueError(f"undeclared file type: {path}") + if path.stat().st_size == 0: + raise ValueError(f"empty file input is not allowed: {path}") + files.append(path) + return sorted(files, key=lambda path: path.relative_to(input_dir).as_posix()) + + +def _artifact_description( + input_dir: Path, source_path: Path, versions: dict[str, str] +) -> tuple[Path, dict[str, object]]: + relative_path = source_path.relative_to(input_dir) + parts = relative_path.parts + filename = source_path.name + if len(parts) >= 2 and parts[0] == "deb" and filename.endswith(".deb"): + match = re.search(r"_(amd64|arm64)\.deb$", filename) + if not match: + raise ValueError(f"unsupported DEB architecture: {filename}") + architecture = match.group(1) + platform = f"ubuntu22.04-{architecture}" + target = Path("deb") / platform / filename + return target, { + "family": "deb", + "platform": platform, + "targetRepository": "tsfile-debian", + "targetPath": f"pool/release/{platform}/{filename}", + "properties": { + **DEB_PROPERTIES, + "deb.architecture": [architecture], + }, + } + if len(parts) >= 2 and parts[0] == "rpm" and filename.endswith(".rpm"): + match = re.search(r"\.(x86_64|aarch64)\.rpm$", filename) + if not match: + raise ValueError(f"unsupported RPM architecture: {filename}") + architecture = match.group(1) + platform = f"almalinux9-{architecture}" + target = Path("rpm") / platform / filename + return target, { + "family": "rpm", + "platform": platform, + "targetRepository": "tsfile-rpm", + "targetPath": f"release/el9/{architecture}/{filename}", + "properties": {}, + } + if ( + len(parts) == 3 + and parts[:2] == ("homebrew", "Formula") + and filename == "tsfile.rb" + ): + target = Path(*parts) + return target, { + "family": "homebrew", + "platform": "homebrew", + "targetRepository": "tsfile", + "targetPath": f"homebrew/release/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", + "properties": {}, + } + if ( + len(parts) == 3 + and parts[:2] == ("homebrew", "bottles") + and (filename.endswith(".tar.gz") or filename.endswith(".bottle.json")) + ): + target = Path(*parts) + return target, { + "family": "homebrew", + "platform": "homebrew", + "targetRepository": "tsfile", + "targetPath": f"homebrew/release/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", + "properties": {}, + } + if len(parts) == 3 and parts[0] == "sdk" and filename.endswith(".tar.gz"): + platform = parts[1] + if platform not in SDK_PLATFORMS: + raise ValueError(f"unsupported SDK platform: {platform}") + target = Path("sdk") / platform / filename + return target, { + "family": "sdk", + "platform": platform, + "targetRepository": "tsfile", + "targetPath": ( + f"sdk/release/versions/{versions['archive_version']}/{platform}/{filename}" + ), + "properties": {}, + } + if len(parts) == 3 and parts[0] == "python" and filename.endswith(".whl"): + platform = parts[1] + if platform not in PYTHON_PLATFORMS: + raise ValueError(f"unsupported Python wheel platform: {platform}") + target = Path("python/wheels") / platform / filename + return target, { + "family": "python", + "platform": platform, + **_wheel_metadata(filename, platform), + } + if len(parts) >= 2 and parts[0] == "windows" and filename.endswith(".zip"): + match = re.search(r"-windows-(x86_64|arm64)\.zip$", filename) + if not match: + raise ValueError(f"unsupported Windows ZIP architecture: {filename}") + platform = f"windows-msvc-{match.group(1)}" + target = Path("windows") / filename + return target, { + "family": "windows", + "platform": platform, + "targetRepository": "tsfile", + "targetPath": f"windows/release/versions/{versions['archive_version']}/{filename}", + "properties": {}, + } + raise ValueError(f"undeclared file type: {relative_path.as_posix()}") + + +def _require_complete_families( + planned: list[tuple[Path, Path, dict[str, object]]], +) -> None: + platforms_by_family = { + family: { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == family + } + for family in ("deb", "rpm", "windows") + } + for family, required_platforms in ( + ("deb", DEB_PLATFORMS), + ("rpm", RPM_PLATFORMS), + ("windows", WINDOWS_PLATFORMS), + ): + missing_platforms = required_platforms - platforms_by_family[family] + if missing_platforms: + raise ValueError( + f"missing {family.upper()} inputs for: " + + ", ".join(sorted(missing_platforms)) + ) + sdk_platforms = { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == "sdk" + } + missing_sdk_platforms = SDK_PLATFORMS - sdk_platforms + if missing_sdk_platforms: + raise ValueError( + "missing SDK inputs for: " + ", ".join(sorted(missing_sdk_platforms)) + ) + python_platforms = { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == "python" + } + missing_python_platforms = PYTHON_PLATFORMS - python_platforms + if missing_python_platforms: + raise ValueError( + "missing Python wheel inputs for: " + + ", ".join(sorted(missing_python_platforms)) + ) + wheel_versions: set[str] = set() + for platform in sorted(PYTHON_PLATFORMS): + wheels = [ + metadata + for _, _, metadata in planned + if metadata["family"] == "python" and metadata["platform"] == platform + ] + tags = [wheel["pythonTag"] for wheel in wheels] + missing = PYTHON_TAGS - set(tags) + if missing: + raise ValueError( + f"missing Python wheel ABIs for {platform}: " + + ", ".join(sorted(missing)) + ) + if len(tags) != len(set(tags)): + raise ValueError(f"duplicate Python wheel ABI for {platform}") + wheel_versions.update(wheel["wheelVersion"] for wheel in wheels) + if len(wheel_versions) != 1: + raise ValueError("Python wheels must all have the same version") + homebrew_paths = { + target.as_posix() + for _, target, metadata in planned + if metadata["family"] == "homebrew" + } + if "homebrew/Formula/tsfile.rb" not in homebrew_paths: + raise ValueError("missing Homebrew Formula input") + if not any(path.endswith(".tar.gz") for path in homebrew_paths): + raise ValueError("missing Homebrew Bottle input") + if not any(path.endswith(".bottle.json") for path in homebrew_paths): + raise ValueError("missing Homebrew Bottle metadata input") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _artifact_paths(output_dir: Path) -> set[Path]: + paths: set[Path] = set() + for path in output_dir.rglob("*"): + if path.is_symlink(): + raise ValueError(f"symbolic link output is not allowed: {path}") + if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"}: + paths.add(path.relative_to(output_dir)) + return paths + + +def verify_bundle(output_dir: Path) -> None: + """Verify manifest hashes, sizes, and path-sorted checksums in a bundle.""" + output_dir = Path(output_dir) + if not output_dir.is_dir() or output_dir.is_symlink(): + raise ValueError(f"bundle directory is not a real directory: {output_dir}") + manifest_path = output_dir / "manifest.json" + checksums_path = output_dir / "SHA256SUMS" + if ( + not manifest_path.is_file() + or manifest_path.is_symlink() + or not checksums_path.is_file() + or checksums_path.is_symlink() + ): + raise ValueError("bundle must contain real manifest.json and SHA256SUMS files") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + artifacts = manifest.get("artifacts") if isinstance(manifest, dict) else None + if not isinstance(artifacts, list): + raise ValueError("manifest artifacts must be a list") + + paths: set[Path] = set() + for artifact in artifacts: + if not isinstance(artifact, dict): + raise ValueError("manifest artifacts must be objects") + relative_text = artifact.get("path") + if not isinstance(relative_text, str): + raise ValueError("manifest artifact path must be a string") + relative = Path(relative_text) + if ( + relative.is_absolute() + or relative.as_posix() != relative_text + or ".." in relative.parts + ): + raise ValueError(f"unsafe manifest path: {relative_text}") + if relative in paths: + raise ValueError(f"duplicate manifest path: {relative_text}") + paths.add(relative) + path = output_dir / relative + if not path.is_file() or path.is_symlink(): + raise ValueError(f"missing artifact: {relative_text}") + sha256 = artifact.get("sha256") + if not isinstance(sha256, str) or re.fullmatch(r"[0-9a-f]{64}", sha256) is None: + raise ValueError(f"invalid artifact SHA-256: {relative_text}") + if _sha256(path) != sha256: + raise ValueError(f"hash mismatch: {relative_text}") + if path.stat().st_size != artifact.get("size"): + raise ValueError(f"size mismatch: {relative_text}") + + actual_checksums = checksums_path.read_text(encoding="utf-8").splitlines() + expected_checksums = [ + f"{artifact['sha256']} {artifact['path']}" + for artifact in sorted(artifacts, key=lambda artifact: artifact["path"]) + ] + if actual_checksums != expected_checksums: + raise ValueError("SHA256SUMS does not match path-sorted manifest artifacts") + if _artifact_paths(output_dir) != paths: + raise ValueError("bundle files do not match manifest artifacts") + + +def assemble( + input_dir: Path, output_dir: Path, versions: dict[str, str], source: dict[str, str] +) -> dict[str, object]: + """Copy approved artifacts and return their deterministic publication manifest.""" + _validate_versions(versions) + _validate_source(source) + input_dir = Path(input_dir) + output_dir = Path(output_dir) + files = _input_files(input_dir) + planned: list[tuple[Path, Path, dict[str, object]]] = [] + target_paths: set[Path] = set() + for source_path in files: + target, metadata = _artifact_description(input_dir, source_path, versions) + if target in target_paths: + raise ValueError(f"duplicate target path: {target.as_posix()}") + target_paths.add(target) + planned.append((source_path, target, metadata)) + _require_complete_families(planned) + release_version = versions["archive_version"] + for _, _, metadata in planned: + if ( + metadata["family"] == "python" + and metadata["wheelVersion"] != release_version + ): + raise ValueError( + f"Python wheel version must match release version {release_version}" + ) + if output_dir.is_symlink(): + raise ValueError(f"output directory must not be a symbolic link: {output_dir}") + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"output directory must be empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + + artifacts: list[dict[str, object]] = [] + for source_path, target, metadata in sorted( + planned, key=lambda item: item[1].as_posix() + ): + destination = output_dir / target + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source_path, destination) + artifact = { + **metadata, + "filename": destination.name, + "path": target.as_posix(), + "size": destination.stat().st_size, + "sha256": _sha256(destination), + } + artifacts.append(artifact) + + checksums = "".join( + f"{artifact['sha256']} {artifact['path']}\n" for artifact in artifacts + ) + (output_dir / "SHA256SUMS").write_text(checksums, encoding="utf-8") + manifest: dict[str, object] = { + "artifacts": artifacts, + "source": source, + "versions": versions, + } + (output_dir / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--versions-json", type=Path) + parser.add_argument("--source-repository") + parser.add_argument("--source-commit") + parser.add_argument("--verify-bundle", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Assemble a bundle from an Actions artifact download directory.""" + arguments = _argument_parser().parse_args(argv) + if arguments.verify_bundle is not None: + if any( + value is not None + for value in ( + arguments.input_dir, + arguments.output_dir, + arguments.versions_json, + arguments.source_repository, + arguments.source_commit, + ) + ): + raise ValueError( + "--verify-bundle cannot be combined with assembly arguments" + ) + verify_bundle(arguments.verify_bundle) + return 0 + missing = [ + name + for name, value in ( + ("--input-dir", arguments.input_dir), + ("--output-dir", arguments.output_dir), + ("--versions-json", arguments.versions_json), + ("--source-repository", arguments.source_repository), + ("--source-commit", arguments.source_commit), + ) + if value is None + ] + if missing: + raise ValueError(f"missing assembly arguments: {', '.join(missing)}") + versions = json.loads(arguments.versions_json.read_text(encoding="utf-8")) + assemble( + arguments.input_dir, + arguments.output_dir, + versions, + {"repository": arguments.source_repository, "commit": arguments.source_commit}, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/scripts/build_python_sdk.sh b/packaging/scripts/build_python_sdk.sh new file mode 100644 index 000000000..888e55a2e --- /dev/null +++ b/packaging/scripts/build_python_sdk.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One build per compatibility baseline, reused by every Python ABI. +set -euo pipefail + +platform="${1:?usage: build_python_sdk.sh }" +archive_version="${2:?usage: build_python_sdk.sh }" +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repository_root" + +case "$platform" in + manylinux_2_28-x86_64|manylinux_2_28-aarch64) + test "$(uname -s)" = Linux + test "$(uname -m)" = "${platform#manylinux_2_28-}" + # This hook runs inside cibuildwheel's manylinux container. + yum install -y java-17-openjdk-devel cmake pkgconfig libuuid-devel libblkid-devel + export JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export PATH="$JAVA_HOME/bin:$PATH" + ;; + macos12-x86_64|macos12-arm64) + test "$(uname -s)" = Darwin + test "$(uname -m)" = "${platform#macos12-}" + export MACOSX_DEPLOYMENT_TARGET=12.0 + export CFLAGS="${CFLAGS:-} -mmacosx-version-min=12.0" + export CXXFLAGS="${CXXFLAGS:-} -mmacosx-version-min=12.0" + export LDFLAGS="${LDFLAGS:-} -mmacosx-version-min=12.0" + ;; + *) + printf 'Unsupported Python SDK platform: %s\n' "$platform" >&2 + exit 1 + ;; +esac + +python3 packaging/scripts/run_maven_with_retry.py -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.version.sync.skip=true \ + -Dtsfile.dependency.source=BUNDLED \ + -Denable.lzma2=ON \ + "-Dtsfile.project.include=$repository_root/packaging/cmake/PythonSDK.cmake" + +# A fixed path lets every Python build consume this SDK without rebuilding C++. +sdk_root="$repository_root/python-sdk" +test ! -e "$sdk_root" +cmake --install cpp/target/build --prefix "$sdk_root" +test -f "$sdk_root/include/tsfile/cwrapper/tsfile_cwrapper.h" +test -d "$sdk_root/lib" +if [[ "$(uname -s)" == Darwin ]]; then + python3 packaging/scripts/verify_macos_runtime.py "$sdk_root" +fi +"$sdk_root/bin/tsfile-cli" --version +sdk_output="${TSFILE_PYTHON_SDK_EXPORT:-$repository_root/python-sdk-output}" +mkdir -p "$sdk_output" +tar -czf "$sdk_output/tsfile-sdk-$platform-$archive_version.tar.gz" python-sdk diff --git a/packaging/scripts/check_python_wheel.py b/packaging/scripts/check_python_wheel.py new file mode 100644 index 000000000..aad85b8a2 --- /dev/null +++ b/packaging/scripts/check_python_wheel.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Reject build-only files, repeated native libraries and ELF debug sections.""" + +import argparse +import hashlib +from pathlib import Path +import struct +from zipfile import ZipFile + + +def elf_debug_sections(data): + if data[:4] != b"\x7fELF": + return [] + if data[4] != 2: + raise ValueError("Expected a 64-bit ELF for the release wheel matrix") + endian = "<" if data[5] == 1 else ">" + offset = struct.unpack_from(endian + "Q", data, 40)[0] + size, count, names_index = struct.unpack_from(endian + "HHH", data, 58) + sections = [ + struct.unpack_from(endian + "IIQQQQIIQQ", data, offset + i * size) + for i in range(count) + ] + names_header = sections[names_index] + names = data[names_header[4] : names_header[4] + names_header[5]] + result = [] + for section in sections: + name = names[section[0] :].split(b"\0", 1)[0].decode("ascii") + if name.startswith((".debug_", ".zdebug_")): + result.append(name) + return result + + +def check_wheel(path): + with ZipFile(path) as wheel: + files = [info for info in wheel.infolist() if not info.is_dir()] + names = [info.filename for info in files] + if len(names) != len(set(names)): + raise ValueError(f"Duplicate ZIP entries in {path}") + forbidden = [ + name + for name in names + if name.startswith("tsfile/include/") + or Path(name).suffix.lower() + in { + ".h", + ".hpp", + ".pxd", + ".pxi", + ".pyx", + ".a", + ".lib", + ".pdb", + ".cc", + ".cpp", + } + ] + if forbidden: + raise ValueError(f"Build-only files in {path}: {forbidden[:8]}") + native = [ + name + for name in names + if Path(name).name.startswith("libtsfile") + or Path(name).name == "tsfile.dll" + ] + if len(native) != 1: + raise ValueError(f"Expected one TsFile runtime library in {path}: {native}") + hashes = {} + for name in names: + if not (name.endswith((".so", ".dylib", ".dll", ".pyd")) or ".so." in name): + continue + data = wheel.read(name) + debug = elf_debug_sections(data) + if debug: + raise ValueError(f"ELF debug sections in {path}/{name}: {debug}") + digest = hashlib.sha256(data).hexdigest() + if digest in hashes: + raise ValueError(f"Repeated binary in {path}: {hashes[digest]}, {name}") + hashes[digest] = name + raw = sum(info.file_size for info in files) + print( + f"PASS: {Path(path).name}: {Path(path).stat().st_size} bytes compressed, " + f"{raw} bytes installed; one runtime library, no build-only files or ELF debug sections" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheelhouse", type=Path) + parser.add_argument("wheels", type=Path, nargs="*") + arguments = parser.parse_args() + wheels = arguments.wheels + if arguments.wheelhouse: + wheels += sorted(arguments.wheelhouse.glob("*.whl")) + if not wheels: + raise ValueError("No wheels found for content validation") + for wheel in wheels: + check_wheel(wheel) + + +if __name__ == "__main__": + main() diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py new file mode 100644 index 000000000..f8bc68a48 --- /dev/null +++ b/packaging/scripts/native_package_versions.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build release versions for native TsFile packages.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Sequence + +_SOURCE_VERSION_RE = re.compile( + r"^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$" +) +_CMAKE_VERSION_RE = re.compile( + r"^\s*set\(\s*TsFile_CPP_VERSION\s+(?P[^\s)]+)\s*\)\s*(?:#.*)?$", + re.MULTILINE, +) + + +def read_cpp_version(path: Path) -> str: + """Read the release version declared by CMake.""" + match = _CMAKE_VERSION_RE.search(path.read_text(encoding="utf-8")) + if match is None or _SOURCE_VERSION_RE.fullmatch(match.group("version")) is None: + raise ValueError( + f"{path} must declare a release TsFile_CPP_VERSION as MAJOR.MINOR.PATCH" + ) + return match.group("version") + + +def build_versions(source_version: str) -> dict[str, str]: + """Create matching archive, Python, DEB, RPM, and Homebrew release versions.""" + if _SOURCE_VERSION_RE.fullmatch(source_version) is None: + raise ValueError("source_version must be a release version MAJOR.MINOR.PATCH") + return { + "base_version": source_version, + "python_version": source_version, + "logical_version": source_version, + "deb_version": f"{source_version}-1", + "rpm_version": source_version, + "rpm_release": "1.el9", + "archive_version": source_version, + "homebrew_version": source_version, + } + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cmake-file", type=Path, required=True) + parser.add_argument("--json-out", type=Path, required=True) + parser.add_argument("--github-output", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Write release versions to JSON and, optionally, GitHub outputs.""" + arguments = _argument_parser().parse_args(argv) + versions = build_versions(read_cpp_version(arguments.cmake_file)) + arguments.json_out.write_text( + json.dumps(versions, indent=2) + "\n", encoding="utf-8" + ) + if arguments.github_output is not None: + with arguments.github_output.open("a+b") as github_output: + if github_output.tell(): + github_output.seek(-1, 2) + if github_output.read(1) != b"\n": + github_output.write(b"\n") + github_output.write( + ( + "\n".join(f"{key}={value}" for key, value in versions.items()) + + "\n" + ).encode("utf-8") + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/scripts/render_homebrew_formula.py b/packaging/scripts/render_homebrew_formula.py new file mode 100644 index 000000000..d13c9065c --- /dev/null +++ b/packaging/scripts/render_homebrew_formula.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Render the release Formula using an immutable source identity.""" + +import argparse +import re +from pathlib import Path + +VALUE_PATTERNS = { + "SOURCE_REPOSITORY": r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "GIT_SHA": r"[0-9a-f]{40}", + "HOMEBREW_VERSION": r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)", + "SOURCE_SHA256": r"[0-9a-f]{64}", +} +TOKEN = re.compile(r"@([A-Z0-9_]+)@") + + +def render_formula( + template: str, values: dict[str, str], bottle_block: str = "" +) -> str: + """Replace declared tokens without interpreting Ruby braces or at-signs.""" + if values.keys() != VALUE_PATTERNS.keys(): + raise ValueError( + "Formula values must contain exactly the declared source fields" + ) + for name, pattern in VALUE_PATTERNS.items(): + if not re.fullmatch(pattern, values[name]): + raise ValueError(f"Invalid Formula value: {name}") + replacements = {**values, "BOTTLE_BLOCK": bottle_block} + unknown = set(TOKEN.findall(template)) - replacements.keys() + if unknown: + raise ValueError(f"Unknown Formula placeholders: {sorted(unknown)}") + if not bottle_block: + # Omit an empty standalone block and its following separator line. + template = re.sub(r"(?m)^[ \t]*@BOTTLE_BLOCK@\n(?:[ \t]*\n)?", "", template) + rendered = TOKEN.sub(lambda match: replacements[match.group(1)], template) + if TOKEN.search(rendered): + raise ValueError("Unresolved Formula placeholder") + if len(re.findall(r"^\s*bottle\s+do\b", rendered, re.MULTILINE)) > 1: + raise ValueError("Duplicate bottle blocks") + return rendered + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--template", type=Path, required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-sha256", required=True) + parser.add_argument("--bottle-block", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + values = { + "SOURCE_REPOSITORY": args.repository, + "GIT_SHA": args.git_sha, + "HOMEBREW_VERSION": args.version, + "SOURCE_SHA256": args.source_sha256, + } + bottle_block = ( + args.bottle_block.read_text(encoding="utf-8") if args.bottle_block else "" + ) + args.output.write_text( + render_formula(args.template.read_text(encoding="utf-8"), values, bottle_block), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/scripts/run_maven_with_retry.py b/packaging/scripts/run_maven_with_retry.py new file mode 100644 index 000000000..ccc950cc5 --- /dev/null +++ b/packaging/scripts/run_maven_with_retry.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Retry Maven only when artifact downloads fail with transient transport errors.""" + +from collections import deque +import os +from pathlib import Path +import re +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[2] +TRANSIENT_DOWNLOAD = re.compile( + r"Could not transfer artifact .*?(?:Connection reset|Connection timed out|" + r"Read timed out|Remote host terminated the handshake|" + r"status code: (?:408|429|500|502|503|504)(?:\D|$))", + re.IGNORECASE, +) + + +def run_maven(arguments, *, root=ROOT, command_prefix=None): + if command_prefix is None: + command_prefix = [str(root / ("mvnw.cmd" if os.name == "nt" else "mvnw"))] + for attempt in range(1, 4): + # Maven caches failed resolutions. Force a new resolution on retry. + options = ["-U"] if attempt > 1 and "-U" not in arguments else [] + command = [*command_prefix, *options, *arguments] + print("+ " + subprocess.list2cmdline(command), flush=True) + tail = deque(maxlen=200) + with subprocess.Popen( + command, + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + ) as process: + for line in process.stdout: + print(line, end="", flush=True) + tail.append(line) + returncode = process.wait() + if returncode == 0: + return 0 + errors = "".join(line for line in tail if "[ERROR]" in line) + if attempt == 3 or TRANSIENT_DOWNLOAD.search(errors) is None: + return returncode + print( + f"Transient Maven artifact download failure; retry {attempt}/2", flush=True + ) + time.sleep(attempt * 2) + raise AssertionError("Unreachable Maven retry state") + + +if __name__ == "__main__": + raise SystemExit(run_maven(sys.argv[1:])) diff --git a/packaging/scripts/verify_macos_runtime.py b/packaging/scripts/verify_macos_runtime.py new file mode 100644 index 000000000..22a833dfe --- /dev/null +++ b/packaging/scripts/verify_macos_runtime.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reject macOS packages that load external third-party dynamic libraries.""" + +import argparse +from pathlib import Path +import re +import subprocess + +LOAD_COMMANDS = { + "LC_LOAD_DYLIB", + "LC_LOAD_WEAK_DYLIB", + "LC_REEXPORT_DYLIB", + "LC_LAZY_LOAD_DYLIB", + "LC_LOAD_UPWARD_DYLIB", +} +MACHO_MAGICS = { + bytes.fromhex(value) + for value in ( + "feedface", + "cefaedfe", + "feedfacf", + "cffaedfe", + "cafebabe", + "bebafeca", + "cafebabf", + "bfbafeca", + ) +} + + +def parse_dependencies(output): + imports = set() + for block in re.split(r"(?m)^Load command \d+\s*$", output): + command = re.search(r"^\s*cmd (LC_\w+)\s*$", block, re.M) + if command is None or command[1] not in LOAD_COMMANDS: + continue + name = re.search(r"^\s*name (.+?) \(offset \d+\)\s*$", block, re.M) + if name is None: + raise ValueError("otool reported a dylib load command without a name") + imports.add(name[1]) + if not imports: + raise ValueError("otool did not report any dylib dependencies") + return imports + + +def validate_dependencies(binary, imports, prefix): + binary = Path(binary).resolve() + prefix = Path(prefix).resolve() + for name in sorted(imports): + if name.startswith(("/usr/lib/", "/System/Library/")): + continue + # BUNDLED codecs must not remain separate dylibs, even when installed + # on the runner. The only non-system dylib is TsFile itself. + if not re.fullmatch(r"libtsfile(?:\.[0-9]+)*\.dylib", Path(name).name): + raise ValueError( + f"{binary.name} needs an external third-party dylib: {name}" + ) + if name.startswith("@rpath/"): + suffix = name.removeprefix("@rpath/") + candidates = [prefix / "lib" / suffix, prefix / suffix] + elif name.startswith("@loader_path/"): + candidates = [binary.parent / name.removeprefix("@loader_path/")] + elif name.startswith("@executable_path/") and binary.name == "tsfile-cli": + candidates = [binary.parent / name.removeprefix("@executable_path/")] + elif Path(name).is_absolute(): + candidates = [Path(name)] + else: + candidates = [] + if not any( + path.is_file() and prefix in path.resolve().parents for path in candidates + ): + raise ValueError(f"{binary.name} needs an unshipped TsFile dylib: {name}") + + +def verify_prefix(prefix): + prefix = Path(prefix).resolve() + binaries = set() + for path in prefix.rglob("*"): + if path.is_file(): + with path.open("rb") as stream: + if stream.read(4) in MACHO_MAGICS: + binaries.add(path.resolve()) + if not binaries: + raise ValueError(f"No Mach-O binaries found under {prefix}") + for binary in sorted(binaries): + output = subprocess.run( + ["otool", "-l", str(binary)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout + imports = parse_dependencies(output) + validate_dependencies(binary, imports, prefix) + print( + f"{binary.name}: all dylib dependencies are macOS system libraries or packaged TsFile" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("prefix", type=Path) + verify_prefix(parser.parse_args().prefix) + + +if __name__ == "__main__": + main() diff --git a/packaging/scripts/verify_windows_runtime.py b/packaging/scripts/verify_windows_runtime.py new file mode 100644 index 000000000..79a7c90b4 --- /dev/null +++ b/packaging/scripts/verify_windows_runtime.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reject portable Windows binaries that need an unshipped DLL or dynamic CRT.""" + +import argparse +import re +import subprocess +from pathlib import Path + +# Windows 10 / Server 2022 inbox DLLs used by TsFile and the static CRT. This +# explicit list deliberately does not trust whatever happens to be on PATH or +# in the development runner's System32 (which also contains the VC redist). +SYSTEM_DLLS = { + "advapi32.dll", + "bcrypt.dll", + "crypt32.dll", + "kernel32.dll", + "ntdll.dll", + "ole32.dll", + "oleaut32.dll", + "rpcrt4.dll", + "secur32.dll", + "shell32.dll", + "shlwapi.dll", + "user32.dll", + "version.dll", + "ws2_32.dll", +} +DYNAMIC_CRT = re.compile(r"^(msvcp|msvcr|vcruntime|ucrtbase|api-ms-win-crt-)", re.I) + + +def parse_dependents(output: str) -> set[str]: + imports = set(re.findall(r"^\s+([A-Za-z0-9_.-]+\.dll)\s*$", output, re.M | re.I)) + if not imports: + raise ValueError("dumpbin did not report any DLL imports") + return {name.lower() for name in imports} + + +def validate_dependents(binary: str, imports: set[str], colocated: set[str]) -> None: + for name in sorted(imports): + if DYNAMIC_CRT.match(name): + raise ValueError( + f"{binary} imports {name}; the ZIP requires a static MSVC runtime" + ) + if ( + name not in SYSTEM_DLLS + and not name.startswith("api-ms-win-") + and name not in colocated + ): + raise ValueError(f"{binary} needs an unshipped non-system DLL: {name}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("prefix", type=Path) + args = parser.parse_args() + binaries = sorted( + path + for path in args.prefix.rglob("*") + if path.suffix.lower() in (".exe", ".dll") + ) + if not binaries: + parser.error("the prefix contains no Windows binaries") + for binary in binaries: + output = subprocess.run( + ["dumpbin", "/nologo", "/dependents", str(binary)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout + imports = parse_dependents(output) + colocated = { + path.name.lower() + for path in binary.parent.iterdir() + if path.suffix.lower() == ".dll" + } + validate_dependents(binary.name, imports, colocated) + print( + f"{binary.name}: static CRT; all {len(imports)} DLL dependencies are inbox or colocated" + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb new file mode 100644 index 000000000..f1be69aec --- /dev/null +++ b/packaging/tests/check_native_workflow.rb @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "yaml" +require "shellwords" +require "open3" + +path = ARGV.fetch(0, File.expand_path("../../.github/workflows/native-packages.yml", __dir__)) +workflow = YAML.load_file(path) +triggers = workflow["on"] || workflow[true] # Psych's YAML 1.1 boolean parsing +raise "Workflow must remain manual-only" unless triggers.keys == ["workflow_dispatch"] +raise "Workflow permissions must remain read-only" unless workflow["permissions"] == { "contents" => "read" } + +jobs = workflow.fetch("jobs") +raise "Workflow must be dedicated to release artifacts" unless workflow.fetch("name") == "Build native release artifacts" + +def run_text(job) + job.fetch("steps").filter_map { |step| step["run"] }.join("\n") +end + +%w[build-deb build-deb-arm64 build-rpm build-rpm-aarch64 build-windows build-windows-arm64].each do |name| + text = run_text(jobs.fetch(name)) + raise "#{name} must build C++ through Maven" unless text.include?("-Pwith-cpp package") + raise "#{name} must bundle third-party dependencies" unless text.include?("-Dtsfile.dependency.source=BUNDLED") + raise "#{name} must enable CPack through Maven" unless text.include?("-Denable.cpack=ON") + raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") + raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") + raise "#{name} must use an absolute SDK prefix" unless text.include?("$PWD/sdk/") || text.include?('Join-Path $PWD "sdk/') + unless name.start_with?("build-windows") + raise "#{name} must locate pkg-config without assuming lib" unless text.include?('dirname "$sdk_pkgconfig"') + end + raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") + raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") +end + +rpm_bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } +packages = Shellwords.split(rpm_bootstrap.fetch("run").gsub("\\\n", " ")) +raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") +raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") +raise "AlmaLinux 9 default repositories do not provide ninja-build" if packages.include?("ninja-build") +raise "RPM bootstrap must install make" unless packages.include?("make") +raise "RPM bootstrap must install Java" unless packages.include?("java-17-openjdk-devel") + +windows = run_text(jobs.fetch("build-windows")) +raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-Dtsfile.msvc.static.runtime=ON") && windows.include?("-Dtsfile.dependency.source=BUNDLED") +raise "Windows runtime check must be wired through Maven" unless windows.include?("CheckStaticMSVCRuntime.cmake") && windows.include?("-Dtsfile.project.include") +raise "Windows Maven properties must be quoted for PowerShell" unless windows.include?('"-Dcpp.toolchain=msvc"') && windows.include?('"-Dbuild.type=Release"') +raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 + +raise "Ubuntu ARM64 packages must use a native ARM runner" unless jobs.fetch("build-deb-arm64").fetch("runs-on") == "ubuntu-22.04-arm" +raise "AlmaLinux ARM64 packages must use a native ARM runner" unless jobs.fetch("build-rpm-aarch64").fetch("runs-on") == "ubuntu-24.04-arm" +raise "Windows ARM64 binaries must use the pinned native ARM runner" unless jobs.fetch("build-windows-arm64").fetch("runs-on") == "windows-11-vs2026-arm" +windows_arm = run_text(jobs.fetch("build-windows-arm64")) +raise "Windows ARM64 build must use its Maven toolchain profile" unless windows_arm.include?("-Dcpp.toolchain=msvc-arm64-native") +raise "Windows ARM64 build must put the native compiler first on PATH" unless windows_arm.include?("HostARM64") && windows_arm.include?("Get-Command cl.exe") +raise "Windows ARM64 consumer must use the same Ninja Multi-Config toolchain" unless windows_arm.include?('-G "Ninja Multi-Config"') +raise "Windows ARM64 ZIP must have an architecture-specific name" unless windows_arm.include?("windows-arm64") +raise "Windows ARM64 must not add signed MSIX/AppX packaging" if windows_arm.match?(/msix|appx|signtool/i) +root_pom = File.read(File.expand_path("../../pom.xml", __dir__)) +raise "Windows ARM64 Maven profile must use Ninja Multi-Config" unless root_pom.include?("cpp-msvc-arm64-native") && root_pom.include?("Ninja Multi-Config") + +%w[test-deb-arm64 test-rpm-aarch64].each do |name| + job = jobs.fetch(name) + raise "#{name} must run on a native ARM runner" unless job.fetch("runs-on").end_with?("-arm") + runs = run_text(job) + raise "#{name} must build and run an installed CMake consumer" unless runs.include?("cpp/cmake/tests/projects/InstalledConsumer") +end + +unix_python = jobs.fetch("build-python-unix") +raise "Unix wheels must use the prepared build identity" unless unix_python.fetch("needs") == "prepare" +unix_matrix = unix_python.fetch("strategy").fetch("matrix").fetch("include") +expected_unix_platforms = %w[manylinux_2_28-x86_64 manylinux_2_28-aarch64 macos12-x86_64 macos12-arm64] +raise "Unix wheel matrix must retain both architectures on Linux and macOS" unless unix_matrix.map { |row| row.fetch("name") }.sort == expected_unix_platforms.sort +raise "Linux ARM wheels must use a native ARM runner" unless unix_matrix.find { |row| row["name"] == "manylinux_2_28-aarch64" }.fetch("os") == "ubuntu-22.04-arm" +unix_build = unix_python.fetch("steps").find { |step| step["name"] == "Build wheels from one SDK per compatibility baseline" } +unix_env = unix_build.fetch("env") +%w[X86_64 AARCH64].each do |arch| + raise "Linux wheels must retain manylinux_2_28" unless unix_env.fetch("CIBW_MANYLINUX_#{arch}_IMAGE") == "manylinux_2_28" +end +%w[LINUX MACOS].each do |platform| + raise "#{platform} must build C++ once per wheel baseline" unless unix_env.fetch("CIBW_BEFORE_ALL_#{platform}").include?("build_python_sdk.sh") + raise "#{platform} wheels must point at the staged SDK" unless unix_env.fetch("CIBW_ENVIRONMENT_#{platform}").include?("TSFILE_CPP_BUILD=") +end +raise "macOS wheels must retain the 12.0 deployment target" unless unix_env.fetch("MACOSX_DEPLOYMENT_TARGET") == "12.0" +raise "manylinux must export its SDK archive to the host" unless unix_env.fetch("CIBW_ENVIRONMENT_LINUX").include?("TSFILE_PYTHON_SDK_EXPORT=\"/host") + +windows_python = jobs.fetch("build-python-windows") +raise "Windows wheels must reuse the native MSVC SDK" unless windows_python.fetch("needs") == "build-windows" +raise "Windows wheels must download the MSVC SDK" unless windows_python.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-windows-msvc-x86_64" } +windows_python_text = run_text(windows_python) +windows_wheel_build = windows_python.fetch("steps").find { |step| step["run"].to_s.include?("python -m cibuildwheel") } +raise "Windows wheel runtime must come from the SDK" if windows_wheel_build.fetch("env").key?("TSFILE_MSVC_STATIC_RUNTIME") +raise "Windows wheels must adapt installed DLLs into the Python SDK view" unless windows_python_text.include?('Copy-Item "$sdkRoot/bin/*.dll" "$sdkRoot/lib/"') +raise "Windows wheels must point setup.py at the SDK" unless windows_python_text.include?("TSFILE_CPP_BUILD=") +raise "Windows wheel job must not rebuild the C++ core" if windows_python_text.include?("-Pwith-cpp") +raise "Windows wheels must use MSVC rather than MinGW" unless windows_python.fetch("steps").any? { |step| step["uses"].to_s.start_with?("ilammy/msvc-dev-cmd@") } + +[unix_python, windows_python].each do |job| + build = job.fetch("steps").find { |step| step["run"].to_s.include?("python -m cibuildwheel") } + env = build.fetch("env") + raise "Wheel matrix must retain CPython 3.9 through 3.14" unless env.fetch("CIBW_BUILD").split == %w[cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*] + raise "Wheel tests must run on separate installation jobs" unless env.fetch("CIBW_TEST_SKIP") == "*" + raise "Wheel build jobs must not run package tests" if env.key?("CIBW_TEST_COMMAND") + raise "Every wheel build must validate runtime-only contents" unless run_text(job).include?("packaging/scripts/check_python_wheel.py --wheelhouse wheelhouse") +end +sdk_builder_path = File.expand_path("../scripts/build_python_sdk.sh", __dir__) +sdk_builder = File.read(sdk_builder_path) +%w[-Pwith-cpp -Dtsfile.version.sync.skip=true -Dtsfile.dependency.source=BUNDLED -Denable.lzma2=ON].each do |option| + raise "Python SDK builder must use #{option}" unless sdk_builder.include?(option) +end +raise "Python SDK builder must stage installed headers and libraries" unless sdk_builder.include?("cmake --install cpp/target/build") +raise "Python SDK export must support the container host mount" unless sdk_builder.include?("TSFILE_PYTHON_SDK_EXPORT") +raise "Windows SDKs must retain Python wheel codec coverage" unless windows.include?("-Denable.lzma2=ON") +output, status = Open3.capture2e("bash", "-n", sdk_builder_path) +raise "Invalid Python SDK shell script: #{output}" unless status.success? +raise "The old wheel workflow must be retired" if File.exist?(File.expand_path("../../.github/workflows/wheels.yml", __dir__)) + +homebrew = jobs.fetch("build-homebrew") +raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" +homebrew_text = run_text(homebrew) +raise "Homebrew must build the release Formula" unless homebrew_text.include?("packaging/homebrew/tsfile.rb.in") && !homebrew_text.include?("tsfile-dev") +expected_bottle_root = "https://packages.apache.org/artifactory/tsfile/homebrew/release/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles" +%w[build-homebrew merge-homebrew].each do |name| + raise "#{name} must use the release bottle root" unless jobs.fetch(name).fetch("env").fetch("BOTTLE_ROOT_URL") == expected_bottle_root +end +raise "Homebrew bottles and SDKs must validate dylib dependencies" unless homebrew_text.scan("python3 packaging/scripts/verify_macos_runtime.py").size == 2 +%w[tsfile.rb.in].each do |name| + formula = File.read(File.expand_path("../homebrew/#{name}", __dir__)) + raise "#{name} must use bundled dependencies" unless formula.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") + dependencies = formula.lines.grep(/^\s*depends_on/) + raise "#{name} must not install third-party runtime Formulae" unless dependencies.all? { |line| line.include?("=> :build") } + raise "#{name} must bundle zlib" if formula.include?("uses_from_macos") +end +raise "Homebrew job must stage a macOS SDK from the installed formula" unless homebrew_text.include?("brew --prefix apache/tsfile/tsfile") +raise "Homebrew job must archive the staged macOS SDK" unless homebrew_text.include?("tsfile-sdk-${{ matrix.name }}-$ARCHIVE_VERSION") +raise "Homebrew SDK must be verified from its own staged tree" unless homebrew_text.include?('sdk_root="$PWD/sdk/$sdk_name"') +raise "Homebrew SDK must be archived from the staging root" unless homebrew_text.include?('tar -C sdk -czf "sdk/$sdk_name.tar.gz" "$sdk_name"') +raise "Homebrew prefix capture must tolerate extra stdout lines" unless homebrew_text.include?("brew --prefix apache/tsfile/tsfile | tail -n 1") +raise "Homebrew SDK verification must provide pkg-config" unless homebrew_text.include?("brew install pkgconf") +raise "Homebrew SDK step must keep bash 3.2 compatibility" if homebrew_text.include?("set -euo pipefail") +raise "Homebrew job must upload a macOS SDK artifact" unless homebrew.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-${{ matrix.name }}" } +raise "Homebrew SDK must drop keg bookkeeping" unless homebrew_text.include?("INSTALL_RECEIPT.json") && homebrew_text.include?("sbom.spdx.json") +raise "Homebrew SDK must normalize the dylib identity" unless homebrew_text.include?("install_name_tool -id") +raise "Homebrew SDK must re-sign the normalized dylib" unless homebrew_text.include?("codesign --force --sign -") + +homebrew_merge = run_text(jobs.fetch("merge-homebrew")) +trust = homebrew_merge.index("brew trust apache/tsfile") +merge = homebrew_merge.index("brew bottle --merge") +raise "Homebrew merge must trust its temporary tap before loading the Formula" unless trust && merge && trust < merge + +%w[test-deb test-deb-arm64 test-rpm test-rpm-aarch64 build-windows build-windows-arm64].each do |name| + steps = jobs.fetch(name).fetch("steps") + raise "#{name} must check out the consumer fixture" unless steps.any? { |step| step["uses"].to_s.start_with?("actions/checkout@") } + runs = steps.filter_map { |step| step["run"] }.join("\n") + raise "#{name} must use the installed CMake consumer" unless runs.include?("-S cpp/cmake/tests/projects/InstalledConsumer") +end + +prepare_steps = jobs.fetch("prepare").fetch("steps") +prepare_text = run_text(jobs.fetch("prepare")) +%w[--build-date --run-number --run-attempt].each do |option| + raise "Release versions must not depend on a development build identity" if prepare_text.include?(option) +end +cli_data_upload = prepare_steps.find { |step| step["with"].to_h["name"] == "native-cli-test-data" }.fetch("with") +raise "CLI data must come from the current source checkout" unless cli_data_upload.fetch("path") == "python/tests/resources/*.tsfile" +raise "prepare must use one source checkout for code and CLI data" unless prepare_steps.count { |step| step["uses"].to_s.start_with?("actions/checkout@") } == 1 + +%w[test-deb test-deb-arm64 test-rpm test-rpm-aarch64 test-windows test-homebrew].each do |name| + job = jobs.fetch(name) + steps = job.fetch("steps") + raise "#{name} must download the Python CLI test data" unless steps.any? { |step| step["with"].to_h["name"] == "native-cli-test-data" } + raise "#{name} must run the installed CLI acceptance script" unless run_text(job).include?("packaging/tests/cli_package_smoke.py") + checkout = steps.find { |step| step["uses"].to_s.start_with?("actions/checkout@") }.fetch("with") + raise "#{name} must check out only its acceptance fixtures" if checkout.fetch("sparse-checkout").include?("cpp/src") || checkout.fetch("sparse-checkout").include?("cpp/tools") + raise "#{name} must not rebuild TsFile" if run_text(job).include?("-Pwith-cpp") +end + +%w[test-homebrew test-compatibility-sdk].each do |name| + checkout = jobs.fetch(name).fetch("steps").find { |step| step["uses"].to_s.start_with?("actions/checkout@") } + raise "#{name} must provide the macOS runtime checker for downloaded SDKs" unless checkout.fetch("with").fetch("sparse-checkout").include?("packaging/scripts/verify_macos_runtime.py") +end +raise "Installed Homebrew bottle must validate dylib dependencies" unless run_text(jobs.fetch("test-homebrew")).include?("python3 packaging/scripts/verify_macos_runtime.py") +windows_cli = jobs.fetch("test-windows") +raise "Windows CLI acceptance must wait for both ZIP architectures" unless windows_cli.fetch("needs").sort == %w[build-windows build-windows-arm64].sort +windows_cli_matrix = windows_cli.fetch("strategy").fetch("matrix").fetch("include") +raise "Windows CLI acceptance must cover x86_64 and ARM64" unless windows_cli_matrix.map { |row| row.fetch("name") }.sort == %w[windows-msvc-x86_64 windows-msvc-arm64].sort +raise "Windows CLI acceptance must install the ZIP" unless run_text(windows_cli).include?("Expand-Archive") +homebrew_cli = jobs.fetch("test-homebrew") +raise "Homebrew CLI acceptance must wait for the bottles" unless homebrew_cli.fetch("needs") == "build-homebrew" +raise "Homebrew CLI acceptance must cover both architectures" unless homebrew_cli.fetch("strategy").fetch("matrix").fetch("include").map { |row| row.fetch("name") }.sort == %w[macos-arm64 macos-x86_64].sort +raise "Homebrew CLI acceptance must install a bottle" unless run_text(homebrew_cli).include?("brew install --force-bottle") && run_text(homebrew_cli).include?("poured_from_bottle") +homebrew_install = run_text(homebrew_cli) +raise "Homebrew acceptance must install from its trusted tap" unless homebrew_install.include?("brew install --force-bottle apache/tsfile/tsfile") +raise "Homebrew acceptance must merge bottle checksums" unless homebrew_install.include?("brew bottle --merge --write --no-commit") +raise "Homebrew acceptance must seed the bottle download cache" unless homebrew_install.include?("brew --cache --force-bottle") && homebrew_install.include?('cp "${bottles[0]}" "$cached_bottle"') +raise "Homebrew acceptance must not bypass package path restrictions" if homebrew_install.include?("HOMEBREW_INTERNAL_ALLOW_PACKAGES_FROM_PATHS") +raise "Homebrew CLI acceptance must not build a bottle" if run_text(homebrew_cli).include?("--build-bottle") + +# Every native target runs the real API regressions and Go's own tests after +# downloading and installing a package on a separate runner/container. +%w[test-deb test-deb-arm64 test-rpm test-rpm-aarch64 test-windows test-homebrew].each do |name| + job = jobs.fetch(name) + text = run_text(job) + raise "#{name} must compile installed C++ API regressions" unless text.include?("packaging/tests/cpp_installed_tests.py") + raise "#{name} must run Go's installed-SDK tests" unless text.include?("packaging/tests/go_installed_tests.py") + raise "#{name} must set up Go" unless job.fetch("steps").any? { |step| step["uses"].to_s.start_with?("actions/setup-go@") } + checkout = job.fetch("steps").find { |step| step["uses"].to_s.start_with?("actions/checkout@") }.fetch("with").fetch("sparse-checkout") + %w[cpp/test go packaging/tests packaging/cmake/InstalledGTest].each do |required| + raise "#{name} must check out #{required}" unless checkout.lines.map(&:strip).include?(required) + end +end +windows_text = run_text(jobs.fetch("test-windows")) +raise "Windows cgo must use a verified MinGW compiler" unless windows_text.include?("LLVM-MinGW checksum mismatch") && windows_text.include?("--llvm-bin") +raise "Windows C++ tests must use the native MSVC environment" unless jobs.fetch("test-windows").fetch("steps").any? { |step| step["uses"].to_s.start_with?("ilammy/msvc-dev-cmd@") } + +sdk_archive_jobs = { + "test-deb" => "native-sdk-ubuntu22.04-amd64", + "test-deb-arm64" => "native-sdk-ubuntu22.04-arm64", + "test-rpm" => "native-sdk-almalinux9-x86_64", + "test-rpm-aarch64" => "native-sdk-almalinux9-aarch64", + "test-windows" => "native-sdk-${{ matrix.name }}", + "test-homebrew" => "native-sdk-${{ matrix.name }}", +} +sdk_archive_jobs.each do |name, artifact| + job = jobs.fetch(name) + raise "#{name} must download its standalone SDK" unless job.fetch("steps").any? { |step| step["with"].to_h["name"] == artifact } + raise "#{name} must test the extracted SDK" unless run_text(job).include?("packaging/tests/sdk_archive_tests.py") +end +compatibility_sdk = jobs.fetch("test-compatibility-sdk") +raise "Compatibility SDK tests must wait for wheel baseline builds" unless compatibility_sdk.fetch("needs").sort == %w[prepare build-python-unix].sort +raise "Compatibility SDK tests must cover every Unix baseline" unless compatibility_sdk.fetch("strategy").fetch("matrix").fetch("include").map { |row| row.fetch("name") }.sort == expected_unix_platforms.sort +raise "Compatibility SDK tests must exercise the extracted archive" unless run_text(compatibility_sdk).include?("packaging/tests/sdk_archive_tests.py") +raise "Compatibility SDK tests must not rebuild the product" if run_text(compatibility_sdk).match?(/with-cpp|build_python_sdk/) +compat_checkout = compatibility_sdk.fetch("steps").find { |step| step["uses"].to_s.start_with?("actions/checkout@") }.fetch("with").fetch("sparse-checkout") +raise "Compatibility SDK tests must exclude product sources" if compat_checkout.include?("cpp/src") || compat_checkout.include?("cpp/tools") || compat_checkout.include?("python/tsfile") + +expected_versions = %w[3.9 3.10 3.11 3.12 3.13 3.14] +%w[test-python-unix test-python-windows].each do |name| + job = jobs.fetch(name) + expected_build = name == "test-python-unix" ? "build-python-unix" : "build-python-windows" + raise "#{name} must wait for downloaded wheels" unless job.fetch("needs").sort == ["prepare", expected_build].sort + raise "#{name} must cover every Python ABI" unless job.fetch("strategy").fetch("matrix").fetch("python-version") == expected_versions + checkout = job.fetch("steps").find { |step| step["uses"].to_s.start_with?("actions/checkout@") }.fetch("with").fetch("sparse-checkout") + raise "#{name} must check out Python tests only" unless checkout.lines.map(&:strip).sort == %w[packaging/scripts/check_python_wheel.py packaging/tests/python_installed_tests.py python/tests].sort + raise "#{name} must run installed-wheel pytest" unless run_text(job).include?("packaging/tests/python_installed_tests.py") + report = job.fetch("steps").find { |step| step["name"] == "Upload Python installation test results" } + raise "#{name} must require the pytest report" unless report.fetch("with").fetch("if-no-files-found") == "error" + raise "#{name} must consume the expected Python package version" unless run_text(job).include?("needs.prepare.outputs.python_version") + raise "#{name} must download the built wheels" unless job.fetch("steps").any? { |step| step["with"].to_h["name"].to_s.start_with?("native-python-wheel-") } + raise "#{name} must not compile native code" if run_text(job).match?(/cibuildwheel|with-cpp|build_ext/) +end +python_platforms = jobs.fetch("test-python-unix").fetch("strategy").fetch("matrix").fetch("platform") +raise "Independent Unix pytest must cover the four wheel platforms" unless python_platforms.map { |row| row.fetch("name") }.sort == expected_unix_platforms.sort +raise "Windows pytest must run on Windows" unless jobs.fetch("test-python-windows").fetch("runs-on") == "windows-2022" +cpp_test_entry = File.read(File.expand_path("../../cpp/test/InstalledSDKTests.cmake", __dir__)) +raise "C++ regressions must link the installed CMake target" unless cpp_test_entry.include?("find_package(TsFile CONFIG REQUIRED)") && cpp_test_entry.include?("TsFile::tsfile") + +assemble = jobs.fetch("assemble") +needs = Array(assemble.fetch("needs")) +%w[test-deb test-deb-arm64 test-rpm test-rpm-aarch64 test-windows test-homebrew test-compatibility-sdk test-python-unix test-python-windows build-python-unix build-python-windows merge-homebrew build-windows build-windows-arm64].each do |name| + raise "assemble must depend on #{name}" unless needs.include?(name) +end +assemble_text = run_text(assemble) +%w[native-deb-ubuntu22.04-arm64 native-rpm-almalinux9-aarch64 native-windows-msvc-arm64 native-sdk-ubuntu22.04-amd64 native-sdk-ubuntu22.04-arm64 native-sdk-almalinux9-x86_64 native-sdk-almalinux9-aarch64 native-sdk-windows-msvc-x86_64 native-sdk-windows-msvc-arm64 native-sdk-macos-arm64 native-sdk-macos-x86_64 native-python-wheel-manylinux_2_28-x86_64 native-python-wheel-manylinux_2_28-aarch64 native-python-wheel-macos12-x86_64 native-python-wheel-macos12-arm64 native-python-wheel-windows-msvc-x86_64].each do |name| + raise "assemble must download #{name}" unless assemble.fetch("steps").any? { |step| step["with"].to_h["name"] == name } +end +expected_unix_platforms.each do |platform| + raise "assemble must download the #{platform} SDK" unless assemble.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-#{platform}" } +end +raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle") +raise "workflow must not hard-code a fork repository" if File.read(path).include?("ColinLeeo/tsfile") + +jobs.each_value do |job| + job.fetch("steps").each do |step| + artifact = step["with"].to_h["name"].to_s + raise "artifact names must not repeat the macOS matrix prefix" if artifact.include?("macos-macos") + next unless step["run"] && step.fetch("shell", "bash") == "bash" + output, status = Open3.capture2e("bash", "-n", stdin_data: step["run"]) + raise "Invalid shell in #{step['name']}: #{output}" unless status.success? + end +end +puts "PASS: manual/read-only Maven build hierarchy, artifact consumers, workflow contracts, and shell syntax" diff --git a/packaging/tests/cli_package_smoke.py b/packaging/tests/cli_package_smoke.py new file mode 100644 index 000000000..b89a64f2c --- /dev/null +++ b/packaging/tests/cli_package_smoke.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate an installed CLI with the current source checkout's Python resources.""" + +import argparse +import csv +import hashlib +import io +import math +from pathlib import Path +import shutil +import subprocess +import tempfile + +# Stable fixtures from python/tests/resources in the source checkout. +# Expected data facts come from python/tests/test_load_tsfile_from_iotdb.py. +CASES = ( + { + "file": "simple_tree.tsfile", + "sha256": "bd221e61be15ac57b6197622b3a302015b2bf25a6d3cab84f31e74d919bbec26", + "model": "tree", + "object": "root.ln.wf01.wt01", + "rows": 105, + "schema": {"status": ("FIELD", "BOOLEAN"), "temperature": ("FIELD", "FLOAT")}, + "nulls": {"status": 5, "temperature": 5}, + "sums": {"time": 15960}, + }, + { + "file": "simple_table_t1.tsfile", + "sha256": "a7b7736dbe414d68eff90e792db93669738817a58bd3d560801a3ec5a7b3edef", + "model": "table", + "object": "test", + "rows": 60, + "schema": { + "s0": ("TAG", "STRING"), + "s1": ("TAG", "STRING"), + "s2": ("FIELD", "INT64"), + "s3": ("FIELD", "FLOAT"), + "s4": ("FIELD", "STRING"), + "s5": ("FIELD", "TIMESTAMP"), + "s6": ("FIELD", "INT32"), + "s7": ("FIELD", "DOUBLE"), + "s8": ("FIELD", "DATE"), + "s9": ("FIELD", "TEXT"), + }, + "nulls": {"s2": 8, "s6": 8, "s9": 8}, + "sums": {"s5": 63270, "s6": 3342}, + }, + { + "file": "simple_table_t2.tsfile", + "sha256": "9bea5c683ef22ee01bf3c31c8a066bfdfe8db9cedee5a3360772547980508d3a", + "model": "table", + "object": "test", + "rows": 40, + "schema": { + "s0": ("TAG", "STRING"), + "s1": ("TAG", "STRING"), + "s2": ("FIELD", "INT64"), + }, + "nulls": {"s2": 5, "s6": 5, "s9": 5}, + "sums": {"time": 70404242080000, "s2": 36450, "s5": 41680, "s6": 1800}, + }, + { + "file": "table_with_time_column.tsfile", + "sha256": "22006050d75559466e653d61c4e8182f8f2531c667f2ca7e234b43648cc696fd", + "model": "table", + "object": "table2", + "rows": 25, + "schema": { + "id": ("TIME", "TIMESTAMP"), + "region_id": ("TAG", "STRING"), + "temperature": ("FIELD", "FLOAT"), + "humidity": ("FIELD", "DOUBLE"), + }, + "nulls": {"temperature": 0, "humidity": 0}, + "sums": {}, + }, +) + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def csv_rows(output): + reader = csv.DictReader(io.StringIO(output)) + require(reader.fieldnames, "CLI returned CSV without a header") + rows = list(reader) + require( + all(None not in row and None not in row.values() for row in rows), + "Malformed CSV rows", + ) + return rows + + +def column(rows, name): + # Tree outputs may qualify the measurement with its device path. + matches = [key for key in rows[0] if key == name or key.endswith("." + name)] + require(len(matches) == 1, f"Expected exactly one column for {name}: {matches}") + return [row[matches[0]] for row in rows] + + +def check_case(cli, resources, work, case, timeout): + source = resources / case["file"] + require(source.is_file(), f"Missing Python test resource: {source}") + require( + hashlib.sha256(source.read_bytes()).hexdigest() == case["sha256"], + f"Resource differs from the expected Python test fixture: {source}", + ) + print(f"Testing {case['file']} with {cli}", flush=True) + + def run(*arguments): + command = [str(cli), *map(str, arguments)] + result = subprocess.run( + command, + capture_output=True, + text=True, + encoding="utf-8", + timeout=timeout, + cwd=work, + ) + require( + result.returncode == 0, + f"Command failed ({result.returncode}): {command}\nstdout: {result.stdout}\nstderr: {result.stderr}", + ) + require(not result.stderr, f"Unexpected stderr from {command}: {result.stderr}") + return result.stdout + + objects = csv_rows(run("ls", "-f", "csv", source)) + require( + objects == [{"model": case["model"], "object": case["object"]}], + f"Unexpected objects: {objects}", + ) + metadata = csv_rows(run("meta", "-f", "csv", source)) + require( + len(metadata) == 1 and metadata[0]["model"] == case["model"], + f"Unexpected metadata: {metadata}", + ) + require( + int(metadata[0]["size_bytes"]) == source.stat().st_size + and int(metadata[0]["format_version"]) == 4, + f"Invalid file metadata: {metadata}", + ) + schema = { + row["column"]: (row["category"], row["data_type"]) + for row in csv_rows(run("schema", "-f", "csv", source)) + } + for name, expected in case["schema"].items(): + require( + schema.get(name) == expected, + f"Unexpected schema for {name}: {schema.get(name)}", + ) + + statistics = csv_rows(run("stats", "-f", "csv", source)) + require(statistics, "CLI returned no field statistics") + counts = csv_rows(run("count", "-f", "csv", source)) + scope = ("-d" if case["model"] == "tree" else "-t", case["object"]) + content = run("cat", *scope, "-f", "csv", source) + rows = csv_rows(content) + require(len(rows) == case["rows"], f"Expected {case['rows']} rows, got {len(rows)}") + for name, expected_nulls in case["nulls"].items(): + values = column(rows, name) + require( + values.count("\\N") == expected_nulls, f"Unexpected null count for {name}" + ) + matching = [row for row in counts if row["column"].split(".")[-1] == name] + require(len(matching) == 1, f"Missing/duplicate count for {name}") + require( + int(matching[0]["row_count"]) == case["rows"] + and int(matching[0]["null_count"]) == expected_nulls + and int(matching[0]["non_null_count"]) == case["rows"] - expected_nulls, + f"Incorrect count result for {name}: {matching}", + ) + for name, expected_sum in case["sums"].items(): + actual_sum = sum(int(value) for value in column(rows, name) if value != "\\N") + require(actual_sum == expected_sum, f"Unexpected sum for {name}: {actual_sum}") + if case["model"] == "tree": + values = column(rows, "status") + require( + values.count("true") == 50 and values.count("false") == 50, + "Incorrect BOOLEAN values", + ) + if case["object"] == "table2": + require( + column(rows, "region_id") == ["loc"] * 25, "Incorrect region TAG values" + ) + for name in ("temperature", "humidity"): + require( + math.isclose(sum(map(float, column(rows, name))), 2.5, rel_tol=1e-6), + f"Incorrect FLOAT/DOUBLE values for {name}", + ) + + require( + csv_rows(run("head", *scope, "-n", "3", "-f", "csv", source)) == rows[:3], + "head differs from first three cat rows", + ) + require( + csv_rows(run("cat", *scope, "--offset", "1", "-n", "2", "-f", "csv", source)) + == rows[1:3], + "Incorrect offset/limit", + ) + exported = work / (case["file"] + ".csv") + require( + run("export", *scope, "--type", "csv", "-o", exported, source) == "", + "export unexpectedly wrote to stdout", + ) + require( + exported.read_text(encoding="utf-8") == content, + "Exported CSV differs from cat output", + ) + require("TsFile" in run("sketch", source), "CLI returned no TsFile sketch") + print( + f"PASS: {case['file']}; {len(rows)} rows, schema/counts/values/query/export checked", + flush=True, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cli", required=True, help="Installed tsfile-cli executable or command" + ) + parser.add_argument( + "--resources", + required=True, + type=Path, + help="Python test .tsfile resources from the source checkout", + ) + parser.add_argument( + "--timeout", + type=float, + default=30, + help="Timeout in seconds per CLI invocation", + ) + args = parser.parse_args() + cli = shutil.which(args.cli) + require(cli is not None, f"CLI executable not found: {args.cli}") + require(args.timeout > 0, "Timeout must be positive") + version = subprocess.run( + [cli, "--version"], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + timeout=args.timeout, + ) + print(version.stdout.strip(), flush=True) + with tempfile.TemporaryDirectory(prefix="tsfile-cli-install-") as temporary: + for case in CASES: + check_case( + Path(cli).resolve(), + args.resources.resolve(), + Path(temporary), + case, + args.timeout, + ) + print("PASS: installed CLI validated with all four Python test resources") + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/cpp_installed_tests.py b/packaging/tests/cpp_installed_tests.py new file mode 100644 index 000000000..3850d30f8 --- /dev/null +++ b/packaging/tests/cpp_installed_tests.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Compile existing C++ regression tests against an installed SDK only.""" + +import argparse +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] + + +def package_config(prefix): + candidates = sorted( + { + *prefix.glob("lib*/cmake/TsFile/TsFileConfig.cmake"), + *prefix.glob("lib*/*/cmake/TsFile/TsFileConfig.cmake"), + } + ) + if len(candidates) != 1: + raise ValueError( + f"Expected one installed TsFileConfig.cmake under {prefix}: {candidates}" + ) + return candidates[0].parent + + +def run(command, **kwargs): + print("+ " + subprocess.list2cmdline([str(item) for item in command]), flush=True) + subprocess.run(command, check=True, timeout=900, **kwargs) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prefix", type=Path, required=True) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--gtest-prefix", type=Path) + arguments = parser.parse_args() + prefix = arguments.prefix.resolve() + build = arguments.build_dir.resolve() + config = package_config(prefix) + dependency_prefix = ( + arguments.gtest_prefix.resolve() + if arguments.gtest_prefix is not None + else build / "dependencies" + ) + options = ["-DCMAKE_BUILD_TYPE=Release"] + if sys.platform == "win32": + options += [ + "-G", + "Ninja Multi-Config", + "-DCMAKE_POLICY_DEFAULT_CMP0091=NEW", + "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded", + ] + elif sys.platform == "darwin": + sdk = subprocess.check_output(["xcrun", "--show-sdk-path"], text=True).strip() + options += [f"-DCMAKE_OSX_SYSROOT={sdk}"] + if arguments.gtest_prefix is None: + run( + [ + "cmake", + "-S", + ROOT / "packaging/cmake/InstalledGTest", + "-B", + build / "gtest", + *options, + f"-DCMAKE_INSTALL_PREFIX={dependency_prefix}", + ] + ) + run( + [ + "cmake", + "--build", + build / "gtest", + "--config", + "Release", + "--parallel", + "4", + ] + ) + run(["cmake", "--install", build / "gtest", "--config", "Release"]) + gtest_configs = list(dependency_prefix.glob("lib*/cmake/GTest/GTestConfig.cmake")) + if len(gtest_configs) != 1: + raise ValueError( + f"Expected one installed GoogleTest config under {dependency_prefix}" + ) + run( + [ + "cmake", + "-S", + ROOT / "cpp/test", + "-B", + build / "tests", + *options, + f"-DTsFile_DIR={config}", + f"-DGTest_DIR={gtest_configs[0].parent}", + f"-DCMAKE_PREFIX_PATH={dependency_prefix}", + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF", + "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF", + ] + ) + run(["cmake", "--build", build / "tests", "--config", "Release", "--parallel", "4"]) + environment = os.environ.copy() + if sys.platform == "win32": + environment["PATH"] = ( + str(prefix / "bin") + os.pathsep + environment.get("PATH", "") + ) + run( + [ + "ctest", + "--test-dir", + build / "tests", + "-C", + "Release", + "--output-on-failure", + "--no-tests=error", + "--output-junit", + build / "cpp-results.xml", + ], + env=environment, + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/go_installed_tests.py b/packaging/tests/go_installed_tests.py new file mode 100644 index 000000000..05667d9fb --- /dev/null +++ b/packaging/tests/go_installed_tests.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run Go's own tests against the manually installed native package.""" + +import argparse +import os +from pathlib import Path +import re +import shlex +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def exported_c_symbols(output): + symbols = re.findall( + r"^\s*Name: ([A-Za-z_][A-Za-z0-9_]*)\s*$", output, re.MULTILINE + ) + if not symbols: + raise ValueError("No C exports found in the installed TsFile DLL") + return sorted(set(symbols)) + + +def race_supported(goos, goarch): + return (goos, goarch) in { + ("linux", "amd64"), + ("linux", "arm64"), + ("darwin", "amd64"), + ("darwin", "arm64"), + ("windows", "amd64"), + } + + +def pkgconfig_file(prefix): + candidates = sorted( + { + *prefix.glob("lib*/pkgconfig/libtsfile.pc"), + *prefix.glob("lib*/*/pkgconfig/libtsfile.pc"), + } + ) + if len(candidates) != 1: + raise ValueError( + f"Expected one installed libtsfile.pc under {prefix}: {candidates}" + ) + return candidates[0] + + +def run(command, **kwargs): + print("+ " + subprocess.list2cmdline([str(item) for item in command]), flush=True) + subprocess.run(command, check=True, timeout=900, **kwargs) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prefix", type=Path, required=True) + parser.add_argument("--goarch", choices=("amd64", "arm64"), required=True) + parser.add_argument("--llvm-bin", type=Path) + parser.add_argument("--build-dir", type=Path, default=Path("go-test-output")) + arguments = parser.parse_args() + prefix = arguments.prefix.resolve() + header = prefix / "include/tsfile/cwrapper/tsfile_cwrapper.h" + if not header.is_file(): + raise ValueError(f"Missing installed C header: {header}") + goos, goarch = subprocess.check_output( + ["go", "env", "GOOS", "GOARCH"], text=True + ).split() + if goarch != arguments.goarch: + raise ValueError(f"Go architecture {goarch} does not match {arguments.goarch}") + environment = os.environ.copy() + environment["CGO_ENABLED"] = "1" + if goos == "windows": + if arguments.llvm_bin is None: + raise ValueError("Windows cgo requires --llvm-bin") + tools = arguments.llvm_bin.resolve() + triple = {"amd64": "x86_64", "arm64": "aarch64"}[goarch] + "-w64-mingw32" + environment["CC"] = (tools / f"{triple}-clang.exe").as_posix() + environment["CXX"] = (tools / f"{triple}-clang++.exe").as_posix() + dlls = [prefix / "bin" / name for name in ("tsfile.dll", "libtsfile.dll")] + dlls = [dll for dll in dlls if dll.is_file()] + if len(dlls) != 1: + raise ValueError(f"Expected one installed TsFile DLL: {dlls}") + dll = dlls[0] + output = subprocess.check_output( + [tools / "llvm-readobj.exe", "--coff-exports", dll], text=True + ) + symbols = exported_c_symbols(output) + imports = arguments.build_dir.resolve() / "imports" + imports.mkdir(parents=True, exist_ok=True) + definition = imports / "tsfile.def" + definition.write_text( + f"LIBRARY {dll.name}\nEXPORTS\n" + "\n".join(symbols) + "\n" + ) + machine = {"amd64": "i386:x86-64", "arm64": "arm64"}[goarch] + run( + [ + tools / "llvm-dlltool.exe", + "-m", + machine, + "-d", + definition, + "-l", + imports / "libtsfile.dll.a", + ] + ) + cflags = [ + f"-I{prefix.as_posix()}/include", + f"-I{prefix.as_posix()}/include/tsfile", + ] + ldflags = [f"-L{imports.as_posix()}", "-ltsfile"] + environment["PATH"] = os.pathsep.join( + [str(dll.parent), str(tools), environment.get("PATH", "")] + ) + else: + metadata = pkgconfig_file(prefix) + environment["PKG_CONFIG_PATH"] = str(metadata.parent) + cflags = shlex.split( + subprocess.check_output( + ["pkg-config", "--cflags", "libtsfile"], text=True, env=environment + ) + ) + ldflags = shlex.split( + subprocess.check_output( + ["pkg-config", "--libs", "libtsfile"], text=True, env=environment + ) + ) + ldflags.append(f"-Wl,-rpath,{metadata.parent.parent}") + environment["CGO_CFLAGS"] = shlex.join(cflags) + environment["CGO_LDFLAGS"] = shlex.join(ldflags) + print(f"Testing installed SDK {prefix} with Go {goos}/{goarch}", flush=True) + print(f"CGO_CFLAGS={environment['CGO_CFLAGS']}", flush=True) + print(f"CGO_LDFLAGS={environment['CGO_LDFLAGS']}", flush=True) + run(["go", "version"], env=environment) + for command in ( + ["go", "vet", "./..."], + ["go", "test", "-count=1", "-timeout=10m", "./..."], + ): + run(command, cwd=ROOT / "go", env=environment) + if race_supported(goos, goarch): + run( + ["go", "test", "-race", "-count=1", "-timeout=10m", "./..."], + cwd=ROOT / "go", + env=environment, + ) + else: + print( + f"NOT APPLICABLE: Go's race detector does not support {goos}/{goarch}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/python_installed_tests.py b/packaging/tests/python_installed_tests.py new file mode 100644 index 000000000..0858b1279 --- /dev/null +++ b/packaging/tests/python_installed_tests.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Install one downloaded wheel and run the existing pytest suite in isolation.""" + +import argparse +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def select_wheel(directory, version, python_tag): + pattern = re.compile( + rf"^tsfile-{re.escape(version)}-{python_tag}-{python_tag}-.+\.whl$" + ) + wheels = [ + wheel.resolve() + for wheel in directory.glob("*.whl") + if pattern.fullmatch(wheel.name) + ] + if len(wheels) != 1: + raise ValueError( + f"Expected one tsfile {version} {python_tag} wheel in {directory}: {wheels}" + ) + return wheels[0] + + +def run(command, **kwargs): + print("+ " + subprocess.list2cmdline([str(item) for item in command]), flush=True) + subprocess.run(command, check=True, **kwargs) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheelhouse", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--results", type=Path, required=True) + arguments = parser.parse_args() + python_tag = f"cp{sys.version_info.major}{sys.version_info.minor}" + wheel = select_wheel(arguments.wheelhouse, arguments.version, python_tag) + run( + [ + sys.executable, + str(ROOT / "packaging/scripts/check_python_wheel.py"), + str(wheel), + ], + timeout=60, + ) + # Keep the report outside the temporary cwd even when older Windows + # pathlib versions leave a nonexistent path relative after resolve(). + results = Path(os.path.abspath(arguments.results)) + results.parent.mkdir(parents=True, exist_ok=True) + tests = ROOT / "python/tests" + if not list(tests.glob("test_*.py")): + raise ValueError(f"No existing Python tests found in {tests}") + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--only-binary=:all:", + str(wheel), + "pytest", + "pytest-timeout", + ], + timeout=600, + ) + with tempfile.TemporaryDirectory(prefix="tsfile-installed-pytest-") as directory: + # -I excludes the checkout and PYTHONPATH from imports. The wheel must + # work with its bundled native library on this separate runner. + check = """import faulthandler +faulthandler.dump_traceback_later(30, repeat=True) +from importlib.metadata import version +from pathlib import Path +import sys +import tsfile +faulthandler.cancel_dump_traceback_later() +module = Path(tsfile.__file__).resolve() +checkout = Path(sys.argv[2]).resolve() +if checkout in module.parents: + raise RuntimeError(f'Imported source checkout: {module}') +if version('tsfile') != sys.argv[1]: + raise RuntimeError(f'Unexpected installed version: {version("tsfile")}') +print(f'Installed tsfile {version("tsfile")} from {module}', flush=True) +""" + run( + [ + sys.executable, + "-I", + "-u", + "-X", + "faulthandler", + "-c", + check, + arguments.version, + str(ROOT), + ], + cwd=directory, + timeout=60, + ) + run( + [ + sys.executable, + "-I", + "-u", + "-X", + "faulthandler", + "-m", + "pytest", + "--import-mode=importlib", + f"--rootdir={tests}", + "-o", + f"cache_dir={Path(directory) / 'pytest-cache'}", + "-ra", + "--timeout=120", + "--timeout-method=thread", + "-o", + "faulthandler_timeout=60", + f"--junitxml={results}", + str(tests), + ], + cwd=directory, + timeout=1800, + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/python_wheel_smoke.py b/packaging/tests/python_wheel_smoke.py new file mode 100644 index 000000000..cf7ffc154 --- /dev/null +++ b/packaging/tests/python_wheel_smoke.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Smoke-test an installed wheel, outside the Python source directory.""" + +from importlib.metadata import version +from pathlib import Path +import tempfile + +import tsfile +from tsfile import ( + ColumnCategory, + ColumnSchema, + Field, + RowRecord, + TableSchema, + Tablet, + TimeseriesSchema, + TSDataType, + TsFileReader, + TsFileTableWriter, + TsFileWriter, +) + + +def main(): + repository = Path(__file__).resolve().parents[2] + module = Path(tsfile.__file__).resolve() + if repository / "python" in module.parents: + raise RuntimeError( + f"Imported the source tree instead of the installed wheel: {module}" + ) + + with tempfile.TemporaryDirectory(prefix="tsfile-wheel-smoke-") as temporary: + table_path = str(Path(temporary) / "table.tsfile") + schema = TableSchema( + "sensors", + [ + ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("value", TSDataType.INT64, ColumnCategory.FIELD), + ], + ) + with TsFileTableWriter(table_path, schema) as writer: + tablet = Tablet(["site", "value"], [TSDataType.STRING, TSDataType.INT64], 3) + for row, value in enumerate([10, 20, 30]): + tablet.add_timestamp(row, row) + tablet.add_value_by_name("site", row, "north") + tablet.add_value_by_name("value", row, value) + writer.write_table(tablet) + + with TsFileReader(table_path) as reader: + with reader.query_table("sensors", ["site", "value"], 0, 2) as result: + values = [] + while result.next(): + assert result.get_value_by_name("site") == "north" + values.append(result.get_value_by_name("value")) + assert values == [10, 20, 30], values + + tree_path = str(Path(temporary) / "tree.tsfile") + with TsFileWriter(tree_path) as writer: + writer.register_timeseries( + "root.device", TimeseriesSchema("value", TSDataType.INT64) + ) + for row, value in enumerate([40, 50]): + writer.write_row_record( + RowRecord( + "root.device", row, [Field("value", value, TSDataType.INT64)] + ) + ) + with TsFileReader(tree_path) as reader: + with reader.query_timeseries("root.device", ["value"], 0, 1) as result: + values = [] + while result.next(): + values.append(result.get_value_by_index(2)) + assert values == [40, 50], values + + print( + f"PASS: installed tsfile {version('tsfile')} from {module}; table/tree read-write" + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/sdk_archive_tests.py b/packaging/tests/sdk_archive_tests.py new file mode 100644 index 000000000..3db636f90 --- /dev/null +++ b/packaging/tests/sdk_archive_tests.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Download-contract tests for a relocated SDK archive, without product sources.""" + +import argparse +import json +from pathlib import Path, PurePosixPath +import subprocess +import sys +import tarfile + +ROOT = Path(__file__).resolve().parents[2] + + +def extract_sdk(archive_path, destination): + destination = destination.resolve() + if destination.exists() and any(destination.iterdir()): + raise ValueError(f"SDK extraction directory must be empty: {destination}") + destination.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.name + or ":" in member.name + ): + raise ValueError(f"Unsafe SDK archive path: {member.name}") + if not ( + member.isfile() or member.isdir() or member.issym() or member.islnk() + ): + raise ValueError(f"Unsupported SDK archive entry: {member.name}") + if member.issym() or member.islnk(): + if "\\" in member.linkname or ":" in member.linkname: + raise ValueError(f"Unsafe SDK archive link: {member.name}") + base = ( + (destination / member.name).parent + if member.issym() + else destination + ) + target = (base / member.linkname).resolve() + if target != destination and destination not in target.parents: + raise ValueError( + f"SDK archive link escapes extraction directory: {member.name}" + ) + if hasattr(tarfile, "data_filter"): + archive.extractall(destination, members=members, filter="data") + else: + archive.extractall(destination, members=members) + headers = list(destination.rglob("include/tsfile/cwrapper/tsfile_cwrapper.h")) + if len(headers) != 1: + raise ValueError(f"Expected one SDK installation prefix in {archive_path}") + return headers[0].parents[3] + + +def run(command): + print( + "+ " + subprocess.list2cmdline([str(item) for item in command]), + flush=True, + ) + subprocess.run(command, check=True, timeout=1800) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archives", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--resources", type=Path, required=True) + parser.add_argument("--goarch", choices=("amd64", "arm64"), required=True) + parser.add_argument("--llvm-bin", type=Path) + parser.add_argument("--gtest-prefix", type=Path) + arguments = parser.parse_args() + archives = list(arguments.archives.glob("*.tar.gz")) + if len(archives) != 1: + raise ValueError(f"Expected one downloaded SDK archive: {archives}") + prefix = extract_sdk(archives[0], arguments.destination) + cli = ( + prefix / "bin" / ("tsfile-cli.exe" if sys.platform == "win32" else "tsfile-cli") + ) + if not cli.is_file(): + raise ValueError(f"Missing CLI in SDK archive: {cli}") + metadata = json.loads( + (prefix / "share/tsfile/tsfile-build-config.json").read_text() + ) + if metadata.get("schema_version") != 1 or not metadata.get("runtime_library"): + raise ValueError("Missing shared-library build configuration in SDK") + if sys.platform == "win32" and metadata.get("msvc_runtime") != "MultiThreaded": + raise ValueError("Portable Windows SDK must record its static Release CRT") + print( + f"Testing downloaded SDK {archives[0]} relocated to {prefix}", + flush=True, + ) + if sys.platform == "darwin": + run( + [ + sys.executable, + ROOT / "packaging/scripts/verify_macos_runtime.py", + prefix, + ] + ) + run( + [ + sys.executable, + ROOT / "packaging/tests/cli_package_smoke.py", + "--cli", + cli, + "--resources", + arguments.resources, + ] + ) + cpp = [ + sys.executable, + ROOT / "packaging/tests/cpp_installed_tests.py", + "--prefix", + prefix, + "--build-dir", + arguments.build_dir, + ] + if arguments.gtest_prefix is not None: + cpp += ["--gtest-prefix", arguments.gtest_prefix] + run(cpp) + go = [ + sys.executable, + ROOT / "packaging/tests/go_installed_tests.py", + "--prefix", + prefix, + "--goarch", + arguments.goarch, + "--build-dir", + arguments.build_dir / "go", + ] + if arguments.llvm_bin is not None: + go += ["--llvm-bin", arguments.llvm_bin] + run(go) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py new file mode 100644 index 000000000..2dab920ef --- /dev/null +++ b/packaging/tests/test_assemble_native_packages.py @@ -0,0 +1,591 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/assemble_native_packages.py" +VERSIONS = { + "archive_version": "2.5.0", + "homebrew_version": "2.5.0", + "logical_version": "2.5.0", +} +SOURCE = { + "commit": "abcdef1234567890abcdef1234567890abcdef12", + "repository": "apache/tsfile", +} + + +PYTHON_TAGS = ("cp39", "cp310", "cp311", "cp312", "cp313", "cp314") +WHEEL_VERSION = "2.5.0" +WHEEL_PLATFORMS = { + "manylinux_2_28-x86_64": "manylinux_2_28_x86_64", + "manylinux_2_28-aarch64": "manylinux_2_28_aarch64", + "macos12-x86_64": "macosx_12_0_x86_64", + "macos12-arm64": "macosx_12_0_arm64", + "windows-msvc-x86_64": "win_amd64", +} +PYTHON_SDK_PLATFORMS = tuple( + platform for platform in WHEEL_PLATFORMS if platform != "windows-msvc-x86_64" +) + + +def wheel_filename(platform, tag="cp311", version=WHEEL_VERSION): + return f"tsfile-{version}-{tag}-{tag}-{WHEEL_PLATFORMS[platform]}.whl" + + +def load_assembler_module(): + if not MODULE_PATH.is_file(): + return None + spec = importlib.util.spec_from_file_location( + "assemble_native_packages", MODULE_PATH + ) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load native package assembler from {MODULE_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AssembleNativePackagesTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_assembler_module() + + def require_module(self): + self.assertIsNotNone( + self.module, "native package assembler must exist for final bundle assembly" + ) + return self.module + + def write_fixture(self, directory): + files = { + "deb/tsfile_2.5.0-1_amd64.deb": b"deb artifact\n", + "deb/tsfile_2.5.0-1_arm64.deb": b"arm64 deb artifact\n", + "rpm/tsfile-2.5.0-1.el9.x86_64.rpm": b"rpm artifact\n", + "rpm/tsfile-2.5.0-1.el9.aarch64.rpm": b"aarch64 rpm artifact\n", + "homebrew/Formula/tsfile.rb": b"formula artifact\n", + "homebrew/bottles/tsfile.bottle.tar.gz": b"bottle artifact\n", + "homebrew/bottles/tsfile.bottle.json": b"bottle metadata\n", + "windows/tsfile-2.5.0-windows-x86_64.zip": b"windows artifact\n", + "windows/tsfile-2.5.0-windows-arm64.zip": b"windows arm64 artifact\n", + "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0.tar.gz": b"sdk ubuntu\n", + "sdk/ubuntu22.04-arm64/tsfile-sdk-ubuntu22.04-arm64-2.5.0.tar.gz": b"sdk ubuntu arm64\n", + "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0.tar.gz": b"sdk almalinux\n", + "sdk/almalinux9-aarch64/tsfile-sdk-almalinux9-aarch64-2.5.0.tar.gz": b"sdk almalinux arm64\n", + "sdk/macos-arm64/tsfile-sdk-macos-arm64-2.5.0.tar.gz": b"sdk macos arm64\n", + "sdk/macos-x86_64/tsfile-sdk-macos-x86_64-2.5.0.tar.gz": b"sdk macos x86_64\n", + "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0.tar.gz": b"sdk windows\n", + "sdk/windows-msvc-arm64/tsfile-sdk-windows-msvc-arm64-2.5.0.tar.gz": b"sdk windows arm64\n", + } + for platform in PYTHON_SDK_PLATFORMS: + files[ + f"sdk/{platform}/tsfile-sdk-{platform}-{VERSIONS['archive_version']}.tar.gz" + ] = b"Python compatibility SDK\n" + for platform in WHEEL_PLATFORMS: + for tag in PYTHON_TAGS: + files[f"python/{platform}/{wheel_filename(platform, tag)}"] = ( + b"Python wheel\n" + ) + for relative_path, contents in files.items(): + path = directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + return files + + def test_assembles_publishable_bundle_with_literal_metadata(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + output_directory = Path(temporary_directory) / "bundle" + self.write_fixture(input_directory) + + manifest = module.assemble( + input_directory, output_directory, VERSIONS, SOURCE + ) + + expected_paths = [ + "deb/ubuntu22.04-amd64/tsfile_2.5.0-1_amd64.deb", + "deb/ubuntu22.04-arm64/tsfile_2.5.0-1_arm64.deb", + "homebrew/Formula/tsfile.rb", + "homebrew/bottles/tsfile.bottle.json", + "homebrew/bottles/tsfile.bottle.tar.gz", + "rpm/almalinux9-aarch64/tsfile-2.5.0-1.el9.aarch64.rpm", + "rpm/almalinux9-x86_64/tsfile-2.5.0-1.el9.x86_64.rpm", + "sdk/almalinux9-aarch64/tsfile-sdk-almalinux9-aarch64-2.5.0.tar.gz", + "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0.tar.gz", + "sdk/macos-arm64/tsfile-sdk-macos-arm64-2.5.0.tar.gz", + "sdk/macos-x86_64/tsfile-sdk-macos-x86_64-2.5.0.tar.gz", + "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0.tar.gz", + "sdk/ubuntu22.04-arm64/tsfile-sdk-ubuntu22.04-arm64-2.5.0.tar.gz", + "sdk/windows-msvc-arm64/tsfile-sdk-windows-msvc-arm64-2.5.0.tar.gz", + "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0.tar.gz", + "windows/tsfile-2.5.0-windows-arm64.zip", + "windows/tsfile-2.5.0-windows-x86_64.zip", + ] + expected_paths.extend( + f"sdk/{platform}/tsfile-sdk-{platform}-{VERSIONS['archive_version']}.tar.gz" + for platform in PYTHON_SDK_PLATFORMS + ) + expected_paths.extend( + f"python/wheels/{platform}/{wheel_filename(platform, tag)}" + for platform in WHEEL_PLATFORMS + for tag in PYTHON_TAGS + ) + expected_paths.sort() + self.assertEqual( + sorted( + path.relative_to(output_directory).as_posix() + for path in output_directory.rglob("*") + if path.is_file() + and path.name not in {"SHA256SUMS", "manifest.json"} + ), + expected_paths, + ) + self.assertEqual( + json.loads((output_directory / "manifest.json").read_text()), manifest + ) + self.assertEqual(manifest["source"], SOURCE) + self.assertEqual(manifest["versions"], VERSIONS) + self.assertEqual( + {artifact["family"] for artifact in manifest["artifacts"]}, + {"deb", "homebrew", "python", "rpm", "sdk", "windows"}, + ) + self.assertEqual( + {artifact["platform"] for artifact in manifest["artifacts"]}, + { + "almalinux9-aarch64", + "almalinux9-x86_64", + "homebrew", + "macos-arm64", + "macos-x86_64", + "ubuntu22.04-amd64", + "ubuntu22.04-arm64", + "manylinux_2_28-x86_64", + "manylinux_2_28-aarch64", + "macos12-x86_64", + "macos12-arm64", + "windows-msvc-arm64", + "windows-msvc-x86_64", + }, + ) + wheels = [ + artifact + for artifact in manifest["artifacts"] + if artifact["family"] == "python" + ] + self.assertEqual(len(wheels), 30) + for wheel in wheels: + self.assertEqual(wheel["sdkPlatform"], wheel["platform"]) + self.assertEqual(wheel["wheelVersion"], WHEEL_VERSION) + self.assertEqual(wheel["abiTag"], wheel["pythonTag"]) + for key in ("targetRepository", "targetPath", "properties"): + self.assertNotIn(key, wheel) + for artifact in manifest["artifacts"]: + if artifact["family"] == "python": + continue + self.assertIn( + artifact["targetRepository"], + {"tsfile", "tsfile-debian", "tsfile-rpm"}, + ) + self.assertIn("release/", artifact["targetPath"]) + if artifact["family"] == "deb": + self.assertEqual( + artifact["properties"]["deb.component"], ["release"] + ) + artifact_platforms = { + artifact["filename"]: artifact["platform"] + for artifact in manifest["artifacts"] + } + self.assertEqual( + artifact_platforms["tsfile_2.5.0-1_arm64.deb"], + "ubuntu22.04-arm64", + ) + self.assertEqual( + artifact_platforms["tsfile-2.5.0-1.el9.aarch64.rpm"], + "almalinux9-aarch64", + ) + self.assertEqual( + artifact_platforms["tsfile-2.5.0-windows-arm64.zip"], + "windows-msvc-arm64", + ) + deb_arm64 = next( + artifact + for artifact in manifest["artifacts"] + if artifact["filename"] == "tsfile_2.5.0-1_arm64.deb" + ) + self.assertEqual(deb_arm64["properties"]["deb.architecture"], ["arm64"]) + rpm_arm64 = next( + artifact + for artifact in manifest["artifacts"] + if artifact["filename"] == "tsfile-2.5.0-1.el9.aarch64.rpm" + ) + self.assertEqual( + rpm_arm64["targetPath"], + "release/el9/aarch64/tsfile-2.5.0-1.el9.aarch64.rpm", + ) + for artifact in manifest["artifacts"]: + output_path = output_directory / artifact["path"] + input_path = next( + path + for path in input_directory.rglob(artifact["filename"]) + if path.is_file() + ) + self.assertEqual(artifact["size"], input_path.stat().st_size) + self.assertEqual( + artifact["sha256"], + hashlib.sha256(input_path.read_bytes()).hexdigest(), + ) + + def test_requires_every_python_abi_on_every_platform(self): + module = self.require_module() + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + for platform in WHEEL_PLATFORMS: + for tag in PYTHON_TAGS: + with self.subTest(platform=platform, tag=tag): + source = root / platform / tag + self.write_fixture(source) + ( + source / "python" / platform / wheel_filename(platform, tag) + ).unlink() + with self.assertRaisesRegex( + ValueError, "missing Python wheel ABIs" + ): + module.assemble(source, root / "bundle", VERSIONS, SOURCE) + + def test_rejects_incompatible_wheel_tags_and_duplicate_abis(self): + module = self.require_module() + invalid = [ + ("manylinux_2_28-x86_64", "linux_x86_64", "incompatible Python wheel tags"), + ( + "manylinux_2_28-aarch64", + "manylinux_2_28_x86_64", + "incompatible Python wheel tags", + ), + ("macos12-arm64", "macosx_13_0_arm64", "incompatible Python wheel tags"), + ("windows-msvc-x86_64", "win_arm64", "incompatible Python wheel tags"), + ] + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + for index, (platform, tag, error) in enumerate(invalid): + with self.subTest(platform=platform, tag=tag): + source = root / f"invalid-{index}" + self.write_fixture(source) + original = source / "python" / platform / wheel_filename(platform) + original.rename( + original.with_name( + f"tsfile-{WHEEL_VERSION}-cp311-cp311-{tag}.whl" + ) + ) + with self.assertRaisesRegex(ValueError, error): + module.assemble(source, root / "bundle", VERSIONS, SOURCE) + source = root / "duplicate" + self.write_fixture(source) + wheel = ( + source + / "python/manylinux_2_28-x86_64" + / wheel_filename("manylinux_2_28-x86_64", version="9.0.0") + ) + wheel.write_bytes(b"duplicate ABI with a different version") + with self.assertRaisesRegex(ValueError, "duplicate Python wheel ABI"): + module.assemble(source, root / "bundle", VERSIONS, SOURCE) + + def test_rejects_different_versions_across_wheel_platforms(self): + module = self.require_module() + with tempfile.TemporaryDirectory() as temporary_directory: + source = Path(temporary_directory) / "input" + self.write_fixture(source) + original = ( + source + / "python/windows-msvc-x86_64" + / wheel_filename("windows-msvc-x86_64") + ) + original.rename( + original.with_name( + wheel_filename("windows-msvc-x86_64", version="9.0.0") + ) + ) + with self.assertRaisesRegex(ValueError, "must all have the same version"): + module.assemble( + source, Path(temporary_directory) / "bundle", VERSIONS, SOURCE + ) + + def test_release_bundle_requires_matching_wheel_version(self): + module = self.require_module() + release_versions = { + "base_version": "2.5.0", + "logical_version": "2.5.0", + "archive_version": "2.5.0", + "homebrew_version": "2.5.0", + } + for wheel_version in ("2.5.0", "2.3.2.dev0"): + with self.subTest( + wheel_version=wheel_version + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "input" + output = root / "bundle" + self.write_fixture(source) + for wheel in source.rglob("*.whl"): + wheel.rename( + wheel.with_name( + wheel.name.replace(WHEEL_VERSION, wheel_version) + ) + ) + if wheel_version == "2.5.0": + manifest = module.assemble(source, output, release_versions, SOURCE) + self.assertEqual(manifest["versions"], release_versions) + module.verify_bundle(output) + else: + with self.assertRaisesRegex( + ValueError, "must match release version 2.5.0" + ): + module.assemble(source, output, release_versions, SOURCE) + self.assertFalse(output.exists()) + + def test_rejects_development_or_mismatched_release_versions(self): + module = self.require_module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "input" + self.write_fixture(source) + for field, value in ( + ("archive_version", "2.5.0.dev"), + ("homebrew_version", "2.5.0.dev0"), + ("homebrew_version", "2.6.0"), + ("logical_version", "2.5.0.dev0+build"), + ): + with self.subTest(field=field, value=value): + output = root / "bundle" + with self.assertRaisesRegex(ValueError, "release version"): + module.assemble( + source, output, {**VERSIONS, field: value}, SOURCE + ) + self.assertFalse(output.exists()) + + def test_rejects_bundles_missing_arm_architectures(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + missing_artifacts = ( + ( + "deb/tsfile_2.5.0-1_arm64.deb", + "missing DEB inputs for: ubuntu22.04-arm64", + ), + ( + "rpm/tsfile-2.5.0-1.el9.aarch64.rpm", + "missing RPM inputs for: almalinux9-aarch64", + ), + ( + "windows/tsfile-2.5.0-windows-arm64.zip", + "missing WINDOWS inputs for: windows-msvc-arm64", + ), + ( + "sdk/ubuntu22.04-arm64/tsfile-sdk-ubuntu22.04-arm64-2.5.0.tar.gz", + "missing SDK inputs for: ubuntu22.04-arm64", + ), + ( + "sdk/almalinux9-aarch64/tsfile-sdk-almalinux9-aarch64-2.5.0.tar.gz", + "missing SDK inputs for: almalinux9-aarch64", + ), + ( + "sdk/windows-msvc-arm64/tsfile-sdk-windows-msvc-arm64-2.5.0.tar.gz", + "missing SDK inputs for: windows-msvc-arm64", + ), + ) + for index, (missing_path, error_message) in enumerate(missing_artifacts): + with self.subTest(missing=missing_path): + input_directory = Path(temporary_directory) / f"input-{index}" + self.write_fixture(input_directory) + (input_directory / missing_path).unlink() + + with self.assertRaisesRegex(ValueError, error_message): + module.assemble( + input_directory, + Path(temporary_directory) / f"incomplete-{index}", + VERSIONS, + SOURCE, + ) + + def test_verifies_assembler_checksum_lines_sorted_by_path(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + output_directory = Path(temporary_directory) / "bundle" + self.write_fixture(input_directory) + module.assemble(input_directory, output_directory, VERSIONS, SOURCE) + + checksum_lines = ( + (output_directory / "SHA256SUMS") + .read_text(encoding="utf-8") + .splitlines() + ) + self.assertNotEqual( + checksum_lines, + sorted(checksum_lines), + "fixture must distinguish path ordering from hash ordering", + ) + self.assertIsNone(module.verify_bundle(output_directory)) + + def test_rejects_unsafe_versions_and_empty_source_identity(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + for index, (field, value) in enumerate( + ( + ("archive_version", "../../latest"), + ("homebrew_version", "../latest"), + ("archive_version", ""), + ("homebrew_version", "version\\latest"), + ) + ): + with self.subTest(field=field, value=value): + versions = {**VERSIONS, field: value} + with self.assertRaisesRegex(ValueError, "safe path component"): + module.assemble( + input_directory, + Path(temporary_directory) / f"invalid-{index}", + versions, + SOURCE, + ) + for index, (field, value) in enumerate( + (("commit", ""), ("repository", " ")) + ): + with self.subTest(field=field, value=value): + source = {**SOURCE, field: value} + with self.assertRaisesRegex(ValueError, "nonempty"): + module.assemble( + input_directory, + Path(temporary_directory) / f"empty-{index}", + VERSIONS, + source, + ) + + def test_rejects_unknown_files(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + (input_directory / "deb/README.txt").write_text( + "not a package\n", encoding="utf-8" + ) + + with self.assertRaisesRegex(ValueError, "undeclared file type"): + module.assemble( + input_directory, + Path(temporary_directory) / "bundle", + VERSIONS, + SOURCE, + ) + + def test_rejects_empty_and_symbolic_link_inputs(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + (input_directory / "rpm/tsfile-2.5.0-1.el9.x86_64.rpm").write_bytes(b"") + with self.assertRaisesRegex(ValueError, "empty file"): + module.assemble( + input_directory, + Path(temporary_directory) / "empty", + VERSIONS, + SOURCE, + ) + + self.write_fixture(input_directory) + os.symlink( + input_directory / "deb/tsfile_2.5.0-1_amd64.deb", + input_directory / "deb/linked.deb", + ) + with self.assertRaisesRegex(ValueError, "symbolic link"): + module.assemble( + input_directory, + Path(temporary_directory) / "linked", + VERSIONS, + SOURCE, + ) + + def test_rejects_duplicate_destination_names(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + duplicate = input_directory / "deb/duplicate/tsfile_2.5.0-1_amd64.deb" + duplicate.parent.mkdir() + duplicate.write_bytes(b"another deb artifact\n") + + with self.assertRaisesRegex(ValueError, "duplicate target path"): + module.assemble( + input_directory, + Path(temporary_directory) / "bundle", + VERSIONS, + SOURCE, + ) + + def test_cli_assembles_from_versions_json(self): + self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = Path(temporary_directory) + input_directory = temporary_path / "input" + self.write_fixture(input_directory) + versions_path = temporary_path / "versions.json" + versions_path.write_text(json.dumps(VERSIONS), encoding="utf-8") + output_directory = temporary_path / "bundle" + + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--input-dir", + str(input_directory), + "--output-dir", + str(output_directory), + "--versions-json", + str(versions_path), + "--source-repository", + SOURCE["repository"], + "--source-commit", + SOURCE["commit"], + ], + check=True, + ) + manifest_path = output_directory / "manifest.json" + self.assertTrue(manifest_path.is_file(), "CLI must write manifest.json") + self.assertEqual( + json.loads(manifest_path.read_text(encoding="utf-8"))["source"], SOURCE + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_dependency_download.py b/packaging/tests/test_dependency_download.py new file mode 100644 index 000000000..453872476 --- /dev/null +++ b/packaging/tests/test_dependency_download.py @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest + +ROOT = Path(__file__).resolve().parents[2] +HELPER = ROOT / "cpp/cmake/DownloadWithRetry.cmake" + + +class DependencyDownloadTest(unittest.TestCase): + def download(self, responses): + state = {"requests": 0} + payload = b"verified upstream archive payload" + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + index = state["requests"] + state["requests"] += 1 + status = responses[min(index, len(responses) - 1)] + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + destination = root / "archive.part" + status_file = root / "status.txt" + script = root / "download.cmake" + script.write_text( + "cmake_minimum_required(VERSION 3.11)\n" + f'include("{HELPER.as_posix()}")\n' + f'tsfile_download_with_retry("http://127.0.0.1:{server.server_port}/archive" ' + f'"{destination.as_posix()}" status)\n' + f'file(WRITE "{status_file.as_posix()}" "${{status}}")\n' + ) + subprocess.run( + ["cmake", "-P", str(script)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=30, + ) + status = int(status_file.read_text().split(";", 1)[0]) + contents = ( + destination.read_bytes() if destination.exists() else None + ) + return status, state["requests"], contents, payload + finally: + server.shutdown() + worker.join(timeout=5) + + def test_transient_server_failures_retry_then_succeed(self): + status, requests, contents, payload = self.download([503, 503, 200]) + self.assertEqual(status, 0) + self.assertEqual(requests, 3) + self.assertEqual(contents, payload) + + def test_repeated_failures_stop_after_three_attempts_and_remove_partial_file(self): + status, requests, contents, _ = self.download([503]) + self.assertNotEqual(status, 0) + self.assertEqual(requests, 3) + self.assertIsNone(contents) + + def test_permanent_http_error_is_not_retried(self): + status, requests, contents, _ = self.download([404]) + self.assertNotEqual(status, 0) + self.assertEqual(requests, 1) + self.assertIsNone(contents) + + def test_success_does_not_repeat_the_download(self): + status, requests, contents, payload = self.download([200]) + self.assertEqual(status, 0) + self.assertEqual(requests, 1) + self.assertEqual(contents, payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_installed_test_runners.py b/packaging/tests/test_installed_test_runners.py new file mode 100644 index 000000000..749414c79 --- /dev/null +++ b/packaging/tests/test_installed_test_runners.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +from pathlib import Path +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent + + +def load_runner(name): + spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class InstalledTestRunnersTest(unittest.TestCase): + def test_selects_downloaded_wheel_for_exact_version_and_abi(self): + runner = load_runner("python_installed_tests") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + selected = root / "tsfile-2.5.0-cp312-cp312-manylinux_2_28_x86_64.whl" + selected.touch() + (root / "tsfile-2.3.2.dev0-cp312-cp312-manylinux_2_28_x86_64.whl").touch() + (root / "tsfile-2.5.0-cp311-cp311-manylinux_2_28_x86_64.whl").touch() + self.assertEqual( + runner.select_wheel(root, "2.5.0", "cp312"), selected.resolve() + ) + with self.assertRaisesRegex(ValueError, "Expected one"): + runner.select_wheel(root, "2.5.0", "cp314") + + def test_rejects_ambiguous_downloaded_wheels(self): + runner = load_runner("python_installed_tests") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for platform in ("macosx_12_0_arm64", "macosx_12_0_x86_64"): + (root / f"tsfile-2.5.0-cp312-cp312-{platform}.whl").touch() + with self.assertRaisesRegex(ValueError, "Expected one"): + runner.select_wheel(root, "2.5.0", "cp312") + + def test_cpp_config_is_required_under_the_installed_prefix(self): + runner = load_runner("cpp_installed_tests") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaisesRegex(ValueError, "Expected one installed"): + runner.package_config(root) + config = root / "lib/x86_64-linux-gnu/cmake/TsFile/TsFileConfig.cmake" + config.parent.mkdir(parents=True) + config.touch() + self.assertEqual(runner.package_config(root), config.parent) + + def test_go_pkgconfig_supports_multiarch_and_lib64_installations(self): + runner = load_runner("go_installed_tests") + for libdir in ("lib", "lib64", "lib/x86_64-linux-gnu"): + with self.subTest( + libdir=libdir + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + metadata = root / libdir / "pkgconfig/libtsfile.pc" + metadata.parent.mkdir(parents=True) + metadata.touch() + self.assertEqual(runner.pkgconfig_file(root), metadata) + + def test_windows_go_import_library_uses_only_c_exports(self): + runner = load_runner("go_installed_tests") + output = """File: tsfile.dll +Export { + Ordinal: 1 + Name: tsfile_open +} +Export { + Ordinal: 2 + Name: ?query@TsFileReader@storage@@QEAAXXZ +} +Export { + Name: free_tsfile +} +""" + self.assertEqual( + runner.exported_c_symbols(output), ["free_tsfile", "tsfile_open"] + ) + with self.assertRaisesRegex(ValueError, "No C exports"): + runner.exported_c_symbols("invalid PE input") + + def test_race_runs_on_supported_targets_only(self): + runner = load_runner("go_installed_tests") + for goos in ("linux", "darwin", "windows"): + self.assertTrue(runner.race_supported(goos, "amd64")) + self.assertTrue(runner.race_supported("linux", "arm64")) + self.assertTrue(runner.race_supported("darwin", "arm64")) + self.assertFalse(runner.race_supported("windows", "arm64")) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_macos_runtime.py b/packaging/tests/test_macos_runtime.py new file mode 100644 index 000000000..9f819b4d8 --- /dev/null +++ b/packaging/tests/test_macos_runtime.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +from pathlib import Path +import tempfile +import unittest + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts/verify_macos_runtime.py" +spec = importlib.util.spec_from_file_location("verify_macos_runtime", MODULE_PATH) +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +class MacOSRuntimeTest(unittest.TestCase): + def test_reads_all_dylib_load_commands_but_not_library_identity(self): + output = """library.dylib: +Load command 0 + cmd LC_ID_DYLIB + cmdsize 96 + name /opt/homebrew/Cellar/tsfile/2.5.0/lib/libtsfile.1.dylib (offset 24) +Load command 1 + cmd LC_LOAD_DYLIB + cmdsize 56 + name /usr/lib/libSystem.B.dylib (offset 24) +""" + for index, command in enumerate(sorted(checker.LOAD_COMMANDS), 2): + output += f"Load command {index}\n cmd {command}\n name @rpath/libtsfile.1.dylib (offset 24)\n" + self.assertEqual( + checker.parse_dependencies(output), + {"/usr/lib/libSystem.B.dylib", "@rpath/libtsfile.1.dylib"}, + ) + + def test_allows_system_libraries_and_packaged_tsfile(self): + with tempfile.TemporaryDirectory() as directory: + prefix = Path(directory) + (prefix / "lib").mkdir() + (prefix / "bin").mkdir() + library = prefix / "lib/libtsfile.1.dylib" + library.touch() + checker.validate_dependencies( + prefix / "bin/tsfile-cli", + { + "/usr/lib/libc++.1.dylib", + "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", + "@rpath/libtsfile.1.dylib", + "@loader_path/../lib/libtsfile.1.dylib", + str(library), + }, + prefix, + ) + + def test_rejects_codec_dylibs_even_when_they_exist(self): + with tempfile.TemporaryDirectory() as directory: + prefix = Path(directory) + (prefix / "lib").mkdir() + codec = prefix / "lib/liblz4.1.dylib" + codec.touch() + for name in ( + "/opt/homebrew/opt/snappy/lib/libsnappy.1.dylib", + "/usr/local/opt/lz4/lib/liblz4.1.dylib", + "@rpath/libsnappy.1.dylib", + str(codec), + ): + with self.subTest(name=name), self.assertRaisesRegex( + ValueError, "external third-party dylib" + ): + checker.validate_dependencies( + prefix / "bin/tsfile-cli", {name}, prefix + ) + + def test_rejects_missing_or_external_tsfile_even_through_symlinks(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prefix = root / "sdk" + (prefix / "lib").mkdir(parents=True) + external = root / "libtsfile.1.dylib" + external.touch() + (prefix / "lib/libtsfile.1.dylib").symlink_to(external) + for name in ( + "@rpath/libtsfile.2.dylib", + "@rpath/libtsfile.1.dylib", + str(external), + "@loader_path/../../libtsfile.1.dylib", + ): + with self.subTest(name=name), self.assertRaisesRegex( + ValueError, "unshipped TsFile dylib" + ): + checker.validate_dependencies( + prefix / "bin/tsfile-cli", {name}, prefix + ) + + def test_rejects_empty_or_malformed_inspector_output(self): + for output in ( + "not a Mach-O file", + "Load command 0\n cmd LC_LOAD_DYLIB\n", + ): + with self.subTest(output=output), self.assertRaises(ValueError): + checker.parse_dependencies(output) + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "No Mach-O"): + checker.verify_prefix(directory) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_maven_retry.py b/packaging/tests/test_maven_retry.py new file mode 100644 index 000000000..916c40b8b --- /dev/null +++ b/packaging/tests/test_maven_retry.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Exercise the Maven wrapper retry boundary with real subprocess failures.""" + +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/run_maven_with_retry.py" +SPEC = importlib.util.spec_from_file_location("maven_retry", SCRIPT) +MAVEN = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MAVEN) + + +class MavenRetryTest(unittest.TestCase): + def execute(self, message, failures): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fake = root / "maven.py" + fake.write_text( + "from pathlib import Path\nimport json, sys\n" + "path = Path('calls.json')\n" + "calls = json.loads(path.read_text()) if path.exists() else []\n" + "calls.append(sys.argv[1:])\n" + "path.write_text(json.dumps(calls))\n" + f"if len(calls) <= {failures}:\n" + f" print({message!r}, flush=True)\n sys.exit(1)\n" + "print('[INFO] BUILD SUCCESS', flush=True)\n" + ) + with patch.object(MAVEN.time, "sleep"): + status = MAVEN.run_maven( + ["package"], root=root, command_prefix=[sys.executable, str(fake)] + ) + return status, json.loads((root / "calls.json").read_text()) + + def test_connection_reset_retries_and_refreshes_negative_cache(self): + status, calls = self.execute( + "[ERROR] Could not transfer artifact org.apache.maven:maven-model-builder:pom:3.0 " + "from/to central (https://repo.maven.apache.org/maven2): Connection reset", + 1, + ) + self.assertEqual(status, 0) + self.assertEqual(calls, [["package"], ["-U", "package"]]) + + def test_transient_http_failure_stops_after_three_attempts(self): + status, calls = self.execute( + "[ERROR] Could not transfer artifact example:pom:1 from/to central: " + "status code: 503, reason phrase: Service Unavailable", + 5, + ) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 3) + + def test_missing_artifact_does_not_retry(self): + status, calls = self.execute( + "[ERROR] Could not transfer artifact example:pom:1 from/to central: " + "status code: 404, reason phrase: Not Found", + 5, + ) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 1) + + def test_compiler_failure_does_not_retry(self): + status, calls = self.execute( + "[ERROR] Failed to execute goal exec-maven-plugin: compiler error", 5 + ) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py new file mode 100644 index 000000000..5fb780ccd --- /dev/null +++ b/packaging/tests/test_native_install.py @@ -0,0 +1,475 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Opt-in native configure/install/link regressions (requires CMake and LZ4).""" + +import os +import re +import shlex +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +FIXTURES = ROOT / "cpp/cmake/tests/projects" + + +class PublicHeaderCompatibilityTest(unittest.TestCase): + def test_simd_int64_reduction_normalizes_lane_types(self): + statistic = (ROOT / "cpp/src/common/statistic.h").read_text() + self.assertIn("std::min(simde_mm_cvtsi128_si64(vmin)", statistic) + self.assertIn("std::max(simde_mm_cvtsi128_si64(vmax)", statistic) + source = textwrap.dedent("""\ + #include + #include + + long simde_mm_cvtsi128_si64(int); + long long simde_mm_extract_epi64(int, int); + + std::int64_t reduce_min(int value) { + return std::min(simde_mm_cvtsi128_si64(value), + simde_mm_extract_epi64(value, 1)); + } + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-fsyntax-only", + "-x", + "c++", + "-std=c++11", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + + def test_msvc_old_cplusplus_uses_native_cxx11_features(self): + source = textwrap.dedent("""\ + #include "utils/util_define.h" + class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; + class Base { virtual void reset(); }; + class Derived : public Base { void reset() OVERRIDE; }; + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-E", + "-P", + "-x", + "c++", + "-std=c++11", + "-D_MSC_VER=1930", + "-DTSFILE_STATIC", + "-D__cplusplus=199711L", + f"-I{ROOT / 'cpp/src'}", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout[-4000:]) + self.assertRegex( + result.stdout, r"class Probe \{\s*(?:static_assert|_Static_assert)" + ) + self.assertNotIn("static_assertion_MSVC_old_cplusplus", result.stdout) + self.assertIn("void reset() override", result.stdout) + + def test_msvc_old_cplusplus_does_not_define_nullptr(self): + compatibility = (ROOT / "cpp/src/utils/util_define.h").read_text() + section = re.search( + r"/\* ======== nullptr ======== \*/\n(.*?)\n/\* ======== cache line", + compatibility, + re.S, + ) + self.assertIsNotNone(section) + source = section.group(1) + textwrap.dedent("""\ + + #ifdef nullptr + #error nullptr must remain a native MSVC keyword + #endif + class Base { virtual void reset(); }; + class Derived : public Base { void reset() OVERRIDE; }; + int* pointer = nullptr; + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-E", + "-P", + "-nostdinc", + "-x", + "c++", + "-std=c++11", + "-D_MSC_VER=1930", + "-D__cplusplus=199711L", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("void reset() override", result.stdout) + self.assertRegex(result.stdout, r"int\s*\*\s*pointer\s*=\s*nullptr") + + +class StaticANTLRPackageTest(unittest.TestCase): + """Exercise installed metadata without compiling the ANTLR runtime.""" + + def test_installed_system_antlr_requires_compatible_runtime(self): + with tempfile.TemporaryDirectory(prefix="tsfile-antlr-config-") as temporary: + directory = Path(temporary) + producer = directory / "producer" + (producer / "cmake").mkdir(parents=True) + shutil.copyfile( + ROOT / "cpp/cmake/TsFileStaticDependencies.cmake.in", + producer / "cmake/TsFileStaticDependencies.cmake.in", + ) + (producer / "CMakeLists.txt").write_text(textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.11) + project(StaticANTLRPackage NONE) + include(CMakePackageConfigHelpers) + include("${TSFILE_CMAKE_DIR}/DependencySource.cmake") + include("${TSFILE_CMAKE_DIR}/ANTLR4Dependency.cmake") + set(CMAKE_INSTALL_LIBDIR lib) + set(ENABLE_ANTLR4 ON) + set(ENABLE_THREADS OFF) + set(TSFILE_BUILD_SHARED OFF) + add_library(tsfile INTERFACE) + add_library(TsFile::ANTLR4 INTERFACE IMPORTED) + include("${TSFILE_CMAKE_DIR}/TsFileStaticDependencies.cmake") + configure_package_config_file( + "${TSFILE_CMAKE_DIR}/TsFileConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + INSTALL_DESTINATION lib/cmake/TsFile) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + DESTINATION lib/cmake/TsFile) + install(TARGETS tsfile EXPORT TsFileTargets) + install(EXPORT TsFileTargets NAMESPACE TsFile:: + DESTINATION lib/cmake/TsFile) + """)) + consumer = directory / "consumer" + consumer.mkdir() + (consumer / "CMakeLists.txt").write_text(textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.11) + project(StaticANTLRConsumer NONE) + find_package(TsFile CONFIG REQUIRED) + get_target_property(runtime TsFile::Static_ANTLR4 + INTERFACE_LINK_LIBRARIES) + if (NOT runtime STREQUAL EXPECTED_TARGET OR NOT TARGET ${runtime}) + message(FATAL_ERROR "Static TsFile lost its system target") + endif () + if (NOT TARGET utf8cpp) + message(FATAL_ERROR "Static TsFile lost utf8cpp compatibility") + endif () + """)) + + def configure(source, build, *options): + return subprocess.run( + [ + "cmake", + "-S", + str(source), + "-B", + str(build), + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=FALSE", + "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE", + "-DCMAKE_FIND_USE_CMAKE_SYSTEM_PATH=FALSE", + "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=FALSE", + f"-DCMAKE_MAKE_PROGRAM={shutil.which('make')}", + "-G", + "Unix Makefiles", + *options, + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + def package(prefix, version, target, version_field): + prefix.mkdir(parents=True) + (prefix / "utf8cppConfig.cmake").write_text( + "add_library(utf8cpp INTERFACE IMPORTED)\n" + ) + if version == "missing-package": + return + content = "" + if version is not None: + content += f'set({version_field} "{version}")\n' + if target is not None: + content += ( + f"add_library({target} INTERFACE IMPORTED)\n" + f"set_property(TARGET {target} PROPERTY " + "INTERFACE_LINK_LIBRARIES utf8cpp)\n" + ) + (prefix / "antlr4-runtime-config.cmake").write_text(content) + + targets = ( + "antlr4_static", + "antlr4-runtime::antlr4_static", + "antlr4_shared", + "antlr4-runtime::antlr4_shared", + ) + for index, target in enumerate(targets): + build = directory / f"producer-build-{index}" + stage = directory / f"stage-{index}" + build_package = directory / f"build-package-{index}" + package(build_package, "4.9.3", target, "ANTLR_VERSION") + result = configure( + producer, + build, + f"-DTSFILE_CMAKE_DIR={ROOT / 'cpp/cmake'}", + "-DTSFILE_DEPENDENCY_SOURCE=SYSTEM", + f"-DCMAKE_PREFIX_PATH={build_package}", + f"-DCMAKE_INSTALL_PREFIX={stage}", + ) + self.assertEqual(result.returncode, 0, result.stdout) + result = subprocess.run( + ["cmake", "--install", str(build)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + relocated = directory / f"relocated-{index}" + stage.rename(relocated) + cases = ( + ("4.9.3", target, True), + ("4.12.0", target, True), + ("4.13.0", target, False), + ("4.13.2", target, False), + ("4.9.2", target, False), + (None, target, False), + ("4.9.3", None, False), + ("missing-package", None, False), + ) + for field in ("antlr4-runtime_VERSION", "ANTLR_VERSION"): + for case, (version, consumer_target, succeeds) in enumerate(cases): + with self.subTest( + target=target, + field=field, + version=version, + consumer_target=consumer_target, + ): + suffix = f"{index}-{field}-{case}" + runtime = directory / f"runtime-{suffix}" + package(runtime, version, consumer_target, field) + result = configure( + consumer, + directory / f"consumer-{suffix}", + f"-DCMAKE_PREFIX_PATH={relocated};{runtime}", + f"-DEXPECTED_TARGET={target}", + ) + if succeeds: + self.assertEqual(result.returncode, 0, result.stdout) + else: + self.assertNotEqual( + result.returncode, + 0, + f"Installed static TsFile accepted {version}:\n" + + result.stdout, + ) + diagnostic = " ".join(result.stdout.split()) + self.assertIn("compatible system ANTLR4", diagnostic) + self.assertIn(">=4.9.3", diagnostic) + self.assertIn("<4.13.0", diagnostic) + + +@unittest.skipUnless( + os.environ.get("TSFILE_RUN_INSTALL_TESTS") == "1", "opt-in native builds" +) +class NativeInstallTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="tsfile-install-") + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.build = self.directory / "build" + self.stage = self.directory / "stage" + + def run_command(self, *arguments, env=None): + result = subprocess.run( + [str(argument) for argument in arguments], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) + self.assertEqual(result.returncode, 0, result.stdout[-14000:]) + return result.stdout.strip() + + def configure(self, *options): + arguments = [ + "cmake", + "-S", + ROOT / "cpp", + "-B", + self.build, + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_TEST=OFF", + "-DBUILD_TOOLS=OFF", + "-DTSFILE_DEPENDENCY_SOURCE=SYSTEM", + "-DTSFILE_ENABLE_NATIVE_ARCH=OFF", + f"-DCMAKE_INSTALL_PREFIX={self.stage}", + ] + arguments.extend( + f"-DENABLE_{name}=OFF" + for name in ( + "ANTLR4", + "SNAPPY", + "LZ4", + "LZOKAY", + "ZLIB", + "ZSTD", + "LZMA2", + "SIMD", + ) + ) + self.run_command(*arguments, *options) + + def install(self): + self.run_command("cmake", "--build", self.build, "--parallel", "8") + self.run_command("cmake", "--install", self.build, "--config", "Release") + relocated = self.directory / "relocated" + self.stage.rename(relocated) + self.stage = relocated + for path in self.stage.rglob("*.cmake"): + self.assertNotIn(str(self.build), path.read_text(), str(path)) + self.assertNotIn(str(ROOT / "cpp"), path.read_text(), str(path)) + + def consumer(self, fixture, target, env=None, *options): + build = self.directory / fixture + self.run_command( + "cmake", + "-S", + FIXTURES / fixture, + "-B", + build, + f"-DCMAKE_PREFIX_PATH={self.stage}", + *options, + env=env, + ) + self.run_command("cmake", "--build", build, "--parallel", "8", env=env) + self.run_command(build / target, env=env) + + def test_static_system_lz4_install_and_relocated_consumer(self): + self.configure("-DTSFILE_BUILD_SHARED=OFF", "-DENABLE_LZ4=ON") + self.install() + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + + def test_static_bundled_install_and_relocated_consumer(self): + self.configure( + "-DTSFILE_BUILD_SHARED=OFF", + "-DTSFILE_DEPENDENCY_SOURCE=BUNDLED", + "-DENABLE_ANTLR4=ON", + "-DENABLE_ZLIB=ON", + "-DENABLE_LZOKAY=ON", + "-DTSFILE_DEPENDENCY_OFFLINE=ON", + f"-DTSFILE_DEPENDENCY_CACHE={os.environ['TSFILE_TEST_DEPENDENCY_CACHE']}", + ) + self.install() + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + + def test_multiarch_pkgconfig_install_and_relocated_consumers(self): + self.configure("-DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu") + self.install() + env = dict( + os.environ, + PKG_CONFIG_PATH=str(self.stage / "lib/x86_64-linux-gnu/pkgconfig"), + ) + prefix = self.run_command( + "pkg-config", "--variable=prefix", "libtsfile", env=env + ) + self.assertEqual(Path(prefix).resolve(), self.stage.resolve()) + self.consumer("PkgConfigConsumer", "tsfile_pkgconfig_consumer", env) + self.run_command( + self.directory / "PkgConfigConsumer/tsfile_pkgconfig_cpp_consumer", env=env + ) + + def test_installed_smoke_requires_a_library_symbol(self): + self.configure() + self.install() + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-std=c++11", + f"-I{self.stage / 'include'}", + f"-I{self.stage / 'include/tsfile'}", + str(FIXTURES / "InstalledConsumer/main.cc"), + "-o", + str(self.directory / "unlinked"), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertNotEqual( + result.returncode, 0, "Smoke consumer linked successfully without libtsfile" + ) + self.assertIn("get_global_time_encoding", result.stdout) + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + for formula in ("tsfile.rb.in",): + source = re.search( + r"write <<~CPP\n(.*?)\n\s+CPP", + (ROOT / "packaging/homebrew" / formula).read_text(), + re.S, + ).group(1) + source_path = self.directory / f"{formula}.cc" + source_path.write_text(textwrap.dedent(source)) + command = [ + *shlex.split(os.environ.get("CXX", "c++")), + "-std=c++11", + f"-I{self.stage / 'include'}", + str(source_path), + "-o", + str(self.directory / "brew-consumer"), + ] + unlinked = subprocess.run( + command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + self.assertNotEqual(unlinked.returncode, 0, formula) + self.assertIn("get_global_time_encoding", unlinked.stdout) + self.run_command( + *command, + f"-L{self.stage / 'lib'}", + f"-Wl,-rpath,{self.stage / 'lib'}", + "-ltsfile", + ) + self.run_command(self.directory / "brew-consumer") + + def test_static_msvc_runtime_reaches_dependency_and_project_targets(self): + self.configure( + "-DTSFILE_MSVC_STATIC_RUNTIME=ON", + "-DTSFILE_DEPENDENCY_SOURCE=BUNDLED", + "-DENABLE_ZLIB=ON", + "-DTSFILE_DEPENDENCY_OFFLINE=ON", + f"-DTSFILE_DEPENDENCY_CACHE={os.environ['TSFILE_TEST_DEPENDENCY_CACHE']}", + f"-DCMAKE_PROJECT_TsFile_CPP_INCLUDE={ROOT / 'cpp/cmake/tests/CheckStaticMSVCRuntime.cmake'}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py new file mode 100644 index 000000000..20b472162 --- /dev/null +++ b/packaging/tests/test_native_package_versions.py @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import json +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/native_package_versions.py" +CPP_CMAKE_FILE = REPOSITORY_ROOT / "cpp/CMakeLists.txt" +CPP_VERSION_PATTERN = re.compile(r"^set\(TsFile_CPP_VERSION\s+(\S+)\)", re.MULTILINE) + + +def repository_cpp_version(): + match = CPP_VERSION_PATTERN.search(CPP_CMAKE_FILE.read_text(encoding="utf-8")) + if match is None: + raise AssertionError(f"cannot find TsFile_CPP_VERSION in {CPP_CMAKE_FILE}") + return match.group(1) + + +class NativePackageVersionsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + spec = importlib.util.spec_from_file_location( + "native_package_versions", MODULE_PATH + ) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def test_reads_repository_release_version(self): + self.assertEqual( + self.module.read_cpp_version(CPP_CMAKE_FILE), repository_cpp_version() + ) + + def test_builds_exact_release_package_versions(self): + self.assertEqual( + self.module.build_versions("2.5.0"), + { + "base_version": "2.5.0", + "python_version": "2.5.0", + "logical_version": "2.5.0", + "deb_version": "2.5.0-1", + "rpm_version": "2.5.0", + "rpm_release": "1.el9", + "archive_version": "2.5.0", + "homebrew_version": "2.5.0", + }, + ) + self.assertEqual(self.module.build_versions("3.0.0")["deb_version"], "3.0.0-1") + + def test_rejects_development_prerelease_and_invalid_versions(self): + with tempfile.TemporaryDirectory() as directory: + cmake_file = Path(directory) / "CMakeLists.txt" + for version in ( + "2.5.0.dev", + "2.5.0.dev0", + "2.5.0rc1", + "2.5.dev", + "2.5", + "02.5.0", + "2.5.0+build", + "2.5.0-release", + "", + ): + with self.subTest(version=version): + with self.assertRaisesRegex(ValueError, "release"): + self.module.build_versions(version) + cmake_file.write_text(f"set(TsFile_CPP_VERSION {version})\n") + with self.assertRaisesRegex(ValueError, "release"): + self.module.read_cpp_version(cmake_file) + + def test_cli_rejects_development_checkout_before_writing_outputs(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cmake_file = root / "CMakeLists.txt" + cmake_file.write_text("set(TsFile_CPP_VERSION 2.5.0.dev)\n") + json_out = root / "versions.json" + github_out = root / "github-output.txt" + result = subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(cmake_file), + "--json-out", + str(json_out), + "--github-output", + str(github_out), + ], + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("release TsFile_CPP_VERSION", result.stderr) + self.assertFalse(json_out.exists()) + self.assertFalse(github_out.exists()) + + def test_cli_writes_json_and_github_outputs(self): + expected = self.module.build_versions(repository_cpp_version()) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + json_out = root / "versions.json" + github_out = root / "github-output.txt" + github_out.write_text("existing=value\n") + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(CPP_CMAKE_FILE), + "--json-out", + str(json_out), + "--github-output", + str(github_out), + ], + check=True, + ) + self.assertEqual(json.loads(json_out.read_text()), expected) + self.assertEqual( + github_out.read_text().splitlines(), + [ + "existing=value", + *[f"{key}={value}" for key, value in expected.items()], + ], + ) + + def test_cli_preserves_existing_github_output_bytes(self): + version = repository_cpp_version() + for existing in ( + b"", + b"existing=value", + b"existing=value\n", + b"existing=value\r\n", + ): + with self.subTest( + existing=existing + ), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + github_out = root / "github-output.txt" + github_out.write_bytes(existing) + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(CPP_CMAKE_FILE), + "--json-out", + str(root / "versions.json"), + "--github-output", + str(github_out), + ], + check=True, + ) + separator = b"\n" if existing and not existing.endswith(b"\n") else b"" + self.assertTrue( + github_out.read_bytes().startswith( + existing + separator + f"base_version={version}\n".encode() + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_python_wheel_contents.py b/packaging/tests/test_python_wheel_contents.py new file mode 100644 index 000000000..6d015862b --- /dev/null +++ b/packaging/tests/test_python_wheel_contents.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Validate real wheel ZIP layouts and ELF debug-section rejection.""" + +import importlib.util +from pathlib import Path +import struct +import tempfile +import unittest +from zipfile import ZipFile + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "wheel_contents", ROOT / "packaging/scripts/check_python_wheel.py" +) +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +class WheelContentsTest(unittest.TestCase): + def check(self, files): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "tsfile.whl" + with ZipFile(path, "w") as wheel: + for name, data in files.items(): + wheel.writestr(name, data) + checker.check_wheel(path) + + def test_runtime_only_wheel_is_accepted(self): + self.check( + { + "tsfile/libtsfile.so.1": b"native", + "tsfile/reader.so": b"binding", + "tsfile/__init__.py": b"", + } + ) + + def test_sdk_headers_and_import_libraries_are_rejected(self): + for name in [ + "tsfile/include/tsfile/cwrapper/tsfile_cwrapper.h", + "tsfile/tsfile.lib", + "tsfile/tsfile_cpp.pxd", + ]: + with self.subTest(name=name), self.assertRaisesRegex( + ValueError, "Build-only" + ): + self.check({"tsfile/tsfile.dll": b"native", name: b"build-input"}) + + def test_library_alias_copies_are_rejected(self): + with self.assertRaisesRegex(ValueError, "one TsFile runtime library"): + self.check( + {"tsfile/libtsfile.so": b"native", "tsfile/libtsfile.so.1": b"native"} + ) + + def test_elf_debug_sections_are_rejected(self): + # ELF64 fixture: null section, section-name string table, debug section. + strings = b"\0.shstrtab\0.debug_info\0" + data = bytearray(64 + 3 * 64 + len(strings) + 1) + data[:6] = b"\x7fELF\x02\x01" + struct.pack_into(":Debug>", "Release", "/MT"), + ("MultiThreaded$<$:Debug>", "Debug", "/MTd"), + ("MultiThreaded$<$:Debug>DLL", "Release", "/MD"), + ("MultiThreaded$<$:Debug>DLL", "Debug", "/MDd"), + ]: + with self.subTest(runtime=runtime, configuration=configuration): + build = root / (flag[1:] + "-build") + subprocess.run( + [ + "cmake", + "-S", + str(root), + "-B", + str(build), + f"-DCMAKE_BUILD_TYPE={configuration}", + f"-DTEST_RUNTIME={runtime}", + ], + check=True, + capture_output=True, + text=True, + ) + generated = list(build.rglob("tsfile-build-config.json")) + self.assertEqual(len(generated), 1) + config = support.read_sdk_config( + build, generated[0].parent / "library" + ) + self.assertEqual(support.msvc_runtime_flag(config), flag) + self.assertEqual(config["configuration"], configuration) + prefix = root / (flag[1:] + "-sdk") + subprocess.run( + [ + "cmake", + "--install", + str(build), + "--prefix", + str(prefix), + "--config", + configuration, + ], + check=True, + capture_output=True, + ) + moved = root / (flag[1:] + "-relocated") + shutil.move(prefix, moved) + self.assertEqual( + support.read_sdk_config(moved, moved / "lib/library"), config + ) + + def test_missing_or_invalid_runtime_metadata_fails_instead_of_guessing(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + with self.assertRaisesRegex(FileNotFoundError, r"Rebuild/install C\+\+"): + support.read_sdk_config(root, root / "library") + for config in [ + {"compiler": "MSVC", "msvc_runtime": ""}, + {"compiler": "GNU", "msvc_runtime": "MultiThreaded"}, + ]: + with self.subTest(config=config), self.assertRaises(ValueError): + support.msvc_runtime_flag(config) + + def test_staging_one_soname_removes_stale_alias_copies(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + package = root / "tsfile" + package.mkdir() + source = root / "libtsfile.so.2.5.0" + source.write_bytes(b"native-library") + for name in ("libtsfile.so", "libtsfile.so.1", "libtsfile.so.2.5.0"): + (package / name).write_bytes(b"old-copy") + target = support.copy_runtime_library( + package, source, {"runtime_library": "libtsfile.so.1"} + ) + self.assertEqual([p.name for p in package.iterdir()], ["libtsfile.so.1"]) + self.assertEqual(target.read_bytes(), source.read_bytes()) + + def test_runtime_filename_cannot_escape_package(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + info = root / "share/tsfile" + info.mkdir(parents=True) + (info / "tsfile-build-config.json").write_text( + json.dumps({"schema_version": 1, "runtime_library": "../tsfile.dll"}) + ) + with self.assertRaisesRegex(ValueError, "shared runtime library"): + support.read_sdk_config(root, root / "lib/tsfile.dll") diff --git a/packaging/tests/test_windows_runtime.py b/packaging/tests/test_windows_runtime.py new file mode 100644 index 000000000..8258f00a0 --- /dev/null +++ b/packaging/tests/test_windows_runtime.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts/verify_windows_runtime.py" + + +class WindowsRuntimeTest(unittest.TestCase): + def checker(self): + self.assertTrue( + MODULE_PATH.is_file(), "Windows artifacts need a runtime dependency check" + ) + spec = importlib.util.spec_from_file_location( + "verify_windows_runtime", MODULE_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_allows_only_inbox_or_colocated_package_dlls(self): + module = self.checker() + imports = module.parse_dependents( + "File Type: EXECUTABLE IMAGE\n\n Image has the following dependencies:\n\n" + " tsfile.dll\n KERNEL32.dll\n api-ms-win-core-synch-l1-2-0.dll\n\n" + " Summary\n 1000 .data\n" + ) + module.validate_dependents("tsfile-cli.exe", imports, {"tsfile.dll"}) + self.assertEqual(len(imports), 3) + + def test_rejects_dynamic_crt_even_when_runner_or_package_provides_it(self): + module = self.checker() + for dll in ( + "VCRUNTIME140.dll", + "MSVCP140.dll", + "ucrtbase.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + ): + with self.subTest(dll=dll), self.assertRaisesRegex( + ValueError, "static MSVC runtime" + ): + module.validate_dependents("tsfile.dll", {dll.lower()}, {dll.lower()}) + + def test_rejects_unshipped_codec_dll(self): + module = self.checker() + with self.assertRaisesRegex(ValueError, "snappy.dll"): + module.validate_dependents("tsfile.dll", {"snappy.dll"}, {"tsfile.dll"}) + + def test_rejects_unrecognized_dumpbin_output(self): + with self.assertRaisesRegex(ValueError, "DLL imports"): + self.checker().parse_dependents( + "fatal error LNK1107: invalid or corrupt file" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/pom.xml b/pom.xml index fcf8528d8..609762ae9 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,14 @@ pom Apache TsFile Project Parent POM + + true 17 17 17 @@ -544,6 +552,22 @@ python + + + with-python-only + + python + + + ${maven.multiModuleProjectDirectory}/cpp/target/build + false + + .java-17-and-above @@ -747,6 +771,25 @@ ${msvc.generator} + + + cpp-msvc-arm64-native + + + cpp.toolchain + msvc-arm64-native + + + + Ninja Multi-Config + + .skipTests diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 000000000..1981d9212 --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +include _build_support.py +recursive-include tsfile *.pyx *.pxd *.pxi +include tsfile/python_random_access_read_file.h +prune tsfile/include diff --git a/python/VersionUpdater.groovy b/python/VersionUpdater.groovy index a586fe936..fa2c5c970 100644 --- a/python/VersionUpdater.groovy +++ b/python/VersionUpdater.groovy @@ -19,6 +19,17 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Synchronize the version in setup.py and the one used in the maven pom. + +// Native package jobs derive their identity from native_package_versions.py. +// Keep that identity stable by leaving source versions untouched in those jobs. +def skipVersionSync = project.properties.getProperty("tsfile.version.sync.skip") +if (skipVersionSync == null) { + skipVersionSync = System.getProperty("tsfile.version.sync.skip") +} +if (skipVersionSync != null && skipVersionSync.equalsIgnoreCase("true")) { + println "Skipping source version synchronization for native packaging" + return +} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// def currentMavenVersion = project.version as String diff --git a/python/_build_support.py b/python/_build_support.py new file mode 100644 index 000000000..b050c6378 --- /dev/null +++ b/python/_build_support.py @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Read native SDK build metadata and prepare runtime-only wheel files.""" + +import json +from pathlib import Path +import shutil +import subprocess + +MSVC_RUNTIME_FLAGS = { + "MultiThreaded": "/MT", + "MultiThreadedDLL": "/MD", + "MultiThreadedDebug": "/MTd", + "MultiThreadedDebugDLL": "/MDd", +} + + +def read_sdk_config(sdk, library): + candidates = [ + Path(sdk) / "share/tsfile/tsfile-build-config.json", + Path(library).parent / "tsfile-build-config.json", + ] + for path in candidates: + if path.is_file(): + config = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(config, dict) or config.get("schema_version") != 1: + raise ValueError(f"Unsupported TsFile SDK build metadata: {path}") + name = config.get("runtime_library") + if ( + not isinstance(name, str) + or not name + or name in {".", ".."} + or "/" in name + or "\\" in name + or ":" in name + or "\0" in name + ): + raise ValueError( + f"SDK does not describe a shared runtime library: {path}" + ) + return config + raise FileNotFoundError( + "Missing tsfile-build-config.json in the native SDK. " + "Rebuild/install C++ from this checkout before building Python. " + f"Checked: {candidates}" + ) + + +def msvc_runtime_flag(config): + if config.get("compiler") != "MSVC": + raise ValueError("An MSVC Python extension requires an MSVC-built SDK") + runtime = config.get("msvc_runtime") + if runtime not in MSVC_RUNTIME_FLAGS: + raise ValueError(f"Unsupported SDK MSVC runtime: {runtime!r}") + return MSVC_RUNTIME_FLAGS[runtime] + + +def copy_runtime_library(package, library, config): + package = Path(package) + # Repeated ABI builds must not retain aliases from an earlier SDK view. + for pattern in ("libtsfile.so*", "libtsfile*.dylib", "tsfile.dll", "libtsfile.dll"): + for stale in package.glob(pattern): + stale.unlink() + target = package / config["runtime_library"] + shutil.copy2(library, target) + return target + + +def strip_wheel_binaries(build_lib, platform): + # Run before auditwheel/delocate rewrite load commands. Only wheel build + # copies are stripped; SDKs and in-place development builds keep symbols. + if platform not in {"linux", "darwin"}: + return + for path in sorted((Path(build_lib) / "tsfile").rglob("*")): + if not path.is_file() or not ( + path.name.endswith((".so", ".dylib")) or ".so." in path.name + ): + continue + if platform == "linux": + subprocess.run(["strip", "--strip-unneeded", str(path)], check=True) + else: + subprocess.run(["xcrun", "strip", "-S", "-x", str(path)], check=True) + subprocess.run( + ["codesign", "--force", "--sign", "-", str(path)], check=True + ) diff --git a/python/check_cpp_build.py b/python/check_cpp_build.py new file mode 100755 index 000000000..b8e6f61a8 --- /dev/null +++ b/python/check_cpp_build.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Validate that the C++ build tree required by the Python module exists. + +The Python module links against the native library produced by the ``cpp`` +module. When the two are built together (``-Pwith-python``) the artifacts are +always present. When the Python module is built on its own +(``-Pwith-python-only``) they must already exist, so this check turns a late and +opaque failure inside ``setup.py`` into an early, actionable one. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def missing_parts(build_root: Path) -> list[Path]: + """Return the required subdirectories of *build_root* that are absent.""" + return [ + part + for part in (build_root / "include", build_root / "lib") + if not part.is_dir() + ] + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print(f"usage: {Path(argv[0]).name} ", file=sys.stderr) + return 2 + + build_root = Path(argv[1]).expanduser().resolve() + missing = missing_parts(build_root) + if not missing: + return 0 + + print( + f"error: TsFile C++ build not found at {build_root}", + file=sys.stderr, + ) + for part in missing: + print(f" missing directory: {part}", file=sys.stderr) + print( + " Run './mvnw -Pwith-cpp package' first, " + "or set -Dtsfile.cpp.build=.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/python/pom.xml b/python/pom.xml index 1e11b3634..f47c92ac2 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -29,6 +29,13 @@ TsFile: Python 2.6.0 + + ${project.basedir}/../cpp/target/build tsfile ${project.build.directory}/build-wrapper-output @@ -69,7 +76,36 @@ org.codehaus.mojo exec-maven-plugin + + + + ${tsfile.cpp.build} + + + + + check-cpp-build + validate + + exec + + + ${tsfile.cpp.check.skip} + ${python.exe.bin} + + ${project.basedir}/check_cpp_build.py + ${tsfile.cpp.build} + + + python-venv diff --git a/python/pyproject.toml b/python/pyproject.toml index cd28b3d54..dfcd58be6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -30,7 +30,7 @@ build-backend = "setuptools.build_meta" [project] name = "tsfile" -version = "2.3.2.dev" +version = "2.5.0" requires-python = ">=3.9" description = "TsFile Python" readme = {file = "README.md", content-type = "text/markdown"} @@ -58,23 +58,21 @@ Issues = "https://github.com/apache/tsfile/issues" [tool.setuptools] package-dir = {"" = "."} +include-package-data = false [tool.setuptools.packages.find] where = ["."] include = ["tsfile*"] +namespaces = false [tool.setuptools.package-data] tsfile = [ - "python_random_access_read_file.h", - "*.pxd", - "*.pxi", - "*.so", - "*.so.*", - "*.dylib", - "*.dylib.*", - "*.dll", - "*.dll.a", - "*.lib", - "*.lib.a", - "include/**/*" + "libtsfile.so*", + "libtsfile*.dylib", + "tsfile.dll", + "libtsfile.dll", + "libstdc++-6.dll", + "libgcc_s_seh-1.dll", + "libwinpthread-1.dll", + "licenses/*" ] diff --git a/python/setup.py b/python/setup.py index fe19fc91c..f843201a7 100644 --- a/python/setup.py +++ b/python/setup.py @@ -20,17 +20,28 @@ import platform import shutil import sys + import numpy as np from pathlib import Path from Cython.Build import cythonize from setuptools import setup, Extension from setuptools.command.build_ext import build_ext +from setuptools.command.build_py import build_py ROOT = Path(__file__).parent.resolve() PKG = ROOT / "tsfile" -version = "2.3.2.dev" +# PEP 517 hooks execute setup.py without adding its directory to sys.path. +sys.path.insert(0, str(ROOT)) +from _build_support import ( + copy_runtime_library, + msvc_runtime_flag, + read_sdk_config, + strip_wheel_binaries, +) + +version = "2.5.0" def _find_cpp_build(): @@ -92,10 +103,9 @@ def _find_lib(root, patterns): if not candidates: raise FileNotFoundError("missing libtsfile.so* in build output") src = candidates[0] - dst = PKG / src.name - shutil.copy2(src, dst) - link_name = PKG / "libtsfile.so" - shutil.copy2(src, link_name) + sdk_config = read_sdk_config(CPP_OUT, src) + copy_runtime_library(PKG, src, sdk_config) + native_link_dir = src.parent elif sys.platform == "darwin": candidates = sorted(CPP_LIB.rglob("libtsfile.*.dylib")) or list( @@ -104,10 +114,9 @@ def _find_lib(root, patterns): if not candidates: raise FileNotFoundError("missing libtsfile*.dylib in build output") src = candidates[0] - dst = PKG / src.name - shutil.copy2(src, dst) - link_name = PKG / "libtsfile.dylib" - shutil.copy2(src, link_name) + sdk_config = read_sdk_config(CPP_OUT, src) + copy_runtime_library(PKG, src, sdk_config) + native_link_dir = src.parent elif sys.platform == "win32": # The shared library is named tsfile.dll (MSVC) or libtsfile.dll (MinGW). @@ -133,8 +142,9 @@ def _find_lib(root, patterns): # name (libtsfile.dll): the import library (libtsfile.dll.a) has that name # baked in, so Cython .pyd files record "libtsfile.dll" in their PE import # table and Windows must find it by that exact name at runtime. - dll_dst = PKG / dll_src.name - shutil.copy2(dll_src, dll_dst) + sdk_config = read_sdk_config(CPP_OUT, dll_src) + dll_dst = copy_runtime_library(PKG, dll_src, sdk_config) + native_link_dir = PKG # Copy import library with a name matching the DLL. if win_toolchain == "mingw": @@ -189,9 +199,34 @@ def _find_lib(root, patterns): raise RuntimeError(f"Unsupported platform: {sys.platform}") +# Retain license/notice texts even though SDK headers are build inputs only. +license_dir = PKG / "licenses" +license_dir.mkdir(exist_ok=True) +for document in ("LICENSE", "NOTICE"): + candidates = [CPP_OUT / "share/doc/tsfile" / document, ROOT.parent / document] + source = next((path for path in candidates if path.is_file()), None) + if source is None: + raise FileNotFoundError(f"Missing SDK {document}: {candidates}") + shutil.copy2(source, license_dir / document) + + +class BuildPy(build_py): + def run(self): + if "bdist_wheel" in self.distribution.script_args: + # Reused build directories must not carry old SDK headers, import + # libraries or binary aliases into a new runtime-only wheel. + staging = Path(self.build_lib) / "tsfile" + if staging.resolve() == PKG: + raise ValueError("Wheel build_lib must be outside the source package") + shutil.rmtree(staging, ignore_errors=True) + super().run() + + class BuildExt(build_ext): def run(self): super().run() + if "bdist_wheel" in self.distribution.script_args and not self.inplace: + strip_wheel_binaries(self.build_lib, sys.platform) def finalize_options(self): # MinGW must be requested explicitly; MSVC is the default Windows @@ -205,8 +240,11 @@ def finalize_options(self): extra_link_args = [] runtime_library_dirs = [] libraries = [] -library_dirs = [str(PKG)] +library_dirs = [str(native_link_dir)] include_dirs = [str(PKG), np.get_include(), str(PKG / "include")] +sdk_include = PKG / "include" / "tsfile" +if sdk_include.is_dir(): + include_dirs.append(str(sdk_include)) if sys.platform.startswith("linux"): libraries = ["tsfile"] @@ -233,6 +271,8 @@ def finalize_options(self): ] else: # msvc libraries = ["tsfile"] + # The SDK records its resolved CRT; callers do not configure it twice. + extra_compile_args.append(msvc_runtime_flag(sdk_config)) # cl.exe rejects the GCC-style flags above. Mirror the options the # C++ module itself is built with (see cpp/CMakeLists.txt). C++17 is # required because Cython 3 emits inline variables (error C7525); the @@ -293,7 +333,7 @@ def finalize_options(self): version=version, packages=["tsfile", "tsfile.dataset"], package_dir={"": "."}, - include_package_data=True, + include_package_data=False, ext_modules=cythonize(exts, compiler_directives={"language_level": 3}), - cmdclass={"build_ext": BuildExt}, + cmdclass={"build_ext": BuildExt, "build_py": BuildPy}, )