diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fd23ac2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/iotdb-tools-thrift-artifacts.yml b/.github/workflows/iotdb-tools-thrift-artifacts.yml new file mode 100644 index 0000000..a283455 --- /dev/null +++ b/.github/workflows/iotdb-tools-thrift-artifacts.yml @@ -0,0 +1,223 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Build IoTDB Tools Thrift Artifacts + +on: + workflow_dispatch: + inputs: + git_ref: + description: Optional branch, tag, or commit SHA to build + required: false + type: string + pull_request: + paths: + - .github/workflows/iotdb-tools-thrift-artifacts.yml + - iotdb-tools-thrift/** + +permissions: + contents: read + +concurrency: + group: iotdb-tools-thrift-artifacts-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + name: Build ${{ matrix.classifier }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - classifier: linux-x86_64 + runner: ubuntu-24.04 + platform: linux + maven: ./mvnw + - classifier: linux-aarch64 + runner: ubuntu-24.04-arm + platform: linux + maven: ./mvnw + - classifier: mac-x86_64 + runner: macos-15-intel + platform: macos + maven: ./mvnw + - classifier: mac-aarch64 + runner: macos-15 + platform: macos + maven: ./mvnw + - classifier: windows-x86_64 + runner: windows-2022 + platform: windows + maven: .\mvnw.cmd + - classifier: windows-aarch64 + runner: windows-11-arm + platform: windows + maven: .\mvnw.cmd + + steps: + - name: Check out source + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.git_ref || github.ref }} + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: maven + + - name: Install Linux build prerequisites + if: matrix.platform == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y flex bison g++ make + + - name: Install macOS build prerequisites + if: matrix.platform == 'macos' + run: brew install bison + + - name: Install Windows build prerequisites + if: matrix.platform == 'windows' + shell: pwsh + run: choco install winflexbison3 -y --no-progress + + - name: Verify active Maven classifier + working-directory: iotdb-tools-thrift + shell: pwsh + run: | + $expected = "${{ matrix.classifier }}" + $actual = & "${{ matrix.maven }}" -q help:evaluate "-Dexpression=os.classifier" "-DforceStdout" + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + $actual = ($actual | Select-Object -Last 1).Trim() + Write-Host "Expected classifier: $expected" + Write-Host "Maven classifier: $actual" + if ($actual -ne $expected) { + throw "Maven activated classifier '$actual', expected '$expected'" + } + + - name: Build artifact + working-directory: iotdb-tools-thrift + shell: pwsh + run: | + & "${{ matrix.maven }}" clean package -DskipTests + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + - name: Verify archive contents + working-directory: iotdb-tools-thrift + shell: pwsh + run: | + $classifier = "${{ matrix.classifier }}" + $zip = Get-ChildItem -Path target -Filter "*-$classifier.zip" | Select-Object -First 1 + if (-not $zip) { + throw "No zip file found for classifier $classifier" + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead($zip.FullName) + try { + $files = $archive.Entries | Where-Object { $_.Name } | ForEach-Object { $_.FullName } + } finally { + $archive.Dispose() + } + + $expectedBinary = if ($classifier.StartsWith("windows")) { "bin/Release/thrift.exe" } else { "bin/thrift" } + Write-Host "Archive: $($zip.Name)" + $files | ForEach-Object { Write-Host " - $_" } + + if ($files.Count -ne 1) { + throw "Expected exactly one file in the archive, found $($files.Count)" + } + if ($files -notcontains $expectedBinary) { + throw "Expected archive to contain $expectedBinary" + } + + $extractDir = Join-Path $env:RUNNER_TEMP "thrift-$classifier" + Remove-Item -Recurse -Force $extractDir -ErrorAction SilentlyContinue + Expand-Archive -Path $zip.FullName -DestinationPath $extractDir + $binary = Join-Path $extractDir $expectedBinary + if (-not (Test-Path $binary)) { + throw "Expected binary not found after extraction: $expectedBinary" + } + if (-not $classifier.StartsWith("windows")) { + chmod +x $binary + } + + $version = & $binary --version + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + $version = ($version | Select-Object -Last 1).Trim() + Write-Host "Thrift version output: $version" + if ($version -ne "Thrift version 0.23.0") { + throw "Unexpected thrift version output: $version" + } + + if ($classifier.StartsWith("linux")) { + $fileInfo = & file $binary + Write-Host "file output: $fileInfo" + if ($fileInfo -notmatch "statically linked") { + throw "Expected Linux thrift binary to be statically linked" + } + } + + - name: Upload platform artifact + uses: actions/upload-artifact@v4 + with: + name: iotdb-tools-thrift-${{ matrix.classifier }} + path: iotdb-tools-thrift/target/*-${{ matrix.classifier }}.zip + if-no-files-found: error + compression-level: 0 + retention-days: 14 + + bundle: + name: Bundle all platform artifacts + runs-on: ubuntu-24.04 + needs: build + steps: + - name: Download platform artifacts + uses: actions/download-artifact@v4 + with: + path: dist + pattern: iotdb-tools-thrift-* + merge-multiple: true + + - name: Verify bundled artifacts + shell: bash + run: | + set -euo pipefail + ls -lah dist + test "$(find dist -maxdepth 1 -name '*.zip' | wc -l)" -eq 6 + for classifier in linux-x86_64 linux-aarch64 mac-x86_64 mac-aarch64 windows-x86_64 windows-aarch64; do + test "$(find dist -maxdepth 1 -name "*-${classifier}.zip" | wc -l)" -eq 1 + done + + - name: Upload bundled artifact + uses: actions/upload-artifact@v4 + with: + name: iotdb-tools-thrift-all-platforms + path: dist/*.zip + if-no-files-found: error + compression-level: 0 + retention-days: 14 diff --git a/.gitignore b/.gitignore index dceb35b..bb3e9b2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ **/.DS_Store derby-tsfile-db /iotdb-tools-thrift/target/ +/iotdb-tools-thrift/prebuilt-artifacts/ /iotdb-tools-thrift/.mvn/wrapper/maven-wrapper.jar diff --git a/iotdb-tools-thrift/README.md b/iotdb-tools-thrift/README.md index f26c8eb..00bf7f0 100644 --- a/iotdb-tools-thrift/README.md +++ b/iotdb-tools-thrift/README.md @@ -21,31 +21,116 @@ # Releasing the IoTDB Tools: Thrift - ./mvnw clean deploy -P apache-release +This module publishes platform-specific Apache Thrift compiler archives used by +IoTDB builds. The archives contain the `thrift` executable only. + +## Prerequisites + +Install the following software before building this module: + +- JDK +- Flex +- Bison +- A C/C++ build toolchain supported by CMake +- GPG configured for Apache release signing +- Apache Nexus credentials configured as `apache.releases.https` in Maven `settings.xml` + +Linux profiles build the `thrift` executable with `-static`, so the build host +needs the static runtime libraries required by its C/C++ toolchain. + +Use `mvnw.cmd` instead of `./mvnw` on Windows. + +## Build Locally + +Run the following command from this directory: + + ./mvnw clean package -DskipTests + +The archive is generated under `target/`. Check that it contains `bin/thrift` +and that the binary reports the expected Apache Thrift version. + +## Build Release Artifacts With GitHub Actions + +The `Build IoTDB Tools Thrift Artifacts` workflow builds the six platform zip +artifacts without signing or deploying them. Trigger it manually from GitHub +Actions, optionally passing a branch, tag, or commit SHA in the `git_ref` input. +The workflow verifies that each generated compiler reports the expected Apache +Thrift version, and that Linux compilers are statically linked. + +The workflow uploads one bundled artifact named +`iotdb-tools-thrift-all-platforms`. Download and extract that artifact into this +directory: + + iotdb-tools-thrift/prebuilt-artifacts/ -On the first run, comment out the "stagingRepositoryId" property. -This will make the build deploy the rc in a new staging repository in nexus. +If you use the GitHub CLI, run: -As soon as the build is finished, go to https://repository.apache.org/#stagingRepositories and take the new repository id from there and comment in the property again and update the value to that of the staging repository and commit that. + gh run download --name iotdb-tools-thrift-all-platforms --dir prebuilt-artifacts -This will make it easy for the other platform deployment. +The directory must contain these files: -Then checkout the repo on the other platforms and run the following on each of the other platforms: +- `iotdb-tools-thrift-${project.version}-linux-x86_64.zip` +- `iotdb-tools-thrift-${project.version}-linux-aarch64.zip` +- `iotdb-tools-thrift-${project.version}-mac-x86_64.zip` +- `iotdb-tools-thrift-${project.version}-mac-aarch64.zip` +- `iotdb-tools-thrift-${project.version}-windows-x86_64.zip` +- `iotdb-tools-thrift-${project.version}-windows-aarch64.zip` + +`${project.version}` is the Maven project version, for example `0.23.0.0`. + +Verify the archives before deploying them. Each archive should contain only the +`bin/thrift` executable, or `bin/Release/thrift.exe` on Windows, and the +executable should report the expected Apache Thrift version. + +## Deploy Prebuilt Artifacts to Nexus + +Run the deploy locally from this directory. This signs and deploys the six +prebuilt platform artifacts from `prebuilt-artifacts/`: + + ./mvnw clean deploy -P apache-release,prebuilt-artifacts + +Use `prebuilt.artifacts.dir` if the downloaded artifacts are in another +directory: + + ./mvnw clean deploy -P apache-release,prebuilt-artifacts -Dprebuilt.artifacts.dir=/path/to/prebuilt-artifacts + +This creates a new staging repository in Nexus. After the deploy completes, open +https://repository.apache.org/#stagingRepositories and verify the uploaded +artifacts. + +If you need to re-run the local deploy into an existing staging repository, pass +that exact staging repository id: + + ./mvnw clean deploy -P apache-release,prebuilt-artifacts -DstagingRepositoryId=orgapacheiotdb-1234 + +The `stagingRepositoryId` value must be an existing Nexus staging repository id. +Do not use a made-up id. + +## Deploy by Building on Each Platform + +If you do not use the prebuilt artifacts workflow, you can still deploy by +building on each platform. + +Run the first deploy on one platform without `stagingRepositoryId`: ./mvnw clean deploy -P apache-release -> Note: For some reason you will see errors a the end when deploying the other artifacts, however in all cases I did see the artifacts deployed correctly. +This creates a new staging repository in Nexus. After the deploy completes, open +https://repository.apache.org/#stagingRepositories and copy the generated staging +repository id, for example `orgapacheiotdb-1234`. -Once this has been run on each of the supported platforms, go back to Nexus and close the staging repository. +Run the deploy on each remaining platform with that exact staging repository id: -## Prerequisites + ./mvnw clean deploy -P apache-release -DstagingRepositoryId=orgapacheiotdb-1234 -The following software needs to be installed in order to build the thrift module: +Supported classifiers are: -- Java -- Flex -- Bison -- Boost -- Ssl +- `linux-x86_64` +- `linux-aarch64` +- `mac-x86_64` +- `mac-aarch64` +- `windows-x86_64` +- `windows-aarch64` -Please look in the IoTDB documentation for information on how to install them on your particular OS. +After all platform archives have been deployed, verify the staging repository in +Nexus, close it, and continue with the Apache release vote and release process. diff --git a/iotdb-tools-thrift/README_zh.md b/iotdb-tools-thrift/README_zh.md new file mode 100644 index 0000000..3bdf09d --- /dev/null +++ b/iotdb-tools-thrift/README_zh.md @@ -0,0 +1,133 @@ + + +# 发布 IoTDB Tools: Thrift + +本模块用于发布 IoTDB 构建过程中使用的、按平台区分的 Apache Thrift +compiler 压缩包。压缩包中只包含 `thrift` 可执行文件。 + +## 环境准备 + +构建本模块前需要安装以下软件： + +- JDK +- Flex +- Bison +- CMake 支持的 C/C++ 构建工具链 +- 已配置好 Apache 发布签名的 GPG +- 在 Maven `settings.xml` 中配置好 server id 为 `apache.releases.https` 的 Apache Nexus 凭据 + +Linux profile 会用 `-static` 链接 `thrift` 可执行文件，因此构建机器需要具备 +当前 C/C++ 工具链静态链接所需的运行时库。 + +Windows 上请使用 `mvnw.cmd` 替代 `./mvnw`。 + +## 本地构建 + +在当前目录执行： + + ./mvnw clean package -DskipTests + +构建产物会生成到 `target/` 目录下。请确认压缩包中包含 `bin/thrift`，并确认该 +二进制文件输出的 Apache Thrift 版本符合预期。 + +## 使用 GitHub Actions 构建发布 artifacts + +`Build IoTDB Tools Thrift Artifacts` workflow 会构建六个平台的 zip artifacts， +但不会签名，也不会 deploy 到 Nexus。请在 GitHub Actions 页面手动触发该 +workflow，可以通过 `git_ref` 输入指定要构建的分支、tag 或 commit SHA。 +workflow 会验证每个平台生成的 compiler 是否输出预期的 Apache Thrift 版本， +并确认 Linux compiler 是静态链接的。 + +workflow 会上传一个名为 `iotdb-tools-thrift-all-platforms` 的汇总 artifact。 +下载并解压到当前模块的以下目录： + + iotdb-tools-thrift/prebuilt-artifacts/ + +如果使用 GitHub CLI，可以执行： + + gh run download --name iotdb-tools-thrift-all-platforms --dir prebuilt-artifacts + +该目录中必须包含以下文件： + +- `iotdb-tools-thrift-${project.version}-linux-x86_64.zip` +- `iotdb-tools-thrift-${project.version}-linux-aarch64.zip` +- `iotdb-tools-thrift-${project.version}-mac-x86_64.zip` +- `iotdb-tools-thrift-${project.version}-mac-aarch64.zip` +- `iotdb-tools-thrift-${project.version}-windows-x86_64.zip` +- `iotdb-tools-thrift-${project.version}-windows-aarch64.zip` + +`${project.version}` 是 Maven 项目版本，例如 `0.23.0.0`。 + +deploy 前请先检查这些压缩包。每个压缩包应当只包含 `bin/thrift` 可执行文件， +Windows 平台为 `bin/Release/thrift.exe`，并且该可执行文件输出的 Apache +Thrift 版本符合预期。 + +## 将预构建 artifacts 发布到 Nexus + +在当前目录本地执行 deploy。该命令会对 `prebuilt-artifacts/` 下的六个平台 +artifacts 签名并 deploy： + + ./mvnw clean deploy -P apache-release,prebuilt-artifacts + +如果下载目录不是默认的 `prebuilt-artifacts/`，可以通过 +`prebuilt.artifacts.dir` 指定： + + ./mvnw clean deploy -P apache-release,prebuilt-artifacts -Dprebuilt.artifacts.dir=/path/to/prebuilt-artifacts + +这一步会在 Nexus 中创建新的 staging repository。deploy 完成后，打开 +https://repository.apache.org/#stagingRepositories，检查上传的 artifacts。 + +如果需要重新 deploy 到已有 staging repository，请传入真实存在的 staging +repository id： + + ./mvnw clean deploy -P apache-release,prebuilt-artifacts -DstagingRepositoryId=orgapacheiotdb-1234 + +`stagingRepositoryId` 必须是 Nexus 里真实存在的 staging repository id，不能 +随便填写一个不存在的 id。 + +## 在每个平台本地构建并发布 + +如果不使用预构建 artifacts workflow，也可以继续按平台分别构建并 deploy。 + +先在一个平台上执行第一次 deploy，不要传入 `stagingRepositoryId`： + + ./mvnw clean deploy -P apache-release + +这一步会在 Nexus 中创建新的 staging repository。deploy 完成后，打开 +https://repository.apache.org/#stagingRepositories，复制新创建出来的 staging +repository id，例如 `orgapacheiotdb-1234`。 + +然后在其余平台上执行 deploy，并传入第一次创建出来的 staging repository id： + + ./mvnw clean deploy -P apache-release -DstagingRepositoryId=orgapacheiotdb-1234 + +支持的平台 classifier 包括： + +- `linux-x86_64` +- `linux-aarch64` +- `mac-x86_64` +- `mac-aarch64` +- `windows-x86_64` +- `windows-aarch64` + +所有平台压缩包都 deploy 完成后，在 Nexus 中检查 staging repository，确认无误后 +close，然后继续 Apache release vote 和 release 流程。 diff --git a/iotdb-tools-thrift/pom.xml b/iotdb-tools-thrift/pom.xml index 0de594d..a4c6395 100644 --- a/iotdb-tools-thrift/pom.xml +++ b/iotdb-tools-thrift/pom.xml @@ -33,20 +33,17 @@ contains nothing else as the compiled thrift binary without any modification. The fourth segment allows us to re-deploy, if we need to change the config. --> - 0.20.0.0 + 0.23.0.0 pom IoTDB-Tools: Thrift Local build of the Apache Thrift compiler. - 0.20.0 - ON - ON - ON - ON + 0.23.0 Unix Makefiles Release + ${project.basedir}/prebuilt-artifacts @@ -96,48 +93,39 @@ + + + - - + - + - - + + + + - + - + - - - - - - - - - - - - - @@ -159,14 +147,14 @@ org.apache.maven.plugins maven-assembly-plugin + binary-assembly package single @@ -208,11 +196,21 @@ + + com.googlecode.maven-download-plugin + download-maven-plugin + 1.7.1 + com.googlecode.cmake-maven-project cmake-maven-plugin 3.29.3-b2 + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + org.sonatype.plugins nexus-staging-maven-plugin @@ -231,7 +229,6 @@ apache.releases.https https://repository.apache.org true - orgapacheiotdb-1158 @@ -239,6 +236,28 @@ + + reuse-staging-repository + + + stagingRepositoryId + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + + ${stagingRepositoryId} + + + + + + + .os-unix @@ -434,5 +453,94 @@ Visual Studio 17 2022 + + + prebuilt-artifacts + + + + com.googlecode.maven-download-plugin + download-maven-plugin + + + get-thrift + none + + + + + com.googlecode.cmake-maven-project + cmake-maven-plugin + + + cmake-generate + none + + + cmake-compile + none + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + binary-assembly + none + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-prebuilt-artifacts + package + + attach-artifact + + + + + ${prebuilt.artifacts.dir}/${project.artifactId}-${project.version}-linux-x86_64.zip + zip + linux-x86_64 + + + ${prebuilt.artifacts.dir}/${project.artifactId}-${project.version}-linux-aarch64.zip + zip + linux-aarch64 + + + ${prebuilt.artifacts.dir}/${project.artifactId}-${project.version}-mac-x86_64.zip + zip + mac-x86_64 + + + ${prebuilt.artifacts.dir}/${project.artifactId}-${project.version}-mac-aarch64.zip + zip + mac-aarch64 + + + ${prebuilt.artifacts.dir}/${project.artifactId}-${project.version}-windows-x86_64.zip + zip + windows-x86_64 + + + ${prebuilt.artifacts.dir}/${project.artifactId}-${project.version}-windows-aarch64.zip + zip + windows-aarch64 + + + + + + + + + diff --git a/iotdb-tools-thrift/src/main/assembly/thrift.xml b/iotdb-tools-thrift/src/main/assembly/thrift.xml index ce0de79..fb0c7b1 100644 --- a/iotdb-tools-thrift/src/main/assembly/thrift.xml +++ b/iotdb-tools-thrift/src/main/assembly/thrift.xml @@ -35,31 +35,5 @@ bin - - - ${project.build.directory}/build - - lib/*.a - lib/Release/*.lib - lib/Debug/*.lib - - / - - - - ${project.build.directory}/thrift-${thrift.version}/lib/cpp/src - - ** - - include - - - - ${project.build.directory}/build/thrift - - config.h - - include/thrift -