Faiss test

.github/actions/setup-faiss/action.yml

@@ -0,0 +1,86 @@
+name: 'Setup FAISS'
+description: 'Install native dependencies and build FAISS library'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install native dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          build-essential \
+          libopenblas-dev \
+          liblapack-dev \
+          patchelf \
+          libgomp1 \
+          wget
+
+    - name: Install GCC 9
+      shell: bash
+      run: |
+        sudo apt-get install -y gcc-9 g++-9
+        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+        gcc --version
+        g++ --version
+        # Verify GCC version is >= 9.3.0
+        # Use -dumpfullversion for full version, fall back to -dumpversion
+        GCC_VERSION=$(gcc -dumpfullversion 2>/dev/null || gcc -dumpversion)
+        echo "GCC version: $GCC_VERSION"
+        # Extract major version
+        GCC_MAJOR=$(echo "$GCC_VERSION" | cut -d. -f1)
+        if [[ "$GCC_MAJOR" -lt 9 ]]; then
+          echo "ERROR: GCC major version must be >= 9, got $GCC_MAJOR"
+          exit 1
+        fi
+        echo "GCC version check passed: $GCC_VERSION (major: $GCC_MAJOR)"
+
+    - name: Install CMake 3.30.1
+      shell: bash
+      run: |
+        CMAKE_VERSION="3.30.1"
+        wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
+        tar -xzf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
+        sudo mv cmake-${CMAKE_VERSION}-linux-x86_64 /opt/cmake
+        sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
+        sudo ln -sf /opt/cmake/bin/ctest /usr/local/bin/ctest
+        sudo ln -sf /opt/cmake/bin/cpack /usr/local/bin/cpack
+        cmake --version
+        # Verify CMake version
+        CMAKE_INSTALLED=$(cmake --version | head -n1 | awk '{print $3}')
+        echo "CMake version: $CMAKE_INSTALLED"
+        if [[ "$(printf '%s\n' "3.30.1" "$CMAKE_INSTALLED" | sort -V | head -n1)" != "3.30.1" ]]; then
+          echo "ERROR: CMake version must be >= 3.30.1, got $CMAKE_INSTALLED"
+          exit 1
+        fi
+
+    - name: Install FAISS
+      shell: bash
+      run: |
+        # Clone and build FAISS
+        git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
+        cd /tmp/faiss
+        cmake -B build \
+          -DFAISS_ENABLE_GPU=OFF \
+          -DFAISS_ENABLE_PYTHON=OFF \
+          -DBUILD_TESTING=OFF \
+          -DCMAKE_BUILD_TYPE=Release
+        cmake --build build -j $(nproc)
+        sudo cmake --install build
+
+    - name: Build native library
+      shell: bash
+      run: |
+        cd paimon-faiss-jni
+        ./scripts/build-native.sh --clean --fat-lib
+
+    - name: Build paimon-faiss-jni
+      shell: bash
+      run: |
+        mvn -B clean install -pl paimon-faiss-jni -am -DskipTests -Ppaimon-faiss-vector
+
+    - name: Build paimon-faiss
+      shell: bash
+      run: |
+        mvn -B clean install -pl paimon-faiss -am -DskipTests -Ppaimon-faiss-vector

.github/workflows/faiss-vector-index-tests.yml

@@ -0,0 +1,92 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Faiss Vector Index Tests
+
+on:
+  push:
+    paths:
+      - 'paimon-faiss/**'
+      - 'paimon-faiss-jni/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
+  pull_request:
+    paths:
+      - 'paimon-faiss/**'
+      - 'paimon-faiss-jni/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
+
+env:
+  JDK_VERSION: 8
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Setup FAISS and build paimon-faiss
+        uses: ./.github/actions/setup-faiss
+
+      - name: List bundled libraries
+        run: |
+          echo "=== Bundled libraries ==="
+          ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
+          echo ""
+          echo "=== Library dependencies ==="
+          ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
+
+      - name: Test paimon-faiss-jni
+        timeout-minutes: 10
+        run: |
+          mvn -T 1C  -B test -pl paimon-faiss-jni -DskipFaissTests=false -Ppaimon-faiss-vector
+        env:
+          MAVEN_OPTS: -Xmx2048m
+
+      - name: Test paimon-faiss
+        timeout-minutes: 30
+        run: |
+          mvn -T 1C  -B test -pl paimon-faiss -Ppaimon-faiss-vector
+        env:
+          MAVEN_OPTS: -Xmx4096m
+
+      - name: Build Vector E2E Test Module
+        run: mvn -T 2C -B clean install -DskipTests -Pspark3,flink1,paimon-faiss-vector -pl paimon-vector-e2e-test -am
+
+      - name: Run Vector E2E Tests
+        timeout-minutes: 30
+        run: |
+          # run tests with random timezone to find out timezone related bugs
+          . .github/workflows/utils.sh
+          jvm_timezone=$(random_timezone)
+          echo "JVM timezone is set to $jvm_timezone"
+          mvn -T 2C -B verify -Pspark3,flink1,paimon-faiss-vector -pl paimon-vector-e2e-test -Duser.timezone=$jvm_timezone
+        env:
+          MAVEN_OPTS: -Xmx4096m