diff --git a/.github/actions/run-inference/action.yml b/.github/actions/run-inference/action.yml new file mode 100644 index 00000000..314a8be5 --- /dev/null +++ b/.github/actions/run-inference/action.yml @@ -0,0 +1,62 @@ +name: Run Inference +description: Run one llama-tornado inference pass and write the metrics + sidecar files. + +inputs: + backend: + description: 'GPU backend (opencl or ptx)' + required: true + model_file: + description: 'Model filename inside $MODELS_DIR (e.g. Llama-3.2-1B-Instruct-F16.gguf)' + required: true + model: + description: 'Human-readable model name for the sidecar (e.g. Llama-3.2-1B-Instruct)' + required: true + quantization: + description: 'Quantization type (e.g. F16, Q8_0)' + required: true + configuration: + description: 'Configuration key for the sidecar (e.g. standard, prefill-decode)' + required: true + flags: + description: 'Extra CLI flags passed to llama-tornado (omit for standard run)' + required: false + default: '' + metrics_file: + description: 'Absolute path for the output metrics JSON file' + required: true + prompt: + description: 'Prompt to pass to the model' + required: false + default: 'Say hello' + +runs: + using: composite + steps: + - name: Run inference + shell: bash + working-directory: ${{ github.workspace }} + env: + JAVA_TOOL_OPTIONS: >- + -Dllama.metrics.format=json + -Dllama.metrics.output=file + -Dllama.metrics.file=${{ inputs.metrics_file }} + METRICS_FILE: ${{ inputs.metrics_file }} + run: | + # Run inference and emit raw metrics JSON via JAVA_TOOL_OPTIONS + ./llama-tornado --gpu --${{ inputs.backend }} \ + --model $MODELS_DIR/${{ inputs.model_file }} \ + --prompt "${{ inputs.prompt }}" \ + ${{ inputs.flags }} + + # Write metadata sidecar so process_metrics.py can identify each metrics file + SIDECAR="${METRICS_FILE%.json}.meta.json" + python3 scripts/write_metrics_sidecar.py \ + --out "$SIDECAR" \ + backend="${{ inputs.backend }}" \ + task=llama-inference \ + model_file=${{ inputs.model_file }} \ + model=${{ inputs.model }} \ + quantization=${{ inputs.quantization }} \ + configuration=${{ inputs.configuration }} \ + "flags=${{ inputs.flags }}" \ + prompt="${{ inputs.prompt }}" diff --git a/.github/actions/setup-java/action.yml b/.github/actions/setup-java/action.yml new file mode 100644 index 00000000..334b5432 --- /dev/null +++ b/.github/actions/setup-java/action.yml @@ -0,0 +1,22 @@ +name: Setup Java +description: Install and activate a JDK via SDKMAN. Exports JAVA_HOME and updates PATH. + +inputs: + java_version: + description: 'SDKMAN Java version identifier (e.g. 21.0.2-open)' + required: true + +runs: + using: composite + steps: + - name: Set up Java with SDKMAN + shell: bash + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + if ! sdk list java | grep -q "${{ inputs.java_version }}"; then + sdk install java "${{ inputs.java_version }}" + fi + sdk use java "${{ inputs.java_version }}" + echo "JAVA_HOME=$HOME/.sdkman/candidates/java/current" >> $GITHUB_ENV + echo "$HOME/.sdkman/candidates/java/current/bin" >> $GITHUB_PATH + java -version diff --git a/.github/actions/setup-tornadovm/action.yml b/.github/actions/setup-tornadovm/action.yml new file mode 100644 index 00000000..3b1c5070 --- /dev/null +++ b/.github/actions/setup-tornadovm/action.yml @@ -0,0 +1,81 @@ +name: Setup TornadoVM +description: Build TornadoVM once per backend and reuse across runs via a local SHA sentinel. Exports TORNADOVM_HOME and updates PATH for all subsequent steps. + +inputs: + backend: + description: 'TornadoVM backend to build (opencl or ptx)' + required: true + +runs: + using: composite + steps: + - name: Get TornadoVM HEAD SHA + id: tornado_sha + shell: bash + run: | + SHA=$(git ls-remote https://github.com/beehive-lab/TornadoVM HEAD | cut -f1) + echo "sha=$SHA" >> $GITHUB_OUTPUT + + - name: Check local build sentinel + id: sentinel + shell: bash + run: | + SENTINEL="$TORNADO_ROOT/.built-sha" + if [ -f "$SENTINEL" ] && [ "$(cat $SENTINEL)" = "${{ steps.tornado_sha.outputs.sha }}" ]; then + echo "up-to-date=true" >> $GITHUB_OUTPUT + else + echo "up-to-date=false" >> $GITHUB_OUTPUT + fi + + - name: Clone TornadoVM master + if: steps.sentinel.outputs.up-to-date != 'true' + shell: bash + run: | + rm -rf $TORNADO_ROOT + git clone --depth 1 --branch master \ + https://github.com/beehive-lab/TornadoVM.git \ + $TORNADO_ROOT + + - name: Set up Python venv for TornadoVM + if: steps.sentinel.outputs.up-to-date != 'true' + shell: bash + run: | + python3 -m venv $TORNADO_ROOT/venv + source $TORNADO_ROOT/venv/bin/activate + python --version + + - name: Build TornadoVM + if: steps.sentinel.outputs.up-to-date != 'true' + shell: bash + run: | + cd $TORNADO_ROOT + mkdir -p graalJars && cp $GRAAL_JARS/* graalJars/ + source venv/bin/activate + echo "=== Building TornadoVM ===" + make BACKEND=${{ inputs.backend }} + echo "${{ steps.tornado_sha.outputs.sha }}" > .built-sha + + - name: Find TornadoVM SDK directory + id: find_sdk + shell: bash + run: | + SDK_DIR=$(find $TORNADO_ROOT/dist -type d -maxdepth 3 -path "*/tornadovm-*-${{ inputs.backend }}" | head -n 1) + if [ -z "$SDK_DIR" ]; then + echo "::error::Could not locate TornadoVM SDK directory!" + find $TORNADO_ROOT/dist -maxdepth 5 -type d + exit 1 + fi + echo "sdk_dir=$SDK_DIR" >> $GITHUB_OUTPUT + + # Runs on both fresh build and sentinel hit — sets TORNADOVM_HOME and PATH + # for all subsequent steps in the calling workflow. + - name: Configure TornadoVM environment + shell: bash + run: | + FULL_SDK="${{ steps.find_sdk.outputs.sdk_dir }}" + echo "Detected TornadoVM SDK: $FULL_SDK" + echo "TORNADOVM_HOME=$FULL_SDK" >> $GITHUB_ENV + echo "$FULL_SDK/bin" >> $GITHUB_PATH + echo "$JAVA_HOME/bin" >> $GITHUB_PATH + which tornado || { echo "::error::tornado not in PATH"; exit 1; } + tornado --devices diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml index a17c9228..2a19099f 100644 --- a/.github/workflows/build-and-run.yml +++ b/.github/workflows/build-and-run.yml @@ -8,11 +8,10 @@ on: types: [opened, synchronize, reopened] env: - JAVA_HOME: /opt/jenkins/jdks/graal-23.1.0/jdk-21.0.3 - TORNADO_ROOT: ${{ github.workspace }}/GPULlama3.java/external/tornadovm - LLAMA_ROOT: ${{ github.workspace }} + JAVA_VERSION: 21.0.2-open GRAAL_JARS: /opt/graalJars MODELS_DIR: /opt/models + QUARKUS_PORT: 8081 # History file committed back to the repo on push to main PERF_HISTORY_FILE: docs/perf-history.jsonl @@ -31,14 +30,17 @@ jobs: cd ${{ github.workspace }} # ./mvnw -T12C -Pspotless spotless:check - build-and-run: + # Build: TornadoVM → GPULlama3 → Quarkus LangChain4j + # max-parallel: 1 ensures the opencl and ptx variants run sequentially so + # there are no workspace conflicts between matrix jobs. + build: if: github.repository == 'beehive-lab/GPULlama3.java' runs-on: [self-hosted] needs: code-quality timeout-minutes: 30 - strategy: fail-fast: true + max-parallel: 1 matrix: backend: - name: opencl @@ -47,483 +49,265 @@ jobs: steps: - name: Checkout GPULlama3 uses: actions/checkout@v4 + with: + clean: false - - name: Clone TornadoVM master - run: | - git clone --depth 1 --branch master \ - https://github.com/beehive-lab/TornadoVM.git \ - $TORNADO_ROOT - - - name: Set up Python venv for TornadoVM - run: | - python3 -m venv $TORNADO_ROOT/venv - source $TORNADO_ROOT/venv/bin/activate - python --version + - name: Set up Java + uses: ./.github/actions/setup-java + with: + java_version: ${{ env.JAVA_VERSION }} - - name: Build TornadoVM - run: | - cd $TORNADO_ROOT - mkdir -p graalJars && cp $GRAAL_JARS/* graalJars/ - source venv/bin/activate - echo "=== Building TornadoVM ===" - - make BACKEND=${{ matrix.backend.name }} - - echo "=== Searching for TornadoVM SDK directory ===" - SDK_DIR=$(find dist -type d -maxdepth 3 -path "*/tornadovm-*-${{ matrix.backend.name }}" | head -n 1) - if [ -z "$SDK_DIR" ]; then - echo "::error::Could not locate TornadoVM SDK directory!" - find dist -maxdepth 5 -type d - exit 1 - fi - FULL_SDK="${PWD}/${SDK_DIR}" - echo "Detected TornadoVM SDK: $FULL_SDK" - - # Export for current shell session - export TORNADOVM_HOME="$FULL_SDK" - export PATH="$FULL_SDK/bin:$JAVA_HOME/bin:$PATH" - - # Save for subsequent steps - echo "TORNADOVM_HOME=$FULL_SDK" >> $GITHUB_ENV - echo "PATH=$PATH" >> $GITHUB_ENV - - echo "=== Checking tornado CLI ===" - which tornado || { echo "::error::tornado not in PATH"; exit 1; } - tornado --devices + - name: Setup TornadoVM + uses: ./.github/actions/setup-tornadovm + env: + TORNADO_ROOT: ${{ runner.tool_cache }}/tornadovm/tornadovm-${{ matrix.backend.name }} + with: + backend: ${{ matrix.backend.name }} - name: Build GPULlama3.java run: | - cd ${{ github.workspace }} - echo "Using TORNADOVM_HOME=$TORNADOVM_HOME" - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" tornado --version - ./mvnw clean package -DskipTests + # Strip any pre-existing -SNAPSHOT suffix before appending, making this step idempotent + # across sequential matrix variants (ptx runs after opencl on the same workspace). + BASE_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT$//') + GPULLAMA3_VERSION="${BASE_VERSION}-SNAPSHOT" + echo "GPULlama3 version: $GPULLAMA3_VERSION" + ./mvnw versions:set -DnewVersion=$GPULLAMA3_VERSION + ./mvnw clean install -DskipTests + echo "GPULLAMA3_VERSION=$GPULLAMA3_VERSION" >> $GITHUB_ENV + + - name: Clone Quarkus LangChain4j + run: | + rm -rf quarkus-langchain4j + git clone --depth 1 https://github.com/quarkiverse/quarkus-langchain4j.git - - name: FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Standard - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-standard.json + - name: Build Quarkus LangChain4j run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Llama-3.2-1B-Instruct-F16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Llama-3.2-1B-Instruct-F16.gguf \ - model=Llama-3.2-1B-Instruct \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + cd ${{ github.workspace }}/quarkus-langchain4j + sed -i 's/.*<\/gpu-llama3\.version>/'$GPULLAMA3_VERSION'<\/gpu-llama3.version>/' pom.xml + # -Dtornado activates the TornadoVM profile; -am builds only the gpu-llama3 module + deps + mvn clean install -pl integration-tests/gpu-llama3 -am -DskipTests -Dtornado - - name: FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Prefill-Decode + standalone-inference: + if: github.repository == 'beehive-lab/GPULlama3.java' + runs-on: [self-hosted] + needs: build + timeout-minutes: 30 + strategy: + fail-fast: true + matrix: + backend: + - name: opencl + - name: ptx + + steps: + - name: Checkout GPULlama3 + uses: actions/checkout@v4 + with: + clean: false + + - name: Set up Java + uses: ./.github/actions/setup-java + with: + java_version: ${{ env.JAVA_VERSION }} + + - name: Setup TornadoVM + uses: ./.github/actions/setup-tornadovm env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-prefill-decode.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Llama-3.2-1B-Instruct-F16.gguf \ - --prompt "Say hello" \ - --with-prefill-decode - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-prefill-decode.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Llama-3.2-1B-Instruct-F16.gguf \ - model=Llama-3.2-1B-Instruct \ - quantization=F16 \ - configuration=prefill-decode \ - "flags=--with-prefill-decode" \ - prompt="Say hello" + TORNADO_ROOT: ${{ runner.tool_cache }}/tornadovm/tornadovm-${{ matrix.backend.name }} + with: + backend: ${{ matrix.backend.name }} + + # Test standalone mode per model family and quantization + # Note: variants can be represented with matrices + - name: FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Standard + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Llama-3.2-1B-Instruct-F16.gguf + model: Llama-3.2-1B-Instruct + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-standard.json + + - name: FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Prefill-Decode + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Llama-3.2-1B-Instruct-F16.gguf + model: Llama-3.2-1B-Instruct + quantization: F16 + configuration: prefill-decode + flags: --with-prefill-decode + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-prefill-decode.json - name: FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Batch-Prefill-Decode - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-batch-prefill-decode.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Llama-3.2-1B-Instruct-F16.gguf \ - --prompt "Say hello" \ - --with-prefill-decode --batch-prefill-size 32 - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-batch-prefill-decode.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Llama-3.2-1B-Instruct-F16.gguf \ - model=Llama-3.2-1B-Instruct \ - quantization=F16 \ - configuration=batch-prefill-decode \ - "flags=--with-prefill-decode --batch-prefill-size 32" \ - prompt="Say hello" - - # ── PTX-only: CUDA-graph variants ──────────────────────────────────────── + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Llama-3.2-1B-Instruct-F16.gguf + model: Llama-3.2-1B-Instruct + quantization: F16 + configuration: batch-prefill-decode + flags: --with-prefill-decode --batch-prefill-size 32 + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-batch-prefill-decode.json + + # PTX-only: CUDA-graph variants - name: PTX - FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Prefill-Decode-CUDA-Graphs if: matrix.backend.name == 'ptx' - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-ptx-llama-1b-f16-prefill-decode-cuda-graphs.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --ptx \ - --model $MODELS_DIR/Llama-3.2-1B-Instruct-F16.gguf \ - --prompt "Say hello" \ - --with-prefill-decode \ - --cuda-graphs - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-ptx-llama-1b-f16-prefill-decode-cuda-graphs.meta.json" \ - backend=ptx \ - task=llama-inference \ - model_file=Llama-3.2-1B-Instruct-F16.gguf \ - model=Llama-3.2-1B-Instruct \ - quantization=F16 \ - configuration=prefill-decode-cuda-graphs \ - "flags=--with-prefill-decode --cuda-graphs" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Llama-3.2-1B-Instruct-F16.gguf + model: Llama-3.2-1B-Instruct + quantization: F16 + configuration: prefill-decode-cuda-graphs + flags: --with-prefill-decode --cuda-graphs + metrics_file: ${{ runner.temp }}/metrics-ptx-llama-1b-f16-prefill-decode-cuda-graphs.json - name: PTX - FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Batch-Prefill-Decode-CUDA-Graphs if: matrix.backend.name == 'ptx' - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-ptx-llama-1b-f16-batch-prefill-decode-cuda-graphs.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --ptx \ - --model $MODELS_DIR/Llama-3.2-1B-Instruct-F16.gguf \ - --prompt "Say hello" \ - --with-prefill-decode --batch-prefill-size 32 \ - --cuda-graphs - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-ptx-llama-1b-f16-batch-prefill-decode-cuda-graphs.meta.json" \ - backend=ptx \ - task=llama-inference \ - model_file=Llama-3.2-1B-Instruct-F16.gguf \ - model=Llama-3.2-1B-Instruct \ - quantization=F16 \ - configuration=batch-prefill-decode-cuda-graphs \ - "flags=--with-prefill-decode --batch-prefill-size 32 --cuda-graphs" \ - prompt="Say hello" - - # ── Additional models — standard inference, all backends ───────────────── + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Llama-3.2-1B-Instruct-F16.gguf + model: Llama-3.2-1B-Instruct + quantization: F16 + configuration: batch-prefill-decode-cuda-graphs + flags: --with-prefill-decode --batch-prefill-size 32 --cuda-graphs + metrics_file: ${{ runner.temp }}/metrics-ptx-llama-1b-f16-batch-prefill-decode-cuda-graphs.json + - name: FP16 - Run Qwen3-4B-f16.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-4b-f16-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Qwen3-4B-f16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-4b-f16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Qwen3-4B-f16.gguf \ - model=Qwen3-4B \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Qwen3-4B-f16.gguf + model: Qwen3-4B + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-4b-f16-standard.json - name: FP16 - Run Mistral-7B-Instruct-v0.3.fp16.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-fp16-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Mistral-7B-Instruct-v0.3.fp16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-fp16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Mistral-7B-Instruct-v0.3.fp16.gguf \ - model=Mistral-7B-Instruct-v0.3 \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Mistral-7B-Instruct-v0.3.fp16.gguf + model: Mistral-7B-Instruct-v0.3 + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-fp16-standard.json - name: FP16 - Run Qwen2.5-1.5b-instruct-fp16.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-fp16-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/qwen2.5-1.5b-instruct-fp16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-fp16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=qwen2.5-1.5b-instruct-fp16.gguf \ - model=Qwen2.5-1.5B-Instruct \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: qwen2.5-1.5b-instruct-fp16.gguf + model: Qwen2.5-1.5B-Instruct + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-fp16-standard.json - name: FP16 - Run Phi-3-mini-4k-instruct-fp16.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-fp16-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Phi-3-mini-4k-instruct-fp16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-fp16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Phi-3-mini-4k-instruct-fp16.gguf \ - model=Phi-3-mini-4k-instruct \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Phi-3-mini-4k-instruct-fp16.gguf + model: Phi-3-mini-4k-instruct + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-fp16-standard.json - name: FP16 - Run Granite-3.2-2b-instruct-f16.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-f16-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/granite-3.2-2b-instruct-f16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-f16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=granite-3.2-2b-instruct-f16.gguf \ - model=Granite-3.2-2B-Instruct \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: granite-3.2-2b-instruct-f16.gguf + model: Granite-3.2-2B-Instruct + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-f16-standard.json - name: FP16 - Run Granite-4.0-1b-F16.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-4-0-1b-f16-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/granite-4.0-1b-F16.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-4-0-1b-f16-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=granite-4.0-1b-F16.gguf \ - model=Granite-4.0-1B \ - quantization=F16 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: granite-4.0-1b-F16.gguf + model: Granite-4.0-1B + quantization: F16 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-4-0-1b-f16-standard.json - name: Q8 - Run Llama-3.2-1B-Instruct-Q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Llama-3.2-1B-Instruct-Q8_0.gguf \ - model=Llama-3.2-1B-Instruct \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Llama-3.2-1B-Instruct-Q8_0.gguf + model: Llama-3.2-1B-Instruct + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-q8-standard.json - name: Q8 - Run Qwen3-0.6B-Q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0-6b-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Qwen3-0.6B-Q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0-6b-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Qwen3-0.6B-Q8_0.gguf \ - model=Qwen3-0.6B \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Qwen3-0.6B-Q8_0.gguf + model: Qwen3-0.6B + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0-6b-q8-standard.json - name: Q8 - Run Phi-3-mini-4k-instruct-Q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Phi-3-mini-4k-instruct-Q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Phi-3-mini-4k-instruct-Q8_0.gguf \ - model=Phi-3-mini-4k-instruct \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Phi-3-mini-4k-instruct-Q8_0.gguf + model: Phi-3-mini-4k-instruct + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-q8-standard.json - name: Q8 - Run Qwen2.5-1.5b-instruct-q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/qwen2.5-1.5b-instruct-q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=qwen2.5-1.5b-instruct-q8_0.gguf \ - model=Qwen2.5-1.5B-Instruct \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" - - - name: Q8 - Mistral-7B-Instruct-v0.3.Q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/Mistral-7B-Instruct-v0.3.Q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=Mistral-7B-Instruct-v0.3.Q8_0.gguf \ - model=Mistral-7B-Instruct-v0.3 \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: qwen2.5-1.5b-instruct-q8_0.gguf + model: Qwen2.5-1.5B-Instruct + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-q8-standard.json + + - name: Q8 - Run Mistral-7B-Instruct-v0.3.Q8_0.gguf + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: Mistral-7B-Instruct-v0.3.Q8_0.gguf + model: Mistral-7B-Instruct-v0.3 + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-q8-standard.json - name: Q8 - Run Granite-3.2-2b-instruct-Q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/granite-3.2-2b-instruct-Q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=granite-3.2-2b-instruct-Q8_0.gguf \ - model=Granite-3.2-2B-Instruct \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: granite-3.2-2b-instruct-Q8_0.gguf + model: Granite-3.2-2B-Instruct + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-q8-standard.json - name: Q8 - Run Granite-4.0-1b-Q8_0.gguf - env: - JAVA_TOOL_OPTIONS: >- - -Dllama.metrics.format=json - -Dllama.metrics.output=file - -Dllama.metrics.file=${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-4-0-1b-q8-standard.json - run: | - cd ${{ github.workspace }} - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ - --model $MODELS_DIR/granite-4.0-1b-Q8_0.gguf \ - --prompt "Say hello" - python3 scripts/write_metrics_sidecar.py \ - --out "${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-4-0-1b-q8-standard.meta.json" \ - backend="${{ matrix.backend.name }}" \ - task=llama-inference \ - model_file=granite-4.0-1b-Q8_0.gguf \ - model=Granite-4.0-1B \ - quantization=Q8_0 \ - configuration=standard \ - flags="" \ - prompt="Say hello" - - # ── Upload metrics for the publish job ──────────────────────────────────── + uses: ./.github/actions/run-inference + with: + backend: ${{ matrix.backend.name }} + model_file: granite-4.0-1b-Q8_0.gguf + model: Granite-4.0-1B + quantization: Q8_0 + configuration: standard + metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-4-0-1b-q8-standard.json + + # Upload metrics for the publish job - name: Upload metrics artifacts if: always() uses: actions/upload-artifact@v4 @@ -532,7 +316,102 @@ jobs: path: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-*.json if-no-files-found: warn - # ── Separate job: collect all matrix metrics and update history ─────────────── + # Test integration with Quarkus-langchain4j + quarkus-langchain4j-integration: + if: github.repository == 'beehive-lab/GPULlama3.java' + runs-on: [self-hosted] + needs: build + timeout-minutes: 10 + strategy: + fail-fast: true + matrix: + backend: + - name: opencl + - name: ptx + + steps: + - name: Checkout GPULlama3 + uses: actions/checkout@v4 + with: + clean: false + + - name: Set up Java + uses: ./.github/actions/setup-java + with: + java_version: ${{ env.JAVA_VERSION }} + + - name: Setup TornadoVM + uses: ./.github/actions/setup-tornadovm + env: + TORNADO_ROOT: ${{ runner.tool_cache }}/tornadovm/tornadovm-${{ matrix.backend.name }} + with: + backend: ${{ matrix.backend.name }} + + - name: Verify GPULlama3 Dependency + run: | + cd ${{ github.workspace }}/quarkus-langchain4j/integration-tests/gpu-llama3 + mvn dependency:tree | grep "io.github.beehive-lab:gpu-llama3" + + - name: Start Quarkus Application + run: | + cd ${{ github.workspace }}/quarkus-langchain4j/integration-tests/gpu-llama3 + java @"$TORNADOVM_HOME/tornado-argfile" \ + -Dtornado.device.memory=8GB \ + -Dquarkus.http.port=$QUARKUS_PORT \ + -jar target/quarkus-app/quarkus-run.jar & + APP_PID=$! + echo "APP_PID=$APP_PID" >> $GITHUB_ENV + for i in {1..30}; do + if curl -s http://localhost:$QUARKUS_PORT/q/health > /dev/null 2>&1; then + echo "Application ready after ${i} seconds" + break + elif [ $i -eq 30 ]; then + echo "::error::Application failed to start within 30 seconds" + kill $APP_PID || true + exit 1 + else + [ $((i % 5)) -eq 0 ] && echo "Still waiting... (${i}s)" + sleep 1 + fi + done + + - name: Trigger Blocking Endpoint + run: | + for attempt in 1 2 3; do + echo "Attempt $attempt of 3 for blocking endpoint..." + HTTP_RESPONSE=$(curl -s -w "%{http_code}" http://localhost:$QUARKUS_PORT/chat/blocking) + HTTP_CODE="${HTTP_RESPONSE: -3}" + if [ "$HTTP_CODE" = "200" ]; then + echo "SUCCESS: HTTP $HTTP_CODE" + echo "Response body: ${HTTP_RESPONSE%???}" + break + fi + echo "Failed: HTTP $HTTP_CODE" + [ $attempt -lt 3 ] && sleep 2 + [ $attempt -eq 3 ] && { echo "::error::Blocking endpoint failed after 3 attempts"; exit 1; } + done + + - name: Trigger Streaming Endpoint + run: | + for attempt in 1 2 3; do + echo "Attempt $attempt of 3 for streaming endpoint..." + HTTP_CODE=$(timeout 10s curl -s -o /dev/null -w "%{http_code}" http://localhost:$QUARKUS_PORT/chat/streaming) + if [ "$HTTP_CODE" = "200" ]; then + echo "SUCCESS: HTTP $HTTP_CODE" + break + fi + echo "Failed: HTTP $HTTP_CODE" + [ $attempt -lt 3 ] && sleep 2 + [ $attempt -eq 3 ] && { echo "::error::Streaming endpoint failed after 3 attempts"; exit 1; } + done + + - name: Cleanup & Shutdown + if: always() + run: | + kill $APP_PID || true + wait $APP_PID 2>/dev/null || true + + # Collect all matrix metrics and update history publish-performance-history: # Guard: only commit history on real pushes to main, not on PRs or forks. # Prevents duplicate entries from PR runs and avoids push-permission errors on forks. @@ -540,8 +419,9 @@ jobs: github.repository == 'beehive-lab/GPULlama3.java' && github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: [self-hosted] - needs: build-and-run + needs: standalone-inference timeout-minutes: 15 steps: @@ -559,24 +439,36 @@ jobs: run: | python3 scripts/process_metrics.py \ --metrics-dir "${{ runner.temp }}/metrics-artifacts" \ - --commit "${{ github.sha }}" \ - --branch "${{ github.ref_name }}" \ - --run-id "${{ github.run_id }}" \ - --run-number "${{ github.run_number }}" \ - --run-attempt "${{ github.run_attempt }}" \ - --workflow "${{ github.workflow }}" \ - --history "$PERF_HISTORY_FILE" + --commit "${{ github.sha }}" \ + --branch "${{ github.ref_name }}" \ + --run-id "${{ github.run_id }}" \ + --run-number "${{ github.run_number }}" \ + --run-attempt "${{ github.run_attempt }}" \ + --workflow "${{ github.workflow }}" \ + --history "$PERF_HISTORY_FILE" - name: Commit performance history run: | - git config user.name "github-actions[bot]" + SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-8) + + git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + git add "$PERF_HISTORY_FILE" - git diff --cached --quiet && echo "No history changes to commit" && exit 0 - git commit -m "perf: record run #${{ github.run_number }} @ ${GITHUB_SHA::8}" + + git diff --cached --quiet && \ + echo "No history changes to commit" && exit 0 + + git commit -m "perf: record run #${{ github.run_number }} @ ${SHORT_SHA}" + for attempt in 1 2 3; do git pull --rebase origin main && git push && break || { - [ $attempt -lt 3 ] && { echo "Attempt $attempt failed, retrying in $((attempt * 5))s..."; sleep $((attempt * 5)); } \ - || { echo "::error::Failed to push after 3 attempts"; exit 1; } + if [ $attempt -lt 3 ]; then + echo "Attempt $attempt failed, retrying in $((attempt * 5))s..." + sleep $((attempt * 5)) + else + echo "::error::Failed to push after 3 attempts" + exit 1 + fi } done diff --git a/.github/workflows/deploy-maven-central.yml b/.github/workflows/deploy-maven-central.yml index 88bcee8e..53f62226 100644 --- a/.github/workflows/deploy-maven-central.yml +++ b/.github/workflows/deploy-maven-central.yml @@ -31,19 +31,18 @@ jobs: matrix: jdk: - name: jdk21 - java_home: /opt/jenkins/jdks/graal-23.1.0/jdk-21.0.3 + java_version: 21.0.2-open - name: jdk25 - java_home: /opt/jenkins/jdks/jdk-25.0.2 - env: - JAVA_HOME: ${{ matrix.jdk.java_home }} + java_version: 25.0.2-open steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup environment - run: | - echo "$JAVA_HOME/bin" >> $GITHUB_PATH + - name: Set up Java + uses: ./.github/actions/setup-java + with: + java_version: ${{ matrix.jdk.java_version }} - name: Configure Maven settings run: | diff --git a/.github/workflows/integration-quarkus-langchain4j.yml b/.github/workflows/integration-quarkus-langchain4j.yml deleted file mode 100644 index 8ee6e900..00000000 --- a/.github/workflows/integration-quarkus-langchain4j.yml +++ /dev/null @@ -1,260 +0,0 @@ -name: Integration Quarkus-LangChain4j - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - types: [opened, synchronize, reopened] - schedule: - # Run every Saturday at 02:30 UTC to catch dependency breakages - - cron: '30 2 * * 6' - workflow_dispatch: - inputs: - quarkus_langchain4j_version: - description: 'Quarkus LangChain4j version to test against' - required: false - type: string - quarkus_version: - description: 'Quarkus platform version to test against' - required: false - type: string - -env: - JAVA_HOME: /opt/jenkins/jdks/graal-23.1.0/jdk-21.0.3 - TORNADO_ROOT: ${{ github.workspace }}/GPULlama3.java/external/tornadovm - GRAAL_JARS: /opt/graalJars - QUARKUS_PORT: 8081 - -jobs: - quarkus-integration-test: - if: github.repository == 'beehive-lab/GPULlama3.java' - runs-on: [self-hosted] - timeout-minutes: 30 - strategy: - fail-fast: true - matrix: - backend: - - name: opencl - - name: ptx - - steps: - - name: Checkout GPULlama3 - uses: actions/checkout@v4 - - # Step 1: Clone and build TornadoVM - - name: Clone TornadoVM master - run: | - git clone --depth 1 --branch master \ - https://github.com/beehive-lab/TornadoVM.git \ - $TORNADO_ROOT - - name: Set up Python venv for TornadoVM - run: | - python3 -m venv $TORNADO_ROOT/venv - source $TORNADO_ROOT/venv/bin/activate - python --version - - name: Build TornadoVM - run: | - cd $TORNADO_ROOT - mkdir -p graalJars && cp $GRAAL_JARS/* graalJars/ - source venv/bin/activate - echo "=== Building TornadoVM ===" - - make BACKEND=${{ matrix.backend.name }} - - echo "=== Searching for TornadoVM SDK directory ===" - SDK_DIR=$(find dist -type d -maxdepth 3 -path "*/tornadovm-*-${{ matrix.backend.name }}" | head -n 1) - if [ -z "$SDK_DIR" ]; then - echo "::error::Could not locate TornadoVM SDK directory!" - find dist -maxdepth 5 -type d - exit 1 - fi - FULL_SDK="${PWD}/${SDK_DIR}" - echo "Detected TornadoVM SDK: $FULL_SDK" - - # Export for current shell session - export TORNADOVM_HOME="$FULL_SDK" - export PATH="$FULL_SDK/bin:$JAVA_HOME/bin:$PATH" - - # Save for subsequent steps - echo "TORNADOVM_HOME=$FULL_SDK" >> $GITHUB_ENV - echo "PATH=$PATH" >> $GITHUB_ENV - - echo "=== Checking tornado CLI ===" - which tornado || { echo "::error::tornado not in PATH"; exit 1; } - tornado --devices - - # Step 2: Build GPULlama3.java - - name: Build GPULlama3.java - run: | - cd ${{ github.workspace }} - echo "Using TORNADOVM_HOME=$TORNADOVM_HOME" - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - tornado --version - - # Append SNAPSHOT to GPULlama3 version - GPULLAMA3_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout) - GPULLAMA3_VERSION="${GPULLAMA3_VERSION}-SNAPSHOT" - echo "GPULlama3 version: $GPULLAMA3_VERSION" - ./mvnw versions:set -DnewVersion=$GPULLAMA3_VERSION - - # Build - ./mvnw clean install -DskipTests - - # Save GPULlama3.java version for subsequent steps - echo "GPULLAMA3_VERSION=$GPULLAMA3_VERSION" >> $GITHUB_ENV - - # Step 3: Clone Quarkus LangChain4j - - name: Clone Quarkus LangChain4j - run: | - cd ${{ github.workspace }} - git clone https://github.com/quarkiverse/quarkus-langchain4j.git - - # Step 4: Build Quarkus LangChain4j with current GPULlama3.java - - name: Build Quarkus LangChain4j - run: | - cd ${{ github.workspace }}/quarkus-langchain4j - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - - # Update the GPULlama3 version used by quarkus-langchain4j - POM="pom.xml" - sed -i 's/.*<\/gpu-llama3\.version>/'$GPULLAMA3_VERSION'<\/gpu-llama3.version>/' "$POM" - - # Use reactor to build *only *GPULlama3 integration test + dependencies - # This recompiles everything with the same Java version, avoiding compatibility issues - # The -Dtornado flag activates the TornadoVM profile which includes gpu-llama3 module - mvn clean install -pl integration-tests/gpu-llama3 -am -DskipTests -Dtornado - - # Step 4.5: Verify GPULlama3 JAR - - name: Verify GPULlama3 Dependency - run: | - cd ${{ github.workspace }}/quarkus-langchain4j/integration-tests/gpu-llama3 - echo "GPULlama3 dependency used by Quarkus-LangChain4j:" - mvn dependency:tree | grep "io.github.beehive-lab:gpu-llama3" - - # Step 5: Start Quarkus Application and Wait for Startup - - name: Start Quarkus Application and Wait for Startup - run: | - cd ${{ github.workspace }}/quarkus-langchain4j/integration-tests/gpu-llama3 - export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - - echo "Starting Quarkus application on port $QUARKUS_PORT..." - - # Start the Quarkus application in the background - java @"$TORNADOVM_HOME/tornado-argfile" \ - -Dtornado.device.memory=8GB \ - -Dquarkus.http.port=$QUARKUS_PORT \ - -jar target/quarkus-app/quarkus-run.jar & - APP_PID=$! - - if [ -z "$APP_PID" ]; then - echo "ERROR: Failed to start Quarkus application" - exit 1 - fi - - echo "Waiting for Quarkus application to start..." - - # Wait for application to be ready - for i in {1..30}; do - if curl -s http://localhost:$QUARKUS_PORT/q/health > /dev/null 2>&1; then - echo "Application ready after ${i} seconds" - echo "Health endpoint: http://localhost:$QUARKUS_PORT/q/health" - break - elif [ $i -eq 30 ]; then - echo "ERROR: Application failed to start within 30 seconds" - echo "Debugging info:" - echo "- Port: $QUARKUS_PORT" - echo "- Process ID: $APP_PID" - echo "- Health URL: http://localhost:$QUARKUS_PORT/q/health" - kill $APP_PID || true - exit 1 - else - [ $((i % 5)) -eq 0 ] && echo "Still waiting... (${i}s)" - sleep 1 - fi - done - - # Step 6: Run test 1 - - name: Trigger Blocking Endpoint - run: | - MAX_ATTEMPTS=3 - ATTEMPT=1 - SUCCESS=false - - while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do - echo "Attempt $ATTEMPT of $MAX_ATTEMPTS for blocking endpoint..." - - # Trigger endpoint - HTTP_RESPONSE=$(curl -s -w "%{http_code}" http://localhost:$QUARKUS_PORT/chat/blocking) - HTTP_RESPONSE_CODE="${HTTP_RESPONSE: -3}" - HTTP_RESPONSE_BODY="${HTTP_RESPONSE%???}" - - # Check response code - if [ "$HTTP_RESPONSE_CODE" = "200" ]; then - echo "SUCCESS: Blocking endpoint returned HTTP code: $HTTP_RESPONSE_CODE" - echo "HTTP Response body: $HTTP_RESPONSE_BODY" - SUCCESS=true - break - else - echo "Attempt $ATTEMPT failed: Blocking endpoint returned HTTP code $HTTP_RESPONSE_CODE" - echo "Response body: $HTTP_RESPONSE_BODY" - - if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then - echo "Retrying in 2 seconds..." - sleep 2 - fi - fi - - ATTEMPT=$((ATTEMPT + 1)) - done - - if [ "$SUCCESS" = false ]; then - echo "ERROR: Blocking endpoint failed after $MAX_ATTEMPTS attempts" - exit 1 - fi - - # Step 7: Run test 2 - - name: Trigger Streaming Endpoint - run: | - MAX_ATTEMPTS=3 - ATTEMPT=1 - SUCCESS=false - - while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do - echo "Attempt $ATTEMPT of $MAX_ATTEMPTS for streaming endpoint..." - - # Trigger endpoint - HTTP_RESPONSE=$(timeout 10s curl -s -w "%{http_code}" http://localhost:$QUARKUS_PORT/chat/streaming) - HTTP_RESPONSE_CODE="${HTTP_RESPONSE: -3}" - #HTTP_RESPONSE_BODY="$HTTP_RESPONSE%???}" - - # Check response code - if [ "$HTTP_RESPONSE_CODE" == "200" ]; then - echo "SUCCESS: Streaming endpoint returned HTTP code: ${HTTP_RESPONSE: -3}" - # do not show ugly streaming response body - #echo "HTTP Response body: $HTTP_RESPONSE_BODY" - SUCCESS=true - break - else - echo "Attempt $ATTEMPT failed: Streaming endpoint returned HTTP code $HTTP_RESPONSE_CODE" - - if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then - echo "Retrying in 2 seconds..." - sleep 2 - fi - fi - - ATTEMPT=$((ATTEMPT + 1)) - done - - if [ "$SUCCESS" = false ]; then - echo "ERROR: Streaming endpoint failed after $MAX_ATTEMPTS attempts" - exit 1 - fi - - # Step 8: Cleanup & Shutdown - - name: Cleanup & Shutdown - run: | - # Clean shutdown - kill $APP_PID || true - wait $APP_PID 2>/dev/null || true diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 3ff71f90..00d6c63b 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -20,6 +20,7 @@ on: env: VERSION: ${{ inputs.version }} PREV_VERSION: ${{ inputs.previous_version }} + JAVA_VERSION: 21.0.2-open jobs: prepare-release: @@ -29,8 +30,6 @@ jobs: contents: write pull-requests: write timeout-minutes: 15 - env: - JAVA_HOME: /opt/jenkins/jdks/graal-23.1.0/jdk-21.0.3 steps: - name: Validate version format @@ -48,9 +47,10 @@ jobs: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup environment - run: | - echo "$JAVA_HOME/bin" >> $GITHUB_PATH + - name: Set up Java + uses: ./.github/actions/setup-java + with: + java_version: ${{ env.JAVA_VERSION }} - name: Configure Git run: | diff --git a/.github/workflows/rerun-workflow.yml b/.github/workflows/rerun-workflow.yml index e4b14e75..62eec7d3 100644 --- a/.github/workflows/rerun-workflow.yml +++ b/.github/workflows/rerun-workflow.yml @@ -47,8 +47,30 @@ jobs: core.setOutput('is_help', 'false'); } - - name: Get PR SHA + - name: Check commenter permissions + id: check_permission if: steps.help.outputs.is_help != 'true' + uses: actions/github-script@v7 + with: + script: | + const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: context.payload.comment.user.login + }); + const authorized = ['write', 'admin'].includes(perm.permission); + if (!authorized) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `@${context.payload.comment.user.login} You need write permission to trigger workflow reruns.` + }); + } + core.setOutput('authorized', authorized ? 'true' : 'false'); + + - name: Get PR SHA + if: steps.help.outputs.is_help != 'true' && steps.check_permission.outputs.authorized == 'true' id: pr uses: actions/github-script@v7 with: @@ -64,7 +86,7 @@ jobs: console.log(`PR head ref: ${pr.head.ref}`); - name: Add reaction to comment - if: steps.help.outputs.is_help != 'true' + if: steps.help.outputs.is_help != 'true' && steps.check_permission.outputs.authorized == 'true' uses: actions/github-script@v7 with: script: | @@ -76,7 +98,7 @@ jobs: }); - name: Post start comment - if: steps.help.outputs.is_help != 'true' + if: steps.help.outputs.is_help != 'true' && steps.check_permission.outputs.authorized == 'true' uses: actions/github-script@v7 with: script: | @@ -92,7 +114,7 @@ jobs: }); - name: Rerun failed workflows - if: steps.help.outputs.is_help != 'true' + if: steps.help.outputs.is_help != 'true' && steps.check_permission.outputs.authorized == 'true' uses: actions/github-script@v7 with: script: | @@ -167,7 +189,7 @@ jobs: console.log(`Reran ${rerunCount} workflow(s)`); - name: Post completion comment - if: always() && steps.help.outputs.is_help != 'true' + if: always() && steps.help.outputs.is_help != 'true' && steps.check_permission.outputs.authorized == 'true' uses: actions/github-script@v7 with: script: |