Support Data Center precompiled driver container for Arm (Ubuntu 24.04) #533
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -28,6 +28,7 @@ jobs: | |||||
| kernel_flavors: ${{ steps.extract_driver_branch.outputs.kernel_flavors }} | ||||||
| dist: ${{ steps.extract_driver_branch.outputs.dist }} | ||||||
| lts_kernel: ${{ steps.extract_driver_branch.outputs.lts_kernel }} | ||||||
| platforms: ${{ steps.extract_driver_branch.outputs.platforms }} | ||||||
| steps: | ||||||
| - name: Checkout code | ||||||
| uses: actions/checkout@v6 | ||||||
|
|
@@ -54,6 +55,11 @@ jobs: | |||||
| lts_kernel_json=$(printf '%s\n' "${LTS_KERNEL[@]}" | jq -R . | jq -cs .) | ||||||
| echo "lts_kernel=$lts_kernel_json" >> $GITHUB_OUTPUT | ||||||
|
|
||||||
| # platforms for precompiled build (amd64 always; arm64 for ubuntu24.04) | ||||||
| PLATFORMS=("amd64" "arm64") | ||||||
| platforms_json=$(printf '%s\n' "${PLATFORMS[@]}" | jq -R . | jq -cs .) | ||||||
| echo "platforms=$platforms_json" >> $GITHUB_OUTPUT | ||||||
|
|
||||||
| precompiled-build-image: | ||||||
| needs: set-driver-version-matrix | ||||||
| runs-on: linux-amd64-cpu4 | ||||||
|
|
@@ -63,13 +69,16 @@ jobs: | |||||
| flavor: ${{ fromJson(needs.set-driver-version-matrix.outputs.kernel_flavors) }} | ||||||
| dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }} | ||||||
| lts_kernel: ${{ fromJson(needs.set-driver-version-matrix.outputs.lts_kernel) }} | ||||||
| platform: ${{ fromJson(needs.set-driver-version-matrix.outputs.platforms) }} | ||||||
| exclude: | ||||||
| - dist: ubuntu24.04 | ||||||
| driver_branch: 535 | ||||||
| - lts_kernel: 5.15 | ||||||
| dist: ubuntu24.04 | ||||||
| - flavor: azure-fde | ||||||
| dist: ubuntu22.04 | ||||||
| - dist: ubuntu22.04 | ||||||
| platform: arm64 | ||||||
| steps: | ||||||
| - uses: actions/checkout@v6 | ||||||
| name: Check out code | ||||||
|
|
@@ -83,6 +92,7 @@ jobs: | |||||
| GENERATE_ARTIFACTS="false" | ||||||
| echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV | ||||||
| echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV | ||||||
| echo "DOCKER_BUILD_PLATFORM_OPTIONS=--platform=linux/${{ matrix.platform }}" >> $GITHUB_ENV | ||||||
|
|
||||||
| - name: Set up QEMU | ||||||
| uses: docker/setup-qemu-action@v3 | ||||||
|
|
@@ -100,6 +110,7 @@ jobs: | |||||
| env: | ||||||
| IMAGE_NAME: ghcr.io/nvidia/driver | ||||||
| LTS_KERNEL: ${{ matrix.lts_kernel }} | ||||||
| DOCKER_BUILD_PLATFORM_OPTIONS: --platform=linux/${{ matrix.platform }} | ||||||
| run: | | ||||||
| if [[ "${{ matrix.dist }}" == "ubuntu22.04" ]]; then | ||||||
| BASE_TARGET="jammy" | ||||||
|
|
@@ -120,6 +131,7 @@ jobs: | |||||
| IMAGE_NAME: ghcr.io/nvidia/driver | ||||||
| PRECOMPILED: "true" | ||||||
| DIST: signed_${{ matrix.dist }} | ||||||
| DOCKER_BUILD_PLATFORM_OPTIONS: --platform=linux/${{ matrix.platform }} | ||||||
| run: | | ||||||
| source kernel_version.txt && \ | ||||||
| make DRIVER_VERSIONS=${DRIVER_VERSIONS} DRIVER_BRANCH=${{ matrix.driver_branch }} build-${DIST}-${DRIVER_VERSION} | ||||||
|
|
@@ -136,34 +148,40 @@ jobs: | |||||
| elif [[ "${{ matrix.dist }}" == "ubuntu24.04" ]]; then | ||||||
| BASE_TARGET="noble" | ||||||
| fi | ||||||
| if [[ "${{ matrix.platform }}" == "amd64" ]]; then | ||||||
| PLATFORM_SUFFIX="" | ||||||
| else | ||||||
| PLATFORM_SUFFIX="-${{ matrix.platform }}" | ||||||
| fi | ||||||
| BASE_ARTIFACT="base-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${{ matrix.dist }}${PLATFORM_SUFFIX}" | ||||||
| DRIVER_ARTIFACT="driver-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${{ matrix.dist }}${PLATFORM_SUFFIX}" | ||||||
| KERNEL_VERSION_ARTIFACT="kernel-version-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${{ matrix.dist }}${PLATFORM_SUFFIX}" | ||||||
| echo "BASE_ARTIFACT_NAME=${BASE_ARTIFACT}" >> $GITHUB_ENV | ||||||
| echo "DRIVER_ARTIFACT_NAME=${DRIVER_ARTIFACT}" >> $GITHUB_ENV | ||||||
| echo "KERNEL_VERSION_ARTIFACT_NAME=${KERNEL_VERSION_ARTIFACT}" >> $GITHUB_ENV | ||||||
| tar -cvf "${KERNEL_VERSION_ARTIFACT}.tar" kernel_version.txt | ||||||
| docker save "${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${{ matrix.flavor }}-${{ matrix.driver_branch }}" -o "${BASE_ARTIFACT}.tar" | ||||||
| docker save "${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${KERNEL_VERSION}-${{ matrix.dist }}" -o "${DRIVER_ARTIFACT}.tar" | ||||||
|
|
||||||
| - name: Upload base image as an artifact | ||||||
| uses: actions/upload-artifact@v6 | ||||||
| with: | ||||||
| name: ${{ env.BASE_ARTIFACT_NAME }} | ||||||
| path: ./${{ env.BASE_ARTIFACT_NAME }}.tar | ||||||
| retention-days: 1 | ||||||
|
|
||||||
| - name: Upload build image as an artifact | ||||||
| uses: actions/upload-artifact@v6 | ||||||
| with: | ||||||
| name: ${{ env.DRIVER_ARTIFACT_NAME }} | ||||||
| path: ./${{ env.DRIVER_ARTIFACT_NAME }}.tar | ||||||
| retention-days: 1 | ||||||
|
|
||||||
| - name: Upload kernel version as an artifact | ||||||
| uses: actions/upload-artifact@v6 | ||||||
| with: | ||||||
| name: ${{ env.KERNEL_VERSION_ARTIFACT_NAME }} | ||||||
| path: ./${{ env.KERNEL_VERSION_ARTIFACT_NAME }}.tar | ||||||
| retention-days: 1 | ||||||
|
|
||||||
| determine-e2e-test-matrix: | ||||||
|
|
@@ -172,9 +190,12 @@ jobs: | |||||
| matrix: | ||||||
| dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }} | ||||||
| lts_kernel: ${{ fromJson(needs.set-driver-version-matrix.outputs.lts_kernel) }} | ||||||
| platform: ${{ fromJson(needs.set-driver-version-matrix.outputs.platforms) }} | ||||||
| exclude: | ||||||
| - lts_kernel: 5.15 | ||||||
| dist: ubuntu24.04 | ||||||
| - platform: arm64 | ||||||
| dist: ubuntu22.04 | ||||||
| needs: | ||||||
| - precompiled-build-image | ||||||
| - set-driver-version-matrix | ||||||
|
|
@@ -193,7 +214,7 @@ jobs: | |||||
| with: | ||||||
| pattern: kernel-version-* | ||||||
| path: ./kernel-version-artifacts | ||||||
| merge-multiple: true | ||||||
|
|
||||||
| - name: Set kernel version | ||||||
| env: | ||||||
|
|
@@ -205,29 +226,34 @@ jobs: | |||||
| KERNEL_FLAVORS=($(echo "$kernel_flavors_json" | jq -r '.[]')) | ||||||
| driver_branch_json='${{ needs.set-driver-version-matrix.outputs.driver_branch }}' | ||||||
| DRIVER_BRANCHES=($(echo "$driver_branch_json" | jq -r '.[]')) | ||||||
| if [[ "${{ matrix.platform }}" == "arm64" ]]; then | ||||||
| PLATFORM_SUFFIX="-arm64" | ||||||
| else | ||||||
| PLATFORM_SUFFIX="" | ||||||
| fi | ||||||
| echo "PLATFORM_SUFFIX=${PLATFORM_SUFFIX}" >> $GITHUB_ENV | ||||||
| # remove 535 driver branch for ubuntu24.04 | ||||||
| if [ "$DIST" == "ubuntu24.04" ]; then | ||||||
| DRIVER_BRANCHES=($(for branch in "${DRIVER_BRANCHES[@]}"; do | ||||||
| [[ $branch != "535" ]] && echo "$branch" | ||||||
| done)) | ||||||
| fi | ||||||
| source ./tests/scripts/ci-precompiled-helpers.sh | ||||||
| KERNEL_VERSIONS=($(get_kernel_versions_to_test KERNEL_FLAVORS[@] DRIVER_BRANCHES[@] $DIST $LTS_KERNEL $PLATFORM_SUFFIX)) | ||||||
| if [ -z "$KERNEL_VERSIONS" ]; then | ||||||
| # no new kernel release | ||||||
| echo "Skipping e2e tests" | ||||||
| exit 0 | ||||||
| fi | ||||||
| # Convert array to JSON format and assign | ||||||
| echo "[]" > ./matrix_values_${{ matrix.dist }}_${{ matrix.lts_kernel }}${PLATFORM_SUFFIX}.json | ||||||
| printf '%s\n' "${KERNEL_VERSIONS[@]}" | jq -R . | jq -s . > ./matrix_values_${{ matrix.dist }}_${{ matrix.lts_kernel }}${PLATFORM_SUFFIX}.json | ||||||
|
|
||||||
| - name: Upload kernel matrix values as artifacts | ||||||
| uses: actions/upload-artifact@v6 | ||||||
| with: | ||||||
| name: matrix-values-${{ matrix.dist }}-${{ matrix.lts_kernel }}${{ env.PLATFORM_SUFFIX }} | ||||||
| path: ./matrix_values_${{ matrix.dist }}_${{ matrix.lts_kernel }}${{ env.PLATFORM_SUFFIX }}.json | ||||||
| retention-days: 1 | ||||||
|
|
||||||
| collect-e2e-test-matrix: | ||||||
|
|
@@ -256,24 +282,12 @@ jobs: | |||||
| echo "matrix_values_not_empty=0" >> $GITHUB_OUTPUT | ||||||
| kernel_versions=() | ||||||
|
|
||||||
| # Read and merge kernel_version values from all platform artifacts (amd64 and arm64) | ||||||
| for f in $(find ./matrix-values-artifacts -name "matrix_values_*.json" -type f 2>/dev/null); do | ||||||
| value=$(jq -r '.[]' "$f") | ||||||
| kernel_versions+=($value) | ||||||
| echo "matrix_values_not_empty=1" >> $GITHUB_OUTPUT | ||||||
| kernel_versions=($(printf '%s\n' "${kernel_versions[@]}" | sort -u)) | ||||||
| done | ||||||
| echo "Collected Kernel Versions: ${kernel_versions[@]}" | ||||||
| combined_values=$(printf '%s\n' "${kernel_versions[@]}" | jq -R . | jq -s -c . | tr -d ' \n') | ||||||
|
|
@@ -334,6 +348,16 @@ jobs: | |||||
| echo "${{ secrets.AWS_SSH_KEY }}" > ${{ github.workspace }}/key.pem && chmod 400 ${{ github.workspace }}/key.pem | ||||||
| echo "PRIVATE_REGISTRY=ghcr.io" >> $GITHUB_ENV | ||||||
| KERNEL_VERSION="${{ matrix.kernel_version }}" | ||||||
| if [[ "${KERNEL_VERSION}" == *-arm64 ]]; then | ||||||
| PLATFORM=arm64 | ||||||
| PLATFORM_SUFFIX="-arm64" | ||||||
| KERNEL_VERSION="${KERNEL_VERSION%-arm64}" | ||||||
| else | ||||||
| PLATFORM=amd64 | ||||||
| PLATFORM_SUFFIX="" | ||||||
| fi | ||||||
| echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV | ||||||
| echo "PLATFORM_SUFFIX=$PLATFORM_SUFFIX" >> $GITHUB_ENV | ||||||
| # Extract the last segment after the last dash | ||||||
| DIST=${KERNEL_VERSION##*-} | ||||||
| echo "DIST=$DIST" >> $GITHUB_ENV | ||||||
|
|
@@ -342,22 +366,29 @@ jobs: | |||||
| driver_branch_json="${{ needs.set-driver-version-matrix.outputs.driver_branch }}" | ||||||
| DRIVER_BRANCHES=($(echo "$driver_branch_json" | jq -r '.[]')) | ||||||
| echo "DRIVER_BRANCHES=${DRIVER_BRANCHES[*]}" >> $GITHUB_ENV | ||||||
| - name: Set kernel version and architecture in holodeck_ubuntu.yaml | ||||||
| run: | | ||||||
| yq eval '.spec += {"kernel": {"version": strenv(KERNEL_VERSION)}}' -i tests/holodeck_ubuntu.yaml | ||||||
| if [[ "$DIST" == "ubuntu24.04" ]]; then | ||||||
| yq eval '.spec.instance.os = "ubuntu-24.04"' -i tests/holodeck_ubuntu.yaml | ||||||
| fi | ||||||
| if [[ "$PLATFORM" == "arm64" ]]; then | ||||||
| yq eval '.spec.instance.image.architecture = strenv(PLATFORM)' -i tests/holodeck_ubuntu.yaml | ||||||
| yq eval '.spec.instance.type = "g5g.xlarge"' -i tests/holodeck_ubuntu.yaml | ||||||
| yq eval '.spec.instance.region = "us-west-2"' -i tests/holodeck_ubuntu.yaml | ||||||
| fi | ||||||
|
|
||||||
| - name: Set up Holodeck | ||||||
| uses: NVIDIA/holodeck@main | ||||||
|
Contributor
Author
There was a problem hiding this comment. I will update it and specify the actual version once @ArangoGutierrez releases the new version of Holodeck. There was a problem hiding this comment. The Holodeck version has been changed from a pinned version (v0.2.18) to @main, which is not a best practice for CI/CD workflows. Using @main introduces unpredictability as the main branch could contain breaking changes at any time. The other workflow file (.github/workflows/ci.yaml) uses NVIDIA/holodeck@v0.2.18. Consider using a specific pinned version or tag instead of @main for stability and reproducibility.
Suggested change
|
||||||
| env: | ||||||
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||||||
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||||||
| AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }} | ||||||
| with: | ||||||
| aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||||||
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||||||
| aws_ssh_key: ${{ secrets.AWS_SSH_KEY }} | ||||||
| holodeck_config: "tests/holodeck_ubuntu.yaml" | ||||||
| - name: Get public dns name | ||||||
| id: get_public_dns_name | ||||||
| uses: mikefarah/yq@master | ||||||
|
|
@@ -394,7 +425,7 @@ jobs: | |||||
| TEST_CASE_ARGS="${GPU_OPERATOR_OPTIONS} --set driver.version=${DRIVER_VERSION}" | ||||||
| # add escape character for space | ||||||
| TEST_CASE_ARGS=$(printf '%q ' "$TEST_CASE_ARGS") | ||||||
| IMAGE_PATH="./tests/driver-images-${DRIVER_VERSION}-${KERNEL_VERSION}-${DIST}${{ env.PLATFORM_SUFFIX }}.tar" | ||||||
| ./tests/ci-run-e2e.sh "${TEST_CASE}" "${TEST_CASE_ARGS}" ${IMAGE_PATH} || status=$? | ||||||
| if [ $status -eq 1 ]; then | ||||||
| echo "e2e validation failed for driver version $DRIVER_VERSION with status $status" | ||||||
|
|
@@ -436,6 +467,11 @@ jobs: | |||||
| - name: Set image vars | ||||||
| run: | | ||||||
| echo "PRIVATE_REGISTRY=ghcr.io" >> $GITHUB_ENV | ||||||
| KERNEL_VERSION="${{ matrix.kernel_version }}" | ||||||
| if [[ "${KERNEL_VERSION}" == *-arm64 ]]; then | ||||||
| KERNEL_VERSION="${KERNEL_VERSION%-arm64}" | ||||||
| fi | ||||||
| echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV | ||||||
|
|
||||||
| - name: Download base image artifact | ||||||
| if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }} | ||||||
|
|
@@ -447,9 +483,9 @@ jobs: | |||||
| - name: Publish base image | ||||||
| if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }} | ||||||
| run: | | ||||||
| LTS_KERNEL=$(echo "${{ env.KERNEL_VERSION }}" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/') | ||||||
| KERNEL_FLAVOR=$(echo "${{ env.KERNEL_VERSION }}" | sed -E 's/^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-(.*)-ubuntu[0-9]+\.[0-9]+$/\1/') | ||||||
| DIST=$(echo "${{ env.KERNEL_VERSION }}" | sed -E 's/^.*-(ubuntu[0-9]+\.[0-9]+)$/\1/') | ||||||
| if [[ "${DIST}" == "ubuntu22.04" ]]; then | ||||||
| BASE_TARGET="jammy" | ||||||
| elif [[ "${DIST}" == "ubuntu24.04" ]]; then | ||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
sort -udeduplication command is placed inside the for loop, which means it will be executed on every iteration and will lose accumulated values. The line should be moved outside the loop, after thedonestatement. This will cause incorrect behavior where only the last file's kernel versions are kept after deduplication instead of all accumulated versions.