From c2e15a73af101eb5412db7f6886273cc3fc94fa0 Mon Sep 17 00:00:00 2001 From: Michal Baumgartner Date: Thu, 4 Jun 2026 11:13:56 +0200 Subject: [PATCH 1/3] feat: Publish Toolkit artifacts as OCI images --- .github/actions/push-oci-artifact/action.yml | 63 ++++++++++++++ .github/workflows/cd.yml | 91 ++++++++++++++++++-- 2 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 .github/actions/push-oci-artifact/action.yml diff --git a/.github/actions/push-oci-artifact/action.yml b/.github/actions/push-oci-artifact/action.yml new file mode 100644 index 0000000..6797165 --- /dev/null +++ b/.github/actions/push-oci-artifact/action.yml @@ -0,0 +1,63 @@ +name: 'Push OCI artifact' +description: 'Push one or more files as an OCI artifact' +inputs: + artifact_ref: + description: 'Fully qualified OCI artifact reference' + required: true + artifact_type: + description: 'OCI artifact type' + required: true + files: + description: 'Newline-separated file descriptors in ORAS path:media-type format' + required: true + toolkit_version: + description: 'Toolkit version annotation value' + required: true + annotations: + description: 'Additional newline-separated OCI annotations in key=value format' + default: '' + +runs: + using: "composite" + + steps: + - name: Set up ORAS + uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 + + - name: Push OCI artifact + shell: bash + env: + ARTIFACT_REF: ${{ inputs.artifact_ref }} + ARTIFACT_TYPE: ${{ inputs.artifact_type }} + FILES: ${{ inputs.files }} + TOOLKIT_VERSION: ${{ inputs.toolkit_version }} + EXTRA_ANNOTATIONS: ${{ inputs.annotations }} + run: | + set -euo pipefail + + push_args=( + --artifact-type "${ARTIFACT_TYPE}" + --annotation "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" + --annotation "org.opencontainers.image.revision=${GITHUB_SHA}" + --annotation "org.opencontainers.image.version=${TOOLKIT_VERSION}" + ) + + while IFS= read -r annotation; do + [ -n "${annotation}" ] || continue + push_args+=(--annotation "${annotation}") + done <<< "${EXTRA_ANNOTATIONS}" + + while IFS= read -r file_descriptor; do + [ -n "${file_descriptor}" ] || continue + file_path="${file_descriptor%%:*}" + if [ ! -f "${file_path}" ]; then + echo "Error: OCI artifact file not found at ${file_path}" >&2 + ls -la dist/ || echo "dist/ directory not found" >&2 + exit 1 + fi + push_args+=("${file_descriptor}") + done <<< "${FILES}" + + oras push "${ARTIFACT_REF}" "${push_args[@]}" + oras manifest fetch "${ARTIFACT_REF}" >/dev/null + echo "Pushed ${ARTIFACT_REF}" diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5dee342..d3f55fc 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -118,26 +118,99 @@ jobs: bucket: ${{ env.AWS_PRODUCTION_BUCKET }} aws_role_arn: ${{ secrets.AWS_PRODUCTION_ROLE_ARN }} + # OCI ARTIFACT UPLOADS + - name: Push toolkit bundle OCI artifact + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-bundle:${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }} + artifact_type: application/vnd.deepnote.toolkit.bundle.v1 + files: dist/python${{ matrix.python_version }}.tar:application/vnd.deepnote.toolkit.python-bundle.v1.tar + toolkit_version: ${{ steps.version.outputs.VERSION }} + annotations: com.deepnote.toolkit.python-version=${{ matrix.python_version }} + + - name: Push toolkit installer OCI artifact + if: matrix.python_version == '3.10' + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-installer:${{ steps.version.outputs.VERSION }} + artifact_type: application/vnd.deepnote.toolkit.installer.v1 + files: dist/installer.zip:application/zip + toolkit_version: ${{ steps.version.outputs.VERSION }} + + - name: Upload toolkit constraints artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: toolkit-constraints-${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }} + path: dist/constraints${{ matrix.python_version }}.txt + if-no-files-found: error + + push-toolkit-constraints-oci: + name: Push toolkit constraints OCI artifact + runs-on: ubuntu-latest + needs: build-and-push-artifacts + # Only run for base repo, not forks or dependabot + if: (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Login to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3 + with: + username: deepnotebot + password: ${{ secrets.DOCKERHUB_PASS }} + + - name: Export version + id: version + uses: ./.github/actions/export-version + + - name: Download toolkit constraints artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: toolkit-constraints-${{ steps.version.outputs.VERSION }}-python* + path: dist/ + merge-multiple: true + + - name: Push toolkit constraints OCI artifact + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-constraints:${{ steps.version.outputs.VERSION }} + artifact_type: application/vnd.deepnote.toolkit.constraints.v1 + files: | + dist/constraints3.10.txt:text/plain + dist/constraints3.11.txt:text/plain + dist/constraints3.12.txt:text/plain + dist/constraints3.13.txt:text/plain + toolkit_version: ${{ steps.version.outputs.VERSION }} + build-and-push-artifacts-status: name: All artifacts pushed runs-on: ubuntu-latest - needs: build-and-push-artifacts - # Only run if the build job ran (i.e., not for forks or dependabot) + needs: + - build-and-push-artifacts + - push-toolkit-constraints-oci + # Only run if the artifact jobs ran (i.e., not for forks or dependabot) if: always() && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' steps: - - name: Check matrix job results + - name: Check artifact job results env: BUILD_RESULT: ${{ needs.build-and-push-artifacts.result }} + CONSTRAINTS_RESULT: ${{ needs.push-toolkit-constraints-oci.result }} run: | - result="${BUILD_RESULT}" - if [[ $result == "success" ]]; then - echo "All matrix jobs succeeded" + build_result="${BUILD_RESULT}" + constraints_result="${CONSTRAINTS_RESULT}" + if [[ $build_result == "success" && $constraints_result == "success" ]]; then + echo "All artifact jobs succeeded" exit 0 - elif [[ $result == "cancelled" ]]; then - echo "Matrix jobs were cancelled" + elif [[ $build_result == "cancelled" || $constraints_result == "cancelled" ]]; then + echo "One or more artifact jobs were cancelled" exit 1 else - echo "One or more matrix jobs failed: $result" + echo "One or more artifact jobs failed: build=${build_result}, constraints=${constraints_result}" exit 1 fi From 57c29d8ce3ff97e2dce0b033a7a6c0d810d3a37d Mon Sep 17 00:00:00 2001 From: Michal Baumgartner Date: Thu, 4 Jun 2026 12:02:39 +0200 Subject: [PATCH 2/3] fix: Make artifacts compatible with Kubernetes Image Volumes --- .github/actions/push-oci-artifact/action.yml | 70 +++++++++++++++----- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/.github/actions/push-oci-artifact/action.yml b/.github/actions/push-oci-artifact/action.yml index 6797165..676bf26 100644 --- a/.github/actions/push-oci-artifact/action.yml +++ b/.github/actions/push-oci-artifact/action.yml @@ -1,14 +1,14 @@ -name: 'Push OCI artifact' -description: 'Push one or more files as an OCI artifact' +name: 'Push OCI image volume artifact' +description: 'Push one or more files as an OCI image-volume-compatible artifact' inputs: artifact_ref: - description: 'Fully qualified OCI artifact reference' + description: 'Fully qualified OCI image reference' required: true artifact_type: - description: 'OCI artifact type' + description: 'Logical artifact type label' required: true files: - description: 'Newline-separated file descriptors in ORAS path:media-type format' + description: 'Newline-separated file descriptors in path:media-type format. Media type is ignored for image-volume publishing.' required: true toolkit_version: description: 'Toolkit version annotation value' @@ -24,7 +24,7 @@ runs: - name: Set up ORAS uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 - - name: Push OCI artifact + - name: Push OCI image volume artifact shell: bash env: ARTIFACT_REF: ${{ inputs.artifact_ref }} @@ -35,16 +35,22 @@ runs: run: | set -euo pipefail - push_args=( - --artifact-type "${ARTIFACT_TYPE}" - --annotation "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" - --annotation "org.opencontainers.image.revision=${GITHUB_SHA}" - --annotation "org.opencontainers.image.version=${TOOLKIT_VERSION}" + workdir="$(mktemp -d)" + trap 'rm -rf "${workdir}"' EXIT + + rootfs_dir="${workdir}/rootfs" + mkdir -p "${rootfs_dir}" + + manifest_annotations=( + "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" + "org.opencontainers.image.revision=${GITHUB_SHA}" + "org.opencontainers.image.version=${TOOLKIT_VERSION}" + "com.deepnote.toolkit.artifact-type=${ARTIFACT_TYPE}" ) while IFS= read -r annotation; do [ -n "${annotation}" ] || continue - push_args+=(--annotation "${annotation}") + manifest_annotations+=("${annotation}") done <<< "${EXTRA_ANNOTATIONS}" while IFS= read -r file_descriptor; do @@ -52,12 +58,46 @@ runs: file_path="${file_descriptor%%:*}" if [ ! -f "${file_path}" ]; then echo "Error: OCI artifact file not found at ${file_path}" >&2 - ls -la dist/ || echo "dist/ directory not found" >&2 + ls -la "$(dirname "${file_path}")" >&2 || true exit 1 fi - push_args+=("${file_descriptor}") + cp "${file_path}" "${rootfs_dir}/$(basename "${file_path}")" done <<< "${FILES}" - oras push "${ARTIFACT_REF}" "${push_args[@]}" + layer_tar="${workdir}/rootfs.tar" + layer_tar_gz="${workdir}/rootfs.tar.gz" + tar -C "${rootfs_dir}" -cf "${layer_tar}" . + gzip -n -c "${layer_tar}" > "${layer_tar_gz}" + + layer_diff_id="$(sha256sum "${layer_tar}" | awk '{print $1}')" + + config_path="${workdir}/config.json" + cat > "${config_path}" </dev/null echo "Pushed ${ARTIFACT_REF}" From c43da7d6697c2502d406d28dd0350862f2e7811b Mon Sep 17 00:00:00 2001 From: Michal Baumgartner Date: Thu, 4 Jun 2026 14:09:14 +0200 Subject: [PATCH 3/3] chore: Dynamically build constraint files list --- .github/workflows/cd.yml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d3f55fc..59553cf 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -175,16 +175,34 @@ jobs: path: dist/ merge-multiple: true + - name: Build toolkit constraints OCI files input + id: constraints_oci_files + shell: bash + run: | + set -euo pipefail + + constraint_files="$(mktemp)" + find dist -maxdepth 1 -type f -name 'constraints*.txt' | sort > "${constraint_files}" + if [ ! -s "${constraint_files}" ]; then + echo "Error: no constraint files were downloaded" >&2 + find dist -maxdepth 1 -type f -print >&2 + exit 1 + fi + + { + echo "files<> "${GITHUB_OUTPUT}" + - name: Push toolkit constraints OCI artifact uses: ./.github/actions/push-oci-artifact with: artifact_ref: docker.io/deepnote/toolkit-constraints:${{ steps.version.outputs.VERSION }} artifact_type: application/vnd.deepnote.toolkit.constraints.v1 - files: | - dist/constraints3.10.txt:text/plain - dist/constraints3.11.txt:text/plain - dist/constraints3.12.txt:text/plain - dist/constraints3.13.txt:text/plain + files: ${{ steps.constraints_oci_files.outputs.files }} toolkit_version: ${{ steps.version.outputs.VERSION }} build-and-push-artifacts-status: