feat: Publish Toolkit artifacts as OCI images

.github/actions/push-oci-artifact/action.yml

@@ -0,0 +1,103 @@
+name: 'Push OCI image volume artifact'
+description: 'Push one or more files as an OCI image-volume-compatible artifact'
+inputs:
+  artifact_ref:
+    description: 'Fully qualified OCI image reference'
+    required: true
+  artifact_type:
+    description: 'Logical artifact type label'
+    required: true
+  files:
+    description: 'Newline-separated file descriptors in path:media-type format. Media type is ignored for image-volume publishing.'
+    required: true
+  toolkit_version:
+    description: 'Toolkit version annotation value'
+    required: true
+  annotations:
+    description: 'Additional newline-separated OCI annotations in key=value format'
+    default: ''
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Set up ORAS
+      uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
+
+    - name: Push OCI image volume artifact
+      shell: bash
+      env:
+        ARTIFACT_REF: ${{ inputs.artifact_ref }}
+        ARTIFACT_TYPE: ${{ inputs.artifact_type }}
+        FILES: ${{ inputs.files }}
+        TOOLKIT_VERSION: ${{ inputs.toolkit_version }}
+        EXTRA_ANNOTATIONS: ${{ inputs.annotations }}
+      run: |
+        set -euo pipefail
+
+        workdir="$(mktemp -d)"
+        trap 'rm -rf "${workdir}"' EXIT
+
+        rootfs_dir="${workdir}/rootfs"
+        mkdir -p "${rootfs_dir}"
+
+        manifest_annotations=(
+          "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}"
+          "org.opencontainers.image.revision=${GITHUB_SHA}"
+          "org.opencontainers.image.version=${TOOLKIT_VERSION}"
+          "com.deepnote.toolkit.artifact-type=${ARTIFACT_TYPE}"
+        )
+
+        while IFS= read -r annotation; do
+          [ -n "${annotation}" ] || continue
+          manifest_annotations+=("${annotation}")
+        done <<< "${EXTRA_ANNOTATIONS}"
+
+        while IFS= read -r file_descriptor; do
+          [ -n "${file_descriptor}" ] || continue
+          file_path="${file_descriptor%%:*}"
+          if [ ! -f "${file_path}" ]; then
+            echo "Error: OCI artifact file not found at ${file_path}" >&2
+            ls -la "$(dirname "${file_path}")" >&2 || true
+            exit 1
+          fi
+          cp "${file_path}" "${rootfs_dir}/$(basename "${file_path}")"
+        done <<< "${FILES}"
+
+        layer_tar="${workdir}/rootfs.tar"
+        layer_tar_gz="${workdir}/rootfs.tar.gz"
+        tar -C "${rootfs_dir}" -cf "${layer_tar}" .
+        gzip -n -c "${layer_tar}" > "${layer_tar_gz}"
+
+        layer_diff_id="$(sha256sum "${layer_tar}" | awk '{print $1}')"
+
+        config_path="${workdir}/config.json"
+        cat > "${config_path}" <<EOF
+        {
+          "architecture": "amd64",
+          "os": "linux",
+          "rootfs": {
+            "type": "layers",
+            "diff_ids": ["sha256:${layer_diff_id}"]
+          },
+          "config": {},
+          "history": [{"created_by": "deepnote-toolkit OCI image volume artifact"}]
+        }
+        EOF
+
+        (
+          cd "${workdir}"
+          oras_args=(
+            push
+            "${ARTIFACT_REF}"
+            --config "config.json:application/vnd.oci.image.config.v1+json"
+          )
+          for annotation in "${manifest_annotations[@]}"; do
+            oras_args+=(--annotation "${annotation}")
+          done
+          oras_args+=("rootfs.tar.gz:application/vnd.oci.image.layer.v1.tar+gzip")
+
+          oras "${oras_args[@]}"
+        )
+        oras manifest fetch "${ARTIFACT_REF}" >/dev/null
+        echo "Pushed ${ARTIFACT_REF}"

.github/workflows/cd.yml

@@ -118,26 +118,117 @@ jobs:
           bucket: ${{ env.AWS_PRODUCTION_BUCKET }}
           aws_role_arn: ${{ secrets.AWS_PRODUCTION_ROLE_ARN }}
 
+      # OCI ARTIFACT UPLOADS
+      - name: Push toolkit bundle OCI artifact
+        uses: ./.github/actions/push-oci-artifact
+        with:
+          artifact_ref: docker.io/deepnote/toolkit-bundle:${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }}
+          artifact_type: application/vnd.deepnote.toolkit.bundle.v1
+          files: dist/python${{ matrix.python_version }}.tar:application/vnd.deepnote.toolkit.python-bundle.v1.tar
+          toolkit_version: ${{ steps.version.outputs.VERSION }}
+          annotations: com.deepnote.toolkit.python-version=${{ matrix.python_version }}
+
+      - name: Push toolkit installer OCI artifact
+        if: matrix.python_version == '3.10'
+        uses: ./.github/actions/push-oci-artifact
+        with:
+          artifact_ref: docker.io/deepnote/toolkit-installer:${{ steps.version.outputs.VERSION }}
+          artifact_type: application/vnd.deepnote.toolkit.installer.v1
+          files: dist/installer.zip:application/zip
+          toolkit_version: ${{ steps.version.outputs.VERSION }}
+
+      - name: Upload toolkit constraints artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: toolkit-constraints-${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }}
+          path: dist/constraints${{ matrix.python_version }}.txt
+          if-no-files-found: error
+
+  push-toolkit-constraints-oci:
+    name: Push toolkit constraints OCI artifact
+    runs-on: ubuntu-latest
+    needs: build-and-push-artifacts
+    # Only run for base repo, not forks or dependabot
+    if: (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]'
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+        with:
+          persist-credentials: false
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3
+        with:
+          username: deepnotebot
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Export version
+        id: version
+        uses: ./.github/actions/export-version
+
+      - name: Download toolkit constraints artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        with:
+          pattern: toolkit-constraints-${{ steps.version.outputs.VERSION }}-python*
+          path: dist/
+          merge-multiple: true
+
+      - name: Build toolkit constraints OCI files input
+        id: constraints_oci_files
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          constraint_files="$(mktemp)"
+          find dist -maxdepth 1 -type f -name 'constraints*.txt' | sort > "${constraint_files}"
+          if [ ! -s "${constraint_files}" ]; then
+            echo "Error: no constraint files were downloaded" >&2
+            find dist -maxdepth 1 -type f -print >&2
+            exit 1
+          fi
+
+          {
+            echo "files<<EOF"
+            while IFS= read -r constraint_file; do
+              printf '%s:text/plain\n' "${constraint_file}"
+            done < "${constraint_files}"
+            echo "EOF"
+          } >> "${GITHUB_OUTPUT}"
+
+      - name: Push toolkit constraints OCI artifact
+        uses: ./.github/actions/push-oci-artifact
+        with:
+          artifact_ref: docker.io/deepnote/toolkit-constraints:${{ steps.version.outputs.VERSION }}
+          artifact_type: application/vnd.deepnote.toolkit.constraints.v1
+          files: ${{ steps.constraints_oci_files.outputs.files }}
+          toolkit_version: ${{ steps.version.outputs.VERSION }}
+
   build-and-push-artifacts-status:
     name: All artifacts pushed
     runs-on: ubuntu-latest
-    needs: build-and-push-artifacts
-    # Only run if the build job ran (i.e., not for forks or dependabot)
+    needs:
+      - build-and-push-artifacts
+      - push-toolkit-constraints-oci
+    # Only run if the artifact jobs ran (i.e., not for forks or dependabot)
     if: always() && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]'
     steps:
-      - name: Check matrix job results
+      - name: Check artifact job results
         env:
           BUILD_RESULT: ${{ needs.build-and-push-artifacts.result }}
+          CONSTRAINTS_RESULT: ${{ needs.push-toolkit-constraints-oci.result }}
         run: |
-          result="${BUILD_RESULT}"
-          if [[ $result == "success" ]]; then
-            echo "All matrix jobs succeeded"
+          build_result="${BUILD_RESULT}"
+          constraints_result="${CONSTRAINTS_RESULT}"
+          if [[ $build_result == "success" && $constraints_result == "success" ]]; then
+            echo "All artifact jobs succeeded"
             exit 0
-          elif [[ $result == "cancelled" ]]; then
-            echo "Matrix jobs were cancelled"
+          elif [[ $build_result == "cancelled" || $constraints_result == "cancelled" ]]; then
+            echo "One or more artifact jobs were cancelled"
             exit 1
           else
-            echo "One or more matrix jobs failed: $result"
+            echo "One or more artifact jobs failed: build=${build_result}, constraints=${constraints_result}"
             exit 1
           fi