diff --git a/.github/actions/push-oci-artifact/action.yml b/.github/actions/push-oci-artifact/action.yml new file mode 100644 index 0000000..676bf26 --- /dev/null +++ b/.github/actions/push-oci-artifact/action.yml @@ -0,0 +1,103 @@ +name: 'Push OCI image volume artifact' +description: 'Push one or more files as an OCI image-volume-compatible artifact' +inputs: + artifact_ref: + description: 'Fully qualified OCI image reference' + required: true + artifact_type: + description: 'Logical artifact type label' + required: true + files: + description: 'Newline-separated file descriptors in path:media-type format. Media type is ignored for image-volume publishing.' + required: true + toolkit_version: + description: 'Toolkit version annotation value' + required: true + annotations: + description: 'Additional newline-separated OCI annotations in key=value format' + default: '' + +runs: + using: "composite" + + steps: + - name: Set up ORAS + uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 + + - name: Push OCI image volume artifact + shell: bash + env: + ARTIFACT_REF: ${{ inputs.artifact_ref }} + ARTIFACT_TYPE: ${{ inputs.artifact_type }} + FILES: ${{ inputs.files }} + TOOLKIT_VERSION: ${{ inputs.toolkit_version }} + EXTRA_ANNOTATIONS: ${{ inputs.annotations }} + run: | + set -euo pipefail + + workdir="$(mktemp -d)" + trap 'rm -rf "${workdir}"' EXIT + + rootfs_dir="${workdir}/rootfs" + mkdir -p "${rootfs_dir}" + + manifest_annotations=( + "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" + "org.opencontainers.image.revision=${GITHUB_SHA}" + "org.opencontainers.image.version=${TOOLKIT_VERSION}" + "com.deepnote.toolkit.artifact-type=${ARTIFACT_TYPE}" + ) + + while IFS= read -r annotation; do + [ -n "${annotation}" ] || continue + manifest_annotations+=("${annotation}") + done <<< "${EXTRA_ANNOTATIONS}" + + while IFS= read -r file_descriptor; do + [ -n "${file_descriptor}" ] || continue + file_path="${file_descriptor%%:*}" + if [ ! -f "${file_path}" ]; then + echo "Error: OCI artifact file not found at ${file_path}" >&2 + ls -la "$(dirname "${file_path}")" >&2 || true + exit 1 + fi + cp "${file_path}" "${rootfs_dir}/$(basename "${file_path}")" + done <<< "${FILES}" + + layer_tar="${workdir}/rootfs.tar" + layer_tar_gz="${workdir}/rootfs.tar.gz" + tar -C "${rootfs_dir}" -cf "${layer_tar}" . + gzip -n -c "${layer_tar}" > "${layer_tar_gz}" + + layer_diff_id="$(sha256sum "${layer_tar}" | awk '{print $1}')" + + config_path="${workdir}/config.json" + cat > "${config_path}" </dev/null + echo "Pushed ${ARTIFACT_REF}" diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5dee342..59553cf 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -118,26 +118,117 @@ jobs: bucket: ${{ env.AWS_PRODUCTION_BUCKET }} aws_role_arn: ${{ secrets.AWS_PRODUCTION_ROLE_ARN }} + # OCI ARTIFACT UPLOADS + - name: Push toolkit bundle OCI artifact + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-bundle:${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }} + artifact_type: application/vnd.deepnote.toolkit.bundle.v1 + files: dist/python${{ matrix.python_version }}.tar:application/vnd.deepnote.toolkit.python-bundle.v1.tar + toolkit_version: ${{ steps.version.outputs.VERSION }} + annotations: com.deepnote.toolkit.python-version=${{ matrix.python_version }} + + - name: Push toolkit installer OCI artifact + if: matrix.python_version == '3.10' + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-installer:${{ steps.version.outputs.VERSION }} + artifact_type: application/vnd.deepnote.toolkit.installer.v1 + files: dist/installer.zip:application/zip + toolkit_version: ${{ steps.version.outputs.VERSION }} + + - name: Upload toolkit constraints artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: toolkit-constraints-${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }} + path: dist/constraints${{ matrix.python_version }}.txt + if-no-files-found: error + + push-toolkit-constraints-oci: + name: Push toolkit constraints OCI artifact + runs-on: ubuntu-latest + needs: build-and-push-artifacts + # Only run for base repo, not forks or dependabot + if: (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Login to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3 + with: + username: deepnotebot + password: ${{ secrets.DOCKERHUB_PASS }} + + - name: Export version + id: version + uses: ./.github/actions/export-version + + - name: Download toolkit constraints artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: toolkit-constraints-${{ steps.version.outputs.VERSION }}-python* + path: dist/ + merge-multiple: true + + - name: Build toolkit constraints OCI files input + id: constraints_oci_files + shell: bash + run: | + set -euo pipefail + + constraint_files="$(mktemp)" + find dist -maxdepth 1 -type f -name 'constraints*.txt' | sort > "${constraint_files}" + if [ ! -s "${constraint_files}" ]; then + echo "Error: no constraint files were downloaded" >&2 + find dist -maxdepth 1 -type f -print >&2 + exit 1 + fi + + { + echo "files<> "${GITHUB_OUTPUT}" + + - name: Push toolkit constraints OCI artifact + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-constraints:${{ steps.version.outputs.VERSION }} + artifact_type: application/vnd.deepnote.toolkit.constraints.v1 + files: ${{ steps.constraints_oci_files.outputs.files }} + toolkit_version: ${{ steps.version.outputs.VERSION }} + build-and-push-artifacts-status: name: All artifacts pushed runs-on: ubuntu-latest - needs: build-and-push-artifacts - # Only run if the build job ran (i.e., not for forks or dependabot) + needs: + - build-and-push-artifacts + - push-toolkit-constraints-oci + # Only run if the artifact jobs ran (i.e., not for forks or dependabot) if: always() && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' steps: - - name: Check matrix job results + - name: Check artifact job results env: BUILD_RESULT: ${{ needs.build-and-push-artifacts.result }} + CONSTRAINTS_RESULT: ${{ needs.push-toolkit-constraints-oci.result }} run: | - result="${BUILD_RESULT}" - if [[ $result == "success" ]]; then - echo "All matrix jobs succeeded" + build_result="${BUILD_RESULT}" + constraints_result="${CONSTRAINTS_RESULT}" + if [[ $build_result == "success" && $constraints_result == "success" ]]; then + echo "All artifact jobs succeeded" exit 0 - elif [[ $result == "cancelled" ]]; then - echo "Matrix jobs were cancelled" + elif [[ $build_result == "cancelled" || $constraints_result == "cancelled" ]]; then + echo "One or more artifact jobs were cancelled" exit 1 else - echo "One or more matrix jobs failed: $result" + echo "One or more artifact jobs failed: build=${build_result}, constraints=${constraints_result}" exit 1 fi