diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39cbfd8..cbecbb3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,12 +34,32 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: + prepare: + runs-on: ubuntu-latest + + outputs: + ruby_sha: ${{ steps.ruby_sha.outputs.ruby_sha }} + + steps: + - name: Resolve ruby_sha + id: ruby_sha + run: |- + if [ "$ruby_version" = master ] && [ -z "$ruby_sha" ]; then + for attempt in 1 2 3; do + ruby_sha=$(gh api repos/ruby/ruby/commits/master --jq .sha) && break + sleep 30 + done + test -n "$ruby_sha" + fi + echo "ruby_sha=$ruby_sha" >> "$GITHUB_OUTPUT" + build: + needs: prepare + strategy: fail-fast: false matrix: debug_suffix: ['', '-debug'] - dev_suffix: ['', '-dev'] ubuntu_version: - resolute - noble @@ -49,11 +69,8 @@ jobs: runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} env: - push: false + ruby_sha: ${{ needs.prepare.outputs.ruby_sha }} ubuntu_version: "${{ matrix.ubuntu_version }}" - tag_suffix: "-${{ matrix.arch }}-${{ github.run_id }}" - push_tags: ${{ github.event.inputs.ruby_version || github.event.client_payload.ruby_version || 'master' }}${{ matrix.debug_suffix }}${{ matrix.dev_suffix }}-${{ matrix.ubuntu_version }}-${{ matrix.arch }}-${{ github.run_id }} - dev_suffix: ${{ matrix.dev_suffix }} debug_suffix: ${{ matrix.debug_suffix }} steps: @@ -70,54 +87,46 @@ jobs: username: ${{ secrets.GHCR_USER }} password: ${{ secrets.GHCR_ACCESS_TOKEN }} + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - run: | - if [ "${{ env.dev_suffix }}" = "-dev" ]; then - echo "target=development" >> $GITHUB_ENV - else - echo "target=ruby" >> $GITHUB_ENV - fi if [ "${{ env.debug_suffix }}" = "-debug" ]; then echo "cppflags=-DENABLE_PATH_CHECK=0 -DRUBY_DEBUG=1" >> $GITHUB_ENV echo "optflags=-O3 -fno-inline" >> $GITHUB_ENV fi - - name: Build docker image - run: |- - rake docker:build ruby_version=${{ env.ruby_version }} \ - ubuntu_version=${{ env.ubuntu_version }} \ - arch=linux/${{ matrix.arch }} \ - image_version_suffix=${{ env.debug_suffix }}${{ env.dev_suffix }} \ - tag_suffix=${{ env.tag_suffix }} \ - optflags="${{ env.optflags }}" \ - cppflags="${{ env.cppflags }}" \ - target=${{ env.target }} \ - - - name: List images - run: docker images - - - name: Push docker image to rubylang - if: "${{ env.push_tags }}" + - name: Build and push docker images run: |- - push_tags="${{ env.push_tags }}" - for tag in $push_tags; do - docker push rubylang/ruby:$tag + mkdir -p "$RUNNER_TEMP/digests" + for target in ruby development; do + for attempt in 1 2 3; do + rake docker:build ruby_version=${{ env.ruby_version }} \ + ubuntu_version=${{ env.ubuntu_version }} \ + arch=linux/${{ matrix.arch }} \ + optflags="${{ env.optflags }}" \ + cppflags="${{ env.cppflags }}" \ + target=$target \ + push_registries="rubylang ghcr.io/ruby" \ + metadata_file="$RUNNER_TEMP/$target.json" && break + if [ $attempt = 3 ]; then exit 1; fi + sleep 60 + done + jq -r '."containerimage.digest"' "$RUNNER_TEMP/$target.json" > "$RUNNER_TEMP/digests/$target-${{ matrix.arch }}" done - - name: Push docker image to ghcr.io/ruby - if: "${{ env.push_tags }}" - run: |- - push_tags="${{ env.push_tags }}" - for tag in $push_tags; do - docker tag rubylang/ruby:$tag ghcr.io/ruby/ruby:$tag - docker push ghcr.io/ruby/ruby:$tag - done + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: digests${{ matrix.debug_suffix }}-${{ matrix.ubuntu_version }}-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 - uses: ruby/action-slack@d260b61aa817726d5bedd22dd6cc305787fa4cdd # v4.0.0 with: payload: | { "attachments": [{ - "text": "build ${{ job.status }}: ${{ matrix.ubuntu_version }} ${{ matrix.arch }} ${{ matrix.debug_suffix }} ${{ matrix.dev_suffix }} ", + "text": "build ${{ job.status }}: ${{ matrix.ubuntu_version }} ${{ matrix.arch }} ${{ matrix.debug_suffix }} ", "color": "danger" }] } @@ -129,8 +138,7 @@ jobs: strategy: fail-fast: false matrix: - registry_name: [rubylang, ghcr.io/ruby] - image_version_suffix: ['', '-dev', '-debug', '-debug-dev'] + debug_suffix: ['', '-debug'] ubuntu_version: - resolute - noble @@ -138,7 +146,12 @@ jobs: runs-on: ubuntu-latest - needs: build + needs: [prepare, build] + + if: ${{ !cancelled() && needs.prepare.result == 'success' }} + + env: + ruby_sha: ${{ needs.prepare.outputs.ruby_sha }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -147,51 +160,40 @@ jobs: with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} - if: ${{ matrix.registry_name == 'rubylang' }} - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ secrets.GHCR_USER }} password: ${{ secrets.GHCR_ACCESS_TOKEN }} - if: ${{ matrix.registry_name == 'ghcr.io/ruby' }} - # Make sure the sha of "master-{sha}" tags agrees with the RUBY_REVISION in the image - - name: Extract ruby_sha - run: echo "ruby_sha=$(docker run "$docker_image" ruby -e 'puts RUBY_REVISION')" >> "$GITHUB_ENV" - env: - # In docker:manifest:create: - # tags = ["master#{version_suffix}", ...] #=> "master${{ matrix.image_version_suffix }}" - # tags.collect! {|t| "#{docker_image_name}:#{t}-#{ubuntu_version(ruby_version)}#{tag_suffix}" } #=> "${{ matrix.registry_name}}/ruby:master${{ matrix.image_version_suffix }}-${{ matrix.ubuntu_version }}" - # manifest_name = "#{tags[0]}-#{arch}" #=> "${{ matrix.registry_name}}/ruby:master${{ matrix.image_version_suffix }}-${{ matrix.ubuntu_version }}-amd64" - # manifest_name = "#{manifest_name}-#{manifest_suffix}" if manifest_suffix #=> "${{ matrix.registry_name}}/ruby:master${{ matrix.image_version_suffix }}-${{ matrix.ubuntu_version }}-amd64-${{ github.run_id }}" - docker_image: ${{ matrix.registry_name}}/ruby:master${{ matrix.image_version_suffix }}-${{ matrix.ubuntu_version }}-amd64-${{ github.run_id }} - if: ${{ (github.event.inputs.ruby_version || github.event.client_payload.ruby_version || 'master') == 'master' && (github.event.inputs.ruby_sha || github.event.client_payload.ruby_sha || '') == '' }} - - - name: Create manifest for ${{ matrix.registry_name }} - run: |- - rake docker:manifest:create \ - registry_name="${{ matrix.registry_name }}" \ - ruby_version="${{ env.ruby_version }}" \ - ubuntu_version="${{ matrix.ubuntu_version }}" \ - architectures="amd64 arm64" \ - manifest_suffix=${{ github.run_id }} \ - image_version_suffix=${{ matrix.image_version_suffix }} - - - name: Push manifest to ${{ matrix.registry_name }} + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: digests${{ matrix.debug_suffix }}-${{ matrix.ubuntu_version }}-* + path: ${{ runner.temp }}/digests + merge-multiple: true + + - name: Create and push manifests run: |- - rake docker:manifest:push \ - registry_name="${{ matrix.registry_name }}" \ - ruby_version="${{ env.ruby_version }}" \ - ubuntu_version="${{ matrix.ubuntu_version }}" \ - image_version_suffix=${{ matrix.image_version_suffix }} + for target in ruby development; do + if [ $target = development ]; then dev_suffix=-dev; else dev_suffix=; fi + digests=$(cat "$RUNNER_TEMP/digests/$target-amd64" "$RUNNER_TEMP/digests/$target-arm64") + for registry_name in rubylang ghcr.io/ruby; do + rake docker:manifest:create \ + registry_name="$registry_name" \ + ruby_version="${{ env.ruby_version }}" \ + ubuntu_version="${{ matrix.ubuntu_version }}" \ + image_version_suffix=${{ matrix.debug_suffix }}$dev_suffix \ + digests="$digests" + done + done - uses: ruby/action-slack@d260b61aa817726d5bedd22dd6cc305787fa4cdd # v4.0.0 with: payload: | { "attachments": [{ - "text": "deploy_multiarch ${{ job.status }}: ${{ matrix.registry_name }} / ${{ matrix.ubuntu_version }} ${{ matrix.image_version_suffix }} ", + "text": "deploy_multiarch ${{ job.status }}: ${{ matrix.ubuntu_version }} ${{ matrix.debug_suffix }} ", "color": "danger" }] } diff --git a/Rakefile b/Rakefile index fb493aa..ba4c4ee 100644 --- a/Rakefile +++ b/Rakefile @@ -1,19 +1,34 @@ LATEST_UBUNTU_VERSION = "noble" LATEST_RUBY_VERSION = "4.0" +def with_retry(attempts: 3, wait: 10) + tries = 0 + begin + yield + rescue => e + tries += 1 + raise if tries >= attempts + warn "#{e.class}: #{e.message} (retrying in #{wait}s)" + sleep wait + retry + end +end + def download(url) require "net/http" url = URI.parse(url) - Net::HTTP.start(url.hostname, url.port, :use_ssl => (url.scheme == "https")) do |http| - path = url.path - path += "?#{url.query}" if url.query - request = Net::HTTP::Get.new(url.path) - http.request(request) do |response| - case response - when Net::HTTPSuccess - return response.read_body - else - response.error! + with_retry do + Net::HTTP.start(url.hostname, url.port, :use_ssl => (url.scheme == "https")) do |http| + path = url.path + path += "?#{url.query}" if url.query + request = Net::HTTP::Get.new(url.path) + http.request(request) do |response| + case response + when Net::HTTPSuccess + return response.read_body + else + response.error! + end end end end @@ -94,7 +109,9 @@ def ruby_version_exist?(version) require "net/http" require "uri" ver2 = version.split('.')[0,2].join('.') - Net::HTTP.get_response(URI.parse("https://cache.ruby-lang.org/pub/ruby/#{ver2}/ruby-#{version}.tar.gz")).code == "200" + with_retry do + Net::HTTP.get_response(URI.parse("https://cache.ruby-lang.org/pub/ruby/#{ver2}/ruby-#{version}.tar.gz")).code == "200" + end end namespace :debug do @@ -227,15 +244,21 @@ namespace :docker do def github_api_get(path, accept: "application/vnd.github+json", token: ENV.fetch("GITHUB_TOKEN")) require "net/http" require "uri" - Net::HTTP.get_response(URI.join("https://api.github.com", path), { - "Accept" => accept, - "Authorization" => token&.then { "Bearer #{token}" }, - }.compact).tap(&:value).then(&:body) + with_retry do + Net::HTTP.get_response(URI.join("https://api.github.com", path), { + "Accept" => accept, + "Authorization" => token&.then { "Bearer #{token}" }, + }.compact).tap(&:value).then(&:body) + end + end + + def ruby_master_commit_hash + ENV.fetch("ruby_sha", "").empty? ? get_ruby_master_head_hash : ENV.fetch("ruby_sha") end def make_tags(ruby_version, version_suffix=nil, tag_suffix=nil) if ruby_version == "master" - commit_hash = ENV.fetch("ruby_sha", "").empty? ? get_ruby_master_head_hash : ENV.fetch("ruby_sha") + commit_hash = ruby_master_commit_hash commit_date = get_date_at_commit(commit_hash).then do |date| date_offset = Integer(ENV.fetch("ruby_commit_date_offset", "0")) date += date_offset * 24 * 60 * 60 @@ -253,7 +276,7 @@ namespace :docker do return ruby_version, tags end - desc "Build Docker image for Ruby (env: ruby_version, ubuntu_version, arch, image_version_suffix, tag_suffix, tag, cppflags, optflags, target)" + desc "Build Docker image for Ruby (env: ruby_version, ubuntu_version, arch, image_version_suffix, tag_suffix, tag, cppflags, optflags, target, push_registries, metadata_file)" task :build do ruby_version = default_ruby_version unless ruby_version_exist?(ruby_version) @@ -265,8 +288,19 @@ namespace :docker do target = ENV.fetch("target", "ruby") arch = ENV.fetch("arch", "linux/amd64") - ruby_version, tags = make_tags(ruby_version, version_suffix, tag_suffix) - tags << "#{docker_image_name}:#{tag}" if !tag.empty? + if (push_registries = ENV["push_registries"]) + # Pushed untagged. docker:manifest:create tags the digests later. + ruby_version = "master:#{ruby_master_commit_hash}" if ruby_version == "master" + tags = [] + images = push_registries.split.map {|name| "#{name}/ruby" }.join(",") + build_cmd_args = ['buildx', 'build', '--platform', arch, + '--output', %Q(type=image,"name=#{images}",push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=false)] + build_cmd_args.push('--metadata-file', ENV["metadata_file"]) if ENV.key?("metadata_file") + else + ruby_version, tags = make_tags(ruby_version, version_suffix, tag_suffix) + tags << "#{docker_image_name}:#{tag}" if !tag.empty? + build_cmd_args = arch =~ /arm/ ? ['buildx', 'build', '--platform', arch, '--load'] : ['build'] + end build_args = [ "RUBY_VERSION=#{ruby_version}", @@ -292,8 +326,6 @@ namespace :docker do IO.write('tmp/ruby/.keep', '') end - build_cmd_args = arch =~ /arm/ ? ['buildx', 'build', '--platform', arch, '--load'] : ['build'] - sh 'docker', *build_cmd_args, '-f', 'Dockerfile', *tags.map {|tag| ["-t", tag] }.flatten, *build_args.map {|arg| ["--build-arg", arg] }.flatten, @@ -306,54 +338,22 @@ namespace :docker do end namespace :manifest do - desc "Create multi-architecture Docker manifests (env: ruby_version, architectures, manifest_suffix, image_version_suffix)" + desc "Create and push multi-architecture Docker manifests from image digests (env: registry_name, ruby_version, ubuntu_version, image_version_suffix, digests)" task :create do ruby_version = ENV.fetch("ruby_version") - architectures = ENV.fetch("architectures").split(' ') - manifest_suffix = ENV.fetch("manifest_suffix", nil) image_version_suffix = ENV["image_version_suffix"] + sources = ENV.fetch("digests").split.map {|digest| "#{docker_image_name}@#{digest}" } _, tags = make_tags(ruby_version, image_version_suffix) - amend_args = architectures.map {|arch| - # "-#{arch}-#{manifest_suffix}" should match `tag_suffix` on `docker:build` - manifest_name = "#{tags[0]}-#{arch}" - manifest_name = "#{manifest_name}-#{manifest_suffix}" if manifest_suffix - ['--amend', manifest_name] - }.flatten - - latest_tag = nil - tags.each do |tag| - sh 'docker', 'manifest', 'create', "#{tag}", *amend_args - if tag =~ /#{LATEST_UBUNTU_VERSION}/ - non_ubuntu_tag = tag.sub(/-#{LATEST_UBUNTU_VERSION}/, '') - sh 'docker', 'manifest', 'create', "#{non_ubuntu_tag}", *amend_args - if image_version_suffix.empty? && ruby_version =~ /\A#{Regexp.escape(LATEST_RUBY_VERSION)}\.\d+\z/ && latest_tag.nil? - latest_tag = tag.sub(/#{ruby_version}-#{LATEST_UBUNTU_VERSION}/, "latest") - sh 'docker', 'manifest', 'create', "#{latest_tag}", *amend_args - end - end + latest_ubuntu_tags = tags.grep(/#{LATEST_UBUNTU_VERSION}/) + tags += latest_ubuntu_tags.map {|tag| tag.sub(/-#{LATEST_UBUNTU_VERSION}/, '') } + if image_version_suffix.empty? && ruby_version =~ /\A#{Regexp.escape(LATEST_RUBY_VERSION)}\.\d+\z/ && !latest_ubuntu_tags.empty? + tags << latest_ubuntu_tags.first.sub(/#{ruby_version}-#{LATEST_UBUNTU_VERSION}/, "latest") end - end - desc "Push multi-architecture Docker manifests to registry (env: ruby_version, image_version_suffix)" - task :push do - ruby_version = ENV["ruby_version"] - image_version_suffix = ENV["image_version_suffix"] - - _, tags = make_tags(ruby_version, image_version_suffix) - - latest_tag = nil - tags.each do |tag| - sh 'docker', 'manifest', 'push', "#{tag}" - if tag =~ /#{LATEST_UBUNTU_VERSION}/ - non_ubuntu_tag = tag.sub(/-#{LATEST_UBUNTU_VERSION}/, '') - sh 'docker', 'manifest', 'push', "#{non_ubuntu_tag}" - if image_version_suffix.empty? && ruby_version =~ /\A#{Regexp.escape(LATEST_RUBY_VERSION)}\.\d+\z/ && latest_tag.nil? - latest_tag = tag.sub(/#{ruby_version}-#{LATEST_UBUNTU_VERSION}/, "latest") - sh 'docker', 'manifest', 'push', "#{latest_tag}" - end - end + with_retry do + sh 'docker', 'buildx', 'imagetools', 'create', *tags.map {|tag| ["-t", tag] }.flatten, *sources end end end