diff --git a/lib/cibuildgem/cli.rb b/lib/cibuildgem/cli.rb
index 8cd2018..e76fb0b 100644
--- a/lib/cibuildgem/cli.rb
+++ b/lib/cibuildgem/cli.rb
@@ -3,6 +3,7 @@
 require "thor"
 require "rake/extensiontask"
 require "prism"
+require "open3"
 
 module Cibuildgem
   class CLI < Thor
@@ -112,7 +113,16 @@ def release
         pathname = Pathname(file)
         next if pathname.directory? || pathname.extname != ".gem"
 
-        Kernel.system("gem push #{file}", exception: true)
+        out, status = Open3.capture2e("gem push #{file}")
+        next if status.success?
+
+        if out =~ /Repushing of gem versions is not allowed/
+          puts "Gem #{file} already exists on RubyGems.org, skipping..."
+
+          next
+        end
+
+        raise out
       end
     end
 
diff --git a/test/cli_test.rb b/test/cli_test.rb
index 05f7dff..7a1feeb 100644
--- a/test/cli_test.rb
+++ b/test/cli_test.rb
@@ -2,6 +2,7 @@
 
 require "test_helper"
 require "English"
+require "open3"
 
 module Cibuildgem
   class CLITest < Minitest::Test
@@ -129,14 +130,20 @@ def test_ci_template_when_passed_a_test_command_and_workdir
       FileUtils.rm_rf("test/fixtures/dummy_gem/.github")
     end
 
-    def test_release
+    def test_release_succeeds
       FileUtils.touch("tmp/foo.gem")
       FileUtils.touch("tmp/bar.gem")
       FileUtils.touch("tmp/some_file")
 
+      status = Struct.new(:success?)
       gem_pushed = []
+      callable = proc do |gem_name|
+        gem_pushed << gem_name
 
-      Kernel.stub(:system, ->(gem, _) { gem_pushed << gem }) do
+        ["", status.new(true)]
+      end
+
+      Open3.stub(:capture2e, callable) do
         CLI.start(["release", "--glob", "tmp/*"])
       end
 
@@ -147,6 +154,59 @@ def test_release
       FileUtils.rm_rf("tmp/some_file")
     end
 
+    def test_release_when_gem_was_already_pushed
+      FileUtils.touch("tmp/foo.gem")
+      FileUtils.touch("tmp/bar.gem")
+      FileUtils.touch("tmp/some_file")
+
+      status = Struct.new(:success?)
+      gem_pushed = []
+      callable = proc do |gem_name|
+        gem_pushed << gem_name
+
+        if gem_name == "gem push tmp/bar.gem"
+          ["Repushing of gem versions is not allowed", status.new(false)]
+        else
+          ["", status.new(true)]
+        end
+      end
+
+      Open3.stub(:capture2e, callable) do
+        out, _ = capture_subprocess_io do
+          CLI.start(["release", "--glob", "tmp/*"])
+        end
+
+        assert_equal("Gem tmp/bar.gem already exists on RubyGems.org, skipping...\n", out)
+      end
+
+      assert_equal(["gem push tmp/bar.gem", "gem push tmp/foo.gem"], gem_pushed.sort)
+    ensure
+      FileUtils.rm_rf("tmp/foo.gem")
+      FileUtils.rm_rf("tmp/bar.gem")
+      FileUtils.rm_rf("tmp/some_file")
+    end
+
+    def test_release_fails
+      FileUtils.touch("tmp/foo.gem")
+      FileUtils.touch("tmp/bar.gem")
+      FileUtils.touch("tmp/some_file")
+
+      status = Struct.new(:success?)
+      callable = proc do
+        ["Something went wrong", status.new(false)]
+      end
+
+      Open3.stub(:capture2e, callable) do
+        assert_raises(RuntimeError) do
+          CLI.start(["release", "--glob", "tmp/*"])
+        end
+      end
+    ensure
+      FileUtils.rm_rf("tmp/foo.gem")
+      FileUtils.rm_rf("tmp/bar.gem")
+      FileUtils.rm_rf("tmp/some_file")
+    end
+
     def test_print_ruby_cc_version
       out, _ = capture_subprocess_io do
         Dir.chdir("test/fixtures/dummy_gem") do