Detect RFC 8785 duplicate keys that differ only in encoding

lib/json/ext.rb

@@ -101,12 +101,22 @@ def shareable_lambda(block) # :nodoc:
     })
 
     generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash, as_json) {
-      if as_json
+      # Keys are converted when an as_json hook is given, or when a String or
+      # Symbol key is not UTF-8, since it can then match another key once both
+      # are written as UTF-8.
+      needs_conversion = as_json || hash.each_key.any? do |key|
+        next false unless String === key || Symbol === key
+        string = key.to_s
+        !(string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?))
+      end
+      if needs_conversion
         converted = {}
         hash.each do |key, value|
           string = (String === key || Symbol === key) ? key.to_s : nil
-          unless string && (string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?))
-            key = as_json.call(key, true)
+          if !as_json && !string
+            # Other keys are converted with #to_s by the generator.
+          elsif !string || !(string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?))
+            key = as_json ? as_json.call(key, true) : string
             unless String === key || Symbol === key
               raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key)
             end

test/json/json_generator_test.rb

@@ -1316,6 +1316,20 @@ def key.to_s
     assert_equal '{"1":1,"b":2}', JSON.generate({ 'b' => 2, 1 => 1 }, options.merge(strict: false))
   end
 
+  def test_rfc8785_duplicate_keys_in_different_encodings
+    latin1 = "\u00e9".encode(Encoding::ISO_8859_1)
+    hash = { latin1 => 1, "\u00e9" => 2 }
+    assert_equal 2, hash.size
+
+    error = assert_raise JSON::GeneratorError do
+      JSON.generate(hash, rfc8785: true)
+    end
+    assert_includes error.message, "detected duplicate key"
+
+    assert_rfc8785 "{\"\u00e9\":1}", { latin1 => 1 }
+    assert_rfc8785 "{\"a\":1,\"\u00e9\":2}", { latin1 => 2, "a" => 1 }
+  end
+
   def test_rfc8785_key_order_with_default_external_encoding
     # https://www.rfc-editor.org/rfc/rfc8785.html#section-3.2.3
     hash = { "\u20ac" => 1, "\r" => 2, "\ufb33" => 3, '1' => 4,