From e5774d00f798dbd07fc95d385e446bf4a80e418d Mon Sep 17 00:00:00 2001 From: ydah Date: Mon, 12 Oct 2026 17:17:01 +0900 Subject: [PATCH 1/3] Add shared RFC 9839 Unicode repertoire classifiers Define the character classes and repertoire masks used by the C, Java, and Ruby implementations, including strict UTF-8 decoding and replacement helpers. Package the Java helper in both extension jars. Verify the Ruby reference against the RFC ranges for every code point, with boundary and malformed UTF-8 maximal-subpart coverage. --- Rakefile | 2 + ext/json/ext/unicode.h | 172 +++++++++++++++++++++++++++ java/src/json/ext/UnicodeSubset.java | 130 ++++++++++++++++++++ lib/json/common.rb | 1 + lib/json/unicode_subset.rb | 74 ++++++++++++ test/json/unicode_subset_test.rb | 64 ++++++++++ 6 files changed, 443 insertions(+) create mode 100644 ext/json/ext/unicode.h create mode 100644 java/src/json/ext/UnicodeSubset.java create mode 100644 lib/json/unicode_subset.rb create mode 100644 test/json/unicode_subset_test.rb diff --git a/Rakefile b/Rakefile index db8ea3ac4..e18c78b51 100644 --- a/Rakefile +++ b/Rakefile @@ -167,6 +167,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' parser_classes = FileList[ "json/ext/ByteList*.class", "json/ext/OptionsReader*.class", + "json/ext/UnicodeSubset*.class", "json/ext/Parser*.class", "json/ext/Ryu*.class", "json/ext/RuntimeInfo*.class", @@ -187,6 +188,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' generator_classes = FileList[ "json/ext/*ByteList*.class", "json/ext/OptionsReader*.class", + "json/ext/UnicodeSubset*.class", "json/ext/EscapeScanner*.class", "json/ext/Generator*.class", "json/ext/RuntimeInfo*.class", diff --git a/ext/json/ext/unicode.h b/ext/json/ext/unicode.h new file mode 100644 index 000000000..5418e9e74 --- /dev/null +++ b/ext/json/ext/unicode.h @@ -0,0 +1,172 @@ +#ifndef JSON_UNICODE_H +#define JSON_UNICODE_H + +enum { + JSON_UNICODE_ILL_FORMED = 1 << 0, + JSON_UNICODE_SURROGATE = 1 << 1, + JSON_UNICODE_LEGACY_C0 = 1 << 2, + JSON_UNICODE_DEL = 1 << 3, + JSON_UNICODE_C1 = 1 << 4, + JSON_UNICODE_NONCHAR_FDD0 = 1 << 5, + JSON_UNICODE_NONCHAR_BMP_END = 1 << 6, + JSON_UNICODE_NONCHAR_SUPP = 1 << 7, + JSON_UNICODE_SCALARS = JSON_UNICODE_ILL_FORMED | JSON_UNICODE_SURROGATE, + JSON_UNICODE_XML_CHARACTERS = JSON_UNICODE_SCALARS | JSON_UNICODE_LEGACY_C0 | JSON_UNICODE_NONCHAR_BMP_END, + JSON_UNICODE_ASSIGNABLES = 0xFF, +}; + +static const uint8_t json_unicode_ascii_classes[256] = { + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, +}; + +static inline uint8_t json_unicode_classify(uint32_t cp) +{ + if (cp < 0x80) return json_unicode_ascii_classes[cp]; + if (cp > 0x10FFFF) return JSON_UNICODE_ILL_FORMED; + if (cp >= 0xD800 && cp <= 0xDFFF) return JSON_UNICODE_SURROGATE; + if (cp >= 0x80 && cp <= 0x9F) return JSON_UNICODE_C1; + if (cp >= 0xFDD0 && cp <= 0xFDEF) return JSON_UNICODE_NONCHAR_FDD0; + if ((cp & 0xFFFF) >= 0xFFFE) { + return cp <= 0xFFFF ? JSON_UNICODE_NONCHAR_BMP_END : JSON_UNICODE_NONCHAR_SUPP; + } + return 0; +} + +/* Consume a maximal subpart, as defined by Unicode's U+FFFD substitution rules. */ +static inline size_t json_unicode_decode(const char *ptr, const char *end, uint32_t *cp) +{ + const unsigned char *p = (const unsigned char *)ptr; + unsigned char first = p[0]; + size_t length; + unsigned char lower = 0x80, upper = 0xBF; + uint32_t value; + if (first < 0x80) { + *cp = first; + return 1; + } + *cp = UINT32_MAX; + if (first >= 0xC2 && first <= 0xDF) { + length = 2; + value = first & 0x1F; + } else if (first >= 0xE0 && first <= 0xEF) { + length = 3; + value = first & 0x0F; + if (first == 0xE0) lower = 0xA0; + if (first == 0xED) upper = 0x9F; + } else if (first >= 0xF0 && first <= 0xF4) { + length = 4; + value = first & 0x07; + if (first == 0xF0) lower = 0x90; + if (first == 0xF4) upper = 0x8F; + } else { + return 1; + } + for (size_t i = 1; i < length; i++) { + if ((size_t)(end - ptr) <= i || p[i] < lower || p[i] > upper) return i; + value = (value << 6) | (p[i] & 0x3F); + lower = 0x80; + upper = 0xBF; + } + *cp = value; + return length; +} + +#ifdef HAVE_SIMD_SSE2 +static inline TARGET_SSE2 +#else +static inline +#endif +const char *json_unicode_scan(const char *p, const char *end, uint8_t mask, uint32_t *cp, size_t *length) +{ + while (p < end) { + if ((unsigned char)*p >= 0x20 && (unsigned char)*p < 0x7F) { +#ifdef HAVE_SIMD_NEON + while ((size_t)(end - p) >= sizeof(uint8x16_t)) { + uint8x16_t bytes = vld1q_u8((const unsigned char *)p); + uint8x16_t matches = vorrq_u8(vcltq_u8(bytes, vdupq_n_u8(0x20)), vcgeq_u8(bytes, vdupq_n_u8(0x7F))); + uint64_t non_ascii = neon_match_mask(matches); + if (non_ascii) { + p += trailing_zeros64(non_ascii) >> 2; + break; + } + p += sizeof(bytes); + } +#elif defined(HAVE_SIMD_SSE2) + while ((size_t)(end - p) >= sizeof(__m128i)) { + __m128i bytes = _mm_loadu_si128((const __m128i *)p); + __m128i matches = _mm_or_si128(_mm_cmplt_epu8(bytes, _mm_set1_epi8(0x20)), _mm_cmpgt_epu8(bytes, _mm_set1_epi8(0x7E))); + int non_ascii = _mm_movemask_epi8(matches); + if (non_ascii) { + p += trailing_zeros(non_ascii); + break; + } + p += sizeof(bytes); + } +#endif + while ((size_t)(end - p) >= sizeof(uint64_t)) { + uint64_t bytes; + memcpy(&bytes, p, sizeof(bytes)); + if ((bytes & UINT64_C(0x8080808080808080)) || + ((bytes - UINT64_C(0x2020202020202020)) & UINT64_C(0x8080808080808080)) || + ((bytes + UINT64_C(0x0101010101010101)) & UINT64_C(0x8080808080808080))) break; + p += sizeof(bytes); + } + while (p < end && (unsigned char)*p >= 0x20 && (unsigned char)*p < 0x7F) { + p++; + } + if (p == end) break; + } + /* U+1000..CFFF are allowed by every repertoire. */ + while ((size_t)(end - p) >= 3 && + (unsigned char)p[0] >= 0xE1 && (unsigned char)p[0] <= 0xEC && + ((unsigned char)p[1] & 0xC0) == 0x80 && + ((unsigned char)p[2] & 0xC0) == 0x80) { + p += 3; + } + if (p == end) break; + *length = json_unicode_decode(p, end, cp); + if (json_unicode_classify(*cp) & mask) return p; + p += *length; + } + return NULL; +} + +static inline uint8_t json_unicode_subset_mask(VALUE value) +{ + if (NIL_P(value)) return 0; + if (value == ID2SYM(rb_intern("scalars"))) return JSON_UNICODE_SCALARS; + if (value == ID2SYM(rb_intern("xml_characters"))) return JSON_UNICODE_XML_CHARACTERS; + if (value == ID2SYM(rb_intern("assignables"))) return JSON_UNICODE_ASSIGNABLES; + rb_raise(rb_eArgError, "unicode_subset must be nil, :scalars, :xml_characters, or :assignables"); +} + +static inline const char *json_unicode_subset_name(uint8_t mask) +{ + switch (mask) { + case JSON_UNICODE_SCALARS: return "scalars"; + case JSON_UNICODE_XML_CHARACTERS: return "xml_characters"; + case JSON_UNICODE_ASSIGNABLES: return "assignables"; + default: return NULL; + } +} + +static inline VALUE json_unicode_subset_value(uint8_t mask) +{ + return mask ? ID2SYM(rb_intern(json_unicode_subset_name(mask))) : Qnil; +} + +static inline bool json_unicode_replace(VALUE value) +{ + if (value == ID2SYM(rb_intern("raise"))) return false; + if (value == ID2SYM(rb_intern("replace"))) return true; + rb_raise(rb_eArgError, "on_invalid_char must be :raise or :replace"); +} + +#endif diff --git a/java/src/json/ext/UnicodeSubset.java b/java/src/json/ext/UnicodeSubset.java new file mode 100644 index 000000000..3594ae96f --- /dev/null +++ b/java/src/json/ext/UnicodeSubset.java @@ -0,0 +1,130 @@ +package json.ext; + +import org.jruby.RubySymbol; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.ByteList; + +/** RFC 9839 code point classes and strict UTF-8 decoding. */ +final class UnicodeSubset { + static final int ILL_FORMED = 1; + static final int SURROGATE = 2; + static final int LEGACY_C0 = 4; + static final int DEL = 8; + static final int C1 = 16; + static final int NONCHAR_FDD0 = 32; + static final int NONCHAR_BMP_END = 64; + static final int NONCHAR_SUPP = 128; + static final int SCALARS = ILL_FORMED | SURROGATE; + static final int XML_CHARACTERS = SCALARS | LEGACY_C0 | NONCHAR_BMP_END; + static final int ASSIGNABLES = 255; + + private static final int[] ASCII_CLASSES = new int[128]; + static { + for (int cp = 0; cp < 0x20; cp++) { + if (cp != 9 && cp != 10 && cp != 13) ASCII_CLASSES[cp] = LEGACY_C0; + } + ASCII_CLASSES[0x7f] = DEL; + } + + static int mask(ThreadContext context, IRubyObject value) { + if (value == null || value.isNil()) return 0; + if (value instanceof RubySymbol) { + switch (((RubySymbol)value).idString()) { + case "scalars": return SCALARS; + case "xml_characters": return XML_CHARACTERS; + case "assignables": return ASSIGNABLES; + } + } + throw context.runtime.newArgumentError("unicode_subset must be nil, :scalars, :xml_characters, or :assignables"); + } + + static boolean replace(ThreadContext context, IRubyObject value) { + if (value == null) return false; + if (value instanceof RubySymbol) { + switch (((RubySymbol)value).idString()) { + case "raise": return false; + case "replace": return true; + } + } + throw context.runtime.newArgumentError("on_invalid_char must be :raise or :replace"); + } + + static String name(int mask) { + switch (mask) { + case SCALARS: return "scalars"; + case XML_CHARACTERS: return "xml_characters"; + case ASSIGNABLES: return "assignables"; + default: return null; + } + } + + static int classify(int cp) { + if (cp < 0 || cp > 0x10ffff) return ILL_FORMED; + if (cp < 0x80) return ASCII_CLASSES[cp]; + if (cp <= 0x9f) return C1; + if (cp >= 0xd800 && cp <= 0xdfff) return SURROGATE; + if (cp >= 0xfdd0 && cp <= 0xfdef) return NONCHAR_FDD0; + if ((cp & 0xffff) >= 0xfffe) return cp < 0x10000 ? NONCHAR_BMP_END : NONCHAR_SUPP; + return 0; + } + + // The high word is the code point (-1 if malformed); the low word is its + // byte length, or the maximal subpart length for an ill-formed sequence. + static long decode(byte[] bytes, int start, int end) { + int first = bytes[start] & 0xff; + if (first < 0x80) return ((long)first << 32) | 1; + int length; + int cp; + int min = 0x80; + int max = 0xbf; + if (first >= 0xc2 && first <= 0xdf) { + length = 2; + cp = first & 0x1f; + } else if (first >= 0xe0 && first <= 0xef) { + length = 3; + cp = first & 0xf; + if (first == 0xe0) min = 0xa0; + if (first == 0xed) max = 0x9f; + } else if (first >= 0xf0 && first <= 0xf4) { + length = 4; + cp = first & 7; + if (first == 0xf0) min = 0x90; + if (first == 0xf4) max = 0x8f; + } else { + return (-1L << 32) | 1; + } + for (int i = 1; i < length; i++) { + if (start + i == end) return (-1L << 32) | i; + int b = bytes[start + i] & 0xff; + if (b < min || b > max) return (-1L << 32) | i; + cp = (cp << 6) | (b & 0x3f); + min = 0x80; + max = 0xbf; + } + return ((long)cp << 32) | length; + } + + static String message(int cp, int mask) { + String character = cp < 0 ? "invalid UTF-8 sequence" : String.format("U+%04X", cp); + return character + " is not allowed by unicode_subset: :" + name(mask); + } + + static void append(ByteList out, int cp) { + if (cp < 0x80) { + out.append(cp); + } else if (cp < 0x800) { + out.append(0xc0 | (cp >>> 6)); + out.append(0x80 | (cp & 0x3f)); + } else if (cp < 0x10000) { + out.append(0xe0 | (cp >>> 12)); + out.append(0x80 | ((cp >>> 6) & 0x3f)); + out.append(0x80 | (cp & 0x3f)); + } else { + out.append(0xf0 | (cp >>> 18)); + out.append(0x80 | ((cp >>> 12) & 0x3f)); + out.append(0x80 | ((cp >>> 6) & 0x3f)); + out.append(0x80 | (cp & 0x3f)); + } + } +} diff --git a/lib/json/common.rb b/lib/json/common.rb index 4b2d6d87f..97d3e69f8 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'json/version' +require 'json/unicode_subset' module JSON module ParserOptions # :nodoc: diff --git a/lib/json/unicode_subset.rb b/lib/json/unicode_subset.rb new file mode 100644 index 000000000..181eacd9d --- /dev/null +++ b/lib/json/unicode_subset.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module JSON + # Internal reference implementation of RFC 9839 section 4. + module UnicodeSubset # :nodoc: + ILL_FORMED = 1 + SURROGATE = 2 + LEGACY_C0 = 4 + DEL = 8 + C1 = 16 + NONCHAR_FDD0 = 32 + NONCHAR_BMP_END = 64 + NONCHAR_SUPP = 128 + + MASKS = { + nil => 0, + scalars: ILL_FORMED | SURROGATE, + xml_characters: ILL_FORMED | SURROGATE | LEGACY_C0 | NONCHAR_BMP_END, + assignables: 255, + }.freeze + + PATTERNS = { + scalars: /(?!)/.freeze, + xml_characters: /[\u0000-\u0008\u000B\u000C\u000E-\u001F\uFFFE\uFFFF]/.freeze, + assignables: Regexp.new("[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF" + + (1..16).map { |plane| [(plane << 16) + 0xFFFE, (plane << 16) + 0xFFFF].pack('U*') }.join + "]").freeze, + }.freeze + + def self.mask(subset) + unless nil.equal?(subset) || Symbol === subset + raise ArgumentError, "invalid unicode_subset: #{subset.inspect}" + end + MASKS.fetch(subset) { raise ArgumentError, "invalid unicode_subset: #{subset.inspect}" } + end + + def self.classify(cp) + return ILL_FORMED unless (0..0x10FFFF).cover?(cp) + return SURROGATE if (0xD800..0xDFFF).cover?(cp) + return LEGACY_C0 if cp < 0x20 && cp != 9 && cp != 10 && cp != 13 + return DEL if cp == 0x7F + return C1 if (0x80..0x9F).cover?(cp) + return NONCHAR_FDD0 if (0xFDD0..0xFDEF).cover?(cp) + return cp <= 0xFFFF ? NONCHAR_BMP_END : NONCHAR_SUPP if cp & 0xFFFF >= 0xFFFE + 0 + end + + # Returns [codepoint, byte offset]; nil codepoint denotes ill-formed UTF-8. + # Callers transcode non-UTF-8 input before checking its repertoire. + def self.violation(string, subset) + forbidden = mask(subset) + return if forbidden == 0 + + if string.valid_encoding? + match = PATTERNS.fetch(subset).match(string) + [match[0].ord, match.pre_match.bytesize] if match + else + offset = 0 + string.each_char do |char| + return [nil, offset] unless char.valid_encoding? + return [char.ord, offset] unless classify(char.ord) & forbidden == 0 + offset += char.bytesize + end + nil + end + end + + def self.scrub(string, subset) + return string if mask(subset) == 0 + string = string.scrub("\uFFFD") unless string.valid_encoding? + pattern = PATTERNS.fetch(subset) + pattern.match?(string) ? string.gsub(pattern, "\uFFFD") : string + end + end +end diff --git a/test/json/unicode_subset_test.rb b/test/json/unicode_subset_test.rb new file mode 100644 index 000000000..da84f9c13 --- /dev/null +++ b/test/json/unicode_subset_test.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require_relative 'test_helper' +require 'json/unicode_subset' + +class JSONUnicodeSubsetTest < Test::Unit::TestCase + # RFC 9839 section 4, expressed as allowed ranges independently of the classifier. + RANGES = { + scalars: [0..0xD7FF, 0xE000..0x10FFFF], + xml_characters: [9..10, 13..13, 0x20..0xD7FF, 0xE000..0xFFFD, 0x10000..0x10FFFF], + assignables: [9..10, 13..13, 0x20..0x7E, 0xA0..0xD7FF, 0xE000..0xFDCF, 0xFDF0..0xFFFD] + + (1..16).map { |plane| (plane << 16)..((plane << 16) + 0xFFFD) }, + }.freeze + + def test_classifier_matches_rfc_abnf_for_every_codepoint + RANGES.each do |subset, ranges| + mask = JSON::UnicodeSubset.mask(subset) + allowed = Array.new(0x110000, false) + ranges.each { |range| range.each { |cp| allowed[cp] = true } } + mismatch = (0..0x10FFFF).find do |cp| + allowed[cp] != (JSON::UnicodeSubset.classify(cp) & mask == 0) + end + assert_nil mismatch, "#{subset}: #{mismatch && format('U+%04X', mismatch)}" + end + end + + def test_boundaries_and_regexps + points = [0, 8, 9, 10, 11, 12, 13, 14, 31, 32, 0x7E, 0x7F, 0x80, 0x9F, 0xA0, + 0xD7FF, 0xE000, 0xFDCF, 0xFDD0, 0xFDEF, 0xFDF0, 0xFFFD, 0xFFFE, 0xFFFF, + 0x10000, 0x1FFFD, 0x1FFFE, 0x10FFFD, 0x10FFFE, 0x10FFFF] + RANGES.each do |subset, ranges| + points.each do |cp| + char = [cp].pack('U') + allowed = ranges.any? { |range| range.cover?(cp) } + assert_equal allowed, !JSON::UnicodeSubset::PATTERNS.fetch(subset).match?(char) + assert_equal allowed ? nil : [cp, 3], JSON::UnicodeSubset.violation("日#{char}", subset) + assert_equal allowed ? char : "\uFFFD", JSON::UnicodeSubset.scrub(char, subset) + end + end + end + + def test_malformed_utf8_uses_maximal_subparts + ["\x80", "\xC0\xAF", "\xE1\x80", "\xED\xA0\x80", "\xF4\x90\x80\x80", "\xF0\x90a"].each do |string| + RANGES.each_key do |subset| + assert_equal [nil, 1], JSON::UnicodeSubset.violation("a#{string}", subset) + assert_equal string.scrub, JSON::UnicodeSubset.scrub(string, subset) + end + end + end + + def test_clean_string_is_not_copied + string = 'hello 日本 😀' + RANGES.each_key { |subset| assert_same string, JSON::UnicodeSubset.scrub(string, subset) } + end + + def test_invalid_subset + impersonator = Object.new + def impersonator.hash; :scalars.hash; end + def impersonator.eql?(other); true; end + [false, true, 'scalars', :unknown, impersonator].each do |subset| + assert_raise(ArgumentError) { JSON::UnicodeSubset.mask(subset) } + end + end +end From b0a2e8f305b4e4db98ce4da5a9c537e3c122ce96 Mon Sep 17 00:00:00 2001 From: ydah Date: Mon, 12 Oct 2026 17:17:28 +0900 Subject: [PATCH 2/3] Support RFC 9839 Unicode repertoires when parsing JSON Add unicode_subset and on_invalid_char options to the C and JRuby parsers and forward them through Coder. Validate decoded keys and values, report source positions and json_path, and support maximal-subpart replacement and unpaired surrogate escapes. Preserve the default behavior. Retain validation provenance for cached keys and check resumable chunk boundaries. Include shared parser tests and document the parsing options. --- CHANGES.md | 1 + README.md | 25 +++ ext/json/ext/parser/parser.c | 207 +++++++++++++++++++++--- java/src/json/ext/Parser.java | 93 ++++++++++- java/src/json/ext/StringDecoder.java | 21 ++- lib/json.rb | 28 ++++ lib/json/common.rb | 2 + test/json/unicode_parser_test.rb | 232 +++++++++++++++++++++++++++ 8 files changed, 588 insertions(+), 21 deletions(-) create mode 100644 test/json/unicode_parser_test.rb diff --git a/CHANGES.md b/CHANGES.md index 9ef0ce94a..fc342c23b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* Add opt-in `unicode_subset` and `on_invalid_char` options for RFC 9839 Unicode repertoires in parsing, including Coder#load and ResumableParser. * JRuby: parser errors now include the position (`line`, `column` and the message suffix) as well as `json_path`, matching the C extension. ### 2026-09-09 (3.0.2) diff --git a/README.md b/README.md index ab550ad7c..be258b5ab 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,31 @@ JSON.generate(data) You can also use the `pretty_generate` method (which formats the output more verbosely and nicely). +## Standards compliance + +The optional `unicode_subset` setting restricts object keys and string values to +one of the repertoires in [RFC 9839](https://www.rfc-editor.org/rfc/rfc9839.html): + +* `:scalars` requires well-formed Unicode, excluding surrogates. +* `:xml_characters` also excludes legacy C0 controls and U+FFFE/U+FFFF. +* `:assignables` also excludes DEL, C1 controls, and all noncharacters. + +Unassigned and private-use codepoints remain allowed. The default, `nil`, keeps +the existing behavior. These options work with parsing, `JSON::Coder#load`, +and `JSON::ResumableParser` (where available). + +```ruby +JSON.parse('"\u0089"', unicode_subset: :assignables) # raises JSON::ParserError +JSON.parse('"\u0089"', unicode_subset: :assignables, on_invalid_char: :replace) # => "�" +``` + +`on_invalid_char` defaults to `:raise`. With `:replace`, forbidden codepoints and +each maximal subpart of malformed UTF-8 become U+FFFD. With no subset selected, +replacement only relaxes parsing of unpaired `\uXXXX` surrogate escapes. +Escapes cannot bypass a subset restriction; `allow_control_characters` only +relaxes JSON syntax. Keep duplicate-key detection enabled when replacing +characters, since different keys can become equal after replacement. + ## Casting non native types JSON documents can only support Hashes, Arrays, Strings, Integers and Floats. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b456a4e23..0c4dca5c0 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1,6 +1,7 @@ #include "../json.h" #include "../vendor/fast_float_parser.h" #include "../simd/simd.h" +#include "../unicode.h" static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string; @@ -12,7 +13,8 @@ static ID i_uminus; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments, sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names, - sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key; + sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key, + sym_unicode_subset, sym_on_invalid_char; static int binary_encindex; static int utf8_encindex; @@ -58,6 +60,8 @@ static VALUE rb_str_to_interned_str(VALUE str) #define JSON_RVALUE_CACHE_CAPA 63 typedef struct rvalue_cache_struct { int length; + uint8_t forbidden_classes; + bool symbolize; VALUE entries[JSON_RVALUE_CACHE_CAPA]; } rvalue_cache; @@ -149,7 +153,7 @@ ALWAYS_INLINE(static) int rstring_cache_cmp(const char *str, const long length, } } -ALWAYS_INLINE(static) VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length) +ALWAYS_INLINE(static) VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length, bool create) { int low = 0; int high = cache->length - 1; @@ -168,6 +172,8 @@ ALWAYS_INLINE(static) VALUE rstring_cache_fetch(rvalue_cache *cache, const char } } + if (!create) return Qundef; + VALUE rstring = build_interned_string(str, length); if (cache->length < JSON_RVALUE_CACHE_CAPA) { @@ -176,7 +182,7 @@ ALWAYS_INLINE(static) VALUE rstring_cache_fetch(rvalue_cache *cache, const char return rstring; } -static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const long length) +static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const long length, bool create) { int low = 0; int high = cache->length - 1; @@ -195,6 +201,8 @@ static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const lon } } + if (!create) return Qundef; + VALUE rsymbol = build_symbol(str, length); if (cache->length < JSON_RVALUE_CACHE_CAPA) { @@ -419,6 +427,8 @@ typedef struct JSON_ParserStruct { bool allow_invalid_escape; bool symbolize_names; bool freeze; + uint8_t forbidden_classes; + bool replace_invalid_chars; } JSON_ParserConfig; typedef struct JSON_ParserStateStruct { @@ -427,6 +437,8 @@ typedef struct JSON_ParserStateStruct { const char *start; const char *cursor; const char *end; + long line_offset; + rb_len_t column_offset; rvalue_stack *value_stack; json_frame_stack *frames; rvalue_cache name_cache; @@ -585,7 +597,6 @@ static inline char peek(JSON_ParserState *state) static void cursor_position(JSON_ParserState *state, long *line_out, rb_len_t *column_out) { - JSON_ASSERT(!state->parser); JSON_ASSERT(state->cursor); JSON_ASSERT(state->cursor <= state->end); @@ -595,12 +606,13 @@ static void cursor_position(JSON_ParserState *state, long *line_out, rb_len_t *c } const char *cursor = state->cursor; - long line = 1; + long line = state->line_offset + 1; while (cursor > state->start && cursor[-1] != '\n') { cursor--; } rb_len_t column = rb_enc_strlen(cursor, state->cursor, enc_utf8) + 1; + if (cursor == state->start) column += state->column_offset; while (cursor > state->start) { if (*--cursor == '\n') { @@ -755,7 +767,7 @@ static const signed char digit_values[256] = { static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe) { - if (RB_UNLIKELY(sp > spe - 4)) { + if (RB_UNLIKELY(spe - sp < 4)) { raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); } @@ -920,18 +932,109 @@ static inline bool json_string_cacheable_p(const char *string, size_t length) return length <= JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH && rb_isalpha(string[0]); } +NORETURN(static) void json_raise_unicode_error(JSON_ParserState *state, JSON_ParserConfig *config, const char *at, uint32_t cp) +{ + state->cursor = at; + VALUE message; + const char *subset = json_unicode_subset_name(config->forbidden_classes); + if (cp == UINT32_MAX) { + message = rb_sprintf("invalid UTF-8 is not allowed by unicode_subset: :%s", subset); + } else { + message = rb_sprintf("U+%04X is not allowed by unicode_subset: :%s", cp, subset); + } + long line; + rb_len_t column; + cursor_position(state, &line, &column); + rb_str_catf(message, " at line %ld column %"PRIdLEN, line, column); + rb_exc_raise(parse_error_new(state, message, line, column, false)); +} + +static inline char *json_string_unicode_copy(JSON_ParserState *state, JSON_ParserConfig *config, char *buffer, const char *p, const char *end, const char *escape) +{ + uint32_t cp; + size_t length; + const char *invalid; + while ((invalid = json_unicode_scan(p, end, config->forbidden_classes, &cp, &length))) { + if (!config->replace_invalid_chars) json_raise_unicode_error(state, config, invalid == escape ? invalid - 1 : invalid, cp); + MEMCPY(buffer, p, char, invalid - p); + buffer += invalid - p; + buffer += convert_UTF32_to_UTF8(buffer, 0xFFFD); + p = invalid + length; + } + MEMCPY(buffer, p, char, end - p); + return buffer + (end - p); +} + +static inline size_t json_string_buffer_size(size_t size, JSON_ParserConfig *config) +{ + if (config->forbidden_classes && config->replace_invalid_chars) { + if (size > LONG_MAX / 3) rb_memerror(); + return size * 3; + } + return size; +} + +NOINLINE(static) VALUE json_string_subset_fastpath(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name) +{ + bool intern = is_name || config->freeze; + bool symbolize = is_name && config->symbolize_names; + size_t bufferSize = stringEnd - string; + bool cacheable = is_name && state->in_array && json_string_cacheable_p(string, bufferSize); + + if (cacheable) { + rvalue_cache *cache = &state->name_cache; + // Config can be reinitialized by on_load while this cache is in use. + if (cache->forbidden_classes != config->forbidden_classes || cache->symbolize != symbolize) { + cache->length = 0; + cache->forbidden_classes = config->forbidden_classes; + cache->symbolize = symbolize; + } + VALUE cached = symbolize ? rsymbol_cache_fetch(cache, string, bufferSize, false) : + rstring_cache_fetch(cache, string, bufferSize, false); + if (cached != Qundef) return cached; + } + + uint32_t cp; + size_t length; + const char *invalid = json_unicode_scan(string, stringEnd, config->forbidden_classes, &cp, &length); + if (invalid) { + if (!config->replace_invalid_chars) json_raise_unicode_error(state, config, invalid, cp); + VALUE result = rb_str_buf_new(json_string_buffer_size(bufferSize, config)); + rb_enc_associate_index(result, utf8_encindex); + char *buffer = RSTRING_PTR(result); + char *end = json_string_unicode_copy(state, config, buffer, string, stringEnd, NULL); + rb_str_set_len(result, end - buffer); + if (symbolize) return rb_str_intern(result); + return intern ? rb_str_to_interned_str(result) : result; + } + + if (cacheable) { + return symbolize ? rsymbol_cache_fetch(&state->name_cache, string, bufferSize, true) : + rstring_cache_fetch(&state->name_cache, string, bufferSize, true); + } + return build_string(string, stringEnd, intern, symbolize); +} + static inline VALUE json_string_fastpath(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name) { + if (RB_UNLIKELY(config->forbidden_classes)) { + return json_string_subset_fastpath(state, config, string, stringEnd, is_name); + } bool intern = is_name || config->freeze; bool symbolize = is_name && config->symbolize_names; size_t bufferSize = stringEnd - string; if (is_name && state->in_array && RB_LIKELY(json_string_cacheable_p(string, bufferSize))) { + state->name_cache.forbidden_classes = 0; + if (RB_UNLIKELY(state->name_cache.symbolize != symbolize)) { + state->name_cache.length = 0; + state->name_cache.symbolize = symbolize; + } VALUE cached_key; if (RB_UNLIKELY(symbolize)) { - cached_key = rsymbol_cache_fetch(&state->name_cache, string, bufferSize); + cached_key = rsymbol_cache_fetch(&state->name_cache, string, bufferSize, true); } else { - cached_key = rstring_cache_fetch(&state->name_cache, string, bufferSize); + cached_key = rstring_cache_fetch(&state->name_cache, string, bufferSize, true); } if (RB_LIKELY(cached_key)) { @@ -974,19 +1077,34 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser bool symbolize = is_name && config->symbolize_names; size_t bufferSize = stringEnd - string; const char *p = string, *pe = string, *bufferStart; + const char *escaped_char = NULL; char *buffer; - VALUE result = rb_str_buf_new(bufferSize); + VALUE result = rb_str_buf_new(json_string_buffer_size(bufferSize, config)); rb_enc_associate_index(result, utf8_encindex); buffer = RSTRING_PTR(result); bufferStart = buffer; -#define APPEND_CHAR(chr) *buffer++ = chr; p = ++pe; +#define APPEND_CHAR(chr) do { \ + uint32_t cp = (unsigned char)(chr); \ + if (RB_UNLIKELY(config->forbidden_classes && (config->forbidden_classes & json_unicode_classify(cp)))) { \ + if (!config->replace_invalid_chars) json_raise_unicode_error(state, config, pe - 1, cp); \ + buffer += convert_UTF32_to_UTF8(buffer, 0xFFFD); \ + } else { \ + *buffer++ = (char)cp; \ + } \ + p = ++pe; \ +} while (0) while (pe < stringEnd && (pe = json_next_backslash(pe, stringEnd, positions))) { if (pe > p) { - MEMCPY(buffer, p, char, pe - p); - buffer += pe - p; + if (RB_UNLIKELY(config->forbidden_classes)) { + buffer = json_string_unicode_copy(state, config, buffer, p, pe, escaped_char); + escaped_char = NULL; + } else { + MEMCPY(buffer, p, char, pe - p); + buffer += pe - p; + } } switch (*++pe) { case '"': @@ -1012,6 +1130,7 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser APPEND_CHAR('\f'); break; case 'u': { + const char *escape = pe - 1; uint32_t ch = unescape_unicode(state, ++pe, stringEnd); pe += 3; /* To handle values above U+FFFF, we take a sequence of @@ -1026,26 +1145,47 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser */ if ((ch & 0xFC00) == 0xD800) { pe++; - if (RB_LIKELY((pe <= stringEnd - 6) && memcmp(pe, "\\u", 2) == 0)) { + if (RB_LIKELY((stringEnd - pe >= 6) && memcmp(pe, "\\u", 2) == 0)) { uint32_t sur = unescape_unicode(state, pe + 2, stringEnd); if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) { + if (config->replace_invalid_chars) { + ch = 0xFFFD; + pe--; + goto append_unicode; + } + if (config->forbidden_classes) json_raise_unicode_error(state, config, escape, ch); raise_syntax_error_at("invalid surrogate pair at %s", state, p); } ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF)); pe += 5; } else { + if (config->replace_invalid_chars) { + ch = 0xFFFD; + pe--; + goto append_unicode; + } + if (config->forbidden_classes) json_raise_unicode_error(state, config, escape, ch); raise_syntax_error_at("incomplete surrogate pair at %s", state, p); break; } } else if ((ch & 0xFC00) == 0xDC00) { + if (config->replace_invalid_chars) { + ch = 0xFFFD; + goto append_unicode; + } + if (config->forbidden_classes) json_raise_unicode_error(state, config, escape, ch); raise_syntax_error_at("unpaired trailing surrogate at %s", state, p); break; } - int unescape_len = convert_UTF32_to_UTF8(buffer, ch); - buffer += unescape_len; + if (RB_UNLIKELY(config->forbidden_classes && (config->forbidden_classes & json_unicode_classify(ch)))) { + if (!config->replace_invalid_chars) json_raise_unicode_error(state, config, escape, ch); + ch = 0xFFFD; + } + append_unicode: + buffer += convert_UTF32_to_UTF8(buffer, ch); p = ++pe; break; } @@ -1062,7 +1202,12 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser } if (config->allow_invalid_escape) { - APPEND_CHAR(*pe); + if ((unsigned char)*pe >= 0x80 && config->forbidden_classes) { + escaped_char = pe; + p = pe++; + } else { + APPEND_CHAR(*pe); + } } else { raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1); } @@ -1072,8 +1217,12 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser #undef APPEND_CHAR if (stringEnd > p) { - MEMCPY(buffer, p, char, stringEnd - p); - buffer += stringEnd - p; + if (RB_UNLIKELY(config->forbidden_classes)) { + buffer = json_string_unicode_copy(state, config, buffer, p, stringEnd, escaped_char); + } else { + MEMCPY(buffer, p, char, stringEnd - p); + buffer += stringEnd - p; + } } rb_str_set_len(result, buffer - bufferStart); @@ -2020,6 +2169,8 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } + else if (key == sym_unicode_subset) { config->forbidden_classes = json_unicode_subset_mask(val); } + else if (key == sym_on_invalid_char) { config->replace_invalid_chars = json_unicode_replace(val); } else if (key == sym_on_load) { if (RTEST(val) && !rb_obj_is_proc(val)) { val = rb_check_funcall(val, rb_intern("to_proc"), 0, NULL); @@ -2409,6 +2560,17 @@ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock); +static void json_unicode_discard_prefix(JSON_ResumableParser *parser) +{ + if (parser->config.forbidden_classes && parser->state.start) { + long line; + rb_len_t column; + cursor_position(&parser->state, &line, &column); + parser->state.line_offset = line - 1; + parser->state.column_offset = column - 1; + } +} + /* * call-seq: self << string -> self * @@ -2429,6 +2591,7 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) const size_t remaining = parser->state.end - parser->state.cursor; if (!remaining) { + json_unicode_discard_prefix(parser); if (parser->buffer) { json_str_clear(parser->buffer); } @@ -2444,12 +2607,15 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str))); rb_enc_associate_index(new_buffer, utf8_encindex); + json_unicode_discard_prefix(parser); + char *old_ptr = RSTRING_PTR(parser->buffer); memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining); rb_str_set_len(new_buffer, remaining); offset = 0; parser->buffer = new_buffer; } else if (consumed > (size / 2) && size >= 512) { + json_unicode_discard_prefix(parser); rb_str_modify(parser->buffer); char *old_ptr = RSTRING_PTR(parser->buffer); memmove(old_ptr, old_ptr + consumed, remaining); @@ -2571,6 +2737,7 @@ static VALUE cResumableParser_parse(VALUE self) json_eat_whitespace(&parser->state, &parser->config, false); if (eos(&parser->state)) { + json_unicode_discard_prefix(parser); json_str_clear(parser->buffer); parser->buffer = Qfalse; parser->state.start = parser->state.cursor = parser->state.end = 0; @@ -2646,6 +2813,8 @@ static VALUE cResumableParser_clear(VALUE self) parser->value_stack.head = 0; parser->state.name_cache.length = 0; parser->state.current_nesting = 0; + parser->state.line_offset = 0; + parser->state.column_offset = 0; parser->state.in_array = 1; parser->state.start = parser->state.cursor = parser->state.end = NULL; return self; @@ -2901,6 +3070,8 @@ void Init_parser(void) sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape")); sym_symbolize_names = ID2SYM(rb_intern("symbolize_names")); sym_freeze = ID2SYM(rb_intern("freeze")); + sym_unicode_subset = ID2SYM(rb_intern("unicode_subset")); + sym_on_invalid_char = ID2SYM(rb_intern("on_invalid_char")); sym_on_load = ID2SYM(rb_intern("on_load")); sym_decimal_class = ID2SYM(rb_intern("decimal_class")); sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 6fa5d4383..636d14171 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -51,6 +51,8 @@ public class Parser extends RubyObject { private boolean allowControlCharacters; private boolean allowInvalidEscape; private boolean allowDuplicateKey; + private int forbiddenClasses; + private boolean replaceInvalidChars; private boolean symbolizeNames; private boolean freeze; private RubyProc onLoadProc; @@ -112,6 +114,8 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); + this.forbiddenClasses = UnicodeSubset.mask(context, opts.get("unicode_subset")); + this.replaceInvalidChars = UnicodeSubset.replace(context, opts.get("on_invalid_char")); this.freeze = opts.getBool("freeze", false); this.onLoadProc = opts.getProc("on_load"); @@ -288,7 +292,7 @@ private ParserSession(Parser config, RubyString source, RuntimeInfo info) { this.view = new ByteList(data, false); this.begin = byteList.begin(); this.end = begin + byteList.length(); - this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape); + this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape, config.replaceInvalidChars); } public IRubyObject parse(ThreadContext context) { @@ -681,6 +685,16 @@ private IRubyObject parseString(boolean isName) { cursor = q + 1; // past closing quote + if (config.forbiddenClasses != 0) { + ByteList content = decodeSubsetString(contentStart, q); + if (isName) { + return internedKey(content.unsafeBytes(), content.begin(), content.realSize()); + } + RubyString string = context.runtime.newString(content); + string.setEncoding(UTF8Encoding.INSTANCE); + return config.freeze ? context.runtime.freezeAndDedupString(string) : string; + } + // Note: When running multiple read-world benchmarks in the same JVM, // this seems consistently faster than "if (isName && plain)" // and only handling the ASCII-only path in the cache. @@ -728,6 +742,83 @@ private IRubyObject parseString(boolean isName) { return string; } + private RaiseException stringError(int position, String message) { + cursor = position; + return parseError(message); + } + + private int unicodeEscape(int start, int end) { + if (start + 6 > end) { + throw stringError(start, "incomplete unicode character escape sequence"); + } + int cp = 0; + for (int i = start + 2; i < start + 6; i++) { + int digit = data[i] & 0xff; + int value = digit >= '0' && digit <= '9' ? digit - '0' : + digit >= 'a' && digit <= 'f' ? digit - 'a' + 10 : + digit >= 'A' && digit <= 'F' ? digit - 'A' + 10 : -1; + if (value < 0) throw stringError(start, "incomplete unicode character escape sequence"); + cp = (cp << 4) | value; + } + return cp; + } + + private ByteList decodeSubsetString(int start, int end) { + ByteList out = new ByteList(end - start); + int mask = config.forbiddenClasses; + for (int p = start; p < end;) { + int position = p; + int cp = data[p] & 0xff; + if (cp == '\\') { + if (++p == end) throw stringError(position, "invalid escape character in string"); + cp = data[p++] & 0xff; + if (cp < 0x20 && !config.allowControlCharacters) { + throw stringError(position, "invalid ASCII control character in string"); + } + switch (cp) { + case 'b': cp = '\b'; break; + case 'f': cp = '\f'; break; + case 'n': cp = '\n'; break; + case 'r': cp = '\r'; break; + case 't': cp = '\t'; break; + case '"': case '/': case '\\': break; + case 'u': + cp = unicodeEscape(position, end); + p = position + 6; + if (cp >= 0xd800 && cp <= 0xdbff && p + 6 <= end && data[p] == '\\' && data[p + 1] == 'u') { + int low = unicodeEscape(p, end); + if (low >= 0xdc00 && low <= 0xdfff) { + cp = Character.toCodePoint((char)cp, (char)low); + p += 6; + } + } + if (cp >= 0xd800 && cp <= 0xdfff && config.replaceInvalidChars) cp = 0xfffd; + break; + default: + if (!config.allowInvalidEscape) throw stringError(position, "invalid escape character in string"); + p--; + long decoded = UnicodeSubset.decode(data, p, end); + cp = (int)(decoded >> 32); + p += (int)decoded; + break; + } + } else { + if (cp < 0x20 && !config.allowControlCharacters) { + throw stringError(position, "invalid ASCII control character in string"); + } + long decoded = UnicodeSubset.decode(data, p, end); + cp = (int)(decoded >> 32); + p += (int)decoded; + } + if ((UnicodeSubset.classify(cp) & mask) != 0) { + if (!config.replaceInvalidChars) throw stringError(position, UnicodeSubset.message(cp, mask)); + cp = 0xfffd; + } + UnicodeSubset.append(out, cp); + } + return out; + } + private ByteList decodeString(ThreadContext context, ByteList byteList, int start, int end, boolean isAscii) { if (isAscii) { return decoder.decodeNoValidate(context, byteList, start, end, chunks); diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index 362a0d685..bf93fa15e 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -25,6 +25,7 @@ final class StringDecoder extends ByteListTranscoder { private int surrogatePairStart = -1; private boolean allowControlCharacters = false; private boolean allowInvalidEscape = false; + private final boolean replaceInvalidChars; private final StringScanner scanner = StringScanner.getInstance(); @@ -33,9 +34,10 @@ final class StringDecoder extends ByteListTranscoder { // Array used for writing multibyte characters into the buffer at once private final byte[] aux = new byte[4]; - public StringDecoder(boolean allowControlCharacters, boolean allowInvalidEscape) { + public StringDecoder(boolean allowControlCharacters, boolean allowInvalidEscape, boolean replaceInvalidChars) { this.allowControlCharacters = allowControlCharacters; this.allowInvalidEscape = allowInvalidEscape; + this.replaceInvalidChars = replaceInvalidChars; } ByteList decode(ThreadContext context, ByteList src, int start, int end) { @@ -148,7 +150,8 @@ private void handleEscapeSequence(ThreadContext context) throws IOException { handleLowSurrogate(context, (char)cp); } else if (Character.isLowSurrogate((char)cp)) { // low surrogate with no high surrogate - throw invalidUtf8(context); + if (!replaceInvalidChars) throw invalidUtf8(context); + writeUtf8Char(0xfffd); } else { writeUtf8Char(cp); } @@ -171,6 +174,20 @@ protected void append(byte[] origin, int start, int length) throws IOException { } private void handleLowSurrogate(ThreadContext context, char highSurrogate) throws IOException { + if (replaceInvalidChars) { + int next = pos; + if (pos + 6 <= srcEnd && src.charAt(pos) == '\\' && src.charAt(pos + 1) == 'u') { + pos += 2; + int low = readHex(context); + if (Character.isLowSurrogate((char)low)) { + writeUtf8Char(Character.toCodePoint(highSurrogate, (char)low)); + return; + } + } + pos = next; + writeUtf8Char(0xfffd); + return; + } surrogatePairStart = charStart; ensureMin(context, 1); int lowSurrogate = readUtf8Char(context); diff --git a/lib/json.rb b/lib/json.rb index 1969b026d..478a44fd0 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -259,6 +259,34 @@ # ruby = JSON.parse(source, array_class: Set) # ruby # => Set["foo", 1.0, true, false, nil] # +# --- +# +# Option +unicode_subset+ restricts object keys and string values to a repertoire +# defined by {RFC 9839}[https://www.rfc-editor.org/rfc/rfc9839.html]: +# +# - +nil+ (default): preserve existing behavior. +# - +:scalars+: reject malformed UTF-8 and surrogates. +# - +:xml_characters+: also reject legacy C0 controls (except tab, LF and CR), +# and U+FFFE/U+FFFF. +# - +:assignables+: also reject DEL, C1 controls and all noncharacters. +# +# Unassigned and private-use codepoints are allowed by all three subsets. +# Restrictions apply after escape decoding and before symbolization or freezing. +# +allow_control_characters+ relaxes syntax only; it cannot bypass the subset. +# +# Option +on_invalid_char+ is +:raise+ (default) or +:replace+. +# Violations raise JSON::ParserError with the source position and +json_path+, +# or become U+FFFD. Malformed UTF-8 is replaced once per maximal subpart. +# With +unicode_subset: nil+, +:replace+ only permits unpaired surrogate escapes, +# replacing each with U+FFFD; other syntax and encoding behavior is unchanged. +# +# JSON.parse('"\u0089"', unicode_subset: :assignables) # Raises JSON::ParserError +# JSON.parse('"\u0089"', unicode_subset: :assignables, on_invalid_char: :replace) # => "�" +# +# Keep +allow_duplicate_key+ disabled with replacement: different keys may +# become equal, and duplicate detection compares the replaced keys. +# Unsupported option values raise ArgumentError. +# # === Generating \JSON # # To generate a Ruby \String containing \JSON data, diff --git a/lib/json/common.rb b/lib/json/common.rb index 97d3e69f8..892386625 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -797,6 +797,8 @@ class Coder freeze allow_duplicate_key decimal_class + unicode_subset + on_invalid_char ).freeze private_constant :PARSER_OPTIONS diff --git a/test/json/unicode_parser_test.rb b/test/json/unicode_parser_test.rb new file mode 100644 index 000000000..b3cff8b65 --- /dev/null +++ b/test/json/unicode_parser_test.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: true + +require_relative 'test_helper' + +class JSONUnicodeParserTest < Test::Unit::TestCase + SUBSETS = [:scalars, :xml_characters, :assignables].freeze + BOUNDARIES = [0, 8, 9, 10, 11, 12, 13, 14, 31, 32, 126, 127, 128, 159, 160, + 0xD7FF, 0xE000, 0xFDCF, 0xFDD0, 0xFDEF, 0xFDF0, 0xFFFD, 0xFFFE, 0xFFFF, + 0x10000, 0x1FFFD, 0x1FFFE, 0x1FFFF, 0x10FFFD, 0x10FFFE, 0x10FFFF].freeze + + def test_subset_boundaries + SUBSETS.each do |subset| + BOUNDARIES.each do |cp| + char = cp.chr(Encoding::UTF_8) + [char, unicode_escape(cp)].each do |source| + [false, true].each do |key| + json = key ? %({"#{source}":1}) : %(["#{source}"]) + options = { unicode_subset: subset, allow_control_characters: true } + expected = allowed?(cp, subset) ? char : "\uFFFD" + result = key ? { expected => 1 } : [expected] + assert_equal result, JSON.parse(json, **options, on_invalid_char: :replace), [subset, cp, key, source].inspect + if allowed?(cp, subset) + assert_equal result, JSON.parse(json, **options) + else + error = assert_raise(JSON::ParserError) { JSON.parse(json, **options) } + assert_include error.message, 'U+%04X' % cp + assert_include error.message, "unicode_subset: :#{subset}" + end + end + end + end + end + end + + def test_key_cache_after_subset_changes + omit 'Parser config reinitialization is specific to the C extension' if RUBY_ENGINE == 'jruby' + + config = nil + on_load = lambda do |value| + config.send(:initialize, unicode_subset: :assignables) if value == 0 + value + end + config = JSON::Parser::Config.new(unicode_subset: :scalars, on_load: on_load) + error = assert_raise(JSON::ParserError) { config.parse(%([{"a\u0085":0},{"a\u0085":1}])) } + assert_include error.message, 'U+0085' + + on_load = lambda do |value| + if value == 0 + config.send(:initialize, unicode_subset: nil, on_load: on_load) + elsif value == 1 + config.send(:initialize, unicode_subset: :assignables) + end + value + end + config = JSON::Parser::Config.new(unicode_subset: :assignables, on_load: on_load) + error = assert_raise(JSON::ParserError) { config.parse(%([{"a":0},{"a\u0085":1},{"a\u0085":2}])) } + assert_include error.message, 'U+0085' + + on_load = lambda do |value| + config.send(:initialize, unicode_subset: nil, symbolize_names: false) if value == 0 + value + end + config = JSON::Parser::Config.new(unicode_subset: :assignables, symbolize_names: true, on_load: on_load) + assert_equal [{ a: 0 }, { 'a' => 1 }], config.parse('[{"a":0},{"a":1}]') + end + + def test_invalid_utf8_maximal_subparts + invalid = ["\x80", "\xC0\xAF", "\xE0\x80\x80", "\xED\xA0\x80", "\xF4\x90\x80\x80", + "\xF5\x80\x80\x80", "\xE1", "\xE1\x80", "\xF1\x80\x80", "\xE1\x80z", "\xC2z"] + SUBSETS.each do |subset| + invalid.each do |bytes| + [bytes, "a\\tb#{bytes}c", "#{bytes}\\nc"].each do |source| + [false, true].each do |key| + json = key ? %({"#{source}":1}) : %(["#{source}"]) + error = assert_raise(JSON::ParserError) { JSON.parse(json, unicode_subset: subset) } + assert_include error.message, 'invalid UTF-8' + expected = JSON.parse(json.scrub, unicode_subset: subset) + assert_equal expected, JSON.parse(json, unicode_subset: subset, on_invalid_char: :replace) + end + end + end + end + end + + def test_orphan_surrogates + { + '\uD800' => "\uFFFD", '\uDFFF' => "\uFFFD", '\uD800x' => "\uFFFDx", + '\uD800\u0041' => "\uFFFDA", '\uD800\uD800\uDC00' => "\uFFFD\u{10000}", + '\uDC00\uD800' => "\uFFFD\uFFFD", '\uD800\n' => "\uFFFD\n", + }.each do |source, expected| + [nil, *SUBSETS].each do |subset| + assert_equal expected, JSON.parse(%("#{source}"), unicode_subset: subset, on_invalid_char: :replace) + assert_raise(JSON::ParserError) { JSON.parse(%("#{source}"), unicode_subset: subset) } + end + end + ['\uD800\uZZZZ', '\uZZZZ', '\uD800\u123', '\u123'].each do |source| + assert_raise(JSON::ParserError) { JSON.parse(%("#{source}"), on_invalid_char: :replace) } + end + end + + def test_default_and_nil_subset_compatibility + source = "[\"\xED\xA0\x80\u0089\uFFFF\"]" + expected = JSON.parse(source) + assert_equal expected, JSON.parse(source, unicode_subset: nil) + assert_equal expected, JSON.parse(source, on_invalid_char: :replace) + assert_equal ["\u0000\u0089\u{7FFFF}"], JSON.parse('["\u0000\u0089\uD9BF\uDFFF"]') + end + + def test_option_validation + [false, true, 0, 'scalars', :unknown].each do |value| + assert_raise(ArgumentError) { JSON.parse('null', unicode_subset: value) } + end + [nil, false, true, 0, 'replace', :unknown].each do |value| + assert_raise(ArgumentError) { JSON.parse('null', on_invalid_char: value) } + end + end + + def test_source_positions_and_path + ['\u0089', "\u0089"].each do |source| + error = assert_raise(JSON::ParserError) do + JSON.parse("{\"outer\": [\n\"日#{source}\"]}", unicode_subset: :assignables) + end + assert_equal 2, error.line + assert_equal 3, error.column + assert_equal '$.outer[0]', error.json_path + end + error = assert_raise(JSON::ParserError) do + JSON.parse('{"outer":{"a\uFFFF":0}}', unicode_subset: :assignables) + end + assert_equal '$.outer', error.json_path + assert_equal 13, error.column + end + + def test_replacement_before_symbolizing_freezing_and_duplicate_detection + result = JSON.parse('[{"a\u0089":"b\uFFFF"},{"a\u0089":"b\uFFFF"}]', + unicode_subset: :assignables, on_invalid_char: :replace, symbolize_names: true, freeze: true) + assert_equal [{ :"a\uFFFD" => "b\uFFFD" }, { :"a\uFFFD" => "b\uFFFD" }], result + assert_predicate result[0].values[0], :frozen? + assert_raise(JSON::ParserError) do + JSON.parse('{"\u0089":1,"\uFFFF":2}', unicode_subset: :assignables, on_invalid_char: :replace) + end + assert_equal({ "\uFFFD" => 2 }, JSON.parse('{"\u0089":1,"\uFFFF":2}', + unicode_subset: :assignables, on_invalid_char: :replace, allow_duplicate_key: true)) + end + + def test_permissive_syntax_options_do_not_bypass_subset + assert_raise(JSON::ParserError) { JSON.parse("\"\x01\"", unicode_subset: :assignables) } + assert_equal "\uFFFD", JSON.parse("\"\x01\"", unicode_subset: :assignables, + allow_control_characters: true, on_invalid_char: :replace) + assert_equal "é", JSON.parse('"\é"', unicode_subset: :assignables, allow_invalid_escape: true) + assert_equal "\uFFFD", JSON.parse("\"\\\u0089\"", unicode_subset: :assignables, + allow_invalid_escape: true, on_invalid_char: :replace) + error = assert_raise(JSON::ParserError) do + JSON.parse("[\"\\\u0089\"]", unicode_subset: :assignables, allow_invalid_escape: true) + end + assert_equal 3, error.column + ["\x00", "\x01"].each do |char| + assert_raise(JSON::ParserError) do + JSON.parse("\"\\#{char}\"", unicode_subset: :scalars, allow_invalid_escape: true) + end + assert_equal char, JSON.parse("\"\\#{char}\"", unicode_subset: :scalars, + allow_invalid_escape: true, allow_control_characters: true) + end + assert_equal "\n\r\t\uFFFD\uFFFD", JSON.parse('"\n\r\t\b\f"', + unicode_subset: :assignables, on_invalid_char: :replace) + end + + def test_resumable_closed_incomplete_unicode_escape + omit 'JRuby does not implement ResumableParser' if RUBY_ENGINE == 'jruby' + ['"\u123"', '"\uD800\u123"'].each do |source| + [{ unicode_subset: :scalars }, { on_invalid_char: :replace }].each do |options| + parser = JSON::ResumableParser.new(**options) + parser << source + assert_raise(JSON::ParserError) { parser.parse } + end + end + end + + def test_resumable_chunk_boundaries + omit 'JRuby does not implement ResumableParser' if RUBY_ENGINE == 'jruby' + sources = ['{"a\u0089":["\uD800\u0041","\uD9BF\uDFFF"]}', + "{\"日\u0089\":[\"\xE1\x80z\u{1FFFF}\"]}"] + sources.each do |source| + expected = JSON.parse(source, unicode_subset: :assignables, on_invalid_char: :replace) + (0..source.bytesize).each do |split| + parser = JSON::ResumableParser.new(unicode_subset: :assignables, on_invalid_char: :replace) + parser << source.byteslice(0, split) + complete = parser.parse + parser << source.byteslice(split..-1) + assert parser.parse unless complete + assert_equal expected, parser.value + end + end + end + + def test_resumable_source_positions_after_buffer_compaction + omit 'JRuby does not implement ResumableParser' if RUBY_ENGINE == 'jruby' + ['["日",' + '"x",' * 140 + "\n" + '"日\uFFFF"]', + '["日",' + '"x",' * 140 + '"日\uFFFF"]'].each do |source| + expected = assert_raise(JSON::ParserError) { JSON.parse(source, unicode_subset: :assignables) } + [false, true].each do |frozen_chunks| + (0..source.bytesize).each do |split| + parser = JSON::ResumableParser.new(unicode_subset: :assignables) + error = assert_raise(JSON::ParserError) do + [source.byteslice(0, split), source.byteslice(split..-1)].each do |chunk| + parser << (frozen_chunks ? chunk.freeze : chunk) + parser.parse + end + end + assert_equal [expected.line, expected.column, expected.json_path], + [error.line, error.column, error.json_path], [split, frozen_chunks].inspect + end + end + end + end + + private + + def unicode_escape(cp) + return '\u%04X' % cp if cp <= 0xFFFF + cp -= 0x10000 + '\u%04X\u%04X' % [0xD800 + (cp >> 10), 0xDC00 + (cp & 0x3FF)] + end + + def allowed?(cp, subset) + return true if subset == :scalars + return false if cp < 32 && ![9, 10, 13].include?(cp) + return false if cp == 0xFFFE || cp == 0xFFFF + return true if subset == :xml_characters + !(127..159).cover?(cp) && !(0xFDD0..0xFDEF).cover?(cp) && (cp & 0xFFFF) < 0xFFFE + end +end From 6e9d4baa838fa5e10051178c38a5ae5e9e1d9252 Mon Sep 17 00:00:00 2001 From: ydah Date: Mon, 12 Oct 2026 17:17:39 +0900 Subject: [PATCH 3/3] Support RFC 9839 Unicode repertoires when generating JSON Validate or replace prohibited characters in C, JRuby, and TruffleRuby generation, including State and Coder. Preserve invalid_object, detect replacement key collisions, and reject replacement with RFC 8785. Fuse C validation with escaping, retain IO snapshots across callbacks, and keep normalization off the common path. Preserve default-path speed with compact numeric dispatch and direct output for single-digit integers. Include generator and API integration tests, Ractor coverage, and docs. Keep TruffleRuby's no-argument Object#to_json fallback working. On CRuby 4.0.6 arm64 macOS versus cdcfdb8, all 11 encoder fixtures remain within 1% regression for JSON.generate and Coder. Nine paired rounds put default Twitter generation at -0.31% and assignables overhead at 12.34%. --- CHANGES.md | 2 +- README.md | 6 +- ext/json/ext/generator/generator.c | 337 ++++++++++++++++++++++--- java/src/json/ext/Generator.java | 80 +++++- java/src/json/ext/GeneratorState.java | 50 +++- lib/json.rb | 9 + lib/json/common.rb | 2 + lib/json/ext.rb | 18 +- lib/json/ext/generator/state.rb | 2 + lib/json/truffle_ruby/generator.rb | 105 +++++++- test/json/json_generator_test.rb | 22 ++ test/json/ractor_test.rb | 14 ++ test/json/unicode_generator_test.rb | 348 ++++++++++++++++++++++++++ test/json/unicode_integration_test.rb | 138 ++++++++++ 14 files changed, 1075 insertions(+), 58 deletions(-) create mode 100644 test/json/unicode_generator_test.rb create mode 100644 test/json/unicode_integration_test.rb diff --git a/CHANGES.md b/CHANGES.md index fc342c23b..edaea6557 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ### Unreleased -* Add opt-in `unicode_subset` and `on_invalid_char` options for RFC 9839 Unicode repertoires in parsing, including Coder#load and ResumableParser. +* Add opt-in `unicode_subset` and `on_invalid_char` options for RFC 9839 Unicode repertoires in parsing and generation, including Coder and ResumableParser. * JRuby: parser errors now include the position (`line`, `column` and the message suffix) as well as `json_path`, matching the C extension. ### 2026-09-09 (3.0.2) diff --git a/README.md b/README.md index be258b5ab..cde4b87bd 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,12 @@ one of the repertoires in [RFC 9839](https://www.rfc-editor.org/rfc/rfc9839.html * `:assignables` also excludes DEL, C1 controls, and all noncharacters. Unassigned and private-use codepoints remain allowed. The default, `nil`, keeps -the existing behavior. These options work with parsing, `JSON::Coder#load`, +the existing behavior. These options work with parsing, generation, `JSON::Coder`, and `JSON::ResumableParser` (where available). ```ruby JSON.parse('"\u0089"', unicode_subset: :assignables) # raises JSON::ParserError +JSON.generate("\u0089", unicode_subset: :assignables) # raises JSON::GeneratorError JSON.parse('"\u0089"', unicode_subset: :assignables, on_invalid_char: :replace) # => "�" ``` @@ -73,6 +74,9 @@ Escapes cannot bypass a subset restriction; `allow_control_characters` only relaxes JSON syntax. Keep duplicate-key detection enabled when replacing characters, since different keys can become equal after replacement. +Replacement cannot be combined with `rfc8785: true`. Raw `JSON::Fragment` contents +and custom `to_json` output remain the caller's responsibility. + ## Casting non native types JSON documents can only support Hashes, Arrays, Strings, Integers and Floats. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 38c88f231..84a6df626 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -6,6 +6,7 @@ #include #include "../simd/simd.h" +#include "../unicode.h" /* ruby api and some helpers */ @@ -28,6 +29,8 @@ typedef struct JSON_Generator_StateStruct { bool script_safe; bool strict; bool rfc8785; + uint8_t forbidden_classes; + bool replace_invalid_chars; VALUE sort_keys; } JSON_Generator_State; @@ -36,7 +39,8 @@ static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_ static ID i_to_s, i_to_json, i_new, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, - sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys, sym_rfc8785; + sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys, sym_rfc8785, + sym_unicode_subset, sym_on_invalid_char; #define GET_STATE_TO(self, state) \ @@ -812,8 +816,8 @@ convert_string_subclass(VALUE key) static bool enc_utf8_compatible_p(int enc_idx) { - if (enc_idx == usascii_encindex) return true; if (enc_idx == utf8_encindex) return true; + if (enc_idx == usascii_encindex) return true; return false; } @@ -851,12 +855,59 @@ static inline bool valid_json_string_p(VALUE str) return false; } +NOINLINE(static) VALUE replace_invalid_unicode(struct generate_json_data *data, VALUE str, VALUE original, + rb_len_t offset, uint32_t cp, size_t length) +{ + JSON_Generator_State *state = data->state; + if (!state->replace_invalid_chars) { + const char *subset = json_unicode_subset_name(state->forbidden_classes); + if (cp == UINT32_MAX) { + raise_generator_error(original, "invalid UTF-8 sequence is not allowed by unicode_subset: :%s", subset); + } + raise_generator_error(original, "U+%04X is not allowed by unicode_subset: :%s", cp, subset); + } + + VALUE result = rb_utf8_str_new(RSTRING_PTR(str), offset); + const char *invalid; + do { + rb_str_cat(result, "\xEF\xBF\xBD", 3); + offset += length; + const char *start = RSTRING_PTR(str) + offset; + const char *end = RSTRING_PTR(str) + RSTRING_LEN(str); + invalid = json_unicode_scan(start, end, state->forbidden_classes, &cp, &length); + rb_len_t valid_length = (invalid ? invalid : end) - start; + rb_str_cat(result, start, valid_length); + offset += valid_length; + } while (invalid); + RB_GC_GUARD(str); + return result; +} + +NOINLINE(static) VALUE enforce_unicode_subset(struct generate_json_data *data, VALUE str, VALUE original) +{ + const char *start = RSTRING_PTR(str), *end = start + RSTRING_LEN(str); + uint32_t cp; + size_t length; + const char *invalid = json_unicode_scan(start, end, data->state->forbidden_classes, &cp, &length); + if (RB_LIKELY(!invalid)) return str; + return replace_invalid_unicode(data, str, original, invalid - start, cp, length); +} + +static bool unicode_bytes_p(VALUE str) +{ + int encoding = RB_ENCODING_GET_INLINED(str); + return enc_utf8_compatible_p(encoding) || encoding == rb_ascii8bit_encindex(); +} + NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) { if (!as_json_called && data->state->strict && RTEST(data->state->as_json)) { VALUE coerced_str = json_call_as_json(data->state, str, Qfalse); if (coerced_str != str) { if (RB_TYPE_P(coerced_str, T_STRING)) { + if (data->state->forbidden_classes && unicode_bytes_p(coerced_str)) { + coerced_str = enforce_unicode_subset(data, coerced_str, str); + } if (!valid_json_string_p(coerced_str)) { raise_generator_error(str, "source sequence is illegal/malformed utf-8"); } @@ -871,6 +922,9 @@ NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data, } } + if (data->state->forbidden_classes && unicode_bytes_p(str)) { + str = enforce_unicode_subset(data, str, str); + } str = rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); Check_Type(str, T_STRING); if (!valid_json_string_p(str)) { @@ -881,16 +935,139 @@ NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data, ALWAYS_INLINE(static) VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) { - if (RB_LIKELY(valid_json_string_p(str))) { + if (RB_UNLIKELY(!valid_json_string_p(str))) { + VALUE original = str; + str = convert_invalid_encoding(data, str, as_json_called, is_key); + if (data->state->forbidden_classes && RB_TYPE_P(str, T_STRING)) { + str = enforce_unicode_subset(data, str, original); + } return str; } - else { - return convert_invalid_encoding(data, str, as_json_called, is_key); + if (RB_UNLIKELY(data->state->forbidden_classes) && + (data->state->replace_invalid_chars || data->state->ascii_only || data->state->script_safe)) { + str = enforce_unicode_subset(data, str, str); } + return str; +} + +static const unsigned char unicode_escape_table[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ['"'] = 1, ['\\'] = 1, [0x7F] = 1, [0xC2] = 1, + [0xEF] = 1, [0xF0] = 1, [0xF1] = 1, [0xF2] = 1, [0xF3] = 1, [0xF4] = 1, +}; + +#ifdef HAVE_SIMD_NEON +ALWAYS_INLINE(static) uint64_t unicode_chunk_mask(const char *ptr) +{ + uint8x16_t bytes = vld1q_u8((const unsigned char *)ptr); + uint8x16_t matches = vorrq_u8(vcltq_u8(veorq_u8(bytes, vdupq_n_u8(2)), vdupq_n_u8(33)), + vceqq_u8(bytes, vdupq_n_u8('\\'))); + matches = vorrq_u8(matches, vceqq_u8(bytes, vdupq_n_u8(0x7F))); + matches = vorrq_u8(matches, vceqq_u8(bytes, vdupq_n_u8(0xC2))); + matches = vorrq_u8(matches, vcgeq_u8(bytes, vdupq_n_u8(0xEF))); + return neon_match_mask(matches); +} +#elif defined(HAVE_SIMD_SSE2) +ALWAYS_INLINE(static) TARGET_SSE2 int unicode_chunk_mask(const char *ptr) +{ + __m128i bytes = _mm_loadu_si128((const __m128i *)ptr); + __m128i matches = _mm_or_si128(_mm_cmplt_epu8(_mm_xor_si128(bytes, _mm_set1_epi8(2)), _mm_set1_epi8(33)), + _mm_cmpeq_epi8(bytes, _mm_set1_epi8('\\'))); + matches = _mm_or_si128(matches, _mm_cmpeq_epi8(bytes, _mm_set1_epi8(0x7F))); + matches = _mm_or_si128(matches, _mm_cmpeq_epi8(bytes, _mm_set1_epi8(0xC2))); + matches = _mm_or_si128(matches, _mm_cmpge_epu8(bytes, _mm_set1_epi8(0xEF))); + return _mm_movemask_epi8(matches); +} +#endif + +/* The coderange has already established UTF-8 validity; inspect only possible forbidden characters. */ +#ifdef HAVE_SIMD_SSE2 +static TARGET_SSE2 +#else +static +#endif +void convert_UTF8_to_JSON_unicode(search_state *search, uint8_t mask, VALUE original) +{ + while (search->ptr < search->end) { +#ifdef HAVE_SIMD_NEON + while ((size_t)(search->end - search->ptr) >= sizeof(uint8x16_t)) { + uint64_t candidates = unicode_chunk_mask(search->ptr); + if (candidates) { + search->ptr += trailing_zeros64(candidates) >> 2; + break; + } + search->ptr += sizeof(uint8x16_t); + } +#elif defined(HAVE_SIMD_SSE2) + while ((size_t)(search->end - search->ptr) >= sizeof(__m128i)) { + int candidates = unicode_chunk_mask(search->ptr); + if (candidates) { + search->ptr += trailing_zeros(candidates); + break; + } + search->ptr += sizeof(__m128i); + } +#endif +#ifdef HAVE_SIMD + size_t remaining = search->end - search->ptr; + if (remaining >= SIMD_MINIMUM_THRESHOLD && remaining < 16) { + char *bytes = copy_remaining_bytes(search, 16, remaining); +#ifdef HAVE_SIMD_NEON + uint64_t candidates = unicode_chunk_mask(bytes); +#else + int candidates = unicode_chunk_mask(bytes); +#endif + if (!candidates) { + fbuffer_consumed(search->buffer, remaining); + search->ptr = search->cursor = search->end; + return; + } +#ifdef HAVE_SIMD_NEON + search->ptr += trailing_zeros64(candidates) >> 2; +#else + search->ptr += trailing_zeros(candidates); +#endif + } +#endif + while (search->ptr < search->end && !unicode_escape_table[(unsigned char)*search->ptr]) { + search->ptr++; + } + if (search->ptr == search->end) break; + + uint32_t cp; + size_t length = json_unicode_decode(search->ptr, search->end, &cp); + if (json_unicode_classify(cp) & mask) { + raise_generator_error(original, "U+%04X is not allowed by unicode_subset: :%s", cp, + json_unicode_subset_name(mask)); + } + if (cp < 0x80 && escape_table_basic[cp]) { + search_flush(search); + escape_UTF8_char_basic(search); + } else { + search->ptr += length; + } + } + search_flush(search); +} + +NOINLINE(static) void raw_generate_json_string_unicode(FBuffer *buffer, uint8_t mask, VALUE obj) +{ + JSON_ASSERT(valid_json_string_p(obj)); + VALUE str = buffer->io ? rb_str_new_frozen(obj) : obj; + fbuffer_append_char(buffer, '"'); + search_state search; + search.buffer = buffer; + search.ptr = search.cursor = RSTRING_PTR(str); + search.end = search.ptr + RSTRING_LEN(str); + convert_UTF8_to_JSON_unicode(&search, mask, obj); + fbuffer_append_char(buffer, '"'); + RB_GC_GUARD(str); } static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { + JSON_ASSERT(valid_json_string_p(obj)); VALUE str = obj; if (RB_UNLIKELY(buffer->io)) { // IO writes can mutate the original string while we are reading it. @@ -912,33 +1089,40 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data search.chunk_end = NULL; #endif /* HAVE_SIMD */ - switch (json_str_coderange(str)) { - case ENC_CODERANGE_7BIT: - case ENC_CODERANGE_VALID: - if (RB_UNLIKELY(data->state->ascii_only)) { - convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table); - } else if (RB_UNLIKELY(data->state->script_safe)) { - convert_UTF8_to_script_safe_JSON(&search); - } else { - convert_UTF8_to_JSON(&search); - } - break; - default: - raise_generator_error(obj, "source sequence is illegal/malformed utf-8"); - break; + if (RB_UNLIKELY(data->state->ascii_only)) { + convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table); + } else if (RB_UNLIKELY(data->state->script_safe)) { + convert_UTF8_to_script_safe_JSON(&search); + } else { + convert_UTF8_to_JSON(&search); } fbuffer_append_char(buffer, '"'); RB_GC_GUARD(str); } -static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +ALWAYS_INLINE(static) void raw_generate_json_string_with_subset(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +{ + if (RB_UNLIKELY(data->state->forbidden_classes > JSON_UNICODE_SCALARS) && + !data->state->replace_invalid_chars && !data->state->ascii_only && !data->state->script_safe) { + raw_generate_json_string_unicode(buffer, data->state->forbidden_classes, obj); + } else { + raw_generate_json_string(buffer, data, obj); + } +} + +NOINLINE(static) void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { obj = ensure_valid_encoding(data, obj, false, false); - raw_generate_json_string(buffer, data, obj); + if (RB_TYPE_P(obj, T_STRING)) { + raw_generate_json_string_with_subset(buffer, data, obj); + } else { + generate_json(buffer, data, obj); + } } struct hash_foreach_arg { VALUE hash; + VALUE seen_keys; struct generate_json_data *data; int first_key_type; bool first; @@ -948,7 +1132,7 @@ struct hash_foreach_arg { NOINLINE(static) void json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) { - if (arg->mixed_keys_encountered) { + if (arg->seen_keys || arg->mixed_keys_encountered) { return; } arg->mixed_keys_encountered = true; @@ -1026,8 +1210,16 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) key_to_s = ensure_valid_encoding(data, key_to_s, as_json_called, true); + if (arg->seen_keys) { + if (RTEST(rb_hash_lookup(arg->seen_keys, key_to_s))) { + raise_generator_error(arg->hash, "detected duplicate key %"PRIsVALUE" in %"PRIsVALUE, + rb_inspect(key_to_s), rb_inspect(arg->hash)); + } + rb_hash_aset(arg->seen_keys, key_to_s, Qtrue); + } + if (RB_LIKELY(RBASIC_CLASS(key_to_s) == rb_cString)) { - raw_generate_json_string(buffer, data, key_to_s); + raw_generate_json_string_with_subset(buffer, data, key_to_s); } else { generate_json(buffer, data, key_to_s); } @@ -1051,8 +1243,9 @@ static inline long increase_depth(struct generate_json_data *data) static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { if (RB_UNLIKELY(data->state->rfc8785)) { - VALUE args[2] = {obj, data->state->strict ? data->state->as_json : Qnil}; - obj = rb_proc_call_with_block(rfc8785_sort_keys_proc, 2, args, Qnil); + VALUE args[3] = {obj, data->state->strict ? data->state->as_json : Qnil, + json_unicode_subset_value(data->state->forbidden_classes)}; + obj = rb_proc_call_with_block(rfc8785_sort_keys_proc, 3, args, Qnil); Check_Type(obj, T_HASH); } else if (RB_UNLIKELY(data->state->sort_keys)) { obj = rb_proc_call_with_block(data->state->sort_keys, 1, &obj, Qnil); @@ -1074,7 +1267,11 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat .data = data, .first = true, }; + if (data->state->forbidden_classes && data->state->replace_invalid_chars && !data->state->allow_duplicate_key) { + arg.seen_keys = rb_hash_new(); + } rb_hash_foreach(obj, json_object_i, (VALUE)&arg); + RB_GC_GUARD(arg.seen_keys); depth = --data->depth; if (RB_UNLIKELY(data->state->object_nl)) { @@ -1158,7 +1355,12 @@ static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *dat generate_json_rfc8785_number(buffer, obj); return; } - fbuffer_append_long(buffer, FIX2LONG(obj)); + long number = FIX2LONG(obj); + if ((unsigned long)number < 10) { + fbuffer_append_char(buffer, (char)('0' + number)); + } else { + fbuffer_append_long(buffer, number); + } } static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj) @@ -1217,6 +1419,14 @@ static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *d fbuffer_append_str(buffer, fragment); } +NOINLINE(static) VALUE normalize_json_string(struct generate_json_data *data, VALUE obj, bool as_json_called) +{ + if (as_json_called && !data->state->forbidden_classes) { + raise_generator_error(obj, "source sequence is illegal/malformed utf-8"); + } + return ensure_valid_encoding(data, obj, as_json_called, false); +} + static inline void generate_json_general(FBuffer *buffer, struct generate_json_data *data, VALUE obj, bool fallback) { bool as_json_called = false; @@ -1262,11 +1472,21 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d generate_string: if (RB_LIKELY(valid_json_string_p(obj))) { - raw_generate_json_string(buffer, data, obj); + if (RB_UNLIKELY(data->state->forbidden_classes)) { + generate_json_string(buffer, data, obj); + } else { + raw_generate_json_string(buffer, data, obj); + } } else if (as_json_called) { - raise_generator_error(obj, "source sequence is illegal/malformed utf-8"); + obj = normalize_json_string(data, obj, true); + raw_generate_json_string_with_subset(buffer, data, obj); } else { - obj = ensure_valid_encoding(data, obj, false, false); + bool unicode_subset = data->state->forbidden_classes != 0; + obj = normalize_json_string(data, obj, false); + if (unicode_subset && RB_TYPE_P(obj, T_STRING)) { + raw_generate_json_string_with_subset(buffer, data, obj); + break; + } as_json_called = true; goto start; } @@ -1307,7 +1527,14 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { - generate_json_general(buffer, data, obj, true); + // Avoid the general dispatch's stack and register overhead for immediate numbers. + if (RB_FIXNUM_P(obj)) { + generate_json_fixnum(buffer, data, obj); + } else if (RB_FLONUM_P(obj)) { + generate_json_float(buffer, data, obj); + } else { + generate_json_general(buffer, data, obj, true); + } } static void generate_json_no_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj) @@ -1326,7 +1553,8 @@ static void validate_rfc8785(JSON_Generator_State *state) state->array_nl ? "array_nl" : state->ascii_only ? "ascii_only" : state->script_safe ? "script_safe" : - state->allow_nan ? "allow_nan" : NULL; + state->allow_nan ? "allow_nan" : + state->replace_invalid_chars ? "on_invalid_char: :replace" : NULL; if (option) { rb_raise(rb_eArgError, "%s cannot be used with rfc8785", option); } @@ -1878,10 +2106,49 @@ static VALUE cState_rfc8785_set(VALUE self, VALUE value) { rb_check_frozen(self); GET_STATE(self); + if (RTEST(value) && state->replace_invalid_chars) { + rb_raise(rb_eArgError, "on_invalid_char: :replace cannot be used with rfc8785"); + } state->rfc8785 = RTEST(value); return state->rfc8785 ? Qtrue : Qfalse; } +/* Returns the Unicode character repertoire, or nil when unrestricted. */ +static VALUE cState_unicode_subset(VALUE self) +{ + GET_STATE(self); + return json_unicode_subset_value(state->forbidden_classes); +} + +/* Sets the Unicode character repertoire used for strings and object keys. */ +static VALUE cState_unicode_subset_set(VALUE self, VALUE value) +{ + rb_check_frozen(self); + GET_STATE(self); + state->forbidden_classes = json_unicode_subset_mask(value); + return value; +} + +/* Returns :raise or :replace for characters outside the selected repertoire. */ +static VALUE cState_on_invalid_char(VALUE self) +{ + GET_STATE(self); + return ID2SYM(rb_intern(state->replace_invalid_chars ? "replace" : "raise")); +} + +/* Sets how characters outside the selected repertoire are handled. */ +static VALUE cState_on_invalid_char_set(VALUE self, VALUE value) +{ + rb_check_frozen(self); + GET_STATE(self); + bool replace = json_unicode_replace(value); + if (replace && state->rfc8785) { + rb_raise(rb_eArgError, "on_invalid_char: :replace cannot be used with rfc8785"); + } + state->replace_invalid_chars = replace; + return value; +} + static VALUE cState_allow_duplicate_key_p(VALUE self) { GET_STATE(self); @@ -1980,6 +2247,8 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_strict) { state->strict = RTEST(val); } else if (key == sym_allow_duplicate_key) { state->allow_duplicate_key = RTEST(val); } else if (key == sym_rfc8785) { state->rfc8785 = RTEST(val); } + else if (key == sym_unicode_subset) { state->forbidden_classes = json_unicode_subset_mask(val); } + else if (key == sym_on_invalid_char) { state->replace_invalid_chars = json_unicode_replace(val); } else if (key == sym_as_json) { VALUE proc = RTEST(val) ? as_json_config(val) : Qfalse; state->as_json_single_arg = proc && rb_proc_arity(proc) == 1; @@ -2132,6 +2401,10 @@ void Init_generator(void) rb_define_method(cState, "sort_keys=", cState_sort_keys_set, 1); rb_define_method(cState, "rfc8785?", cState_rfc8785_p, 0); rb_define_method(cState, "rfc8785=", cState_rfc8785_set, 1); + rb_define_method(cState, "unicode_subset", cState_unicode_subset, 0); + rb_define_method(cState, "unicode_subset=", cState_unicode_subset_set, 1); + rb_define_method(cState, "on_invalid_char", cState_on_invalid_char, 0); + rb_define_method(cState, "on_invalid_char=", cState_on_invalid_char_set, 1); rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); @@ -2162,6 +2435,8 @@ void Init_generator(void) sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); sym_sort_keys = ID2SYM(rb_intern("sort_keys")); sym_rfc8785 = ID2SYM(rb_intern("rfc8785")); + sym_unicode_subset = ID2SYM(rb_intern("unicode_subset")); + sym_on_invalid_char = ID2SYM(rb_intern("on_invalid_char")); usascii_encindex = rb_usascii_encindex(); utf8_encindex = rb_utf8_encindex(); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 3b05812f5..3e9379f5d 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -7,6 +7,7 @@ import org.jcodings.Encoding; import org.jcodings.specific.UTF8Encoding; +import org.jcodings.specific.ASCIIEncoding; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyBasicObject; @@ -590,7 +591,7 @@ static void generateHash(ThreadContext context, Session session, RubyHash object if (state.rfc8785()) { IRubyObject asJSON = state.strict() ? state.as_json_get(context) : context.nil; - object = (RubyHash)Helpers.invoke(context, GeneratorState.rfc8785SortKeysProc, "call", object, asJSON); + object = (RubyHash)Helpers.invoke(context, GeneratorState.rfc8785SortKeysProc, "call", object, asJSON, state.unicode_subset_get(context)); } else if (state.getSortKeysProc() != null) { object = (RubyHash)Helpers.invoke(context, state.getSortKeysProc(), "call", object); } @@ -606,13 +607,17 @@ static void generateHash(ThreadContext context, Session session, RubyHash object boolean firstPair = true; HashKeyTracker tracker = new HashKeyTracker(object); + RubyHash emittedKeys = state.getForbiddenClasses() != 0 && state.replaceInvalidChars() && !state.getAllowDuplicateKey() + ? RubyHash.newHash(context.runtime) : null; for (RubyHash.RubyHashEntry entry : (Set) object.directEntrySet()) { - if (firstPair) { - tracker.trackFirst(context, session, (IRubyObject)entry.getKey()); - } else { - tracker.track(context, session, (IRubyObject)entry.getKey()); + if (emittedKeys == null) { + if (firstPair) { + tracker.trackFirst(context, session, (IRubyObject)entry.getKey()); + } else { + tracker.track(context, session, (IRubyObject)entry.getKey()); + } } - processEntry(context, session, buffer, entry, firstPair, objectNl, indent, spaceBefore, space); + processEntry(context, session, buffer, object, entry, firstPair, objectNl, indent, spaceBefore, space, emittedKeys); firstPair = false; } int oldDepth = state.decreaseDepth(); @@ -640,7 +645,7 @@ private static IRubyObject castKey(ThreadContext context, IRubyObject key) { } } - private static void processEntry(ThreadContext context, Session session, OutputStream buffer, RubyHash.RubyHashEntry entry, boolean firstPair, ByteList objectNl, byte[] indent, ByteList spaceBefore, ByteList space) { + private static void processEntry(ThreadContext context, Session session, OutputStream buffer, RubyHash object, RubyHash.RubyHashEntry entry, boolean firstPair, ByteList objectNl, byte[] indent, ByteList spaceBefore, ByteList space, RubyHash emittedKeys) { StringEncoder encoder = session.getStringEncoder(context); IRubyObject key = (IRubyObject) entry.getKey(); @@ -655,9 +660,11 @@ private static void processEntry(ThreadContext context, Session session, OutputS Ruby runtime = context.runtime; + GeneratorState state = session.getState(context); IRubyObject keyStr = castKey(context, key); - if (keyStr == null || !(keyStr instanceof RubyString) || !encoder.hasValidEncoding((RubyString)keyStr)) { - GeneratorState state = session.getState(context); + RubyString originalKey = keyStr instanceof RubyString ? (RubyString)keyStr : null; + if (keyStr == null || !(keyStr instanceof RubyString) || + !encoder.hasValidEncoding((RubyString)keyStr)) { if (state.strict()) { if (state.getAsJSON() != null) { key = state.getAsJSON().call(context, key, context.getRuntime().getTrue()); @@ -673,8 +680,20 @@ private static void processEntry(ThreadContext context, Session session, OutputS } } + if (state.getForbiddenClasses() != 0) { + Utils.ensureString(keyStr); + keyStr = prepareUnicodeString(context, state, (RubyString)keyStr, originalKey == null ? (RubyString)keyStr : originalKey); + if (emittedKeys != null) { + if (emittedKeys.hasKey(keyStr)) { + throw Utils.buildGeneratorError(context, object, + "detected duplicate key " + keyStr.callMethod(context, "inspect") + " in " + + object.callMethod(context, "inspect")).toThrowable(); + } + emittedKeys.op_aset(context, keyStr, context.tru); + } + } if (keyStr.getMetaClass() == runtime.getString()) { - generateString(context, session, (RubyString) keyStr, buffer); + encoder.generate(context, (RubyString)keyStr, buffer); } else { Utils.ensureString(keyStr); generateFor(context, session, keyStr, buffer); @@ -703,6 +722,7 @@ int guessSize(ThreadContext context, Session session, RubyString object) { void generate(ThreadContext context, Session session, RubyString object, OutputStream buffer) throws IOException { GeneratorState state = session.getState(context); StringEncoder encoder = session.getStringEncoder(context); + RubyString original = object; if (state.strict() && !encoder.hasValidEncoding(object) && state.getAsJSON() != null) { IRubyObject value = state.getAsJSON().call(context, object, context.getRuntime().getFalse()); @@ -714,12 +734,48 @@ void generate(ThreadContext context, Session session, RubyString object, OutputS return; } } - generateString(context, session, object, buffer); + if (state.getForbiddenClasses() != 0) object = prepareUnicodeString(context, state, object, original); + encoder.generate(context, object, buffer); + } + } + + private static RubyString prepareUnicodeString(ThreadContext context, GeneratorState state, RubyString string, RubyString original) { + string = string.getEncoding() == ASCIIEncoding.INSTANCE + ? string : StringEncoder.ensureValidEncoding(context, string); + ByteList bytes = string.getByteList(); + byte[] data = bytes.unsafeBytes(); + int start = bytes.begin(); + int end = start + bytes.realSize(); + int runStart = start; + ByteList replaced = null; + for (int p = start; p < end;) { + long decoded = UnicodeSubset.decode(data, p, end); + int cp = (int)(decoded >> 32); + int length = (int)decoded; + if ((UnicodeSubset.classify(cp) & state.getForbiddenClasses()) != 0) { + if (!state.replaceInvalidChars()) { + throw Utils.buildGeneratorError(context, original, UnicodeSubset.message(cp, state.getForbiddenClasses())).toThrowable(); + } + if (replaced == null) replaced = new ByteList(bytes.realSize()); + replaced.append(data, runStart, p - runStart); + UnicodeSubset.append(replaced, 0xfffd); + runStart = p + length; + } + p += length; } + if (replaced == null) return StringEncoder.ensureValidEncoding(context, string); + replaced.append(data, runStart, end - runStart); + replaced.setEncoding(UTF8Encoding.INSTANCE); + return context.runtime.newString(replaced); } static void generateString(ThreadContext context, Session session, RubyString object, OutputStream buffer) throws IOException { - session.getStringEncoder(context).generate(context, object, buffer); + GeneratorState state = session.getState(context); + if (state.getForbiddenClasses() != 0) { + STRING_HANDLER.generate(context, session, object, buffer); + } else { + session.getStringEncoder(context).generate(context, object, buffer); + } } private static class FragmentHandler extends Handler { diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index a0f3dfc21..fc0705624 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -35,6 +35,8 @@ */ public class GeneratorState extends RubyObject { private boolean allowDuplicateKey = false; + private int forbiddenClasses; + private boolean replaceInvalidChars; private static IRubyObject defaultSortKeyProc; public static IRubyObject rfc8785NumberFormatterProc; @@ -251,6 +253,8 @@ public IRubyObject initialize_copy(ThreadContext context, IRubyObject vOrig) { this.depth = orig.depth; this.allowDuplicateKey = orig.allowDuplicateKey; + this.forbiddenClasses = orig.forbiddenClasses; + this.replaceInvalidChars = orig.replaceInvalidChars; this.sortKeys = orig.sortKeys; return this; @@ -517,6 +521,9 @@ public IRubyObject rfc8785_p(ThreadContext context) { @JRubyMethod(name="rfc8785=") public IRubyObject rfc8785_set(ThreadContext context, IRubyObject rfc8785) { checkFrozen(); + if (rfc8785.isTrue() && replaceInvalidChars) { + throw context.runtime.newArgumentError("on_invalid_char: :replace cannot be used with rfc8785"); + } this.rfc8785 = rfc8785.isTrue(); return rfc8785; } @@ -525,6 +532,42 @@ public boolean rfc8785() { return this.rfc8785; } + public int getForbiddenClasses() { + return forbiddenClasses; + } + + public boolean replaceInvalidChars() { + return replaceInvalidChars; + } + + @JRubyMethod(name="unicode_subset") + public IRubyObject unicode_subset_get(ThreadContext context) { + return forbiddenClasses == 0 ? context.nil : context.runtime.newSymbol(UnicodeSubset.name(forbiddenClasses)); + } + + @JRubyMethod(name="unicode_subset=") + public IRubyObject unicode_subset_set(ThreadContext context, IRubyObject value) { + checkFrozen(); + forbiddenClasses = UnicodeSubset.mask(context, value); + return value; + } + + @JRubyMethod(name="on_invalid_char") + public IRubyObject on_invalid_char_get(ThreadContext context) { + return context.runtime.newSymbol(replaceInvalidChars ? "replace" : "raise"); + } + + @JRubyMethod(name="on_invalid_char=") + public IRubyObject on_invalid_char_set(ThreadContext context, IRubyObject value) { + checkFrozen(); + boolean replace = UnicodeSubset.replace(context, value); + if (replace && rfc8785) { + throw context.runtime.newArgumentError("on_invalid_char: :replace cannot be used with rfc8785"); + } + replaceInvalidChars = replace; + return value; + } + public void validateRfc8785(ThreadContext context) { if (!rfc8785) return; @@ -535,7 +578,8 @@ public void validateRfc8785(ThreadContext context) { !arrayNl.isEmpty() ? "array_nl" : asciiOnly ? "ascii_only" : scriptSafe ? "script_safe" : - allowNaN ? "allow_nan" : null; + allowNaN ? "allow_nan" : + replaceInvalidChars ? "on_invalid_char: :replace" : null; if (option != null) { throw context.runtime.newArgumentError(option + " cannot be used with rfc8785"); } @@ -617,6 +661,8 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { if (opts.hasKey("sort_keys")) sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); rfc8785 = opts.getBool("rfc8785", rfc8785); + if (opts.hasKey("unicode_subset")) forbiddenClasses = UnicodeSubset.mask(context, opts.get("unicode_subset")); + if (opts.hasKey("on_invalid_char")) replaceInvalidChars = UnicodeSubset.replace(context, opts.get("on_invalid_char")); opts.ensureEmpty(); validateRfc8785(context); @@ -658,6 +704,8 @@ public RubyHash to_h(ThreadContext context) { result.op_aset(context, runtime.newSymbol("buffer_initial_length"), buffer_initial_length_get(context)); result.op_aset(context, runtime.newSymbol("sort_keys"), sort_keys_get(context)); result.op_aset(context, runtime.newSymbol("allow_duplicate_key"), allow_duplicate_key_p(context)); + result.op_aset(context, runtime.newSymbol("unicode_subset"), unicode_subset_get(context)); + result.op_aset(context, runtime.newSymbol("on_invalid_char"), on_invalid_char_get(context)); for (String name: getInstanceVariableNameList()) { result.op_aset(context, runtime.newSymbol(name.substring(1)), getInstanceVariables().getInstanceVariable(name)); diff --git a/lib/json.rb b/lib/json.rb index 478a44fd0..423c8da8f 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -450,6 +450,15 @@ # and enabled +ascii_only+, +script_safe+, or +allow_nan+ raise ArgumentError. # JSON.dump defaults +allow_nan+ to +false+ when +rfc8785+ is enabled. # JSON::Fragment contents are inserted as is; the caller must ensure they are canonical. +# - Options +unicode_subset+ and +on_invalid_char+ have the same values as for +# parsing and apply to object keys and string values before escaping. +# Violations raise JSON::GeneratorError with +invalid_object+ set to the +# original string, or produce a replaced copy without changing the input. +# With no subset selected, +:replace+ does not relax generation's encoding checks. +# +on_invalid_char: :replace+ cannot be combined with +rfc8785: true+. +# +ascii_only+ and +script_safe+ do not bypass repertoire validation. +# Replacement can make distinct keys equal; keep duplicate detection enabled. +# Raw JSON::Fragment contents and custom +to_json+ output are not inspected. # # In this example, +obj+ is used first to generate the shortest # \JSON data (no whitespace), then again with all formatting options diff --git a/lib/json/common.rb b/lib/json/common.rb index 892386625..881ff2c17 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -806,6 +806,8 @@ class Coder max_nesting allow_nan allow_duplicate_key + unicode_subset + on_invalid_char )).freeze private_constant :EXCLUDED_GENERATOR_OPTIONS diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 7d7b09b27..48af7e881 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -100,7 +100,7 @@ def shareable_lambda(block) # :nodoc: end }) - generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash, as_json) { + generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash, as_json, unicode_subset = nil) { if as_json converted = {} hash.each do |key, value| @@ -127,7 +127,21 @@ def shareable_lambda(block) # :nodoc: end hash = converted end - hash.sort_by { |k,| k.to_s.encode(Encoding::UTF_16) }.to_h + hash.sort_by do |key,| + string = key.to_s + if unicode_subset + utf8 = string.encoding == Encoding::BINARY ? string.dup.force_encoding(Encoding::UTF_8) : string.encode(Encoding::UTF_8) + if violation = UnicodeSubset.violation(utf8, unicode_subset) + cp = violation.first + character = cp ? format('U+%04X', cp) : 'invalid UTF-8 sequence' + raise GeneratorError.new("#{character} is not allowed by unicode_subset: #{unicode_subset.inspect}", string) + end + end + string.encode(Encoding::UTF_16) + rescue EncodingError => error + raise unless unicode_subset + raise GeneratorError.new(error.message, string) + end.to_h }) JSON.generator = generator diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index 92c8fc480..3e611dee9 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -56,6 +56,8 @@ def to_h buffer_initial_length: buffer_initial_length, sort_keys: sort_keys, rfc8785: rfc8785?, + unicode_subset: unicode_subset, + on_invalid_char: on_invalid_char, } allow_duplicate_key = allow_duplicate_key? diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 1cacd76c5..dae151edf 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -171,6 +171,8 @@ def initialize(opts = nil) @sort_keys = false @allow_duplicate_key = false @rfc8785 = false + @unicode_subset = nil + @on_invalid_char = :raise _configure(**opts) if opts end @@ -222,7 +224,33 @@ def as_json=(value) attr_reader :sort_keys # Controls whether RFC8785 (canonicalization) mode is enabled. Otherwise returns false. - attr_writer :rfc8785 + def rfc8785=(value) + raise FrozenError, "can't modify frozen #{self.class}: #{inspect}" if frozen? + if value && @on_invalid_char == :replace + raise ArgumentError, "on_invalid_char: :replace cannot be used with rfc8785" + end + @rfc8785 = value + end + + # The Unicode character repertoire used for strings and object keys. + attr_reader :unicode_subset, :on_invalid_char + + def unicode_subset=(value) + raise FrozenError, "can't modify frozen #{self.class}: #{inspect}" if frozen? + UnicodeSubset.mask(value) + @unicode_subset = value + end + + def on_invalid_char=(value) + raise FrozenError, "can't modify frozen #{self.class}: #{inspect}" if frozen? + unless :raise.equal?(value) || :replace.equal?(value) + raise ArgumentError, "on_invalid_char must be :raise or :replace" + end + if value == :replace && @rfc8785 + raise ArgumentError, "on_invalid_char: :replace cannot be used with rfc8785" + end + @on_invalid_char = value + end def sort_keys=(value) # :nodoc: type_error = false @@ -318,6 +346,7 @@ def validate_rfc8785 # :nodoc: elsif @ascii_only then :ascii_only elsif @script_safe then :script_safe elsif @allow_nan then :allow_nan + elsif @on_invalid_char == :replace then "on_invalid_char: :replace" end raise ArgumentError, "#{option} cannot be used with rfc8785" if option end @@ -344,7 +373,8 @@ def configure(options) array_nl: @array_nl, allow_nan: @allow_nan, as_json: @as_json, ascii_only: @ascii_only, sort_keys: @sort_keys, depth: @depth, buffer_initial_length: @buffer_initial_length, allow_duplicate_key: @allow_duplicate_key, script_safe: @script_safe, strict: @strict, - max_nesting: @max_nesting, rfc8785: @rfc8785 + max_nesting: @max_nesting, rfc8785: @rfc8785, + unicode_subset: @unicode_subset, on_invalid_char: @on_invalid_char ) if depth.negative? raise ArgumentError, "depth must be >= 0 (got #{depth})" @@ -373,7 +403,12 @@ def configure(options) @script_safe = script_safe @strict = strict @max_nesting = max_nesting || 0 - self.rfc8785 = rfc8785 + self.unicode_subset = unicode_subset + unless :raise.equal?(on_invalid_char) || :replace.equal?(on_invalid_char) + raise ArgumentError, "on_invalid_char must be :raise or :replace" + end + @on_invalid_char = on_invalid_char + @rfc8785 = rfc8785 validate_rfc8785 end @@ -402,11 +437,38 @@ def to_h sort_keys: @sort_keys, allow_duplicate_key: @allow_duplicate_key, rfc8785: @rfc8785, + unicode_subset: @unicode_subset, + on_invalid_char: @on_invalid_char, } end alias to_hash to_h + def prepare_string(string, original = string) # :nodoc: + if [Encoding::UTF_8, Encoding::US_ASCII, Encoding::BINARY].include?(string.encoding) + utf8 = string.encoding == Encoding::UTF_8 ? string : string.dup.force_encoding(Encoding::UTF_8) + string = enforce_unicode_subset(utf8, original) if UnicodeSubset.violation(utf8, @unicode_subset) + string = string.encode(Encoding::UTF_8) unless string.encoding == Encoding::UTF_8 + else + string = enforce_unicode_subset(string.encode(Encoding::UTF_8), original) + end + string + rescue EncodingError => error + raise GeneratorError.new(error.message, original) + end + + private def enforce_unicode_subset(string, original) + violation = UnicodeSubset.violation(string, @unicode_subset) + return string unless violation + if @on_invalid_char == :replace + UnicodeSubset.scrub(string, @unicode_subset) + else + cp, = violation + character = cp ? 'U+%04X' % cp : 'invalid UTF-8 sequence' + raise GeneratorError.new("#{character} is not allowed by unicode_subset: #{@unicode_subset.inspect}", original) + end + end + # Generates a valid JSON document from object +obj+ and # returns the result. If no valid JSON document can be # created this method raises a @@ -444,8 +506,9 @@ def generate(obj, anIO = nil) buf << '{' first = true key_type = nil + seen_keys = {} if @unicode_subset && @on_invalid_char == :replace && !@allow_duplicate_key if @rfc8785 - obj = State.rfc8785_sort_keys_proc.call(obj, @strict && @as_json) + obj = State.rfc8785_sort_keys_proc.call(obj, @strict && @as_json, @unicode_subset) elsif @sort_keys obj = @sort_keys.call(obj) end @@ -453,7 +516,7 @@ def generate(obj, anIO = nil) if first key_type = k.class else - if key_type && !@allow_duplicate_key && key_type != k.class + if key_type && !seen_keys && !@allow_duplicate_key && key_type != k.class key_type = nil # stop checking JSON.send(:on_mixed_keys_hash, obj) end @@ -461,6 +524,13 @@ def generate(obj, anIO = nil) end key_str = k.to_s + key_str = prepare_string(key_str) if @unicode_subset && key_str.is_a?(String) + if seen_keys + if seen_keys.key?(key_str) + raise GeneratorError.new("detected duplicate key #{key_str.inspect} in #{obj.inspect}", obj) + end + seen_keys[key_str] = true + end if key_str.class == String fast_serialize_string(key_str, buf) elsif key_str.is_a?(String) @@ -505,6 +575,7 @@ def generate(obj, anIO = nil) # Assumes !@ascii_only, !@script_safe private def fast_serialize_string(string, buf) # :nodoc: + string = prepare_string(string) if @unicode_subset buf << '"' unless string.encoding == ::Encoding::UTF_8 begin @@ -545,7 +616,7 @@ def to_json(state = nil, *) end end else - to_s.to_json + state&.unicode_subset ? to_s.to_json(state) : to_s.to_json end end end @@ -581,20 +652,22 @@ def json_transform(state) result = "{#{state.object_nl}" first = true key_type = nil + seen_keys = {} if state.unicode_subset && state.on_invalid_char == :replace && !state.allow_duplicate_key? indent = !state.object_nl.empty? hash = self if state.rfc8785? - hash = State.rfc8785_sort_keys_proc.call(hash, state.strict? && state.as_json) + hash = State.rfc8785_sort_keys_proc.call(hash, state.strict? && state.as_json, state.unicode_subset) elsif state.sort_keys hash = state.sort_keys.call(hash) end hash.each { |key, value| + original_key = key if first key_type = key.class else - if key_type && !state.allow_duplicate_key? && key_type != key.class + if key_type && !seen_keys && !state.allow_duplicate_key? && key_type != key.class key_type = nil # stop checking JSON.send(:on_mixed_keys_hash, hash) end @@ -611,13 +684,23 @@ def json_transform(state) raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key) end - unless Generator.valid_encoding?(key) + unless state.unicode_subset || Generator.valid_encoding?(key) raise GeneratorError.new("source sequence is illegal/malformed utf-8", key) end end key_str = key.to_s if key_str.is_a?(String) + if state.unicode_subset + original_string = original_key.is_a?(String) ? original_key : key_str + key_str = state.prepare_string(key_str, original_string) + end + if seen_keys + if seen_keys.key?(key_str) + raise GeneratorError.new("detected duplicate key #{key_str.inspect} in #{hash.inspect}", hash) + end + seen_keys[key_str] = true + end key_json = key_str.to_json(state) else raise TypeError, "#{key.class}#to_s returns an instance of #{key_str.class}, expected a String" @@ -790,7 +873,9 @@ def to_json(state = nil, *args) end end - if string.encoding == ::Encoding::UTF_8 + if state.unicode_subset + string = state.prepare_string(string, self) + elsif string.encoding == ::Encoding::UTF_8 unless string.valid_encoding? raise GeneratorError.new("source sequence is illegal/malformed utf-8", self) end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 7cf76c958..60d0a300c 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -56,6 +56,17 @@ def test_dump_unenclosed_hash assert_equal '{"a":1,"b":2}', dump(a: 1, b: 2) end + def test_generate_small_integers + values = [-2**63, -2**62, -10, -1, *(0..10), 2**62 - 1, 2**63] * 20 + expected = "[#{values.join(',')}]" + assert_equal expected, JSON.generate(values) + assert_equal expected, JSON::Coder.new.dump(values) + + io = StringIO.new + JSON.dump(values, io, buffer_initial_length: 1) + assert_equal expected, io.string + end + def test_dump_strict assert_equal '{}', dump({}, strict: true) @@ -367,6 +378,8 @@ def test_state_defaults space_before: "", sort_keys: false, rfc8785: false, + unicode_subset: nil, + on_invalid_char: :raise, }.sort_by { |n,| n.to_s }.to_h, state.to_h.sort_by { |n,| n.to_s }.to_h) state = JSON::State.new(allow_duplicate_key: true) @@ -387,6 +400,8 @@ def test_state_defaults space_before: "", sort_keys: false, rfc8785: false, + unicode_subset: nil, + on_invalid_char: :raise, }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) end @@ -964,6 +979,13 @@ def test_custom_object_as_keys assert_match "FakeString", error.message end + def test_object_to_json_without_state + object = Object.new + expected = object.to_s.to_json + assert_equal expected, object.to_json + assert_equal expected, object.to_json(nil) + end + def test_to_json_called_with_state_object object = Object.new called = false diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index 9e28f13c1..f3d88db72 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -102,6 +102,20 @@ def test_coder assert_predicate status, :success? end + def test_unicode_subset_coder + coder = JSON::Coder.new(unicode_subset: :assignables, on_invalid_char: :replace).freeze + assert Ractor.shareable?(coder) + pid = fork do + Warning[:experimental] = false + result = Ractor.new(coder) do |shared_coder| + [shared_coder.load('"\u0089"'), shared_coder.dump("\u0000")] + end.value + exit(result == ["\uFFFD", "\"\uFFFD\""] ? 0 : 1) + end + _, status = Process.waitpid2(pid) + assert_predicate status, :success? + end + class NonNative def initialize(value) @value = value diff --git a/test/json/unicode_generator_test.rb b/test/json/unicode_generator_test.rb new file mode 100644 index 000000000..21cf0d777 --- /dev/null +++ b/test/json/unicode_generator_test.rb @@ -0,0 +1,348 @@ +# frozen_string_literal: true + +require_relative 'test_helper' + +class JSONUnicodeGeneratorTest < Test::Unit::TestCase + SUBSETS = [:scalars, :xml_characters, :assignables].freeze + BOUNDARIES = [ + 0, 8, 9, 10, 11, 12, 13, 14, 31, 32, 126, 127, 128, 159, 160, + 0xD7FF, 0xE000, 0xFDCF, 0xFDD0, 0xFDEF, 0xFDF0, 0xFFFD, + 0xFFFE, 0xFFFF, 0x10000, 0x1FFFD, 0x1FFFE, 0x10FFFD, 0x10FFFE, 0x10FFFF, + ].freeze + + def test_options + state = JSON::State.new + assert_nil state.unicode_subset + assert_equal :raise, state.on_invalid_char + assert_nil state.to_h[:unicode_subset] + assert_equal :raise, state.to_h[:on_invalid_char] + + state.configure(unicode_subset: :assignables, on_invalid_char: :replace) + assert_equal :assignables, state.unicode_subset + assert_equal :replace, state.on_invalid_char + assert_equal state.to_h, JSON::State.new(state.to_h).to_h + assert_equal state.to_h, state.dup.to_h + assert_equal '"�"', state.freeze.generate("\u0089") + assert_raise(FrozenError) { state.unicode_subset = nil } + assert_raise(FrozenError) { state.on_invalid_char = :raise } + end + + def test_invalid_option_values + impostor = Object.new + def impostor.equal?(_other) + true + end + [false, true, 0, 'scalars', :scalar, Object.new].each do |value| + assert_raise(ArgumentError) { JSON.generate(nil, unicode_subset: value) } + assert_raise(ArgumentError) { JSON::State.new.unicode_subset = value } + end + [nil, false, true, 0, 'replace', :ignore, Object.new, impostor].each do |value| + assert_raise(ArgumentError) { JSON.generate(nil, on_invalid_char: value) } + assert_raise(ArgumentError) { JSON::State.new.on_invalid_char = value } + end + end + + def test_boundaries + SUBSETS.each do |subset| + BOUNDARIES.each do |cp| + string = cp.chr(Encoding::UTF_8).freeze + allowed = case subset + when :scalars + true + when :xml_characters + cp == 9 || cp == 10 || cp == 13 || (cp >= 32 && cp != 0xFFFE && cp != 0xFFFF) + when :assignables + cp == 9 || cp == 10 || cp == 13 || + (cp >= 32 && !(127..159).cover?(cp) && !(0xFDD0..0xFDEF).cover?(cp) && (cp & 0xFFFF) < 0xFFFE) + end + [{}, {ascii_only: true}, {script_safe: true}, {max_nesting: 0}].each do |format| + options = format.merge(unicode_subset: subset) + if allowed + assert_equal string, JSON.parse(JSON.generate(string, **options)) + assert_equal({string => string}, JSON.parse(JSON.generate({string => string}, **options))) + else + error = assert_raise(JSON::GeneratorError) { JSON.generate(string, **options) } + assert_same string, error.invalid_object + assert_equal 'U+%04X is not allowed by unicode_subset: :%s' % [cp, subset], error.message + error = assert_raise(JSON::GeneratorError) { JSON.generate({string => 1}, **options) } + assert_same string, error.invalid_object + end + replacement = allowed ? string : "\uFFFD" + options[:on_invalid_char] = :replace + assert_equal replacement, JSON.parse(JSON.generate(string, **options)) + assert_equal({replacement => replacement}, JSON.parse(JSON.generate({string => string}, **options))) + end + end + end + end + + def test_malformed_utf8_maximal_subparts + cases = { + [0xFF] => "�", + [0x80] => "�", + [0xC0, 0xAF] => "��", + [0xE1, 0x80] => "�", + [0xE1, 0x80, 0x41] => "�A", + [0xED, 0xA0, 0x80] => "���", + [0xF0, 0x90, 0x80] => "�", + [0xF4, 0x90, 0x80, 0x80] => "����", + [0xF5, 0x80, 0x80, 0x80] => "����", + } + cases.each do |bytes, replacement| + [Encoding::UTF_8, Encoding::BINARY].each do |encoding| + original = bytes.pack('C*').force_encoding(encoding).freeze + SUBSETS.each do |subset| + error = assert_raise(JSON::GeneratorError) { JSON.generate(original, unicode_subset: subset) } + assert_same original, error.invalid_object + assert_equal "invalid UTF-8 sequence is not allowed by unicode_subset: :#{subset}", error.message + options = {unicode_subset: subset, on_invalid_char: :replace} + assert_equal replacement, JSON.parse(JSON.generate(original, **options)) + assert_equal({replacement => 1}, JSON.parse(JSON.generate({original => 1}, **options))) + assert_equal replacement, JSON.parse(original.to_json(options)) + assert_equal replacement, JSON::Coder.new(**options).load(JSON::Coder.new(**options).dump(original)) + assert_equal bytes, original.bytes + assert_equal encoding, original.encoding + end + end + end + end + + def test_forbidden_characters_at_scan_boundaries + [0, 0x7F, 0x80, 0x9F, 0xFDD0, 0xFDEF, 0xFFFE, 0x1FFFE, 0x10FFFF].each do |cp| + 32.times do |offset| + string = (("a" * offset) + cp.chr(Encoding::UTF_8) + "日本語\n" + ("z" * 32)).freeze + error = assert_raise(JSON::GeneratorError) { JSON.generate(string, unicode_subset: :assignables) } + assert_same string, error.invalid_object + assert_equal 'U+%04X is not allowed by unicode_subset: :assignables' % cp, error.message + end + end + end + + def test_io_callback_cannot_disable_validation_of_current_string + string = (("a" * 64) + "\n" + ("a" * 64) + "\u0089").freeze + [string, [nil, nil, nil, string]].each do |object| + state = JSON::State.new(unicode_subset: :assignables, buffer_initial_length: 16) + io = StringIO.new + io.define_singleton_method(:write) do |part| + state.unicode_subset = nil + super(part) + end + error = assert_raise(JSON::GeneratorError) { state.generate(object, io) } + assert_same string, error.invalid_object + assert_equal 'U+0089 is not allowed by unicode_subset: :assignables', error.message + end + end + + def test_replacement_across_generation_entry_points + options = {unicode_subset: :assignables, on_invalid_char: :replace} + string = "a\u0000\u0089\u{7FFFF}z" + object = {string => [string]} + expected = '{"a���z":["a���z"]}' + assert_equal expected, JSON.generate(object, **options) + assert_equal expected, JSON.dump(object, **options) + assert_equal expected, object.to_json(options) + assert_equal expected, JSON::Coder.new(**options).dump(object) + assert_equal expected, JSON::State.new(options).generate(object) + assert_equal expected, JSON.generate(object, **options, max_nesting: 0) + io = StringIO.new + assert_same io, JSON.dump(object, io, **options) + assert_equal expected, io.string + io = StringIO.new + assert_same io, JSON::Coder.new(**options).dump(object, io) + assert_equal expected, io.string + assert_equal '"�"', JSON.generate(:"\u0089", **options, strict: true) + assert_equal '"�"', JSON.generate(:"\u0089", **options) + object = Object.new + def object.to_s + "\u0089" + end + assert_equal '"�"', JSON.generate(object, **options) + assert_equal '"�"', object.to_json(options) + end + + def test_transcoded_strings + [Encoding::UTF_16LE, Encoding::UTF_16BE, Encoding::ISO_8859_1].each do |encoding| + string = "é\u0089".encode(encoding).freeze + options = {unicode_subset: :assignables} + error = assert_raise(JSON::GeneratorError) { JSON.generate(string, **options) } + assert_same string, error.invalid_object + assert_equal 'U+0089 is not allowed by unicode_subset: :assignables', error.message + assert_equal '"é�"', JSON.generate(string, **options, on_invalid_char: :replace) + assert_equal '{"é�":1}', JSON.generate({string => 1}, **options, on_invalid_char: :replace) + end + assert_raise(JSON::GeneratorError) { JSON.generate("é".b, unicode_subset: :scalars, on_invalid_char: :replace) } + end + + def test_encoding_callback_precedes_replacement + string = "\xFF".dup.force_encoding(Encoding::UTF_8) + calls = [] + coder = JSON::Coder.new(unicode_subset: :assignables, on_invalid_char: :replace) do |object| + calls << object + "from callback\u0089" + end + assert_equal '"from callback�"', coder.dump(string) + assert_equal [string], calls + + calls.clear + assert_equal '["from callback�"]', coder.dump([string]) + assert_equal [string], calls + calls.clear + assert_equal '{"value":"from callback�"}', coder.dump({'value' => string}) + assert_equal [string], calls + + calls.clear + assert_equal '{"from callback�":1}', coder.dump({string => 1}) + assert_equal [string], calls + + coder = JSON::Coder.new(unicode_subset: :assignables, on_invalid_char: :replace) { |object| object } + assert_equal '"�"', coder.dump(string) + coder = JSON::Coder.new(unicode_subset: :assignables, on_invalid_char: :replace) { 42 } + assert_equal '42', coder.dump(string) + + string.freeze + coder = JSON::Coder.new(unicode_subset: :assignables) { "\u0089" } + error = assert_raise(JSON::GeneratorError) { coder.dump(string) } + assert_same string, error.invalid_object + error = assert_raise(JSON::GeneratorError) { coder.dump({string => 1}) } + assert_same string, error.invalid_object + end + + def test_malformed_string_returned_by_callback + omit 'Callback invocation counts are specific to the C generator' unless RUBY_ENGINE == 'ruby' + + object = Object.new + string = "\xFF".dup.force_encoding(Encoding::UTF_8).freeze + calls = [] + coder = JSON::Coder.new(unicode_subset: :assignables, on_invalid_char: :replace) do |value| + calls << value + string + end + assert_equal '"�"', coder.dump(object) + assert_equal [object], calls + + [nil, :assignables].each do |subset| + calls.clear + coder = JSON::Coder.new(unicode_subset: subset) do |value| + calls << value + string + end + error = assert_raise(JSON::GeneratorError) { coder.dump(object) } + assert_same string, error.invalid_object + assert_equal [object], calls + end + end + + def test_string_subclasses_returned_by_encoding_callback + subclass = Class.new(String) + invalid = "\xFF".dup.force_encoding(Encoding::UTF_8).freeze + + SUBSETS.each do |subset| + calls = [] + coder = JSON::Coder.new(unicode_subset: subset) do |value| + calls << value + subclass.new('ok') + end + assert_equal '["ok"]', coder.dump([invalid]) + assert_equal [invalid], calls + state = JSON::State.new(unicode_subset: subset, strict: true) + state.as_json = proc do |_value| + state.unicode_subset = nil + subclass.new('ok') + end + assert_equal '["ok"]', state.generate([invalid]) + end + end + + def test_replacement_key_collisions + options = {unicode_subset: :assignables, on_invalid_char: :replace} + [ + {"\u0089" => 1, "\u008A" => 2}, + {"\u0089" => 1, "�" => 2}, + {"\u0089" => 1, :"�" => 2}, + {"\xFF".dup.force_encoding(Encoding::UTF_8) => 1, "�" => 2}, + ].each do |object| + [{}, {max_nesting: 0}, {ascii_only: true}].each do |format| + merged = options.merge(format) + error = assert_raise(JSON::GeneratorError) { JSON.generate(object, **merged) } + assert_same object, error.invalid_object + assert_equal "detected duplicate key #{'�'.inspect} in #{object.inspect}", error.message + error = assert_raise(JSON::GeneratorError) { object.to_json(merged) } + assert_same object, error.invalid_object + assert_equal "detected duplicate key #{'�'.inspect} in #{object.inspect}", error.message + output = JSON.generate(object, **merged, allow_duplicate_key: true) + assert_equal '{"�":1,"�":2}', output.gsub('\\ufffd', '�') + end + error = assert_raise(JSON::GeneratorError) { JSON::Coder.new(**options).dump(object) } + assert_same object, error.invalid_object + assert_equal "detected duplicate key #{'�'.inspect} in #{object.inspect}", error.message + end + object = {"\u0089" => 1, "safe" => {"\u0089" => 2}} + assert_equal '{"�":1,"safe":{"�":2}}', JSON.generate(object, **options) + end + + def test_rfc8785_rejects_replacement + [nil, *SUBSETS].each do |subset| + assert_raise(ArgumentError) { JSON.generate(nil, rfc8785: true, unicode_subset: subset, on_invalid_char: :replace) } + assert_raise(ArgumentError) { JSON::State.new(on_invalid_char: :replace, unicode_subset: subset, rfc8785: true) } + end + state = JSON::State.new(rfc8785: true) + assert_raise(ArgumentError) { state.on_invalid_char = :replace } + state = JSON::State.new(on_invalid_char: :replace) + assert_raise(ArgumentError) { state.rfc8785 = true } + state = JSON::State.new + assert_raise(ArgumentError) { state.configure(rfc8785: true, on_invalid_char: :replace) } + assert_raise(ArgumentError) { state.generate(nil) } + assert_equal '"é"', JSON.generate('é', rfc8785: true, unicode_subset: :assignables) + assert_raise(JSON::GeneratorError) { JSON.generate("\u0089", rfc8785: true, unicode_subset: :assignables) } + ["\u0089", "\xFF".dup.force_encoding(Encoding::UTF_8)].each do |key| + key.freeze + error = assert_raise(JSON::GeneratorError) { JSON.generate({key => 1}, rfc8785: true, unicode_subset: :assignables) } + assert_same key, error.invalid_object + assert_match(/not allowed by unicode_subset: :assignables/, error.message) + end + end + + def test_rfc8785_with_converted_keys + key = Object.new + def key.to_s + raise 'must not validate or sort the original key' + end + converted_key = 'a' + as_json = ->(object, is_key) { is_key ? converted_key : object } + options = {rfc8785: true, strict: true, as_json: as_json, unicode_subset: :assignables} + object = [{'b' => 2, key => 1}] + generators = [ + -> { JSON.generate(object, options) }, + -> { object.to_json(options) }, + -> { JSON::Coder.new(**options).dump(object) }, + -> { + io = StringIO.new + JSON::State.new(options).generate(object, io) + io.string + }, + ] + generators.each { |generate| assert_equal '[{"a":1,"b":2}]', generate.call } + + converted_key = "\u0089" + generators.each do |generate| + error = assert_raise(JSON::GeneratorError) { generate.call } + assert_equal converted_key, error.invalid_object + assert_match(/U\+0089 is not allowed by unicode_subset: :assignables/, error.message) + end + end + + def test_no_subset_preserves_generation + string = "\u0000\u0089\u{7FFFF}" + assert_equal JSON.generate(string), JSON.generate(string, unicode_subset: nil, on_invalid_char: :replace) + assert_raise(JSON::GeneratorError) { JSON.generate("\xFF".dup.force_encoding(Encoding::UTF_8), on_invalid_char: :replace) } + assert_equal '"é"', JSON.generate('é', unicode_subset: nil) + if RUBY_ENGINE == 'truffleruby' + object = Object.new + def object.to_s + 'é/' + end + assert_equal '"é/"', JSON.generate(object, ascii_only: true, script_safe: true) + assert_equal '"é/"', object.to_json(ascii_only: true, script_safe: true) + end + end +end diff --git a/test/json/unicode_integration_test.rb b/test/json/unicode_integration_test.rb new file mode 100644 index 000000000..4bcd9cd8d --- /dev/null +++ b/test/json/unicode_integration_test.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require_relative 'test_helper' + +class JSONUnicodeIntegrationTest < Test::Unit::TestCase + SUBSETS = [:scalars, :xml_characters, :assignables].freeze + + def test_coder_applies_options_to_both_directions + SUBSETS.each do |subset| + coder = JSON::Coder.new(unicode_subset: subset, on_invalid_char: :replace) + string = "\u0000\u0089\u{7FFFF}" + expected = JSON::UnicodeSubset.scrub(string, subset) + assert_equal expected, coder.load(coder.dump(string)) + assert_equal expected, coder.load('"\u0000\u0089\uD9BF\uDFFF"') + io = StringIO.new + assert_same io, coder.dump(string, io) + assert_equal expected, JSON.parse(io.string) + end + end + + def test_load_dump_and_direct_to_json + options = { unicode_subset: :assignables, on_invalid_char: :replace } + object = { "a\u0089" => ["\u0000日\uFFFF"] } + expected = { "a\uFFFD" => ["\uFFFD日\uFFFD"] } + assert_equal expected, JSON.load(JSON.generate(object), nil, **options) + assert_equal expected, JSON.parse(JSON.dump(object, **options)) + assert_equal expected, JSON.parse(object.to_json(options)) + assert_equal expected, JSON.parse(JSON.generate(object, **options, max_nesting: false)) + assert_equal expected, JSON.parse(JSON.generate(object, **options, ascii_only: true)) + assert_equal expected, JSON.parse(JSON.generate(object, **options, script_safe: true)) + end + + def test_rfc_example + source = '{"example": "\u0000\u0089\uDEAD\uD9BF\uDFFF"}' + { + scalars: "\u0000\u0089\uFFFD\u{7FFFF}", + xml_characters: "\uFFFD\u0089\uFFFD\u{7FFFF}", + assignables: "\uFFFD" * 4, + }.each do |subset, expected| + assert_raise(JSON::ParserError) { JSON.parse(source, unicode_subset: subset) } + assert_equal({ 'example' => expected }, JSON.parse(source, unicode_subset: subset, on_invalid_char: :replace)) + end + end + + def test_replacement_precedes_symbolization_freezing_and_duplicate_detection + options = { unicode_subset: :assignables, on_invalid_char: :replace } + result = JSON.parse('{"a\u0089":"\uFFFF"}', **options, symbolize_names: true, freeze: true) + assert_equal({ :"a\uFFFD" => "\uFFFD" }, result) + assert_predicate result, :frozen? + assert_predicate result.values.first, :frozen? + assert_raise(JSON::ParserError) { JSON.parse('{"\u0000":1,"\u0089":2}', **options) } + assert_equal({ "\uFFFD" => 2 }, JSON.parse('{"\u0000":1,"\u0089":2}', **options, allow_duplicate_key: true)) + assert_raise(JSON::GeneratorError) { JSON.generate({ "\u0000" => 1, "\u0089" => 2 }, **options) } + assert_equal "{\"\uFFFD\":1,\"\uFFFD\":2}", JSON.generate({ "\u0000" => 1, "\u0089" => 2 }, **options, allow_duplicate_key: true) + end + + def test_nil_subset_replaces_only_unpaired_surrogate_escapes + { + '"\uD800"' => "\uFFFD", + '"\uDC00"' => "\uFFFD", + '"\uD800a"' => "\uFFFDa", + '"\uD800\u0041"' => "\uFFFDA", + '"\uD800\uD800\uDC00"' => "\uFFFD\u{10000}", + '"\u0000\u0089\uFFFF"' => "\u0000\u0089\uFFFF", + }.each do |source, expected| + assert_equal expected, JSON.parse(source, on_invalid_char: :replace) + end + ['"\uD800\uZZZZ"', '"\uD800\u"', '"\uD800\\"'].each do |source| + assert_raise(JSON::ParserError) { JSON.parse(source, on_invalid_char: :replace) } + end + assert_raise(JSON::GeneratorError) { JSON.generate("\xED\xA0\x80", on_invalid_char: :replace) } + end + + def test_error_location_and_path + ['\\u0089', "\u0089"].each do |bad| + error = assert_raise(JSON::ParserError) do + JSON.parse("{\n \"a\": [\"日#{bad}\"]}", unicode_subset: :assignables) + end + assert_equal 2, error.line + assert_equal 11, error.column + assert_equal '$.a[0]', error.json_path + assert_include error.message, 'U+0089' + assert_include error.message, 'unicode_subset: :assignables' + end + end + + def test_all_scalar_codepoints_match_reference + # Includes unassigned and private-use codepoints, which all subsets permit. + string = ((0..0xD7FF).to_a + (0xE000..0x10FFFF).to_a).pack('U*') + source = JSON.generate(string) + SUBSETS.each do |subset| + expected = JSON::UnicodeSubset.scrub(string, subset) + options = { unicode_subset: subset, on_invalid_char: :replace } + assert_equal expected, JSON.parse(source, **options) + assert_equal expected, JSON.parse(JSON.generate(string, **options)) + assert_equal expected, JSON.parse(JSON.generate(expected, unicode_subset: subset)) + end + end + + def test_random_malformed_utf8_matches_reference + random = Random.new(9839) + SUBSETS.each do |subset| + 100.times do + string = Array.new(30) { random.rand(0x20..0xFF) }.pack('C*').force_encoding(Encoding::UTF_8) + source = '"' + string.b.gsub(/["\\]/n) { |char| '\\' + char } + '"' + expected = JSON::UnicodeSubset.scrub(string, subset) + options = { unicode_subset: subset, on_invalid_char: :replace } + assert_equal expected, JSON.parse(source, **options) + assert_equal expected, JSON.parse(JSON.generate(string, **options)) + end + end + end + + def test_canonical_key_validation + ["\u0089", "\xFF", "\xFF".b, "é".b].each do |key| + assert_raise(JSON::GeneratorError) do + JSON.generate({ key => 1 }, rfc8785: true, unicode_subset: :assignables) + end + end + end + + def test_malformed_utf8_after_multibyte_runs + prefix = "\u1000日\uCFFF" * 8 + ["\xE1", "\xEC\x80", "\xE1\x80 ", "\xEC \x80"].each do |invalid| + ["", "日本"].each do |suffix| + string = prefix + invalid + suffix + source = '"' + string + '"' + SUBSETS.each do |subset| + options = { unicode_subset: subset } + assert_raise(JSON::ParserError) { JSON.parse(source, **options) } + assert_raise(JSON::GeneratorError) { JSON.generate(string, **options) } + assert_equal string.scrub, JSON.parse(source, **options, on_invalid_char: :replace) + assert_equal string.scrub, JSON.parse(JSON.generate(string, **options, on_invalid_char: :replace)) + end + end + end + end +end