@@ -716,6 +716,24 @@ def test_lookup_name_normalization(self, encoding, separators, upper):
716716 self .assertEqual ("x" .encode (variant ), "x" .encode (encoding ))
717717
718718
719+ # The expected failures below each generate only inputs that are known to
720+ # fail, so that they start passing as a whole once the bug is fixed.
721+
722+ # Printable ASCII, which every codec below encodes to itself.
723+ printable_ascii = st .characters (min_codepoint = 0x20 , max_codepoint = 0x7e )
724+ ascii_text = st .text (alphabet = printable_ascii , max_size = 32 )
725+ ascii_bytes = st .binary (max_size = 32 ).map (lambda b : bytes (c & 0x7f for c in b ))
726+
727+
728+ def _incomplete_long_sequence (encoding ):
729+ # The first two bytes of a three- or four-byte sequence, where the
730+ # second byte is ASCII: the stateless decoder escapes the first byte
731+ # and decodes the second as itself.
732+ if encoding == "gb18030" :
733+ return st .tuples (st .integers (0x81 , 0xfe ), st .integers (0x30 , 0x39 ))
734+ return st .tuples (st .just (0x8f ), st .integers (0 , 0x7f ))
735+
736+
719737class MultibyteSurrogateescapeBug (unittest .TestCase ):
720738 """An incomplete sequence of three or four bytes at the end of the
721739 input makes the incremental decoder drop whatever follows the first
@@ -724,12 +742,15 @@ class MultibyteSurrogateescapeBug(unittest.TestCase):
724742
725743 @unittest .expectedFailure
726744 @hypothesis .given (
727- encoding = st .sampled_from (sorted (LONG_SEQUENCE_ENCODINGS )),
728- data = data )
729- @hypothesis .example (encoding = "gb18030" , data = b"\x80 " b"6" )
730- @hypothesis .example (encoding = "euc_jp" , data = b"\x8f \x00 " )
731- @hypothesis .example (encoding = "euc_jis_2004" , data = b"\xa8 \x8f l" )
732- def test_trailing_bytes_after_escape (self , encoding , data ):
745+ case = st .sampled_from (sorted (LONG_SEQUENCE_ENCODINGS )).flatmap (
746+ lambda e : st .tuples (st .just (e ), ascii_bytes ,
747+ _incomplete_long_sequence (e ))))
748+ @hypothesis .example (case = ("gb18030" , b"" , (0x80 , ord ("6" ))))
749+ @hypothesis .example (case = ("euc_jp" , b"a" , (0x8f , 0 )))
750+ @hypothesis .example (case = ("euc_jis_2004" , b"" , (0x8f , ord ("l" ))))
751+ def test_trailing_bytes_after_escape (self , case ):
752+ encoding , prefix , tail = case
753+ data = prefix + bytes (tail )
733754 expected = _outcome (lambda : data .decode (encoding , "surrogateescape" ))
734755 decoder = codecs .getincrementaldecoder (encoding )("surrogateescape" )
735756 self .assertEqual (_outcome (lambda : decoder .decode (data , True )),
@@ -741,44 +762,45 @@ class RoundTripBugs(unittest.TestCase):
741762 ENCODE_ONLY_CHARACTERS for how the round-trip tests avoid them."""
742763
743764 def _check_round_trip (self , encoding , text ):
744- try :
745- encoded = text .encode (encoding )
746- except UnicodeEncodeError :
747- return
765+ encoded = text .encode (encoding )
748766 self .assertEqual (encoded .decode (encoding ), text )
749767
750768 # The ISO-2022 encoders pass U+001B through as a bare ESC byte, which
751769 # their own decoders then reject or misread as an escape sequence;
752770 # iso2022_kr does the same with its shift characters U+000E and U+000F.
771+ # The control character goes last, where it never forms a sequence.
753772 @unittest .expectedFailure
754- @hypothesis .given (encoding = st .sampled_from (sorted (ISO2022_ENCODINGS )),
755- text = text )
756- @hypothesis .example (encoding = "iso2022_jp" , text = "\x1b " )
757- @hypothesis .example (encoding = "iso2022_jp_2004" , text = "\x1b \xa0 " )
758- @hypothesis .example (encoding = "iso2022_kr" , text = "a\x1b \uac00 " )
759- @hypothesis .example (encoding = "iso2022_kr" , text = "\x0e " )
760- @hypothesis .example (encoding = "iso2022_kr" , text = "a\x0f \uac00 " )
761- def test_iso2022_escape (self , encoding , text ):
762- self ._check_round_trip (encoding , text )
773+ @hypothesis .given (
774+ case = st .sampled_from (sorted (ISO2022_ENCODINGS )).flatmap (
775+ lambda e : st .tuples (
776+ st .just (e ), ascii_text ,
777+ st .sampled_from (ENCODE_ONLY_CHARACTERS [e ]))))
778+ @hypothesis .example (case = ("iso2022_jp" , "" , "\x1b " ))
779+ @hypothesis .example (case = ("iso2022_jp_2004" , "a" , "\x1b " ))
780+ @hypothesis .example (case = ("iso2022_kr" , "" , "\x0e " ))
781+ @hypothesis .example (case = ("iso2022_kr" , "a" , "\x0f " ))
782+ def test_iso2022_escape (self , case ):
783+ encoding , prefix , control = case
784+ self ._check_round_trip (encoding , prefix + control )
763785
764786 # euc_kr encodes the Hangul filler U+3164 as the two bytes that start
765787 # an eight-byte composed syllable, and will not decode them alone
766788 # (gh-101863, closed as not a bug; recorded here as the asymmetry it is).
767789 @unittest .expectedFailure
768- @hypothesis .given (text = text )
769- @hypothesis .example (text = " \u3164 " )
770- @hypothesis .example (text = "a\u3164 \uac00 " )
771- def test_euc_kr_filler (self , text ):
772- self ._check_round_trip ("euc_kr" , text )
790+ @hypothesis .given (prefix = ascii_text , suffix = ascii_text )
791+ @hypothesis .example (prefix = "" , suffix = " " )
792+ @hypothesis .example (prefix = "a" , suffix = "b " )
793+ def test_euc_kr_filler (self , prefix , suffix ):
794+ self ._check_round_trip ("euc_kr" , prefix + " \u3164 " + suffix )
773795
774796 # iso2022_jp_3 encodes with the JIS X 0213:2000 table but decodes with
775797 # the 2004 one, where the same code point means U+9B1C.
776798 @unittest .expectedFailure
777- @hypothesis .given (text = text )
778- @hypothesis .example (text = " \u9b1d " )
779- @hypothesis .example (text = "a\u9b1d \u3042 " )
780- def test_iso2022_jp_3_mapping (self , text ):
781- self ._check_round_trip ("iso2022_jp_3" , text )
799+ @hypothesis .given (prefix = ascii_text , suffix = ascii_text )
800+ @hypothesis .example (prefix = "" , suffix = " " )
801+ @hypothesis .example (prefix = "a" , suffix = "b " )
802+ def test_iso2022_jp_3_mapping (self , prefix , suffix ):
803+ self ._check_round_trip ("iso2022_jp_3" , prefix + " \u9b1d " + suffix )
782804
783805
784806class StreamReaderTruncationBug (unittest .TestCase ):
@@ -790,7 +812,10 @@ class StreamReaderTruncationBug(unittest.TestCase):
790812 @hypothesis .given (
791813 encoding = st .sampled_from (["utf_8" , "utf_16_le" , "utf_32_be" ,
792814 "utf_16" , "raw_unicode_escape" ]),
793- text = text , errors = st .sampled_from (["strict" , "replace" ]))
815+ # No backslashes: raw_unicode_escape leaves them alone, and an even
816+ # number in front of the chopped escape turns it into literal text.
817+ text = text .map (lambda s : s .replace ("\\ " , "" )),
818+ errors = st .sampled_from (["strict" , "replace" ]))
794819 @hypothesis .example (encoding = "utf_8" , text = "" , errors = "strict" )
795820 @hypothesis .example (encoding = "utf_8" , text = "a" , errors = "replace" )
796821 @hypothesis .example (encoding = "utf_16" , text = "\u3042 " , errors = "strict" )
@@ -809,15 +834,18 @@ class PunycodeIncrementalBug(unittest.TestCase):
809834 passing in gh-64331). Punycode is left out of the chunked tests above
810835 as a whole-string codec."""
811836
837+ # Two non-empty ASCII chunks give two "-" terminated labels where the
838+ # whole string gives one.
812839 @unittest .expectedFailure
813- @hypothesis .given (text = text , sizes = sizes )
814- @hypothesis .example (text = "ab" , sizes = [1 ])
815- @hypothesis .example (text = "b\xfc cher" , sizes = [2 , 2 ])
816- def test_chunked_encode (self , text , sizes ):
840+ @hypothesis .given (
841+ head = st .text (alphabet = printable_ascii , min_size = 1 , max_size = 32 ),
842+ tail = st .text (alphabet = printable_ascii , min_size = 1 , max_size = 32 ))
843+ @hypothesis .example (head = "a" , tail = "b" )
844+ @hypothesis .example (head = "b\xfc " , tail = "cher" )
845+ def test_chunked_encode (self , head , tail ):
817846 encoder = codecs .getincrementalencoder ("punycode" )()
818- parts = [encoder .encode (chunk ) for chunk in _chunks (text , sizes )]
819- parts .append (encoder .encode ("" , final = True ))
820- self .assertEqual (b"" .join (parts ), text .encode ("punycode" ))
847+ chunked = encoder .encode (head ) + encoder .encode (tail , final = True )
848+ self .assertEqual (chunked , (head + tail ).encode ("punycode" ))
821849
822850
823851if __name__ == "__main__" :
0 commit comments