gh-155742: Add tests on the PyMarshal C API

Add Lib/test/test_capi/test_marshal.py and
Modules/_testcapi/marshal.c files.

Author: vstinner

Lib/test/test_capi/test_marshal.py

@@ -0,0 +1,217 @@
+# Test PyMarshal C API
+
+import marshal
+import struct
+import unittest
+from test.support import import_helper
+from test.support import os_helper
+
+_testcapi = import_helper.import_module('_testcapi')
+
+NULL = None
+Py_MARSHAL_VERSION = _testcapi.Py_MARSHAL_VERSION
+
+def noop_func():
+    pass
+
+SIMPLE_OBJECT = 123
+# Only test a few objects: see test_marshal for more exhaustive tests
+TEST_OBJECTS = (
+    '\u20ac',
+    b'abc',
+    True,
+    45.6,
+    7+8j,
+    SIMPLE_OBJECT,
+    # Check that serializing code object is allowed (allow_code = 1)
+    noop_func.__code__,
+)
+
+# Invalid marshal data
+JUNK_BYTES = b'\xff' * 32
+
+
+def read_file(filename):
+    with open(filename, 'rb') as fp:
+        return fp.read()
+
+
+def write_file(filename, data):
+    with open(filename, 'wb') as fp:
+        fp.write(data)
+
+
+class CAPIUnicodeTest(unittest.TestCase):
+    def check_object(self, obj2, obj):
+        self.assertEqual(obj2, obj)
+        self.assertEqual(type(obj2), type(obj))
+
+    def test_pymarshal_readobjectfromstring(self):
+        # Test PyMarshal_ReadObjectFromString()
+        readobjectfromstring = _testcapi.pymarshal_readobjectfromstring
+        for obj in TEST_OBJECTS:
+            for version in range(Py_MARSHAL_VERSION + 1):
+                with self.subTest(obj=obj, version=version):
+                    data = marshal.dumps(obj, version)
+                    obj2 = readobjectfromstring(data)
+                    self.check_object(obj2, obj)
+
+        data = marshal.dumps(SIMPLE_OBJECT, Py_MARSHAL_VERSION)
+        data = data[:-1]  # truncate
+        with self.assertRaises(EOFError):
+            readobjectfromstring(data)
+
+        with self.assertRaisesRegex(ValueError, 'bad marshal data'):
+            readobjectfromstring(JUNK_BYTES)
+
+    def test_pymarshal_writeobjecttostring(self):
+        # Test PyMarshal_WriteObjectToString()
+        writeobjecttostring = _testcapi.pymarshal_writeobjecttostring
+        for version in range(Py_MARSHAL_VERSION + 1):
+            for obj in TEST_OBJECTS:
+                with self.subTest(obj=obj, version=version):
+                    data = writeobjecttostring(obj, version)
+                    obj2 = marshal.loads(data)
+                    self.check_object(obj2, obj)
+
+            with self.assertRaises(SystemError):
+                writeobjecttostring(NULL, version)
+
+    def test_pymarshal_writeobjecttofile(self):
+        # Test PyMarshal_WriteObjectToFile()
+        writeobjecttofile = _testcapi.pymarshal_writeobjecttofile
+
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        for version in range(Py_MARSHAL_VERSION + 1):
+            for obj in TEST_OBJECTS:
+                with self.subTest(obj=obj, version=version):
+                    writeobjecttofile(obj, filename, version)
+                    data = read_file(filename)
+                    obj2 = marshal.loads(data)
+                    self.check_object(obj2, obj)
+
+            with self.assertRaises(SystemError):
+                writeobjecttofile(NULL, filename, version)
+
+    def test_pymarshal_writelongtofile(self):
+        # Test PyMarshal_WriteLongToFile()
+        writelongtofile = _testcapi.pymarshal_writelongtofile
+
+        def mask32(value):
+            res = value & (2 ** 32  - 1)
+            if res >= 2147483648:
+                return res - 4294967296
+            else:
+                return res
+
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        limit = 2 ** 31
+        for version in range(Py_MARSHAL_VERSION + 1):
+            for value in (
+                _testcapi.LONG_MIN,
+                _testcapi.LONG_MAX,
+                -limit - 2,
+                -limit,
+                -limit + 2,
+                limit - 2,
+                limit,
+                limit + 2,
+                0,
+                123,
+                -123,
+            ):
+                with self.subTest(value=value, version=version):
+                    writelongtofile(value, filename, version)
+                    data = read_file(filename)
+                    self.assertEqual(len(data), 4)
+                    value2 = struct.unpack('<i', data)[0]
+                    self.assertEqual(value2, mask32(value))
+
+    def test_pymarshal_readshortfromfile(self):
+        # Test PyMarshal_ReadShortFromFile()
+        readshortfromfile = _testcapi.pymarshal_readshortfromfile
+
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        for value in (
+            -2**15,
+            2**15-1,
+            0,
+            123,
+            -123,
+        ):
+            with self.subTest(value=value):
+                data = struct.pack('<h', value)
+                write_file(filename, data)
+                value2 = readshortfromfile(filename)
+                self.assertEqual(value2, value)
+
+        write_file(filename, b'\x00')  # less than 2 bytes
+        with self.assertRaises(EOFError):
+            readshortfromfile(filename)
+
+    def test_pymarshal_readlongfromfile(self):
+        # Test PyMarshal_ReadLongFromFile()
+        readlongfromfile = _testcapi.pymarshal_readlongfromfile
+
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        for value in (
+            -2**31,
+            2**31-1,
+            0,
+            123,
+            -123,
+        ):
+            with self.subTest(value=value):
+                data = struct.pack('<i', value)
+                write_file(filename, data)
+                value2 = readlongfromfile(filename)
+                self.assertEqual(value2, value)
+
+        write_file(filename, b'\x00\x01\x02')  # less than 4 bytes
+        with self.assertRaises(EOFError):
+            readlongfromfile(filename)
+
+    def check_read_object(self, read_object_func):
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        version = Py_MARSHAL_VERSION
+        for obj in TEST_OBJECTS:
+            with self.subTest(obj=obj):
+                data = marshal.dumps(obj, version)
+                data += b'abc'  # following data is ignored
+                write_file(filename, data)
+                obj2 = read_object_func(filename)
+                self.check_object(obj2, obj)
+
+        data = marshal.dumps(SIMPLE_OBJECT, version)
+        data = data[:-1]  # truncate
+        write_file(filename, data)
+        with self.assertRaises(EOFError):
+            read_object_func(filename)
+
+        write_file(filename, JUNK_BYTES)
+        with self.assertRaisesRegex(ValueError, 'bad marshal data'):
+            read_object_func(filename)
+
+    def test_pymarshal_readobjetfromfile(self):
+        # Test PyMarshal_ReadObjectFromFile()
+        readobjectfromfile = _testcapi.pymarshal_readobjectfromfile
+        self.check_read_object(readobjectfromfile)
+
+    def test_pymarshal_readlastobjetfromfile(self):
+        # Test PyMarshal_ReadLastObjectFromFile()
+        readlastobjectfromfile = _testcapi.pymarshal_readlastobjectfromfile
+        self.check_read_object(readlastobjectfromfile)
+
+
+if __name__ == "__main__":
+    unittest.main()

Modules/Setup.stdlib.in

@@ -173,7 +173,7 @@
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tokenizer.c _testinternalcapi/tuple.c _testinternalcapi/typecache.c
-@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c
+@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c _testcapi/marshal.c
 @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
 @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c