From 51b736cf9ee2ca964e41a27427493776259b3fa1 Mon Sep 17 00:00:00 2001 From: Lorenzo Martini Date: Mon, 9 Nov 2026 15:21:37 +0000 Subject: [PATCH] [MINOR][PYTHON] Avoid parsing sys.version when detecting PyPy in cloudpickle --- python/pyspark/cloudpickle/cloudpickle.py | 3 +-- python/pyspark/tests/test_serializers.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/python/pyspark/cloudpickle/cloudpickle.py b/python/pyspark/cloudpickle/cloudpickle.py index e600b35f28422..3a7924a2466e1 100644 --- a/python/pyspark/cloudpickle/cloudpickle.py +++ b/python/pyspark/cloudpickle/cloudpickle.py @@ -64,7 +64,6 @@ import opcode import pickle from pickle import _getattribute as _pickle_getattribute -import platform import struct import sys import threading @@ -95,7 +94,7 @@ _DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() _DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() -PYPY = platform.python_implementation() == "PyPy" +PYPY = sys.implementation.name == "pypy" builtin_code_type = None if PYPY: diff --git a/python/pyspark/tests/test_serializers.py b/python/pyspark/tests/test_serializers.py index 10fb77b65b006..dc6d12988a4e0 100644 --- a/python/pyspark/tests/test_serializers.py +++ b/python/pyspark/tests/test_serializers.py @@ -17,7 +17,9 @@ import collections import math import pickle +import subprocess import sys +import textwrap import unittest from pyspark import serializers @@ -45,6 +47,21 @@ class SerializationTestCase(unittest.TestCase): + def test_cloudpickle_with_packaged_python_version(self): + script = textwrap.dedent( + """ + import sys + sys.version = sys.version.replace(" ", " | packaged by example | ", 1) + + from pyspark import cloudpickle + + func = cloudpickle.loads(cloudpickle.dumps(lambda x: x + 1)) + assert func(1) == 2 + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + def test_namedtuple(self): from collections import namedtuple