From c55fec1e338b52e37515d95a45fe03bd62be236f Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 29 Sep 2020 23:17:46 +0800 Subject: [PATCH 01/21] Forward-port test_xpickle from Python 2 to 3. --- Lib/test/libregrtest/cmdline.py | 6 +- Lib/test/test_xpickle.py | 258 ++++++++++++++++++ Lib/test/xpickle_worker.py | 89 ++++++ .../2020-09-29-23-14-01.bpo-31391.IZr2P8.rst | 2 + 4 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_xpickle.py create mode 100644 Lib/test/xpickle_worker.py create mode 100644 Misc/NEWS.d/next/Tests/2020-09-29-23-14-01.bpo-31391.IZr2P8.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index a4bac79c8af680..b510100cdeb15b 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -115,6 +115,10 @@ tzdata - Run tests that require timezone data. + xpickle - Test pickle and _pickle against Python 3.6, 3.7, 3.8 + and 3.9 to test backwards compatibility. These tests + may take very long to complete. + To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the option '-uall,-gui'. @@ -138,7 +142,7 @@ # - tzdata: while needed to validate fully test_datetime, it makes # test_datetime too slow (15-20 min on some buildbots) and so is disabled by # default (see bpo-30822). -RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata', 'xpickle') class _ArgParser(argparse.ArgumentParser): diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py new file mode 100644 index 00000000000000..5e97f2db5ed70f --- /dev/null +++ b/Lib/test/test_xpickle.py @@ -0,0 +1,258 @@ +# This test covers backwards compatibility with +# previous version of Python by bouncing pickled objects through Python 3.6 +# and Python 3.9 by running xpickle_worker.py. +import pathlib +import pickle +import subprocess +import sys + + +from test import support +from test import pickletester +from test.test_pickle import PyPicklerTests + +try: + import _pickle + has_c_implementation = True +except ModuleNotFoundError: + has_c_implementation = False + +is_windows = sys.platform.startswith('win') + +# Map python version to a tuple containing the name of a corresponding valid +# Python binary to execute and its arguments. +py_executable_map = {} + +def highest_proto_for_py_version(py_version): + """Finds the highest supported pickle protocol for a given Python version. + Args: + py_version: a 2-tuple of the major, minor version. Eg. Python 3.7 would + be (3, 7) + Returns: + int for the highest supported pickle protocol + """ + major = sys.version_info.major + minor = sys.version_info.minor + # Versions older than py 3 only supported up until protocol 2. + if py_version < (3, 0): + return 2 + elif py_version < (3, 4): + return 3 + elif py_version < (3, 8): + return 4 + elif py_version <= (major, minor): + return 5 + else: + # Safe option. + return 2 + +def have_python_version(py_version): + """Check whether a Python binary exists for the given py_version and has + support. This respects your PATH. + For Windows, it will first try to use the py launcher specified in PEP 397. + Otherwise (and for all other platforms), it will attempt to check for + python.. + + Eg. given a *py_version* of (3, 7), the function will attempt to try + 'py -3.7' (for Windows) first, then 'python3.7', and return + ['py', '-3.7'] (on Windows) or ['python3.7'] on other platforms. + + Args: + py_version: a 2-tuple of the major, minor version. Eg. python 3.7 would + be (3, 7) + Returns: + List/Tuple containing the Python binary name and its required arguments, + or None if no valid binary names found. + """ + python_str = ".".join(map(str, py_version)) + targets = [('py', f'-{python_str}'), (f'python{python_str}',)] + if py_version not in py_executable_map: + for target in targets[0 if is_windows else 1:]: + worker = subprocess.Popen([*target, '-c','import test.support'], + shell=is_windows) + worker.communicate() + if worker.returncode == 0: + py_executable_map[py_version] = target + + return py_executable_map.get(py_version, None) + + + +class AbstractCompatTests(PyPicklerTests): + py_version = None + _OLD_HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL + + def setUp(self): + self.assertTrue(self.py_version) + if not have_python_version(self.py_version): + py_version_str = ".".join(map(str, self.py_version)) + self.skipTest(f'Python {py_version_str} not available') + + # Override the default pickle protocol to match what xpickle worker + # will be running. + highest_protocol = highest_proto_for_py_version(self.py_version) + pickletester.protocols = range(highest_protocol + 1) + pickle.HIGHEST_PROTOCOL = highest_protocol + + def tearDown(self): + # Set the highest protocol back to the default. + pickletester.protocols = range(pickle.HIGHEST_PROTOCOL + 1) + pickle.HIGHEST_PROTOCOL = self._OLD_HIGHEST_PROTOCOL + + @staticmethod + def send_to_worker(python, obj, proto, **kwargs): + """Bounce a pickled object through another version of Python. + This will pickle the object, send it to a child process where it will + be unpickled, then repickled and sent back to the parent process. + Args: + python: list containing the python binary to start and its arguments + obj: object to pickle. + proto: pickle protocol number to use. + kwargs: other keyword arguments to pass into pickle.dumps() + Returns: + The pickled data received from the child process. + """ + target = pathlib.Path(__file__).parent / 'xpickle_worker.py' + data = super().dumps((proto, obj), proto, **kwargs) + worker = subprocess.Popen([*python, target], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # For windows bpo-17023. + shell=is_windows) + stdout, stderr = worker.communicate(data) + if worker.returncode == 0: + return stdout + # if the worker fails, it will write the exception to stdout + try: + exception = pickle.loads(stdout) + except (pickle.UnpicklingError, EOFError): + raise RuntimeError(stderr) + else: + if isinstance(exception, Exception): + # To allow for tests which test for errors. + raise exception + else: + raise RuntimeError(stderr) + + + def dumps(self, arg, proto=0, **kwargs): + # Skip tests that require buffer_callback arguments since + # there isn't a reliable way to marshal/pickle the callback and ensure + # it works in a different Python version. + if 'buffer_callback' in kwargs: + self.skipTest('Test does not support "buffer_callback" argument.') + python = py_executable_map[self.py_version] + return self.send_to_worker(python, arg, proto, **kwargs) + + @staticmethod + def loads(*args, **kwargs): + return super().loads(*args, **kwargs) + + # A scaled-down version of test_bytes from pickletester, to reduce + # the number of calls to self.dumps() and hence reduce the number of + # child python processes forked. This allows the test to complete + # much faster (the one from pickletester takes 3-4 minutes when running + # under text_xpickle). + def test_bytes(self): + for proto in pickletester.protocols: + for s in b'', b'xyz', b'xyz'*100: + p = self.dumps(s, proto) + self.assert_is_copy(s, self.loads(p)) + s = bytes(range(256)) + p = self.dumps(s, proto) + self.assert_is_copy(s, self.loads(p)) + s = bytes([i for i in range(256) for _ in range(2)]) + p = self.dumps(s, proto) + self.assert_is_copy(s, self.loads(p)) + + # These tests are disabled because they require some special setup + # on the worker that's hard to keep in sync. + test_global_ext1 = None + test_global_ext2 = None + test_global_ext4 = None + + # Backwards compatibility was explicitly broken in r67934 to fix a bug. + test_unicode_high_plane = None + + # These tests fail because they require classes from pickletester + # which cannot be properly imported by the xpickle worker. + test_c_methods = None + test_py_methods = None + test_nested_names = None + + test_recursive_dict_key = None + test_recursive_nested_names = None + test_recursive_set = None + + # Attribute lookup problems are expected, disable the test + test_dynamic_class = None + +# Base class for tests using Python 3.7 and earlier +class CompatLowerPython37(AbstractCompatTests): + # Python versions 3.7 and earlier are incompatible with these tests: + + # This version does not support buffers + test_in_band_buffers = None + + +# Base class for tests using Python 3.6 and earlier +class CompatLowerPython36(CompatLowerPython37): + # Python versions 3.6 and earlier are incompatible with these tests: + # This version has changes in framing using protocol 4 + test_framing_large_objects = None + + # These fail for protocol 0 + test_simple_newobj = None + test_complex_newobj = None + test_complex_newobj_ex = None + + +# Test backwards compatibility with Python 3.6. +class PicklePython36Compat(CompatLowerPython36): + py_version = (3, 6) + +# Test backwards compatibility with Python 3.7. +class PicklePython37Compat(CompatLowerPython37): + py_version = (3, 7) + +# Test backwards compatibility with Python 3.8. +class PicklePython38Compat(AbstractCompatTests): + py_version = (3, 8) + +# Test backwards compatibility with Python 3.9. +class PicklePython39Compat(AbstractCompatTests): + py_version = (3, 9) + + +if has_c_implementation: + class CPicklePython36Compat(PicklePython36Compat): + pickler = pickle._Pickler + unpickler = pickle._Unpickler + + class CPicklePython37Compat(PicklePython37Compat): + pickler = pickle._Pickler + unpickler = pickle._Unpickler + + class CPicklePython38Compat(PicklePython38Compat): + pickler = pickle._Pickler + unpickler = pickle._Unpickler + + class CPicklePython39Compat(PicklePython39Compat): + pickler = pickle._Pickler + unpickler = pickle._Unpickler + +def test_main(): + support.requires('xpickle') + tests = [PicklePython36Compat, + PicklePython37Compat, PicklePython38Compat, + PicklePython39Compat] + if has_c_implementation: + tests.extend([CPicklePython36Compat, + CPicklePython37Compat, CPicklePython38Compat, + CPicklePython39Compat]) + support.run_unittest(*tests) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py new file mode 100644 index 00000000000000..f2dfb893cbaedc --- /dev/null +++ b/Lib/test/xpickle_worker.py @@ -0,0 +1,89 @@ +# This script is called by test_xpickle as a subprocess to load and dump +# pickles in a different Python version. +import importlib.util +import os +import pickle +import sys + + +# This allows the xpickle worker to import pickletester.py, which it needs +# since some of the pickle objects hold references to pickletester.py. +# Also provides the test library over the platform's native one since +# pickletester requires some test.support functions (such as os_helper) +# which are not available in versions below Python 3.10. +test_mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__), + '__init__.py')) +spec = importlib.util.spec_from_file_location('test', test_mod_path) +test_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(test_module) +sys.modules['test'] = test_module + + +# To unpickle certain objects, the structure of the class needs to be known. +# These classes are mostly copies from pickletester.py. + +class Nested: + class A: + class B: + class C: + pass +class C: + pass + +class D(C): + pass + +class E(C): + pass + +class H(object): + pass + +class K(object): + pass + +class Subclass(tuple): + class Nested(str): + pass + +class PyMethodsTest: + @staticmethod + def cheese(): + pass + + @classmethod + def wine(cls): + pass + + def biscuits(self): + pass + + class Nested: + "Nested class" + @staticmethod + def ketchup(): + pass + @classmethod + def maple(cls): + pass + def pie(self): + pass + + +class Recursive: + pass + + +in_stream = sys.stdin.buffer +out_stream = sys.stdout.buffer + +try: + message = pickle.load(in_stream) + protocol, obj = message + pickle.dump(obj, out_stream, protocol) +except Exception as e: + # dump the exception to stdout and write to stderr, then exit + pickle.dump(e, out_stream) + sys.stderr.write(repr(e)) + sys.exit(1) + diff --git a/Misc/NEWS.d/next/Tests/2020-09-29-23-14-01.bpo-31391.IZr2P8.rst b/Misc/NEWS.d/next/Tests/2020-09-29-23-14-01.bpo-31391.IZr2P8.rst new file mode 100644 index 00000000000000..60b7fdc8066318 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2020-09-29-23-14-01.bpo-31391.IZr2P8.rst @@ -0,0 +1,2 @@ +Forward-port test_xpickle from Python 2 to Python 3 and add the resource +back to test's command line. From edd799d5628c3aaae100211b4c0c03ba0f0f7b6c Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 29 Sep 2020 23:31:48 +0800 Subject: [PATCH 02/21] remove staticmethod decorators, silence errors --- Lib/test/test_xpickle.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 5e97f2db5ed70f..a29497ea4de41e 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -1,6 +1,7 @@ # This test covers backwards compatibility with # previous version of Python by bouncing pickled objects through Python 3.6 # and Python 3.9 by running xpickle_worker.py. +import os import pathlib import pickle import subprocess @@ -67,12 +68,15 @@ def have_python_version(py_version): python_str = ".".join(map(str, py_version)) targets = [('py', f'-{python_str}'), (f'python{python_str}',)] if py_version not in py_executable_map: - for target in targets[0 if is_windows else 1:]: - worker = subprocess.Popen([*target, '-c','import test.support'], - shell=is_windows) - worker.communicate() - if worker.returncode == 0: - py_executable_map[py_version] = target + with open(os.devnull, 'w') as devnull: + for target in targets[0 if is_windows else 1:]: + worker = subprocess.Popen([*target, '-c','import test.support'], + stdout=devnull, + stderr=devnull, + shell=is_windows) + worker.communicate() + if worker.returncode == 0: + py_executable_map[py_version] = target return py_executable_map.get(py_version, None) @@ -99,8 +103,7 @@ def tearDown(self): pickletester.protocols = range(pickle.HIGHEST_PROTOCOL + 1) pickle.HIGHEST_PROTOCOL = self._OLD_HIGHEST_PROTOCOL - @staticmethod - def send_to_worker(python, obj, proto, **kwargs): + def send_to_worker(self, python, obj, proto, **kwargs): """Bounce a pickled object through another version of Python. This will pickle the object, send it to a child process where it will be unpickled, then repickled and sent back to the parent process. @@ -145,8 +148,7 @@ def dumps(self, arg, proto=0, **kwargs): python = py_executable_map[self.py_version] return self.send_to_worker(python, arg, proto, **kwargs) - @staticmethod - def loads(*args, **kwargs): + def loads(self, *args, **kwargs): return super().loads(*args, **kwargs) # A scaled-down version of test_bytes from pickletester, to reduce From 2dc67fe209d58115725e187f57982269668966cd Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 29 Sep 2020 23:59:29 +0800 Subject: [PATCH 03/21] make patchcheck --- Lib/test/libregrtest/cmdline.py | 4 ++-- Lib/test/test_xpickle.py | 6 +++--- Lib/test/xpickle_worker.py | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index b510100cdeb15b..eb050d1a951e57 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -115,8 +115,8 @@ tzdata - Run tests that require timezone data. - xpickle - Test pickle and _pickle against Python 3.6, 3.7, 3.8 - and 3.9 to test backwards compatibility. These tests + xpickle - Test pickle and _pickle against Python 3.6, 3.7, 3.8 + and 3.9 to test backwards compatibility. These tests may take very long to complete. To enable all resources except one, use '-uall,-'. For diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index a29497ea4de41e..9f8a4b16521383 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -247,14 +247,14 @@ class CPicklePython39Compat(PicklePython39Compat): def test_main(): support.requires('xpickle') tests = [PicklePython36Compat, - PicklePython37Compat, PicklePython38Compat, + PicklePython37Compat, PicklePython38Compat, PicklePython39Compat] if has_c_implementation: tests.extend([CPicklePython36Compat, - CPicklePython37Compat, CPicklePython38Compat, + CPicklePython37Compat, CPicklePython38Compat, CPicklePython39Compat]) support.run_unittest(*tests) if __name__ == '__main__': - test_main() + test_main() diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py index f2dfb893cbaedc..53a7d48b18136c 100644 --- a/Lib/test/xpickle_worker.py +++ b/Lib/test/xpickle_worker.py @@ -86,4 +86,3 @@ class Recursive: pickle.dump(e, out_stream) sys.stderr.write(repr(e)) sys.exit(1) - From c463dec84901f1d6dfca6328c74fc6b6244e799a Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 30 Sep 2020 21:43:44 +0800 Subject: [PATCH 04/21] Implement some suggestions by iritkatriel --- Lib/test/test_xpickle.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 9f8a4b16521383..f11c064a909016 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -87,7 +87,8 @@ class AbstractCompatTests(PyPicklerTests): _OLD_HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL def setUp(self): - self.assertTrue(self.py_version) + self.assertIsNotNone(self.py_version, + msg='Needs a python version tuple') if not have_python_version(self.py_version): py_version_str = ".".join(map(str, self.py_version)) self.skipTest(f'Python {py_version_str} not available') @@ -100,23 +101,21 @@ def setUp(self): def tearDown(self): # Set the highest protocol back to the default. - pickletester.protocols = range(pickle.HIGHEST_PROTOCOL + 1) pickle.HIGHEST_PROTOCOL = self._OLD_HIGHEST_PROTOCOL + pickletester.protocols = range(pickle.HIGHEST_PROTOCOL + 1) - def send_to_worker(self, python, obj, proto, **kwargs): + @staticmethod + def send_to_worker(python, data): """Bounce a pickled object through another version of Python. - This will pickle the object, send it to a child process where it will + This will send data to a child process where it will be unpickled, then repickled and sent back to the parent process. Args: python: list containing the python binary to start and its arguments - obj: object to pickle. - proto: pickle protocol number to use. - kwargs: other keyword arguments to pass into pickle.dumps() + data: bytes object to send to the child process Returns: The pickled data received from the child process. """ target = pathlib.Path(__file__).parent / 'xpickle_worker.py' - data = super().dumps((proto, obj), proto, **kwargs) worker = subprocess.Popen([*python, target], stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -145,8 +144,9 @@ def dumps(self, arg, proto=0, **kwargs): # it works in a different Python version. if 'buffer_callback' in kwargs: self.skipTest('Test does not support "buffer_callback" argument.') + data = super().dumps((proto, arg), proto, **kwargs) python = py_executable_map[self.py_version] - return self.send_to_worker(python, arg, proto, **kwargs) + return self.send_to_worker(python, data) def loads(self, *args, **kwargs): return super().loads(*args, **kwargs) From 7ba3de8625909457a60e968cf1023307c0c9975a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 18:57:57 +0200 Subject: [PATCH 05/21] Fix compatibility. --- Lib/test/pickletester.py | 11 +++ Lib/test/support/__init__.py | 22 +++-- Lib/test/test_xpickle.py | 174 +++++++++++++++++------------------ 3 files changed, 111 insertions(+), 96 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 4e3468bfcde9c3..1d940754fb30f1 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2630,6 +2630,7 @@ def check(code, exc): class AbstractPickleTests: # Subclass must define self.dumps, self.loads. + py_version = sys.version_info # for test_xpickle optimized = False _testdata = AbstractUnpickleTests._testdata @@ -3218,8 +3219,15 @@ def test_builtin_types(self): self.assertIs(self.loads(s), t) def test_builtin_exceptions(self): + new_names = {'EncodingWarning': (3, 10), + 'BaseExceptionGroup': (3, 11), + 'ExceptionGroup': (3, 11), + '_IncompleteInputError': (3, 13), + 'PythonFinalizationError': (3, 13)} for t in builtins.__dict__.values(): if isinstance(t, type) and issubclass(t, BaseException): + if t.__name__ in new_names and self.py_version < new_names[t.__name__]: + continue for proto in protocols: s = self.dumps(t, proto) u = self.loads(s) @@ -3231,8 +3239,11 @@ def test_builtin_exceptions(self): self.assertIs(u, t) def test_builtin_functions(self): + new_names = {'aiter': (3, 10), 'anext': (3, 10)} for t in builtins.__dict__.values(): if isinstance(t, types.BuiltinFunctionType): + if t.__name__ in new_names and self.py_version < new_names[t.__name__]: + continue for proto in protocols: s = self.dumps(t, proto) self.assertIs(self.loads(s), t) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 84fd43fd396914..6181064e447705 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1,9 +1,10 @@ """Supporting definitions for the Python regression tests.""" +from __future__ import annotations # for test_xpickle + if __name__ != 'test.support': raise ImportError('support must be imported from the test package') -import annotationlib import contextlib import functools import inspect @@ -645,7 +646,7 @@ def requires_working_socket(*, module=False): return unittest.skipUnless(has_socket_support, msg) -@functools.cache +@functools.lru_cache() def has_remote_subprocess_debugging(): """Check if we have permissions to debug subprocesses remotely. @@ -2545,7 +2546,7 @@ def requires_venv_with_pip(): return unittest.skipUnless(ctypes, 'venv: pip requires ctypes') -@functools.cache +@functools.lru_cache() def _findwheel(pkgname): """Try to find a wheel with the package specified as pkgname. @@ -2789,7 +2790,10 @@ def exceeds_recursion_limit(): Py_TRACE_REFS = hasattr(sys, 'getobjects') -_JIT_ENABLED = sys._jit.is_enabled() +try: + _JIT_ENABLED = sys._jit.is_enabled() +except AttributeError: + _JIT_ENABLED = False requires_jit_enabled = unittest.skipUnless(_JIT_ENABLED, "requires JIT enabled") requires_jit_disabled = unittest.skipIf(_JIT_ENABLED, "requires JIT disabled") @@ -2996,10 +3000,8 @@ def force_color(color: bool): import _colorize from .os_helper import EnvironmentVarGuard - with ( - swap_attr(_colorize, "can_colorize", lambda *, file=None: color), - EnvironmentVarGuard() as env, - ): + with swap_attr(_colorize, "can_colorize", lambda *, file=None: color), \ + EnvironmentVarGuard() as env: env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS") env.set("FORCE_COLOR" if color else "NO_COLOR", "1") yield @@ -3218,9 +3220,11 @@ def __init__( self.__forward_is_class__ = is_class self.__forward_module__ = module self.__owner__ = owner + import annotationlib + self._ForwardRef = annotationlib.ForwardRef def __eq__(self, other): - if not isinstance(other, (EqualToForwardRef, annotationlib.ForwardRef)): + if not isinstance(other, (EqualToForwardRef, self._ForwardRef)): return NotImplemented return ( self.__forward_arg__ == other.__forward_arg__ diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index f11c064a909016..ea73eb4523d92f 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -1,16 +1,17 @@ # This test covers backwards compatibility with # previous version of Python by bouncing pickled objects through Python 3.6 # and Python 3.9 by running xpickle_worker.py. +import io import os import pathlib import pickle import subprocess import sys +import unittest from test import support from test import pickletester -from test.test_pickle import PyPicklerTests try: import _pickle @@ -70,38 +71,41 @@ def have_python_version(py_version): if py_version not in py_executable_map: with open(os.devnull, 'w') as devnull: for target in targets[0 if is_windows else 1:]: - worker = subprocess.Popen([*target, '-c','import test.support'], - stdout=devnull, - stderr=devnull, - shell=is_windows) - worker.communicate() - if worker.returncode == 0: - py_executable_map[py_version] = target + try: + worker = subprocess.Popen([*target, '-c','import test.support'], + stdout=devnull, + stderr=devnull, + shell=is_windows) + worker.communicate() + if worker.returncode == 0: + py_executable_map[py_version] = target + break + except FileNotFoundError: + pass return py_executable_map.get(py_version, None) - -class AbstractCompatTests(PyPicklerTests): +class AbstractCompatTests(pickletester.AbstractPickleTests): py_version = None _OLD_HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL - def setUp(self): - self.assertIsNotNone(self.py_version, - msg='Needs a python version tuple') - if not have_python_version(self.py_version): - py_version_str = ".".join(map(str, self.py_version)) - self.skipTest(f'Python {py_version_str} not available') - + @classmethod + def setUpClass(cls): + assert cls.py_version is not None, 'Needs a python version tuple' + if not have_python_version(cls.py_version): + py_version_str = ".".join(map(str, cls.py_version)) + raise unittest.SkipTest(f'Python {py_version_str} not available') # Override the default pickle protocol to match what xpickle worker # will be running. - highest_protocol = highest_proto_for_py_version(self.py_version) + highest_protocol = highest_proto_for_py_version(cls.py_version) pickletester.protocols = range(highest_protocol + 1) pickle.HIGHEST_PROTOCOL = highest_protocol - def tearDown(self): + @classmethod + def tearDownClass(cls): # Set the highest protocol back to the default. - pickle.HIGHEST_PROTOCOL = self._OLD_HIGHEST_PROTOCOL + pickle.HIGHEST_PROTOCOL = cls._OLD_HIGHEST_PROTOCOL pickletester.protocols = range(pickle.HIGHEST_PROTOCOL + 1) @staticmethod @@ -131,6 +135,11 @@ def send_to_worker(python, data): except (pickle.UnpicklingError, EOFError): raise RuntimeError(stderr) else: + if support.verbose > 1: + print() + print(f'{data = }') + print(f'{stdout = }') + print(f'{stderr = }') if isinstance(exception, Exception): # To allow for tests which test for errors. raise exception @@ -144,12 +153,18 @@ def dumps(self, arg, proto=0, **kwargs): # it works in a different Python version. if 'buffer_callback' in kwargs: self.skipTest('Test does not support "buffer_callback" argument.') - data = super().dumps((proto, arg), proto, **kwargs) + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump((proto, arg)) + f.seek(0) + data = bytes(f.read()) python = py_executable_map[self.py_version] return self.send_to_worker(python, data) - def loads(self, *args, **kwargs): - return super().loads(*args, **kwargs) + def loads(self, buf, **kwds): + f = io.BytesIO(buf) + u = self.unpickler(f, **kwds) + return u.load() # A scaled-down version of test_bytes from pickletester, to reduce # the number of calls to self.dumps() and hence reduce the number of @@ -174,9 +189,6 @@ def test_bytes(self): test_global_ext2 = None test_global_ext4 = None - # Backwards compatibility was explicitly broken in r67934 to fix a bug. - test_unicode_high_plane = None - # These tests fail because they require classes from pickletester # which cannot be properly imported by the xpickle worker. test_c_methods = None @@ -185,76 +197,64 @@ def test_bytes(self): test_recursive_dict_key = None test_recursive_nested_names = None + test_recursive_nested_names2 = None test_recursive_set = None # Attribute lookup problems are expected, disable the test test_dynamic_class = None + test_evil_class_mutating_dict = None -# Base class for tests using Python 3.7 and earlier -class CompatLowerPython37(AbstractCompatTests): - # Python versions 3.7 and earlier are incompatible with these tests: - - # This version does not support buffers - test_in_band_buffers = None - - -# Base class for tests using Python 3.6 and earlier -class CompatLowerPython36(CompatLowerPython37): - # Python versions 3.6 and earlier are incompatible with these tests: - # This version has changes in framing using protocol 4 - test_framing_large_objects = None + # Expected exception is raised during unpickling in a subprocess. + test_pickle_setstate_None = None - # These fail for protocol 0 - test_simple_newobj = None - test_complex_newobj = None - test_complex_newobj_ex = None - - -# Test backwards compatibility with Python 3.6. -class PicklePython36Compat(CompatLowerPython36): - py_version = (3, 6) - -# Test backwards compatibility with Python 3.7. -class PicklePython37Compat(CompatLowerPython37): - py_version = (3, 7) - -# Test backwards compatibility with Python 3.8. -class PicklePython38Compat(AbstractCompatTests): - py_version = (3, 8) - -# Test backwards compatibility with Python 3.9. -class PicklePython39Compat(AbstractCompatTests): - py_version = (3, 9) +class PyPicklePythonCompat(AbstractCompatTests): + pickler = pickle._Pickler + unpickler = pickle._Unpickler if has_c_implementation: - class CPicklePython36Compat(PicklePython36Compat): - pickler = pickle._Pickler - unpickler = pickle._Unpickler - - class CPicklePython37Compat(PicklePython37Compat): - pickler = pickle._Pickler - unpickler = pickle._Unpickler - - class CPicklePython38Compat(PicklePython38Compat): - pickler = pickle._Pickler - unpickler = pickle._Unpickler - - class CPicklePython39Compat(PicklePython39Compat): - pickler = pickle._Pickler - unpickler = pickle._Unpickler - -def test_main(): - support.requires('xpickle') - tests = [PicklePython36Compat, - PicklePython37Compat, PicklePython38Compat, - PicklePython39Compat] - if has_c_implementation: - tests.extend([CPicklePython36Compat, - CPicklePython37Compat, CPicklePython38Compat, - CPicklePython39Compat]) - support.run_unittest(*tests) + class CPicklePythonCompat(AbstractCompatTests): + pickler = _pickle.Pickler + unpickler = _pickle.Unpickler + + +skip_tests = { + (3, 6): [ + # This version has changes in framing using protocol 4 + 'test_framing_large_objects', + + # These fail for protocol 0 + 'test_simple_newobj', + 'test_complex_newobj', + 'test_complex_newobj_ex', + ], + (3, 7): [ + # This version does not support buffers + 'test_in_band_buffers', + ], +} + + +def make_test(py_version, base): + class_dict = {'py_version': py_version} + for key, value in skip_tests.items(): + if py_version <= key: + for test_name in value: + class_dict[test_name] = None + name = base.__name__.replace('Python', 'Python%d%d' % py_version) + return type(name, (base, unittest.TestCase), class_dict) + +def load_tests(loader, tests, pattern): + major = sys.version_info.major + assert major == 3 + for minor in range(sys.version_info.minor): + test_class = make_test((major, minor), PyPicklePythonCompat) + tests.addTest(loader.loadTestsFromTestCase(test_class)) + if has_c_implementation: + test_class = make_test((major, minor), CPicklePythonCompat) + tests.addTest(loader.loadTestsFromTestCase(test_class)) + return tests if __name__ == '__main__': - test_main() + unittest.main() From 0b0635054c667be370760f6413fdfa0e819e402e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 20:27:18 +0200 Subject: [PATCH 06/21] Fix test_inheritance_pep563. --- Lib/test/pickletester.py | 3 ++- Lib/test/support/__init__.py | 8 +++----- Lib/test/test_xpickle.py | 3 +++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 1d940754fb30f1..7978977ed75213 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3239,7 +3239,7 @@ def test_builtin_exceptions(self): self.assertIs(u, t) def test_builtin_functions(self): - new_names = {'aiter': (3, 10), 'anext': (3, 10)} + new_names = {'breakpoint': (3, 7), 'aiter': (3, 10), 'anext': (3, 10)} for t in builtins.__dict__.values(): if isinstance(t, types.BuiltinFunctionType): if t.__name__ in new_names and self.py_version < new_names[t.__name__]: @@ -3259,6 +3259,7 @@ def test_proto(self): else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) + def test_bad_proto(self): oob = protocols[-1] + 1 # a future protocol build_none = pickle.NONE + pickle.STOP badpickle = pickle.PROTO + bytes([oob]) + build_none diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 6181064e447705..a69dddeca68e3d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1,7 +1,5 @@ """Supporting definitions for the Python regression tests.""" -from __future__ import annotations # for test_xpickle - if __name__ != 'test.support': raise ImportError('support must be imported from the test package') @@ -271,7 +269,7 @@ class USEROBJECTFLAGS(ctypes.Structure): reason = "unable to detect macOS launchd job manager" else: if managername != "Aqua": - reason = f"{managername=} -- can only run in a macOS GUI session" + reason = f"{managername!r} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -803,7 +801,7 @@ def sortdict(dict): return "{%s}" % withcommas -def run_code(code: str, extra_names: dict[str, object] | None = None) -> dict[str, object]: +def run_code(code: str, extra_names: 'dict[str, object] | None' = None) -> 'dict[str, object]': """Run a piece of code after dedenting it, and return its global namespace.""" ns = {} if extra_names: @@ -3053,7 +3051,7 @@ def new_setUpClass(cls): return cls -def make_clean_env() -> dict[str, str]: +def make_clean_env() -> 'dict[str, str]': clean_env = os.environ.copy() for k in clean_env.copy(): if k.startswith("PYTHON"): diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index ea73eb4523d92f..a1fd4740676419 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -231,6 +231,9 @@ class CPicklePythonCompat(AbstractCompatTests): (3, 7): [ # This version does not support buffers 'test_in_band_buffers', + + # No protocol validation in this version + 'test_bad_proto', ], } From 0604d9d20e44c7ea0a56392390ebd433040489d2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 20:29:16 +0200 Subject: [PATCH 07/21] Use subprocess.DEVNULL. --- Lib/test/test_xpickle.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index a1fd4740676419..4a5b2db7481445 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -69,19 +69,18 @@ def have_python_version(py_version): python_str = ".".join(map(str, py_version)) targets = [('py', f'-{python_str}'), (f'python{python_str}',)] if py_version not in py_executable_map: - with open(os.devnull, 'w') as devnull: - for target in targets[0 if is_windows else 1:]: - try: - worker = subprocess.Popen([*target, '-c','import test.support'], - stdout=devnull, - stderr=devnull, - shell=is_windows) - worker.communicate() - if worker.returncode == 0: - py_executable_map[py_version] = target - break - except FileNotFoundError: - pass + for target in targets[0 if is_windows else 1:]: + try: + worker = subprocess.Popen([*target, '-c','import test.support'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=is_windows) + worker.communicate() + if worker.returncode == 0: + py_executable_map[py_version] = target + break + except FileNotFoundError: + pass return py_executable_map.get(py_version, None) From 5a28b4df259a8003f2caf1139573f509b8d783e1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 20:41:03 +0200 Subject: [PATCH 08/21] Simplify highest_proto_for_py_version(). --- Lib/test/test_xpickle.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 4a5b2db7481445..a3a3fc9da673d2 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -25,6 +25,12 @@ # Python binary to execute and its arguments. py_executable_map = {} +protocols_map = { + 3: (3, 0), + 4: (3, 4), + 5: (3, 8), +} + def highest_proto_for_py_version(py_version): """Finds the highest supported pickle protocol for a given Python version. Args: @@ -33,20 +39,12 @@ def highest_proto_for_py_version(py_version): Returns: int for the highest supported pickle protocol """ - major = sys.version_info.major - minor = sys.version_info.minor - # Versions older than py 3 only supported up until protocol 2. - if py_version < (3, 0): - return 2 - elif py_version < (3, 4): - return 3 - elif py_version < (3, 8): - return 4 - elif py_version <= (major, minor): - return 5 - else: - # Safe option. - return 2 + proto = 2 + for p, v in protocols_map.items(): + if py_version < v: + break + proto = p + return proto def have_python_version(py_version): """Check whether a Python binary exists for the given py_version and has From 1ab120311feb5c809115f05a88edd89e799f43a1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 20:46:27 +0200 Subject: [PATCH 09/21] Get rid of pathlib. --- Lib/test/test_xpickle.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index a3a3fc9da673d2..885346e2de3012 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -3,7 +3,6 @@ # and Python 3.9 by running xpickle_worker.py. import io import os -import pathlib import pickle import subprocess import sys @@ -116,7 +115,7 @@ def send_to_worker(python, data): Returns: The pickled data received from the child process. """ - target = pathlib.Path(__file__).parent / 'xpickle_worker.py' + target = os.path.join(os.path.dirname(__file__), 'xpickle_worker.py') worker = subprocess.Popen([*python, target], stdin=subprocess.PIPE, stdout=subprocess.PIPE, From 7101adbd1f4c84edf6744fd69619927b345af2f9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 21:08:45 +0200 Subject: [PATCH 10/21] Partially support 3.5. --- Lib/test/pickletester.py | 13 ++++++++----- Lib/test/test_xpickle.py | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 7978977ed75213..4ce7205ff8a518 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3219,11 +3219,14 @@ def test_builtin_types(self): self.assertIs(self.loads(s), t) def test_builtin_exceptions(self): - new_names = {'EncodingWarning': (3, 10), - 'BaseExceptionGroup': (3, 11), - 'ExceptionGroup': (3, 11), - '_IncompleteInputError': (3, 13), - 'PythonFinalizationError': (3, 13)} + new_names = { + 'ModuleNotFoundError': (3, 6), + 'EncodingWarning': (3, 10), + 'BaseExceptionGroup': (3, 11), + 'ExceptionGroup': (3, 11), + '_IncompleteInputError': (3, 13), + 'PythonFinalizationError': (3, 13), + } for t in builtins.__dict__.values(): if isinstance(t, type) and issubclass(t, BaseException): if t.__name__ in new_names and self.py_version < new_names[t.__name__]: diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 885346e2de3012..ccf12946a77ede 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -1,6 +1,6 @@ # This test covers backwards compatibility with -# previous version of Python by bouncing pickled objects through Python 3.6 -# and Python 3.9 by running xpickle_worker.py. +# previous version of Python by bouncing pickled objects through Python 3.5 +# and the current version by running xpickle_worker.py. import io import os import pickle @@ -68,7 +68,7 @@ def have_python_version(py_version): if py_version not in py_executable_map: for target in targets[0 if is_windows else 1:]: try: - worker = subprocess.Popen([*target, '-c','import test.support'], + worker = subprocess.Popen([*target, '-c', 'pass'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=is_windows) @@ -246,7 +246,7 @@ def make_test(py_version, base): def load_tests(loader, tests, pattern): major = sys.version_info.major assert major == 3 - for minor in range(sys.version_info.minor): + for minor in range(5, sys.version_info.minor): test_class = make_test((major, minor), PyPicklePythonCompat) tests.addTest(loader.loadTestsFromTestCase(test_class)) if has_c_implementation: From 8a23399be651734533bb00f48fe3ba394761dc38 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 22:38:07 +0200 Subject: [PATCH 11/21] Only share tested classes. Full support of 3.5+. --- Lib/test/picklecommon.py | 373 +++++++++++++++++++++++++++++++++++ Lib/test/pickletester.py | 373 +---------------------------------- Lib/test/support/__init__.py | 26 ++- Lib/test/test_xpickle.py | 3 - Lib/test/xpickle_worker.py | 6 +- 5 files changed, 391 insertions(+), 390 deletions(-) create mode 100644 Lib/test/picklecommon.py diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py new file mode 100644 index 00000000000000..e1c18d61b9196f --- /dev/null +++ b/Lib/test/picklecommon.py @@ -0,0 +1,373 @@ +class C: + def __eq__(self, other): + return self.__dict__ == other.__dict__ + +class D(C): + def __init__(self, arg): + pass + +class E(C): + def __getinitargs__(self): + return () + +import __main__ +__main__.C = C +C.__module__ = "__main__" +__main__.D = D +D.__module__ = "__main__" +__main__.E = E +E.__module__ = "__main__" + +# Simple mutable object. +class Object: + pass + +# Hashable immutable key object containing unheshable mutable data. +class K: + def __init__(self, value): + self.value = value + + def __reduce__(self): + # Shouldn't support the recursion itself + return K, (self.value,) + +class myint(int): + def __init__(self, x): + self.str = str(x) + +class initarg(C): + + def __init__(self, a, b): + self.a = a + self.b = b + + def __getinitargs__(self): + return self.a, self.b + +class metaclass(type): + pass + +class use_metaclass(object, metaclass=metaclass): + pass + + +# Test classes for reduce_ex + +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + +class REX_one(object): + """No __reduce_ex__ here, but inheriting it from object""" + _reduce_called = 0 + def __reduce__(self): + self._reduce_called = 1 + return REX_one, () + +class REX_two(object): + """No __reduce__ here, but inheriting it from object""" + _proto = None + def __reduce_ex__(self, proto): + self._proto = proto + return REX_two, () + +class REX_three(object): + _proto = None + def __reduce_ex__(self, proto): + self._proto = proto + return REX_two, () + def __reduce__(self): + raise TestFailed("This __reduce__ shouldn't be called") + +class REX_four(object): + """Calling base class method should succeed""" + _proto = None + def __reduce_ex__(self, proto): + self._proto = proto + return object.__reduce_ex__(self, proto) + +class REX_five(object): + """This one used to fail with infinite recursion""" + _reduce_called = 0 + def __reduce__(self): + self._reduce_called = 1 + return object.__reduce__(self) + +class REX_six(object): + """This class is used to check the 4th argument (list iterator) of + the reduce protocol. + """ + def __init__(self, items=None): + self.items = items if items is not None else [] + def __eq__(self, other): + return type(self) is type(other) and self.items == other.items + def append(self, item): + self.items.append(item) + def __reduce__(self): + return type(self), (), None, iter(self.items), None + +class REX_seven(object): + """This class is used to check the 5th argument (dict iterator) of + the reduce protocol. + """ + def __init__(self, table=None): + self.table = table if table is not None else {} + def __eq__(self, other): + return type(self) is type(other) and self.table == other.table + def __setitem__(self, key, value): + self.table[key] = value + def __reduce__(self): + return type(self), (), None, None, iter(self.table.items()) + +class REX_state(object): + """This class is used to check the 3th argument (state) of + the reduce protocol. + """ + def __init__(self, state=None): + self.state = state + def __eq__(self, other): + return type(self) is type(other) and self.state == other.state + def __setstate__(self, state): + self.state = state + def __reduce__(self): + return type(self), (), self.state + +class REX_None: + """ Setting __reduce_ex__ to None should fail """ + __reduce_ex__ = None + +class R_None: + """ Setting __reduce__ to None should fail """ + __reduce__ = None + +class C_None_setstate: + """ Setting __setstate__ to None should fail """ + def __getstate__(self): + return 1 + + __setstate__ = None + + +# Test classes for newobj + +class MyInt(int): + sample = 1 + +class MyFloat(float): + sample = 1.0 + +class MyComplex(complex): + sample = 1.0 + 0.0j + +class MyStr(str): + sample = "hello" + +class MyUnicode(str): + sample = "hello \u1234" + +class MyTuple(tuple): + sample = (1, 2, 3) + +class MyList(list): + sample = [1, 2, 3] + +class MyDict(dict): + sample = {"a": 1, "b": 2} + +class MySet(set): + sample = {"a", "b"} + +class MyFrozenSet(frozenset): + sample = frozenset({"a", "b"}) + +myclasses = [MyInt, MyFloat, + MyComplex, + MyStr, MyUnicode, + MyTuple, MyList, MyDict, MySet, MyFrozenSet] + +class MyIntWithNew(int): + def __new__(cls, value): + raise AssertionError + +class MyIntWithNew2(MyIntWithNew): + __new__ = int.__new__ + + +class SlotList(MyList): + __slots__ = ["foo"] + +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 + def __init__(self, *args, **kwargs): + # raise an error, to make sure this isn't called + raise TypeError("SimpleNewObj.__init__() didn't expect to get called") + def __eq__(self, other): + return int(self) == int(other) and self.__dict__ == other.__dict__ + +class ComplexNewObj(SimpleNewObj): + def __getnewargs__(self): + return ('%X' % self, 16) + +class ComplexNewObjEx(SimpleNewObj): + def __getnewargs_ex__(self): + return ('%X' % self,), {'base': 16} + +class BadGetattr: + def __getattr__(self, key): + self.foo + +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + + +class ZeroCopyBytes(bytes): + readonly = True + c_contiguous = True + f_contiguous = True + zero_copy_reconstruct = True + + def __reduce_ex__(self, protocol): + if protocol >= 5: + import pickle + return type(self)._reconstruct, (pickle.PickleBuffer(self),), None + else: + return type(self)._reconstruct, (bytes(self),) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, bytes(self)) + + __str__ = __repr__ + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + obj = m.obj + if type(obj) is cls: + # Zero-copy + return obj + else: + return cls(obj) + + +class ZeroCopyBytearray(bytearray): + readonly = False + c_contiguous = True + f_contiguous = True + zero_copy_reconstruct = True + + def __reduce_ex__(self, protocol): + if protocol >= 5: + import pickle + return type(self)._reconstruct, (pickle.PickleBuffer(self),), None + else: + return type(self)._reconstruct, (bytes(self),) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, bytes(self)) + + __str__ = __repr__ + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + obj = m.obj + if type(obj) is cls: + # Zero-copy + return obj + else: + return cls(obj) + + +try: + import _testbuffer +except ImportError: + _testbuffer = None + +if _testbuffer is not None: + + class PicklableNDArray: + # A not-really-zero-copy picklable ndarray, as the ndarray() + # constructor doesn't allow for it + + zero_copy_reconstruct = False + + def __init__(self, *args, **kwargs): + self.array = _testbuffer.ndarray(*args, **kwargs) + + def __getitem__(self, idx): + cls = type(self) + new = cls.__new__(cls) + new.array = self.array[idx] + return new + + @property + def readonly(self): + return self.array.readonly + + @property + def c_contiguous(self): + return self.array.c_contiguous + + @property + def f_contiguous(self): + return self.array.f_contiguous + + def __eq__(self, other): + if not isinstance(other, PicklableNDArray): + return NotImplemented + return (other.array.format == self.array.format and + other.array.shape == self.array.shape and + other.array.strides == self.array.strides and + other.array.readonly == self.array.readonly and + other.array.tobytes() == self.array.tobytes()) + + def __ne__(self, other): + if not isinstance(other, PicklableNDArray): + return NotImplemented + return not (self == other) + + def __repr__(self): + return ("{name}(shape={array.shape}," + "strides={array.strides}, " + "bytes={array.tobytes()})").format( + name=type(self).__name__, array=self.array.shape) + + def __reduce_ex__(self, protocol): + if not self.array.contiguous: + raise NotImplementedError("Reconstructing a non-contiguous " + "ndarray does not seem possible") + ndarray_kwargs = {"shape": self.array.shape, + "strides": self.array.strides, + "format": self.array.format, + "flags": (0 if self.readonly + else _testbuffer.ND_WRITABLE)} + import pickle + pb = pickle.PickleBuffer(self.array) + if protocol >= 5: + return (type(self)._reconstruct, + (pb, ndarray_kwargs)) + else: + # Need to serialize the bytes in physical order + with pb.raw() as m: + return (type(self)._reconstruct, + (m.tobytes(), ndarray_kwargs)) + + @classmethod + def _reconstruct(cls, obj, kwargs): + with memoryview(obj) as m: + # For some reason, ndarray() wants a list of integers... + # XXX This only works if format == 'B' + items = list(m.tobytes()) + return cls(items, **kwargs) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 4ce7205ff8a518..2988f22268207d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -33,6 +33,8 @@ from test.support.os_helper import TESTFN from test.support import threading_helper from test.support.warnings_helper import save_restore_warnings_filters +from test import picklecommon +from test.picklecommon import * from pickle import bytes_types @@ -141,58 +143,6 @@ def restore(self): if pair is not None: copyreg.add_extension(pair[0], pair[1], code) -class C: - def __eq__(self, other): - return self.__dict__ == other.__dict__ - -class D(C): - def __init__(self, arg): - pass - -class E(C): - def __getinitargs__(self): - return () - -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - -# Simple mutable object. -class Object: - pass - -# Hashable immutable key object containing unheshable mutable data. -class K: - def __init__(self, value): - self.value = value - - def __reduce__(self): - # Shouldn't support the recursion itself - return K, (self.value,) - -class myint(int): - def __init__(self, x): - self.str = str(x) - -class initarg(C): - - def __init__(self, a, b): - self.a = a - self.b = b - - def __getinitargs__(self): - return self.a, self.b - -class metaclass(type): - pass - -class use_metaclass(object, metaclass=metaclass): - pass - class pickling_metaclass(type): def __eq__(self, other): return (type(self) == type(other) and @@ -207,138 +157,6 @@ def create_dynamic_class(name, bases): return result -class ZeroCopyBytes(bytes): - readonly = True - c_contiguous = True - f_contiguous = True - zero_copy_reconstruct = True - - def __reduce_ex__(self, protocol): - if protocol >= 5: - return type(self)._reconstruct, (pickle.PickleBuffer(self),), None - else: - return type(self)._reconstruct, (bytes(self),) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, bytes(self)) - - __str__ = __repr__ - - @classmethod - def _reconstruct(cls, obj): - with memoryview(obj) as m: - obj = m.obj - if type(obj) is cls: - # Zero-copy - return obj - else: - return cls(obj) - - -class ZeroCopyBytearray(bytearray): - readonly = False - c_contiguous = True - f_contiguous = True - zero_copy_reconstruct = True - - def __reduce_ex__(self, protocol): - if protocol >= 5: - return type(self)._reconstruct, (pickle.PickleBuffer(self),), None - else: - return type(self)._reconstruct, (bytes(self),) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, bytes(self)) - - __str__ = __repr__ - - @classmethod - def _reconstruct(cls, obj): - with memoryview(obj) as m: - obj = m.obj - if type(obj) is cls: - # Zero-copy - return obj - else: - return cls(obj) - - -if _testbuffer is not None: - - class PicklableNDArray: - # A not-really-zero-copy picklable ndarray, as the ndarray() - # constructor doesn't allow for it - - zero_copy_reconstruct = False - - def __init__(self, *args, **kwargs): - self.array = _testbuffer.ndarray(*args, **kwargs) - - def __getitem__(self, idx): - cls = type(self) - new = cls.__new__(cls) - new.array = self.array[idx] - return new - - @property - def readonly(self): - return self.array.readonly - - @property - def c_contiguous(self): - return self.array.c_contiguous - - @property - def f_contiguous(self): - return self.array.f_contiguous - - def __eq__(self, other): - if not isinstance(other, PicklableNDArray): - return NotImplemented - return (other.array.format == self.array.format and - other.array.shape == self.array.shape and - other.array.strides == self.array.strides and - other.array.readonly == self.array.readonly and - other.array.tobytes() == self.array.tobytes()) - - def __ne__(self, other): - if not isinstance(other, PicklableNDArray): - return NotImplemented - return not (self == other) - - def __repr__(self): - return (f"{type(self)}(shape={self.array.shape}," - f"strides={self.array.strides}, " - f"bytes={self.array.tobytes()})") - - def __reduce_ex__(self, protocol): - if not self.array.contiguous: - raise NotImplementedError("Reconstructing a non-contiguous " - "ndarray does not seem possible") - ndarray_kwargs = {"shape": self.array.shape, - "strides": self.array.strides, - "format": self.array.format, - "flags": (0 if self.readonly - else _testbuffer.ND_WRITABLE)} - pb = pickle.PickleBuffer(self.array) - if protocol >= 5: - return (type(self)._reconstruct, - (pb, ndarray_kwargs)) - else: - # Need to serialize the bytes in physical order - with pb.raw() as m: - return (type(self)._reconstruct, - (m.tobytes(), ndarray_kwargs)) - - @classmethod - def _reconstruct(cls, obj, kwargs): - with memoryview(obj) as m: - # For some reason, ndarray() wants a list of integers... - # XXX This only works if format == 'B' - items = list(m.tobytes()) - return cls(items, **kwargs) - - # DATA0 .. DATA4 are the pickles we expect under the various protocols, for # the object returned by create_data(). @@ -3416,15 +3234,10 @@ def test_newobj_overridden_new(self): def test_newobj_not_class(self): # Issue 24552 - global SimpleNewObj - save = SimpleNewObj o = SimpleNewObj.__new__(SimpleNewObj) b = self.dumps(o, 4) - try: - SimpleNewObj = 42 + with support.swap_attr(picklecommon, 'SimpleNewObj', 42): self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b) - finally: - SimpleNewObj = save # Register a type with copyreg, with extension code extcode. Pickle # an object of that type. Check that the resulting pickle uses opcode @@ -4434,110 +4247,6 @@ def test_huge_str_64b(self, size): data = None -# Test classes for reduce_ex - -class R: - def __init__(self, reduce=None): - self.reduce = reduce - def __reduce__(self, proto): - return self.reduce - -class REX: - def __init__(self, reduce_ex=None): - self.reduce_ex = reduce_ex - def __reduce_ex__(self, proto): - return self.reduce_ex - -class REX_one(object): - """No __reduce_ex__ here, but inheriting it from object""" - _reduce_called = 0 - def __reduce__(self): - self._reduce_called = 1 - return REX_one, () - -class REX_two(object): - """No __reduce__ here, but inheriting it from object""" - _proto = None - def __reduce_ex__(self, proto): - self._proto = proto - return REX_two, () - -class REX_three(object): - _proto = None - def __reduce_ex__(self, proto): - self._proto = proto - return REX_two, () - def __reduce__(self): - raise TestFailed("This __reduce__ shouldn't be called") - -class REX_four(object): - """Calling base class method should succeed""" - _proto = None - def __reduce_ex__(self, proto): - self._proto = proto - return object.__reduce_ex__(self, proto) - -class REX_five(object): - """This one used to fail with infinite recursion""" - _reduce_called = 0 - def __reduce__(self): - self._reduce_called = 1 - return object.__reduce__(self) - -class REX_six(object): - """This class is used to check the 4th argument (list iterator) of - the reduce protocol. - """ - def __init__(self, items=None): - self.items = items if items is not None else [] - def __eq__(self, other): - return type(self) is type(other) and self.items == other.items - def append(self, item): - self.items.append(item) - def __reduce__(self): - return type(self), (), None, iter(self.items), None - -class REX_seven(object): - """This class is used to check the 5th argument (dict iterator) of - the reduce protocol. - """ - def __init__(self, table=None): - self.table = table if table is not None else {} - def __eq__(self, other): - return type(self) is type(other) and self.table == other.table - def __setitem__(self, key, value): - self.table[key] = value - def __reduce__(self): - return type(self), (), None, None, iter(self.table.items()) - -class REX_state(object): - """This class is used to check the 3th argument (state) of - the reduce protocol. - """ - def __init__(self, state=None): - self.state = state - def __eq__(self, other): - return type(self) is type(other) and self.state == other.state - def __setstate__(self, state): - self.state = state - def __reduce__(self): - return type(self), (), self.state - -class REX_None: - """ Setting __reduce_ex__ to None should fail """ - __reduce_ex__ = None - -class R_None: - """ Setting __reduce__ to None should fail """ - __reduce__ = None - -class C_None_setstate: - """ Setting __setstate__ to None should fail """ - def __getstate__(self): - return 1 - - __setstate__ = None - class CustomError(Exception): pass @@ -4552,82 +4261,6 @@ def __call__(self, *args, **kwargs): pass -# Test classes for newobj - -class MyInt(int): - sample = 1 - -class MyFloat(float): - sample = 1.0 - -class MyComplex(complex): - sample = 1.0 + 0.0j - -class MyStr(str): - sample = "hello" - -class MyUnicode(str): - sample = "hello \u1234" - -class MyTuple(tuple): - sample = (1, 2, 3) - -class MyList(list): - sample = [1, 2, 3] - -class MyDict(dict): - sample = {"a": 1, "b": 2} - -class MySet(set): - sample = {"a", "b"} - -class MyFrozenSet(frozenset): - sample = frozenset({"a", "b"}) - -myclasses = [MyInt, MyFloat, - MyComplex, - MyStr, MyUnicode, - MyTuple, MyList, MyDict, MySet, MyFrozenSet] - -class MyIntWithNew(int): - def __new__(cls, value): - raise AssertionError - -class MyIntWithNew2(MyIntWithNew): - __new__ = int.__new__ - - -class SlotList(MyList): - __slots__ = ["foo"] - -# Ruff "redefined while unused" false positive here due to `global` variables -# being assigned (and then restored) from within test methods earlier in the file -class SimpleNewObj(int): # noqa: F811 - def __init__(self, *args, **kwargs): - # raise an error, to make sure this isn't called - raise TypeError("SimpleNewObj.__init__() didn't expect to get called") - def __eq__(self, other): - return int(self) == int(other) and self.__dict__ == other.__dict__ - -class ComplexNewObj(SimpleNewObj): - def __getnewargs__(self): - return ('%X' % self, 16) - -class ComplexNewObjEx(SimpleNewObj): - def __getnewargs_ex__(self): - return ('%X' % self,), {'base': 16} - -class BadGetattr: - def __getattr__(self, key): - self.foo - -class NoNew: - def __getattribute__(self, name): - if name == '__new__': - raise AttributeError - return super().__getattribute__(name) - - class AbstractPickleModuleTests: def test_dump_closed_file(self): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a69dddeca68e3d..84fd43fd396914 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3,6 +3,7 @@ if __name__ != 'test.support': raise ImportError('support must be imported from the test package') +import annotationlib import contextlib import functools import inspect @@ -269,7 +270,7 @@ class USEROBJECTFLAGS(ctypes.Structure): reason = "unable to detect macOS launchd job manager" else: if managername != "Aqua": - reason = f"{managername!r} -- can only run in a macOS GUI session" + reason = f"{managername=} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -644,7 +645,7 @@ def requires_working_socket(*, module=False): return unittest.skipUnless(has_socket_support, msg) -@functools.lru_cache() +@functools.cache def has_remote_subprocess_debugging(): """Check if we have permissions to debug subprocesses remotely. @@ -801,7 +802,7 @@ def sortdict(dict): return "{%s}" % withcommas -def run_code(code: str, extra_names: 'dict[str, object] | None' = None) -> 'dict[str, object]': +def run_code(code: str, extra_names: dict[str, object] | None = None) -> dict[str, object]: """Run a piece of code after dedenting it, and return its global namespace.""" ns = {} if extra_names: @@ -2544,7 +2545,7 @@ def requires_venv_with_pip(): return unittest.skipUnless(ctypes, 'venv: pip requires ctypes') -@functools.lru_cache() +@functools.cache def _findwheel(pkgname): """Try to find a wheel with the package specified as pkgname. @@ -2788,10 +2789,7 @@ def exceeds_recursion_limit(): Py_TRACE_REFS = hasattr(sys, 'getobjects') -try: - _JIT_ENABLED = sys._jit.is_enabled() -except AttributeError: - _JIT_ENABLED = False +_JIT_ENABLED = sys._jit.is_enabled() requires_jit_enabled = unittest.skipUnless(_JIT_ENABLED, "requires JIT enabled") requires_jit_disabled = unittest.skipIf(_JIT_ENABLED, "requires JIT disabled") @@ -2998,8 +2996,10 @@ def force_color(color: bool): import _colorize from .os_helper import EnvironmentVarGuard - with swap_attr(_colorize, "can_colorize", lambda *, file=None: color), \ - EnvironmentVarGuard() as env: + with ( + swap_attr(_colorize, "can_colorize", lambda *, file=None: color), + EnvironmentVarGuard() as env, + ): env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS") env.set("FORCE_COLOR" if color else "NO_COLOR", "1") yield @@ -3051,7 +3051,7 @@ def new_setUpClass(cls): return cls -def make_clean_env() -> 'dict[str, str]': +def make_clean_env() -> dict[str, str]: clean_env = os.environ.copy() for k in clean_env.copy(): if k.startswith("PYTHON"): @@ -3218,11 +3218,9 @@ def __init__( self.__forward_is_class__ = is_class self.__forward_module__ = module self.__owner__ = owner - import annotationlib - self._ForwardRef = annotationlib.ForwardRef def __eq__(self, other): - if not isinstance(other, (EqualToForwardRef, self._ForwardRef)): + if not isinstance(other, (EqualToForwardRef, annotationlib.ForwardRef)): return NotImplemented return ( self.__forward_arg__ == other.__forward_arg__ diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index ccf12946a77ede..d524acbfdcac01 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -190,11 +190,8 @@ def test_bytes(self): test_c_methods = None test_py_methods = None test_nested_names = None - - test_recursive_dict_key = None test_recursive_nested_names = None test_recursive_nested_names2 = None - test_recursive_set = None # Attribute lookup problems are expected, disable the test test_dynamic_class = None diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py index 53a7d48b18136c..944d742cfa22e7 100644 --- a/Lib/test/xpickle_worker.py +++ b/Lib/test/xpickle_worker.py @@ -12,11 +12,11 @@ # pickletester requires some test.support functions (such as os_helper) # which are not available in versions below Python 3.10. test_mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__), - '__init__.py')) -spec = importlib.util.spec_from_file_location('test', test_mod_path) + 'picklecommon.py')) +spec = importlib.util.spec_from_file_location('test.picklecommon', test_mod_path) test_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(test_module) -sys.modules['test'] = test_module +sys.modules['test.picklecommon'] = test_module # To unpickle certain objects, the structure of the class needs to be known. From d482988a927f85ed5760333c28b1ec19afdd1066 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Jan 2026 22:49:51 +0200 Subject: [PATCH 12/21] Clean up xpickle_worker.py. --- Lib/test/xpickle_worker.py | 62 ++------------------------------------ 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py index 944d742cfa22e7..e55b24405696fc 100644 --- a/Lib/test/xpickle_worker.py +++ b/Lib/test/xpickle_worker.py @@ -6,11 +6,8 @@ import sys -# This allows the xpickle worker to import pickletester.py, which it needs -# since some of the pickle objects hold references to pickletester.py. -# Also provides the test library over the platform's native one since -# pickletester requires some test.support functions (such as os_helper) -# which are not available in versions below Python 3.10. +# This allows the xpickle worker to import picklecommon.py, which it needs +# since some of the pickle objects hold references to picklecommon.py. test_mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'picklecommon.py')) spec = importlib.util.spec_from_file_location('test.picklecommon', test_mod_path) @@ -19,61 +16,6 @@ sys.modules['test.picklecommon'] = test_module -# To unpickle certain objects, the structure of the class needs to be known. -# These classes are mostly copies from pickletester.py. - -class Nested: - class A: - class B: - class C: - pass -class C: - pass - -class D(C): - pass - -class E(C): - pass - -class H(object): - pass - -class K(object): - pass - -class Subclass(tuple): - class Nested(str): - pass - -class PyMethodsTest: - @staticmethod - def cheese(): - pass - - @classmethod - def wine(cls): - pass - - def biscuits(self): - pass - - class Nested: - "Nested class" - @staticmethod - def ketchup(): - pass - @classmethod - def maple(cls): - pass - def pie(self): - pass - - -class Recursive: - pass - - in_stream = sys.stdin.buffer out_stream = sys.stdout.buffer From c00dbc20c6db0cbf792f2bc1b739cc2de610da72 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Jan 2026 09:53:27 +0200 Subject: [PATCH 13/21] Fix test_pickle. Enable more tests in test_xpickle. --- Lib/test/picklecommon.py | 64 +++++++++++--- Lib/test/pickletester.py | 180 +++++++++++++++++---------------------- Lib/test/test_xpickle.py | 3 - 3 files changed, 133 insertions(+), 114 deletions(-) diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py index e1c18d61b9196f..26c428058e97f3 100644 --- a/Lib/test/picklecommon.py +++ b/Lib/test/picklecommon.py @@ -1,7 +1,12 @@ +# Classes used for pickle testing. +# They are moved to separate file, so they can be loaded +# in other Python version for test_xpickle. + class C: def __eq__(self, other): return self.__dict__ == other.__dict__ +# For test_load_classic_instance class D(C): def __init__(self, arg): pass @@ -31,10 +36,12 @@ def __reduce__(self): # Shouldn't support the recursion itself return K, (self.value,) +# For test_misc class myint(int): def __init__(self, x): self.str = str(x) +# For test_misc and test_getinitargs class initarg(C): def __init__(self, a, b): @@ -44,6 +51,7 @@ def __init__(self, a, b): def __getinitargs__(self): return self.a, self.b +# For test_metaclass class metaclass(type): pass @@ -140,14 +148,17 @@ def __setstate__(self, state): def __reduce__(self): return type(self), (), self.state +# For test_reduce_ex_None class REX_None: """ Setting __reduce_ex__ to None should fail """ __reduce_ex__ = None +# For test_reduce_None class R_None: """ Setting __reduce__ to None should fail """ __reduce__ = None +# For test_pickle_setstate_None class C_None_setstate: """ Setting __setstate__ to None should fail """ def __getstate__(self): @@ -158,6 +169,8 @@ def __getstate__(self): # Test classes for newobj +# For test_newobj_generic and test_newobj_proxies + class MyInt(int): sample = 1 @@ -193,6 +206,7 @@ class MyFrozenSet(frozenset): MyStr, MyUnicode, MyTuple, MyList, MyDict, MySet, MyFrozenSet] +# For test_newobj_overridden_new class MyIntWithNew(int): def __new__(cls, value): raise AssertionError @@ -201,6 +215,7 @@ class MyIntWithNew2(MyIntWithNew): __new__ = int.__new__ +# For test_newobj_list_slots class SlotList(MyList): __slots__ = ["foo"] @@ -221,16 +236,6 @@ class ComplexNewObjEx(SimpleNewObj): def __getnewargs_ex__(self): return ('%X' % self,), {'base': 16} -class BadGetattr: - def __getattr__(self, key): - self.foo - -class NoNew: - def __getattribute__(self, name): - if name == '__new__': - raise AttributeError - return super().__getattribute__(name) - class ZeroCopyBytes(bytes): readonly = True @@ -371,3 +376,42 @@ def _reconstruct(cls, obj, kwargs): # XXX This only works if format == 'B' items = list(m.tobytes()) return cls(items, **kwargs) + + +# For test_nested_names +class Nested: + class A: + class B: + class C: + pass + +# For test_py_methods +class PyMethodsTest: + @staticmethod + def cheese(): + return "cheese" + @classmethod + def wine(cls): + assert cls is PyMethodsTest + return "wine" + def biscuits(self): + assert isinstance(self, PyMethodsTest) + return "biscuits" + class Nested: + "Nested class" + @staticmethod + def ketchup(): + return "ketchup" + @classmethod + def maple(cls): + assert cls is PyMethodsTest.Nested + return "maple" + def pie(self): + assert isinstance(self, PyMethodsTest.Nested) + return "pie" + +# For test_c_methods +class Subclass(tuple): + class Nested(str): + pass + diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 2988f22268207d..79df7b82840709 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -15,6 +15,7 @@ import types import unittest import weakref +import __main__ from textwrap import dedent from http.cookies import SimpleCookie @@ -1348,9 +1349,9 @@ def check(key, exc): self.loads(b'\x82\x01.') check(None, ValueError) check((), ValueError) - check((__name__,), (TypeError, ValueError)) - check((__name__, "MyList", "x"), (TypeError, ValueError)) - check((__name__, None), (TypeError, ValueError)) + check((MyList.__module__,), (TypeError, ValueError)) + check((MyList.__module__, "MyList", "x"), (TypeError, ValueError)) + check((MyList.__module__, None), (TypeError, ValueError)) check((None, "MyList"), (TypeError, ValueError)) def test_bad_reduce(self): @@ -1664,7 +1665,7 @@ def test_bad_reduce_result(self): self.assertEqual(str(cm.exception), '__reduce__ must return a string or tuple, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((print,)) for proto in protocols: @@ -1674,7 +1675,7 @@ def test_bad_reduce_result(self): self.assertEqual(str(cm.exception), 'tuple returned by __reduce__ must contain 2 through 6 elements') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((print, (), None, None, None, None, None)) for proto in protocols: @@ -1684,7 +1685,7 @@ def test_bad_reduce_result(self): self.assertEqual(str(cm.exception), 'tuple returned by __reduce__ must contain 2 through 6 elements') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_bad_reconstructor(self): obj = REX((42, ())) @@ -1696,7 +1697,7 @@ def test_bad_reconstructor(self): 'first item of the tuple returned by __reduce__ ' 'must be callable, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_reconstructor(self): obj = REX((UnpickleableCallable(), ())) @@ -1705,8 +1706,8 @@ def test_unpickleable_reconstructor(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX reconstructor', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor', + f'when serializing {REX.__module__}.REX object']) def test_bad_reconstructor_args(self): obj = REX((print, [])) @@ -1718,7 +1719,7 @@ def test_bad_reconstructor_args(self): 'second item of the tuple returned by __reduce__ ' 'must be a tuple, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_reconstructor_args(self): obj = REX((print, (1, 2, UNPICKLEABLE))) @@ -1728,8 +1729,8 @@ def test_unpickleable_reconstructor_args(self): self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_args(self): obj = REX((copyreg.__newobj__, ())) @@ -1741,7 +1742,7 @@ def test_bad_newobj_args(self): 'tuple index out of range', '__newobj__ expected at least 1 argument, got 0'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj__, [REX])) for proto in protocols[2:]: @@ -1752,7 +1753,7 @@ def test_bad_newobj_args(self): 'second item of the tuple returned by __reduce__ ' 'must be a tuple, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_class(self): obj = REX((copyreg.__newobj__, (NoNew(),))) @@ -1762,9 +1763,9 @@ def test_bad_newobj_class(self): self.dumps(obj, proto) self.assertIn(str(cm.exception), { 'first argument to __newobj__() has no __new__', - f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + f'first argument to __newobj__() must be a class, not {NoNew.__module__}.NoNew'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_wrong_newobj_class(self): obj = REX((copyreg.__newobj__, (str,))) @@ -1775,7 +1776,7 @@ def test_wrong_newobj_class(self): self.assertEqual(str(cm.exception), f'first argument to __newobj__() must be {REX!r}, not {str!r}') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_newobj_class(self): class LocalREX(REX): pass @@ -1803,13 +1804,13 @@ def test_unpickleable_newobj_args(self): if proto >= 2: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', - 'when serializing test.pickletester.REX __new__ arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX __new__ arguments', + f'when serializing {REX.__module__}.REX object']) else: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 3', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_ex_args(self): obj = REX((copyreg.__newobj_ex__, ())) @@ -1821,7 +1822,7 @@ def test_bad_newobj_ex_args(self): 'not enough values to unpack (expected 3, got 0)', '__newobj_ex__ expected 3 arguments, got 0'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj_ex__, 42)) for proto in protocols[2:]: @@ -1832,7 +1833,7 @@ def test_bad_newobj_ex_args(self): 'second item of the tuple returned by __reduce__ ' 'must be a tuple, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) if self.pickler is pickle._Pickler: @@ -1843,7 +1844,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'Value after * must be an iterable, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) else: for proto in protocols[2:]: with self.subTest(proto=proto): @@ -1852,7 +1853,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'second argument to __newobj_ex__() must be a tuple, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, (), []))) if self.pickler is pickle._Pickler: @@ -1863,7 +1864,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'Value after ** must be a mapping, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) else: for proto in protocols[2:]: with self.subTest(proto=proto): @@ -1872,7 +1873,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'third argument to __newobj_ex__() must be a dict, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_ex__class(self): obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) @@ -1882,9 +1883,9 @@ def test_bad_newobj_ex__class(self): self.dumps(obj, proto) self.assertIn(str(cm.exception), { 'first argument to __newobj_ex__() has no __new__', - f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + f'first argument to __newobj_ex__() must be a class, not {NoNew.__module__}.NoNew'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_wrong_newobj_ex_class(self): if self.pickler is not pickle._Pickler: @@ -1897,7 +1898,7 @@ def test_wrong_newobj_ex_class(self): self.assertEqual(str(cm.exception), f'first argument to __newobj_ex__() must be {REX}, not {str}') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_newobj_ex_class(self): class LocalREX(REX): pass @@ -1933,22 +1934,22 @@ def test_unpickleable_newobj_ex_args(self): if proto >= 4: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', - 'when serializing test.pickletester.REX __new__ arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX __new__ arguments', + f'when serializing {REX.__module__}.REX object']) elif proto >= 2: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 3', 'when serializing tuple item 1', 'when serializing functools.partial state', 'when serializing functools.partial object', - 'when serializing test.pickletester.REX reconstructor', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor', + f'when serializing {REX.__module__}.REX object']) else: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', 'when serializing tuple item 1', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_newobj_ex_kwargs(self): obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) @@ -1959,22 +1960,22 @@ def test_unpickleable_newobj_ex_kwargs(self): if proto >= 4: self.assertEqual(cm.exception.__notes__, [ "when serializing dict item 'a'", - 'when serializing test.pickletester.REX __new__ arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX __new__ arguments', + f'when serializing {REX.__module__}.REX object']) elif proto >= 2: self.assertEqual(cm.exception.__notes__, [ "when serializing dict item 'a'", 'when serializing tuple item 2', 'when serializing functools.partial state', 'when serializing functools.partial object', - 'when serializing test.pickletester.REX reconstructor', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor', + f'when serializing {REX.__module__}.REX object']) else: self.assertEqual(cm.exception.__notes__, [ "when serializing dict item 'a'", 'when serializing tuple item 2', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_state(self): obj = REX_state(UNPICKLEABLE) @@ -1983,8 +1984,8 @@ def test_unpickleable_state(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX_state state', - 'when serializing test.pickletester.REX_state object']) + f'when serializing {REX_state.__module__}.REX_state state', + f'when serializing {REX_state.__module__}.REX_state object']) def test_bad_state_setter(self): if self.pickler is pickle._Pickler: @@ -1998,7 +1999,7 @@ def test_bad_state_setter(self): 'sixth item of the tuple returned by __reduce__ ' 'must be callable, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_state_setter(self): obj = REX((print, (), 'state', None, None, UnpickleableCallable())) @@ -2007,8 +2008,8 @@ def test_unpickleable_state_setter(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX state setter', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX state setter', + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_state_with_state_setter(self): obj = REX((print, (), UNPICKLEABLE, None, None, print)) @@ -2017,8 +2018,8 @@ def test_unpickleable_state_with_state_setter(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX state', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX state', + f'when serializing {REX.__module__}.REX object']) def test_bad_object_list_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -2033,7 +2034,7 @@ def test_bad_object_list_items(self): 'fourth item of the tuple returned by __reduce__ ' 'must be an iterator, not int'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. @@ -2046,7 +2047,7 @@ def test_bad_object_list_items(self): 'fourth item of the tuple returned by __reduce__ ' 'must be an iterator, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_object_list_items(self): obj = REX_six([1, 2, UNPICKLEABLE]) @@ -2055,8 +2056,8 @@ def test_unpickleable_object_list_items(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX_six item 2', - 'when serializing test.pickletester.REX_six object']) + f'when serializing {REX_six.__module__}.REX_six item 2', + f'when serializing {REX_six.__module__}.REX_six object']) def test_bad_object_dict_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -2071,7 +2072,7 @@ def test_bad_object_dict_items(self): 'fifth item of the tuple returned by __reduce__ ' 'must be an iterator, not int'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) for proto in protocols: obj = REX((dict, (), None, None, iter([('a',)]))) @@ -2082,7 +2083,7 @@ def test_bad_object_dict_items(self): 'not enough values to unpack (expected 2, got 1)', 'dict items iterator must return 2-tuples'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. @@ -2094,7 +2095,7 @@ def test_bad_object_dict_items(self): self.assertEqual(str(cm.exception), 'dict items iterator must return 2-tuples') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_object_dict_items(self): obj = REX_seven({'a': UNPICKLEABLE}) @@ -2103,8 +2104,8 @@ def test_unpickleable_object_dict_items(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - "when serializing test.pickletester.REX_seven item 'a'", - 'when serializing test.pickletester.REX_seven object']) + f"when serializing {REX_seven.__module__}.REX_seven item 'a'", + f'when serializing {REX_seven.__module__}.REX_seven object']) def test_unpickleable_list_items(self): obj = [1, [2, 3, UNPICKLEABLE]] @@ -2197,15 +2198,15 @@ def test_unpickleable_frozenset_items(self): def test_global_lookup_error(self): # Global name does not exist obj = REX('spam') - obj.__module__ = __name__ + obj.__module__ = 'test.picklecommon' for proto in protocols: with self.subTest(proto=proto): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + f"Can't pickle {obj!r}: it's not found as test.picklecommon.spam") self.assertEqual(str(cm.exception.__context__), - f"module '{__name__}' has no attribute 'spam'") + "module 'test.picklecommon' has no attribute 'spam'") obj.__module__ = 'nonexisting' for proto in protocols: @@ -2426,7 +2427,7 @@ def persistent_id(self, obj): def test_bad_ext_code(self): # This should never happen in normal circumstances, because the type # and the value of the extension code is checked in copyreg.add_extension(). - key = (__name__, 'MyList') + key = (MyList.__module__, 'MyList') def check(code, exc): assert key not in copyreg._extension_registry assert code not in copyreg._inverted_registry @@ -3246,14 +3247,14 @@ def test_newobj_not_class(self): def produce_global_ext(self, extcode, opcode): e = ExtensionSaver(extcode) try: - copyreg.add_extension(__name__, "MyList", extcode) + copyreg.add_extension(MyList.__module__, "MyList", extcode) x = MyList([1, 2, 3]) x.foo = 42 x.bar = "hello" # Dump using protocol 1 for comparison. s1 = self.dumps(x, 1) - self.assertIn(__name__.encode("utf-8"), s1) + self.assertIn(MyList.__module__.encode(), s1) self.assertIn(b"MyList", s1) self.assertFalse(opcode_in_pickle(opcode, s1)) @@ -3262,7 +3263,7 @@ def produce_global_ext(self, extcode, opcode): # Dump using protocol 2 for test. s2 = self.dumps(x, 2) - self.assertNotIn(__name__.encode("utf-8"), s2) + self.assertNotIn(MyList.__module__.encode(), s2) self.assertNotIn(b"MyList", s2) self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) @@ -3762,12 +3763,6 @@ def concatenate_chunks(self): chunk_sizes) def test_nested_names(self): - global Nested - class Nested: - class A: - class B: - class C: - pass for proto in range(pickle.HIGHEST_PROTOCOL + 1): for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: with self.subTest(proto=proto, obj=obj): @@ -3799,31 +3794,6 @@ class Recursive: del Recursive.ref # break reference loop def test_py_methods(self): - global PyMethodsTest - class PyMethodsTest: - @staticmethod - def cheese(): - return "cheese" - @classmethod - def wine(cls): - assert cls is PyMethodsTest - return "wine" - def biscuits(self): - assert isinstance(self, PyMethodsTest) - return "biscuits" - class Nested: - "Nested class" - @staticmethod - def ketchup(): - return "ketchup" - @classmethod - def maple(cls): - assert cls is PyMethodsTest.Nested - return "maple" - def pie(self): - assert isinstance(self, PyMethodsTest.Nested) - return "pie" - py_methods = ( PyMethodsTest.cheese, PyMethodsTest.wine, @@ -3857,11 +3827,6 @@ def pie(self): self.assertRaises(TypeError, self.dumps, descr, proto) def test_c_methods(self): - global Subclass - class Subclass(tuple): - class Nested(str): - pass - c_methods = ( # bound built-in method ("abcd".index, ("c",)), @@ -4256,10 +4221,23 @@ def __reduce__(self): UNPICKLEABLE = Unpickleable() +# For test_unpickleable_reconstructor and test_unpickleable_state_setter class UnpickleableCallable(Unpickleable): def __call__(self, *args, **kwargs): pass +# For test_bad_getattr +class BadGetattr: + def __getattr__(self, key): + self.foo + +# For test_bad_newobj_class and test_bad_newobj_ex__class +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + class AbstractPickleModuleTests: diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index d524acbfdcac01..42b2fb05b05506 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -187,9 +187,6 @@ def test_bytes(self): # These tests fail because they require classes from pickletester # which cannot be properly imported by the xpickle worker. - test_c_methods = None - test_py_methods = None - test_nested_names = None test_recursive_nested_names = None test_recursive_nested_names2 = None From 66a19ef9f7ddee5f3eca5d5128dbaa41b435f704 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Jan 2026 10:49:02 +0200 Subject: [PATCH 14/21] Enable more tests. --- Lib/test/picklecommon.py | 83 ------------------------------ Lib/test/pickletester.py | 107 +++++++++++++++++++++++++++++++++++---- Lib/test/test_xpickle.py | 35 ++++--------- 3 files changed, 109 insertions(+), 116 deletions(-) diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py index 26c428058e97f3..1df77f6c987fda 100644 --- a/Lib/test/picklecommon.py +++ b/Lib/test/picklecommon.py @@ -295,89 +295,6 @@ def _reconstruct(cls, obj): return cls(obj) -try: - import _testbuffer -except ImportError: - _testbuffer = None - -if _testbuffer is not None: - - class PicklableNDArray: - # A not-really-zero-copy picklable ndarray, as the ndarray() - # constructor doesn't allow for it - - zero_copy_reconstruct = False - - def __init__(self, *args, **kwargs): - self.array = _testbuffer.ndarray(*args, **kwargs) - - def __getitem__(self, idx): - cls = type(self) - new = cls.__new__(cls) - new.array = self.array[idx] - return new - - @property - def readonly(self): - return self.array.readonly - - @property - def c_contiguous(self): - return self.array.c_contiguous - - @property - def f_contiguous(self): - return self.array.f_contiguous - - def __eq__(self, other): - if not isinstance(other, PicklableNDArray): - return NotImplemented - return (other.array.format == self.array.format and - other.array.shape == self.array.shape and - other.array.strides == self.array.strides and - other.array.readonly == self.array.readonly and - other.array.tobytes() == self.array.tobytes()) - - def __ne__(self, other): - if not isinstance(other, PicklableNDArray): - return NotImplemented - return not (self == other) - - def __repr__(self): - return ("{name}(shape={array.shape}," - "strides={array.strides}, " - "bytes={array.tobytes()})").format( - name=type(self).__name__, array=self.array.shape) - - def __reduce_ex__(self, protocol): - if not self.array.contiguous: - raise NotImplementedError("Reconstructing a non-contiguous " - "ndarray does not seem possible") - ndarray_kwargs = {"shape": self.array.shape, - "strides": self.array.strides, - "format": self.array.format, - "flags": (0 if self.readonly - else _testbuffer.ND_WRITABLE)} - import pickle - pb = pickle.PickleBuffer(self.array) - if protocol >= 5: - return (type(self)._reconstruct, - (pb, ndarray_kwargs)) - else: - # Need to serialize the bytes in physical order - with pb.raw() as m: - return (type(self)._reconstruct, - (m.tobytes(), ndarray_kwargs)) - - @classmethod - def _reconstruct(cls, obj, kwargs): - with memoryview(obj) as m: - # For some reason, ndarray() wants a list of integers... - # XXX This only works if format == 'B' - items = list(m.tobytes()) - return cls(items, **kwargs) - - # For test_nested_names class Nested: class A: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 79df7b82840709..dca522980ad629 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -158,6 +158,83 @@ def create_dynamic_class(name, bases): return result +if _testbuffer is not None: + + class PicklableNDArray: + # A not-really-zero-copy picklable ndarray, as the ndarray() + # constructor doesn't allow for it + + zero_copy_reconstruct = False + + def __init__(self, *args, **kwargs): + self.array = _testbuffer.ndarray(*args, **kwargs) + + def __getitem__(self, idx): + cls = type(self) + new = cls.__new__(cls) + new.array = self.array[idx] + return new + + @property + def readonly(self): + return self.array.readonly + + @property + def c_contiguous(self): + return self.array.c_contiguous + + @property + def f_contiguous(self): + return self.array.f_contiguous + + def __eq__(self, other): + if not isinstance(other, PicklableNDArray): + return NotImplemented + return (other.array.format == self.array.format and + other.array.shape == self.array.shape and + other.array.strides == self.array.strides and + other.array.readonly == self.array.readonly and + other.array.tobytes() == self.array.tobytes()) + + def __ne__(self, other): + if not isinstance(other, PicklableNDArray): + return NotImplemented + return not (self == other) + + def __repr__(self): + return ("{name}(shape={array.shape}," + "strides={array.strides}, " + "bytes={array.tobytes()})").format( + name=type(self).__name__, array=self.array.shape) + + def __reduce_ex__(self, protocol): + if not self.array.contiguous: + raise NotImplementedError("Reconstructing a non-contiguous " + "ndarray does not seem possible") + ndarray_kwargs = {"shape": self.array.shape, + "strides": self.array.strides, + "format": self.array.format, + "flags": (0 if self.readonly + else _testbuffer.ND_WRITABLE)} + pb = pickle.PickleBuffer(self.array) + if protocol >= 5: + return (type(self)._reconstruct, + (pb, ndarray_kwargs)) + else: + # Need to serialize the bytes in physical order + with pb.raw() as m: + return (type(self)._reconstruct, + (m.tobytes(), ndarray_kwargs)) + + @classmethod + def _reconstruct(cls, obj, kwargs): + with memoryview(obj) as m: + # For some reason, ndarray() wants a list of integers... + # XXX This only works if format == 'B' + items = list(m.tobytes()) + return cls(items, **kwargs) + + # DATA0 .. DATA4 are the pickles we expect under the various protocols, for # the object returned by create_data(). @@ -3082,6 +3159,8 @@ def test_proto(self): self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) def test_bad_proto(self): + if self.py_version < (3, 8): + self.skipTest('No protocol validation in this version') oob = protocols[-1] + 1 # a future protocol build_none = pickle.NONE + pickle.STOP badpickle = pickle.PROTO + bytes([oob]) + build_none @@ -3363,7 +3442,10 @@ def test_simple_newobj(self): with self.subTest(proto=proto): s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nI64206', s) # INT + if self.py_version >= (3, 7): + self.assertIn(b'\nI64206', s) # INT + else: # for test_xpickle + self.assertIn(b'64206', s) # INT or LONG else: self.assertIn(b'M\xce\xfa', s) # BININT2 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), @@ -3379,7 +3461,10 @@ def test_complex_newobj(self): with self.subTest(proto=proto): s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nI64206', s) # INT + if self.py_version >= (3, 7): + self.assertIn(b'\nI64206', s) # INT + else: # for test_xpickle + self.assertIn(b'64206', s) # INT or LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: @@ -3395,11 +3480,14 @@ def test_complex_newobj(self): def test_complex_newobj_ex(self): x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ x.abc = 666 - for proto in protocols: + for proto in protocols if self.py_version >= (3, 6) else protocols[4:]: with self.subTest(proto=proto): s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nI64206', s) # INT + if self.py_version >= (3, 7): + self.assertIn(b'\nI64206', s) # INT + else: # for test_xpickle + self.assertIn(b'64206', s) # INT or LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: @@ -3648,11 +3736,12 @@ def test_framing_large_objects(self): [len(x) for x in unpickled]) # Perform full equality check if the lengths match. self.assertEqual(obj, unpickled) - n_frames = count_opcode(pickle.FRAME, pickled) - # A single frame for small objects between - # first two large objects. - self.assertEqual(n_frames, 1) - self.check_frame_opcodes(pickled) + if self.py_version >= (3, 7): + n_frames = count_opcode(pickle.FRAME, pickled) + # A single frame for small objects between + # first two large objects. + self.assertEqual(n_frames, 1) + self.check_frame_opcodes(pickled) def test_optional_frames(self): if pickle.HIGHEST_PROTOCOL < 4: diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 42b2fb05b05506..a4b52698a78820 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -82,6 +82,7 @@ def have_python_version(py_version): return py_executable_map.get(py_version, None) +@support.requires_resource('cpu') class AbstractCompatTests(pickletester.AbstractPickleTests): py_version = None _OLD_HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL @@ -197,6 +198,16 @@ def test_bytes(self): # Expected exception is raised during unpickling in a subprocess. test_pickle_setstate_None = None + # Other Python version may not have NumPy. + test_buffers_numpy = None + + # Skip tests that require buffer_callback arguments since + # there isn't a reliable way to marshal/pickle the callback and ensure + # it works in a different Python version. + test_in_band_buffers = None + test_buffers_error = None + test_oob_buffers = None + test_oob_buffers_writable_to_readonly = None class PyPicklePythonCompat(AbstractCompatTests): pickler = pickle._Pickler @@ -208,32 +219,8 @@ class CPicklePythonCompat(AbstractCompatTests): unpickler = _pickle.Unpickler -skip_tests = { - (3, 6): [ - # This version has changes in framing using protocol 4 - 'test_framing_large_objects', - - # These fail for protocol 0 - 'test_simple_newobj', - 'test_complex_newobj', - 'test_complex_newobj_ex', - ], - (3, 7): [ - # This version does not support buffers - 'test_in_band_buffers', - - # No protocol validation in this version - 'test_bad_proto', - ], -} - - def make_test(py_version, base): class_dict = {'py_version': py_version} - for key, value in skip_tests.items(): - if py_version <= key: - for test_name in value: - class_dict[test_name] = None name = base.__name__.replace('Python', 'Python%d%d' % py_version) return type(name, (base, unittest.TestCase), class_dict) From 3c75f488d0509d3745e99c75927f4902a1dcfb50 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Jan 2026 10:55:25 +0200 Subject: [PATCH 15/21] Use enterClassContext(). --- Lib/test/test_xpickle.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index a4b52698a78820..8bcc660cbd5ec6 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -85,7 +85,6 @@ def have_python_version(py_version): @support.requires_resource('cpu') class AbstractCompatTests(pickletester.AbstractPickleTests): py_version = None - _OLD_HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL @classmethod def setUpClass(cls): @@ -96,14 +95,10 @@ def setUpClass(cls): # Override the default pickle protocol to match what xpickle worker # will be running. highest_protocol = highest_proto_for_py_version(cls.py_version) - pickletester.protocols = range(highest_protocol + 1) - pickle.HIGHEST_PROTOCOL = highest_protocol - - @classmethod - def tearDownClass(cls): - # Set the highest protocol back to the default. - pickle.HIGHEST_PROTOCOL = cls._OLD_HIGHEST_PROTOCOL - pickletester.protocols = range(pickle.HIGHEST_PROTOCOL + 1) + cls.enterClassContext(support.swap_attr(pickletester, 'protocols', + range(highest_protocol + 1))) + cls.enterClassContext(support.swap_attr(pickle, 'HIGHEST_PROTOCOL', + highest_protocol)) @staticmethod def send_to_worker(python, data): From f4758a40029fce86bf88d39fd8f9e46067d17f7c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Jan 2026 14:01:44 +0200 Subject: [PATCH 16/21] Support Python 3.2+. --- Lib/test/pickletester.py | 158 +++++++++++++++++++++++++++++-------- Lib/test/test_xpickle.py | 4 +- Lib/test/xpickle_worker.py | 15 +++- 3 files changed, 136 insertions(+), 41 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index dca522980ad629..3340ecd0d936b7 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2800,7 +2800,7 @@ def test_recursive_multi(self): self.assertEqual(list(x[0].attr.keys()), [1]) self.assertIs(x[0].attr[1], x) - def _test_recursive_collection_and_inst(self, factory): + def _test_recursive_collection_and_inst(self, factory, oldminproto=0): # Mutable object containing a collection containing the original # object. o = Object() @@ -2818,6 +2818,8 @@ def _test_recursive_collection_and_inst(self, factory): # collection. o = o.attr for proto in protocols: + if self.py_version < (3, 4) and proto < oldminproto: + continue s = self.dumps(o, proto) x = self.loads(s) self.assertIsInstance(x, t) @@ -2835,25 +2837,25 @@ def test_recursive_dict_and_inst(self): self._test_recursive_collection_and_inst(dict.fromkeys) def test_recursive_set_and_inst(self): - self._test_recursive_collection_and_inst(set) + self._test_recursive_collection_and_inst(set, oldminproto=4) def test_recursive_frozenset_and_inst(self): - self._test_recursive_collection_and_inst(frozenset) + self._test_recursive_collection_and_inst(frozenset, oldminproto=4) def test_recursive_list_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyList) + self._test_recursive_collection_and_inst(MyList, oldminproto=2) def test_recursive_tuple_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyTuple) + self._test_recursive_collection_and_inst(MyTuple, oldminproto=4) def test_recursive_dict_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyDict.fromkeys) + self._test_recursive_collection_and_inst(MyDict.fromkeys, oldminproto=2) def test_recursive_set_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MySet) + self._test_recursive_collection_and_inst(MySet, oldminproto=4) def test_recursive_frozenset_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyFrozenSet) + self._test_recursive_collection_and_inst(MyFrozenSet, oldminproto=4) def test_recursive_inst_state(self): # Mutable object containing itself. @@ -2926,8 +2928,11 @@ def test_bytes(self): self.assert_is_copy(s, self.loads(p)) def test_bytes_memoization(self): + array_types = [bytes] + if self.py_version >= (3, 4): + array_types += [ZeroCopyBytes] for proto in protocols: - for array_type in [bytes, ZeroCopyBytes]: + for array_type in array_types: for s in b'', b'xyz', b'xyz'*100: with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): b = array_type(s) @@ -2964,8 +2969,11 @@ def test_bytearray(self): self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) def test_bytearray_memoization(self): + array_types = [bytearray] + if self.py_version >= (3, 4): + array_types += [ZeroCopyBytearray] for proto in protocols: - for array_type in [bytearray, ZeroCopyBytearray]: + for array_type in array_types: for s in b'', b'xyz', b'xyz'*100: with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): b = array_type(s) @@ -3039,10 +3047,13 @@ def test_float_format(self): def test_reduce(self): for proto in protocols: - inst = AAA() - dumped = self.dumps(inst, proto) - loaded = self.loads(dumped) - self.assertEqual(loaded, REDUCE_A) + if self.py_version < (3, 4) and proto < 3: + continue + with self.subTest(proto=proto): + inst = AAA() + dumped = self.dumps(inst, proto) + loaded = self.loads(dumped) + self.assertEqual(loaded, REDUCE_A) def test_getinitargs(self): for proto in protocols: @@ -3076,6 +3087,8 @@ def test_structseq(self): s = self.dumps(t, proto) u = self.loads(s) self.assert_is_copy(t, u) + if self.py_version < (3, 4): + continue t = os.stat(os.curdir) s = self.dumps(t, proto) u = self.loads(s) @@ -3087,12 +3100,16 @@ def test_structseq(self): self.assert_is_copy(t, u) def test_ellipsis(self): + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') for proto in protocols: s = self.dumps(..., proto) u = self.loads(s) self.assertIs(..., u) def test_notimplemented(self): + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') for proto in protocols: s = self.dumps(NotImplemented, proto) u = self.loads(s) @@ -3100,6 +3117,8 @@ def test_notimplemented(self): def test_singleton_types(self): # Issue #6477: Test that types of built-in singletons can be pickled. + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') singletons = [None, ..., NotImplemented] for singleton in singletons: for proto in protocols: @@ -3110,12 +3129,34 @@ def test_singleton_types(self): def test_builtin_types(self): for t in builtins.__dict__.values(): if isinstance(t, type) and not issubclass(t, BaseException): + if t is str and self.py_version < (3, 4): + continue + if t.__name__ == 'BuiltinImporter' and self.py_version < (3, 3): + continue for proto in protocols: - s = self.dumps(t, proto) - self.assertIs(self.loads(s), t) + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) def test_builtin_exceptions(self): new_names = { + 'BlockingIOError': (3, 3), + 'BrokenPipeError': (3, 3), + 'ChildProcessError': (3, 3), + 'ConnectionError': (3, 3), + 'ConnectionAbortedError': (3, 3), + 'ConnectionRefusedError': (3, 3), + 'ConnectionResetError': (3, 3), + 'FileExistsError': (3, 3), + 'FileNotFoundError': (3, 3), + 'InterruptedError': (3, 3), + 'IsADirectoryError': (3, 3), + 'NotADirectoryError': (3, 3), + 'PermissionError': (3, 3), + 'ProcessLookupError': (3, 3), + 'TimeoutError': (3, 3), + 'RecursionError': (3, 5), + 'StopAsyncIteration': (3, 5), 'ModuleNotFoundError': (3, 6), 'EncodingWarning': (3, 10), 'BaseExceptionGroup': (3, 11), @@ -3128,14 +3169,17 @@ def test_builtin_exceptions(self): if t.__name__ in new_names and self.py_version < new_names[t.__name__]: continue for proto in protocols: - s = self.dumps(t, proto) - u = self.loads(s) - if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: - self.assertIs(u, OSError) - elif proto <= 2 and issubclass(t, ImportError): - self.assertIs(u, ImportError) - else: - self.assertIs(u, t) + if self.py_version < (3, 3) and proto < 3: + continue + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) def test_builtin_functions(self): new_names = {'breakpoint': (3, 7), 'aiter': (3, 10), 'anext': (3, 10)} @@ -3144,8 +3188,9 @@ def test_builtin_functions(self): if t.__name__ in new_names and self.py_version < new_names[t.__name__]: continue for proto in protocols: - s = self.dumps(t, proto) - self.assertIs(self.loads(s), t) + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) # Tests for protocol 2 @@ -3160,7 +3205,7 @@ def test_proto(self): def test_bad_proto(self): if self.py_version < (3, 8): - self.skipTest('No protocol validation in this version') + self.skipTest('no protocol validation in Python < 3.8') oob = protocols[-1] + 1 # a future protocol build_none = pickle.NONE + pickle.STOP badpickle = pickle.PROTO + bytes([oob]) + build_none @@ -3272,6 +3317,8 @@ def test_newobj_list(self): def test_newobj_generic(self): for proto in protocols: for C in myclasses: + if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode): + continue B = C.__base__ x = C(C.sample) x.foo = 42 @@ -3290,6 +3337,8 @@ def test_newobj_proxies(self): classes.remove(c) for proto in protocols: for C in classes: + if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode): + continue B = C.__base__ x = C(C.sample) x.foo = 42 @@ -3314,6 +3363,8 @@ def test_newobj_overridden_new(self): def test_newobj_not_class(self): # Issue 24552 + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') o = SimpleNewObj.__new__(SimpleNewObj) b = self.dumps(o, 4) with support.swap_attr(picklecommon, 'SimpleNewObj', 42): @@ -3448,9 +3499,10 @@ def test_simple_newobj(self): self.assertIn(b'64206', s) # INT or LONG else: self.assertIn(b'M\xce\xfa', s) # BININT2 - self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), - 2 <= proto) - self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) + if not (self.py_version < (3, 5) and proto == 4): + self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), + 2 <= proto) + self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) y = self.loads(s) # will raise TypeError if __init__ called self.assert_is_copy(x, y) @@ -3471,9 +3523,10 @@ def test_complex_newobj(self): self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE else: self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE - self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), - 2 <= proto) - self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) + if not (self.py_version < (3, 5) and proto == 4): + self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), + 2 <= proto) + self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) y = self.loads(s) # will raise TypeError if __init__ called self.assert_is_copy(x, y) @@ -3608,6 +3661,8 @@ def test_large_pickles(self): def test_int_pickling_efficiency(self): # Test compacity of int representation (see issue #12744) + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') for proto in protocols: with self.subTest(proto=proto): pickles = [self.dumps(2**n, proto) for n in range(70)] @@ -3852,7 +3907,12 @@ def concatenate_chunks(self): chunk_sizes) def test_nested_names(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') + # required protocol 4 in Python 3.4 for proto in range(pickle.HIGHEST_PROTOCOL + 1): + if self.py_version < (3, 5) and proto < 4: + continue for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: with self.subTest(proto=proto, obj=obj): unpickled = self.loads(self.dumps(obj, proto)) @@ -3883,10 +3943,21 @@ class Recursive: del Recursive.ref # break reference loop def test_py_methods(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') py_methods = ( - PyMethodsTest.cheese, PyMethodsTest.wine, PyMethodsTest().biscuits, + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for method in py_methods: + with self.subTest(proto=proto, method=method): + unpickled = self.loads(self.dumps(method, proto)) + self.assertEqual(method(), unpickled()) + + # required protocol 4 in Python 3.4 + py_methods = ( + PyMethodsTest.cheese, PyMethodsTest.Nested.ketchup, PyMethodsTest.Nested.maple, PyMethodsTest.Nested().pie @@ -3896,6 +3967,8 @@ def test_py_methods(self): (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): + if self.py_version < (3, 5) and proto < 4: + continue for method in py_methods: with self.subTest(proto=proto, method=method): unpickled = self.loads(self.dumps(method, proto)) @@ -3916,6 +3989,8 @@ def test_py_methods(self): self.assertRaises(TypeError, self.dumps, descr, proto) def test_c_methods(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') c_methods = ( # bound built-in method ("abcd".index, ("c",)), @@ -3936,7 +4011,6 @@ def test_c_methods(self): # subclass methods (Subclass([1,2,2]).count, (2,)), (Subclass.count, (Subclass([1,2,2]), 2)), - (Subclass.Nested("sweet").count, ("e",)), (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -3945,6 +4019,18 @@ def test_c_methods(self): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(*args), unpickled(*args)) + # required protocol 4 in Python 3.4 + c_methods = ( + (Subclass.Nested("sweet").count, ("e",)), + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + if self.py_version < (3, 5) and proto < 4: + continue + for method, args in c_methods: + with self.subTest(proto=proto, method=method): + unpickled = self.loads(self.dumps(method, proto)) + self.assertEqual(method(*args), unpickled(*args)) + descriptors = ( bytearray.__dict__['maketrans'], # built-in static method descriptor dict.__dict__['fromkeys'], # built-in class method descriptor @@ -3955,6 +4041,8 @@ def test_c_methods(self): self.assertRaises(TypeError, self.dumps, descr, proto) def test_compat_pickle(self): + if self.py_version < (3, 4): + self.skipTest("doesn't work in Python < 3.4'") tests = [ (range(1, 7), '__builtin__', 'xrange'), (map(int, '123'), 'itertools', 'imap'), diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 8bcc660cbd5ec6..94cb58dd3d49fe 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -1,5 +1,5 @@ # This test covers backwards compatibility with -# previous version of Python by bouncing pickled objects through Python 3.5 +# previous version of Python by bouncing pickled objects through Python 3.2 # and the current version by running xpickle_worker.py. import io import os @@ -222,7 +222,7 @@ def make_test(py_version, base): def load_tests(loader, tests, pattern): major = sys.version_info.major assert major == 3 - for minor in range(5, sys.version_info.minor): + for minor in range(2, sys.version_info.minor): test_class = make_test((major, minor), PyPicklePythonCompat) tests.addTest(loader.loadTestsFromTestCase(test_class)) if has_c_implementation: diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py index e55b24405696fc..6192ebf88f2d53 100644 --- a/Lib/test/xpickle_worker.py +++ b/Lib/test/xpickle_worker.py @@ -10,10 +10,17 @@ # since some of the pickle objects hold references to picklecommon.py. test_mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'picklecommon.py')) -spec = importlib.util.spec_from_file_location('test.picklecommon', test_mod_path) -test_module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(test_module) -sys.modules['test.picklecommon'] = test_module +if sys.version_info >= (3, 5): + spec = importlib.util.spec_from_file_location('test.picklecommon', test_mod_path) + test_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(test_module) + sys.modules['test.picklecommon'] = test_module +else: + test_module = type(sys)('test.picklecommon') + sys.modules['test.picklecommon'] = test_module + with open(test_mod_path, 'rb') as f: + sources = f.read() + exec(sources, vars(test_module)) in_stream = sys.stdin.buffer From d027d776962cba350bf522a31384a2dc48654576 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Jan 2026 14:28:32 +0200 Subject: [PATCH 17/21] Fix lint error. --- Lib/test/picklecommon.py | 2 +- Lib/test/pickletester.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py index 1df77f6c987fda..81e4e96a5c3192 100644 --- a/Lib/test/picklecommon.py +++ b/Lib/test/picklecommon.py @@ -93,7 +93,7 @@ def __reduce_ex__(self, proto): self._proto = proto return REX_two, () def __reduce__(self): - raise TestFailed("This __reduce__ shouldn't be called") + raise AssertionError("This __reduce__ shouldn't be called") class REX_four(object): """Calling base class method should succeed""" diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 3340ecd0d936b7..55e3937fced6e8 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -27,7 +27,7 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locales, no_tracing, + run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget From 540170367bb05b4c5ff6dd2b3b9ec6be4f1d3782 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Jan 2026 18:09:55 +0200 Subject: [PATCH 18/21] Silence lint warning. --- Lib/test/picklecommon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py index 81e4e96a5c3192..faf6c322774cdd 100644 --- a/Lib/test/picklecommon.py +++ b/Lib/test/picklecommon.py @@ -331,4 +331,3 @@ def pie(self): class Subclass(tuple): class Nested(str): pass - From 21e5130dbfd37081b37f3d0d84d73837ff26eedd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 5 Jan 2026 12:33:32 +0200 Subject: [PATCH 19/21] Support Python 2.7. --- Lib/test/picklecommon.py | 26 +++- Lib/test/pickletester.py | 258 +++++++++++++++++++++++-------------- Lib/test/test_xpickle.py | 21 +-- Lib/test/xpickle_worker.py | 6 +- 4 files changed, 200 insertions(+), 111 deletions(-) diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py index faf6c322774cdd..fece50f17fecca 100644 --- a/Lib/test/picklecommon.py +++ b/Lib/test/picklecommon.py @@ -2,6 +2,8 @@ # They are moved to separate file, so they can be loaded # in other Python version for test_xpickle. +import sys + class C: def __eq__(self, other): return self.__dict__ == other.__dict__ @@ -55,8 +57,15 @@ def __getinitargs__(self): class metaclass(type): pass +if sys.version_info >= (3,): + # Syntax not compatible with Python 2 + exec(''' class use_metaclass(object, metaclass=metaclass): pass +''') +else: + class use_metaclass(object): + __metaclass__ = metaclass # Test classes for reduce_ex @@ -174,6 +183,13 @@ def __getstate__(self): class MyInt(int): sample = 1 +if sys.version_info >= (3,): + class MyLong(int): + sample = 1 +else: + class MyLong(long): + sample = long(1) + class MyFloat(float): sample = 1.0 @@ -183,8 +199,12 @@ class MyComplex(complex): class MyStr(str): sample = "hello" -class MyUnicode(str): - sample = "hello \u1234" +if sys.version_info >= (3,): + class MyUnicode(str): + sample = "hello \u1234" +else: + class MyUnicode(unicode): + sample = unicode(r"hello \u1234", "raw-unicode-escape") class MyTuple(tuple): sample = (1, 2, 3) @@ -201,7 +221,7 @@ class MySet(set): class MyFrozenSet(frozenset): sample = frozenset({"a", "b"}) -myclasses = [MyInt, MyFloat, +myclasses = [MyInt, MyLong, MyFloat, MyComplex, MyStr, MyUnicode, MyTuple, MyList, MyDict, MySet, MyFrozenSet] diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 55e3937fced6e8..5179b7d16ad614 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2539,24 +2539,33 @@ def setUp(self): def test_misc(self): # test various datatypes not tested by testdata for proto in protocols: - x = myint(4) - s = self.dumps(x, proto) - y = self.loads(s) - self.assert_is_copy(x, y) + with self.subTest('myint', proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') + x = myint(4) + s = self.dumps(x, proto) + y = self.loads(s) + self.assert_is_copy(x, y) - x = (1, ()) - s = self.dumps(x, proto) - y = self.loads(s) - self.assert_is_copy(x, y) + with self.subTest('tuple', proto=proto): + x = (1, ()) + s = self.dumps(x, proto) + y = self.loads(s) + self.assert_is_copy(x, y) - x = initarg(1, x) - s = self.dumps(x, proto) - y = self.loads(s) - self.assert_is_copy(x, y) + with self.subTest('initarg', proto=proto): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') + x = initarg(1, x) + s = self.dumps(x, proto) + y = self.loads(s) + self.assert_is_copy(x, y) # XXX test __reduce__ protocol? def test_roundtrip_equality(self): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') expected = self._testdata for proto in protocols: s = self.dumps(expected, proto) @@ -2776,6 +2785,8 @@ def test_recursive_set(self): def test_recursive_inst(self): # Mutable object containing itself. + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') i = Object() i.attr = i for proto in protocols: @@ -2786,6 +2797,8 @@ def test_recursive_inst(self): self.assertIs(x.attr, x) def test_recursive_multi(self): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') l = [] d = {1:l} i = Object() @@ -2800,62 +2813,70 @@ def test_recursive_multi(self): self.assertEqual(list(x[0].attr.keys()), [1]) self.assertIs(x[0].attr[1], x) - def _test_recursive_collection_and_inst(self, factory, oldminproto=0): + def _test_recursive_collection_and_inst(self, factory, oldminproto=None): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') # Mutable object containing a collection containing the original # object. o = Object() o.attr = factory([o]) t = type(o.attr) - for proto in protocols: - s = self.dumps(o, proto) - x = self.loads(s) - self.assertIsInstance(x.attr, t) - self.assertEqual(len(x.attr), 1) - self.assertIsInstance(list(x.attr)[0], Object) - self.assertIs(list(x.attr)[0], x) + with self.subTest('obj -> {t.__name__} -> obj'): + for proto in protocols: + with self.subTest(proto=proto): + s = self.dumps(o, proto) + x = self.loads(s) + self.assertIsInstance(x.attr, t) + self.assertEqual(len(x.attr), 1) + self.assertIsInstance(list(x.attr)[0], Object) + self.assertIs(list(x.attr)[0], x) # Collection containing a mutable object containing the original # collection. o = o.attr - for proto in protocols: - if self.py_version < (3, 4) and proto < oldminproto: - continue - s = self.dumps(o, proto) - x = self.loads(s) - self.assertIsInstance(x, t) - self.assertEqual(len(x), 1) - self.assertIsInstance(list(x)[0], Object) - self.assertIs(list(x)[0].attr, x) + with self.subTest(f'{t.__name__} -> obj -> {t.__name__}'): + if self.py_version < (3, 4) and oldminproto is None: + self.skipTest('not supported in Python < 3.4') + for proto in protocols: + with self.subTest(proto=proto): + if self.py_version < (3, 4) and proto < oldminproto: + self.skipTest('not supported in Python < 3.4') + s = self.dumps(o, proto) + x = self.loads(s) + self.assertIsInstance(x, t) + self.assertEqual(len(x), 1) + self.assertIsInstance(list(x)[0], Object) + self.assertIs(list(x)[0].attr, x) def test_recursive_list_and_inst(self): - self._test_recursive_collection_and_inst(list) + self._test_recursive_collection_and_inst(list, oldminproto=0) def test_recursive_tuple_and_inst(self): - self._test_recursive_collection_and_inst(tuple) + self._test_recursive_collection_and_inst(tuple, oldminproto=0) def test_recursive_dict_and_inst(self): - self._test_recursive_collection_and_inst(dict.fromkeys) + self._test_recursive_collection_and_inst(dict.fromkeys, oldminproto=0) def test_recursive_set_and_inst(self): - self._test_recursive_collection_and_inst(set, oldminproto=4) + self._test_recursive_collection_and_inst(set) def test_recursive_frozenset_and_inst(self): - self._test_recursive_collection_and_inst(frozenset, oldminproto=4) + self._test_recursive_collection_and_inst(frozenset) def test_recursive_list_subclass_and_inst(self): self._test_recursive_collection_and_inst(MyList, oldminproto=2) def test_recursive_tuple_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyTuple, oldminproto=4) + self._test_recursive_collection_and_inst(MyTuple) def test_recursive_dict_subclass_and_inst(self): self._test_recursive_collection_and_inst(MyDict.fromkeys, oldminproto=2) def test_recursive_set_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MySet, oldminproto=4) + self._test_recursive_collection_and_inst(MySet) def test_recursive_frozenset_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyFrozenSet, oldminproto=4) + self._test_recursive_collection_and_inst(MyFrozenSet) def test_recursive_inst_state(self): # Mutable object containing itself. @@ -2909,6 +2930,8 @@ def test_unicode_high_plane(self): def test_unicode_memoization(self): # Repeated str is re-used (even when escapes added). + if self.py_version < (3, 0): + self.skipTest('not supported in Python < 3.0') for proto in protocols: for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': p = self.dumps((s, s), proto) @@ -2934,20 +2957,21 @@ def test_bytes_memoization(self): for proto in protocols: for array_type in array_types: for s in b'', b'xyz', b'xyz'*100: + b = array_type(s) + expected = (b, b) if self.py_version >= (3, 0) else (b.decode(),)*2 with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): - b = array_type(s) p = self.dumps((b, b), proto) x, y = self.loads(p) self.assertIs(x, y) - self.assert_is_copy((b, b), (x, y)) + self.assert_is_copy(expected, (x, y)) + b2 = array_type(s) with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): - b1, b2 = array_type(s), array_type(s) - p = self.dumps((b1, b2), proto) - # Note that (b1, b2) = self.loads(p) might have identical - # components, i.e., b1 is b2, but this is not always the + p = self.dumps((b, b2), proto) + # Note that (b, b2) = self.loads(p) might have identical + # components, i.e., b is b2, but this is not always the # case if the content is large (equality still holds). - self.assert_is_copy((b1, b2), self.loads(p)) + self.assert_is_copy(expected, self.loads(p)) def test_bytearray(self): for proto in protocols: @@ -3047,15 +3071,17 @@ def test_float_format(self): def test_reduce(self): for proto in protocols: - if self.py_version < (3, 4) and proto < 3: - continue with self.subTest(proto=proto): + if self.py_version < (3, 4) and proto < 3: + self.skipTest('str is not interoperable with Python < 3.4') inst = AAA() dumped = self.dumps(inst, proto) loaded = self.loads(dumped) self.assertEqual(loaded, REDUCE_A) def test_getinitargs(self): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') for proto in protocols: inst = initarg(1, 2) dumped = self.dumps(inst, proto) @@ -3063,6 +3089,7 @@ def test_getinitargs(self): self.assert_is_copy(inst, loaded) def test_metaclass(self): + self.assertEqual(type(use_metaclass), metaclass) a = use_metaclass() for proto in protocols: s = self.dumps(a, proto) @@ -3088,6 +3115,8 @@ def test_structseq(self): u = self.loads(s) self.assert_is_copy(t, u) if self.py_version < (3, 4): + # module 'os' has no attributes '_make_stat_result' and + # '_make_statvfs_result' continue t = os.stat(os.curdir) s = self.dumps(t, proto) @@ -3103,17 +3132,19 @@ def test_ellipsis(self): if self.py_version < (3, 3): self.skipTest('not supported in Python < 3.3') for proto in protocols: - s = self.dumps(..., proto) - u = self.loads(s) - self.assertIs(..., u) + with self.subTest(proto=proto): + s = self.dumps(..., proto) + u = self.loads(s) + self.assertIs(..., u) def test_notimplemented(self): if self.py_version < (3, 3): self.skipTest('not supported in Python < 3.3') for proto in protocols: - s = self.dumps(NotImplemented, proto) - u = self.loads(s) - self.assertIs(NotImplemented, u) + with self.subTest(proto=proto): + s = self.dumps(NotImplemented, proto) + u = self.loads(s) + self.assertIs(NotImplemented, u) def test_singleton_types(self): # Issue #6477: Test that types of built-in singletons can be pickled. @@ -3121,17 +3152,22 @@ def test_singleton_types(self): self.skipTest('not supported in Python < 3.3') singletons = [None, ..., NotImplemented] for singleton in singletons: + t = type(singleton) for proto in protocols: - s = self.dumps(type(singleton), proto) - u = self.loads(s) - self.assertIs(type(singleton), u) + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + u = self.loads(s) + self.assertIs(t, u) def test_builtin_types(self): + new_names = { + 'bytes': (3, 0), + 'BuiltinImporter': (3, 3), + 'str': (3, 4), # not interoperable with Python < 3.4 + } for t in builtins.__dict__.values(): if isinstance(t, type) and not issubclass(t, BaseException): - if t is str and self.py_version < (3, 4): - continue - if t.__name__ == 'BuiltinImporter' and self.py_version < (3, 3): + if t.__name__ in new_names and self.py_version < new_names[t.__name__]: continue for proto in protocols: with self.subTest(name=t.__name__, proto=proto): @@ -3169,9 +3205,9 @@ def test_builtin_exceptions(self): if t.__name__ in new_names and self.py_version < new_names[t.__name__]: continue for proto in protocols: - if self.py_version < (3, 3) and proto < 3: - continue with self.subTest(name=t.__name__, proto=proto): + if self.py_version < (3, 3) and proto < 3: + self.skipTest('exception classes are not interoperable with Python < 3.3') s = self.dumps(t, proto) u = self.loads(s) if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: @@ -3182,7 +3218,14 @@ def test_builtin_exceptions(self): self.assertIs(u, t) def test_builtin_functions(self): - new_names = {'breakpoint': (3, 7), 'aiter': (3, 10), 'anext': (3, 10)} + new_names = { + '__build_class__': (3, 0), + 'ascii': (3, 0), + 'exec': (3, 0), + 'breakpoint': (3, 7), + 'aiter': (3, 10), + 'anext': (3, 10), + } for t in builtins.__dict__.values(): if isinstance(t, types.BuiltinFunctionType): if t.__name__ in new_names and self.py_version < new_names[t.__name__]: @@ -3317,49 +3360,56 @@ def test_newobj_list(self): def test_newobj_generic(self): for proto in protocols: for C in myclasses: - if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode): - continue - B = C.__base__ - x = C(C.sample) - x.foo = 42 - s = self.dumps(x, proto) - y = self.loads(s) - detail = (proto, C, B, x, y, type(y)) - self.assert_is_copy(x, y) # XXX revisit - self.assertEqual(B(x), B(y), detail) - self.assertEqual(x.__dict__, y.__dict__, detail) + with self.subTest(proto=proto, C=C): + if self.py_version < (3, 0) and proto < 2 and C in (MyInt, MyStr): + self.skipTest('int and str subclasses are not interoperable with Python 2') + if (3, 0) <= self.py_version < (3, 4) and proto < 2 and C in (MyStr, MyUnicode): + self.skipTest('str subclasses are not interoperable with Python < 3.4') + B = C.__base__ + x = C(C.sample) + x.foo = 42 + s = self.dumps(x, proto) + y = self.loads(s) + detail = (proto, C, B, x, y, type(y)) + self.assert_is_copy(x, y) # XXX revisit + self.assertEqual(B(x), B(y), detail) + self.assertEqual(x.__dict__, y.__dict__, detail) def test_newobj_proxies(self): # NEWOBJ should use the __class__ rather than the raw type classes = myclasses[:] # Cannot create weakproxies to these classes - for c in (MyInt, MyTuple): + for c in (MyInt, MyLong, MyTuple): classes.remove(c) for proto in protocols: for C in classes: - if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode): - continue - B = C.__base__ - x = C(C.sample) - x.foo = 42 - p = weakref.proxy(x) - s = self.dumps(p, proto) - y = self.loads(s) - self.assertEqual(type(y), type(x)) # rather than type(p) - detail = (proto, C, B, x, y, type(y)) - self.assertEqual(B(x), B(y), detail) - self.assertEqual(x.__dict__, y.__dict__, detail) + with self.subTest(proto=proto, C=C): + if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode): + self.skipTest('str subclasses are not interoperable with Python < 3.4') + B = C.__base__ + x = C(C.sample) + x.foo = 42 + p = weakref.proxy(x) + s = self.dumps(p, proto) + y = self.loads(s) + self.assertEqual(type(y), type(x)) # rather than type(p) + detail = (proto, C, B, x, y, type(y)) + self.assertEqual(B(x), B(y), detail) + self.assertEqual(x.__dict__, y.__dict__, detail) def test_newobj_overridden_new(self): # Test that Python class with C implemented __new__ is pickleable for proto in protocols: - x = MyIntWithNew2(1) - x.foo = 42 - s = self.dumps(x, proto) - y = self.loads(s) - self.assertIs(type(y), MyIntWithNew2) - self.assertEqual(int(y), 1) - self.assertEqual(y.foo, 42) + with self.subTest(proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') + x = MyIntWithNew2(1) + x.foo = 42 + s = self.dumps(x, proto) + y = self.loads(s) + self.assertIs(type(y), MyIntWithNew2) + self.assertEqual(int(y), 1) + self.assertEqual(y.foo, 42) def test_newobj_not_class(self): # Issue 24552 @@ -3491,6 +3541,8 @@ def test_simple_newobj(self): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') s = self.dumps(x, proto) if proto < 1: if self.py_version >= (3, 7): @@ -3511,6 +3563,8 @@ def test_complex_newobj(self): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') s = self.dumps(x, proto) if proto < 1: if self.py_version >= (3, 7): @@ -3520,7 +3574,10 @@ def test_complex_newobj(self): elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: - self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE + if self.py_version >= (3, 0): + self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE + else: # for test_xpickle + self.assertIn(b'U\x04FACE', s) # SHORT_BINSTRING else: self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE if not (self.py_version < (3, 5) and proto == 4): @@ -3628,6 +3685,8 @@ def test_many_puts_and_gets(self): def test_attribute_name_interning(self): # Test that attribute names of pickled objects are interned when # unpickling. + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') for proto in protocols: x = C() x.foo = 42 @@ -3657,6 +3716,8 @@ def test_large_pickles(self): dumped = self.dumps(data, proto) loaded = self.loads(dumped) self.assertEqual(len(loaded), len(data)) + if self.py_version < (3, 0): + data = (1, min, 'xy' * (30 * 1024), len) self.assertEqual(loaded, data) def test_int_pickling_efficiency(self): @@ -3683,10 +3744,13 @@ def test_appends_on_non_lists(self): # Issue #17720 obj = REX_six([1, 2, 3]) for proto in protocols: - if proto == 0: - self._check_pickling_with_opcode(obj, pickle.APPEND, proto) - else: - self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) + with self.subTest(proto=proto): + if proto == 0: + self._check_pickling_with_opcode(obj, pickle.APPEND, proto) + else: + if self.py_version < (3, 0): + self.skipTest('not supported in Python < 3.0') + self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) def test_setitems_on_non_dicts(self): obj = REX_seven({1: -1, 2: -2, 3: -3}) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py index 94cb58dd3d49fe..459944afd20ef7 100644 --- a/Lib/test/test_xpickle.py +++ b/Lib/test/test_xpickle.py @@ -1,6 +1,5 @@ -# This test covers backwards compatibility with -# previous version of Python by bouncing pickled objects through Python 3.2 -# and the current version by running xpickle_worker.py. +# This test covers backwards compatibility with previous versions of Python +# by bouncing pickled objects through Python versions by running xpickle_worker.py. import io import os import pickle @@ -164,6 +163,8 @@ def loads(self, buf, **kwds): # much faster (the one from pickletester takes 3-4 minutes when running # under text_xpickle). def test_bytes(self): + if self.py_version < (3, 0): + self.skipTest('not supported in Python < 3.0') for proto in pickletester.protocols: for s in b'', b'xyz', b'xyz'*100: p = self.dumps(s, proto) @@ -220,14 +221,18 @@ def make_test(py_version, base): return type(name, (base, unittest.TestCase), class_dict) def load_tests(loader, tests, pattern): - major = sys.version_info.major - assert major == 3 - for minor in range(2, sys.version_info.minor): - test_class = make_test((major, minor), PyPicklePythonCompat) + def add_tests(py_version): + test_class = make_test(py_version, PyPicklePythonCompat) tests.addTest(loader.loadTestsFromTestCase(test_class)) if has_c_implementation: - test_class = make_test((major, minor), CPicklePythonCompat) + test_class = make_test(py_version, CPicklePythonCompat) tests.addTest(loader.loadTestsFromTestCase(test_class)) + + major = sys.version_info.major + assert major == 3 + add_tests((2, 7)) + for minor in range(2, sys.version_info.minor): + add_tests((major, minor)) return tests diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py index 6192ebf88f2d53..3f27e3d48205b2 100644 --- a/Lib/test/xpickle_worker.py +++ b/Lib/test/xpickle_worker.py @@ -1,6 +1,5 @@ # This script is called by test_xpickle as a subprocess to load and dump # pickles in a different Python version. -import importlib.util import os import pickle import sys @@ -11,6 +10,7 @@ test_mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'picklecommon.py')) if sys.version_info >= (3, 5): + import importlib.util spec = importlib.util.spec_from_file_location('test.picklecommon', test_mod_path) test_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(test_module) @@ -23,8 +23,8 @@ exec(sources, vars(test_module)) -in_stream = sys.stdin.buffer -out_stream = sys.stdout.buffer +in_stream = getattr(sys.stdin, 'buffer', sys.stdin) +out_stream = getattr(sys.stdout, 'buffer', sys.stdout) try: message = pickle.load(in_stream) From 1f7038caf2e2ce94d4aad73d0a18d11b389de5c5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 6 Jan 2026 10:10:52 +0200 Subject: [PATCH 20/21] Fix RuntimeWarning in 2.7. --- Lib/test/xpickle_worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py index 3f27e3d48205b2..03191372bf7ac8 100644 --- a/Lib/test/xpickle_worker.py +++ b/Lib/test/xpickle_worker.py @@ -18,6 +18,7 @@ else: test_module = type(sys)('test.picklecommon') sys.modules['test.picklecommon'] = test_module + sys.modules['test'] = type(sys)('test') with open(test_mod_path, 'rb') as f: sources = f.read() exec(sources, vars(test_module)) From 552f3583435a81d3216b28efd7bd71de4f0d0fde Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 6 Jan 2026 11:11:36 +0200 Subject: [PATCH 21/21] Add more explicit skips. --- Lib/test/pickletester.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 5179b7d16ad614..91677e8bd53488 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2762,6 +2762,8 @@ def test_recursive_tuple_and_dict_like_key(self): self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table) def test_recursive_set(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') # Set containing an immutable object containing the original set. y = set() y.add(K(y)) @@ -2840,7 +2842,7 @@ def _test_recursive_collection_and_inst(self, factory, oldminproto=None): for proto in protocols: with self.subTest(proto=proto): if self.py_version < (3, 4) and proto < oldminproto: - self.skipTest('not supported in Python < 3.4') + self.skipTest(f'requires protocol {oldminproto} in Python < 3.4') s = self.dumps(o, proto) x = self.loads(s) self.assertIsInstance(x, t) @@ -3588,10 +3590,14 @@ def test_complex_newobj(self): self.assert_is_copy(x, y) def test_complex_newobj_ex(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ x.abc = 666 - for proto in protocols if self.py_version >= (3, 6) else protocols[4:]: + for proto in protocols: with self.subTest(proto=proto): + if self.py_version < (3, 6) and proto < 4: + self.skipTest('requires protocol 4 in Python < 3.6') s = self.dumps(x, proto) if proto < 1: if self.py_version >= (3, 7): @@ -3749,7 +3755,7 @@ def test_appends_on_non_lists(self): self._check_pickling_with_opcode(obj, pickle.APPEND, proto) else: if self.py_version < (3, 0): - self.skipTest('not supported in Python < 3.0') + self.skipTest('not supported in Python 2') self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) def test_setitems_on_non_dicts(self): @@ -3815,6 +3821,8 @@ def check_frame_opcodes(self, pickled): @support.skip_if_pgo_task @support.requires_resource('cpu') def test_framing_many_objects(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') obj = list(range(10**5)) for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): @@ -3830,6 +3838,8 @@ def test_framing_many_objects(self): self.check_frame_opcodes(pickled) def test_framing_large_objects(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') N = 1024 * 1024 small_items = [[i] for i in range(10)] obj = [b'x' * N, *small_items, b'y' * N, 'z' * N] @@ -3863,8 +3873,8 @@ def test_framing_large_objects(self): self.check_frame_opcodes(pickled) def test_optional_frames(self): - if pickle.HIGHEST_PROTOCOL < 4: - return + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') def remove_frames(pickled, keep_frame=None): """Remove frame opcodes from the given pickle.""" @@ -3906,6 +3916,9 @@ def remove_frames(pickled, keep_frame=None): @support.skip_if_pgo_task def test_framed_write_sizes_with_delayed_writer(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') + class ChunkAccumulator: """Accumulate pickler output in a list of raw chunks.""" def __init__(self):