From 25bb266fc876b344e31e0b5634a4db94912c1aba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 01:16:30 +0200 Subject: [PATCH 001/124] gh-109748: Fix venv test_zippath_from_non_installed_posix() (#109872) Fix test_zippath_from_non_installed_posix() of test_venv: don't copy __pycache__/ sub-directories, because they can be modified by other Python tests running in parallel. --- Lib/test/test_venv.py | 10 +++++++++- .../2023-09-26-00-49-18.gh-issue-109748.nxlT1i.rst | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-26-00-49-18.gh-issue-109748.nxlT1i.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index a894bb10bd04da..eb83aa39425515 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -559,6 +559,13 @@ def test_zippath_from_non_installed_posix(self): platlibdir, stdlib_zip) additional_pythonpath_for_non_installed = [] + + # gh-109748: Don't copy __pycache__/ sub-directories, because they can + # be modified by other Python tests running in parallel. + ignored_names = {'__pycache__'} + def ignore_pycache(src, names): + return ignored_names + # Copy stdlib files to the non-installed python so venv can # correctly calculate the prefix. for eachpath in sys.path: @@ -575,7 +582,8 @@ def test_zippath_from_non_installed_posix(self): if os.path.isfile(fn): shutil.copy(fn, libdir) elif os.path.isdir(fn): - shutil.copytree(fn, os.path.join(libdir, name)) + shutil.copytree(fn, os.path.join(libdir, name), + ignore=ignore_pycache) else: additional_pythonpath_for_non_installed.append( eachpath) diff --git a/Misc/NEWS.d/next/Tests/2023-09-26-00-49-18.gh-issue-109748.nxlT1i.rst b/Misc/NEWS.d/next/Tests/2023-09-26-00-49-18.gh-issue-109748.nxlT1i.rst new file mode 100644 index 00000000000000..840366ba8d1611 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-26-00-49-18.gh-issue-109748.nxlT1i.rst @@ -0,0 +1,3 @@ +Fix ``test_zippath_from_non_installed_posix()`` of test_venv: don't copy +``__pycache__/`` sub-directories, because they can be modified by other Python +tests running in parallel. Patch by Victor Stinner. From e9791ba35175171170ff09094ea46b91fc18c654 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 25 Sep 2023 19:46:58 -0400 Subject: [PATCH 002/124] gh-88233: zipfile: refactor _strip_extra (#102084) * Refactor zipfile._strip_extra to use higher level abstractions for extras instead of a heavy-state loop. * Add blurb * Remove _strip_extra and use _Extra.strip directly. * Use memoryview to avoid unnecessary copies while splitting Extras. --- Lib/test/test_zipfile/test_core.py | 46 +++++++------- Lib/zipfile/__init__.py | 60 ++++++++++++------- ...3-02-20-12-00-11.gh-issue-88233.o5Zb0t.rst | 2 + 3 files changed, 62 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-20-12-00-11.gh-issue-88233.o5Zb0t.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 9960259c4cde0c..0f6c0f2107ce6b 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3203,14 +3203,14 @@ def test_no_data(self): b = s.pack(2, 0) c = s.pack(3, 0) - self.assertEqual(b'', zipfile._strip_extra(a, (self.ZIP64_EXTRA,))) - self.assertEqual(b, zipfile._strip_extra(b, (self.ZIP64_EXTRA,))) + self.assertEqual(b'', zipfile._Extra.strip(a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b, (self.ZIP64_EXTRA,))) self.assertEqual( - b+b"z", zipfile._strip_extra(b+b"z", (self.ZIP64_EXTRA,))) + b+b"z", zipfile._Extra.strip(b+b"z", (self.ZIP64_EXTRA,))) - self.assertEqual(b+c, zipfile._strip_extra(a+b+c, (self.ZIP64_EXTRA,))) - self.assertEqual(b+c, zipfile._strip_extra(b+a+c, (self.ZIP64_EXTRA,))) - self.assertEqual(b+c, zipfile._strip_extra(b+c+a, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(a+b+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+a+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+c+a, (self.ZIP64_EXTRA,))) def test_with_data(self): s = struct.Struct(" Date: Tue, 26 Sep 2023 02:07:12 +0200 Subject: [PATCH 003/124] gh-109401: Fix threading barrier test_default_timeout() (#109875) Increase timeouts. Barrier default timeout should be long enough to spawn 4 threads on a slow CI. --- Lib/test/lock_tests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index a4f52cb20ad301..0890ec87afd1c6 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -1014,13 +1014,15 @@ def test_default_timeout(self): """ Test the barrier's default timeout """ - # create a barrier with a low default timeout - barrier = self.barriertype(self.N, timeout=0.3) + # gh-109401: Barrier timeout should be long enough + # to create 4 threads on a slow CI. + timeout = 1.0 + barrier = self.barriertype(self.N, timeout=timeout) def f(): i = barrier.wait() if i == self.N // 2: - # One thread is later than the default timeout of 0.3s. - time.sleep(1.0) + # One thread is later than the default timeout. + time.sleep(timeout * 2) self.assertRaises(threading.BrokenBarrierError, barrier.wait) self.run_threads(f) From 4091deba88946841044b0a54090492a2fd903d42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 03:05:07 +0200 Subject: [PATCH 004/124] gh-109739: regrtest disables load tracker if refleak (#109871) regrtest: Fix reference leak check on Windows. Disable the load tracker on Windows in the reference leak check mode (-R option). --- Lib/test/libregrtest/main.py | 16 +++++++++++++--- ...023-09-25-23-59-37.gh-issue-109739.MUn7K5.rst | 3 +++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-25-23-59-37.gh-issue-109739.MUn7K5.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a9dd08702deb59..0ec25a06b6e175 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -20,7 +20,8 @@ StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, - display_header, cleanup_temp_dir) + display_header, cleanup_temp_dir, + MS_WINDOWS) class Regrtest: @@ -435,7 +436,15 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: setup_process() - self.logger.start_load_tracker() + if self.hunt_refleak and not self.num_workers: + # gh-109739: WindowsLoadTracker thread interfers with refleak check + use_load_tracker = False + else: + # WindowsLoadTracker is only needed on Windows + use_load_tracker = MS_WINDOWS + + if use_load_tracker: + self.logger.start_load_tracker() try: if self.num_workers: self._run_tests_mp(runtests, self.num_workers) @@ -448,7 +457,8 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.want_rerun and self.results.need_rerun(): self.rerun_failed_tests(runtests) finally: - self.logger.stop_load_tracker() + if use_load_tracker: + self.logger.stop_load_tracker() self.display_summary() self.finalize_tests(tracer) diff --git a/Misc/NEWS.d/next/Tests/2023-09-25-23-59-37.gh-issue-109739.MUn7K5.rst b/Misc/NEWS.d/next/Tests/2023-09-25-23-59-37.gh-issue-109739.MUn7K5.rst new file mode 100644 index 00000000000000..291524c758ca68 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-25-23-59-37.gh-issue-109739.MUn7K5.rst @@ -0,0 +1,3 @@ +regrtest: Fix reference leak check on Windows. Disable the load tracker on +Windows in the reference leak check mode (-R option). Patch by Victor +Stinner. From 0b4e090422db5f959184353d53552d1675f74212 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 26 Sep 2023 10:06:07 +0300 Subject: [PATCH 005/124] gh-109370: Fix unexpected traceback output in test_concurrent_futures (GH-109780) Follow-up of gh-107219. * Only close the connection writer on Windows. * Also use existing constant _winapi.ERROR_OPERATION_ABORTED instead of WSA_OPERATION_ABORTED. --- Lib/concurrent/futures/process.py | 3 ++- Lib/multiprocessing/connection.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 48d8db3ed423a5..011e79a5e73d5a 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -521,7 +521,8 @@ def terminate_broken(self, cause): # gh-107219: Close the connection writer which can unblock # Queue._feed() if it was stuck in send_bytes(). - self.call_queue._writer.close() + if sys.platform == 'win32': + self.call_queue._writer.close() # clean up resources self.join_executor_internals() diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 7c425a2d8e7034..dbbf106f680964 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -42,7 +42,6 @@ BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. -WSA_OPERATION_ABORTED = 995 _mmap_counter = itertools.count() @@ -300,7 +299,7 @@ def _send_bytes(self, buf): finally: self._send_ov = None nwritten, err = ov.GetOverlappedResult(True) - if err == WSA_OPERATION_ABORTED: + if err == _winapi.ERROR_OPERATION_ABORTED: # close() was called by another thread while # WaitForMultipleObjects() was waiting for the overlapped # operation. From 7c61a361fc2e93375e22849fffbc20b60e94dbde Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 26 Sep 2023 10:46:09 +0300 Subject: [PATCH 006/124] gh-101100: Fix Sphinx warnings in `Doc/library/weakref.rst` (#109881) --- Doc/library/weakref.rst | 19 +++++++++---------- Doc/tools/.nitignore | 1 - 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 1406b663c6a8e2..d6e062df945c64 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -111,7 +111,7 @@ See :ref:`__slots__ documentation ` for details. Exceptions raised by the callback will be noted on the standard error output, but cannot be propagated; they are handled in exactly the same way as exceptions - raised from an object's :meth:`__del__` method. + raised from an object's :meth:`~object.__del__` method. Weak references are :term:`hashable` if the *object* is hashable. They will maintain their hash value even after the *object* was deleted. If @@ -221,8 +221,7 @@ than needed. Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. :class:`WeakValueDictionary` objects have an additional method that has the -same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` -objects. +same issues as the :meth:`WeakKeyDictionary.keyrefs` method. .. method:: WeakValueDictionary.valuerefs() @@ -281,7 +280,7 @@ objects. Exceptions raised by finalizer callbacks during garbage collection will be shown on the standard error output, but cannot be propagated. They are handled in the same way as exceptions raised - from an object's :meth:`__del__` method or a weak reference's + from an object's :meth:`~object.__del__` method or a weak reference's callback. When the program exits, each remaining live finalizer is called @@ -523,18 +522,18 @@ is still alive. For instance obj dead or exiting -Comparing finalizers with :meth:`__del__` methods -------------------------------------------------- +Comparing finalizers with :meth:`~object.__del__` methods +--------------------------------------------------------- Suppose we want to create a class whose instances represent temporary directories. The directories should be deleted with their contents when the first of the following events occurs: * the object is garbage collected, -* the object's :meth:`remove` method is called, or +* the object's :meth:`!remove` method is called, or * the program exits. -We might try to implement the class using a :meth:`__del__` method as +We might try to implement the class using a :meth:`~object.__del__` method as follows:: class TempDir: @@ -553,12 +552,12 @@ follows:: def __del__(self): self.remove() -Starting with Python 3.4, :meth:`__del__` methods no longer prevent +Starting with Python 3.4, :meth:`~object.__del__` methods no longer prevent reference cycles from being garbage collected, and module globals are no longer forced to :const:`None` during :term:`interpreter shutdown`. So this code should work without any issues on CPython. -However, handling of :meth:`__del__` methods is notoriously implementation +However, handling of :meth:`~object.__del__` methods is notoriously implementation specific, since it depends on internal details of the interpreter's garbage collector implementation. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4188edc18f9036..f260a571661b76 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -133,7 +133,6 @@ Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst Doc/library/urllib.request.rst -Doc/library/weakref.rst Doc/library/wsgiref.rst Doc/library/xml.dom.minidom.rst Doc/library/xml.dom.pulldom.rst From 8ac2085b80eca4d9b2a1093d0a7da020fd12e11a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 26 Sep 2023 10:56:33 +0300 Subject: [PATCH 007/124] gh-109631: Allow interruption of short repeated regex matches (GH-109867) Counting for signal checking now continues in new match from the point where it ended in the previous match instead of starting from 0. --- .../Library/2023-09-25-23-00-37.gh-issue-109631.eWSqpO.rst | 3 +++ Modules/_sre/sre.h | 1 + Modules/_sre/sre_lib.h | 6 ++++-- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-25-23-00-37.gh-issue-109631.eWSqpO.rst diff --git a/Misc/NEWS.d/next/Library/2023-09-25-23-00-37.gh-issue-109631.eWSqpO.rst b/Misc/NEWS.d/next/Library/2023-09-25-23-00-37.gh-issue-109631.eWSqpO.rst new file mode 100644 index 00000000000000..58af2e57068267 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-25-23-00-37.gh-issue-109631.eWSqpO.rst @@ -0,0 +1,3 @@ +:mod:`re` functions such as :func:`re.findall`, :func:`re.split`, +:func:`re.search` and :func:`re.sub` which perform short repeated matches +can now be interrupted by user. diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h index f60078d6bb999b..83d89d57b11199 100644 --- a/Modules/_sre/sre.h +++ b/Modules/_sre/sre.h @@ -95,6 +95,7 @@ typedef struct { size_t data_stack_base; /* current repeat context */ SRE_REPEAT *repeat; + unsigned int sigcount; } SRE_STATE; typedef struct { diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index ae80009fd63bbe..3c805aeeca0974 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -564,7 +564,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) Py_ssize_t alloc_pos, ctx_pos = -1; Py_ssize_t ret = 0; int jump; - unsigned int sigcount=0; + unsigned int sigcount = state->sigcount; SRE(match_context)* ctx; SRE(match_context)* nextctx; @@ -1567,8 +1567,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ctx_pos = ctx->last_ctx_pos; jump = ctx->jump; DATA_POP_DISCARD(ctx); - if (ctx_pos == -1) + if (ctx_pos == -1) { + state->sigcount = sigcount; return ret; + } DATA_LOOKUP_AT(SRE(match_context), ctx, ctx_pos); switch (jump) { From 2897142d2ec0930a8991af964c798b68fb6dcadd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 12:43:45 +0200 Subject: [PATCH 008/124] gh-109832: concurrent.futures test_deadlock restores sys.stderr (#109887) test_error_at_task_unpickle() and test_error_during_result_unpickle_in_result_handler() now restore sys.stderr which is overriden by _raise_error_ignore_stderr(). --- Lib/test/test_concurrent_futures/test_deadlock.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index a76e075c3be180..af702542081ad9 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -145,6 +145,9 @@ def test_exit_at_task_unpickle(self): self._check_crash(BrokenProcessPool, id, ExitAtUnpickle()) def test_error_at_task_unpickle(self): + # gh-109832: Restore stderr overriden by _raise_error_ignore_stderr() + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + # Check problem occurring while unpickling a task on workers self._check_crash(BrokenProcessPool, id, ErrorAtUnpickle()) @@ -180,6 +183,9 @@ def test_error_during_result_pickle_on_worker(self): self._check_crash(PicklingError, _return_instance, ErrorAtPickle) def test_error_during_result_unpickle_in_result_handler(self): + # gh-109832: Restore stderr overriden by _raise_error_ignore_stderr() + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + # Check problem occurring while unpickling a task in # the result_handler thread self._check_crash(BrokenProcessPool, From 0eb98837b60bc58e57ad3e2b35c6b0e9ab634678 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 26 Sep 2023 13:57:25 +0200 Subject: [PATCH 009/124] gh-109593: Fix reentrancy issue in multiprocessing resource_tracker (#109629) --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/multiprocessing/resource_tracker.py | 34 ++++++++++++++++-- Lib/test/lock_tests.py | 36 +++++++++++++++++++ Lib/test/test_importlib/test_locks.py | 2 ++ Lib/test/test_threading.py | 3 ++ Lib/threading.py | 7 ++++ ...-09-22-20-16-44.gh-issue-109593.LboaNM.rst | 1 + Modules/_threadmodule.c | 14 ++++++++ 7 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-22-20-16-44.gh-issue-109593.LboaNM.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 3783c1ffc6e4a9..8e41f461cc934e 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -51,15 +51,31 @@ }) +class ReentrantCallError(RuntimeError): + pass + + class ResourceTracker(object): def __init__(self): - self._lock = threading.Lock() + self._lock = threading.RLock() self._fd = None self._pid = None + def _reentrant_call_error(self): + # gh-109629: this happens if an explicit call to the ResourceTracker + # gets interrupted by a garbage collection, invoking a finalizer (*) + # that itself calls back into ResourceTracker. + # (*) for example the SemLock finalizer + raise ReentrantCallError( + "Reentrant call into the multiprocessing resource tracker") + def _stop(self): with self._lock: + # This should not happen (_stop() isn't called by a finalizer) + # but we check for it anyway. + if self._lock._recursion_count() > 1: + return self._reentrant_call_error() if self._fd is None: # not running return @@ -81,6 +97,9 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' with self._lock: + if self._lock._recursion_count() > 1: + # The code below is certainly not reentrant-safe, so bail out + return self._reentrant_call_error() if self._fd is not None: # resource tracker was launched before, is it still running? if self._check_alive(): @@ -159,7 +178,17 @@ def unregister(self, name, rtype): self._send('UNREGISTER', name, rtype) def _send(self, cmd, name, rtype): - self.ensure_running() + try: + self.ensure_running() + except ReentrantCallError: + # The code below might or might not work, depending on whether + # the resource tracker was already running and still alive. + # Better warn the user. + # (XXX is warnings.warn itself reentrant-safe? :-) + warnings.warn( + f"ResourceTracker called reentrantly for resource cleanup, " + f"which is unsupported. " + f"The {rtype} object {name!r} might leak.") msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') if len(msg) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF @@ -176,6 +205,7 @@ def _send(self, cmd, name, rtype): unregister = _resource_tracker.unregister getfd = _resource_tracker.getfd + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 0890ec87afd1c6..e53e24b18f2760 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -330,6 +330,42 @@ def test_release_save_unacquired(self): lock.release() self.assertRaises(RuntimeError, lock._release_save) + def test_recursion_count(self): + lock = self.locktype() + self.assertEqual(0, lock._recursion_count()) + lock.acquire() + self.assertEqual(1, lock._recursion_count()) + lock.acquire() + lock.acquire() + self.assertEqual(3, lock._recursion_count()) + lock.release() + self.assertEqual(2, lock._recursion_count()) + lock.release() + lock.release() + self.assertEqual(0, lock._recursion_count()) + + phase = [] + + def f(): + lock.acquire() + phase.append(None) + while len(phase) == 1: + _wait() + lock.release() + phase.append(None) + + with threading_helper.wait_threads_exit(): + start_new_thread(f, ()) + while len(phase) == 0: + _wait() + self.assertEqual(len(phase), 1) + self.assertEqual(0, lock._recursion_count()) + phase.append(None) + while len(phase) == 2: + _wait() + self.assertEqual(len(phase), 3) + self.assertEqual(0, lock._recursion_count()) + def test_different_thread(self): # Cannot release from a different thread lock = self.locktype() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index ba9cf51c261d52..7091c36aaaf761 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -29,6 +29,8 @@ class ModuleLockAsRLockTests: test_timeout = None # _release_save() unsupported test_release_save_unacquired = None + # _recursion_count() unsupported + test_recursion_count = None # lock status in repr unsupported test_repr = None test_locked_repr = None diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 9c16c4044f66a8..71fcad268b8036 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1783,6 +1783,9 @@ class ConditionAsRLockTests(lock_tests.RLockTests): # Condition uses an RLock by default and exports its API. locktype = staticmethod(threading.Condition) + def test_recursion_count(self): + self.skipTest("Condition does not expose _recursion_count()") + class ConditionTests(lock_tests.ConditionTests): condtype = staticmethod(threading.Condition) diff --git a/Lib/threading.py b/Lib/threading.py index f6bbdb0d0c80ee..31cefd2143a8c4 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -245,6 +245,13 @@ def _release_save(self): def _is_owned(self): return self._owner == get_ident() + # Internal method used for reentrancy checks + + def _recursion_count(self): + if self._owner != get_ident(): + return 0 + return self._count + _PyRLock = _RLock diff --git a/Misc/NEWS.d/next/Library/2023-09-22-20-16-44.gh-issue-109593.LboaNM.rst b/Misc/NEWS.d/next/Library/2023-09-22-20-16-44.gh-issue-109593.LboaNM.rst new file mode 100644 index 00000000000000..292aea0be24dfb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-22-20-16-44.gh-issue-109593.LboaNM.rst @@ -0,0 +1 @@ +Avoid deadlocking on a reentrant call to the multiprocessing resource tracker. Such a reentrant call, though unlikely, can happen if a GC pass invokes the finalizer for a multiprocessing object such as SemLock. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index fa98df516b8b10..e77e30dfe5e821 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -490,6 +490,18 @@ PyDoc_STRVAR(rlock_release_save_doc, \n\ For internal use by `threading.Condition`."); +static PyObject * +rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) +{ + unsigned long tid = PyThread_get_thread_ident(); + return PyLong_FromUnsignedLong( + self->rlock_owner == tid ? self->rlock_count : 0UL); +} + +PyDoc_STRVAR(rlock_recursion_count_doc, +"_recursion_count() -> int\n\ +\n\ +For internal use by reentrancy checks."); static PyObject * rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) @@ -565,6 +577,8 @@ static PyMethodDef rlock_methods[] = { METH_VARARGS, rlock_acquire_restore_doc}, {"_release_save", (PyCFunction)rlock_release_save, METH_NOARGS, rlock_release_save_doc}, + {"_recursion_count", (PyCFunction)rlock_recursion_count, + METH_NOARGS, rlock_recursion_count_doc}, {"__enter__", _PyCFunction_CAST(rlock_acquire), METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, {"__exit__", (PyCFunction)rlock_release, From 8100612bac2df1cbbb3a4cf646c4b82febf7807f Mon Sep 17 00:00:00 2001 From: lohaswinner Date: Tue, 26 Sep 2023 22:12:32 +0900 Subject: [PATCH 010/124] no-issue: Fix a typo in the parameter name of random.expovariate. (gh-109902) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 76f1f00dbd34dd..4874a6c77faa99 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -752,7 +752,7 @@ random * Add :func:`random.binomialvariate`. (Contributed by Raymond Hettinger in :gh:`81620`.) -* Add a default of ``lamb=1.0`` to :func:`random.expovariate`. +* Add a default of ``lambd=1.0`` to :func:`random.expovariate`. (Contributed by Raymond Hettinger in :gh:`100234`.) shutil From 19bf3986958fc8269a1eb6d741bb60c91d6b5e58 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 26 Sep 2023 08:20:17 -0500 Subject: [PATCH 011/124] More informative docstrings in the random module (gh-109745) --- Lib/random.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/Lib/random.py b/Lib/random.py index 84bbfc5df1bf23..1d789b107904fb 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -492,7 +492,14 @@ def choices(self, population, weights=None, *, cum_weights=None, k=1): ## -------------------- real-valued distributions ------------------- def uniform(self, a, b): - "Get a random number in the range [a, b) or [a, b] depending on rounding." + """Get a random number in the range [a, b) or [a, b] depending on rounding. + + The mean (expected value) and variance of the random variable are: + + E[X] = (a + b) / 2 + Var[X] = (b - a) ** 2 / 12 + + """ return a + (b - a) * self.random() def triangular(self, low=0.0, high=1.0, mode=None): @@ -503,6 +510,11 @@ def triangular(self, low=0.0, high=1.0, mode=None): http://en.wikipedia.org/wiki/Triangular_distribution + The mean (expected value) and variance of the random variable are: + + E[X] = (low + high + mode) / 3 + Var[X] = (low**2 + high**2 + mode**2 - low*high - low*mode - high*mode) / 18 + """ u = self.random() try: @@ -593,12 +605,15 @@ def expovariate(self, lambd=1.0): positive infinity if lambd is positive, and from negative infinity to 0 if lambd is negative. - """ - # lambd: rate lambd = 1/mean - # ('lambda' is a Python reserved word) + The mean (expected value) and variance of the random variable are: + + E[X] = 1 / lambd + Var[X] = 1 / lambd ** 2 + """ # we use 1-random() instead of random() to preclude the # possibility of taking the log of zero. + return -_log(1.0 - self.random()) / lambd def vonmisesvariate(self, mu, kappa): @@ -654,8 +669,12 @@ def gammavariate(self, alpha, beta): pdf(x) = -------------------------------------- math.gamma(alpha) * beta ** alpha + The mean (expected value) and variance of the random variable are: + + E[X] = alpha * beta + Var[X] = alpha * beta ** 2 + """ - # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 # Warning: a few older sources define the gamma distribution in terms # of alpha > -1.0 @@ -714,6 +733,11 @@ def betavariate(self, alpha, beta): Conditions on the parameters are alpha > 0 and beta > 0. Returned values range between 0 and 1. + The mean (expected value) and variance of the random variable are: + + E[X] = alpha / (alpha + beta) + Var[X] = alpha * beta / ((alpha + beta)**2 * (alpha + beta + 1)) + """ ## See ## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html @@ -766,6 +790,11 @@ def binomialvariate(self, n=1, p=0.5): Returns an integer in the range: 0 <= X <= n + The mean (expected value) and variance of the random variable are: + + E[X] = n * p + Var[x] = n * p * (1 - p) + """ # Error check inputs and handle edge cases if n < 0: From 859618c8cd5de86a975e68d7e5d20c04bc5db2e5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 17:22:50 +0200 Subject: [PATCH 012/124] gh-109566, regrtest: Add --fast-ci and --slow-ci options (#109570) * Add --fast-ci and --slow-ci options to libregrtest: * --fast-ci uses a default timeout of 10 minutes and "-u all,-cpu" (skip slowest tests). * --slow-ci uses a default timeout of 20 minues and "-u all" (run all tests). * regrtest header now lists test resources. * Makefile changes: * "make test", "make hostrunnertest" and "make coverage-report" now use --fast-ci option and TESTTIMEOUT variable. * "make buildbottest" now uses "--slow-ci". Remove options which became redundant with "--slow-ci". * "make testall" and "make testuniversal" now use --slow-ci option and TESTTIMEOUT variable. * "make testall" now uses "find -exec rm ..." instead of "find ... -print|xargs rm ...", same as "make clean". * GitHub Actions workflow: * Ubuntu and Address Sanitizer jobs now use "make test". Remove options which became redundant with "--fast-ci". * Windows jobs now use --fast-ci option. * Use -j0 to detect the number of CPUs. * Set Makefile TESTTIMEOUT default to an empty string, since --slow-ci and --fast-ci use different default timeout. It's now accepted to pass "--timeout=" to regrtest: treated as not timeout. * Tools/scripts/run_tests.py now uses --fast-ci option. * Tools/buildbot/test.bat now uses --slow-ci option. Remove --timeout=1200 option, redundant with --slow-ci. --- .github/workflows/build.yml | 10 ++-- Doc/using/configure.rst | 15 ++++- Lib/test/libregrtest/cmdline.py | 55 ++++++++++++++++++- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/results.py | 4 +- Lib/test/libregrtest/utils.py | 9 ++- Lib/test/test_regrtest.py | 50 ++++++++++++++++- Makefile.pre.in | 45 +++++++-------- ...-09-19-13-33-20.gh-issue-109566.aX0g9o.rst | 4 ++ Tools/buildbot/test.bat | 6 +- Tools/scripts/run_tests.py | 12 +--- 11 files changed, 161 insertions(+), 51 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f9d0f4da09be7..28ebc1643bd694 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -182,7 +182,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci build_win_amd64: name: 'Windows (x64)' @@ -201,7 +201,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci build_win_arm64: name: 'Windows (arm64)' @@ -252,7 +252,7 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: make test build_ubuntu: name: 'Ubuntu' @@ -319,7 +319,7 @@ jobs: run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: xvfb-run make test build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -535,7 +535,7 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: xvfb-run make test all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 763f9778776990..82074750ec59a8 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -964,9 +964,18 @@ Main Makefile targets You can use the configure :option:`--enable-optimizations` option to make this the default target of the ``make`` command (``make all`` or just ``make``). -* ``make buildbottest``: Build Python and run the Python test suite, the same - way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds) - to change the test timeout (1200 by default: 20 minutes). + +* ``make test``: Build Python and run the Python test suite with ``--slow-ci`` + option. Variables: + + * ``TESTOPTS``: additional regrtest command line options. + * ``TESTPYTHONOPTS``: additional Python command line options. + * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes). + +* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci`` + option and default timeout of 20 minutes, instead of ``--fast-ci`` option + and a default timeout of 10 minutes. + * ``make install``: Build and install Python. * ``make regen-all``: Regenerate (almost) all generated files; ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 99f28152f1a1c7..a0a8504fe8f606 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -4,6 +4,8 @@ import sys from test.support import os_helper +from .utils import MS_WINDOWS + USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] @@ -145,6 +147,7 @@ class Namespace(argparse.Namespace): def __init__(self, **kwargs) -> None: + self.ci = False self.testdir = None self.verbose = 0 self.quiet = False @@ -209,7 +212,13 @@ def _create_parser(): # We add help explicitly to control what argument group it renders under. group.add_argument('-h', '--help', action='help', help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, + group.add_argument('--fast-ci', action='store_true', + help='Fast Continuous Integration (CI) mode used by ' + 'GitHub Actions') + group.add_argument('--slow-ci', action='store_true', + help='Slow Continuous Integration (CI) mode used by ' + 'buildbot workers') + group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' 'more than TIMEOUT seconds; disabled if TIMEOUT ' 'is negative or equals to zero') @@ -384,7 +393,49 @@ def _parse_args(args, **kwargs): for arg in ns.args: if arg.startswith('-'): parser.error("unrecognized arguments: %s" % arg) - sys.exit(1) + + if ns.timeout is not None: + # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT + # can be used by "make buildbottest" and "make test". + if ns.timeout != "": + try: + ns.timeout = float(ns.timeout) + except ValueError: + parser.error(f"invalid timeout value: {ns.timeout!r}") + else: + ns.timeout = None + + # Continuous Integration (CI): common options for fast/slow CI modes + if ns.slow_ci or ns.fast_ci: + # Similar to options: + # + # -j0 --randomize --fail-env-changed --fail-rerun --rerun + # --slowest --verbose3 --nowindows + if ns.use_mp is None: + ns.use_mp = 0 + ns.randomize = True + ns.fail_env_changed = True + ns.fail_rerun = True + ns.rerun = True + ns.print_slow = True + ns.verbose3 = True + if MS_WINDOWS: + ns.nowindows = True # Silence alerts under Windows + + # When both --slow-ci and --fast-ci options are present, + # --slow-ci has the priority + if ns.slow_ci: + # Similar to: -u "all" --timeout=1200 + if not ns.use: + ns.use = [['all']] + if ns.timeout is None: + ns.timeout = 1200 # 20 minutes + elif ns.fast_ci: + # Similar to: -u "all,-cpu" --timeout=600 + if not ns.use: + ns.use = [['all', '-cpu']] + if ns.timeout is None: + ns.timeout = 600 # 10 minutes if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0ec25a06b6e175..2cd79a1eae5c91 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -425,7 +425,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if (self.want_header or not(self.pgo or self.quiet or self.single_test_run or tests or self.cmdline_args)): - display_header() + display_header(self.use_resources) if self.randomize: print("Using random seed", self.random_seed) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 1a8619fb62be2a..35df50d581ff6a 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -8,11 +8,13 @@ printlist, count, format_duration) +# Python uses exit code 1 when an exception is not catched +# argparse.ArgumentParser.error() uses exit code 2 EXITCODE_BAD_TEST = 2 EXITCODE_ENV_CHANGED = 3 EXITCODE_NO_TESTS_RAN = 4 EXITCODE_RERUN_FAIL = 5 -EXITCODE_INTERRUPTED = 130 +EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 class TestResults: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 6af949cea9c926..f3f0eb53b32100 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -547,7 +547,7 @@ def adjust_rlimit_nofile(): f"{new_fd_limit}: {err}.") -def display_header(): +def display_header(use_resources: tuple[str, ...]): encoding = sys.stdout.encoding # Print basic platform information @@ -569,6 +569,13 @@ def display_header(): print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + + if use_resources: + print(f"== resources ({len(use_resources)}): " + f"{', '.join(sorted(use_resources))}") + else: + print(f"== resources: (all disabled, use -u option)") + # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 4b819cbbb8dfc3..15aab609ed1ba7 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -23,8 +23,9 @@ from test import support from test.support import os_helper, TestStats, without_optimizer from test.libregrtest import cmdline -from test.libregrtest import utils +from test.libregrtest import main from test.libregrtest import setup +from test.libregrtest import utils from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -75,8 +76,15 @@ def test_help(self): def test_timeout(self): ns = self.parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) + + # negative, zero and empty string are treated as "no timeout" + for value in ('-1', '0', ''): + with self.subTest(value=value): + ns = self.parse_args([f'--timeout={value}']) + self.assertEqual(ns.timeout, None) + self.checkError(['--timeout'], 'expected one argument') - self.checkError(['--timeout', 'foo'], 'invalid float value') + self.checkError(['--timeout', 'foo'], 'invalid timeout value:') def test_wait(self): ns = self.parse_args(['--wait']) @@ -366,6 +374,44 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') + def check_ci_mode(self, args, use_resources): + ns = cmdline._parse_args(args) + if utils.MS_WINDOWS: + self.assertTrue(ns.nowindows) + + # Check Regrtest attributes which are more reliable than Namespace + # which has an unclear API + regrtest = main.Regrtest(ns) + self.assertNotEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.want_rerun) + self.assertTrue(regrtest.randomize) + self.assertIsNone(regrtest.random_seed) + self.assertTrue(regrtest.fail_env_changed) + self.assertTrue(regrtest.fail_rerun) + self.assertTrue(regrtest.print_slowest) + self.assertTrue(regrtest.output_on_failure) + self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) + return regrtest + + def test_fast_ci(self): + args = ['--fast-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 10 * 60) + + def test_fast_ci_resource(self): + # it should be possible to override resources + args = ['--fast-ci', '-u', 'network'] + use_resources = ['network'] + self.check_ci_mode(args, use_resources) + + def test_slow_ci(self): + args = ['--slow-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 20 * 60) + @dataclasses.dataclass(slots=True) class Rerun: diff --git a/Makefile.pre.in b/Makefile.pre.in index d123fa3e6f4a47..ccbacfc8571851 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -771,7 +771,7 @@ coverage-report: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures - $(TESTRUNNER) $(TESTOPTS) || true + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) || true @ # build lcov report $(MAKE) coverage-lcov @@ -1844,7 +1844,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) TESTOPTS= $(EXTRATESTOPTS) TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py -TESTTIMEOUT= 1200 +TESTTIMEOUT= # Remove "test_python_*" directories of previous failed test jobs. # Pass TESTOPTS options because it can contain --tempdir option. @@ -1854,9 +1854,10 @@ cleantest: all # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. +# Similar to buildbottest, but use --fast-ci option, instead of --slow-ci. .PHONY: test test: all - $(TESTRUNNER) $(TESTOPTS) + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. # In the past, we've had problems where bugs in the marshalling or @@ -1867,43 +1868,43 @@ test: all # sample data. .PHONY: testall testall: all - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - -$(TESTRUNNER) -u all $(TESTOPTS) - $(TESTRUNNER) -u all $(TESTOPTS) + -find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py + -find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. .PHONY: testuniversal testuniversal: all - @if [ `arch` != 'i386' ]; then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ - fi - $(TESTRUNNER) -u all $(TESTOPTS) - $(RUNSHARED) /usr/libexec/oah/translate \ - ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) + @if [ `arch` != 'i386' ]; then \ + echo "This can only be used on OSX/i386" ;\ + exit 1 ;\ + fi + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) + $(RUNSHARED) /usr/libexec/oah/translate \ + ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) # Like testall, but with only one pass and without multiple processes. # Run an optional script to include information about the build environment. .PHONY: buildbottest buildbottest: all - -@if which pybuildbot.identify >/dev/null 2>&1; then \ - pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ - fi - $(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --fail-rerun --timeout=$(TESTTIMEOUT) $(TESTOPTS) + -@if which pybuildbot.identify >/dev/null 2>&1; then \ + pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ + fi + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Like testall, but run Python tests with HOSTRUNNER directly. .PHONY: hostrunnertest hostrunnertest: all - $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS) + $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) .PHONY: pythoninfo pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo -QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \ +QUICKTESTOPTS= -x test_subprocess test_io \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing_fork test_multiprocessing_spawn \ test_multiprocessing_forkserver \ @@ -1912,7 +1913,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \ .PHONY: quicktest quicktest: all - $(TESTRUNNER) $(QUICKTESTOPTS) + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) $(QUICKTESTOPTS) # SSL tests .PHONY: multisslcompile diff --git a/Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst b/Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst new file mode 100644 index 00000000000000..10f90132c37ec9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst @@ -0,0 +1,4 @@ +regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a +default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests). +``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all +tests). Patch by Victor Stinner. diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index c1b2605a4b2c7e..781f9a4c8206c8 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -5,7 +5,7 @@ setlocal set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH set here=%~dp0 set rt_opts=-q -d -set regrtest_args=-j1 +set regrtest_args= set arm32_ssh= :CheckOpts @@ -23,7 +23,7 @@ if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecuti if "%arm32_ssh%"=="true" goto :Arm32Ssh :NativeExecution -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% --slow-ci %regrtest_args% exit /b %ERRORLEVEL% :Arm32Ssh @@ -35,7 +35,7 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp -set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 %regrtest_args% %TEMP_ARGS% +set rt_args=%rt_opts% --slow-ci %dashU% %regrtest_args% %TEMP_ARGS% ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& cd %REMOTE_PYTHON_DIR% & %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% set ERR=%ERRORLEVEL% scp %SSH_SERVER%:"%REMOTE_PYTHON_DIR%test-results.xml" "%PYTHON_SOURCE%\test-results.xml" diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py index 445a34ae3e8eee..c62ae82dd788d5 100644 --- a/Tools/scripts/run_tests.py +++ b/Tools/scripts/run_tests.py @@ -18,9 +18,6 @@ def is_multiprocess_flag(arg): return arg.startswith('-j') or arg.startswith('--multiprocess') -def is_resource_use_flag(arg): - return arg.startswith('-u') or arg.startswith('--use') - def is_python_flag(arg): return arg.startswith('-p') or arg.startswith('--python') @@ -56,20 +53,13 @@ def main(regrtest_args): args.extend(test.support.args_from_interpreter_flags()) args.extend(['-m', 'test', # Run the test suite - '-r', # Randomize test order - '-w', # Re-run failed tests in verbose mode + '--fast-ci', # Fast Continuous Integration mode ]) - if sys.platform == 'win32': - args.append('-n') # Silence alerts under Windows if not any(is_multiprocess_flag(arg) for arg in regrtest_args): if cross_compile and hostrunner: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. args.extend(['-j', '2']) - else: - args.extend(['-j', '0']) # Use all CPU cores - if not any(is_resource_use_flag(arg) for arg in regrtest_args): - args.extend(['-u', 'all,-largefile,-audio,-gui']) if cross_compile and hostrunner: # If HOSTRUNNER is set and -p/--python option is not given, then From ecd813f054e0dee890d484b8210e202175abd632 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 26 Sep 2023 17:57:17 +0100 Subject: [PATCH 013/124] GH-109187: Improve symlink loop handling in `pathlib.Path.resolve()` (GH-109192) Treat symlink loops like other errors: in strict mode, raise `OSError`, and in non-strict mode, do not raise any exception. --- Doc/library/pathlib.rst | 14 ++++++++----- Lib/pathlib.py | 21 +------------------ Lib/test/test_pathlib.py | 13 +++++++----- ...-09-09-17-09-54.gh-issue-109187.dIayNW.rst | 3 +++ 4 files changed, 21 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-09-17-09-54.gh-issue-109187.dIayNW.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 22360b22fd924b..48d6176d26bb8f 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1381,15 +1381,19 @@ call fails (for example because the path doesn't exist). >>> p.resolve() PosixPath('/home/antoine/pathlib/setup.py') - If the path doesn't exist and *strict* is ``True``, :exc:`FileNotFoundError` - is raised. If *strict* is ``False``, the path is resolved as far as possible - and any remainder is appended without checking whether it exists. If an - infinite loop is encountered along the resolution path, :exc:`RuntimeError` - is raised. + If a path doesn't exist or a symlink loop is encountered, and *strict* is + ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is + resolved as far as possible and any remainder is appended without checking + whether it exists. .. versionchanged:: 3.6 The *strict* parameter was added (pre-3.6 behavior is strict). + .. versionchanged:: 3.13 + Symlink loops are treated like other errors: :exc:`OSError` is raised in + strict mode, and no exception is raised in non-strict mode. In previous + versions, :exc:`RuntimeError` is raised no matter the value of *strict*. + .. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) Glob the given relative *pattern* recursively. This is like calling diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f4ec315da6b4fa..bd5f61b0b7c878 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1230,26 +1230,7 @@ def resolve(self, strict=False): normalizing it. """ - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - - try: - s = os.path.realpath(self, strict=strict) - except OSError as e: - check_eloop(e) - raise - p = self.with_segments(s) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p + return self.with_segments(os.path.realpath(self, strict=strict)) def owner(self): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 09df3fe471fc3e..484a5e6c3bd64d 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -3178,10 +3178,11 @@ def test_absolute(self): self.assertEqual(str(P('//a').absolute()), '//a') self.assertEqual(str(P('//a/b').absolute()), '//a/b') - def _check_symlink_loop(self, *args, strict=True): + def _check_symlink_loop(self, *args): path = self.cls(*args) - with self.assertRaises(RuntimeError): - print(path.resolve(strict)) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) @unittest.skipIf( is_emscripten or is_wasi, @@ -3240,7 +3241,8 @@ def test_resolve_loop(self): os.symlink('linkZ/../linkZ', join('linkZ')) self._check_symlink_loop(BASE, 'linkZ') # Non-strict - self._check_symlink_loop(BASE, 'linkZ', 'foo', strict=False) + p = self.cls(BASE, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) # Loops with absolute symlinks. os.symlink(join('linkU/inside'), join('linkU')) self._check_symlink_loop(BASE, 'linkU') @@ -3249,7 +3251,8 @@ def test_resolve_loop(self): os.symlink(join('linkW/../linkW'), join('linkW')) self._check_symlink_loop(BASE, 'linkW') # Non-strict - self._check_symlink_loop(BASE, 'linkW', 'foo', strict=False) + q = self.cls(BASE, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) def test_glob(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2023-09-09-17-09-54.gh-issue-109187.dIayNW.rst b/Misc/NEWS.d/next/Library/2023-09-09-17-09-54.gh-issue-109187.dIayNW.rst new file mode 100644 index 00000000000000..31b3ef77807cde --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-09-17-09-54.gh-issue-109187.dIayNW.rst @@ -0,0 +1,3 @@ +:meth:`pathlib.Path.resolve` now treats symlink loops like other errors: in +strict mode, :exc:`OSError` is raised, and in non-strict mode, no exception +is raised. From fbfec5642edd9d7690bbff088ee43c08e8067044 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 20:46:52 +0200 Subject: [PATCH 014/124] gh-109566: regrtest reexecutes the process (#109909) When --fast-ci or --slow-ci option is used, regrtest now replaces the current process with a new process to add "-u -W default -bb -E" options to Python. Changes: * PCbuild/rt.bat and Tools/scripts/run_tests.py no longer need to add "-u -W default -bb -E" options to Python: it's now done by regrtest. * Fix Tools/scripts/run_tests.py: flush stdout before replacing the process. Previously, buffered messages were lost. --- Lib/test/__main__.py | 2 +- Lib/test/libregrtest/cmdline.py | 5 ++ Lib/test/libregrtest/main.py | 41 +++++++++++-- Lib/test/test_regrtest.py | 58 ++++++++++++++++++- ...-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst | 3 + PCbuild/rt.bat | 2 +- Tools/scripts/run_tests.py | 10 +--- 7 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index e5780b784b4b05..42553fa32867d3 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,2 @@ from test.libregrtest.main import main -main() +main(reexec=True) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index a0a8504fe8f606..bc969969e068eb 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -184,6 +184,7 @@ def __init__(self, **kwargs) -> None: self.threshold = None self.fail_rerun = False self.tempdir = None + self.no_reexec = False super().__init__(**kwargs) @@ -343,6 +344,8 @@ def _create_parser(): help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', help='remove old test_python_* directories') + group.add_argument('--no-reexec', action='store_true', + help="internal option, don't use it") return parser @@ -421,6 +424,8 @@ def _parse_args(args, **kwargs): ns.verbose3 = True if MS_WINDOWS: ns.nowindows = True # Silence alerts under Windows + else: + ns.no_reexec = True # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2cd79a1eae5c91..a93f532e9cb586 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,6 +1,7 @@ import os import random import re +import shlex import sys import time @@ -20,7 +21,7 @@ StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, - display_header, cleanup_temp_dir, + display_header, cleanup_temp_dir, print_warning, MS_WINDOWS) @@ -47,7 +48,7 @@ class Regrtest: directly to set the values that would normally be set by flags on the command line. """ - def __init__(self, ns: Namespace): + def __init__(self, ns: Namespace, reexec: bool = False): # Log verbosity self.verbose: int = int(ns.verbose) self.quiet: bool = ns.quiet @@ -69,6 +70,7 @@ def __init__(self, ns: Namespace): self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks + self.want_reexec: bool = (reexec and not ns.no_reexec) # Select tests if ns.match_tests: @@ -95,6 +97,7 @@ def __init__(self, ns: Namespace): self.worker_json: StrJSON | None = ns.worker_json # Options to run tests + self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.fail_fast: bool = ns.failfast self.fail_env_changed: bool = ns.fail_env_changed self.fail_rerun: bool = ns.fail_rerun @@ -483,7 +486,37 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: # processes. return self._run_tests(selected, tests) + def _reexecute_python(self): + if self.python_cmd: + # Do nothing if --python=cmd option is used + return + + python_opts = [ + '-u', # Unbuffered stdout and stderr + '-W', 'default', # Add warnings filter 'default' + '-bb', # Error on bytes/str comparison + '-E', # Ignore PYTHON* environment variables + ] + + cmd = [*sys.orig_argv, "--no-reexec"] + cmd[1:1] = python_opts + + # Make sure that messages before execv() are logged + sys.stdout.flush() + sys.stderr.flush() + + try: + os.execv(cmd[0], cmd) + # execv() do no return and so we don't get to this line on success + except OSError as exc: + cmd_text = shlex.join(cmd) + print_warning(f"Failed to reexecute Python: {exc!r}\n" + f"Command: {cmd_text}") + def main(self, tests: TestList | None = None): + if self.want_reexec and self.ci_mode: + self._reexecute_python() + if self.junit_filename and not os.path.isabs(self.junit_filename): self.junit_filename = os.path.abspath(self.junit_filename) @@ -515,7 +548,7 @@ def main(self, tests: TestList | None = None): sys.exit(exitcode) -def main(tests=None, **kwargs): +def main(tests=None, reexec=False, **kwargs): """Run the Python suite.""" ns = _parse_args(sys.argv[1:], **kwargs) - Regrtest(ns).main(tests=tests) + Regrtest(ns, reexec=reexec).main(tests=tests) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 15aab609ed1ba7..2b77300c079c05 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -382,7 +382,8 @@ def check_ci_mode(self, args, use_resources): # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API regrtest = main.Regrtest(ns) - self.assertNotEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.ci_mode) + self.assertEqual(regrtest.num_workers, -1) self.assertTrue(regrtest.want_rerun) self.assertTrue(regrtest.randomize) self.assertIsNone(regrtest.random_seed) @@ -1960,6 +1961,61 @@ def test_dev_mode(self): self.check_executed_tests(output, tests, stats=len(tests), parallel=True) + def check_reexec(self, option): + # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + code = textwrap.dedent(r""" + import sys + import unittest + try: + from _testinternalcapi import get_config + except ImportError: + get_config = None + + class WorkerTests(unittest.TestCase): + @unittest.skipUnless(get_config is None, 'need get_config()') + def test_config(self): + config = get_config()['config'] + # -u option + self.assertEqual(config['buffered_stdio'], 0) + # -W default option + self.assertTrue(config['warnoptions'], ['default']) + # -bb option + self.assertTrue(config['bytes_warning'], 2) + # -E option + self.assertTrue(config['use_environment'], 0) + + # test if get_config() is not available + def test_unbuffered(self): + # -u option + self.assertFalse(sys.stdout.line_buffering) + self.assertFalse(sys.stderr.line_buffering) + + def test_python_opts(self): + # -W default option + self.assertTrue(sys.warnoptions, ['default']) + # -bb option + self.assertEqual(sys.flags.bytes_warning, 2) + # -E option + self.assertTrue(sys.flags.ignore_environment) + """) + testname = self.create_test(code=code) + + cmd = [sys.executable, + "-m", "test", option, + f'--testdir={self.tmptestdir}', + testname] + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + self.assertEqual(proc.returncode, 0, proc) + + def test_reexec_fast_ci(self): + self.check_reexec("--fast-ci") + + def test_reexec_slow_ci(self): + self.check_reexec("--slow-ci") + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Misc/NEWS.d/next/Tests/2023-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst b/Misc/NEWS.d/next/Tests/2023-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst new file mode 100644 index 00000000000000..d865f629fdb05b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst @@ -0,0 +1,3 @@ +regrtest: When ``--fast-ci`` or ``--slow-ci`` option is used, regrtest now +replaces the current process with a new process to add ``-u -W default -bb -E`` +options to Python. Patch by Victor Stinner. diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 33f4212e14567d..7ae7141bfc4eaa 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -48,7 +48,7 @@ if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 set exe=%prefix%\python%suffix%.exe -set cmd="%exe%" %dashO% -u -Wd -E -bb -m test %regrtestargs% +set cmd="%exe%" %dashO% -m test %regrtestargs% if defined qmode goto Qmode echo Deleting .pyc files ... diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py index c62ae82dd788d5..3e3d15d3b0da5c 100644 --- a/Tools/scripts/run_tests.py +++ b/Tools/scripts/run_tests.py @@ -23,11 +23,7 @@ def is_python_flag(arg): def main(regrtest_args): - args = [sys.executable, - '-u', # Unbuffered stdout and stderr - '-W', 'default', # Warnings set to 'default' - '-bb', # Warnings about bytes/bytearray - ] + args = [sys.executable] cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: @@ -47,7 +43,6 @@ def main(regrtest_args): } else: environ = os.environ.copy() - args.append("-E") # Allow user-specified interpreter options to override our defaults. args.extend(test.support.args_from_interpreter_flags()) @@ -70,7 +65,8 @@ def main(regrtest_args): args.extend(regrtest_args) - print(shlex.join(args)) + print(shlex.join(args), flush=True) + if sys.platform == 'win32': from subprocess import call sys.exit(call(args)) From ae1d99c2ed9d44b2554129f3a85b97a31119bccc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 21:33:59 +0200 Subject: [PATCH 015/124] Remove concurrent.futures deadcode: process_result_item() (#109906) process_result_item() cannot be called with an int anymore, the protocol changed. --- Lib/concurrent/futures/process.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 011e79a5e73d5a..73bdcbe8693991 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -444,24 +444,14 @@ def process_result_item(self, result_item): # Process the received a result_item. This can be either the PID of a # worker that exited gracefully or a _ResultItem - if isinstance(result_item, int): - # Clean shutdown of a worker using its PID - # (avoids marking the executor broken) - assert self.is_shutting_down() - p = self.processes.pop(result_item) - p.join() - if not self.processes: - self.join_executor_internals() - return - else: - # Received a _ResultItem so mark the future as completed. - work_item = self.pending_work_items.pop(result_item.work_id, None) - # work_item can be None if another process terminated (see above) - if work_item is not None: - if result_item.exception: - work_item.future.set_exception(result_item.exception) - else: - work_item.future.set_result(result_item.result) + # Received a _ResultItem so mark the future as completed. + work_item = self.pending_work_items.pop(result_item.work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + if result_item.exception: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) def is_shutting_down(self): # Check whether we should start shutting down the executor. From b1e4f6e83e8916005caa3f751f25fb58cccbf812 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 21:34:50 +0200 Subject: [PATCH 016/124] gh-109276, gh-109508: Fix libregrtest stdout (#109903) Remove replace_stdout(): call sys.stdout.reconfigure() instead of set the error handler to backslashreplace. display_header() logs an empty line and flush stdout. Remove encoding workaround in display_header() since stdout error handler is now set to backslashreplace earlier. --- Doc/using/configure.rst | 2 +- Lib/test/libregrtest/main.py | 14 ++++++-- Lib/test/libregrtest/setup.py | 4 +-- Lib/test/libregrtest/utils.py | 61 ++++++++--------------------------- 4 files changed, 27 insertions(+), 54 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 82074750ec59a8..a9555199a2ac24 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -965,7 +965,7 @@ Main Makefile targets this the default target of the ``make`` command (``make all`` or just ``make``). -* ``make test``: Build Python and run the Python test suite with ``--slow-ci`` +* ``make test``: Build Python and run the Python test suite with ``--fast-ci`` option. Variables: * ``TESTOPTS``: additional regrtest command line options. diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a93f532e9cb586..e1cb22afcc9900 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -513,9 +513,11 @@ def _reexecute_python(self): print_warning(f"Failed to reexecute Python: {exc!r}\n" f"Command: {cmd_text}") - def main(self, tests: TestList | None = None): - if self.want_reexec and self.ci_mode: - self._reexecute_python() + def _init(self): + # Set sys.stdout encoder error handler to backslashreplace, + # similar to sys.stderr error handler, to avoid UnicodeEncodeError + # when printing a traceback or any other non-encodable character. + sys.stdout.reconfigure(errors="backslashreplace") if self.junit_filename and not os.path.isabs(self.junit_filename): self.junit_filename = os.path.abspath(self.junit_filename) @@ -524,6 +526,12 @@ def main(self, tests: TestList | None = None): self.tmp_dir = get_temp_dir(self.tmp_dir) + def main(self, tests: TestList | None = None): + if self.want_reexec and self.ci_mode: + self._reexecute_python() + + self._init() + if self.want_cleanup: cleanup_temp_dir(self.tmp_dir) sys.exit(0) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 204f10fe839792..f0d8d7ebaa2fdb 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,7 +11,7 @@ from .runtests import RunTests from .utils import ( setup_unraisable_hook, setup_threading_excepthook, fix_umask, - replace_stdout, adjust_rlimit_nofile) + adjust_rlimit_nofile) UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" @@ -49,7 +49,7 @@ def setup_process(): faulthandler.register(signum, chain=True, file=stderr_fd) adjust_rlimit_nofile() - replace_stdout() + support.record_original_stdout(sys.stdout) # Some times __path__ and __file__ are not absolute (e.g. while running from diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index f3f0eb53b32100..acf35723a1abb0 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,4 +1,3 @@ -import atexit import contextlib import faulthandler import locale @@ -495,32 +494,6 @@ def normalize_test_name(test_full_name, *, is_error=False): return short_name -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - stdout = sys.stdout - try: - fd = stdout.fileno() - except ValueError: - # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper - # object. Leaving sys.stdout unchanged. - # - # Catch ValueError to catch io.UnsupportedOperation on TextIOBase - # and ValueError on a closed stream. - return - - sys.stdout = open(fd, 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - - def adjust_rlimit_nofile(): """ On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) @@ -548,20 +521,12 @@ def adjust_rlimit_nofile(): def display_header(use_resources: tuple[str, ...]): - encoding = sys.stdout.encoding - # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), "%s-endian" % sys.byteorder) print("== Python build:", ' '.join(get_build_info())) - - cwd = os.getcwd() - # gh-109508: support.os_helper.FS_NONASCII, used by get_work_dir(), cannot - # be encoded to the filesystem encoding on purpose, escape non-encodable - # characters with backslashreplace error handler. - formatted_cwd = cwd.encode(encoding, "backslashreplace").decode(encoding) - print("== cwd:", formatted_cwd) + print("== cwd:", os.getcwd()) cpu_count = os.cpu_count() if cpu_count: @@ -588,18 +553,18 @@ def display_header(use_resources: tuple[str, ...]): sanitizers.append("memory") if ubsan: sanitizers.append("undefined behavior") - if not sanitizers: - return - - print(f"== sanitizers: {', '.join(sanitizers)}") - for sanitizer, env_var in ( - (asan, "ASAN_OPTIONS"), - (msan, "MSAN_OPTIONS"), - (ubsan, "UBSAN_OPTIONS"), - ): - options= os.environ.get(env_var) - if sanitizer and options is not None: - print(f"== {env_var}={options!r}") + if sanitizers: + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") + + print(flush=True) def cleanup_temp_dir(tmp_dir: StrPath): From 2ef2fffe3be953b91852585c75188d5475b09474 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 26 Sep 2023 22:58:46 +0300 Subject: [PATCH 017/124] gh-109845: Make test_ftplib more stable under load (GH-109912) recv() can return partial data cut in the middle of a multibyte character. Test raw binary data instead of data incorrectly decoded by parts. --- Lib/test/test_ftplib.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index bebd1bbb9e2703..2f191ea7a44c16 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -32,7 +32,7 @@ DEFAULT_ENCODING = 'utf-8' # the dummy data returned by server over the data channel when # RETR, LIST, NLST, MLSD commands are issued -RETR_DATA = 'abcde12345\r\n' * 1000 + 'non-ascii char \xAE\r\n' +RETR_DATA = 'abcde\xB9\xB2\xB3\xA4\xA6\r\n' * 1000 LIST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n' NLST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n' MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" @@ -67,11 +67,11 @@ class DummyDTPHandler(asynchat.async_chat): def __init__(self, conn, baseclass): asynchat.async_chat.__init__(self, conn) self.baseclass = baseclass - self.baseclass.last_received_data = '' + self.baseclass.last_received_data = bytearray() self.encoding = baseclass.encoding def handle_read(self): - new_data = self.recv(1024).decode(self.encoding, 'replace') + new_data = self.recv(1024) self.baseclass.last_received_data += new_data def handle_close(self): @@ -107,7 +107,7 @@ def __init__(self, conn, encoding=DEFAULT_ENCODING): self.in_buffer = [] self.dtp = None self.last_received_cmd = None - self.last_received_data = '' + self.last_received_data = bytearray() self.next_response = '' self.next_data = None self.rest = None @@ -590,19 +590,17 @@ def test_abort(self): self.client.abort() def test_retrbinary(self): - def callback(data): - received.append(data.decode(self.client.encoding)) received = [] - self.client.retrbinary('retr', callback) - self.check_data(''.join(received), RETR_DATA) + self.client.retrbinary('retr', received.append) + self.check_data(b''.join(received), + RETR_DATA.encode(self.client.encoding)) def test_retrbinary_rest(self): - def callback(data): - received.append(data.decode(self.client.encoding)) for rest in (0, 10, 20): received = [] - self.client.retrbinary('retr', callback, rest=rest) - self.check_data(''.join(received), RETR_DATA[rest:]) + self.client.retrbinary('retr', received.append, rest=rest) + self.check_data(b''.join(received), + RETR_DATA[rest:].encode(self.client.encoding)) def test_retrlines(self): received = [] @@ -612,7 +610,8 @@ def test_retrlines(self): def test_storbinary(self): f = io.BytesIO(RETR_DATA.encode(self.client.encoding)) self.client.storbinary('stor', f) - self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + self.check_data(self.server.handler_instance.last_received_data, + RETR_DATA.encode(self.server.encoding)) # test new callback arg flag = [] f.seek(0) @@ -631,7 +630,8 @@ def test_storlines(self): data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding) f = io.BytesIO(data) self.client.storlines('stor', f) - self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + self.check_data(self.server.handler_instance.last_received_data, + RETR_DATA.encode(self.server.encoding)) # test new callback arg flag = [] f.seek(0) @@ -649,7 +649,7 @@ def test_nlst(self): def test_dir(self): l = [] - self.client.dir(lambda x: l.append(x)) + self.client.dir(l.append) self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) def test_mlsd(self): @@ -889,12 +889,10 @@ def test_makepasv(self): def test_transfer(self): def retr(): - def callback(data): - received.append(data.decode(self.client.encoding)) received = [] - self.client.retrbinary('retr', callback) - self.assertEqual(len(''.join(received)), len(RETR_DATA)) - self.assertEqual(''.join(received), RETR_DATA) + self.client.retrbinary('retr', received.append) + self.assertEqual(b''.join(received), + RETR_DATA.encode(self.client.encoding)) self.client.set_pasv(True) retr() self.client.set_pasv(False) From 4390c131489db6169ca0a709b196935c7eac0b5f Mon Sep 17 00:00:00 2001 From: OmniTroid Date: Tue, 26 Sep 2023 22:22:00 +0200 Subject: [PATCH 018/124] Fix argument ordering of embuilder command documented in `Tools/wasm/README.md` (GH-109863) --- Tools/wasm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index e6dd4d5f00abde..8ef63c6dcd9ddc 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -79,7 +79,7 @@ PIC. To populate the build cache, run: ```shell . /opt/emsdk/emsdk_env.sh embuilder build zlib bzip2 MINIMAL_PIC -embuilder build --pic zlib bzip2 MINIMAL_PIC +embuilder --pic build zlib bzip2 MINIMAL_PIC ``` From 87ddfa74e2d37f6837351ed2bafc7d6d55fe2fd0 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 26 Sep 2023 22:24:44 +0100 Subject: [PATCH 019/124] GH-109190: Copyedit 3.12 What's New: Deprecations (#109766) --- Doc/whatsnew/3.12.rst | 245 ++++++++++++++++++++++++++---------------- 1 file changed, 153 insertions(+), 92 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4874a6c77faa99..dfe2ea886a9209 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -580,19 +580,10 @@ asyncio :class:`asyncio.ThreadedChildWatcher`. (Contributed by Kumar Aditya in :gh:`98024`.) -* The child watcher classes :class:`asyncio.MultiLoopChildWatcher`, - :class:`asyncio.FastChildWatcher`, :class:`asyncio.AbstractChildWatcher` - and :class:`asyncio.SafeChildWatcher` are deprecated and - will be removed in Python 3.14. It is recommended to not manually - configure a child watcher as the event loop now uses the best available - child watcher for each platform (:class:`asyncio.PidfdChildWatcher` - if supported and :class:`asyncio.ThreadedChildWatcher` otherwise). - (Contributed by Kumar Aditya in :gh:`94597`.) - -* :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, - :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and - :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated - and will be removed in Python 3.14. +* The event loop now uses the best available child watcher for each platform + (:class:`asyncio.PidfdChildWatcher` if supported and + :class:`asyncio.ThreadedChildWatcher` otherwise), so manually + configuring a child watcher is not recommended. (Contributed by Kumar Aditya in :gh:`94597`.) * Add *loop_factory* parameter to :func:`asyncio.run` to allow specifying @@ -1046,15 +1037,52 @@ Demos and Tools Deprecated ========== -* :mod:`asyncio`: The :meth:`~asyncio.get_event_loop` method of the - default event loop policy now emits a :exc:`DeprecationWarning` if there - is no current event loop set and it decides to create one. - (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following :mod:`ast` features have been deprecated in documentation since + Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime + when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`asyncio`: + + * The child watcher classes :class:`asyncio.MultiLoopChildWatcher`, + :class:`asyncio.FastChildWatcher`, :class:`asyncio.AbstractChildWatcher` + and :class:`asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, + :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * The :meth:`~asyncio.get_event_loop` method of the + default event loop policy now emits a :exc:`DeprecationWarning` if there + is no current event loop set and it decides to create one. + (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) * :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are deprecated and replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + * :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and :meth:`~datetime.datetime.utcfromtimestamp` are deprecated and will be removed in a future version. Instead, use timezone-aware objects to represent @@ -1063,12 +1091,55 @@ Deprecated :const:`datetime.UTC`. (Contributed by Paul Ganssle in :gh:`103857`.) +* :mod:`email`: Deprecate the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib.abc`: Deprecated the following classes, scheduled for removal in + Python 3.14: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +* :mod:`itertools`: Deprecate the support for copy, deepcopy, and pickle operations, + which is undocumented, inefficient, historically buggy, and inconsistent. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) + * :mod:`os`: The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on Windows are deprecated. In a future release, they will contain the last metadata change time, consistent with other platforms. For now, they still contain the creation time, which is also available in the new ``st_birthtime`` field. (Contributed by Steve Dower in :gh:`99726`.) +* :mod:`multiprocessing`: In Python 3.14, the default :mod:`multiprocessing` + start method will change to a safer one on Linux, BSDs, + and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`contexts and start methods + `. + +* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` + are deprecated and will be removed in Python 3.14; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: The module has two undocumented ``master_open()`` and ``slave_open()`` + functions that have been deprecated since Python 2 but only gained a + proper :exc:`DeprecationWarning` in 3.12. Remove them in 3.14. + (Contributed by Soumendra Ganguly and Gregory P. Smith in :gh:`85984`.) + * :mod:`os`: On POSIX platforms, :func:`os.fork` can now raise a :exc:`DeprecationWarning` when it can detect being called from a multithreaded process. There has always been a fundamental incompatibility @@ -1083,22 +1154,23 @@ Deprecated :mod:`concurrent.futures` the fix is to use a different :mod:`multiprocessing` start method such as ``"spawn"`` or ``"forkserver"``. -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated as will be removed +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated and will be removed in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) * :mod:`sqlite3`: - * :ref:`default adapters and converters - ` are now deprecated. - Instead, use the :ref:`sqlite3-adapter-converter-recipes` - and tailor them to your needs. - (Contributed by Erlend E. Aasland in :gh:`90016`.) - - * In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted - when :ref:`named placeholders ` are used together with - parameters supplied as a :term:`sequence` instead of as a :class:`dict`. - Starting from Python 3.14, using named placeholders with parameters supplied - as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. - (Contributed by Erlend E. Aasland in :gh:`101698`.) + + * :ref:`default adapters and converters + ` are now deprecated. + Instead, use the :ref:`sqlite3-adapter-converter-recipes` + and tailor them to your needs. + (Contributed by Erlend E. Aasland in :gh:`90016`.) + + * In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted + when :ref:`named placeholders ` are used together with + parameters supplied as a :term:`sequence` instead of as a :class:`dict`. + Starting from Python 3.14, using named placeholders with parameters supplied + as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`101698`.) * :mod:`sys`: The :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback` fields are deprecated. Use :data:`sys.last_exc` instead. @@ -1108,16 +1180,24 @@ Deprecated Python 3.14, when ``'data'`` filter will become the default. See :ref:`tarfile-extraction-filter` for details. -* :mod:`typing`: :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` - and :class:`collections.abc.Sized`. (:gh:`94309`.) +* :mod:`typing`: + + * :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` + and :class:`collections.abc.Sized`. (:gh:`94309`.) + + * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + :exc:`DeprecationWarning` to be emitted when it is used. + (Contributed by Alex Waygood in :gh:`91896`.) * :mod:`xml.etree.ElementTree`: The module now emits :exc:`DeprecationWarning` when testing the truth value of an :class:`xml.etree.ElementTree.Element`. Before, the Python implementation emitted :exc:`FutureWarning`, and the C implementation emitted nothing. + (Contributed by Jacob Walls in :gh:`83122`.) -* The 3-arg signatures (type, value, traceback) of :meth:`~coroutine.throw`, - :meth:`~generator.throw` and :meth:`~agen.athrow` are deprecated and +* The 3-arg signatures (type, value, traceback) of :meth:`coroutine throw() + `, :meth:`generator throw() ` and + :meth:`async generator throw() ` are deprecated and may be removed in a future version of Python. Use the single-arg versions of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) @@ -1126,12 +1206,21 @@ Deprecated :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) +* Setting ``__package__`` or ``__cached__`` on a module is deprecated, + and will cease to be set or taken into consideration by the import system in Python 3.14. + (Contributed by Brett Cannon in :gh:`65961`.) + * The bitwise inversion operator (``~``) on bool is deprecated. It will throw an error in Python 3.14. Use ``not`` for logical negation of bools instead. In the rare case that you really need the bitwise inversion of the underlying - ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann + ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) +* Accessing ``co_lnotab`` on code objects was deprecated in Python 3.10 via :pep:`626`, + but it only got a proper :exc:`DeprecationWarning` in 3.12, + therefore it will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`101866`.) + Pending Removal in Python 3.13 ------------------------------ @@ -1180,14 +1269,13 @@ APIs: Pending Removal in Python 3.14 ------------------------------ +The following APIs have been deprecated +and will be removed in Python 3.14. + * :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` are deprecated - and will be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`92248`.) + of :class:`!argparse.BooleanOptionalAction` -* :mod:`ast`: The following :mod:`ast` features have been deprecated in documentation since - Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime - when they are accessed or used, and will be removed in Python 3.14: +* :mod:`ast`: * :class:`!ast.Num` * :class:`!ast.Str` @@ -1195,75 +1283,48 @@ Pending Removal in Python 3.14 * :class:`!ast.NameConstant` * :class:`!ast.Ellipsis` - Use :class:`ast.Constant` instead. - (Contributed by Serhiy Storchaka in :gh:`90953`.) +* :mod:`asyncio`: -* :mod:`asyncio`: the *msg* parameter of both - :meth:`asyncio.Future.cancel` and - :meth:`asyncio.Task.cancel` (:gh:`90985`) + * :class:`!asyncio.MultiLoopChildWatcher` + * :class:`!asyncio.FastChildWatcher` + * :class:`!asyncio.AbstractChildWatcher` + * :class:`!asyncio.SafeChildWatcher` + * :func:`!asyncio.set_child_watcher` + * :func:`!asyncio.get_child_watcher`, + * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` -* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) +* :mod:`collections.abc`: :class:`!collections.abc.ByteString`. -* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. - (Contributed by Alan Williams in :gh:`72346`.) +* :mod:`email`: the *isdst* parameter in :func:`email.utils.localtime`. -* :mod:`importlib.abc`: Deprecated the following classes, scheduled for removal in - Python 3.14: +* :mod:`importlib.abc`: * :class:`!importlib.abc.ResourceReader` * :class:`!importlib.abc.Traversable` * :class:`!importlib.abc.TraversableResources` - Use :mod:`importlib.resources.abc` classes instead: +* :mod:`itertools`: Support for copy, deepcopy, and pickle operations. - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` - - (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) - -* :mod:`itertools`: The module had undocumented, inefficient, historically buggy, - and inconsistent support for copy, deepcopy, and pickle operations. - This will be removed in 3.14 for a significant reduction in code - volume and maintenance burden. - (Contributed by Raymond Hettinger in :gh:`101588`.) +* :mod:`pkgutil`: -* :mod:`multiprocessing`: The default :mod:`multiprocessing` start method will change to a safer one on - Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently - the default (:gh:`84559`). Adding a runtime warning about this was deemed too - disruptive as the majority of code is not expected to care. Use the - :func:`~multiprocessing.get_context` or - :func:`~multiprocessing.set_start_method` APIs to explicitly specify when - your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + * :func:`!pkgutil.find_loader` + * :func:`!pkgutil.get_loader`. -* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` - now raise :exc:`DeprecationWarning`; - use :func:`importlib.util.find_spec` instead. - (Contributed by Nikita Sobolev in :gh:`97850`.) +* :mod:`pty`: -* :mod:`pty`: The module has two undocumented ``master_open()`` and ``slave_open()`` - functions that have been deprecated since Python 2 but only gained a - proper :exc:`DeprecationWarning` in 3.12. Remove them in 3.14. + * :func:`!pty.master_open` + * :func:`!pty.slave_open` -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated in 3.12, - and will be removed in 3.14. +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` -* :mod:`typing`: :class:`typing.ByteString`, deprecated since Python 3.9, now causes a - :exc:`DeprecationWarning` to be emitted when it is used. +* :mod:`typing`: :class:`!typing.ByteString` -* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element` - is deprecated and will raise an exception in Python 3.14. +* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`. -* ``__package__`` and ``__cached__`` will cease to be set or taken - into consideration by the import system (:gh:`97879`). +* The ``__package__`` and ``__cached__`` attributes on module objects. -* Accessing ``co_lnotab`` was deprecated in :pep:`626` since 3.10 - and was planned to be removed in 3.12 - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) +* The ``co_lnotab`` attribute of code objects. Pending Removal in Future Versions ---------------------------------- From 9abba715e3225b8e4c4b7dd0ed528ef3a3057bea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 23:59:11 +0200 Subject: [PATCH 020/124] gh-109566: Fix regrtest code adding Python options (#109926) * On Windows, use subprocess.run() instead of os.execv(). * Only add needed options * Rename reexec parameter to _add_python_opts. * Rename --no-reexec option to --dont-add-python-opts. --- Lib/test/__main__.py | 2 +- Lib/test/libregrtest/cmdline.py | 7 ++-- Lib/test/libregrtest/main.py | 65 +++++++++++++++++++++------------ Lib/test/test_regrtest.py | 26 +++++++------ 4 files changed, 62 insertions(+), 38 deletions(-) diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 42553fa32867d3..82b50ad2c6e777 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,2 @@ from test.libregrtest.main import main -main(reexec=True) +main(_add_python_opts=True) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index bc969969e068eb..c180bb76222a89 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -184,7 +184,7 @@ def __init__(self, **kwargs) -> None: self.threshold = None self.fail_rerun = False self.tempdir = None - self.no_reexec = False + self._add_python_opts = True super().__init__(**kwargs) @@ -344,7 +344,8 @@ def _create_parser(): help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', help='remove old test_python_* directories') - group.add_argument('--no-reexec', action='store_true', + group.add_argument('--dont-add-python-opts', dest='_add_python_opts', + action='store_false', help="internal option, don't use it") return parser @@ -425,7 +426,7 @@ def _parse_args(args, **kwargs): if MS_WINDOWS: ns.nowindows = True # Silence alerts under Windows else: - ns.no_reexec = True + ns._add_python_opts = False # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1cb22afcc9900..c31d5ff187c56a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -48,7 +48,7 @@ class Regrtest: directly to set the values that would normally be set by flags on the command line. """ - def __init__(self, ns: Namespace, reexec: bool = False): + def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Log verbosity self.verbose: int = int(ns.verbose) self.quiet: bool = ns.quiet @@ -70,7 +70,11 @@ def __init__(self, ns: Namespace, reexec: bool = False): self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks - self.want_reexec: bool = (reexec and not ns.no_reexec) + + ci_mode = (ns.fast_ci or ns.slow_ci) + self.want_add_python_opts: bool = (_add_python_opts + and ns._add_python_opts + and ci_mode) # Select tests if ns.match_tests: @@ -97,7 +101,6 @@ def __init__(self, ns: Namespace, reexec: bool = False): self.worker_json: StrJSON | None = ns.worker_json # Options to run tests - self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.fail_fast: bool = ns.failfast self.fail_env_changed: bool = ns.fail_env_changed self.fail_rerun: bool = ns.fail_rerun @@ -486,32 +489,48 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: # processes. return self._run_tests(selected, tests) - def _reexecute_python(self): - if self.python_cmd: - # Do nothing if --python=cmd option is used - return + def _add_python_opts(self): + python_opts = [] + + # Unbuffered stdout and stderr + if not sys.stdout.write_through: + python_opts.append('-u') + + # Add warnings filter 'default' + if 'default' not in sys.warnoptions: + python_opts.extend(('-W', 'default')) - python_opts = [ - '-u', # Unbuffered stdout and stderr - '-W', 'default', # Add warnings filter 'default' - '-bb', # Error on bytes/str comparison - '-E', # Ignore PYTHON* environment variables - ] + # Error on bytes/str comparison + if sys.flags.bytes_warning < 2: + python_opts.append('-bb') - cmd = [*sys.orig_argv, "--no-reexec"] + # Ignore PYTHON* environment variables + if not sys.flags.ignore_environment: + python_opts.append('-E') + + if not python_opts: + return + + cmd = [*sys.orig_argv, "--dont-add-python-opts"] cmd[1:1] = python_opts # Make sure that messages before execv() are logged sys.stdout.flush() sys.stderr.flush() + cmd_text = shlex.join(cmd) try: - os.execv(cmd[0], cmd) - # execv() do no return and so we don't get to this line on success - except OSError as exc: - cmd_text = shlex.join(cmd) - print_warning(f"Failed to reexecute Python: {exc!r}\n" + if hasattr(os, 'execv') and not MS_WINDOWS: + os.execv(cmd[0], cmd) + # execv() do no return and so we don't get to this line on success + else: + import subprocess + proc = subprocess.run(cmd) + sys.exit(proc.returncode) + except Exception as exc: + print_warning(f"Failed to change Python options: {exc!r}\n" f"Command: {cmd_text}") + # continue executing main() def _init(self): # Set sys.stdout encoder error handler to backslashreplace, @@ -527,8 +546,8 @@ def _init(self): self.tmp_dir = get_temp_dir(self.tmp_dir) def main(self, tests: TestList | None = None): - if self.want_reexec and self.ci_mode: - self._reexecute_python() + if self.want_add_python_opts: + self._add_python_opts() self._init() @@ -556,7 +575,7 @@ def main(self, tests: TestList | None = None): sys.exit(exitcode) -def main(tests=None, reexec=False, **kwargs): +def main(tests=None, _add_python_opts=False, **kwargs): """Run the Python suite.""" ns = _parse_args(sys.argv[1:], **kwargs) - Regrtest(ns, reexec=reexec).main(tests=tests) + Regrtest(ns, _add_python_opts=_add_python_opts).main(tests=tests) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2b77300c079c05..3ece31be9af3c3 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -382,7 +382,6 @@ def check_ci_mode(self, args, use_resources): # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API regrtest = main.Regrtest(ns) - self.assertTrue(regrtest.ci_mode) self.assertEqual(regrtest.num_workers, -1) self.assertTrue(regrtest.want_rerun) self.assertTrue(regrtest.randomize) @@ -413,6 +412,11 @@ def test_slow_ci(self): regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 20 * 60) + def test_dont_add_python_opts(self): + args = ['--dont-add-python-opts'] + ns = cmdline._parse_args(args) + self.assertFalse(ns._add_python_opts) + @dataclasses.dataclass(slots=True) class Rerun: @@ -1984,22 +1988,23 @@ def test_config(self): # -E option self.assertTrue(config['use_environment'], 0) - # test if get_config() is not available - def test_unbuffered(self): + def test_python_opts(self): # -u option - self.assertFalse(sys.stdout.line_buffering) - self.assertFalse(sys.stderr.line_buffering) + self.assertTrue(sys.__stdout__.write_through) + self.assertTrue(sys.__stderr__.write_through) - def test_python_opts(self): # -W default option self.assertTrue(sys.warnoptions, ['default']) + # -bb option self.assertEqual(sys.flags.bytes_warning, 2) + # -E option self.assertTrue(sys.flags.ignore_environment) """) testname = self.create_test(code=code) + # Use directly subprocess to control the exact command line cmd = [sys.executable, "-m", "test", option, f'--testdir={self.tmptestdir}', @@ -2010,11 +2015,10 @@ def test_python_opts(self): text=True) self.assertEqual(proc.returncode, 0, proc) - def test_reexec_fast_ci(self): - self.check_reexec("--fast-ci") - - def test_reexec_slow_ci(self): - self.check_reexec("--slow-ci") + def test_add_python_opts(self): + for opt in ("--fast-ci", "--slow-ci"): + with self.subTest(opt=opt): + self.check_reexec(opt) class TestUtils(unittest.TestCase): From 9dbfe2dc8e7bba25e52f9470ae6969821a365297 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 00:26:34 +0200 Subject: [PATCH 021/124] gh-107888: Fix test_mmap.test_access_parameter() on macOS 14 (#109928) --- Lib/test/test_mmap.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index bab868600895c1..92c99d645b25cc 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -255,10 +255,15 @@ def test_access_parameter(self): # Try writing with PROT_EXEC and without PROT_WRITE prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0) with open(TESTFN, "r+b") as f: - m = mmap.mmap(f.fileno(), mapsize, prot=prot) - self.assertRaises(TypeError, m.write, b"abcdef") - self.assertRaises(TypeError, m.write_byte, 0) - m.close() + try: + m = mmap.mmap(f.fileno(), mapsize, prot=prot) + except PermissionError: + # on macOS 14, PROT_READ | PROT_WRITE is not allowed + pass + else: + self.assertRaises(TypeError, m.write, b"abcdef") + self.assertRaises(TypeError, m.write_byte, 0) + m.close() def test_bad_file_desc(self): # Try opening a bad file descriptor... From a829356f86d597e4dfe92e236a6d711c8a464f16 Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Tue, 26 Sep 2023 18:35:49 -0400 Subject: [PATCH 022/124] gh-109098: Fuzz re module instead of internal sre (#109911) * gh-109098: Fuzz re module instead of internal sre * Fix c-analyzer globals test failure * Put globals exception in ignored.tsv --- Modules/_xxtestfuzz/fuzzer.c | 45 +++++++++++----------------- Tools/c-analyzer/cpython/ignored.tsv | 6 ++-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 54f8a42273401f..816ba09c8fd7de 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -193,37 +193,33 @@ static int fuzz_json_loads(const char* data, size_t size) { #define MAX_RE_TEST_SIZE 0x10000 -PyObject* sre_compile_method = NULL; -PyObject* sre_error_exception = NULL; -int SRE_FLAG_DEBUG = 0; +PyObject* re_compile_method = NULL; +PyObject* re_error_exception = NULL; +int RE_FLAG_DEBUG = 0; /* Called by LLVMFuzzerTestOneInput for initialization */ static int init_sre_compile(void) { /* Import sre_compile.compile and sre.error */ - PyObject* sre_compile_module = PyImport_ImportModule("sre_compile"); - if (sre_compile_module == NULL) { + PyObject* re_module = PyImport_ImportModule("re"); + if (re_module == NULL) { return 0; } - sre_compile_method = PyObject_GetAttrString(sre_compile_module, "compile"); - if (sre_compile_method == NULL) { + re_compile_method = PyObject_GetAttrString(re_module, "compile"); + if (re_compile_method == NULL) { return 0; } - PyObject* sre_constants = PyImport_ImportModule("sre_constants"); - if (sre_constants == NULL) { + re_error_exception = PyObject_GetAttrString(re_module, "error"); + if (re_error_exception == NULL) { return 0; } - sre_error_exception = PyObject_GetAttrString(sre_constants, "error"); - if (sre_error_exception == NULL) { - return 0; - } - PyObject* debug_flag = PyObject_GetAttrString(sre_constants, "SRE_FLAG_DEBUG"); + PyObject* debug_flag = PyObject_GetAttrString(re_module, "DEBUG"); if (debug_flag == NULL) { return 0; } - SRE_FLAG_DEBUG = PyLong_AsLong(debug_flag); + RE_FLAG_DEBUG = PyLong_AsLong(debug_flag); return 1; } -/* Fuzz _sre.compile(x) */ +/* Fuzz re.compile(x) */ static int fuzz_sre_compile(const char* data, size_t size) { /* Ignore really long regex patterns that will timeout the fuzzer */ if (size > MAX_RE_TEST_SIZE) { @@ -236,7 +232,7 @@ static int fuzz_sre_compile(const char* data, size_t size) { uint16_t flags = ((uint16_t*) data)[0]; /* We remove the SRE_FLAG_DEBUG if present. This is because it prints to stdout which greatly decreases fuzzing speed */ - flags &= ~SRE_FLAG_DEBUG; + flags &= ~RE_FLAG_DEBUG; /* Pull the pattern from the remaining bytes */ PyObject* pattern_bytes = PyBytes_FromStringAndSize(data + 2, size - 2); @@ -249,9 +245,9 @@ static int fuzz_sre_compile(const char* data, size_t size) { return 0; } - /* compiled = _sre.compile(data[2:], data[0:2] */ + /* compiled = re.compile(data[2:], data[0:2] */ PyObject* compiled = PyObject_CallFunctionObjArgs( - sre_compile_method, pattern_bytes, flags_obj, NULL); + re_compile_method, pattern_bytes, flags_obj, NULL); /* Ignore ValueError as the fuzzer will more than likely generate some invalid combination of flags */ if (compiled == NULL && PyErr_ExceptionMatches(PyExc_ValueError)) { @@ -267,7 +263,7 @@ static int fuzz_sre_compile(const char* data, size_t size) { PyErr_Clear(); } /* Ignore re.error */ - if (compiled == NULL && PyErr_ExceptionMatches(sre_error_exception)) { + if (compiled == NULL && PyErr_ExceptionMatches(re_error_exception)) { PyErr_Clear(); } @@ -531,13 +527,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_compile) static int SRE_COMPILE_INITIALIZED = 0; if (!SRE_COMPILE_INITIALIZED && !init_sre_compile()) { - if (!PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { - PyErr_Print(); - abort(); - } - else { - PyErr_Clear(); - } + PyErr_Print(); + abort(); } else { SRE_COMPILE_INITIALIZED = 1; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 336b0281bda85d..1f398701a7a5b5 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -580,15 +580,15 @@ Modules/_testmultiphase.c - uninitialized_def - Modules/_testsinglephase.c - global_state - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - -Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - +Modules/_xxtestfuzz/fuzzer.c - RE_FLAG_DEBUG - Modules/_xxtestfuzz/fuzzer.c - ast_literal_eval_method - Modules/_xxtestfuzz/fuzzer.c - compiled_patterns - Modules/_xxtestfuzz/fuzzer.c - csv_error - Modules/_xxtestfuzz/fuzzer.c - csv_module - Modules/_xxtestfuzz/fuzzer.c - json_loads_method - Modules/_xxtestfuzz/fuzzer.c - regex_patterns - -Modules/_xxtestfuzz/fuzzer.c - sre_compile_method - -Modules/_xxtestfuzz/fuzzer.c - sre_error_exception - +Modules/_xxtestfuzz/fuzzer.c - re_compile_method - +Modules/_xxtestfuzz/fuzzer.c - re_error_exception - Modules/_xxtestfuzz/fuzzer.c - struct_error - Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - From 3538930d87e6bdd2bfffa3f674a62cc91d359d31 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 02:01:48 +0200 Subject: [PATCH 023/124] gh-101100: Fix Sphinx warnings in Doc/using/configure.rst (#109931) --- Doc/tools/.nitignore | 1 - Doc/using/configure.rst | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f260a571661b76..1dd7676eab057e 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -153,7 +153,6 @@ Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst Doc/using/cmdline.rst -Doc/using/configure.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index a9555199a2ac24..9403c19a695776 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -99,8 +99,8 @@ General Options .. cmdoption:: --enable-loadable-sqlite-extensions - Support loadable extensions in the :mod:`_sqlite` extension module (default - is no). + Support loadable extensions in the :mod:`!_sqlite` extension module (default + is no) of the :mod:`sqlite3` module. See the :meth:`sqlite3.Connection.enable_load_extension` method of the :mod:`sqlite3` module. @@ -198,7 +198,7 @@ General Options Some Linux distribution packaging policies recommend against bundling dependencies. For example, Fedora installs wheel packages in the ``/usr/share/python-wheels/`` directory and don't install the - :mod:`ensurepip._bundled` package. + :mod:`!ensurepip._bundled` package. .. versionadded:: 3.10 @@ -469,7 +469,7 @@ Install Options .. cmdoption:: --disable-test-modules Don't build nor install test modules, like the :mod:`test` package or the - :mod:`_testcapi` extension module (built and installed by default). + :mod:`!_testcapi` extension module (built and installed by default). .. versionadded:: 3.10 @@ -615,7 +615,7 @@ Effects of a debug build: * Display all warnings by default: the list of default warning filters is empty in the :mod:`warnings` module. * Add ``d`` to :data:`sys.abiflags`. -* Add :func:`sys.gettotalrefcount` function. +* Add :func:`!sys.gettotalrefcount` function. * Add :option:`-X showrefcount <-X>` command line option. * Add :option:`-d` command line option and :envvar:`PYTHONDEBUG` environment variable to debug the parser. @@ -637,7 +637,7 @@ Effects of a debug build: * Check that deallocator functions don't change the current exception. * The garbage collector (:func:`gc.collect` function) runs some basic checks on objects consistency. - * The :c:macro:`Py_SAFE_DOWNCAST()` macro checks for integer underflow and + * The :c:macro:`!Py_SAFE_DOWNCAST()` macro checks for integer underflow and overflow when downcasting from wide types to narrow types. See also the :ref:`Python Development Mode ` and the @@ -664,7 +664,7 @@ Debug options Effects: * Define the ``Py_TRACE_REFS`` macro. - * Add :func:`sys.getobjects` function. + * Add :func:`!sys.getobjects` function. * Add :envvar:`PYTHONDUMPREFS` environment variable. The :envvar:`PYTHONDUMPREFS` environment variable can be used to dump @@ -748,7 +748,7 @@ Libraries options .. cmdoption:: --with-system-expat - Build the :mod:`pyexpat` module using an installed ``expat`` library + Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). .. cmdoption:: --with-system-libmpdec From e721f7a95186452339dc9e57630d639d549b2521 Mon Sep 17 00:00:00 2001 From: Tom Gillespie Date: Tue, 26 Sep 2023 23:34:15 -0400 Subject: [PATCH 024/124] Remove loop from docstring for asyncio.streams.open_connection (#108528) --- Lib/asyncio/streams.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index b7ad365709b19e..bc84e53b8443cf 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -67,9 +67,8 @@ async def start_server(client_connected_cb, host=None, port=None, *, positional host and port, with various optional keyword arguments following. The return value is the same as loop.create_server(). - Additional optional keyword arguments are loop (to set the event loop - instance to use) and limit (to set the buffer limit passed to the - StreamReader). + Additional optional keyword argument is limit (to set the buffer + limit passed to the StreamReader). The return value is the same as loop.create_server(), i.e. a Server object which can be used to stop the service. From 0e28d0f7a1bc3776cc07e0f8b91bc43fcdbb4206 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 27 Sep 2023 05:59:42 +0100 Subject: [PATCH 025/124] GH-109190: Copyedit 3.12 What's New: Deprecations (``os`` fix) (#109927) Merge the two ``os`` entries --- Doc/whatsnew/3.12.rst | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index dfe2ea886a9209..cad8dd7db7228a 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1114,12 +1114,6 @@ Deprecated volume and maintenance burden. (Contributed by Raymond Hettinger in :gh:`101588`.) -* :mod:`os`: The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on - Windows are deprecated. In a future release, they will contain the last - metadata change time, consistent with other platforms. For now, they still - contain the creation time, which is also available in the new ``st_birthtime`` - field. (Contributed by Steve Dower in :gh:`99726`.) - * :mod:`multiprocessing`: In Python 3.14, the default :mod:`multiprocessing` start method will change to a safer one on Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently @@ -1140,15 +1134,23 @@ Deprecated proper :exc:`DeprecationWarning` in 3.12. Remove them in 3.14. (Contributed by Soumendra Ganguly and Gregory P. Smith in :gh:`85984`.) -* :mod:`os`: On POSIX platforms, :func:`os.fork` can now raise a - :exc:`DeprecationWarning` when it can detect being called from a - multithreaded process. There has always been a fundamental incompatibility - with the POSIX platform when doing so. Even if such code *appeared* to work. - We added the warning to to raise awareness as issues encounted by code doing - this are becoming more frequent. See the :func:`os.fork` documentation for - more details along with `this discussion on fork being incompatible with threads - `_ for *why* we're now surfacing this - longstanding platform compatibility problem to developers. +* :mod:`os`: + + * The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on + Windows are deprecated. In a future release, they will contain the last + metadata change time, consistent with other platforms. For now, they still + contain the creation time, which is also available in the new ``st_birthtime`` + field. (Contributed by Steve Dower in :gh:`99726`.) + + * On POSIX platforms, :func:`os.fork` can now raise a + :exc:`DeprecationWarning` when it can detect being called from a + multithreaded process. There has always been a fundamental incompatibility + with the POSIX platform when doing so. Even if such code *appeared* to work. + We added the warning to to raise awareness as issues encounted by code doing + this are becoming more frequent. See the :func:`os.fork` documentation for + more details along with `this discussion on fork being incompatible with threads + `_ for *why* we're now surfacing this + longstanding platform compatibility problem to developers. When this warning appears due to usage of :mod:`multiprocessing` or :mod:`concurrent.futures` the fix is to use a different From 1512d6c6ee2a770afb339bbb74c1b990116f7f89 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 10:18:39 +0200 Subject: [PATCH 026/124] gh-109615: Fix test_tools.test_freeze SRCDIR (#109935) Fix copy_source_tree() function of test_tools.test_freeze: * Don't copy SRC_DIR/build/ anymore. This directory is modified by other tests running in parallel. * Add test.support.copy_python_src_ignore(). * Use sysconfig to get the source directory. * Use sysconfig.get_config_var() to get CONFIG_ARGS variable. --- Lib/test/libregrtest/utils.py | 2 +- Lib/test/support/__init__.py | 27 ++++++++++++++++++ Lib/test/test_support.py | 22 ++++++++++++++ Lib/test/test_venv.py | 10 ++----- Tools/freeze/test/freeze.py | 54 +++++------------------------------ 5 files changed, 59 insertions(+), 56 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index acf35723a1abb0..76922178d7b6d6 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -355,7 +355,7 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: if not support.is_wasi: tmp_dir = sysconfig.get_config_var('abs_builddir') if tmp_dir is None: - # bpo-30284: On Windows, only srcdir is available. Using + # gh-74470: On Windows, only srcdir is available. Using # abs_builddir mostly matters on UNIX when building Python # out of the source tree, especially when the source tree # is read only. diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8a4555ce16fbb6..4fcb8999579a82 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2565,3 +2565,30 @@ def wrapper(*args, **kwargs): finally: _testinternalcapi.set_optimizer(save_opt) return wrapper + + +_BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({ + # SRC_DIR/.git + '.git', + # ignore all __pycache__/ sub-directories + '__pycache__', +}) + +# Ignore function for shutil.copytree() to copy the Python source code. +def copy_python_src_ignore(path, names): + ignored = _BASE_COPY_SRC_DIR_IGNORED_NAMES + if os.path.basename(path) == 'Doc': + ignored |= { + # SRC_DIR/Doc/build/ + 'build', + # SRC_DIR/Doc/venv/ + 'venv', + } + + # check if we are at the root of the source code + elif 'Modules' in names: + ignored |= { + # SRC_DIR/build/ + 'build', + } + return ignored diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 5b57c5fd54a68d..af38db596d8528 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -7,6 +7,7 @@ import stat import subprocess import sys +import sysconfig import tempfile import textwrap import unittest @@ -800,6 +801,27 @@ def test_set_memlimit(self): support.max_memuse = old_max_memuse support.real_max_memuse = old_real_max_memuse + def test_copy_python_src_ignore(self): + src_dir = sysconfig.get_config_var('srcdir') + src_dir = os.path.abspath(src_dir) + + ignored = {'.git', '__pycache__'} + + # Source code directory + names = os.listdir(src_dir) + self.assertEqual(support.copy_python_src_ignore(src_dir, names), + ignored | {'build'}) + + # Doc/ directory + path = os.path.join(src_dir, 'Doc') + self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), + ignored | {'build', 'venv'}) + + # An other directory + path = os.path.join(src_dir, 'Objects') + self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), + ignored) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index eb83aa39425515..0ffe3e1d0cc498 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -21,7 +21,7 @@ skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_emscripten, is_wasi, requires_venv_with_pip, TEST_HOME_DIR, - requires_resource) + requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -560,12 +560,6 @@ def test_zippath_from_non_installed_posix(self): stdlib_zip) additional_pythonpath_for_non_installed = [] - # gh-109748: Don't copy __pycache__/ sub-directories, because they can - # be modified by other Python tests running in parallel. - ignored_names = {'__pycache__'} - def ignore_pycache(src, names): - return ignored_names - # Copy stdlib files to the non-installed python so venv can # correctly calculate the prefix. for eachpath in sys.path: @@ -583,7 +577,7 @@ def ignore_pycache(src, names): shutil.copy(fn, libdir) elif os.path.isdir(fn): shutil.copytree(fn, os.path.join(libdir, name), - ignore=ignore_pycache) + ignore=copy_python_src_ignore) else: additional_pythonpath_for_non_installed.append( eachpath) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 92e97cb261719c..bb15941464e3d1 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -1,14 +1,15 @@ import os import os.path -import re import shlex import shutil import subprocess +import sysconfig +from test import support TESTS_DIR = os.path.dirname(__file__) TOOL_ROOT = os.path.dirname(TESTS_DIR) -SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT)) +SRCDIR = os.path.abspath(sysconfig.get_config_var('srcdir')) MAKE = shutil.which('make') FREEZE = os.path.join(TOOL_ROOT, 'freeze.py') @@ -75,56 +76,17 @@ def ensure_opt(args, name, value): def copy_source_tree(newroot, oldroot): - print(f'copying the source tree into {newroot}...') + print(f'copying the source tree from {oldroot} to {newroot}...') if os.path.exists(newroot): if newroot == SRCDIR: raise Exception('this probably isn\'t what you wanted') shutil.rmtree(newroot) - def ignore_non_src(src, names): - """Turns what could be a 1000M copy into a 100M copy.""" - # Don't copy the ~600M+ of needless git repo metadata. - # source only, ignore cached .pyc files. - subdirs_to_skip = {'.git', '__pycache__'} - if os.path.basename(src) == 'Doc': - # Another potential ~250M+ of non test related data. - subdirs_to_skip.add('build') - subdirs_to_skip.add('venv') - return subdirs_to_skip - - shutil.copytree(oldroot, newroot, ignore=ignore_non_src) + shutil.copytree(oldroot, newroot, ignore=support.copy_python_src_ignore) if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) -def get_makefile_var(builddir, name): - regex = re.compile(rf'^{name} *=\s*(.*?)\s*$') - filename = os.path.join(builddir, 'Makefile') - try: - infile = open(filename, encoding='utf-8') - except FileNotFoundError: - return None - with infile: - for line in infile: - m = regex.match(line) - if m: - value, = m.groups() - return value or '' - return None - - -def get_config_var(builddir, name): - python = os.path.join(builddir, 'python') - if os.path.isfile(python): - cmd = [python, '-c', - f'import sysconfig; print(sysconfig.get_config_var("{name}"))'] - try: - return _run_stdout(cmd) - except subprocess.CalledProcessError: - pass - return get_makefile_var(builddir, name) - - ################################## # freezing @@ -151,10 +113,8 @@ def prepare(script=None, outdir=None): # Run configure. print(f'configuring python in {builddir}...') - cmd = [ - os.path.join(srcdir, 'configure'), - *shlex.split(get_config_var(SRCDIR, 'CONFIG_ARGS') or ''), - ] + config_args = shlex.split(sysconfig.get_config_var('CONFIG_ARGS') or '') + cmd = [os.path.join(srcdir, 'configure'), *config_args] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') ensure_opt(cmd, 'prefix', prefix) From b1aebf1e6576680d606068d17e2208259573e061 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 10:51:44 +0200 Subject: [PATCH 027/124] gh-109565: Fix concurrent.futures test_future_times_out() (#109949) as_completed() uses a timeout of 100 ms instead of 10 ms. Windows monotonic clock resolution is around 15.6 ms. --- Lib/test/test_concurrent_futures/test_as_completed.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_as_completed.py b/Lib/test/test_concurrent_futures/test_as_completed.py index 2b3bec8cafbcb0..c90b0021d85fc7 100644 --- a/Lib/test/test_concurrent_futures/test_as_completed.py +++ b/Lib/test/test_concurrent_futures/test_as_completed.py @@ -42,11 +42,14 @@ def test_future_times_out(self): EXCEPTION_FUTURE, SUCCESSFUL_FUTURE} - for timeout in (0, 0.01): + # Windows clock resolution is around 15.6 ms + short_timeout = 0.100 + for timeout in (0, short_timeout): with self.subTest(timeout): - future = self.executor.submit(time.sleep, 0.1) completed_futures = set() + future = self.executor.submit(time.sleep, short_timeout * 10) + try: for f in futures.as_completed( already_completed | {future}, From 91fb8daa2494df4dd6a841ca8c742a03175c7ecd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 12:01:16 +0200 Subject: [PATCH 028/124] gh-109566: Fix regrtest Python options for WASM/WASI (#109954) WASM and WASI buildbots use multiple PYTHON environment variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. Don't use -E if the --python=COMMAND option is used. --- Lib/test/libregrtest/main.py | 9 ++++++--- Lib/test/libregrtest/worker.py | 6 +++++- Lib/test/test_regrtest.py | 13 +++++++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c31d5ff187c56a..45a68a8465d8e0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -504,9 +504,12 @@ def _add_python_opts(self): if sys.flags.bytes_warning < 2: python_opts.append('-bb') - # Ignore PYTHON* environment variables - if not sys.flags.ignore_environment: - python_opts.append('-E') + # WASM/WASI buildbot builders pass multiple PYTHON environment + # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. + if not self.python_cmd: + # Ignore PYTHON* environment variables + if not sys.flags.ignore_environment: + python_opts.append('-E') if not python_opts: return diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 610e0a8437839d..67f26cfe75fbe4 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -22,11 +22,15 @@ def create_worker_process(runtests: RunTests, output_fd: int, python_cmd = runtests.python_cmd worker_json = runtests.as_json() + python_opts = support.args_from_interpreter_flags() if python_cmd is not None: executable = python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON environment + # variables, such as PYTHONPATH, in the worker process. + python_opts = [opt for opt in python_opts if opt != "-E"] else: executable = (sys.executable,) - cmd = [*executable, *support.args_from_interpreter_flags(), + cmd = [*executable, *python_opts, '-u', # Unbuffered stdout and stderr '-m', 'test.libregrtest.worker', worker_json] diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 3ece31be9af3c3..e0568cb2a91bc2 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1965,16 +1965,20 @@ def test_dev_mode(self): self.check_executed_tests(output, tests, stats=len(tests), parallel=True) - def check_reexec(self, option): + def check_add_python_opts(self, option): # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python code = textwrap.dedent(r""" import sys import unittest + from test import support try: from _testinternalcapi import get_config except ImportError: get_config = None + # WASI/WASM buildbots don't use -E option + use_environment = (support.is_emscripten or support.is_wasi) + class WorkerTests(unittest.TestCase): @unittest.skipUnless(get_config is None, 'need get_config()') def test_config(self): @@ -1986,7 +1990,7 @@ def test_config(self): # -bb option self.assertTrue(config['bytes_warning'], 2) # -E option - self.assertTrue(config['use_environment'], 0) + self.assertTrue(config['use_environment'], use_environment) def test_python_opts(self): # -u option @@ -2000,7 +2004,8 @@ def test_python_opts(self): self.assertEqual(sys.flags.bytes_warning, 2) # -E option - self.assertTrue(sys.flags.ignore_environment) + self.assertEqual(not sys.flags.ignore_environment, + use_environment) """) testname = self.create_test(code=code) @@ -2018,7 +2023,7 @@ def test_python_opts(self): def test_add_python_opts(self): for opt in ("--fast-ci", "--slow-ci"): with self.subTest(opt=opt): - self.check_reexec(opt) + self.check_add_python_opts(opt) class TestUtils(unittest.TestCase): From b89ed9df39851348fbb1552294644f99f6b17d2c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 12:32:12 +0200 Subject: [PATCH 029/124] gh-109615: Fix support test_copy_python_src_ignore() (#109958) Fix the test when run on an installed Python: use "abs_srcdir" of sysconfig, and skip the test if the Python source code cannot be found. * Tools/patchcheck/patchcheck.py, Tools/freeze/test/freeze.py and Lib/test/libregrtest/utils.py now first try to get "abs_srcdir" from sysconfig, before getting "srcdir" from sysconfig. * test.pythoninfo logs sysconfig "abs_srcdir". --- Lib/test/libregrtest/utils.py | 12 +++++++----- Lib/test/pythoninfo.py | 1 + Lib/test/test_support.py | 7 ++++++- Tools/freeze/test/freeze.py | 9 ++++++++- Tools/patchcheck/patchcheck.py | 9 ++++++++- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 76922178d7b6d6..bedf9a5420db64 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -355,11 +355,13 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: if not support.is_wasi: tmp_dir = sysconfig.get_config_var('abs_builddir') if tmp_dir is None: - # gh-74470: On Windows, only srcdir is available. Using - # abs_builddir mostly matters on UNIX when building Python - # out of the source tree, especially when the source tree - # is read only. - tmp_dir = sysconfig.get_config_var('srcdir') + tmp_dir = sysconfig.get_config_var('abs_srcdir') + if not tmp_dir: + # gh-74470: On Windows, only srcdir is available. Using + # abs_builddir mostly matters on UNIX when building + # Python out of the source tree, especially when the + # source tree is read only. + tmp_dir = sysconfig.get_config_var('srcdir') tmp_dir = os.path.join(tmp_dir, 'build') else: # WASI platform diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index c372efaedd313b..0e7528ef97c5f6 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -520,6 +520,7 @@ def collect_sysconfig(info_add): 'SHELL', 'SOABI', 'abs_builddir', + 'abs_srcdir', 'prefix', 'srcdir', ): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index af38db596d8528..134ce24484fa28 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -802,8 +802,13 @@ def test_set_memlimit(self): support.real_max_memuse = old_real_max_memuse def test_copy_python_src_ignore(self): - src_dir = sysconfig.get_config_var('srcdir') + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') src_dir = os.path.abspath(src_dir) + if not os.path.exists(src_dir): + self.skipTest(f"cannot access Python source code directory:" + f" {src_dir!r}") ignored = {'.git', '__pycache__'} diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index bb15941464e3d1..cdf77c57bbb6ae 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -7,9 +7,16 @@ from test import support +def get_python_source_dir(): + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') + return os.path.abspath(src_dir) + + TESTS_DIR = os.path.dirname(__file__) TOOL_ROOT = os.path.dirname(TESTS_DIR) -SRCDIR = os.path.abspath(sysconfig.get_config_var('srcdir')) +SRCDIR = get_python_source_dir() MAKE = shutil.which('make') FREEZE = os.path.join(TOOL_ROOT, 'freeze.py') diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index fa3a43af6e6048..e3959ce428c7c5 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -11,6 +11,13 @@ import untabify +def get_python_source_dir(): + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') + return os.path.abspath(src_dir) + + # Excluded directories which are copies of external libraries: # don't check their coding style EXCLUDE_DIRS = [ @@ -18,7 +25,7 @@ os.path.join('Modules', 'expat'), os.path.join('Modules', 'zlib'), ] -SRCDIR = sysconfig.get_config_var('srcdir') +SRCDIR = get_python_source_dir() def n_files_str(count): From ea285ad8b69c6ed91fe79edb3b0ea4d9cd6e6011 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 27 Sep 2023 13:24:33 +0100 Subject: [PATCH 030/124] gh-109923: set line number on the POP_TOP that follows a RETURN_GENERATOR (#109924) --- Lib/test/test_dis.py | 14 ++++---------- .../2023-09-26-21-26-54.gh-issue-109923.WO3CHi.rst | 1 + Python/flowgraph.c | 6 ++++-- 3 files changed, 9 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-26-21-26-54.gh-issue-109923.WO3CHi.rst diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index d104e5dd904999..8ab0e1ecbc4a7f 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -524,10 +524,8 @@ async def _asyncwith(c): dis_asyncwith = """\ %4d RETURN_GENERATOR - -None POP_TOP - -%4d RESUME 0 + POP_TOP + RESUME 0 %4d LOAD_FAST 0 (c) BEFORE_ASYNC_WITH @@ -598,7 +596,6 @@ async def _asyncwith(c): ExceptionTable: 12 rows """ % (_asyncwith.__code__.co_firstlineno, - _asyncwith.__code__.co_firstlineno, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 2, _asyncwith.__code__.co_firstlineno + 1, @@ -757,10 +754,8 @@ def foo(x): None COPY_FREE_VARS 1 %4d RETURN_GENERATOR - -None POP_TOP - -%4d RESUME 0 + POP_TOP + RESUME 0 LOAD_FAST 0 (.0) >> FOR_ITER 10 (to 34) STORE_FAST 1 (z) @@ -782,7 +777,6 @@ def foo(x): __file__, _h.__code__.co_firstlineno + 3, _h.__code__.co_firstlineno + 3, - _h.__code__.co_firstlineno + 3, ) def load_test(x, y=0): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-26-21-26-54.gh-issue-109923.WO3CHi.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-21-26-54.gh-issue-109923.WO3CHi.rst new file mode 100644 index 00000000000000..f2184592af0051 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-21-26-54.gh-issue-109923.WO3CHi.rst @@ -0,0 +1 @@ +Set line number on the ``POP_TOP`` that follows a ``RETURN_GENERATOR``. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 9c24264cfbb459..9fe387cc9a8e80 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2468,17 +2468,19 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl * of 0. This is because RETURN_GENERATOR pushes an element * with _PyFrame_StackPush before switching stacks. */ + + location loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1); cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, - .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), + .i_loc = loc, .i_target = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); cfg_instr pop_top = { .i_opcode = POP_TOP, .i_oparg = 0, - .i_loc = NO_LOCATION, + .i_loc = loc, .i_target = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); From d9809e84fbf22ed8d90b212a9322260f7074bc9c Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 27 Sep 2023 16:07:28 +0300 Subject: [PATCH 031/124] gh-101100: Fix sphinx warnings in `library/devmode.rst` (#109963) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/devmode.rst | 5 +++-- Doc/tools/.nitignore | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index 914aa45cf9cbc3..5b8a9bd1908456 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -59,8 +59,9 @@ Effects of the Python Development Mode: ``default``. * Call :func:`faulthandler.enable` at Python startup to install handlers for - the :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and - :const:`SIGILL` signals to dump the Python traceback on a crash. + the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, + :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` and + :const:`~signal.SIGILL` signals to dump the Python traceback on a crash. It behaves as if the :option:`-X faulthandler <-X>` command line option is used or if the :envvar:`PYTHONFAULTHANDLER` environment variable is set to diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1dd7676eab057e..a5fb940b150ee8 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -53,7 +53,6 @@ Doc/library/csv.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst -Doc/library/devmode.rst Doc/library/difflib.rst Doc/library/doctest.rst Doc/library/email.charset.rst From 62881a79a81816f075c8809b94438a3d519af0a4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 16:09:23 +0200 Subject: [PATCH 032/124] gh-109566: regrtest doesn't enable --rerun if --python is used (#109969) regrtest: --fast-ci and --slow-ci options no longer enable --rerun if the --python option is used. --- Lib/test/libregrtest/cmdline.py | 3 ++- Lib/test/test_regrtest.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c180bb76222a89..0a863561d5273d 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -420,7 +420,8 @@ def _parse_args(args, **kwargs): ns.randomize = True ns.fail_env_changed = True ns.fail_rerun = True - ns.rerun = True + if ns.python is None: + ns.rerun = True ns.print_slow = True ns.verbose3 = True if MS_WINDOWS: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e0568cb2a91bc2..da1406def55543 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -374,7 +374,7 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') - def check_ci_mode(self, args, use_resources): + def check_ci_mode(self, args, use_resources, rerun=True): ns = cmdline._parse_args(args) if utils.MS_WINDOWS: self.assertTrue(ns.nowindows) @@ -383,7 +383,7 @@ def check_ci_mode(self, args, use_resources): # which has an unclear API regrtest = main.Regrtest(ns) self.assertEqual(regrtest.num_workers, -1) - self.assertTrue(regrtest.want_rerun) + self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) self.assertIsNone(regrtest.random_seed) self.assertTrue(regrtest.fail_env_changed) @@ -400,6 +400,14 @@ def test_fast_ci(self): regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 10 * 60) + def test_fast_ci_python_cmd(self): + args = ['--fast-ci', '--python', 'python -X dev'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources, rerun=False) + self.assertEqual(regrtest.timeout, 10 * 60) + self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev')) + def test_fast_ci_resource(self): # it should be possible to override resources args = ['--fast-ci', '-u', 'network'] From b35f0843fc15486b17bc945dde08b306b8e4e81f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 27 Sep 2023 15:31:55 +0100 Subject: [PATCH 033/124] GH-109190: Copyedit 3.12 What's New: Release highlights (#109770) --- Doc/whatsnew/3.12.rst | 421 +++++++++++++++++++++++++----------------- 1 file changed, 254 insertions(+), 167 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index cad8dd7db7228a..75a784f620f5fd 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -59,36 +59,106 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.12. Brevity is key. +Python 3.12 is the latest stable release of the Python programming language, +with a mix of changes to the language and the standard library. +The library changes focus on cleaning up deprecated APIs, usability, and correctness. +Of note, the :mod:`!distutils` package has been removed from the standard library. +Filesystem support in :mod:`os` and :mod:`pathlib` has seen a number of improvements, +and several modules have better performance. + +The language changes focus on usability, +as :term:`f-strings ` have had many limitations removed +and 'Did you mean ...' suggestions continue to improve. +The new :ref:`type parameter syntax ` +and :keyword:`type` statement improve ergonomics for using :term:`generic types +` and :term:`type aliases ` with static type checkers. + +This article doesn't attempt to provide a complete specification of all new features, +but instead gives a convenient overview. +For full details, you should refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +If you want to understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. + +-------------- .. PEP-sized items next. +New syntax features: + +* :ref:`PEP 695 `, type parameter syntax and the :keyword:`type` statement + New grammar features: -* :ref:`whatsnew312-pep701` +* :ref:`PEP 701 `, :term:`f-strings ` in the grammar Interpreter improvements: -* :ref:`whatsnew312-pep684` +* :ref:`PEP 684 `, a unique per-interpreter :term:`GIL + ` +* :ref:`PEP 669 `, low impact monitoring +* `Improved 'Did you mean ...' suggestions `_ + for :exc:`NameError`, :exc:`ImportError`, and :exc:`SyntaxError` exceptions -* :ref:`whatsnew312-pep669` +Python data model improvements: -New typing features: +* :ref:`PEP 688 `, using the :ref:`buffer protocol + ` from Python + +Significant improvements in the standard library: + +* The :class:`pathlib.Path` class now supports subclassing +* The :mod:`os` module received several improvements for Windows support +* A :ref:`command-line interface ` has been added to the + :mod:`sqlite3` module +* :func:`isinstance` checks against :func:`runtime-checkable protocols + ` enjoy a speed up of between two and 20 times +* The :mod:`asyncio` package has had a number of performance improvements, + with some benchmarks showing a 75% speed up. +* A :ref:`command-line interface ` has been added to the + :mod:`uuid` module +* Due to the changes in :ref:`PEP 701 `, + producing tokens via the :mod:`tokenize` module is up to up to 64% faster. + +Security improvements: -* :ref:`whatsnew312-pep688` +* Replace the builtin :mod:`hashlib` implementations of + SHA1, SHA3, SHA2-384, SHA2-512, and MD5 with formally verified code from the + `HACL* `__ project. + These builtin implementations remain as fallbacks that are only used when + OpenSSL does not provide them. -* :ref:`whatsnew312-pep692` +C API improvements: -* :ref:`whatsnew312-pep695` +* :ref:`PEP 697 `, unstable C API tier +* :ref:`PEP 683 `, immortal objects -* :ref:`whatsnew312-pep698` +CPython implementation improvements: + +* :ref:`PEP 709 `, comprehension inlining +* :ref:`CPython support ` for the Linux ``perf`` profiler +* Implement stack overflow protection on supported platforms + +New typing features: + +* :ref:`PEP 692 `, using :class:`~typing.TypedDict` to + annotate :term:`**kwargs ` +* :ref:`PEP 698 `, :func:`typing.override` decorator Important deprecations, removals or restrictions: -* :pep:`623`: Remove wstr from Unicode +* :pep:`623`: Remove ``wstr`` from Unicode objects in Python's C API, + reducing the size of every :class:`str` object by at least 8 bytes. -* :pep:`632`: Remove the ``distutils`` package. See - `the migration guide `_ - for advice on its replacement. +* :pep:`632`: Remove the :mod:`!distutils` package. + See `the migration guide `_ + for advice replacing the APIs it provided. + The third-party `Setuptools `__ + package continues to provide :mod:`!distutils`, + if you still require it in Python 3.12 and beyond. * :gh:`95299`: Do not pre-install ``setuptools`` in virtual environments created with :mod:`venv`. @@ -97,60 +167,77 @@ Important deprecations, removals or restrictions: run ``pip install setuptools`` in the :ref:`activated ` virtual environment. -Improved Error Messages -======================= +* The :mod:`!asynchat`, :mod:`!asyncore`, and :mod:`!imp` modules have been + removed, along with several :class:`unittest.TestCase` + `method aliases `_. -* Modules from the standard library are now potentially suggested as part of - the error messages displayed by the interpreter when a :exc:`NameError` is - raised to the top level. (Contributed by Pablo Galindo in :gh:`98254`.) - >>> sys.version_info - Traceback (most recent call last): - File "", line 1, in - NameError: name 'sys' is not defined. Did you forget to import 'sys'? +New Features +============ -* Improve the error suggestion for :exc:`NameError` exceptions for instances. - Now if a :exc:`NameError` is raised in a method and the instance has an - attribute that's exactly equal to the name in the exception, the suggestion - will include ``self.`` instead of the closest match in the method - scope. (Contributed by Pablo Galindo in :gh:`99139`.) +.. _whatsnew312-pep695: - >>> class A: - ... def __init__(self): - ... self.blech = 1 - ... - ... def foo(self): - ... somethin = blech - ... - >>> A().foo() - Traceback (most recent call last): - File "", line 1 - somethin = blech - ^^^^^ - NameError: name 'blech' is not defined. Did you mean: 'self.blech'? +PEP 695: Type Parameter Syntax +------------------------------ -* Improve the :exc:`SyntaxError` error message when the user types ``import x - from y`` instead of ``from y import x``. (Contributed by Pablo Galindo in :gh:`98931`.) +Generic classes and functions under :pep:`484` were declared using a verbose syntax +that left the scope of type parameters unclear and required explicit declarations of +variance. - >>> import a.y.z from b.y.z - Traceback (most recent call last): - File "", line 1 - import a.y.z from b.y.z - ^^^^^^^^^^^^^^^^^^^^^^^ - SyntaxError: Did you mean to use 'from ... import ...' instead? +:pep:`695` introduces a new, more compact and explicit way to create +:ref:`generic classes ` and :ref:`functions `:: -* :exc:`ImportError` exceptions raised from failed ``from import - `` statements now include suggestions for the value of ```` based on the - available names in ````. (Contributed by Pablo Galindo in :gh:`91058`.) + def max[T](args: Iterable[T]) -> T: + ... - >>> from collections import chainmap - Traceback (most recent call last): - File "", line 1, in - ImportError: cannot import name 'chainmap' from 'collections'. Did you mean: 'ChainMap'? + class list[T]: + def __getitem__(self, index: int, /) -> T: + ... + def append(self, element: T) -> None: + ... -New Features -============ +In addition, the PEP introduces a new way to declare :ref:`type aliases ` +using the :keyword:`type` statement, which creates an instance of +:class:`~typing.TypeAliasType`:: + + type Point = tuple[float, float] + +Type aliases can also be :ref:`generic `:: + + type Point[T] = tuple[T, T] + +The new syntax allows declaring :class:`~typing.TypeVarTuple` +and :class:`~typing.ParamSpec` parameters, as well as :class:`~typing.TypeVar` +parameters with bounds or constraints:: + + type IntFunc[**P] = Callable[P, int] # ParamSpec + type LabeledTuple[*Ts] = tuple[str, *Ts] # TypeVarTuple + type HashableSequence[T: Hashable] = Sequence[T] # TypeVar with bound + type IntOrStrSequence[T: (int, str)] = Sequence[T] # TypeVar with constraints + +The value of type aliases and the bound and constraints of type variables +created through this syntax are evaluated only on demand (see +:ref:`lazy evaluation `). This means type aliases are able to +refer to other types defined later in the file. + +Type parameters declared through a type parameter list are visible within the +scope of the declaration and any nested scopes, but not in the outer scope. For +example, they can be used in the type annotations for the methods of a generic +class or in the class body. However, they cannot be used in the module scope after +the class is defined. See :ref:`type-params` for a detailed description of the +runtime semantics of type parameters. + +In order to support these scoping semantics, a new kind of scope is introduced, +the :ref:`annotation scope `. Annotation scopes behave for the +most part like function scopes, but interact differently with enclosing class scopes. +In Python 3.13, :term:`annotations ` will also be evaluated in +annotation scopes. + +See :pep:`695` for more details. + +(PEP written by Eric Traut. Implementation by Jelle Zijlstra, Eric Traut, +and others in :gh:`103764`.) .. _whatsnew312-pep701: @@ -244,52 +331,6 @@ are parsed with the PEG parser, error messages can be more precise and show the Maureira-Fredes and Marta Gómez in :gh:`102856`. PEP written by Pablo Galindo, Batuhan Taskaya, Lysandros Nikolaou and Marta Gómez). -.. _whatsnew312-pep709: - -PEP 709: Comprehension inlining -------------------------------- - -Dictionary, list, and set comprehensions are now inlined, rather than creating a -new single-use function object for each execution of the comprehension. This -speeds up execution of a comprehension by up to two times. -See :pep:`709` for further details. - -Comprehension iteration variables remain isolated and don't overwrite a -variable of the same name in the outer scope, nor are they visible after the -comprehension. Inlining does result in a few visible behavior changes: - -* There is no longer a separate frame for the comprehension in tracebacks, - and tracing/profiling no longer shows the comprehension as a function call. -* The :mod:`symtable` module will no longer produce child symbol tables for each - comprehension; instead, the comprehension's locals will be included in the - parent function's symbol table. -* Calling :func:`locals` inside a comprehension now includes variables - from outside the comprehension, and no longer includes the synthetic ``.0`` - variable for the comprehension "argument". -* A comprehension iterating directly over ``locals()`` (e.g. ``[k for k in - locals()]``) may see "RuntimeError: dictionary changed size during iteration" - when run under tracing (e.g. code coverage measurement). This is the same - behavior already seen in e.g. ``for k in locals():``. To avoid the error, first - create a list of keys to iterate over: ``keys = list(locals()); [k for k in - keys]``. - -(Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`.) - -.. _whatsnew312-pep688: - -PEP 688: Making the buffer protocol accessible in Python --------------------------------------------------------- - -:pep:`688` introduces a way to use the :ref:`buffer protocol ` -from Python code. Classes that implement the :meth:`~object.__buffer__` method -are now usable as buffer types. - -The new :class:`collections.abc.Buffer` ABC provides a standard -way to represent buffer objects, for example in type annotations. -The new :class:`inspect.BufferFlags` enum represents the flags that -can be used to customize buffer creation. -(Contributed by Jelle Zijlstra in :gh:`102500`.) - .. _whatsnew312-pep684: PEP 684: A Per-Interpreter GIL @@ -333,7 +374,105 @@ This means that you only pay for what you use, providing support for near-zero overhead debuggers and coverage tools. See :mod:`sys.monitoring` for details. -(Contributed by Mark Shannon in :gh:`103083`.) +(Contributed by Mark Shannon in :gh:`103082`.) + +.. _whatsnew312-pep688: + +PEP 688: Making the buffer protocol accessible in Python +-------------------------------------------------------- + +:pep:`688` introduces a way to use the :ref:`buffer protocol ` +from Python code. Classes that implement the :meth:`~object.__buffer__` method +are now usable as buffer types. + +The new :class:`collections.abc.Buffer` ABC provides a standard +way to represent buffer objects, for example in type annotations. +The new :class:`inspect.BufferFlags` enum represents the flags that +can be used to customize buffer creation. +(Contributed by Jelle Zijlstra in :gh:`102500`.) + +.. _whatsnew312-pep709: + +PEP 709: Comprehension inlining +------------------------------- + +Dictionary, list, and set comprehensions are now inlined, rather than creating a +new single-use function object for each execution of the comprehension. This +speeds up execution of a comprehension by up to two times. +See :pep:`709` for further details. + +Comprehension iteration variables remain isolated and don't overwrite a +variable of the same name in the outer scope, nor are they visible after the +comprehension. Inlining does result in a few visible behavior changes: + +* There is no longer a separate frame for the comprehension in tracebacks, + and tracing/profiling no longer shows the comprehension as a function call. +* The :mod:`symtable` module will no longer produce child symbol tables for each + comprehension; instead, the comprehension's locals will be included in the + parent function's symbol table. +* Calling :func:`locals` inside a comprehension now includes variables + from outside the comprehension, and no longer includes the synthetic ``.0`` + variable for the comprehension "argument". +* A comprehension iterating directly over ``locals()`` (e.g. ``[k for k in + locals()]``) may see "RuntimeError: dictionary changed size during iteration" + when run under tracing (e.g. code coverage measurement). This is the same + behavior already seen in e.g. ``for k in locals():``. To avoid the error, first + create a list of keys to iterate over: ``keys = list(locals()); [k for k in + keys]``. + +(Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`.) + +Improved Error Messages +----------------------- + +* Modules from the standard library are now potentially suggested as part of + the error messages displayed by the interpreter when a :exc:`NameError` is + raised to the top level. (Contributed by Pablo Galindo in :gh:`98254`.) + + >>> sys.version_info + Traceback (most recent call last): + File "", line 1, in + NameError: name 'sys' is not defined. Did you forget to import 'sys'? + +* Improve the error suggestion for :exc:`NameError` exceptions for instances. + Now if a :exc:`NameError` is raised in a method and the instance has an + attribute that's exactly equal to the name in the exception, the suggestion + will include ``self.`` instead of the closest match in the method + scope. (Contributed by Pablo Galindo in :gh:`99139`.) + + >>> class A: + ... def __init__(self): + ... self.blech = 1 + ... + ... def foo(self): + ... somethin = blech + ... + >>> A().foo() + Traceback (most recent call last): + File "", line 1 + somethin = blech + ^^^^^ + NameError: name 'blech' is not defined. Did you mean: 'self.blech'? + +* Improve the :exc:`SyntaxError` error message when the user types ``import x + from y`` instead of ``from y import x``. (Contributed by Pablo Galindo in :gh:`98931`.) + + >>> import a.y.z from b.y.z + Traceback (most recent call last): + File "", line 1 + import a.y.z from b.y.z + ^^^^^^^^^^^^^^^^^^^^^^^ + SyntaxError: Did you mean to use 'from ... import ...' instead? + +* :exc:`ImportError` exceptions raised from failed ``from import + `` statements now include suggestions for the value of ```` based on the + available names in ````. (Contributed by Pablo Galindo in :gh:`91058`.) + + >>> from collections import chainmap + Traceback (most recent call last): + File "", line 1, in + ImportError: cannot import name 'chainmap' from 'collections'. Did you mean: 'ChainMap'? + New Features Related to Type Hints ================================== @@ -398,70 +537,6 @@ See :pep:`698` for more details. (Contributed by Steven Troxler in :gh:`101561`.) -.. _whatsnew312-pep695: - -PEP 695: Type Parameter Syntax ------------------------------- - -Generic classes and functions under :pep:`484` were declared using a verbose syntax -that left the scope of type parameters unclear and required explicit declarations of -variance. - -:pep:`695` introduces a new, more compact and explicit way to create -:ref:`generic classes ` and :ref:`functions `:: - - def max[T](args: Iterable[T]) -> T: - ... - - class list[T]: - def __getitem__(self, index: int, /) -> T: - ... - - def append(self, element: T) -> None: - ... - -In addition, the PEP introduces a new way to declare :ref:`type aliases ` -using the :keyword:`type` statement, which creates an instance of -:class:`~typing.TypeAliasType`:: - - type Point = tuple[float, float] - -Type aliases can also be :ref:`generic `:: - - type Point[T] = tuple[T, T] - -The new syntax allows declaring :class:`~typing.TypeVarTuple` -and :class:`~typing.ParamSpec` parameters, as well as :class:`~typing.TypeVar` -parameters with bounds or constraints:: - - type IntFunc[**P] = Callable[P, int] # ParamSpec - type LabeledTuple[*Ts] = tuple[str, *Ts] # TypeVarTuple - type HashableSequence[T: Hashable] = Sequence[T] # TypeVar with bound - type IntOrStrSequence[T: (int, str)] = Sequence[T] # TypeVar with constraints - -The value of type aliases and the bound and constraints of type variables -created through this syntax are evaluated only on demand (see -:ref:`lazy evaluation `). This means type aliases are able to -refer to other types defined later in the file. - -Type parameters declared through a type parameter list are visible within the -scope of the declaration and any nested scopes, but not in the outer scope. For -example, they can be used in the type annotations for the methods of a generic -class or in the class body. However, they cannot be used in the module scope after -the class is defined. See :ref:`type-params` for a detailed description of the -runtime semantics of type parameters. - -In order to support these scoping semantics, a new kind of scope is introduced, -the :ref:`annotation scope `. Annotation scopes behave for the -most part like function scopes, but interact differently with enclosing class scopes. -In Python 3.13, :term:`annotations ` will also be evaluated in -annotation scopes. - -See :pep:`695` for more details. - -(PEP written by Eric Traut. Implementation by Jelle Zijlstra, Eric Traut, -and others in :gh:`103764`.) - Other Language Changes ====================== @@ -1020,6 +1095,13 @@ CPython bytecode changes * Add the :opcode:`LOAD_SUPER_ATTR` instruction. (Contributed by Carl Meyer and Vladimir Matveev in :gh:`103497`.) +FOR_ITER new behavior is not mentioned +The fact that POP_JUMP_IF_* family of instructions are now real instructions is not mentioned +YIELD_VALUE need for an argument is not mentioned + + + + Demos and Tools =============== @@ -1578,6 +1660,8 @@ unittest * Remove many long-deprecated :mod:`unittest` features: + .. _unittest-TestCase-removed-aliases: + * A number of :class:`~unittest.TestCase` method aliases: ============================ =============================== =============== @@ -1811,6 +1895,7 @@ C API Changes New Features ------------ +.. _whatsnew312-pep697: * :pep:`697`: Introduce the :ref:`Unstable C API tier `, intended for low-level tools like debuggers and JIT compilers. @@ -1938,6 +2023,8 @@ New Features to replace the legacy-api :c:func:`!PyErr_Display`. (Contributed by Irit Katriel in :gh:`102755`). +.. _whatsnew312-pep683: + * :pep:`683`: Introduce *Immortal Objects*, which allows objects to bypass reference counts, and related changes to the C-API: From 773614e03aef29c744d0300bd62fc8254e3c06b6 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 27 Sep 2023 11:24:12 -0400 Subject: [PATCH 034/124] gh-109740: Use 't' in `--disable-gil` SOABI (#109922) Shared libraries for CPython 3.13 are now marked with a 't' for threading. For example, `binascii.cpython-313t-darwin.so`. --- Doc/library/sys.rst | 2 + Lib/test/test_sys.py | 7 ++ ...-09-26-16-00-50.gh-issue-109740.wboWdQ.rst | 2 + Python/dynload_win.c | 10 ++- configure | 65 ++++++++++--------- configure.ac | 33 +++++----- 6 files changed, 71 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-09-26-16-00-50.gh-issue-109740.wboWdQ.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ef818a7da016de..f9f556306f5827 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -22,6 +22,8 @@ always available. .. versionadded:: 3.2 + .. availability:: Unix. + .. function:: addaudithook(hook) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c616a27364b494..16050171ad139d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1210,6 +1210,13 @@ def test_pystats(self): sys._stats_clear() sys._stats_dump() + @test.support.cpython_only + @unittest.skipUnless(hasattr(sys, 'abiflags'), 'need sys.abiflags') + def test_disable_gil_abi(self): + abi_threaded = 't' in sys.abiflags + py_nogil = (sysconfig.get_config_var('Py_NOGIL') == 1) + self.assertEqual(py_nogil, abi_threaded) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Build/2023-09-26-16-00-50.gh-issue-109740.wboWdQ.rst b/Misc/NEWS.d/next/Build/2023-09-26-16-00-50.gh-issue-109740.wboWdQ.rst new file mode 100644 index 00000000000000..f59f462aecd1fc --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-09-26-16-00-50.gh-issue-109740.wboWdQ.rst @@ -0,0 +1,2 @@ +The experimental ``--disable-gil`` configure flag now includes "t" (for "threaded") in +extension ABI tags. diff --git a/Python/dynload_win.c b/Python/dynload_win.c index f69995b8f9e3a1..fcb3cb744047ce 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -15,10 +15,16 @@ #define PYD_DEBUG_SUFFIX "" #endif +#ifdef Py_NOGIL +# define PYD_THREADING_TAG "t" +#else +# define PYD_THREADING_TAG "" +#endif + #ifdef PYD_PLATFORM_TAG -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) "-" PYD_PLATFORM_TAG ".pyd" +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG ".pyd" #else -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) ".pyd" +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG ".pyd" #endif #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" diff --git a/configure b/configure index abae542b528a23..098def9aab08bf 100755 --- a/configure +++ b/configure @@ -1065,6 +1065,7 @@ with_suffix enable_shared with_static_libpython enable_profiling +enable_gil with_pydebug with_trace_refs enable_pystats @@ -1105,7 +1106,6 @@ with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes enable_test_modules -enable_gil ' ac_precious_vars='build_alias host_alias @@ -1792,6 +1792,8 @@ Optional Features: no) --enable-profiling enable C-level code profiling with gprof (default is no) + --disable-gil enable experimental support for running without the + GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) @@ -1806,8 +1808,6 @@ Optional Features: use big digits (30 or 15 bits) for Python longs (default is 30)] --disable-test-modules don't build nor install test modules - --disable-gil enable experimental support for running without the - GIL (default is no) Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] @@ -7845,6 +7845,36 @@ fi ABIFLAGS="" +# Check for --disable-gil +# --disable-gil +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-gil" >&5 +printf %s "checking for --disable-gil... " >&6; } +# Check whether --enable-gil was given. +if test ${enable_gil+y} +then : + enableval=$enable_gil; if test "x$enable_gil" = xyes +then : + disable_gil=no +else $as_nop + disable_gil=yes +fi +else $as_nop + disable_gil=no + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_gil" >&5 +printf "%s\n" "$disable_gil" >&6; } + +if test "$disable_gil" = "yes" +then + +printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h + + # Add "t" for "threaded" + ABIFLAGS="${ABIFLAGS}t" +fi + # Check for --with-pydebug { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-pydebug" >&5 printf %s "checking for --with-pydebug... " >&6; } @@ -23546,6 +23576,7 @@ printf "%s\n" "#define AC_APPLE_UNIVERSAL_BUILD 1" >>confdefs.h # # * The Python implementation (always 'cpython-' for us) # * The major and minor version numbers +# * --disable-gil (adds a 't') # * --with-pydebug (adds a 'd') # # Thus for example, Python 3.2 built with wide unicode, pydebug, and pymalloc, @@ -27724,34 +27755,6 @@ fi printf "%s\n" "$TEST_MODULES" >&6; } -# Check for --disable-gil -# --disable-gil -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-gil" >&5 -printf %s "checking for --disable-gil... " >&6; } -# Check whether --enable-gil was given. -if test ${enable_gil+y} -then : - enableval=$enable_gil; if test "x$enable_gil" = xyes -then : - disable_gil=no -else $as_nop - disable_gil=yes -fi -else $as_nop - disable_gil=no - -fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_gil" >&5 -printf "%s\n" "$disable_gil" >&6; } - -if test "$disable_gil" = "yes" -then - -printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h - -fi - # gh-109054: Check if -latomic is needed to get atomic functions. # On Linux aarch64, GCC may require programs and libraries to be linked # explicitly to libatomic. Call _Py_atomic_or_uint64() which may require diff --git a/configure.ac b/configure.ac index 205a98a992279c..3e6cbc69c21009 100644 --- a/configure.ac +++ b/configure.ac @@ -1489,6 +1489,23 @@ fi AC_SUBST([ABIFLAGS]) ABIFLAGS="" +# Check for --disable-gil +# --disable-gil +AC_MSG_CHECKING([for --disable-gil]) +AC_ARG_ENABLE([gil], + [AS_HELP_STRING([--disable-gil], [enable experimental support for running without the GIL (default is no)])], + [AS_VAR_IF([enable_gil], [yes], [disable_gil=no], [disable_gil=yes])], [disable_gil=no] +) +AC_MSG_RESULT([$disable_gil]) + +if test "$disable_gil" = "yes" +then + AC_DEFINE([Py_NOGIL], [1], + [Define if you want to disable the GIL]) + # Add "t" for "threaded" + ABIFLAGS="${ABIFLAGS}t" +fi + # Check for --with-pydebug AC_MSG_CHECKING([for --with-pydebug]) AC_ARG_WITH([pydebug], @@ -5669,6 +5686,7 @@ AC_C_BIGENDIAN # # * The Python implementation (always 'cpython-' for us) # * The major and minor version numbers +# * --disable-gil (adds a 't') # * --with-pydebug (adds a 'd') # # Thus for example, Python 3.2 built with wide unicode, pydebug, and pymalloc, @@ -6947,21 +6965,6 @@ AC_ARG_ENABLE([test-modules], AC_MSG_RESULT([$TEST_MODULES]) AC_SUBST([TEST_MODULES]) -# Check for --disable-gil -# --disable-gil -AC_MSG_CHECKING([for --disable-gil]) -AC_ARG_ENABLE([gil], - [AS_HELP_STRING([--disable-gil], [enable experimental support for running without the GIL (default is no)])], - [AS_VAR_IF([enable_gil], [yes], [disable_gil=no], [disable_gil=yes])], [disable_gil=no] -) -AC_MSG_RESULT([$disable_gil]) - -if test "$disable_gil" = "yes" -then - AC_DEFINE([Py_NOGIL], [1], - [Define if you want to disable the GIL]) -fi - # gh-109054: Check if -latomic is needed to get atomic functions. # On Linux aarch64, GCC may require programs and libraries to be linked # explicitly to libatomic. Call _Py_atomic_or_uint64() which may require From cc54bcf17b5b5f7681f52baf3acef75b995fa1fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 17:29:20 +0200 Subject: [PATCH 035/124] gh-109615: Fix support test_copy_python_src_ignore() on WASM (#109970) Not only check if src_dir exists, but look also for Lib/os.py landmark. --- Lib/test/test_support.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 134ce24484fa28..e4a246ba3ddd4d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -802,17 +802,25 @@ def test_set_memlimit(self): support.real_max_memuse = old_real_max_memuse def test_copy_python_src_ignore(self): + # Get source directory src_dir = sysconfig.get_config_var('abs_srcdir') if not src_dir: src_dir = sysconfig.get_config_var('srcdir') src_dir = os.path.abspath(src_dir) + + # Check that the source code is available if not os.path.exists(src_dir): self.skipTest(f"cannot access Python source code directory:" f" {src_dir!r}") + landmark = os.path.join(src_dir, 'Lib', 'os.py') + if not os.path.exists(landmark): + self.skipTest(f"cannot access Python source code directory:" + f" {landmark!r} landmark is missing") - ignored = {'.git', '__pycache__'} + # Test support.copy_python_src_ignore() # Source code directory + ignored = {'.git', '__pycache__'} names = os.listdir(src_dir) self.assertEqual(support.copy_python_src_ignore(src_dir, names), ignored | {'build'}) From 74723e11109a320e628898817ab449b3dad9ee96 Mon Sep 17 00:00:00 2001 From: Dale Collison <92315623+dcollison@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:26:41 +0100 Subject: [PATCH 036/124] gh-109461: Update logging module lock to use context manager (#109462) Co-authored-by: Victor Stinner --- Lib/logging/__init__.py | 136 ++++++------------ Lib/logging/config.py | 30 ++-- Lib/logging/handlers.py | 30 +--- Lib/multiprocessing/util.py | 6 +- Lib/test/test_logging.py | 54 +++---- ...-09-15-17-12-53.gh-issue-109461.VNFPTK.rst | 1 + 6 files changed, 81 insertions(+), 176 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-15-17-12-53.gh-issue-109461.VNFPTK.rst diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 2d228e563094c8..eb7e020d1edfc0 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -159,12 +159,9 @@ def addLevelName(level, levelName): This is used when converting levels to text during message formatting. """ - _acquireLock() - try: #unlikely to cause an exception, but you never know... + with _lock: _levelToName[level] = levelName _nameToLevel[levelName] = level - finally: - _releaseLock() if hasattr(sys, "_getframe"): currentframe = lambda: sys._getframe(1) @@ -231,25 +228,27 @@ def _checkLevel(level): # _lock = threading.RLock() -def _acquireLock(): +def _prepareFork(): """ - Acquire the module-level lock for serializing access to shared data. + Prepare to fork a new child process by acquiring the module-level lock. - This should be released with _releaseLock(). + This should be used in conjunction with _afterFork(). """ - if _lock: - try: - _lock.acquire() - except BaseException: - _lock.release() - raise + # Wrap the lock acquisition in a try-except to prevent the lock from being + # abandoned in the event of an asynchronous exception. See gh-106238. + try: + _lock.acquire() + except BaseException: + _lock.release() + raise -def _releaseLock(): +def _afterFork(): """ - Release the module-level lock acquired by calling _acquireLock(). + After a new child process has been forked, release the module-level lock. + + This should be used in conjunction with _prepareFork(). """ - if _lock: - _lock.release() + _lock.release() # Prevent a held logging lock from blocking a child from logging. @@ -264,23 +263,20 @@ def _register_at_fork_reinit_lock(instance): _at_fork_reinit_lock_weakset = weakref.WeakSet() def _register_at_fork_reinit_lock(instance): - _acquireLock() - try: + with _lock: _at_fork_reinit_lock_weakset.add(instance) - finally: - _releaseLock() def _after_at_fork_child_reinit_locks(): for handler in _at_fork_reinit_lock_weakset: handler._at_fork_reinit() - # _acquireLock() was called in the parent before forking. + # _prepareFork() was called in the parent before forking. # The lock is reinitialized to unlocked state. _lock._at_fork_reinit() - os.register_at_fork(before=_acquireLock, + os.register_at_fork(before=_prepareFork, after_in_child=_after_at_fork_child_reinit_locks, - after_in_parent=_releaseLock) + after_in_parent=_afterFork) #--------------------------------------------------------------------------- @@ -883,25 +879,20 @@ def _removeHandlerRef(wr): # set to None. It can also be called from another thread. So we need to # pre-emptively grab the necessary globals and check if they're None, # to prevent race conditions and failures during interpreter shutdown. - acquire, release, handlers = _acquireLock, _releaseLock, _handlerList - if acquire and release and handlers: - acquire() - try: - handlers.remove(wr) - except ValueError: - pass - finally: - release() + handlers, lock = _handlerList, _lock + if lock and handlers: + with lock: + try: + handlers.remove(wr) + except ValueError: + pass def _addHandlerRef(handler): """ Add a handler to the internal cleanup list using a weak reference. """ - _acquireLock() - try: + with _lock: _handlerList.append(weakref.ref(handler, _removeHandlerRef)) - finally: - _releaseLock() def getHandlerByName(name): @@ -946,15 +937,12 @@ def get_name(self): return self._name def set_name(self, name): - _acquireLock() - try: + with _lock: if self._name in _handlers: del _handlers[self._name] self._name = name if name: _handlers[name] = self - finally: - _releaseLock() name = property(get_name, set_name) @@ -1026,11 +1014,8 @@ def handle(self, record): if isinstance(rv, LogRecord): record = rv if rv: - self.acquire() - try: + with self.lock: self.emit(record) - finally: - self.release() return rv def setFormatter(self, fmt): @@ -1058,13 +1043,10 @@ def close(self): methods. """ #get the module data lock, as we're updating a shared structure. - _acquireLock() - try: #unlikely to raise an exception, but you never know... + with _lock: self._closed = True if self._name and self._name in _handlers: del _handlers[self._name] - finally: - _releaseLock() def handleError(self, record): """ @@ -1141,12 +1123,9 @@ def flush(self): """ Flushes the stream. """ - self.acquire() - try: + with self.lock: if self.stream and hasattr(self.stream, "flush"): self.stream.flush() - finally: - self.release() def emit(self, record): """ @@ -1182,12 +1161,9 @@ def setStream(self, stream): result = None else: result = self.stream - self.acquire() - try: + with self.lock: self.flush() self.stream = stream - finally: - self.release() return result def __repr__(self): @@ -1237,8 +1213,7 @@ def close(self): """ Closes the stream. """ - self.acquire() - try: + with self.lock: try: if self.stream: try: @@ -1254,8 +1229,6 @@ def close(self): # Also see Issue #42378: we also rely on # self._closed being set to True there StreamHandler.close(self) - finally: - self.release() def _open(self): """ @@ -1391,8 +1364,7 @@ def getLogger(self, name): rv = None if not isinstance(name, str): raise TypeError('A logger name must be a string') - _acquireLock() - try: + with _lock: if name in self.loggerDict: rv = self.loggerDict[name] if isinstance(rv, PlaceHolder): @@ -1407,8 +1379,6 @@ def getLogger(self, name): rv.manager = self self.loggerDict[name] = rv self._fixupParents(rv) - finally: - _releaseLock() return rv def setLoggerClass(self, klass): @@ -1471,12 +1441,11 @@ def _clear_cache(self): Called when level changes are made """ - _acquireLock() - for logger in self.loggerDict.values(): - if isinstance(logger, Logger): - logger._cache.clear() - self.root._cache.clear() - _releaseLock() + with _lock: + for logger in self.loggerDict.values(): + if isinstance(logger, Logger): + logger._cache.clear() + self.root._cache.clear() #--------------------------------------------------------------------------- # Logger classes and functions @@ -1701,23 +1670,17 @@ def addHandler(self, hdlr): """ Add the specified handler to this logger. """ - _acquireLock() - try: + with _lock: if not (hdlr in self.handlers): self.handlers.append(hdlr) - finally: - _releaseLock() def removeHandler(self, hdlr): """ Remove the specified handler from this logger. """ - _acquireLock() - try: + with _lock: if hdlr in self.handlers: self.handlers.remove(hdlr) - finally: - _releaseLock() def hasHandlers(self): """ @@ -1795,16 +1758,13 @@ def isEnabledFor(self, level): try: return self._cache[level] except KeyError: - _acquireLock() - try: + with _lock: if self.manager.disable >= level: is_enabled = self._cache[level] = False else: is_enabled = self._cache[level] = ( level >= self.getEffectiveLevel() ) - finally: - _releaseLock() return is_enabled def getChild(self, suffix): @@ -1834,16 +1794,13 @@ def _hierlevel(logger): return 1 + logger.name.count('.') d = self.manager.loggerDict - _acquireLock() - try: + with _lock: # exclude PlaceHolders - the last check is to ensure that lower-level # descendants aren't returned - if there are placeholders, a logger's # parent field might point to a grandparent or ancestor thereof. return set(item for item in d.values() if isinstance(item, Logger) and item.parent is self and _hierlevel(item) == 1 + _hierlevel(item.parent)) - finally: - _releaseLock() def __repr__(self): level = getLevelName(self.getEffectiveLevel()) @@ -2102,8 +2059,7 @@ def basicConfig(**kwargs): """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads - _acquireLock() - try: + with _lock: force = kwargs.pop('force', False) encoding = kwargs.pop('encoding', None) errors = kwargs.pop('errors', 'backslashreplace') @@ -2152,8 +2108,6 @@ def basicConfig(**kwargs): if kwargs: keys = ', '.join(kwargs.keys()) raise ValueError('Unrecognised argument(s): %s' % keys) - finally: - _releaseLock() #--------------------------------------------------------------------------- # Utility functions at module level. diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 41283f4d627267..951bba73913cb3 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -83,15 +83,12 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non formatters = _create_formatters(cp) # critical section - logging._acquireLock() - try: + with logging._lock: _clearExistingHandlers() # Handlers add themselves to logging._handlers handlers = _install_handlers(cp, formatters) _install_loggers(cp, handlers, disable_existing_loggers) - finally: - logging._releaseLock() def _resolve(name): @@ -516,8 +513,7 @@ def configure(self): raise ValueError("Unsupported version: %s" % config['version']) incremental = config.pop('incremental', False) EMPTY_DICT = {} - logging._acquireLock() - try: + with logging._lock: if incremental: handlers = config.get('handlers', EMPTY_DICT) for name in handlers: @@ -661,8 +657,6 @@ def configure(self): except Exception as e: raise ValueError('Unable to configure root ' 'logger') from e - finally: - logging._releaseLock() def configure_formatter(self, config): """Configure a formatter from a dictionary.""" @@ -988,9 +982,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): ThreadingTCPServer.__init__(self, (host, port), handler) - logging._acquireLock() - self.abort = 0 - logging._releaseLock() + with logging._lock: + self.abort = 0 self.timeout = 1 self.ready = ready self.verify = verify @@ -1004,9 +997,8 @@ def serve_until_stopped(self): self.timeout) if rd: self.handle_request() - logging._acquireLock() - abort = self.abort - logging._releaseLock() + with logging._lock: + abort = self.abort self.server_close() class Server(threading.Thread): @@ -1027,9 +1019,8 @@ def run(self): self.port = server.server_address[1] self.ready.set() global _listener - logging._acquireLock() - _listener = server - logging._releaseLock() + with logging._lock: + _listener = server server.serve_until_stopped() return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify) @@ -1039,10 +1030,7 @@ def stopListening(): Stop the listening server which was created with a call to listen(). """ global _listener - logging._acquireLock() - try: + with logging._lock: if _listener: _listener.abort = 1 _listener = None - finally: - logging._releaseLock() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 671cc9596b02dd..e75da9b7b1de64 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -683,15 +683,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.sock if sock: self.sock = None sock.close() logging.Handler.close(self) - finally: - self.release() class DatagramHandler(SocketHandler): """ @@ -953,15 +950,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.socket if sock: self.socket = None sock.close() logging.Handler.close(self) - finally: - self.release() def mapPriority(self, levelName): """ @@ -1333,11 +1327,8 @@ def flush(self): This version just zaps the buffer to empty. """ - self.acquire() - try: + with self.lock: self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1387,11 +1378,8 @@ def setTarget(self, target): """ Set the target handler for this handler. """ - self.acquire() - try: + with self.lock: self.target = target - finally: - self.release() def flush(self): """ @@ -1401,14 +1389,11 @@ def flush(self): The record buffer is only cleared if a target has been set. """ - self.acquire() - try: + with self.lock: if self.target: for record in self.buffer: self.target.handle(record) self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1419,12 +1404,9 @@ def close(self): if self.flushOnClose: self.flush() finally: - self.acquire() - try: + with self.lock: self.target = None BufferingHandler.close(self) - finally: - self.release() class QueueHandler(logging.Handler): diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 6ee0d33e88a060..28c77df1c32ea8 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -64,8 +64,7 @@ def get_logger(): global _logger import logging - logging._acquireLock() - try: + with logging._lock: if not _logger: _logger = logging.getLogger(LOGGER_NAME) @@ -79,9 +78,6 @@ def get_logger(): atexit._exithandlers.remove((_exit_function, (), {})) atexit._exithandlers.append((_exit_function, (), {})) - finally: - logging._releaseLock() - return _logger def log_to_stderr(level=None): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 375f65f9d16182..cca02a010b80f4 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -90,8 +90,7 @@ def setUp(self): self._threading_key = threading_helper.threading_setup() logger_dict = logging.getLogger().manager.loggerDict - logging._acquireLock() - try: + with logging._lock: self.saved_handlers = logging._handlers.copy() self.saved_handler_list = logging._handlerList[:] self.saved_loggers = saved_loggers = logger_dict.copy() @@ -101,8 +100,6 @@ def setUp(self): for name in saved_loggers: logger_states[name] = getattr(saved_loggers[name], 'disabled', None) - finally: - logging._releaseLock() # Set two unused loggers self.logger1 = logging.getLogger("\xab\xd7\xbb") @@ -136,8 +133,7 @@ def tearDown(self): self.root_logger.removeHandler(h) h.close() self.root_logger.setLevel(self.original_logging_level) - logging._acquireLock() - try: + with logging._lock: logging._levelToName.clear() logging._levelToName.update(self.saved_level_to_name) logging._nameToLevel.clear() @@ -154,8 +150,6 @@ def tearDown(self): for name in self.logger_states: if logger_states[name] is not None: self.saved_loggers[name].disabled = logger_states[name] - finally: - logging._releaseLock() self.doCleanups() threading_helper.threading_cleanup(*self._threading_key) @@ -739,11 +733,8 @@ def __init__(self): stream=open('/dev/null', 'wt', encoding='utf-8')) def emit(self, record): - self.sub_handler.acquire() - try: + with self.sub_handler.lock: self.sub_handler.emit(record) - finally: - self.sub_handler.release() self.assertEqual(len(logging._handlers), 0) refed_h = _OurHandler() @@ -759,29 +750,22 @@ def emit(self, record): fork_happened__release_locks_and_end_thread = threading.Event() def lock_holder_thread_fn(): - logging._acquireLock() - try: - refed_h.acquire() - try: - # Tell the main thread to do the fork. - locks_held__ready_to_fork.set() - - # If the deadlock bug exists, the fork will happen - # without dealing with the locks we hold, deadlocking - # the child. - - # Wait for a successful fork or an unreasonable amount of - # time before releasing our locks. To avoid a timing based - # test we'd need communication from os.fork() as to when it - # has actually happened. Given this is a regression test - # for a fixed issue, potentially less reliably detecting - # regression via timing is acceptable for simplicity. - # The test will always take at least this long. :( - fork_happened__release_locks_and_end_thread.wait(0.5) - finally: - refed_h.release() - finally: - logging._releaseLock() + with logging._lock, refed_h.lock: + # Tell the main thread to do the fork. + locks_held__ready_to_fork.set() + + # If the deadlock bug exists, the fork will happen + # without dealing with the locks we hold, deadlocking + # the child. + + # Wait for a successful fork or an unreasonable amount of + # time before releasing our locks. To avoid a timing based + # test we'd need communication from os.fork() as to when it + # has actually happened. Given this is a regression test + # for a fixed issue, potentially less reliably detecting + # regression via timing is acceptable for simplicity. + # The test will always take at least this long. :( + fork_happened__release_locks_and_end_thread.wait(0.5) lock_holder_thread = threading.Thread( target=lock_holder_thread_fn, diff --git a/Misc/NEWS.d/next/Library/2023-09-15-17-12-53.gh-issue-109461.VNFPTK.rst b/Misc/NEWS.d/next/Library/2023-09-15-17-12-53.gh-issue-109461.VNFPTK.rst new file mode 100644 index 00000000000000..28f0c16e620146 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-15-17-12-53.gh-issue-109461.VNFPTK.rst @@ -0,0 +1 @@ +:mod:`logging`: Use a context manager for lock acquisition. From f49958c886a2f2608f1008186d588efc2a98b445 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 27 Sep 2023 10:35:46 -0700 Subject: [PATCH 037/124] Enhance TypedDict docs around required/optional keys (#109547) As discussed in comments to #109544, the semantics of this attribute are somewhat confusing. Add a note explaining its limitations and steering users towards __required_keys__ and __optional_keys__ instead. --- Doc/library/typing.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e63b839931822c..8f691932201225 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2404,6 +2404,13 @@ types. >>> Point3D.__total__ True + This attribute reflects *only* the value of the ``total`` argument + to the current ``TypedDict`` class, not whether the class is semantically + total. For example, a ``TypedDict`` with ``__total__`` set to True may + have keys marked with :data:`NotRequired`, or it may inherit from another + ``TypedDict`` with ``total=False``. Therefore, it is generally better to use + :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. + .. attribute:: __required_keys__ .. versionadded:: 3.9 @@ -2439,6 +2446,14 @@ types. .. versionadded:: 3.9 + .. note:: + + If ``from __future__ import annotations`` is used or if annotations + are given as strings, annotations are not evaluated when the + ``TypedDict`` is defined. Therefore, the runtime introspection that + ``__required_keys__`` and ``__optional_keys__`` rely on may not work + properly, and the values of the attributes may be incorrect. + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 From 32466c97c06ee5812923d695195394c736eeb707 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 27 Sep 2023 13:41:06 -0600 Subject: [PATCH 038/124] gh-109793: Allow Switching Interpreters During Finalization (gh-109794) Essentially, we should check the thread ID rather than the thread state pointer. --- Include/cpython/pyatomic.h | 17 +++++++++++++++ Include/internal/pycore_interp.h | 16 ++++++++++++++ Include/internal/pycore_pystate.h | 8 +++++-- Include/internal/pycore_runtime.h | 16 ++++++++++++++ Lib/test/test_interpreters.py | 21 +++++++++++++++++++ ...-09-25-09-24-10.gh-issue-109793.zFQBkv.rst | 4 ++++ Python/pystate.c | 17 ++++++++++++++- 7 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index ab182381b39f00..ce23e13bf3838c 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -501,3 +501,20 @@ static inline void _Py_atomic_fence_release(void); #else # error "no available pyatomic implementation for this platform/compiler" #endif + + +// --- aliases --------------------------------------------------------------- + +#if SIZEOF_LONG == 8 +# define _Py_atomic_load_ulong _Py_atomic_load_uint64 +# define _Py_atomic_load_ulong_relaxed _Py_atomic_load_uint64_relaxed +# define _Py_atomic_store_ulong _Py_atomic_store_uint64 +# define _Py_atomic_store_ulong_relaxed _Py_atomic_store_uint64_relaxed +#elif SIZEOF_LONG == 4 +# define _Py_atomic_load_ulong _Py_atomic_load_uint32 +# define _Py_atomic_load_ulong_relaxed _Py_atomic_load_uint32_relaxed +# define _Py_atomic_store_ulong _Py_atomic_store_uint32 +# define _Py_atomic_store_ulong_relaxed _Py_atomic_store_uint32_relaxed +#else +# error "long must be 4 or 8 bytes in size" +#endif // SIZEOF_LONG diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ba5764e943e676..0912bd175fe4f7 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -93,6 +93,8 @@ struct _is { and _PyInterpreterState_SetFinalizing() to access it, don't access it directly. */ _Py_atomic_address _finalizing; + /* The ID of the OS thread in which we are finalizing. */ + unsigned long _finalizing_id; struct _gc_runtime_state gc; @@ -215,9 +217,23 @@ _PyInterpreterState_GetFinalizing(PyInterpreterState *interp) { return (PyThreadState*)_Py_atomic_load_relaxed(&interp->_finalizing); } +static inline unsigned long +_PyInterpreterState_GetFinalizingID(PyInterpreterState *interp) { + return _Py_atomic_load_ulong_relaxed(&interp->_finalizing_id); +} + static inline void _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tstate) { _Py_atomic_store_relaxed(&interp->_finalizing, (uintptr_t)tstate); + if (tstate == NULL) { + _Py_atomic_store_ulong_relaxed(&interp->_finalizing_id, 0); + } + else { + // XXX Re-enable this assert once gh-109860 is fixed. + //assert(tstate->thread_id == PyThread_get_thread_ident()); + _Py_atomic_store_ulong_relaxed(&interp->_finalizing_id, + tstate->thread_id); + } } diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 9fc8ae903b2ac0..2e568f8aeeb152 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -36,8 +36,12 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) static inline int _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) { - return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL && - interp == &interp->runtime->_main_interpreter); + /* bpo-39877: Access _PyRuntime directly rather than using + tstate->interp->runtime to support calls from Python daemon threads. + After Py_Finalize() has been called, tstate can be a dangling pointer: + point to PyThreadState freed memory. */ + return (_PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL && + interp == &_PyRuntime._main_interpreter); } diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0ddc405f221a1c..cc3a3420befa3d 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -171,6 +171,8 @@ typedef struct pyruntimestate { Use _PyRuntimeState_GetFinalizing() and _PyRuntimeState_SetFinalizing() to access it, don't access it directly. */ _Py_atomic_address _finalizing; + /* The ID of the OS thread in which we are finalizing. */ + unsigned long _finalizing_id; struct pyinterpreters { PyThread_type_lock mutex; @@ -303,9 +305,23 @@ _PyRuntimeState_GetFinalizing(_PyRuntimeState *runtime) { return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->_finalizing); } +static inline unsigned long +_PyRuntimeState_GetFinalizingID(_PyRuntimeState *runtime) { + return _Py_atomic_load_ulong_relaxed(&runtime->_finalizing_id); +} + static inline void _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { _Py_atomic_store_relaxed(&runtime->_finalizing, (uintptr_t)tstate); + if (tstate == NULL) { + _Py_atomic_store_ulong_relaxed(&runtime->_finalizing_id, 0); + } + else { + // XXX Re-enable this assert once gh-109860 is fixed. + //assert(tstate->thread_id == PyThread_get_thread_ident()); + _Py_atomic_store_ulong_relaxed(&runtime->_finalizing_id, + tstate->thread_id); + } } #ifdef __cplusplus diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 90932c0f66f38f..9c0dac7d6c61fb 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,5 +1,6 @@ import contextlib import os +import sys import threading from textwrap import dedent import unittest @@ -487,6 +488,26 @@ def task(): pass +class FinalizationTests(TestBase): + + def test_gh_109793(self): + import subprocess + argv = [sys.executable, '-c', '''if True: + import _xxsubinterpreters as _interpreters + interpid = _interpreters.create() + raise Exception + '''] + proc = subprocess.run(argv, capture_output=True, text=True) + self.assertIn('Traceback', proc.stderr) + if proc.returncode == 0 and support.verbose: + print() + print("--- cmd unexpected succeeded ---") + print(f"stdout:\n{proc.stdout}") + print(f"stderr:\n{proc.stderr}") + print("------") + self.assertEqual(proc.returncode, 1) + + class TestIsShareable(TestBase): def test_default_shareables(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst new file mode 100644 index 00000000000000..d2dc4c830a9031 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst @@ -0,0 +1,4 @@ +The main thread no longer exits prematurely when a subinterpreter +is cleaned up during runtime finalization. The bug was a problem +particularly because, when triggered, the Python process would +always return with a 0 exitcode, even if it failed. diff --git a/Python/pystate.c b/Python/pystate.c index dcc6c112215b30..570b5242600c0c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2964,11 +2964,26 @@ _PyThreadState_MustExit(PyThreadState *tstate) tstate->interp->runtime to support calls from Python daemon threads. After Py_Finalize() has been called, tstate can be a dangling pointer: point to PyThreadState freed memory. */ + unsigned long finalizing_id = _PyRuntimeState_GetFinalizingID(&_PyRuntime); PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime); if (finalizing == NULL) { + // XXX This isn't completely safe from daemon thraeds, + // since tstate might be a dangling pointer. finalizing = _PyInterpreterState_GetFinalizing(tstate->interp); + finalizing_id = _PyInterpreterState_GetFinalizingID(tstate->interp); } - return (finalizing != NULL && finalizing != tstate); + // XXX else check &_PyRuntime._main_interpreter._initial_thread + if (finalizing == NULL) { + return 0; + } + else if (finalizing == tstate) { + return 0; + } + else if (finalizing_id == PyThread_get_thread_ident()) { + /* gh-109793: we must have switched interpreters. */ + return 0; + } + return 1; } From 45cf5b0c69bb5c51f33fc681d90c45147e311ddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Wed, 27 Sep 2023 22:24:10 +0000 Subject: [PATCH 039/124] gh-109955 : Update state transition comments for asyncio.Task (#109910) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/asyncio/tasks.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 21a1b24194bcd8..72f4cc07173f0a 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -73,15 +73,25 @@ class Task(futures._PyFuture): # Inherit Python Task implementation """A coroutine wrapped in a Future.""" # An important invariant maintained while a Task not done: + # _fut_waiter is either None or a Future. The Future + # can be either done() or not done(). + # The task can be in any of 3 states: # - # - Either _fut_waiter is None, and _step() is scheduled; - # - or _fut_waiter is some Future, and _step() is *not* scheduled. + # - 1: _fut_waiter is not None and not _fut_waiter.done(): + # __step() is *not* scheduled and the Task is waiting for _fut_waiter. + # - 2: (_fut_waiter is None or _fut_waiter.done()) and __step() is scheduled: + # the Task is waiting for __step() to be executed. + # - 3: _fut_waiter is None and __step() is *not* scheduled: + # the Task is currently executing (in __step()). # - # The only transition from the latter to the former is through - # _wakeup(). When _fut_waiter is not None, one of its callbacks - # must be _wakeup(). - - # If False, don't log a message if the task is destroyed whereas its + # * In state 1, one of the callbacks of __fut_waiter must be __wakeup(). + # * The transition from 1 to 2 happens when _fut_waiter becomes done(), + # as it schedules __wakeup() to be called (which calls __step() so + # we way that __step() is scheduled). + # * It transitions from 2 to 3 when __step() is executed, and it clears + # _fut_waiter to None. + + # If False, don't log a message if the task is destroyed while its # status is still pending _log_destroy_pending = True From 5bb6f0fcba663e1006f9063d1027ce8bd9f8effb Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 Sep 2023 15:27:44 -0700 Subject: [PATCH 040/124] gh-104909: Split some more insts into ops (#109943) These are the most popular specializations of `LOAD_ATTR` and `STORE_ATTR` that weren't already viable uops: * Split LOAD_ATTR_METHOD_WITH_VALUES * Split LOAD_ATTR_METHOD_NO_DICT * Split LOAD_ATTR_SLOT * Split STORE_ATTR_SLOT * Split STORE_ATTR_INSTANCE_VALUE Also: * Add `-v` flag to code generator which prints a list of non-viable uops (easter-egg: it can print execution counts -- see source) * Double _Py_UOP_MAX_TRACE_LENGTH to 128 I had dropped one of the DEOPT_IF() calls! :-( --- Include/internal/pycore_opcode_metadata.h | 120 +++++++++++--- Include/internal/pycore_uops.h | 2 +- Python/abstract_interp_cases.c.h | 47 ++++++ Python/bytecodes.c | 66 ++++++-- Python/executor_cases.c.h | 139 ++++++++++++++++ Python/generated_cases.c.h | 191 +++++++++++++--------- Tools/cases_generator/analysis.py | 58 +++++++ Tools/cases_generator/generate_cases.py | 11 ++ 8 files changed, 519 insertions(+), 115 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bb37e9a1d1b6b6..16c1637e496033 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -46,31 +46,40 @@ #define _GUARD_TYPE_VERSION 318 #define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 #define _LOAD_ATTR_INSTANCE_VALUE 320 -#define _IS_NONE 321 -#define _ITER_CHECK_LIST 322 -#define _ITER_JUMP_LIST 323 -#define _IS_ITER_EXHAUSTED_LIST 324 -#define _ITER_NEXT_LIST 325 -#define _ITER_CHECK_TUPLE 326 -#define _ITER_JUMP_TUPLE 327 -#define _IS_ITER_EXHAUSTED_TUPLE 328 -#define _ITER_NEXT_TUPLE 329 -#define _ITER_CHECK_RANGE 330 -#define _ITER_JUMP_RANGE 331 -#define _IS_ITER_EXHAUSTED_RANGE 332 -#define _ITER_NEXT_RANGE 333 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 334 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 335 -#define _CHECK_PEP_523 336 -#define _CHECK_FUNCTION_EXACT_ARGS 337 -#define _CHECK_STACK_SPACE 338 -#define _INIT_CALL_PY_EXACT_ARGS 339 -#define _PUSH_FRAME 340 -#define _POP_JUMP_IF_FALSE 341 -#define _POP_JUMP_IF_TRUE 342 -#define _JUMP_TO_TOP 343 -#define _SAVE_CURRENT_IP 344 -#define _INSERT 345 +#define _LOAD_ATTR_SLOT 321 +#define _GUARD_DORV_VALUES 322 +#define _STORE_ATTR_INSTANCE_VALUE 323 +#define _GUARD_TYPE_VERSION_STORE 324 +#define _STORE_ATTR_SLOT 325 +#define _IS_NONE 326 +#define _ITER_CHECK_LIST 327 +#define _ITER_JUMP_LIST 328 +#define _IS_ITER_EXHAUSTED_LIST 329 +#define _ITER_NEXT_LIST 330 +#define _ITER_CHECK_TUPLE 331 +#define _ITER_JUMP_TUPLE 332 +#define _IS_ITER_EXHAUSTED_TUPLE 333 +#define _ITER_NEXT_TUPLE 334 +#define _ITER_CHECK_RANGE 335 +#define _ITER_JUMP_RANGE 336 +#define _IS_ITER_EXHAUSTED_RANGE 337 +#define _ITER_NEXT_RANGE 338 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 339 +#define _GUARD_KEYS_VERSION 340 +#define _LOAD_ATTR_METHOD_WITH_VALUES 341 +#define _LOAD_ATTR_METHOD_NO_DICT 342 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 344 +#define _CHECK_PEP_523 345 +#define _CHECK_FUNCTION_EXACT_ARGS 346 +#define _CHECK_STACK_SPACE 347 +#define _INIT_CALL_PY_EXACT_ARGS 348 +#define _PUSH_FRAME 349 +#define _POP_JUMP_IF_FALSE 350 +#define _POP_JUMP_IF_TRUE 351 +#define _JUMP_TO_TOP 352 +#define _SAVE_CURRENT_IP 353 +#define _INSERT 354 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -356,6 +365,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_ATTR_WITH_HINT: return 1; + case _LOAD_ATTR_SLOT: + return 1; case LOAD_ATTR_SLOT: return 1; case LOAD_ATTR_CLASS: @@ -364,10 +375,18 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; + case _GUARD_DORV_VALUES: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; case STORE_ATTR_INSTANCE_VALUE: return 2; case STORE_ATTR_WITH_HINT: return 2; + case _GUARD_TYPE_VERSION_STORE: + return 1; + case _STORE_ATTR_SLOT: + return 2; case STORE_ATTR_SLOT: return 2; case COMPARE_OP: @@ -478,8 +497,16 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case PUSH_EXC_INFO: return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; case LOAD_ATTR_METHOD_WITH_VALUES: return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; case LOAD_ATTR_METHOD_NO_DICT: return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: @@ -896,18 +923,28 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_WITH_HINT: return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_SLOT: + case _LOAD_ATTR_SLOT: return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_SLOT: + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_CLASS: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_PROPERTY: return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; + case _GUARD_DORV_VALUES: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 0; case STORE_ATTR_INSTANCE_VALUE: return 0; case STORE_ATTR_WITH_HINT: return 0; + case _GUARD_TYPE_VERSION_STORE: + return 1; + case _STORE_ATTR_SLOT: + return 0; case STORE_ATTR_SLOT: return 0; case COMPARE_OP: @@ -1018,8 +1055,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case PUSH_EXC_INFO: return 2; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 2; case LOAD_ATTR_METHOD_WITH_VALUES: return 2; + case _LOAD_ATTR_METHOD_NO_DICT: + return 2; case LOAD_ATTR_METHOD_NO_DICT: return 2; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: @@ -1359,12 +1404,17 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, 0 }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [_GUARD_TYPE_VERSION_STORE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, 0 }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1420,7 +1470,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [SETUP_WITH] = { true, INSTR_FMT_IX, 0 }, [POP_BLOCK] = { true, INSTR_FMT_IX, 0 }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1583,6 +1637,9 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_STORE, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION_STORE, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, @@ -1600,6 +1657,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SET_IP, 7, 3 }, { _SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SET_IP, 7, 3 }, { _SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, @@ -1652,6 +1711,11 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_GUARD_TYPE_VERSION_STORE] = "_GUARD_TYPE_VERSION_STORE", + [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", [_IS_NONE] = "_IS_NONE", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", @@ -1665,6 +1729,10 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_ITER_JUMP_RANGE] = "_ITER_JUMP_RANGE", [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_PEP_523] = "_CHECK_PEP_523", diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index 249f5c010e0092..d8a7d978f1304e 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -10,7 +10,7 @@ extern "C" { #include "pycore_frame.h" // _PyInterpreterFrame -#define _Py_UOP_MAX_TRACE_LENGTH 64 +#define _Py_UOP_MAX_TRACE_LENGTH 128 typedef struct { uint32_t opcode; diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 5a3848cd726245..61b1db9e5a1543 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -474,6 +474,31 @@ break; } + case _LOAD_ATTR_SLOT: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _GUARD_DORV_VALUES: { + break; + } + + case _STORE_ATTR_INSTANCE_VALUE: { + STACK_SHRINK(2); + break; + } + + case _GUARD_TYPE_VERSION_STORE: { + break; + } + + case _STORE_ATTR_SLOT: { + STACK_SHRINK(2); + break; + } + case COMPARE_OP: { STACK_SHRINK(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); @@ -627,6 +652,28 @@ break; } + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + break; + } + + case _GUARD_KEYS_VERSION: { + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 402b27101dbdb6..0f89779fb9245f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1939,10 +1939,7 @@ dummy_func( DECREF_INPUTS(); } - inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -1952,6 +1949,12 @@ dummy_func( DECREF_INPUTS(); } + macro(LOAD_ATTR_SLOT) = + unused/1 + + _GUARD_TYPE_VERSION + + _LOAD_ATTR_SLOT + // NOTE: This action may also deopt + unused/5; + inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); @@ -2019,13 +2022,15 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) { + op(_GUARD_DORV_VALUES, (owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + } + + op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; @@ -2039,6 +2044,12 @@ dummy_func( Py_DECREF(owner); } + macro(STORE_ATTR_INSTANCE_VALUE) = + unused/1 + + _GUARD_TYPE_VERSION_STORE + + _GUARD_DORV_VALUES + + _STORE_ATTR_INSTANCE_VALUE; + inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2080,10 +2091,13 @@ dummy_func( Py_DECREF(owner); } - inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) { + op(_GUARD_TYPE_VERSION_STORE, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + } + + op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -2092,6 +2106,11 @@ dummy_func( Py_DECREF(owner); } + macro(STORE_ATTR_SLOT) = + unused/1 + + _GUARD_TYPE_VERSION_STORE + + _STORE_ATTR_SLOT; + family(COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP) = { COMPARE_OP_FLOAT, COMPARE_OP_INT, @@ -2769,20 +2788,25 @@ dummy_func( exc_info->exc_value = Py_NewRef(new_exc); } - inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, self if (1))) { - assert(oparg & 1); - /* Cached method object */ + op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + + op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { + PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + assert(oparg & 1); + /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2790,10 +2814,16 @@ dummy_func( self = owner; } - inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { + macro(LOAD_ATTR_METHOD_WITH_VALUES) = + unused/1 + + _GUARD_TYPE_VERSION + + _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + + _GUARD_KEYS_VERSION + + _LOAD_ATTR_METHOD_WITH_VALUES; + + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2802,6 +2832,12 @@ dummy_func( self = owner; } + macro(LOAD_ATTR_METHOD_NO_DICT) = + unused/1 + + _GUARD_TYPE_VERSION + + unused/2 + + _LOAD_ATTR_METHOD_NO_DICT; + inst(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index befb972f1e90f5..981db6973f281a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1714,6 +1714,83 @@ break; } + case _LOAD_ATTR_SLOT: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)operand; + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + break; + } + + case _GUARD_DORV_VALUES: { + PyObject *owner; + owner = stack_pointer[-1]; + PyTypeObject *tp = Py_TYPE(owner); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + break; + } + + case _STORE_ATTR_INSTANCE_VALUE: { + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint16_t index = (uint16_t)operand; + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + STACK_SHRINK(2); + break; + } + + case _GUARD_TYPE_VERSION_STORE: { + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)operand; + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + break; + } + + case _STORE_ATTR_SLOT: { + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint16_t index = (uint16_t)operand; + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); + STACK_SHRINK(2); + break; + } + case COMPARE_OP: { PyObject *right; PyObject *left; @@ -2219,6 +2296,68 @@ break; } + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + PyObject *owner; + owner = stack_pointer[-1]; + PyTypeObject *owner_cls = Py_TYPE(owner); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); + break; + } + + case _GUARD_KEYS_VERSION: { + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t keys_version = (uint32_t)operand; + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != + keys_version, LOAD_ATTR); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES: { + PyObject *owner; + PyObject *attr; + PyObject *self; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)operand; + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + STACK_GROW(1); + stack_pointer[-2] = attr; + stack_pointer[-1] = self; + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT: { + PyObject *owner; + PyObject *attr; + PyObject *self; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)operand; + assert(oparg & 1); + PyTypeObject *owner_cls = Py_TYPE(owner); + assert(owner_cls->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + STACK_GROW(1); + stack_pointer[-2] = attr; + stack_pointer[-1] = self; + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; PyObject *callable; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ebb87a86de432e..17df44019a6581 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2504,19 +2504,25 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&next_instr[3].cache); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } @@ -2615,27 +2621,38 @@ TARGET(STORE_ATTR_INSTANCE_VALUE) { PyObject *owner; PyObject *value; + // _GUARD_TYPE_VERSION_STORE owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - else { - Py_DECREF(old_value); + // _GUARD_DORV_VALUES + { + PyTypeObject *tp = Py_TYPE(owner); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + } + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&next_instr[3].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); } - Py_DECREF(owner); STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2694,19 +2711,25 @@ TARGET(STORE_ATTR_SLOT) { PyObject *owner; PyObject *value; + // _GUARD_TYPE_VERSION_STORE owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + { + uint16_t index = read_u16(&next_instr[3].cache); + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); + } STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -3557,28 +3580,42 @@ PyObject *owner; PyObject *attr; PyObject *self; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t keys_version = read_u32(&next_instr[3].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - assert(oparg & 1); - /* Cached method object */ - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyTypeObject *owner_cls = Py_TYPE(owner); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&next_instr[3].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != + keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&next_instr[5].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; @@ -3590,18 +3627,26 @@ PyObject *owner; PyObject *attr; PyObject *self; + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - assert(oparg & 1); - PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(owner_cls->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + { + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&next_instr[5].cache); + assert(oparg & 1); + PyTypeObject *owner_cls = Py_TYPE(owner); + assert(owner_cls->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index b920c0aa8c1c8a..91dcba8ceee13d 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -414,3 +414,61 @@ def check_macro_components( case _: assert_never(uop) return components + + def report_non_viable_uops(self, jsonfile: str) -> None: + print("The following ops are not viable uops:") + skips = { + "CACHE", + "RESERVED", + "INTERPRETER_EXIT", + "JUMP_BACKWARD", + "LOAD_FAST_LOAD_FAST", + "LOAD_CONST_LOAD_FAST", + "STORE_FAST_STORE_FAST", + "_BINARY_OP_INPLACE_ADD_UNICODE", + "POP_JUMP_IF_TRUE", + "POP_JUMP_IF_FALSE", + "_ITER_JUMP_LIST", + "_ITER_JUMP_TUPLE", + "_ITER_JUMP_RANGE", + } + try: + # Secret feature: if bmraw.json exists, print and sort by execution count + counts = load_execution_counts(jsonfile) + except FileNotFoundError as err: + counts = {} + non_viable = [ + instr + for instr in self.instrs.values() + if instr.name not in skips + and not instr.name.startswith("INSTRUMENTED_") + and not instr.is_viable_uop() + ] + non_viable.sort(key=lambda instr: (-counts.get(instr.name, 0), instr.name)) + for instr in non_viable: + if instr.name in counts: + scount = f"{counts[instr.name]:,}" + else: + scount = "" + print(f" {scount:>15} {instr.name:<35}", end="") + if instr.name in self.families: + print(" (unspecialized)", end="") + elif instr.family is not None: + print(f" (specialization of {instr.family.name})", end="") + print() + + +def load_execution_counts(jsonfile: str) -> dict[str, int]: + import json + + with open(jsonfile) as f: + jsondata = json.load(f) + + # Look for keys like "opcode[LOAD_FAST].execution_count" + prefix = "opcode[" + suffix = "].execution_count" + res: dict[str, int] = {} + for key, value in jsondata.items(): + if key.startswith(prefix) and key.endswith(suffix): + res[key[len(prefix) : -len(suffix)]] = value + return res diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 898736248a98f9..9192d1038ab7d6 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -92,6 +92,13 @@ description="Generate the code for the interpreter switch.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) + +arg_parser.add_argument( + "-v", + "--verbose", + help="Print list of non-viable uops and exit", + action="store_true", +) arg_parser.add_argument( "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) @@ -865,6 +872,10 @@ def main() -> None: a.analyze() # Prints messages and sets a.errors on failure if a.errors: sys.exit(f"Found {a.errors} errors") + if args.verbose: + # Load execution counts from bmraw.json, if it exists + a.report_non_viable_uops("bmraw.json") + return # These raise OSError if output can't be written a.write_instructions(args.output, args.emit_line_directives) From f65f9e80fe741c894582a3e413d4e3318c1ed626 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 28 Sep 2023 05:26:42 +0300 Subject: [PATCH 041/124] gh-109818: `reprlib.recursive_repr` copies `__type_params__` (#109819) --- Lib/reprlib.py | 1 + Lib/test/test_reprlib.py | 11 +++++++++++ .../2023-09-25-09-59-59.gh-issue-109818.dLRtT-.rst | 2 ++ 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-25-09-59-59.gh-issue-109818.dLRtT-.rst diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 840dd0e20132b1..05bb1a0eb01795 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -29,6 +29,7 @@ def wrapper(self): wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__type_params__ = getattr(user_function, '__type_params__', ()) wrapper.__wrapped__ = user_function return wrapper diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 502287b620d066..3e93b561c143d8 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -774,5 +774,16 @@ def __repr__(self): self.assertIs(X.f, X.__repr__.__wrapped__) + def test__type_params__(self): + class My: + @recursive_repr() + def __repr__[T: str](self, default: T = '') -> str: + return default + + type_params = My().__repr__.__type_params__ + self.assertEqual(len(type_params), 1) + self.assertEqual(type_params[0].__name__, 'T') + self.assertEqual(type_params[0].__bound__, str) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-09-25-09-59-59.gh-issue-109818.dLRtT-.rst b/Misc/NEWS.d/next/Library/2023-09-25-09-59-59.gh-issue-109818.dLRtT-.rst new file mode 100644 index 00000000000000..184086af2585ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-25-09-59-59.gh-issue-109818.dLRtT-.rst @@ -0,0 +1,2 @@ +Fix :func:`reprlib.recursive_repr` not copying ``__type_params__`` from +decorated function. From 99fba5f156386cf8f4a71321708690cdb9357ffc Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 27 Sep 2023 21:29:39 -0500 Subject: [PATCH 042/124] gh-109812: Fix phrasing for `collections.Counter` (gh-109813) --- Doc/library/collections.rst | 2 +- Misc/ACKS | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 03cb1dca8f816c..43f4ff077b40e0 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -358,7 +358,7 @@ Common patterns for working with :class:`Counter` objects:: list(c) # list unique elements set(c) # convert to a set dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs + c.items() # access the (elem, cnt) pairs Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs c.most_common()[:-n-1:-1] # n least common elements +c # remove zero and negative counts diff --git a/Misc/ACKS b/Misc/ACKS index aaa178fc3b5d08..ccdfae66832f0e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -349,6 +349,7 @@ Robbie Clemons Steve Clift Hervé Coatanhay Riccardo Coccioli +Jacob Coffee Nick Coghlan Josh Cogliati Noam Cohen From 526380e28644236bde9e41b949497ca1ee22653f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 28 Sep 2023 05:40:59 +0100 Subject: [PATCH 043/124] GH-109190: Copyedit 3.12 What's New: Bytecode (#109821) Co-authored-by: Hugo van Kemenade --- Doc/whatsnew/3.12.rst | 32 ++++++++++++++++++++++++++------ Misc/NEWS.d/3.12.0a4.rst | 4 ++-- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 75a784f620f5fd..ec39616d7c9d2b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -701,6 +701,9 @@ dis :data:`dis.hasarg` collection instead. (Contributed by Irit Katriel in :gh:`94216`.) +* Add the :data:`dis.hasexc` collection to signify instructions that set + an exception handler. (Contributed by Irit Katriel in :gh:`94216`.) + fractions --------- @@ -884,6 +887,10 @@ statistics sys --- +* Add the :mod:`sys.monitoring` namespace to expose the new :ref:`PEP 669 + ` monitoring API. + (Contributed by Mark Shannon in :gh:`103082`.) + * Add :func:`sys.activate_stack_trampoline` and :func:`sys.deactivate_stack_trampoline` for activating and deactivating stack profiler trampolines, @@ -1083,9 +1090,27 @@ CPython bytecode changes * Remove the :opcode:`!PRECALL` instruction. (Contributed by Mark Shannon in :gh:`92925`.) +* Add the :opcode:`BINARY_SLICE` and :opcode:`STORE_SLICE` instructions. + (Contributed by Mark Shannon in :gh:`94163`.) + +* Add the :opcode:`CALL_INTRINSIC_1` instructions. + (Contributed by Mark Shannon in :gh:`99005`.) + +* Add the :opcode:`CALL_INTRINSIC_2` instruction. + (Contributed by Irit Katriel in :gh:`101799`.) + +* Add the :opcode:`CLEANUP_THROW` instruction. + (Contributed by Brandt Bucher in :gh:`90997`.) + +* Add the :opcode:`!END_SEND` instruction. + (Contributed by Mark Shannon in :gh:`103082`.) + * Add the :opcode:`LOAD_FAST_AND_CLEAR` instruction as part of the implementation of :pep:`709`. (Contributed by Carl Meyer in :gh:`101441`.) +* Add the :opcode:`LOAD_FAST_CHECK` instruction. + (Contributed by Dennis Sweeney in :gh:`93143`.) + * Add the :opcode:`LOAD_FROM_DICT_OR_DEREF`, :opcode:`LOAD_FROM_DICT_OR_GLOBALS`, and :opcode:`LOAD_LOCALS` opcodes as part of the implementation of :pep:`695`. Remove the :opcode:`!LOAD_CLASSDEREF` opcode, which can be replaced with @@ -1095,12 +1120,7 @@ CPython bytecode changes * Add the :opcode:`LOAD_SUPER_ATTR` instruction. (Contributed by Carl Meyer and Vladimir Matveev in :gh:`103497`.) -FOR_ITER new behavior is not mentioned -The fact that POP_JUMP_IF_* family of instructions are now real instructions is not mentioned -YIELD_VALUE need for an argument is not mentioned - - - +* Add the :opcode:`RETURN_CONST` instruction. (Contributed by Wenyang Wang in :gh:`101632`.) Demos and Tools =============== diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index b3b39024056ccc..75246f3f13503e 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -23,10 +23,10 @@ Remove :opcode:`UNARY_POSITIVE`, :opcode:`ASYNC_GEN_WRAP` and .. nonce: D7H6j4 .. section: Core and Builtins -Add new :opcode:`CALL_INSTRINSIC_1` instruction. Remove +Add new :opcode:`CALL_INTRINSIC_1` instruction. Remove :opcode:`IMPORT_STAR`, :opcode:`PRINT_EXPR` and :opcode:`STOPITERATION_ERROR`, replacing them with the -:opcode:`CALL_INSTRINSIC_1` instruction. +:opcode:`CALL_INTRINSIC_1` instruction. .. From c88037d137a98d7c399c7bd74d5117b5bcae1543 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 28 Sep 2023 00:45:13 -0400 Subject: [PATCH 044/124] gh-109991: Update GitHub CI workflows to use OpenSSL 3.0.11 and multissltests to use 1.1.1w, 3.0.11, and 3.1.3. (gh-110002) --- .github/workflows/build.yml | 8 ++++---- .../2023-09-27-23-31-54.gh-issue-109991.sUUYY8.rst | 2 ++ Tools/ssl/multissltests.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-09-27-23-31-54.gh-issue-109991.sUUYY8.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28ebc1643bd694..a60632dc565235 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -261,7 +261,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1v + OPENSSL_VER: 3.0.11 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -330,7 +330,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1v, 3.0.10, 3.1.2] + openssl_ver: [1.1.1w, 3.0.11, 3.1.3] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -382,7 +382,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 1.1.1v + OPENSSL_VER: 3.0.11 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -491,7 +491,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1v + OPENSSL_VER: 3.0.11 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-09-27-23-31-54.gh-issue-109991.sUUYY8.rst b/Misc/NEWS.d/next/Tools-Demos/2023-09-27-23-31-54.gh-issue-109991.sUUYY8.rst new file mode 100644 index 00000000000000..13c1163ab53443 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-09-27-23-31-54.gh-issue-109991.sUUYY8.rst @@ -0,0 +1,2 @@ +Update GitHub CI workflows to use OpenSSL 3.0.11 and multissltests to use +1.1.1w, 3.0.11, and 3.1.3. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index fc261c770d6944..f066fb52cfd496 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,9 +46,9 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1v", - "3.0.10", - "3.1.2", + "1.1.1w", + "3.0.11", + "3.1.3", ] LIBRESSL_OLD_VERSIONS = [ From 98c0c1de18e9ec02a0dde0a89b9acf9415891de2 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 28 Sep 2023 02:08:39 -0400 Subject: [PATCH 045/124] gh-109991: Update macOS installer to use OpenSSL 3.0.10. (GH-110003) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2023-09-27-22-35-22.gh-issue-109991.-xJzaF.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-09-27-22-35-22.gh-issue-109991.-xJzaF.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index c108ee21a7a2a3..c54de880c20360 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.10", - url="https://www.openssl.org/source/openssl-3.0.10.tar.gz", - checksum='1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323', + name="OpenSSL 3.0.11", + url="https://www.openssl.org/source/openssl-3.0.11.tar.gz", + checksum='b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/macOS/2023-09-27-22-35-22.gh-issue-109991.-xJzaF.rst b/Misc/NEWS.d/next/macOS/2023-09-27-22-35-22.gh-issue-109991.-xJzaF.rst new file mode 100644 index 00000000000000..8d369988274f28 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-09-27-22-35-22.gh-issue-109991.-xJzaF.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.11. From 8f324b7ecd2df3036fab098c4c8ac185ac07b277 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Sep 2023 13:24:15 +0200 Subject: [PATCH 046/124] gh-109972: Split test_gdb.py into test_gdb package (#109977) Split test_gdb.py file into a test_gdb package made of multiple tests, so tests can now be run in parallel. * Create Lib/test/test_gdb/ directory. * Split test_gdb.py into multiple files in Lib/test/test_gdb/ directory. * Move Lib/test/gdb_sample.py to Lib/test/test_gdb/ directory. Update get_sample_script(): use __file__ to locate gdb_sample.py. * Move gdb_has_frame_select() and HAS_PYUP_PYDOWN to test_misc.py. * Explicitly skip test_gdb on Windows. Previously, test_gdb was skipped even if gdb was available because of gdb_has_frame_select(). --- Lib/test/libregrtest/findtests.py | 1 + Lib/test/test_gdb.py | 1065 ----------------- Lib/test/test_gdb/__init__.py | 10 + Lib/test/{ => test_gdb}/gdb_sample.py | 2 +- Lib/test/test_gdb/test_backtrace.py | 134 +++ Lib/test/test_gdb/test_cfunction.py | 83 ++ Lib/test/test_gdb/test_misc.py | 188 +++ Lib/test/test_gdb/test_pretty_print.py | 400 +++++++ Lib/test/test_gdb/util.py | 304 +++++ Makefile.pre.in | 1 + ...-09-28-12-25-19.gh-issue-109972.GYnwIP.rst | 2 + 11 files changed, 1124 insertions(+), 1066 deletions(-) delete mode 100644 Lib/test/test_gdb.py create mode 100644 Lib/test/test_gdb/__init__.py rename Lib/test/{ => test_gdb}/gdb_sample.py (75%) create mode 100644 Lib/test/test_gdb/test_backtrace.py create mode 100644 Lib/test/test_gdb/test_cfunction.py create mode 100644 Lib/test/test_gdb/test_misc.py create mode 100644 Lib/test/test_gdb/test_pretty_print.py create mode 100644 Lib/test/test_gdb/util.py create mode 100644 Misc/NEWS.d/next/Tests/2023-09-28-12-25-19.gh-issue-109972.GYnwIP.rst diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 60f21980c10dd0..96cc3e0d021184 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -19,6 +19,7 @@ "test_asyncio", "test_concurrent_futures", "test_future_stmt", + "test_gdb", "test_multiprocessing_fork", "test_multiprocessing_forkserver", "test_multiprocessing_spawn", diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py deleted file mode 100644 index 5a4394a0993c8d..00000000000000 --- a/Lib/test/test_gdb.py +++ /dev/null @@ -1,1065 +0,0 @@ -# Verify that gdb can pretty-print the various PyObject* types -# -# The code for testing gdb was adapted from similar work in Unladen Swallow's -# Lib/test/test_jit_gdb.py - -import os -import re -import subprocess -import sys -import sysconfig -import textwrap -import unittest - -from test import support -from test.support import findfile, python_is_optimized - -def get_gdb_version(): - try: - cmd = ["gdb", "-nx", "--version"] - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - with proc: - version, stderr = proc.communicate() - - if proc.returncode: - raise Exception(f"Command {' '.join(cmd)!r} failed " - f"with exit code {proc.returncode}: " - f"stdout={version!r} stderr={stderr!r}") - except OSError: - # This is what "no gdb" looks like. There may, however, be other - # errors that manifest this way too. - raise unittest.SkipTest("Couldn't find gdb on the path") - - # Regex to parse: - # 'GNU gdb (GDB; SUSE Linux Enterprise 12) 7.7\n' -> 7.7 - # 'GNU gdb (GDB) Fedora 7.9.1-17.fc22\n' -> 7.9 - # 'GNU gdb 6.1.1 [FreeBSD]\n' -> 6.1 - # 'GNU gdb (GDB) Fedora (7.5.1-37.fc18)\n' -> 7.5 - # 'HP gdb 6.7 for HP Itanium (32 or 64 bit) and target HP-UX 11iv2 and 11iv3.\n' -> 6.7 - match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", version) - if match is None: - raise Exception("unable to parse GDB version: %r" % version) - return (version, int(match.group(1)), int(match.group(2))) - -gdb_version, gdb_major_version, gdb_minor_version = get_gdb_version() -if gdb_major_version < 7: - raise unittest.SkipTest("gdb versions before 7.0 didn't support python " - "embedding. Saw %s.%s:\n%s" - % (gdb_major_version, gdb_minor_version, - gdb_version)) - -if not sysconfig.is_python_build(): - raise unittest.SkipTest("test_gdb only works on source builds at the moment.") - -if ((sysconfig.get_config_var('PGO_PROF_USE_FLAG') or 'xxx') in - (sysconfig.get_config_var('PY_CORE_CFLAGS') or '')): - raise unittest.SkipTest("test_gdb is not reliable on PGO builds") - -# Location of custom hooks file in a repository checkout. -checkout_hook_path = os.path.join(os.path.dirname(sys.executable), - 'python-gdb.py') - -PYTHONHASHSEED = '123' - - -def cet_protection(): - cflags = sysconfig.get_config_var('CFLAGS') - if not cflags: - return False - flags = cflags.split() - # True if "-mcet -fcf-protection" options are found, but false - # if "-fcf-protection=none" or "-fcf-protection=return" is found. - return (('-mcet' in flags) - and any((flag.startswith('-fcf-protection') - and not flag.endswith(("=none", "=return"))) - for flag in flags)) - -# Control-flow enforcement technology -CET_PROTECTION = cet_protection() - - -def run_gdb(*args, **env_vars): - """Runs gdb in --batch mode with the additional arguments given by *args. - - Returns its (stdout, stderr) decoded from utf-8 using the replace handler. - """ - if env_vars: - env = os.environ.copy() - env.update(env_vars) - else: - env = None - # -nx: Do not execute commands from any .gdbinit initialization files - # (issue #22188) - base_cmd = ('gdb', '--batch', '-nx') - if (gdb_major_version, gdb_minor_version) >= (7, 4): - base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path) - proc = subprocess.Popen(base_cmd + args, - # Redirect stdin to prevent GDB from messing with - # the terminal settings - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) - with proc: - out, err = proc.communicate() - return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') - -# Verify that "gdb" was built with the embedded python support enabled: -gdbpy_version, _ = run_gdb("--eval-command=python import sys; print(sys.version_info)") -if not gdbpy_version: - raise unittest.SkipTest("gdb not built with embedded python support") - -if "major=2" in gdbpy_version: - raise unittest.SkipTest("gdb built with Python 2") - -# Verify that "gdb" can load our custom hooks, as OS security settings may -# disallow this without a customized .gdbinit. -_, gdbpy_errors = run_gdb('--args', sys.executable) -if "auto-loading has been declined" in gdbpy_errors: - msg = "gdb security settings prevent use of custom hooks: " - raise unittest.SkipTest(msg + gdbpy_errors.rstrip()) - -def gdb_has_frame_select(): - # Does this build of gdb have gdb.Frame.select ? - stdout, _ = run_gdb("--eval-command=python print(dir(gdb.Frame))") - m = re.match(r'.*\[(.*)\].*', stdout) - if not m: - raise unittest.SkipTest("Unable to parse output from gdb.Frame.select test") - gdb_frame_dir = m.group(1).split(', ') - return "'select'" in gdb_frame_dir - -HAS_PYUP_PYDOWN = gdb_has_frame_select() - -BREAKPOINT_FN='builtin_id' - -@unittest.skipIf(support.PGO, "not useful for PGO") -class DebuggerTests(unittest.TestCase): - - """Test that the debugger can debug Python.""" - - def get_stack_trace(self, source=None, script=None, - breakpoint=BREAKPOINT_FN, - cmds_after_breakpoint=None, - import_site=False, - ignore_stderr=False): - ''' - Run 'python -c SOURCE' under gdb with a breakpoint. - - Support injecting commands after the breakpoint is reached - - Returns the stdout from gdb - - cmds_after_breakpoint: if provided, a list of strings: gdb commands - ''' - # We use "set breakpoint pending yes" to avoid blocking with a: - # Function "foo" not defined. - # Make breakpoint pending on future shared library load? (y or [n]) - # error, which typically happens python is dynamically linked (the - # breakpoints of interest are to be found in the shared library) - # When this happens, we still get: - # Function "textiowrapper_write" not defined. - # emitted to stderr each time, alas. - - # Initially I had "--eval-command=continue" here, but removed it to - # avoid repeated print breakpoints when traversing hierarchical data - # structures - - # Generate a list of commands in gdb's language: - commands = ['set breakpoint pending yes', - 'break %s' % breakpoint, - - # The tests assume that the first frame of printed - # backtrace will not contain program counter, - # that is however not guaranteed by gdb - # therefore we need to use 'set print address off' to - # make sure the counter is not there. For example: - # #0 in PyObject_Print ... - # is assumed, but sometimes this can be e.g. - # #0 0x00003fffb7dd1798 in PyObject_Print ... - 'set print address off', - - 'run'] - - # GDB as of 7.4 onwards can distinguish between the - # value of a variable at entry vs current value: - # http://sourceware.org/gdb/onlinedocs/gdb/Variables.html - # which leads to the selftests failing with errors like this: - # AssertionError: 'v@entry=()' != '()' - # Disable this: - if (gdb_major_version, gdb_minor_version) >= (7, 4): - commands += ['set print entry-values no'] - - if cmds_after_breakpoint: - if CET_PROTECTION: - # bpo-32962: When Python is compiled with -mcet - # -fcf-protection, function arguments are unusable before - # running the first instruction of the function entry point. - # The 'next' command makes the required first step. - commands += ['next'] - commands += cmds_after_breakpoint - else: - commands += ['backtrace'] - - # print commands - - # Use "commands" to generate the arguments with which to invoke "gdb": - args = ['--eval-command=%s' % cmd for cmd in commands] - args += ["--args", - sys.executable] - args.extend(subprocess._args_from_interpreter_flags()) - - if not import_site: - # -S suppresses the default 'import site' - args += ["-S"] - - if source: - args += ["-c", source] - elif script: - args += [script] - - # Use "args" to invoke gdb, capturing stdout, stderr: - out, err = run_gdb(*args, PYTHONHASHSEED=PYTHONHASHSEED) - - if not ignore_stderr: - for line in err.splitlines(): - print(line, file=sys.stderr) - - # bpo-34007: Sometimes some versions of the shared libraries that - # are part of the traceback are compiled in optimised mode and the - # Program Counter (PC) is not present, not allowing gdb to walk the - # frames back. When this happens, the Python bindings of gdb raise - # an exception, making the test impossible to succeed. - if "PC not saved" in err: - raise unittest.SkipTest("gdb cannot walk the frame object" - " because the Program Counter is" - " not present") - - # bpo-40019: Skip the test if gdb failed to read debug information - # because the Python binary is optimized. - for pattern in ( - '(frame information optimized out)', - 'Unable to read information on python frame', - # gh-91960: On Python built with "clang -Og", gdb gets - # "frame=" for _PyEval_EvalFrameDefault() parameter - '(unable to read python frame information)', - # gh-104736: On Python built with "clang -Og" on ppc64le, - # "py-bt" displays a truncated or not traceback, but "where" - # logs this error message: - 'Backtrace stopped: frame did not save the PC', - # gh-104736: When "bt" command displays something like: - # "#1 0x0000000000000000 in ?? ()", the traceback is likely - # truncated or wrong. - ' ?? ()', - ): - if pattern in out: - raise unittest.SkipTest(f"{pattern!r} found in gdb output") - - return out - - def get_gdb_repr(self, source, - cmds_after_breakpoint=None, - import_site=False): - # Given an input python source representation of data, - # run "python -c'id(DATA)'" under gdb with a breakpoint on - # builtin_id and scrape out gdb's representation of the "op" - # parameter, and verify that the gdb displays the same string - # - # Verify that the gdb displays the expected string - # - # For a nested structure, the first time we hit the breakpoint will - # give us the top-level structure - - # NOTE: avoid decoding too much of the traceback as some - # undecodable characters may lurk there in optimized mode - # (issue #19743). - cmds_after_breakpoint = cmds_after_breakpoint or ["backtrace 1"] - gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN, - cmds_after_breakpoint=cmds_after_breakpoint, - import_site=import_site) - # gdb can insert additional '\n' and space characters in various places - # in its output, depending on the width of the terminal it's connected - # to (using its "wrap_here" function) - m = re.search( - # Match '#0 builtin_id(self=..., v=...)' - r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)' - # Match ' at Python/bltinmodule.c'. - # bpo-38239: builtin_id() is defined in Python/bltinmodule.c, - # but accept any "Directory\file.c" to support Link Time - # Optimization (LTO). - r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c', - gdb_output, re.DOTALL) - if not m: - self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) - return m.group(1), gdb_output - - def assertEndsWith(self, actual, exp_end): - '''Ensure that the given "actual" string ends with "exp_end"''' - self.assertTrue(actual.endswith(exp_end), - msg='%r did not end with %r' % (actual, exp_end)) - - def assertMultilineMatches(self, actual, pattern): - m = re.match(pattern, actual, re.DOTALL) - if not m: - self.fail(msg='%r did not match %r' % (actual, pattern)) - - def get_sample_script(self): - return findfile('gdb_sample.py') - -class PrettyPrintTests(DebuggerTests): - def test_getting_backtrace(self): - gdb_output = self.get_stack_trace('id(42)') - self.assertTrue(BREAKPOINT_FN in gdb_output) - - def assertGdbRepr(self, val, exp_repr=None): - # Ensure that gdb's rendering of the value in a debugged process - # matches repr(value) in this process: - gdb_repr, gdb_output = self.get_gdb_repr('id(' + ascii(val) + ')') - if not exp_repr: - exp_repr = repr(val) - self.assertEqual(gdb_repr, exp_repr, - ('%r did not equal expected %r; full output was:\n%s' - % (gdb_repr, exp_repr, gdb_output))) - - @support.requires_resource('cpu') - def test_int(self): - 'Verify the pretty-printing of various int values' - self.assertGdbRepr(42) - self.assertGdbRepr(0) - self.assertGdbRepr(-7) - self.assertGdbRepr(1000000000000) - self.assertGdbRepr(-1000000000000000) - - def test_singletons(self): - 'Verify the pretty-printing of True, False and None' - self.assertGdbRepr(True) - self.assertGdbRepr(False) - self.assertGdbRepr(None) - - def test_dicts(self): - 'Verify the pretty-printing of dictionaries' - self.assertGdbRepr({}) - self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}") - # Python preserves insertion order since 3.6 - self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'foo': 'bar', 'douglas': 42}") - - def test_lists(self): - 'Verify the pretty-printing of lists' - self.assertGdbRepr([]) - self.assertGdbRepr(list(range(5))) - - @support.requires_resource('cpu') - def test_bytes(self): - 'Verify the pretty-printing of bytes' - self.assertGdbRepr(b'') - self.assertGdbRepr(b'And now for something hopefully the same') - self.assertGdbRepr(b'string with embedded NUL here \0 and then some more text') - self.assertGdbRepr(b'this is a tab:\t' - b' this is a slash-N:\n' - b' this is a slash-R:\r' - ) - - self.assertGdbRepr(b'this is byte 255:\xff and byte 128:\x80') - - self.assertGdbRepr(bytes([b for b in range(255)])) - - @support.requires_resource('cpu') - def test_strings(self): - 'Verify the pretty-printing of unicode strings' - # We cannot simply call locale.getpreferredencoding() here, - # as GDB might have been linked against a different version - # of Python with a different encoding and coercion policy - # with respect to PEP 538 and PEP 540. - out, err = run_gdb( - '--eval-command', - 'python import locale; print(locale.getpreferredencoding())') - - encoding = out.rstrip() - if err or not encoding: - raise RuntimeError( - f'unable to determine the preferred encoding ' - f'of embedded Python in GDB: {err}') - - def check_repr(text): - try: - text.encode(encoding) - except UnicodeEncodeError: - self.assertGdbRepr(text, ascii(text)) - else: - self.assertGdbRepr(text) - - self.assertGdbRepr('') - self.assertGdbRepr('And now for something hopefully the same') - self.assertGdbRepr('string with embedded NUL here \0 and then some more text') - - # Test printing a single character: - # U+2620 SKULL AND CROSSBONES - check_repr('\u2620') - - # Test printing a Japanese unicode string - # (I believe this reads "mojibake", using 3 characters from the CJK - # Unified Ideographs area, followed by U+3051 HIRAGANA LETTER KE) - check_repr('\u6587\u5b57\u5316\u3051') - - # Test a character outside the BMP: - # U+1D121 MUSICAL SYMBOL C CLEF - # This is: - # UTF-8: 0xF0 0x9D 0x84 0xA1 - # UTF-16: 0xD834 0xDD21 - check_repr(chr(0x1D121)) - - def test_tuples(self): - 'Verify the pretty-printing of tuples' - self.assertGdbRepr(tuple(), '()') - self.assertGdbRepr((1,), '(1,)') - self.assertGdbRepr(('foo', 'bar', 'baz')) - - @support.requires_resource('cpu') - def test_sets(self): - 'Verify the pretty-printing of sets' - if (gdb_major_version, gdb_minor_version) < (7, 3): - self.skipTest("pretty-printing of sets needs gdb 7.3 or later") - self.assertGdbRepr(set(), "set()") - self.assertGdbRepr(set(['a']), "{'a'}") - # PYTHONHASHSEED is need to get the exact frozenset item order - if not sys.flags.ignore_environment: - self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") - self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") - - # Ensure that we handle sets containing the "dummy" key value, - # which happens on deletion: - gdb_repr, gdb_output = self.get_gdb_repr('''s = set(['a','b']) -s.remove('a') -id(s)''') - self.assertEqual(gdb_repr, "{'b'}") - - @support.requires_resource('cpu') - def test_frozensets(self): - 'Verify the pretty-printing of frozensets' - if (gdb_major_version, gdb_minor_version) < (7, 3): - self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") - self.assertGdbRepr(frozenset(), "frozenset()") - self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") - # PYTHONHASHSEED is need to get the exact frozenset item order - if not sys.flags.ignore_environment: - self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") - self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") - - def test_exceptions(self): - # Test a RuntimeError - gdb_repr, gdb_output = self.get_gdb_repr(''' -try: - raise RuntimeError("I am an error") -except RuntimeError as e: - id(e) -''') - self.assertEqual(gdb_repr, - "RuntimeError('I am an error',)") - - - # Test division by zero: - gdb_repr, gdb_output = self.get_gdb_repr(''' -try: - a = 1 / 0 -except ZeroDivisionError as e: - id(e) -''') - self.assertEqual(gdb_repr, - "ZeroDivisionError('division by zero',)") - - def test_modern_class(self): - 'Verify the pretty-printing of new-style class instances' - gdb_repr, gdb_output = self.get_gdb_repr(''' -class Foo: - pass -foo = Foo() -foo.an_int = 42 -id(foo)''') - m = re.match(r'', gdb_repr) - self.assertTrue(m, - msg='Unexpected new-style class rendering %r' % gdb_repr) - - def test_subclassing_list(self): - 'Verify the pretty-printing of an instance of a list subclass' - gdb_repr, gdb_output = self.get_gdb_repr(''' -class Foo(list): - pass -foo = Foo() -foo += [1, 2, 3] -foo.an_int = 42 -id(foo)''') - m = re.match(r'', gdb_repr) - - self.assertTrue(m, - msg='Unexpected new-style class rendering %r' % gdb_repr) - - def test_subclassing_tuple(self): - 'Verify the pretty-printing of an instance of a tuple subclass' - # This should exercise the negative tp_dictoffset code in the - # new-style class support - gdb_repr, gdb_output = self.get_gdb_repr(''' -class Foo(tuple): - pass -foo = Foo((1, 2, 3)) -foo.an_int = 42 -id(foo)''') - m = re.match(r'', gdb_repr) - - self.assertTrue(m, - msg='Unexpected new-style class rendering %r' % gdb_repr) - - def assertSane(self, source, corruption, exprepr=None): - '''Run Python under gdb, corrupting variables in the inferior process - immediately before taking a backtrace. - - Verify that the variable's representation is the expected failsafe - representation''' - if corruption: - cmds_after_breakpoint=[corruption, 'backtrace'] - else: - cmds_after_breakpoint=['backtrace'] - - gdb_repr, gdb_output = \ - self.get_gdb_repr(source, - cmds_after_breakpoint=cmds_after_breakpoint) - if exprepr: - if gdb_repr == exprepr: - # gdb managed to print the value in spite of the corruption; - # this is good (see http://bugs.python.org/issue8330) - return - - # Match anything for the type name; 0xDEADBEEF could point to - # something arbitrary (see http://bugs.python.org/issue8330) - pattern = '<.* at remote 0x-?[0-9a-f]+>' - - m = re.match(pattern, gdb_repr) - if not m: - self.fail('Unexpected gdb representation: %r\n%s' % \ - (gdb_repr, gdb_output)) - - def test_NULL_ptr(self): - 'Ensure that a NULL PyObject* is handled gracefully' - gdb_repr, gdb_output = ( - self.get_gdb_repr('id(42)', - cmds_after_breakpoint=['set variable v=0', - 'backtrace']) - ) - - self.assertEqual(gdb_repr, '0x0') - - def test_NULL_ob_type(self): - 'Ensure that a PyObject* with NULL ob_type is handled gracefully' - self.assertSane('id(42)', - 'set v->ob_type=0') - - def test_corrupt_ob_type(self): - 'Ensure that a PyObject* with a corrupt ob_type is handled gracefully' - self.assertSane('id(42)', - 'set v->ob_type=0xDEADBEEF', - exprepr='42') - - def test_corrupt_tp_flags(self): - 'Ensure that a PyObject* with a type with corrupt tp_flags is handled' - self.assertSane('id(42)', - 'set v->ob_type->tp_flags=0x0', - exprepr='42') - - def test_corrupt_tp_name(self): - 'Ensure that a PyObject* with a type with corrupt tp_name is handled' - self.assertSane('id(42)', - 'set v->ob_type->tp_name=0xDEADBEEF', - exprepr='42') - - def test_builtins_help(self): - 'Ensure that the new-style class _Helper in site.py can be handled' - - if sys.flags.no_site: - self.skipTest("need site module, but -S option was used") - - # (this was the issue causing tracebacks in - # http://bugs.python.org/issue8032#msg100537 ) - gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) - - m = re.match(r'<_Helper\(\) at remote 0x-?[0-9a-f]+>', gdb_repr) - self.assertTrue(m, - msg='Unexpected rendering %r' % gdb_repr) - - def test_selfreferential_list(self): - '''Ensure that a reference loop involving a list doesn't lead proxyval - into an infinite loop:''' - gdb_repr, gdb_output = \ - self.get_gdb_repr("a = [3, 4, 5] ; a.append(a) ; id(a)") - self.assertEqual(gdb_repr, '[3, 4, 5, [...]]') - - gdb_repr, gdb_output = \ - self.get_gdb_repr("a = [3, 4, 5] ; b = [a] ; a.append(b) ; id(a)") - self.assertEqual(gdb_repr, '[3, 4, 5, [[...]]]') - - def test_selfreferential_dict(self): - '''Ensure that a reference loop involving a dict doesn't lead proxyval - into an infinite loop:''' - gdb_repr, gdb_output = \ - self.get_gdb_repr("a = {} ; b = {'bar':a} ; a['foo'] = b ; id(a)") - - self.assertEqual(gdb_repr, "{'foo': {'bar': {...}}}") - - def test_selfreferential_old_style_instance(self): - gdb_repr, gdb_output = \ - self.get_gdb_repr(''' -class Foo: - pass -foo = Foo() -foo.an_attr = foo -id(foo)''') - self.assertTrue(re.match(r'\) at remote 0x-?[0-9a-f]+>', - gdb_repr), - 'Unexpected gdb representation: %r\n%s' % \ - (gdb_repr, gdb_output)) - - def test_selfreferential_new_style_instance(self): - gdb_repr, gdb_output = \ - self.get_gdb_repr(''' -class Foo(object): - pass -foo = Foo() -foo.an_attr = foo -id(foo)''') - self.assertTrue(re.match(r'\) at remote 0x-?[0-9a-f]+>', - gdb_repr), - 'Unexpected gdb representation: %r\n%s' % \ - (gdb_repr, gdb_output)) - - gdb_repr, gdb_output = \ - self.get_gdb_repr(''' -class Foo(object): - pass -a = Foo() -b = Foo() -a.an_attr = b -b.an_attr = a -id(a)''') - self.assertTrue(re.match(r'\) at remote 0x-?[0-9a-f]+>\) at remote 0x-?[0-9a-f]+>', - gdb_repr), - 'Unexpected gdb representation: %r\n%s' % \ - (gdb_repr, gdb_output)) - - def test_truncation(self): - 'Verify that very long output is truncated' - gdb_repr, gdb_output = self.get_gdb_repr('id(list(range(1000)))') - self.assertEqual(gdb_repr, - "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, " - "14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, " - "27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, " - "40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, " - "53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, " - "66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, " - "79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, " - "92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, " - "104, 105, 106, 107, 108, 109, 110, 111, 112, 113, " - "114, 115, 116, 117, 118, 119, 120, 121, 122, 123, " - "124, 125, 126, 127, 128, 129, 130, 131, 132, 133, " - "134, 135, 136, 137, 138, 139, 140, 141, 142, 143, " - "144, 145, 146, 147, 148, 149, 150, 151, 152, 153, " - "154, 155, 156, 157, 158, 159, 160, 161, 162, 163, " - "164, 165, 166, 167, 168, 169, 170, 171, 172, 173, " - "174, 175, 176, 177, 178, 179, 180, 181, 182, 183, " - "184, 185, 186, 187, 188, 189, 190, 191, 192, 193, " - "194, 195, 196, 197, 198, 199, 200, 201, 202, 203, " - "204, 205, 206, 207, 208, 209, 210, 211, 212, 213, " - "214, 215, 216, 217, 218, 219, 220, 221, 222, 223, " - "224, 225, 226...(truncated)") - self.assertEqual(len(gdb_repr), - 1024 + len('...(truncated)')) - - def test_builtin_method(self): - gdb_repr, gdb_output = self.get_gdb_repr('import sys; id(sys.stdout.readlines)') - self.assertTrue(re.match(r'', - gdb_repr), - 'Unexpected gdb representation: %r\n%s' % \ - (gdb_repr, gdb_output)) - - def test_frames(self): - gdb_output = self.get_stack_trace(''' -import sys -def foo(a, b, c): - return sys._getframe(0) - -f = foo(3, 4, 5) -id(f)''', - breakpoint='builtin_id', - cmds_after_breakpoint=['print (PyFrameObject*)v'] - ) - self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file , line 4, in foo \(a=3.*', - gdb_output, - re.DOTALL), - 'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output)) - -@unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") -class PyListTests(DebuggerTests): - def assertListing(self, expected, actual): - self.assertEndsWith(actual, expected) - - def test_basic_command(self): - 'Verify that the "py-list" command works' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-list']) - - self.assertListing(' 5 \n' - ' 6 def bar(a, b, c):\n' - ' 7 baz(a, b, c)\n' - ' 8 \n' - ' 9 def baz(*args):\n' - ' >10 id(42)\n' - ' 11 \n' - ' 12 foo(1, 2, 3)\n', - bt) - - def test_one_abs_arg(self): - 'Verify the "py-list" command with one absolute argument' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-list 9']) - - self.assertListing(' 9 def baz(*args):\n' - ' >10 id(42)\n' - ' 11 \n' - ' 12 foo(1, 2, 3)\n', - bt) - - def test_two_abs_args(self): - 'Verify the "py-list" command with two absolute arguments' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-list 1,3']) - - self.assertListing(' 1 # Sample script for use by test_gdb.py\n' - ' 2 \n' - ' 3 def foo(a, b, c):\n', - bt) - -SAMPLE_WITH_C_CALL = """ - -from _testcapi import pyobject_vectorcall - -def foo(a, b, c): - bar(a, b, c) - -def bar(a, b, c): - pyobject_vectorcall(baz, (a, b, c), None) - -def baz(*args): - id(42) - -foo(1, 2, 3) - -""" - - -class StackNavigationTests(DebuggerTests): - @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - def test_pyup_command(self): - 'Verify that the "py-up" command works' - bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, - cmds_after_breakpoint=['py-up', 'py-up']) - self.assertMultilineMatches(bt, - r'''^.* -#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) -#[0-9]+ -$''') - - @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") - def test_down_at_bottom(self): - 'Verify handling of "py-down" at the bottom of the stack' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-down']) - self.assertEndsWith(bt, - 'Unable to find a newer python frame\n') - - @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") - def test_up_at_top(self): - 'Verify handling of "py-up" at the top of the stack' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up'] * 5) - self.assertEndsWith(bt, - 'Unable to find an older python frame\n') - - @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - def test_up_then_down(self): - 'Verify "py-up" followed by "py-down"' - bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, - cmds_after_breakpoint=['py-up', 'py-up', 'py-down']) - self.assertMultilineMatches(bt, - r'''^.* -#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) -#[0-9]+ -#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) -$''') - -class PyBtTests(DebuggerTests): - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - def test_bt(self): - 'Verify that the "py-bt" command works' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-bt']) - self.assertMultilineMatches(bt, - r'''^.* -Traceback \(most recent call first\): - - File ".*gdb_sample.py", line 10, in baz - id\(42\) - File ".*gdb_sample.py", line 7, in bar - baz\(a, b, c\) - File ".*gdb_sample.py", line 4, in foo - bar\(a=a, b=b, c=c\) - File ".*gdb_sample.py", line 12, in - foo\(1, 2, 3\) -''') - - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - def test_bt_full(self): - 'Verify that the "py-bt-full" command works' - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-bt-full']) - self.assertMultilineMatches(bt, - r'''^.* -#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) - baz\(a, b, c\) -#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\) - bar\(a=a, b=b, c=c\) -#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 12, in \(\) - foo\(1, 2, 3\) -''') - - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - @support.requires_resource('cpu') - def test_threads(self): - 'Verify that "py-bt" indicates threads that are waiting for the GIL' - cmd = ''' -from threading import Thread - -class TestThread(Thread): - # These threads would run forever, but we'll interrupt things with the - # debugger - def run(self): - i = 0 - while 1: - i += 1 - -t = {} -for i in range(4): - t[i] = TestThread() - t[i].start() - -# Trigger a breakpoint on the main thread -id(42) - -''' - # Verify with "py-bt": - gdb_output = self.get_stack_trace(cmd, - cmds_after_breakpoint=['thread apply all py-bt']) - self.assertIn('Waiting for the GIL', gdb_output) - - # Verify with "py-bt-full": - gdb_output = self.get_stack_trace(cmd, - cmds_after_breakpoint=['thread apply all py-bt-full']) - self.assertIn('Waiting for the GIL', gdb_output) - - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - # Some older versions of gdb will fail with - # "Cannot find new threads: generic error" - # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround - def test_gc(self): - 'Verify that "py-bt" indicates if a thread is garbage-collecting' - cmd = ('from gc import collect\n' - 'id(42)\n' - 'def foo():\n' - ' collect()\n' - 'def bar():\n' - ' foo()\n' - 'bar()\n') - # Verify with "py-bt": - gdb_output = self.get_stack_trace(cmd, - cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt'], - ) - self.assertIn('Garbage-collecting', gdb_output) - - # Verify with "py-bt-full": - gdb_output = self.get_stack_trace(cmd, - cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt-full'], - ) - self.assertIn('Garbage-collecting', gdb_output) - - - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - @support.requires_resource('cpu') - # Some older versions of gdb will fail with - # "Cannot find new threads: generic error" - # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround - # - # gdb will also generate many erroneous errors such as: - # Function "meth_varargs" not defined. - # This is because we are calling functions from an "external" module - # (_testcapimodule) rather than compiled-in functions. It seems difficult - # to suppress these. See also the comment in DebuggerTests.get_stack_trace - def test_pycfunction(self): - 'Verify that "py-bt" displays invocations of PyCFunction instances' - # bpo-46600: If the compiler inlines _null_to_none() in meth_varargs() - # (ex: clang -Og), _null_to_none() is the frame #1. Otherwise, - # meth_varargs() is the frame #1. - expected_frame = r'#(1|2)' - # Various optimizations multiply the code paths by which these are - # called, so test a variety of calling conventions. - for func_name, args in ( - ('meth_varargs', ''), - ('meth_varargs_keywords', ''), - ('meth_o', '[]'), - ('meth_noargs', ''), - ('meth_fastcall', ''), - ('meth_fastcall_keywords', ''), - ): - for obj in ( - '_testcapi', - '_testcapi.MethClass', - '_testcapi.MethClass()', - '_testcapi.MethStatic()', - - # XXX: bound methods don't yet give nice tracebacks - # '_testcapi.MethInstance()', - ): - with self.subTest(f'{obj}.{func_name}'): - cmd = textwrap.dedent(f''' - import _testcapi - def foo(): - {obj}.{func_name}({args}) - def bar(): - foo() - bar() - ''') - # Verify with "py-bt": - gdb_output = self.get_stack_trace( - cmd, - breakpoint=func_name, - cmds_after_breakpoint=['bt', 'py-bt'], - # bpo-45207: Ignore 'Function "meth_varargs" not - # defined.' message in stderr. - ignore_stderr=True, - ) - self.assertIn(f'\n.*") - -class PyLocalsTests(DebuggerTests): - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - def test_basic_command(self): - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up', 'py-locals']) - self.assertMultilineMatches(bt, - r".*\nargs = \(1, 2, 3\)\n.*") - - @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") - @unittest.skipIf(python_is_optimized(), - "Python was compiled with optimizations") - def test_locals_after_up(self): - bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up', 'py-up', 'py-locals']) - self.assertMultilineMatches(bt, - r'''^.* -Locals for foo -a = 1 -b = 2 -c = 3 -Locals for -.*$''') - - -def setUpModule(): - if support.verbose: - print("GDB version %s.%s:" % (gdb_major_version, gdb_minor_version)) - for line in gdb_version.splitlines(): - print(" " * 4 + line) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_gdb/__init__.py b/Lib/test/test_gdb/__init__.py new file mode 100644 index 00000000000000..0261f59adf54bd --- /dev/null +++ b/Lib/test/test_gdb/__init__.py @@ -0,0 +1,10 @@ +# Verify that gdb can pretty-print the various PyObject* types +# +# The code for testing gdb was adapted from similar work in Unladen Swallow's +# Lib/test/test_jit_gdb.py + +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/gdb_sample.py b/Lib/test/test_gdb/gdb_sample.py similarity index 75% rename from Lib/test/gdb_sample.py rename to Lib/test/test_gdb/gdb_sample.py index 4188f50136fb97..a7f23db73ea6e6 100644 --- a/Lib/test/gdb_sample.py +++ b/Lib/test/test_gdb/gdb_sample.py @@ -1,4 +1,4 @@ -# Sample script for use by test_gdb.py +# Sample script for use by test_gdb def foo(a, b, c): bar(a=a, b=b, c=c) diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py new file mode 100644 index 00000000000000..15cbcf169ab9e3 --- /dev/null +++ b/Lib/test/test_gdb/test_backtrace.py @@ -0,0 +1,134 @@ +import textwrap +import unittest +from test import support +from test.support import python_is_optimized + +from .util import setup_module, DebuggerTests, CET_PROTECTION + + +def setUpModule(): + setup_module() + + +class PyBtTests(DebuggerTests): + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_bt(self): + 'Verify that the "py-bt" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-bt']) + self.assertMultilineMatches(bt, + r'''^.* +Traceback \(most recent call first\): + + File ".*gdb_sample.py", line 10, in baz + id\(42\) + File ".*gdb_sample.py", line 7, in bar + baz\(a, b, c\) + File ".*gdb_sample.py", line 4, in foo + bar\(a=a, b=b, c=c\) + File ".*gdb_sample.py", line 12, in + foo\(1, 2, 3\) +''') + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_bt_full(self): + 'Verify that the "py-bt-full" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-bt-full']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) + baz\(a, b, c\) +#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\) + bar\(a=a, b=b, c=c\) +#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 12, in \(\) + foo\(1, 2, 3\) +''') + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + @support.requires_resource('cpu') + def test_threads(self): + 'Verify that "py-bt" indicates threads that are waiting for the GIL' + cmd = ''' +from threading import Thread + +class TestThread(Thread): + # These threads would run forever, but we'll interrupt things with the + # debugger + def run(self): + i = 0 + while 1: + i += 1 + +t = {} +for i in range(4): + t[i] = TestThread() + t[i].start() + +# Trigger a breakpoint on the main thread +id(42) + +''' + # Verify with "py-bt": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['thread apply all py-bt']) + self.assertIn('Waiting for the GIL', gdb_output) + + # Verify with "py-bt-full": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['thread apply all py-bt-full']) + self.assertIn('Waiting for the GIL', gdb_output) + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + # Some older versions of gdb will fail with + # "Cannot find new threads: generic error" + # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround + def test_gc(self): + 'Verify that "py-bt" indicates if a thread is garbage-collecting' + cmd = ('from gc import collect\n' + 'id(42)\n' + 'def foo():\n' + ' collect()\n' + 'def bar():\n' + ' foo()\n' + 'bar()\n') + # Verify with "py-bt": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt'], + ) + self.assertIn('Garbage-collecting', gdb_output) + + # Verify with "py-bt-full": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt-full'], + ) + self.assertIn('Garbage-collecting', gdb_output) + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_wrapper_call(self): + cmd = textwrap.dedent(''' + class MyList(list): + def __init__(self): + super(*[]).__init__() # wrapper_call() + + id("first break point") + l = MyList() + ''') + cmds_after_breakpoint = ['break wrapper_call', 'continue'] + if CET_PROTECTION: + # bpo-32962: same case as in get_stack_trace(): + # we need an additional 'next' command in order to read + # arguments of the innermost function of the call stack. + cmds_after_breakpoint.append('next') + cmds_after_breakpoint.append('py-bt') + + # Verify with "py-bt": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=cmds_after_breakpoint) + self.assertRegex(gdb_output, + r"10 id(42)\n' + ' 11 \n' + ' 12 foo(1, 2, 3)\n', + bt) + + def test_one_abs_arg(self): + 'Verify the "py-list" command with one absolute argument' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-list 9']) + + self.assertListing(' 9 def baz(*args):\n' + ' >10 id(42)\n' + ' 11 \n' + ' 12 foo(1, 2, 3)\n', + bt) + + def test_two_abs_args(self): + 'Verify the "py-list" command with two absolute arguments' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-list 1,3']) + + self.assertListing(' 1 # Sample script for use by test_gdb\n' + ' 2 \n' + ' 3 def foo(a, b, c):\n', + bt) + +SAMPLE_WITH_C_CALL = """ + +from _testcapi import pyobject_vectorcall + +def foo(a, b, c): + bar(a, b, c) + +def bar(a, b, c): + pyobject_vectorcall(baz, (a, b, c), None) + +def baz(*args): + id(42) + +foo(1, 2, 3) + +""" + + +class StackNavigationTests(DebuggerTests): + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_pyup_command(self): + 'Verify that the "py-up" command works' + bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, + cmds_after_breakpoint=['py-up', 'py-up']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) +#[0-9]+ +$''') + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_down_at_bottom(self): + 'Verify handling of "py-down" at the bottom of the stack' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-down']) + self.assertEndsWith(bt, + 'Unable to find a newer python frame\n') + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_up_at_top(self): + 'Verify handling of "py-up" at the top of the stack' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up'] * 5) + self.assertEndsWith(bt, + 'Unable to find an older python frame\n') + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_up_then_down(self): + 'Verify "py-up" followed by "py-down"' + bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, + cmds_after_breakpoint=['py-up', 'py-up', 'py-down']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) +#[0-9]+ +#[0-9]+ Frame 0x-?[0-9a-f]+, for file , line 12, in baz \(args=\(1, 2, 3\)\) +$''') + +class PyPrintTests(DebuggerTests): + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_basic_command(self): + 'Verify that the "py-print" command works' + bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, + cmds_after_breakpoint=['py-up', 'py-print args']) + self.assertMultilineMatches(bt, + r".*\nlocal 'args' = \(1, 2, 3\)\n.*") + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_print_after_up(self): + bt = self.get_stack_trace(source=SAMPLE_WITH_C_CALL, + cmds_after_breakpoint=['py-up', 'py-up', 'py-print c', 'py-print b', 'py-print a']) + self.assertMultilineMatches(bt, + r".*\nlocal 'c' = 3\nlocal 'b' = 2\nlocal 'a' = 1\n.*") + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_printing_global(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-print __name__']) + self.assertMultilineMatches(bt, + r".*\nglobal '__name__' = '__main__'\n.*") + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_printing_builtin(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-print len']) + self.assertMultilineMatches(bt, + r".*\nbuiltin 'len' = \n.*") + +class PyLocalsTests(DebuggerTests): + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_basic_command(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-locals']) + self.assertMultilineMatches(bt, + r".*\nargs = \(1, 2, 3\)\n.*") + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_locals_after_up(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-up', 'py-locals']) + self.assertMultilineMatches(bt, + r'''^.* +Locals for foo +a = 1 +b = 2 +c = 3 +Locals for +.*$''') diff --git a/Lib/test/test_gdb/test_pretty_print.py b/Lib/test/test_gdb/test_pretty_print.py new file mode 100644 index 00000000000000..e31dc66f29684a --- /dev/null +++ b/Lib/test/test_gdb/test_pretty_print.py @@ -0,0 +1,400 @@ +import re +import sys +from test import support + +from .util import ( + BREAKPOINT_FN, gdb_major_version, gdb_minor_version, + run_gdb, setup_module, DebuggerTests) + + +def setUpModule(): + setup_module() + + +class PrettyPrintTests(DebuggerTests): + def test_getting_backtrace(self): + gdb_output = self.get_stack_trace('id(42)') + self.assertTrue(BREAKPOINT_FN in gdb_output) + + def assertGdbRepr(self, val, exp_repr=None): + # Ensure that gdb's rendering of the value in a debugged process + # matches repr(value) in this process: + gdb_repr, gdb_output = self.get_gdb_repr('id(' + ascii(val) + ')') + if not exp_repr: + exp_repr = repr(val) + self.assertEqual(gdb_repr, exp_repr, + ('%r did not equal expected %r; full output was:\n%s' + % (gdb_repr, exp_repr, gdb_output))) + + @support.requires_resource('cpu') + def test_int(self): + 'Verify the pretty-printing of various int values' + self.assertGdbRepr(42) + self.assertGdbRepr(0) + self.assertGdbRepr(-7) + self.assertGdbRepr(1000000000000) + self.assertGdbRepr(-1000000000000000) + + def test_singletons(self): + 'Verify the pretty-printing of True, False and None' + self.assertGdbRepr(True) + self.assertGdbRepr(False) + self.assertGdbRepr(None) + + def test_dicts(self): + 'Verify the pretty-printing of dictionaries' + self.assertGdbRepr({}) + self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}") + # Python preserves insertion order since 3.6 + self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'foo': 'bar', 'douglas': 42}") + + def test_lists(self): + 'Verify the pretty-printing of lists' + self.assertGdbRepr([]) + self.assertGdbRepr(list(range(5))) + + @support.requires_resource('cpu') + def test_bytes(self): + 'Verify the pretty-printing of bytes' + self.assertGdbRepr(b'') + self.assertGdbRepr(b'And now for something hopefully the same') + self.assertGdbRepr(b'string with embedded NUL here \0 and then some more text') + self.assertGdbRepr(b'this is a tab:\t' + b' this is a slash-N:\n' + b' this is a slash-R:\r' + ) + + self.assertGdbRepr(b'this is byte 255:\xff and byte 128:\x80') + + self.assertGdbRepr(bytes([b for b in range(255)])) + + @support.requires_resource('cpu') + def test_strings(self): + 'Verify the pretty-printing of unicode strings' + # We cannot simply call locale.getpreferredencoding() here, + # as GDB might have been linked against a different version + # of Python with a different encoding and coercion policy + # with respect to PEP 538 and PEP 540. + out, err = run_gdb( + '--eval-command', + 'python import locale; print(locale.getpreferredencoding())') + + encoding = out.rstrip() + if err or not encoding: + raise RuntimeError( + f'unable to determine the preferred encoding ' + f'of embedded Python in GDB: {err}') + + def check_repr(text): + try: + text.encode(encoding) + except UnicodeEncodeError: + self.assertGdbRepr(text, ascii(text)) + else: + self.assertGdbRepr(text) + + self.assertGdbRepr('') + self.assertGdbRepr('And now for something hopefully the same') + self.assertGdbRepr('string with embedded NUL here \0 and then some more text') + + # Test printing a single character: + # U+2620 SKULL AND CROSSBONES + check_repr('\u2620') + + # Test printing a Japanese unicode string + # (I believe this reads "mojibake", using 3 characters from the CJK + # Unified Ideographs area, followed by U+3051 HIRAGANA LETTER KE) + check_repr('\u6587\u5b57\u5316\u3051') + + # Test a character outside the BMP: + # U+1D121 MUSICAL SYMBOL C CLEF + # This is: + # UTF-8: 0xF0 0x9D 0x84 0xA1 + # UTF-16: 0xD834 0xDD21 + check_repr(chr(0x1D121)) + + def test_tuples(self): + 'Verify the pretty-printing of tuples' + self.assertGdbRepr(tuple(), '()') + self.assertGdbRepr((1,), '(1,)') + self.assertGdbRepr(('foo', 'bar', 'baz')) + + @support.requires_resource('cpu') + def test_sets(self): + 'Verify the pretty-printing of sets' + if (gdb_major_version, gdb_minor_version) < (7, 3): + self.skipTest("pretty-printing of sets needs gdb 7.3 or later") + self.assertGdbRepr(set(), "set()") + self.assertGdbRepr(set(['a']), "{'a'}") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") + self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") + + # Ensure that we handle sets containing the "dummy" key value, + # which happens on deletion: + gdb_repr, gdb_output = self.get_gdb_repr('''s = set(['a','b']) +s.remove('a') +id(s)''') + self.assertEqual(gdb_repr, "{'b'}") + + @support.requires_resource('cpu') + def test_frozensets(self): + 'Verify the pretty-printing of frozensets' + if (gdb_major_version, gdb_minor_version) < (7, 3): + self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") + self.assertGdbRepr(frozenset(), "frozenset()") + self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") + self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") + + def test_exceptions(self): + # Test a RuntimeError + gdb_repr, gdb_output = self.get_gdb_repr(''' +try: + raise RuntimeError("I am an error") +except RuntimeError as e: + id(e) +''') + self.assertEqual(gdb_repr, + "RuntimeError('I am an error',)") + + + # Test division by zero: + gdb_repr, gdb_output = self.get_gdb_repr(''' +try: + a = 1 / 0 +except ZeroDivisionError as e: + id(e) +''') + self.assertEqual(gdb_repr, + "ZeroDivisionError('division by zero',)") + + def test_modern_class(self): + 'Verify the pretty-printing of new-style class instances' + gdb_repr, gdb_output = self.get_gdb_repr(''' +class Foo: + pass +foo = Foo() +foo.an_int = 42 +id(foo)''') + m = re.match(r'', gdb_repr) + self.assertTrue(m, + msg='Unexpected new-style class rendering %r' % gdb_repr) + + def test_subclassing_list(self): + 'Verify the pretty-printing of an instance of a list subclass' + gdb_repr, gdb_output = self.get_gdb_repr(''' +class Foo(list): + pass +foo = Foo() +foo += [1, 2, 3] +foo.an_int = 42 +id(foo)''') + m = re.match(r'', gdb_repr) + + self.assertTrue(m, + msg='Unexpected new-style class rendering %r' % gdb_repr) + + def test_subclassing_tuple(self): + 'Verify the pretty-printing of an instance of a tuple subclass' + # This should exercise the negative tp_dictoffset code in the + # new-style class support + gdb_repr, gdb_output = self.get_gdb_repr(''' +class Foo(tuple): + pass +foo = Foo((1, 2, 3)) +foo.an_int = 42 +id(foo)''') + m = re.match(r'', gdb_repr) + + self.assertTrue(m, + msg='Unexpected new-style class rendering %r' % gdb_repr) + + def assertSane(self, source, corruption, exprepr=None): + '''Run Python under gdb, corrupting variables in the inferior process + immediately before taking a backtrace. + + Verify that the variable's representation is the expected failsafe + representation''' + if corruption: + cmds_after_breakpoint=[corruption, 'backtrace'] + else: + cmds_after_breakpoint=['backtrace'] + + gdb_repr, gdb_output = \ + self.get_gdb_repr(source, + cmds_after_breakpoint=cmds_after_breakpoint) + if exprepr: + if gdb_repr == exprepr: + # gdb managed to print the value in spite of the corruption; + # this is good (see http://bugs.python.org/issue8330) + return + + # Match anything for the type name; 0xDEADBEEF could point to + # something arbitrary (see http://bugs.python.org/issue8330) + pattern = '<.* at remote 0x-?[0-9a-f]+>' + + m = re.match(pattern, gdb_repr) + if not m: + self.fail('Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_NULL_ptr(self): + 'Ensure that a NULL PyObject* is handled gracefully' + gdb_repr, gdb_output = ( + self.get_gdb_repr('id(42)', + cmds_after_breakpoint=['set variable v=0', + 'backtrace']) + ) + + self.assertEqual(gdb_repr, '0x0') + + def test_NULL_ob_type(self): + 'Ensure that a PyObject* with NULL ob_type is handled gracefully' + self.assertSane('id(42)', + 'set v->ob_type=0') + + def test_corrupt_ob_type(self): + 'Ensure that a PyObject* with a corrupt ob_type is handled gracefully' + self.assertSane('id(42)', + 'set v->ob_type=0xDEADBEEF', + exprepr='42') + + def test_corrupt_tp_flags(self): + 'Ensure that a PyObject* with a type with corrupt tp_flags is handled' + self.assertSane('id(42)', + 'set v->ob_type->tp_flags=0x0', + exprepr='42') + + def test_corrupt_tp_name(self): + 'Ensure that a PyObject* with a type with corrupt tp_name is handled' + self.assertSane('id(42)', + 'set v->ob_type->tp_name=0xDEADBEEF', + exprepr='42') + + def test_builtins_help(self): + 'Ensure that the new-style class _Helper in site.py can be handled' + + if sys.flags.no_site: + self.skipTest("need site module, but -S option was used") + + # (this was the issue causing tracebacks in + # http://bugs.python.org/issue8032#msg100537 ) + gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) + + m = re.match(r'<_Helper\(\) at remote 0x-?[0-9a-f]+>', gdb_repr) + self.assertTrue(m, + msg='Unexpected rendering %r' % gdb_repr) + + def test_selfreferential_list(self): + '''Ensure that a reference loop involving a list doesn't lead proxyval + into an infinite loop:''' + gdb_repr, gdb_output = \ + self.get_gdb_repr("a = [3, 4, 5] ; a.append(a) ; id(a)") + self.assertEqual(gdb_repr, '[3, 4, 5, [...]]') + + gdb_repr, gdb_output = \ + self.get_gdb_repr("a = [3, 4, 5] ; b = [a] ; a.append(b) ; id(a)") + self.assertEqual(gdb_repr, '[3, 4, 5, [[...]]]') + + def test_selfreferential_dict(self): + '''Ensure that a reference loop involving a dict doesn't lead proxyval + into an infinite loop:''' + gdb_repr, gdb_output = \ + self.get_gdb_repr("a = {} ; b = {'bar':a} ; a['foo'] = b ; id(a)") + + self.assertEqual(gdb_repr, "{'foo': {'bar': {...}}}") + + def test_selfreferential_old_style_instance(self): + gdb_repr, gdb_output = \ + self.get_gdb_repr(''' +class Foo: + pass +foo = Foo() +foo.an_attr = foo +id(foo)''') + self.assertTrue(re.match(r'\) at remote 0x-?[0-9a-f]+>', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_selfreferential_new_style_instance(self): + gdb_repr, gdb_output = \ + self.get_gdb_repr(''' +class Foo(object): + pass +foo = Foo() +foo.an_attr = foo +id(foo)''') + self.assertTrue(re.match(r'\) at remote 0x-?[0-9a-f]+>', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + gdb_repr, gdb_output = \ + self.get_gdb_repr(''' +class Foo(object): + pass +a = Foo() +b = Foo() +a.an_attr = b +b.an_attr = a +id(a)''') + self.assertTrue(re.match(r'\) at remote 0x-?[0-9a-f]+>\) at remote 0x-?[0-9a-f]+>', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_truncation(self): + 'Verify that very long output is truncated' + gdb_repr, gdb_output = self.get_gdb_repr('id(list(range(1000)))') + self.assertEqual(gdb_repr, + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, " + "14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, " + "27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, " + "40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, " + "53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, " + "66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, " + "79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, " + "92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, " + "104, 105, 106, 107, 108, 109, 110, 111, 112, 113, " + "114, 115, 116, 117, 118, 119, 120, 121, 122, 123, " + "124, 125, 126, 127, 128, 129, 130, 131, 132, 133, " + "134, 135, 136, 137, 138, 139, 140, 141, 142, 143, " + "144, 145, 146, 147, 148, 149, 150, 151, 152, 153, " + "154, 155, 156, 157, 158, 159, 160, 161, 162, 163, " + "164, 165, 166, 167, 168, 169, 170, 171, 172, 173, " + "174, 175, 176, 177, 178, 179, 180, 181, 182, 183, " + "184, 185, 186, 187, 188, 189, 190, 191, 192, 193, " + "194, 195, 196, 197, 198, 199, 200, 201, 202, 203, " + "204, 205, 206, 207, 208, 209, 210, 211, 212, 213, " + "214, 215, 216, 217, 218, 219, 220, 221, 222, 223, " + "224, 225, 226...(truncated)") + self.assertEqual(len(gdb_repr), + 1024 + len('...(truncated)')) + + def test_builtin_method(self): + gdb_repr, gdb_output = self.get_gdb_repr('import sys; id(sys.stdout.readlines)') + self.assertTrue(re.match(r'', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_frames(self): + gdb_output = self.get_stack_trace(''' +import sys +def foo(a, b, c): + return sys._getframe(0) + +f = foo(3, 4, 5) +id(f)''', + breakpoint='builtin_id', + cmds_after_breakpoint=['print (PyFrameObject*)v'] + ) + self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file , line 4, in foo \(a=3.*', + gdb_output, + re.DOTALL), + 'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output)) diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py new file mode 100644 index 00000000000000..30beb4e14285c7 --- /dev/null +++ b/Lib/test/test_gdb/util.py @@ -0,0 +1,304 @@ +import os +import re +import subprocess +import sys +import sysconfig +import unittest +from test import support + + +MS_WINDOWS = (sys.platform == 'win32') +if MS_WINDOWS: + raise unittest.SkipTest("test_gdb doesn't work on Windows") + + +def get_gdb_version(): + try: + cmd = ["gdb", "-nx", "--version"] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + with proc: + version, stderr = proc.communicate() + + if proc.returncode: + raise Exception(f"Command {' '.join(cmd)!r} failed " + f"with exit code {proc.returncode}: " + f"stdout={version!r} stderr={stderr!r}") + except OSError: + # This is what "no gdb" looks like. There may, however, be other + # errors that manifest this way too. + raise unittest.SkipTest("Couldn't find gdb on the path") + + # Regex to parse: + # 'GNU gdb (GDB; SUSE Linux Enterprise 12) 7.7\n' -> 7.7 + # 'GNU gdb (GDB) Fedora 7.9.1-17.fc22\n' -> 7.9 + # 'GNU gdb 6.1.1 [FreeBSD]\n' -> 6.1 + # 'GNU gdb (GDB) Fedora (7.5.1-37.fc18)\n' -> 7.5 + # 'HP gdb 6.7 for HP Itanium (32 or 64 bit) and target HP-UX 11iv2 and 11iv3.\n' -> 6.7 + match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", version) + if match is None: + raise Exception("unable to parse GDB version: %r" % version) + return (version, int(match.group(1)), int(match.group(2))) + +gdb_version, gdb_major_version, gdb_minor_version = get_gdb_version() +if gdb_major_version < 7: + raise unittest.SkipTest("gdb versions before 7.0 didn't support python " + "embedding. Saw %s.%s:\n%s" + % (gdb_major_version, gdb_minor_version, + gdb_version)) + +if not sysconfig.is_python_build(): + raise unittest.SkipTest("test_gdb only works on source builds at the moment.") + +if ((sysconfig.get_config_var('PGO_PROF_USE_FLAG') or 'xxx') in + (sysconfig.get_config_var('PY_CORE_CFLAGS') or '')): + raise unittest.SkipTest("test_gdb is not reliable on PGO builds") + +# Location of custom hooks file in a repository checkout. +checkout_hook_path = os.path.join(os.path.dirname(sys.executable), + 'python-gdb.py') + +PYTHONHASHSEED = '123' + + +def cet_protection(): + cflags = sysconfig.get_config_var('CFLAGS') + if not cflags: + return False + flags = cflags.split() + # True if "-mcet -fcf-protection" options are found, but false + # if "-fcf-protection=none" or "-fcf-protection=return" is found. + return (('-mcet' in flags) + and any((flag.startswith('-fcf-protection') + and not flag.endswith(("=none", "=return"))) + for flag in flags)) + +# Control-flow enforcement technology +CET_PROTECTION = cet_protection() + + +def run_gdb(*args, **env_vars): + """Runs gdb in --batch mode with the additional arguments given by *args. + + Returns its (stdout, stderr) decoded from utf-8 using the replace handler. + """ + if env_vars: + env = os.environ.copy() + env.update(env_vars) + else: + env = None + # -nx: Do not execute commands from any .gdbinit initialization files + # (issue #22188) + base_cmd = ('gdb', '--batch', '-nx') + if (gdb_major_version, gdb_minor_version) >= (7, 4): + base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path) + proc = subprocess.Popen(base_cmd + args, + # Redirect stdin to prevent GDB from messing with + # the terminal settings + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + with proc: + out, err = proc.communicate() + return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') + +# Verify that "gdb" was built with the embedded python support enabled: +gdbpy_version, _ = run_gdb("--eval-command=python import sys; print(sys.version_info)") +if not gdbpy_version: + raise unittest.SkipTest("gdb not built with embedded python support") + +if "major=2" in gdbpy_version: + raise unittest.SkipTest("gdb built with Python 2") + +# Verify that "gdb" can load our custom hooks, as OS security settings may +# disallow this without a customized .gdbinit. +_, gdbpy_errors = run_gdb('--args', sys.executable) +if "auto-loading has been declined" in gdbpy_errors: + msg = "gdb security settings prevent use of custom hooks: " + raise unittest.SkipTest(msg + gdbpy_errors.rstrip()) + +BREAKPOINT_FN='builtin_id' + + +def setup_module(): + if support.verbose: + print("GDB version %s.%s:" % (gdb_major_version, gdb_minor_version)) + for line in gdb_version.splitlines(): + print(" " * 4 + line) + + +@unittest.skipIf(support.PGO, "not useful for PGO") +class DebuggerTests(unittest.TestCase): + + """Test that the debugger can debug Python.""" + + def get_stack_trace(self, source=None, script=None, + breakpoint=BREAKPOINT_FN, + cmds_after_breakpoint=None, + import_site=False, + ignore_stderr=False): + ''' + Run 'python -c SOURCE' under gdb with a breakpoint. + + Support injecting commands after the breakpoint is reached + + Returns the stdout from gdb + + cmds_after_breakpoint: if provided, a list of strings: gdb commands + ''' + # We use "set breakpoint pending yes" to avoid blocking with a: + # Function "foo" not defined. + # Make breakpoint pending on future shared library load? (y or [n]) + # error, which typically happens python is dynamically linked (the + # breakpoints of interest are to be found in the shared library) + # When this happens, we still get: + # Function "textiowrapper_write" not defined. + # emitted to stderr each time, alas. + + # Initially I had "--eval-command=continue" here, but removed it to + # avoid repeated print breakpoints when traversing hierarchical data + # structures + + # Generate a list of commands in gdb's language: + commands = ['set breakpoint pending yes', + 'break %s' % breakpoint, + + # The tests assume that the first frame of printed + # backtrace will not contain program counter, + # that is however not guaranteed by gdb + # therefore we need to use 'set print address off' to + # make sure the counter is not there. For example: + # #0 in PyObject_Print ... + # is assumed, but sometimes this can be e.g. + # #0 0x00003fffb7dd1798 in PyObject_Print ... + 'set print address off', + + 'run'] + + # GDB as of 7.4 onwards can distinguish between the + # value of a variable at entry vs current value: + # http://sourceware.org/gdb/onlinedocs/gdb/Variables.html + # which leads to the selftests failing with errors like this: + # AssertionError: 'v@entry=()' != '()' + # Disable this: + if (gdb_major_version, gdb_minor_version) >= (7, 4): + commands += ['set print entry-values no'] + + if cmds_after_breakpoint: + if CET_PROTECTION: + # bpo-32962: When Python is compiled with -mcet + # -fcf-protection, function arguments are unusable before + # running the first instruction of the function entry point. + # The 'next' command makes the required first step. + commands += ['next'] + commands += cmds_after_breakpoint + else: + commands += ['backtrace'] + + # print commands + + # Use "commands" to generate the arguments with which to invoke "gdb": + args = ['--eval-command=%s' % cmd for cmd in commands] + args += ["--args", + sys.executable] + args.extend(subprocess._args_from_interpreter_flags()) + + if not import_site: + # -S suppresses the default 'import site' + args += ["-S"] + + if source: + args += ["-c", source] + elif script: + args += [script] + + # Use "args" to invoke gdb, capturing stdout, stderr: + out, err = run_gdb(*args, PYTHONHASHSEED=PYTHONHASHSEED) + + if not ignore_stderr: + for line in err.splitlines(): + print(line, file=sys.stderr) + + # bpo-34007: Sometimes some versions of the shared libraries that + # are part of the traceback are compiled in optimised mode and the + # Program Counter (PC) is not present, not allowing gdb to walk the + # frames back. When this happens, the Python bindings of gdb raise + # an exception, making the test impossible to succeed. + if "PC not saved" in err: + raise unittest.SkipTest("gdb cannot walk the frame object" + " because the Program Counter is" + " not present") + + # bpo-40019: Skip the test if gdb failed to read debug information + # because the Python binary is optimized. + for pattern in ( + '(frame information optimized out)', + 'Unable to read information on python frame', + # gh-91960: On Python built with "clang -Og", gdb gets + # "frame=" for _PyEval_EvalFrameDefault() parameter + '(unable to read python frame information)', + # gh-104736: On Python built with "clang -Og" on ppc64le, + # "py-bt" displays a truncated or not traceback, but "where" + # logs this error message: + 'Backtrace stopped: frame did not save the PC', + # gh-104736: When "bt" command displays something like: + # "#1 0x0000000000000000 in ?? ()", the traceback is likely + # truncated or wrong. + ' ?? ()', + ): + if pattern in out: + raise unittest.SkipTest(f"{pattern!r} found in gdb output") + + return out + + def get_gdb_repr(self, source, + cmds_after_breakpoint=None, + import_site=False): + # Given an input python source representation of data, + # run "python -c'id(DATA)'" under gdb with a breakpoint on + # builtin_id and scrape out gdb's representation of the "op" + # parameter, and verify that the gdb displays the same string + # + # Verify that the gdb displays the expected string + # + # For a nested structure, the first time we hit the breakpoint will + # give us the top-level structure + + # NOTE: avoid decoding too much of the traceback as some + # undecodable characters may lurk there in optimized mode + # (issue #19743). + cmds_after_breakpoint = cmds_after_breakpoint or ["backtrace 1"] + gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN, + cmds_after_breakpoint=cmds_after_breakpoint, + import_site=import_site) + # gdb can insert additional '\n' and space characters in various places + # in its output, depending on the width of the terminal it's connected + # to (using its "wrap_here" function) + m = re.search( + # Match '#0 builtin_id(self=..., v=...)' + r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)' + # Match ' at Python/bltinmodule.c'. + # bpo-38239: builtin_id() is defined in Python/bltinmodule.c, + # but accept any "Directory\file.c" to support Link Time + # Optimization (LTO). + r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c', + gdb_output, re.DOTALL) + if not m: + self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) + return m.group(1), gdb_output + + def assertEndsWith(self, actual, exp_end): + '''Ensure that the given "actual" string ends with "exp_end"''' + self.assertTrue(actual.endswith(exp_end), + msg='%r did not end with %r' % (actual, exp_end)) + + def assertMultilineMatches(self, actual, pattern): + m = re.match(pattern, actual, re.DOTALL) + if not m: + self.fail(msg='%r did not match %r' % (actual, pattern)) + + def get_sample_script(self): + return os.path.join(os.path.dirname(__file__), 'gdb_sample.py') diff --git a/Makefile.pre.in b/Makefile.pre.in index ccbacfc8571851..f7b52bdab316f1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2169,6 +2169,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_email \ test/test_email/data \ test/test_future_stmt \ + test/test_gdb \ test/test_import \ test/test_import/data \ test/test_import/data/circular_imports \ diff --git a/Misc/NEWS.d/next/Tests/2023-09-28-12-25-19.gh-issue-109972.GYnwIP.rst b/Misc/NEWS.d/next/Tests/2023-09-28-12-25-19.gh-issue-109972.GYnwIP.rst new file mode 100644 index 00000000000000..7b6007678388b1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-28-12-25-19.gh-issue-109972.GYnwIP.rst @@ -0,0 +1,2 @@ +Split test_gdb.py file into a test_gdb package made of multiple tests, so tests +can now be run in parallel. Patch by Victor Stinner. From 0baf72696e79191241a2d5cfdfd7e6135115f7b2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 28 Sep 2023 14:51:33 +0300 Subject: [PATCH 047/124] gh-109961: Docs: Fix incorrect rendering of `__replace__` in `copy.rst` (#109968) --- Doc/library/copy.rst | 33 +++++++++++++++++++++------------ Doc/tools/.nitignore | 1 - 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index cc4ca034d07a00..811b2c51dd3d52 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -88,13 +88,22 @@ pickle functions from the :mod:`copyreg` module. single: __deepcopy__() (copy protocol) In order for a class to define its own copy implementation, it can define -special methods :meth:`__copy__` and :meth:`__deepcopy__`. The former is called -to implement the shallow copy operation; no additional arguments are passed. -The latter is called to implement the deep copy operation; it is passed one -argument, the ``memo`` dictionary. If the :meth:`__deepcopy__` implementation needs -to make a deep copy of a component, it should call the :func:`deepcopy` function -with the component as first argument and the memo dictionary as second argument. -The memo dictionary should be treated as an opaque object. +special methods :meth:`~object.__copy__` and :meth:`~object.__deepcopy__`. + +.. method:: object.__copy__(self) + :noindexentry: + + Called to implement the shallow copy operation; + no additional arguments are passed. + +.. method:: object.__deepcopy__(self, memo) + :noindexentry: + + Called to implement the deep copy operation; it is passed one + argument, the *memo* dictionary. If the ``__deepcopy__`` implementation needs + to make a deep copy of a component, it should call the :func:`deepcopy` function + with the component as first argument and the *memo* dictionary as second argument. + The *memo* dictionary should be treated as an opaque object. .. index:: @@ -102,13 +111,13 @@ The memo dictionary should be treated as an opaque object. Function :func:`replace` is more limited than :func:`copy` and :func:`deepcopy`, and only supports named tuples created by :func:`~collections.namedtuple`, -:mod:`dataclasses`, and other classes which define method :meth:`!__replace__`. +:mod:`dataclasses`, and other classes which define method :meth:`~object.__replace__`. - .. method:: __replace__(self, /, **changes) - :noindex: +.. method:: object.__replace__(self, /, **changes) + :noindexentry: -:meth:`!__replace__` should create a new object of the same type, -replacing fields with values from *changes*. + This method should create a new object of the same type, + replacing fields with values from *changes*. .. seealso:: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index a5fb940b150ee8..2478f305884162 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -48,7 +48,6 @@ Doc/library/collections.rst Doc/library/concurrent.futures.rst Doc/library/configparser.rst Doc/library/contextlib.rst -Doc/library/copy.rst Doc/library/csv.rst Doc/library/datetime.rst Doc/library/dbm.rst From 9be283e5e15d5d5685b78a38eb132501f7f3febb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Sep 2023 15:21:15 +0200 Subject: [PATCH 048/124] gh-109594: Fix concurrent.futures test_timeout() (#110018) Fix test_timeout() of test_concurrent_futures.test_wait. Remove the future which may or may not complete depending if it takes longer than the timeout ot not. Keep the second future which does not complete before wait(). Make also the test faster: 0.5 second instead of 6 seconds, so remove @support.requires_resource('walltime') decorator. --- Lib/test/test_concurrent_futures/test_wait.py | 17 +++++++++-------- ...23-09-28-14-47-14.gh-issue-109594.DB5KPP.rst | 4 ++++ 2 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-28-14-47-14.gh-issue-109594.DB5KPP.rst diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index 3f64ca173c02f6..ff486202092c81 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -112,24 +112,25 @@ def test_all_completed(self): future2]), finished) self.assertEqual(set(), pending) - @support.requires_resource('walltime') def test_timeout(self): - future1 = self.executor.submit(mul, 6, 7) - future2 = self.executor.submit(time.sleep, 6) + short_timeout = 0.050 + long_timeout = short_timeout * 10 + + future = self.executor.submit(time.sleep, long_timeout) finished, pending = futures.wait( [CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, - future1, future2], - timeout=5, + future], + timeout=short_timeout, return_when=futures.ALL_COMPLETED) self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, - SUCCESSFUL_FUTURE, - future1]), finished) - self.assertEqual(set([future2]), pending) + SUCCESSFUL_FUTURE]), + finished) + self.assertEqual(set([future]), pending) class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): diff --git a/Misc/NEWS.d/next/Tests/2023-09-28-14-47-14.gh-issue-109594.DB5KPP.rst b/Misc/NEWS.d/next/Tests/2023-09-28-14-47-14.gh-issue-109594.DB5KPP.rst new file mode 100644 index 00000000000000..5a4ae2b0837df6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-28-14-47-14.gh-issue-109594.DB5KPP.rst @@ -0,0 +1,4 @@ +Fix test_timeout() of test_concurrent_futures.test_wait. Remove the future +which may or may not complete depending if it takes longer than the timeout +ot not. Keep the second future which does not complete before wait() +timeout. Patch by Victor Stinner. From 3814bc17230df4cd3bc4d8e2ce0ad36470fba269 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 28 Sep 2023 17:31:32 +0300 Subject: [PATCH 049/124] gh-110020: Fix unused variable warnings in bytecodes.c (GH-110023) --- Python/bytecodes.c | 9 +++------ Python/executor_cases.c.h | 9 +++------ Python/generated_cases.c.h | 9 +++------ 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0f89779fb9245f..f7681bd234a43f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2023,8 +2023,7 @@ dummy_func( } op(_GUARD_DORV_VALUES, (owner -- owner)) { - PyTypeObject *tp = Py_TYPE(owner); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } @@ -2789,8 +2788,7 @@ dummy_func( } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), @@ -2823,8 +2821,7 @@ dummy_func( op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(owner_cls->tp_dictoffset == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 981db6973f281a..55a03c9a23a572 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1736,8 +1736,7 @@ case _GUARD_DORV_VALUES: { PyObject *owner; owner = stack_pointer[-1]; - PyTypeObject *tp = Py_TYPE(owner); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); break; @@ -2299,8 +2298,7 @@ case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { PyObject *owner; owner = stack_pointer[-1]; - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), @@ -2345,8 +2343,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)operand; assert(oparg & 1); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(owner_cls->tp_dictoffset == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 17df44019a6581..2701d416648a20 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2631,8 +2631,7 @@ } // _GUARD_DORV_VALUES { - PyTypeObject *tp = Py_TYPE(owner); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } @@ -3590,8 +3589,7 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), @@ -3639,8 +3637,7 @@ { PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); - PyTypeObject *owner_cls = Py_TYPE(owner); - assert(owner_cls->tp_dictoffset == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); From 7df8b16d28d2418161cef49814b6aca9fb70788d Mon Sep 17 00:00:00 2001 From: Amin Alaee Date: Thu, 28 Sep 2023 17:17:30 +0200 Subject: [PATCH 050/124] gh-109782: Ensure `os.path.isdir` has the same signature on all platforms (GH-109790) --- ...023-09-24-16-43-33.gh-issue-109782.gMC_7z.rst | 2 ++ Modules/clinic/posixmodule.c.h | 16 ++++++++-------- Modules/posixmodule.c | 10 +++++----- 3 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-24-16-43-33.gh-issue-109782.gMC_7z.rst diff --git a/Misc/NEWS.d/next/Library/2023-09-24-16-43-33.gh-issue-109782.gMC_7z.rst b/Misc/NEWS.d/next/Library/2023-09-24-16-43-33.gh-issue-109782.gMC_7z.rst new file mode 100644 index 00000000000000..7612e59dc45412 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-24-16-43-33.gh-issue-109782.gMC_7z.rst @@ -0,0 +1,2 @@ +Ensure the signature of :func:`os.path.isdir` is identical on all platforms. +Patch by Amin Alaee. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index c3e7f86b3e33f1..e77a31b947f45e 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1977,7 +1977,7 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_isdir__doc__, -"_path_isdir($module, /, path)\n" +"_path_isdir($module, /, s)\n" "--\n" "\n" "Return true if the pathname refers to an existing directory."); @@ -1986,7 +1986,7 @@ PyDoc_STRVAR(os__path_isdir__doc__, {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *path); +os__path_isdir_impl(PyObject *module, PyObject *s); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2001,7 +2001,7 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, + .ob_item = { &_Py_ID(s), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2010,7 +2010,7 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"path", NULL}; + static const char * const _keywords[] = {"s", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "_path_isdir", @@ -2018,14 +2018,14 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + PyObject *s; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_isdir_impl(module, path); + s = args[0]; + return_value = os__path_isdir_impl(module, s); exit: return return_value; @@ -11988,4 +11988,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=1dd5aa7495cd6e3a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=51aa26bc6a41e1da input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 096aa043514c85..abf449e25493fa 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4912,25 +4912,25 @@ os__path_splitroot_impl(PyObject *module, path_t *path) /*[clinic input] os._path_isdir - path: 'O' + s: 'O' Return true if the pathname refers to an existing directory. [clinic start generated code]*/ static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=00faea0af309669d input=b1d2571cf7291aaf]*/ +os__path_isdir_impl(PyObject *module, PyObject *s) +/*[clinic end generated code: output=9d87ab3c8b8a4e61 input=c17f7ef21d22d64e]*/ { HANDLE hfile; BOOL close_file = TRUE; FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isdir", "path", 0, 1); + path_t _path = PATH_T_INITIALIZE("isdir", "s", 0, 1); int result; BOOL slow_path = TRUE; FILE_STAT_BASIC_INFORMATION statInfo; - if (!path_converter(path, &_path)) { + if (!path_converter(s, &_path)) { path_cleanup(&_path); if (PyErr_ExceptionMatches(PyExc_ValueError)) { PyErr_Clear(); From c4eda57345f579947b128e6148ab7f77de44bb88 Mon Sep 17 00:00:00 2001 From: Davide Rizzo Date: Thu, 28 Sep 2023 18:34:35 +0200 Subject: [PATCH 051/124] Whitespace fix in asyncio-stream.rst (#110015) --- Doc/library/asyncio-stream.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index bbac1c32b5695f..d8186b6ce75c79 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -157,8 +157,8 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. - .. versionchanged:: 3.11 - Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ From 757cbd4f29c9e89b38b975e0463dc8ed331b2515 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Sep 2023 19:04:01 +0200 Subject: [PATCH 052/124] gh-109972: Enhance test_gdb (#110026) * Split test_pycfunction.py: add test_cfunction_full.py. Split the function into the following 6 functions. In verbose mode, these "pycfunction" tests now log each tested call. * test_pycfunction_noargs() * test_pycfunction_o() * test_pycfunction_varargs() * test_pycfunction_varargs_keywords() * test_pycfunction_fastcall() * test_pycfunction_fastcall_keywords() * Move get_gdb_repr() to PrettyPrintTests. * Replace DebuggerTests.get_sample_script() with SAMPLE_SCRIPT. * Rename checkout_hook_path to CHECKOUT_HOOK_PATH. * Rename gdb_version to GDB_VERSION_TEXT. * Replace (gdb_major_version, gdb_minor_version) with GDB_VERSION. * run_gdb() uses "backslashreplace" error handler instead of "replace". * Add check_gdb() function to util.py. * Enhance support.check_cflags_pgo(): check also for sysconfig PGO_PROF_USE_FLAG (if available) in compiler flags. * Move some SkipTest checks to test_gdb/__init__.py. * Elaborate why gdb cannot be tested on Windows: gdb doesn't support PDB debug symbol files. --- Lib/test/support/__init__.py | 7 +- Lib/test/test_gdb/__init__.py | 24 ++- Lib/test/test_gdb/test_backtrace.py | 6 +- Lib/test/test_gdb/test_cfunction.py | 114 +++++----- Lib/test/test_gdb/test_cfunction_full.py | 36 ++++ Lib/test/test_gdb/test_misc.py | 20 +- Lib/test/test_gdb/test_pretty_print.py | 54 ++++- Lib/test/test_gdb/util.py | 256 +++++++++++------------ 8 files changed, 299 insertions(+), 218 deletions(-) create mode 100644 Lib/test/test_gdb/test_cfunction_full.py diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 4fcb8999579a82..38d5012ba46c08 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -777,14 +777,17 @@ def check_cflags_pgo(): # Check if Python was built with ./configure --enable-optimizations: # with Profile Guided Optimization (PGO). cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' - pgo_options = ( + pgo_options = [ # GCC '-fprofile-use', # clang: -fprofile-instr-use=code.profclangd '-fprofile-instr-use', # ICC "-prof-use", - ) + ] + PGO_PROF_USE_FLAG = sysconfig.get_config_var('PGO_PROF_USE_FLAG') + if PGO_PROF_USE_FLAG: + pgo_options.append(PGO_PROF_USE_FLAG) return any(option in cflags_nodist for option in pgo_options) diff --git a/Lib/test/test_gdb/__init__.py b/Lib/test/test_gdb/__init__.py index 0261f59adf54bd..d74075e456792d 100644 --- a/Lib/test/test_gdb/__init__.py +++ b/Lib/test/test_gdb/__init__.py @@ -4,7 +4,27 @@ # Lib/test/test_jit_gdb.py import os -from test.support import load_package_tests +import sysconfig +import unittest +from test import support + + +MS_WINDOWS = (os.name == 'nt') +if MS_WINDOWS: + # On Windows, Python is usually built by MSVC. Passing /p:DebugSymbols=true + # option to MSBuild produces PDB debug symbols, but gdb doesn't support PDB + # debug symbol files. + raise unittest.SkipTest("test_gdb doesn't work on Windows") + +if support.PGO: + raise unittest.SkipTest("test_gdb is not useful for PGO") + +if not sysconfig.is_python_build(): + raise unittest.SkipTest("test_gdb only works on source builds at the moment.") + +if support.check_cflags_pgo(): + raise unittest.SkipTest("test_gdb is not reliable on PGO builds") + def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index 15cbcf169ab9e3..c41e7cb7c210de 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -3,7 +3,7 @@ from test import support from test.support import python_is_optimized -from .util import setup_module, DebuggerTests, CET_PROTECTION +from .util import setup_module, DebuggerTests, CET_PROTECTION, SAMPLE_SCRIPT def setUpModule(): @@ -15,7 +15,7 @@ class PyBtTests(DebuggerTests): "Python was compiled with optimizations") def test_bt(self): 'Verify that the "py-bt" command works' - bt = self.get_stack_trace(script=self.get_sample_script(), + bt = self.get_stack_trace(script=SAMPLE_SCRIPT, cmds_after_breakpoint=['py-bt']) self.assertMultilineMatches(bt, r'''^.* @@ -35,7 +35,7 @@ def test_bt(self): "Python was compiled with optimizations") def test_bt_full(self): 'Verify that the "py-bt-full" command works' - bt = self.get_stack_trace(script=self.get_sample_script(), + bt = self.get_stack_trace(script=SAMPLE_SCRIPT, cmds_after_breakpoint=['py-bt-full']) self.assertMultilineMatches(bt, r'''^.* diff --git a/Lib/test/test_gdb/test_cfunction.py b/Lib/test/test_gdb/test_cfunction.py index 55796d021511e1..0a62014923e61f 100644 --- a/Lib/test/test_gdb/test_cfunction.py +++ b/Lib/test/test_gdb/test_cfunction.py @@ -1,8 +1,6 @@ -import re import textwrap import unittest from test import support -from test.support import python_is_optimized from .util import setup_module, DebuggerTests @@ -11,10 +9,22 @@ def setUpModule(): setup_module() -@unittest.skipIf(python_is_optimized(), +@unittest.skipIf(support.python_is_optimized(), "Python was compiled with optimizations") @support.requires_resource('cpu') class CFunctionTests(DebuggerTests): + def check(self, func_name, cmd): + # Verify with "py-bt": + gdb_output = self.get_stack_trace( + cmd, + breakpoint=func_name, + cmds_after_breakpoint=['bt', 'py-bt'], + # bpo-45207: Ignore 'Function "meth_varargs" not + # defined.' message in stderr. + ignore_stderr=True, + ) + self.assertIn(f'\n.*") @@ -167,7 +167,7 @@ class PyLocalsTests(DebuggerTests): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") def test_basic_command(self): - bt = self.get_stack_trace(script=self.get_sample_script(), + bt = self.get_stack_trace(script=SAMPLE_SCRIPT, cmds_after_breakpoint=['py-up', 'py-locals']) self.assertMultilineMatches(bt, r".*\nargs = \(1, 2, 3\)\n.*") @@ -176,7 +176,7 @@ def test_basic_command(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") def test_locals_after_up(self): - bt = self.get_stack_trace(script=self.get_sample_script(), + bt = self.get_stack_trace(script=SAMPLE_SCRIPT, cmds_after_breakpoint=['py-up', 'py-up', 'py-locals']) self.assertMultilineMatches(bt, r'''^.* diff --git a/Lib/test/test_gdb/test_pretty_print.py b/Lib/test/test_gdb/test_pretty_print.py index e31dc66f29684a..dfc77d65ab16a4 100644 --- a/Lib/test/test_gdb/test_pretty_print.py +++ b/Lib/test/test_gdb/test_pretty_print.py @@ -3,7 +3,7 @@ from test import support from .util import ( - BREAKPOINT_FN, gdb_major_version, gdb_minor_version, + BREAKPOINT_FN, GDB_VERSION, run_gdb, setup_module, DebuggerTests) @@ -12,6 +12,42 @@ def setUpModule(): class PrettyPrintTests(DebuggerTests): + def get_gdb_repr(self, source, + cmds_after_breakpoint=None, + import_site=False): + # Given an input python source representation of data, + # run "python -c'id(DATA)'" under gdb with a breakpoint on + # builtin_id and scrape out gdb's representation of the "op" + # parameter, and verify that the gdb displays the same string + # + # Verify that the gdb displays the expected string + # + # For a nested structure, the first time we hit the breakpoint will + # give us the top-level structure + + # NOTE: avoid decoding too much of the traceback as some + # undecodable characters may lurk there in optimized mode + # (issue #19743). + cmds_after_breakpoint = cmds_after_breakpoint or ["backtrace 1"] + gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN, + cmds_after_breakpoint=cmds_after_breakpoint, + import_site=import_site) + # gdb can insert additional '\n' and space characters in various places + # in its output, depending on the width of the terminal it's connected + # to (using its "wrap_here" function) + m = re.search( + # Match '#0 builtin_id(self=..., v=...)' + r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)' + # Match ' at Python/bltinmodule.c'. + # bpo-38239: builtin_id() is defined in Python/bltinmodule.c, + # but accept any "Directory\file.c" to support Link Time + # Optimization (LTO). + r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c', + gdb_output, re.DOTALL) + if not m: + self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) + return m.group(1), gdb_output + def test_getting_backtrace(self): gdb_output = self.get_stack_trace('id(42)') self.assertTrue(BREAKPOINT_FN in gdb_output) @@ -75,15 +111,17 @@ def test_strings(self): # as GDB might have been linked against a different version # of Python with a different encoding and coercion policy # with respect to PEP 538 and PEP 540. - out, err = run_gdb( + stdout, stderr = run_gdb( '--eval-command', 'python import locale; print(locale.getpreferredencoding())') - encoding = out.rstrip() - if err or not encoding: + encoding = stdout + if stderr or not encoding: raise RuntimeError( - f'unable to determine the preferred encoding ' - f'of embedded Python in GDB: {err}') + f'unable to determine the Python locale preferred encoding ' + f'of embedded Python in GDB\n' + f'stdout={stdout!r}\n' + f'stderr={stderr!r}') def check_repr(text): try: @@ -122,7 +160,7 @@ def test_tuples(self): @support.requires_resource('cpu') def test_sets(self): 'Verify the pretty-printing of sets' - if (gdb_major_version, gdb_minor_version) < (7, 3): + if GDB_VERSION < (7, 3): self.skipTest("pretty-printing of sets needs gdb 7.3 or later") self.assertGdbRepr(set(), "set()") self.assertGdbRepr(set(['a']), "{'a'}") @@ -141,7 +179,7 @@ def test_sets(self): @support.requires_resource('cpu') def test_frozensets(self): 'Verify the pretty-printing of frozensets' - if (gdb_major_version, gdb_minor_version) < (7, 3): + if GDB_VERSION < (7, 3): self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") self.assertGdbRepr(frozenset(), "frozenset()") self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 30beb4e14285c7..7f4e3cba3534bd 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -1,5 +1,6 @@ import os import re +import shlex import subprocess import sys import sysconfig @@ -7,29 +8,74 @@ from test import support -MS_WINDOWS = (sys.platform == 'win32') -if MS_WINDOWS: - raise unittest.SkipTest("test_gdb doesn't work on Windows") +# Location of custom hooks file in a repository checkout. +CHECKOUT_HOOK_PATH = os.path.join(os.path.dirname(sys.executable), + 'python-gdb.py') + +SAMPLE_SCRIPT = os.path.join(os.path.dirname(__file__), 'gdb_sample.py') +BREAKPOINT_FN = 'builtin_id' + +PYTHONHASHSEED = '123' + + +def clean_environment(): + # Remove PYTHON* environment variables such as PYTHONHOME + return {name: value for name, value in os.environ.items() + if not name.startswith('PYTHON')} + + +# Temporary value until it's initialized by get_gdb_version() below +GDB_VERSION = (0, 0) + +def run_gdb(*args, exitcode=0, **env_vars): + """Runs gdb in --batch mode with the additional arguments given by *args. + + Returns its (stdout, stderr) decoded from utf-8 using the replace handler. + """ + env = clean_environment() + if env_vars: + env.update(env_vars) + + cmd = ['gdb', + # Batch mode: Exit after processing all the command files + # specified with -x/--command + '--batch', + # -nx: Do not execute commands from any .gdbinit initialization + # files (gh-66384) + '-nx'] + if GDB_VERSION >= (7, 4): + cmd.extend(('--init-eval-command', + f'add-auto-load-safe-path {CHECKOUT_HOOK_PATH}')) + cmd.extend(args) + + proc = subprocess.run( + cmd, + # Redirect stdin to prevent gdb from messing with the terminal settings + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf8", errors="backslashreplace", + env=env) + + stdout = proc.stdout + stderr = proc.stderr + if proc.returncode != exitcode: + cmd_text = shlex.join(cmd) + raise Exception(f"{cmd_text} failed with exit code {proc.returncode}, " + f"expected exit code {exitcode}:\n" + f"stdout={stdout!r}\n" + f"stderr={stderr!r}") + + return (stdout, stderr) def get_gdb_version(): try: - cmd = ["gdb", "-nx", "--version"] - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - with proc: - version, stderr = proc.communicate() - - if proc.returncode: - raise Exception(f"Command {' '.join(cmd)!r} failed " - f"with exit code {proc.returncode}: " - f"stdout={version!r} stderr={stderr!r}") + stdout, stderr = run_gdb('--version') except OSError: # This is what "no gdb" looks like. There may, however, be other # errors that manifest this way too. - raise unittest.SkipTest("Couldn't find gdb on the path") + raise unittest.SkipTest("Couldn't find gdb program on the path") # Regex to parse: # 'GNU gdb (GDB; SUSE Linux Enterprise 12) 7.7\n' -> 7.7 @@ -37,32 +83,48 @@ def get_gdb_version(): # 'GNU gdb 6.1.1 [FreeBSD]\n' -> 6.1 # 'GNU gdb (GDB) Fedora (7.5.1-37.fc18)\n' -> 7.5 # 'HP gdb 6.7 for HP Itanium (32 or 64 bit) and target HP-UX 11iv2 and 11iv3.\n' -> 6.7 - match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", version) + match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", stdout) if match is None: - raise Exception("unable to parse GDB version: %r" % version) - return (version, int(match.group(1)), int(match.group(2))) + raise Exception("unable to parse gdb version: %r" % stdout) + version_text = stdout + major = int(match.group(1)) + minor = int(match.group(2)) + version = (major, minor) + return (version_text, version) -gdb_version, gdb_major_version, gdb_minor_version = get_gdb_version() -if gdb_major_version < 7: - raise unittest.SkipTest("gdb versions before 7.0 didn't support python " - "embedding. Saw %s.%s:\n%s" - % (gdb_major_version, gdb_minor_version, - gdb_version)) +GDB_VERSION_TEXT, GDB_VERSION = get_gdb_version() +if GDB_VERSION < (7, 0): + raise unittest.SkipTest( + f"gdb versions before 7.0 didn't support python embedding. " + f"Saw gdb version {GDB_VERSION[0]}.{GDB_VERSION[1]}:\n" + f"{GDB_VERSION_TEXT}") -if not sysconfig.is_python_build(): - raise unittest.SkipTest("test_gdb only works on source builds at the moment.") -if ((sysconfig.get_config_var('PGO_PROF_USE_FLAG') or 'xxx') in - (sysconfig.get_config_var('PY_CORE_CFLAGS') or '')): - raise unittest.SkipTest("test_gdb is not reliable on PGO builds") +def check_usable_gdb(): + # Verify that "gdb" was built with the embedded Python support enabled and + # verify that "gdb" can load our custom hooks, as OS security settings may + # disallow this without a customized .gdbinit. + stdout, stderr = run_gdb( + '--eval-command=python import sys; print(sys.version_info)', + '--args', sys.executable) -# Location of custom hooks file in a repository checkout. -checkout_hook_path = os.path.join(os.path.dirname(sys.executable), - 'python-gdb.py') + if "auto-loading has been declined" in stderr: + raise unittest.SkipTest( + f"gdb security settings prevent use of custom hooks; " + f"stderr: {stderr!r}") -PYTHONHASHSEED = '123' + if not stdout: + raise unittest.SkipTest( + f"gdb not built with embedded python support; " + f"stderr: {stderr!r}") + + if "major=2" in stdout: + raise unittest.SkipTest("gdb built with Python 2") +check_usable_gdb() + +# Control-flow enforcement technology def cet_protection(): cflags = sysconfig.get_config_var('CFLAGS') if not cflags: @@ -74,63 +136,17 @@ def cet_protection(): and any((flag.startswith('-fcf-protection') and not flag.endswith(("=none", "=return"))) for flag in flags)) - -# Control-flow enforcement technology CET_PROTECTION = cet_protection() -def run_gdb(*args, **env_vars): - """Runs gdb in --batch mode with the additional arguments given by *args. - - Returns its (stdout, stderr) decoded from utf-8 using the replace handler. - """ - if env_vars: - env = os.environ.copy() - env.update(env_vars) - else: - env = None - # -nx: Do not execute commands from any .gdbinit initialization files - # (issue #22188) - base_cmd = ('gdb', '--batch', '-nx') - if (gdb_major_version, gdb_minor_version) >= (7, 4): - base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path) - proc = subprocess.Popen(base_cmd + args, - # Redirect stdin to prevent GDB from messing with - # the terminal settings - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) - with proc: - out, err = proc.communicate() - return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') - -# Verify that "gdb" was built with the embedded python support enabled: -gdbpy_version, _ = run_gdb("--eval-command=python import sys; print(sys.version_info)") -if not gdbpy_version: - raise unittest.SkipTest("gdb not built with embedded python support") - -if "major=2" in gdbpy_version: - raise unittest.SkipTest("gdb built with Python 2") - -# Verify that "gdb" can load our custom hooks, as OS security settings may -# disallow this without a customized .gdbinit. -_, gdbpy_errors = run_gdb('--args', sys.executable) -if "auto-loading has been declined" in gdbpy_errors: - msg = "gdb security settings prevent use of custom hooks: " - raise unittest.SkipTest(msg + gdbpy_errors.rstrip()) - -BREAKPOINT_FN='builtin_id' - - def setup_module(): if support.verbose: - print("GDB version %s.%s:" % (gdb_major_version, gdb_minor_version)) - for line in gdb_version.splitlines(): + print(f"gdb version {GDB_VERSION[0]}.{GDB_VERSION[1]}:") + for line in GDB_VERSION_TEXT.splitlines(): print(" " * 4 + line) + print() -@unittest.skipIf(support.PGO, "not useful for PGO") class DebuggerTests(unittest.TestCase): """Test that the debugger can debug Python.""" @@ -163,20 +179,22 @@ def get_stack_trace(self, source=None, script=None, # structures # Generate a list of commands in gdb's language: - commands = ['set breakpoint pending yes', - 'break %s' % breakpoint, - - # The tests assume that the first frame of printed - # backtrace will not contain program counter, - # that is however not guaranteed by gdb - # therefore we need to use 'set print address off' to - # make sure the counter is not there. For example: - # #0 in PyObject_Print ... - # is assumed, but sometimes this can be e.g. - # #0 0x00003fffb7dd1798 in PyObject_Print ... - 'set print address off', - - 'run'] + commands = [ + 'set breakpoint pending yes', + 'break %s' % breakpoint, + + # The tests assume that the first frame of printed + # backtrace will not contain program counter, + # that is however not guaranteed by gdb + # therefore we need to use 'set print address off' to + # make sure the counter is not there. For example: + # #0 in PyObject_Print ... + # is assumed, but sometimes this can be e.g. + # #0 0x00003fffb7dd1798 in PyObject_Print ... + 'set print address off', + + 'run', + ] # GDB as of 7.4 onwards can distinguish between the # value of a variable at entry vs current value: @@ -184,7 +202,7 @@ def get_stack_trace(self, source=None, script=None, # which leads to the selftests failing with errors like this: # AssertionError: 'v@entry=()' != '()' # Disable this: - if (gdb_major_version, gdb_minor_version) >= (7, 4): + if GDB_VERSION >= (7, 4): commands += ['set print entry-values no'] if cmds_after_breakpoint: @@ -237,13 +255,16 @@ def get_stack_trace(self, source=None, script=None, for pattern in ( '(frame information optimized out)', 'Unable to read information on python frame', + # gh-91960: On Python built with "clang -Og", gdb gets # "frame=" for _PyEval_EvalFrameDefault() parameter '(unable to read python frame information)', + # gh-104736: On Python built with "clang -Og" on ppc64le, # "py-bt" displays a truncated or not traceback, but "where" # logs this error message: 'Backtrace stopped: frame did not save the PC', + # gh-104736: When "bt" command displays something like: # "#1 0x0000000000000000 in ?? ()", the traceback is likely # truncated or wrong. @@ -254,42 +275,6 @@ def get_stack_trace(self, source=None, script=None, return out - def get_gdb_repr(self, source, - cmds_after_breakpoint=None, - import_site=False): - # Given an input python source representation of data, - # run "python -c'id(DATA)'" under gdb with a breakpoint on - # builtin_id and scrape out gdb's representation of the "op" - # parameter, and verify that the gdb displays the same string - # - # Verify that the gdb displays the expected string - # - # For a nested structure, the first time we hit the breakpoint will - # give us the top-level structure - - # NOTE: avoid decoding too much of the traceback as some - # undecodable characters may lurk there in optimized mode - # (issue #19743). - cmds_after_breakpoint = cmds_after_breakpoint or ["backtrace 1"] - gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN, - cmds_after_breakpoint=cmds_after_breakpoint, - import_site=import_site) - # gdb can insert additional '\n' and space characters in various places - # in its output, depending on the width of the terminal it's connected - # to (using its "wrap_here" function) - m = re.search( - # Match '#0 builtin_id(self=..., v=...)' - r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)' - # Match ' at Python/bltinmodule.c'. - # bpo-38239: builtin_id() is defined in Python/bltinmodule.c, - # but accept any "Directory\file.c" to support Link Time - # Optimization (LTO). - r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c', - gdb_output, re.DOTALL) - if not m: - self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) - return m.group(1), gdb_output - def assertEndsWith(self, actual, exp_end): '''Ensure that the given "actual" string ends with "exp_end"''' self.assertTrue(actual.endswith(exp_end), @@ -299,6 +284,3 @@ def assertMultilineMatches(self, actual, pattern): m = re.match(pattern, actual, re.DOTALL) if not m: self.fail(msg='%r did not match %r' % (actual, pattern)) - - def get_sample_script(self): - return os.path.join(os.path.dirname(__file__), 'gdb_sample.py') From 7e0fbf5175fcf21dae390ba68b7f49706d62aa49 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Sep 2023 19:12:11 +0200 Subject: [PATCH 053/124] gh-110033: Fix signal test_interprocess_signal() (#110035) Fix test_interprocess_signal() of test_signal. Make sure that the subprocess.Popen object is deleted before the test raising an exception in a signal handler. Otherwise, Popen.__del__() can get the exception which is logged as "Exception ignored in: ...." and the test fails. --- Lib/test/signalinterproctester.py | 8 ++++++++ .../Tests/2023-09-28-18-14-52.gh-issue-110033.2yHMx0.rst | 5 +++++ 2 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-28-18-14-52.gh-issue-110033.2yHMx0.rst diff --git a/Lib/test/signalinterproctester.py b/Lib/test/signalinterproctester.py index cdcd92a8baace6..073c078f45f6d7 100644 --- a/Lib/test/signalinterproctester.py +++ b/Lib/test/signalinterproctester.py @@ -1,3 +1,4 @@ +import gc import os import signal import subprocess @@ -59,6 +60,13 @@ def test_interprocess_signal(self): self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 0, 'SIGALRM': 0}) + # gh-110033: Make sure that the subprocess.Popen is deleted before + # the next test which raises an exception. Otherwise, the exception + # may be raised when Popen.__del__() is executed and so be logged + # as "Exception ignored in: ". + child = None + gc.collect() + with self.assertRaises(SIGUSR1Exception): with self.subprocess_send_signal(pid, "SIGUSR1") as child: self.wait_signal(child, 'SIGUSR1') diff --git a/Misc/NEWS.d/next/Tests/2023-09-28-18-14-52.gh-issue-110033.2yHMx0.rst b/Misc/NEWS.d/next/Tests/2023-09-28-18-14-52.gh-issue-110033.2yHMx0.rst new file mode 100644 index 00000000000000..fb6089377083bf --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-28-18-14-52.gh-issue-110033.2yHMx0.rst @@ -0,0 +1,5 @@ +Fix ``test_interprocess_signal()`` of ``test_signal``. Make sure that the +``subprocess.Popen`` object is deleted before the test raising an exception +in a signal handler. Otherwise, ``Popen.__del__()`` can get the exception +which is logged as ``Exception ignored in: ...`` and the test fails. Patch by +Victor Stinner. From b14f0ab51cb4851b25935279617e388456dcf716 Mon Sep 17 00:00:00 2001 From: Davide Rizzo Date: Thu, 28 Sep 2023 19:25:10 +0200 Subject: [PATCH 054/124] gh-110038: KqueueSelector must count all read/write events (#110039) --- Lib/selectors.py | 7 ++++- Lib/test/test_selectors.py | 29 +++++++++++++++++++ ...-09-28-18-50-33.gh-issue-110038.nx_gCu.rst | 3 ++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-18-50-33.gh-issue-110038.nx_gCu.rst diff --git a/Lib/selectors.py b/Lib/selectors.py index 20367c9152f331..b8e5f6a4f77d89 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -491,6 +491,7 @@ class KqueueSelector(_BaseSelectorImpl): def __init__(self): super().__init__() self._selector = select.kqueue() + self._max_events = 0 def fileno(self): return self._selector.fileno() @@ -502,10 +503,12 @@ def register(self, fileobj, events, data=None): kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_ADD) self._selector.control([kev], 0, 0) + self._max_events += 1 if events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) self._selector.control([kev], 0, 0) + self._max_events += 1 except: super().unregister(fileobj) raise @@ -516,6 +519,7 @@ def unregister(self, fileobj): if key.events & EVENT_READ: kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE) + self._max_events -= 1 try: self._selector.control([kev], 0, 0) except OSError: @@ -525,6 +529,7 @@ def unregister(self, fileobj): if key.events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) + self._max_events -= 1 try: self._selector.control([kev], 0, 0) except OSError: @@ -537,7 +542,7 @@ def select(self, timeout=None): # If max_ev is 0, kqueue will ignore the timeout. For consistent # behavior with the other selector classes, we prevent that here # (using max). See https://bugs.python.org/issue29255 - max_ev = len(self._fd_to_key) or 1 + max_ev = self._max_events or 1 ready = [] try: kev_list = self._selector.control(None, max_ev, timeout) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 33417ca6a11af0..677349c2bfca93 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -285,6 +285,35 @@ def test_select(self): self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + def test_select_read_write(self): + # gh-110038: when a file descriptor is registered for both read and + # write, the two events must be seen on a single call to select(). + s = self.SELECTOR() + self.addCleanup(s.close) + + sock1, sock2 = self.make_socketpair() + sock2.send(b"foo") + my_key = s.register(sock1, selectors.EVENT_READ | selectors.EVENT_WRITE) + + seen_read, seen_write = False, False + result = s.select() + # We get the read and write either in the same result entry or in two + # distinct entries with the same key. + self.assertLessEqual(len(result), 2) + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertEqual(key, my_key) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + if events & selectors.EVENT_READ: + self.assertFalse(seen_read) + seen_read = True + if events & selectors.EVENT_WRITE: + self.assertFalse(seen_write) + seen_write = True + self.assertTrue(seen_read) + self.assertTrue(seen_write) + def test_context_manager(self): s = self.SELECTOR() self.addCleanup(s.close) diff --git a/Misc/NEWS.d/next/Library/2023-09-28-18-50-33.gh-issue-110038.nx_gCu.rst b/Misc/NEWS.d/next/Library/2023-09-28-18-50-33.gh-issue-110038.nx_gCu.rst new file mode 100644 index 00000000000000..6b2abd802fccdc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-18-50-33.gh-issue-110038.nx_gCu.rst @@ -0,0 +1,3 @@ +Fixed an issue that caused :meth:`KqueueSelector.select` to not return all +the ready events in some cases when a file descriptor is registered for both +read and write. From f580edcc6a4c528020afe46c753db713474acad6 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 28 Sep 2023 20:33:28 +0100 Subject: [PATCH 055/124] gh-109889: fix compiler's redundant NOP detection to look past NOPs with no lineno when looking for the next instruction's lineno (#109987) --- Lib/test/test_compile.py | 5 +++++ .../2023-09-27-21-35-49.gh-issue-109889.t5hIRT.rst | 2 ++ Python/flowgraph.c | 12 +++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-27-21-35-49.gh-issue-109889.t5hIRT.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 53e3e8f75aa766..c4452e38934cf8 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1278,6 +1278,11 @@ def f(x): while x: 0 if 1 else 0 + def test_remove_redundant_nop_edge_case(self): + # See gh-109889 + def f(): + a if (1 if b else c) else d + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-27-21-35-49.gh-issue-109889.t5hIRT.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-21-35-49.gh-issue-109889.t5hIRT.rst new file mode 100644 index 00000000000000..8be373f0f6b6cd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-21-35-49.gh-issue-109889.t5hIRT.rst @@ -0,0 +1,2 @@ +Fix the compiler's redundant NOP detection algorithm to skip over NOPs with +no line number when looking for the next instruction's lineno. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 9fe387cc9a8e80..e89ad39b35719b 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1017,7 +1017,17 @@ remove_redundant_nops(basicblock *bb) { } /* or if last instruction in BB and next BB has same line number */ if (next) { - if (lineno == next->b_instr[0].i_loc.lineno) { + location next_loc = NO_LOCATION; + for (int next_i=0; next_i < next->b_iused; next_i++) { + cfg_instr *instr = &next->b_instr[next_i]; + if (instr->i_opcode == NOP && instr->i_loc.lineno == NO_LOCATION.lineno) { + /* Skip over NOPs without location, they will be removed */ + continue; + } + next_loc = instr->i_loc; + break; + } + if (lineno == next_loc.lineno) { continue; } } From cf4c29725636e1a0dd2ebab443613b56ca6c9486 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 28 Sep 2023 17:58:13 -0500 Subject: [PATCH 056/124] gh-109991: Update Windows build to use OpenSSL 3.0.11 (GH-110054) --- .../Windows/2023-09-28-17-09-23.gh-issue-109991.CIMftz.rst | 1 + PCbuild/get_externals.bat | 4 ++-- PCbuild/python.props | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-09-28-17-09-23.gh-issue-109991.CIMftz.rst diff --git a/Misc/NEWS.d/next/Windows/2023-09-28-17-09-23.gh-issue-109991.CIMftz.rst b/Misc/NEWS.d/next/Windows/2023-09-28-17-09-23.gh-issue-109991.CIMftz.rst new file mode 100644 index 00000000000000..ee988f90863426 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-09-28-17-09-23.gh-issue-109991.CIMftz.rst @@ -0,0 +1 @@ +Update Windows build to use OpenSSL 3.0.11. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 1d3abcc3def1fa..fc43ce2b1835d0 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.10 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 set libraries=%libraries% sqlite-3.42.0.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 @@ -76,7 +76,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.10 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index 94faa8221eac5a..35c6e92be45dc9 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,8 +74,8 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-3.0.10\ - $(ExternalsDir)openssl-bin-3.0.10\$(ArchName)\ + $(ExternalsDir)openssl-3.0.11\ + $(ExternalsDir)openssl-bin-3.0.11\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.2.13\ From b488c0d761b2018c10bc5a0e5469b8b209e1a681 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 28 Sep 2023 17:58:49 -0500 Subject: [PATCH 057/124] gh-109991: Remove obsolete NEWS entries for OpenSSL 3.0.10 (GH-110055) --- .../Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst | 2 -- .../next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst | 1 - .../next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst | 1 - 3 files changed, 4 deletions(-) delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst deleted file mode 100644 index c43ee680e8158e..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, -and 3.1.2. diff --git a/Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst b/Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst deleted file mode 100644 index 024a58299caed9..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-09-05-10-08-47.gh-issue-107565.CIMftz.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows build to use OpenSSL 3.0.10. diff --git a/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst deleted file mode 100644 index c238c4760239e1..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 3.0.10. From 5fdcea744024c8a19ddb57057bf5ec2889546c98 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 02:01:24 +0200 Subject: [PATCH 058/124] gh-109960: Remove test_pty timeout of 10 seconds (#110058) In 2003, test_pty got a hardcoded timeout of 10 seconds to prevent hanging on AIX & HPUX "if run after test_openpty": commit 7d8145268ee282f14d6adce9305dc3c1c7ffec14. Since 2003, test_pty was no longer reported to hang on AIX. But today, the test can fail simply because a CI is busy running other tests in parallel. The timeout of 10 seconds is no longer needed, just remove it. Moreover, regrtest now has multiple built-in generic timeout mecanisms. --- Lib/test/test_pty.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index c9c2b42861c6f4..a971f6b0250efb 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -80,17 +80,9 @@ def expectedFailureIfStdinIsTTY(fun): # because pty code is not too portable. class PtyTest(unittest.TestCase): def setUp(self): - old_alarm = signal.signal(signal.SIGALRM, self.handle_sig) - self.addCleanup(signal.signal, signal.SIGALRM, old_alarm) - old_sighup = signal.signal(signal.SIGHUP, self.handle_sighup) self.addCleanup(signal.signal, signal.SIGHUP, old_sighup) - # isatty() and close() can hang on some platforms. Set an alarm - # before running the test to make sure we don't hang forever. - self.addCleanup(signal.alarm, 0) - signal.alarm(10) - # Save original stdin window size. self.stdin_dim = None if _HAVE_WINSZ: @@ -101,9 +93,6 @@ def setUp(self): except tty.error: pass - def handle_sig(self, sig, frame): - self.fail("isatty hung") - @staticmethod def handle_sighup(signum, frame): pass From 4e356ad183eeb567783f4a87fd092573da1e9252 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 02:34:27 +0200 Subject: [PATCH 059/124] gh-109974: Fix threading lock_tests race conditions (#110057) Fix race conditions in test_threading lock tests. Wait until a condition is met rather than using time.sleep() with a hardcoded number of seconds. * Replace sleeping loops with support.sleeping_retry() which raises an exception on timeout. * Add wait_threads_blocked(nthread) which computes a sleep depending on the number of threads. Remove _wait() function. * test_set_and_clear(): use a way longer Event.wait() timeout. * BarrierTests.test_repr(): wait until the 2 threads are waiting for the barrier. Use a way longer timeout for Barrier.wait() timeout. * test_thread_leak() no longer needs to count len(threading.enumerate()): Bunch uses threading_helper.wait_threads_exit() internally which does it in wait_for_finished(). * Add BaseLockTests.wait_phase() which implements a timeout. test_reacquire() and test_recursion_count() use wait_phase(). --- Lib/test/lock_tests.py | 347 +++++++++++------- ...-09-29-00-19-21.gh-issue-109974.Sh_g-r.rst | 3 + 2 files changed, 226 insertions(+), 124 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-29-00-19-21.gh-issue-109974.Sh_g-r.rst diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index e53e24b18f2760..cbaae3afd6dde3 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -19,22 +19,24 @@ "(no _at_fork_reinit method)") -def _wait(): - # A crude wait/yield function not relying on synchronization primitives. - time.sleep(0.01) +def wait_threads_blocked(nthread): + # Arbitrary sleep to wait until N threads are blocked, + # like waiting for a lock. + time.sleep(0.010 * nthread) + class Bunch(object): """ A bunch of threads. """ - def __init__(self, f, n, wait_before_exit=False): + def __init__(self, func, nthread, wait_before_exit=False): """ - Construct a bunch of `n` threads running the same function `f`. + Construct a bunch of `nthread` threads running the same function `func`. If `wait_before_exit` is True, the threads won't terminate until do_finish() is called. """ - self.f = f - self.n = n + self.func = func + self.nthread = nthread self.started = [] self.finished = [] self._can_exit = not wait_before_exit @@ -45,26 +47,30 @@ def task(): tid = threading.get_ident() self.started.append(tid) try: - f() + func() finally: self.finished.append(tid) - while not self._can_exit: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self._can_exit: + break try: - for i in range(n): + for i in range(nthread): start_new_thread(task, ()) except: self._can_exit = True raise def wait_for_started(self): - while len(self.started) < self.n: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.started) >= self.nthread: + break def wait_for_finished(self): - while len(self.finished) < self.n: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.finished) >= self.nthread: + break + # Wait for threads exit self.wait_thread.__exit__(None, None, None) @@ -94,6 +100,12 @@ class BaseLockTests(BaseTestCase): Tests for both recursive and non-recursive locks. """ + def wait_phase(self, phase, expected): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(phase) >= expected: + break + self.assertEqual(len(phase), expected) + def test_constructor(self): lock = self.locktype() del lock @@ -138,15 +150,18 @@ def f(): def test_acquire_contended(self): lock = self.locktype() lock.acquire() - N = 5 def f(): lock.acquire() lock.release() + # Threads block on lock.acquire() + N = 5 b = Bunch(f, N) b.wait_for_started() - _wait() + wait_threads_blocked(N) self.assertEqual(len(b.finished), 0) + + # Threads unblocked lock.release() b.wait_for_finished() self.assertEqual(len(b.finished), N) @@ -174,17 +189,10 @@ def test_thread_leak(self): def f(): lock.acquire() lock.release() - n = len(threading.enumerate()) + # We run many threads in the hope that existing threads ids won't # be recycled. Bunch(f, 15).wait_for_finished() - if len(threading.enumerate()) != n: - # There is a small window during which a Thread instance's - # target function has finished running, but the Thread is still - # alive and registered. Avoid spurious failures by waiting a - # bit more (seen on a buildbot). - time.sleep(0.4) - self.assertEqual(n, len(threading.enumerate())) def test_timeout(self): lock = self.locktype() @@ -242,15 +250,13 @@ def f(): phase.append(None) with threading_helper.wait_threads_exit(): + # Thread blocked on lock.acquire() start_new_thread(f, ()) - while len(phase) == 0: - _wait() - _wait() - self.assertEqual(len(phase), 1) + self.wait_phase(phase, 1) + + # Thread unblocked lock.release() - while len(phase) == 1: - _wait() - self.assertEqual(len(phase), 2) + self.wait_phase(phase, 2) def test_different_thread(self): # Lock can be released from a different thread. @@ -349,21 +355,20 @@ def test_recursion_count(self): def f(): lock.acquire() phase.append(None) - while len(phase) == 1: - _wait() + + self.wait_phase(phase, 2) lock.release() phase.append(None) with threading_helper.wait_threads_exit(): + # Thread blocked on lock.acquire() start_new_thread(f, ()) - while len(phase) == 0: - _wait() - self.assertEqual(len(phase), 1) + self.wait_phase(phase, 1) self.assertEqual(0, lock._recursion_count()) + + # Thread unblocked phase.append(None) - while len(phase) == 2: - _wait() - self.assertEqual(len(phase), 3) + self.wait_phase(phase, 3) self.assertEqual(0, lock._recursion_count()) def test_different_thread(self): @@ -421,10 +426,14 @@ def _check_notify(self, evt): def f(): results1.append(evt.wait()) results2.append(evt.wait()) + + # Threads blocked on first evt.wait() b = Bunch(f, N) b.wait_for_started() - _wait() + wait_threads_blocked(N) self.assertEqual(len(results1), 0) + + # Threads unblocked evt.set() b.wait_for_finished() self.assertEqual(results1, [True] * N) @@ -464,19 +473,22 @@ def f(): self.assertTrue(r) def test_set_and_clear(self): - # Issue #13502: check that wait() returns true even when the event is + # gh-57711: check that wait() returns true even when the event is # cleared before the waiting thread is woken up. - evt = self.eventtype() + event = self.eventtype() results = [] - timeout = 0.250 - N = 5 def f(): - results.append(evt.wait(timeout * 4)) + results.append(event.wait(support.LONG_TIMEOUT)) + + # Threads blocked on event.wait() + N = 5 b = Bunch(f, N) b.wait_for_started() - time.sleep(timeout) - evt.set() - evt.clear() + wait_threads_blocked(N) + + # Threads unblocked + event.set() + event.clear() b.wait_for_finished() self.assertEqual(results, [True] * N) @@ -533,15 +545,14 @@ def _check_notify(self, cond): # Note that this test is sensitive to timing. If the worker threads # don't execute in a timely fashion, the main thread may think they # are further along then they are. The main thread therefore issues - # _wait() statements to try to make sure that it doesn't race ahead - # of the workers. + # wait_threads_blocked() statements to try to make sure that it doesn't + # race ahead of the workers. # Secondly, this test assumes that condition variables are not subject # to spurious wakeups. The absence of spurious wakeups is an implementation # detail of Condition Variables in current CPython, but in general, not # a guaranteed property of condition variables as a programming # construct. In particular, it is possible that this can no longer # be conveniently guaranteed should their implementation ever change. - N = 5 ready = [] results1 = [] results2 = [] @@ -550,57 +561,84 @@ def f(): cond.acquire() ready.append(phase_num) result = cond.wait() + cond.release() results1.append((result, phase_num)) + cond.acquire() ready.append(phase_num) + result = cond.wait() cond.release() results2.append((result, phase_num)) + + N = 5 b = Bunch(f, N) b.wait_for_started() # first wait, to ensure all workers settle into cond.wait() before # we continue. See issues #8799 and #30727. - while len(ready) < 5: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break + ready.clear() self.assertEqual(results1, []) + # Notify 3 threads at first + count1 = 3 cond.acquire() - cond.notify(3) - _wait() + cond.notify(count1) + wait_threads_blocked(count1) + + # Phase 1 phase_num = 1 cond.release() - while len(results1) < 3: - _wait() - self.assertEqual(results1, [(True, 1)] * 3) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) >= count1: + break + + self.assertEqual(results1, [(True, 1)] * count1) self.assertEqual(results2, []) - # make sure all awaken workers settle into cond.wait() - while len(ready) < 3: - _wait() + + # Wait until awaken workers are blocked on cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= count1 : + break + # Notify 5 threads: they might be in their first or second wait cond.acquire() cond.notify(5) - _wait() + wait_threads_blocked(N) + + # Phase 2 phase_num = 2 cond.release() - while len(results1) + len(results2) < 8: - _wait() - self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2) - self.assertEqual(results2, [(True, 2)] * 3) - # make sure all workers settle into cond.wait() - while len(ready) < 5: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= (N + count1): + break + + count2 = N - count1 + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1) + + # Make sure all workers settle into cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break + # Notify all threads: they are all in their second wait cond.acquire() cond.notify_all() - _wait() + wait_threads_blocked(N) + + # Phase 3 phase_num = 3 cond.release() - while len(results2) < 5: - _wait() - self.assertEqual(results1, [(True, 1)] * 3 + [(True,2)] * 2) - self.assertEqual(results2, [(True, 2)] * 3 + [(True, 3)] * 2) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results2) >= N: + break + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1 + [(True, 3)] * count2) b.wait_for_finished() def test_notify(self): @@ -611,19 +649,22 @@ def test_notify(self): def test_timeout(self): cond = self.condtype() + timeout = 0.5 results = [] - N = 5 def f(): cond.acquire() t1 = time.monotonic() - result = cond.wait(0.5) + result = cond.wait(timeout) t2 = time.monotonic() cond.release() results.append((t2 - t1, result)) + + N = 5 Bunch(f, N).wait_for_finished() self.assertEqual(len(results), N) + for dt, result in results: - self.assertTimeout(dt, 0.5) + self.assertTimeout(dt, timeout) # Note that conceptually (that"s the condition variable protocol) # a wait() may succeed even if no one notifies us and before any # timeout occurs. Spurious wakeups can occur. @@ -636,13 +677,13 @@ def test_waitfor(self): state = 0 def f(): with cond: - result = cond.wait_for(lambda : state==4) + result = cond.wait_for(lambda: state == 4) self.assertTrue(result) self.assertEqual(state, 4) b = Bunch(f, 1) b.wait_for_started() for i in range(4): - time.sleep(0.01) + time.sleep(0.010) with cond: state += 1 cond.notify() @@ -660,14 +701,16 @@ def f(): self.assertFalse(result) self.assertTimeout(dt, 0.1) success.append(None) + b = Bunch(f, 1) b.wait_for_started() # Only increment 3 times, so state == 4 is never reached. for i in range(3): - time.sleep(0.01) + time.sleep(0.010) with cond: state += 1 cond.notify() + b.wait_for_finished() self.assertEqual(len(success), 1) @@ -697,70 +740,107 @@ def test_acquire_destroy(self): del sem def test_acquire_contended(self): - sem = self.semtype(7) + sem_value = 7 + sem = self.semtype(sem_value) sem.acquire() - N = 10 + sem_results = [] results1 = [] results2 = [] phase_num = 0 - def f(): + + def func(): sem_results.append(sem.acquire()) results1.append(phase_num) + sem_results.append(sem.acquire()) results2.append(phase_num) - b = Bunch(f, 10) + + def wait_count(count): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= count: + break + + # Phase 0 + N = 10 + b = Bunch(func, N) b.wait_for_started() - while len(results1) + len(results2) < 6: - _wait() - self.assertEqual(results1 + results2, [0] * 6) + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 phase_num = 1 - for i in range(7): + for i in range(sem_value): sem.release() - while len(results1) + len(results2) < 13: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) + count2 = sem_value + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 phase_num = 2 - for i in range(6): + count3 = (sem_value - 1) + for i in range(count3): sem.release() - while len(results1) + len(results2) < 19: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) # The semaphore is still locked self.assertFalse(sem.acquire(False)) + # Final release, to let the last thread finish + count4 = 1 sem.release() b.wait_for_finished() - self.assertEqual(sem_results, [True] * (6 + 7 + 6 + 1)) + self.assertEqual(sem_results, + [True] * (count1 + count2 + count3 + count4)) def test_multirelease(self): - sem = self.semtype(7) + sem_value = 7 + sem = self.semtype(sem_value) sem.acquire() + results1 = [] results2 = [] phase_num = 0 - def f(): + def func(): sem.acquire() results1.append(phase_num) + sem.acquire() results2.append(phase_num) - b = Bunch(f, 10) + + def wait_count(count): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= count: + break + + # Phase 0 + b = Bunch(func, 10) b.wait_for_started() - while len(results1) + len(results2) < 6: - _wait() - self.assertEqual(results1 + results2, [0] * 6) + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 phase_num = 1 - sem.release(7) - while len(results1) + len(results2) < 13: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) + count2 = sem_value + sem.release(count2) + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 phase_num = 2 - sem.release(6) - while len(results1) + len(results2) < 19: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) + count3 = sem_value - 1 + sem.release(count3) + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) # The semaphore is still locked self.assertFalse(sem.acquire(False)) + # Final release, to let the last thread finish sem.release() b.wait_for_finished() @@ -806,10 +886,14 @@ def test_default_value(self): def f(): sem.acquire() sem.release() + + # Thread blocked on sem.acquire() b = Bunch(f, 1) b.wait_for_started() - _wait() + wait_threads_blocked(1) self.assertFalse(b.finished) + + # Thread unblocked sem.release() b.wait_for_finished() @@ -882,6 +966,7 @@ class BarrierTests(BaseTestCase): def setUp(self): self.barrier = self.barriertype(self.N, timeout=self.defaultTimeout) + def tearDown(self): self.barrier.abort() @@ -979,8 +1064,9 @@ def f(): i = self.barrier.wait() if i == self.N//2: # Wait until the other threads are all in the barrier. - while self.barrier.n_waiting < self.N-1: - time.sleep(0.001) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self.barrier.n_waiting >= (self.N - 1): + break self.barrier.reset() else: try: @@ -1068,16 +1154,29 @@ def test_single_thread(self): b.wait() def test_repr(self): - b = self.barriertype(3) - self.assertRegex(repr(b), r"<\w+\.Barrier at .*: waiters=0/3>") + barrier = self.barriertype(3) + timeout = support.LONG_TIMEOUT + self.assertRegex(repr(barrier), r"<\w+\.Barrier at .*: waiters=0/3>") def f(): - b.wait(3) - bunch = Bunch(f, 2) + barrier.wait(timeout) + + # Threads blocked on barrier.wait() + N = 2 + bunch = Bunch(f, N) bunch.wait_for_started() - time.sleep(0.2) - self.assertRegex(repr(b), r"<\w+\.Barrier at .*: waiters=2/3>") - b.wait(3) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if barrier.n_waiting >= N: + break + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=2/3>") + + # Threads unblocked + barrier.wait(timeout) bunch.wait_for_finished() - self.assertRegex(repr(b), r"<\w+\.Barrier at .*: waiters=0/3>") - b.abort() - self.assertRegex(repr(b), r"<\w+\.Barrier at .*: broken>") + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=0/3>") + + # Abort the barrier + barrier.abort() + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: broken>") diff --git a/Misc/NEWS.d/next/Tests/2023-09-29-00-19-21.gh-issue-109974.Sh_g-r.rst b/Misc/NEWS.d/next/Tests/2023-09-29-00-19-21.gh-issue-109974.Sh_g-r.rst new file mode 100644 index 00000000000000..a130cf690a57cb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-29-00-19-21.gh-issue-109974.Sh_g-r.rst @@ -0,0 +1,3 @@ +Fix race conditions in test_threading lock tests. Wait until a condition is met +rather than using :func:`time.sleep` with a hardcoded number of seconds. Patch +by Victor Stinner. From bd4518c60c9df356cf5e05b81305e3644ebb5e70 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 02:41:12 +0200 Subject: [PATCH 060/124] gh-110036: multiprocessing Popen.terminate() catches PermissionError (#110037) On Windows, multiprocessing Popen.terminate() now catchs PermissionError and get the process exit code. If the process is still running, raise again the PermissionError. Otherwise, the process terminated as expected: store its exit code. --- Lib/multiprocessing/popen_spawn_win32.py | 11 +++++++++-- Lib/test/_test_multiprocessing.py | 5 +++-- .../2023-09-28-18-53-11.gh-issue-110036.fECxTj.rst | 5 +++++ 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-18-53-11.gh-issue-110036.fECxTj.rst diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 4d60ffc030bea6..af044305709e56 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -14,6 +14,7 @@ # # +# Exit code used by Popen.terminate() TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") @@ -122,9 +123,15 @@ def terminate(self): if self.returncode is None: try: _winapi.TerminateProcess(int(self._handle), TERMINATE) - except OSError: - if self.wait(timeout=1.0) is None: + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + code = _winapi.GetExitCodeProcess(int(self._handle)) + if code == _winapi.STILL_ACTIVE: raise + self.returncode = code + else: + self.returncode = -signal.SIGTERM kill = terminate diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 756d6808518fc4..39666dd331db0b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -557,13 +557,14 @@ def handler(*args): def test_terminate(self): exitcode = self._kill_process(multiprocessing.Process.terminate) - if os.name != 'nt': - self.assertEqual(exitcode, -signal.SIGTERM) + self.assertEqual(exitcode, -signal.SIGTERM) def test_kill(self): exitcode = self._kill_process(multiprocessing.Process.kill) if os.name != 'nt': self.assertEqual(exitcode, -signal.SIGKILL) + else: + self.assertEqual(exitcode, -signal.SIGTERM) def test_cpu_count(self): try: diff --git a/Misc/NEWS.d/next/Library/2023-09-28-18-53-11.gh-issue-110036.fECxTj.rst b/Misc/NEWS.d/next/Library/2023-09-28-18-53-11.gh-issue-110036.fECxTj.rst new file mode 100644 index 00000000000000..ddb11b5c3546a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-18-53-11.gh-issue-110036.fECxTj.rst @@ -0,0 +1,5 @@ +On Windows, multiprocessing ``Popen.terminate()`` now catchs +:exc:`PermissionError` and get the process exit code. If the process is +still running, raise again the :exc:`PermissionError`. Otherwise, the +process terminated as expected: store its exit code. Patch by Victor +Stinner. From 235aacdeed71afa6572ffad15155e781cc70bad1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 02:51:22 +0200 Subject: [PATCH 061/124] gh-109566: regrtest _add_python_opts() handles KeyboardInterrupt (#110062) In the subprocess code path, wait until the child process completes with a timeout of EXIT_TIMEOUT seconds. Fix create_worker_process() regression: use start_new_session=True if USE_PROCESS_GROUP is true. WorkerThread.wait_stopped() uses a timeout of 60 seconds, instead of 30 seconds. --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++++----- Lib/test/libregrtest/run_workers.py | 13 ++++++++----- Lib/test/libregrtest/utils.py | 2 +- Lib/test/libregrtest/worker.py | 9 ++++++--- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 45a68a8465d8e0..dcb2c5870de176 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,18 +11,18 @@ from .cmdline import _parse_args, Namespace from .findtests import findtests, split_test_packages, list_cases from .logger import Logger +from .pgo import setup_pgo_tests from .result import State +from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME -from .pgo import setup_pgo_tests -from .results import TestResults from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, - MS_WINDOWS) + MS_WINDOWS, EXIT_TIMEOUT) class Regrtest: @@ -525,10 +525,23 @@ def _add_python_opts(self): try: if hasattr(os, 'execv') and not MS_WINDOWS: os.execv(cmd[0], cmd) - # execv() do no return and so we don't get to this line on success + # On success, execv() do no return. + # On error, it raises an OSError. else: import subprocess - proc = subprocess.run(cmd) + with subprocess.Popen(cmd) as proc: + try: + proc.wait() + except KeyboardInterrupt: + # There is no need to call proc.terminate(): on CTRL+C, + # SIGTERM is also sent to the child process. + try: + proc.wait(timeout=EXIT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + sys.exit(EXITCODE_INTERRUPTED) + sys.exit(proc.returncode) except Exception as exc: print_warning(f"Failed to change Python options: {exc!r}\n" diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 89cc50b7c158d2..41ed7b0bac01ad 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -42,7 +42,10 @@ assert MAIN_PROCESS_TIMEOUT >= PROGRESS_UPDATE # Time to wait until a worker completes: should be immediate -JOIN_TIMEOUT = 30.0 # seconds +WAIT_COMPLETED_TIMEOUT = 30.0 # seconds + +# Time to wait a killed process (in seconds) +WAIT_KILLED_TIMEOUT = 60.0 # We do not use a generator so multiple threads can call next(). @@ -138,7 +141,7 @@ def _kill(self) -> None: if USE_PROCESS_GROUP: what = f"{self} process group" else: - what = f"{self}" + what = f"{self} process" print(f"Kill {what}", file=sys.stderr, flush=True) try: @@ -390,10 +393,10 @@ def _wait_completed(self) -> None: popen = self._popen try: - popen.wait(JOIN_TIMEOUT) + popen.wait(WAIT_COMPLETED_TIMEOUT) except (subprocess.TimeoutExpired, OSError) as exc: print_warning(f"Failed to wait for {self} completion " - f"(timeout={format_duration(JOIN_TIMEOUT)}): " + f"(timeout={format_duration(WAIT_COMPLETED_TIMEOUT)}): " f"{exc!r}") def wait_stopped(self, start_time: float) -> None: @@ -414,7 +417,7 @@ def wait_stopped(self, start_time: float) -> None: break dt = time.monotonic() - start_time self.log(f"Waiting for {self} thread for {format_duration(dt)}") - if dt > JOIN_TIMEOUT: + if dt > WAIT_KILLED_TIMEOUT: print_warning(f"Failed to join {self} in {format_duration(dt)}") break diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index bedf9a5420db64..46451152b8859f 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -541,7 +541,7 @@ def display_header(use_resources: tuple[str, ...]): print(f"== resources ({len(use_resources)}): " f"{', '.join(sorted(use_resources))}") else: - print(f"== resources: (all disabled, use -u option)") + print("== resources: (all disabled, use -u option)") # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 67f26cfe75fbe4..a9c8be0bb65d08 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -41,14 +41,15 @@ def create_worker_process(runtests: RunTests, output_fd: int, env['TEMP'] = tmp_dir env['TMP'] = tmp_dir + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + # # Emscripten and WASI Python must start in the Python source code directory # to get 'python.js' or 'python.wasm' file. Then worker_process() changes # to a temporary directory created to run tests. work_dir = os_helper.SAVEDCWD - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. kwargs: dict[str, Any] = dict( env=env, stdout=output_fd, @@ -58,6 +59,8 @@ def create_worker_process(runtests: RunTests, output_fd: int, close_fds=True, cwd=work_dir, ) + if USE_PROCESS_GROUP: + kwargs['start_new_session'] = True # Pass json_file to the worker process json_file = runtests.json_file From 2e37a38bcbfbe1357436e030538290e7d00b668d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 04:04:06 +0200 Subject: [PATCH 062/124] gh-110052: Fix faulthandler for freed tstate (#110069) faulthandler now detected freed interp and freed tstate, and no longer dereference them. --- Modules/faulthandler.c | 3 +-- Python/traceback.c | 47 +++++++++++++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index b051c71b3ade9b..4b6bf68be07202 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -174,7 +174,6 @@ faulthandler_dump_traceback(int fd, int all_threads, PyInterpreterState *interp) { static volatile int reentrant = 0; - PyThreadState *tstate; if (reentrant) return; @@ -189,7 +188,7 @@ faulthandler_dump_traceback(int fd, int all_threads, fault if the thread released the GIL, and so this function cannot be used. Read the thread specific storage (TSS) instead: call PyGILState_GetThisThreadState(). */ - tstate = PyGILState_GetThisThreadState(); + PyThreadState *tstate = PyGILState_GetThisThreadState(); if (all_threads) { (void)_Py_DumpTracebackThreads(fd, NULL, tstate); diff --git a/Python/traceback.c b/Python/traceback.c index a75b7833af4e05..7e791d0a59bd82 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1215,23 +1215,45 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "\n"); } +static int +tstate_is_freed(PyThreadState *tstate) +{ + if (_PyMem_IsPtrFreed(tstate)) { + return 1; + } + if (_PyMem_IsPtrFreed(tstate->interp)) { + return 1; + } + return 0; +} + + +static int +interp_is_freed(PyInterpreterState *interp) +{ + return _PyMem_IsPtrFreed(interp); +} + + static void dump_traceback(int fd, PyThreadState *tstate, int write_header) { - _PyInterpreterFrame *frame; - unsigned int depth; - if (write_header) { PUTS(fd, "Stack (most recent call first):\n"); } - frame = tstate->current_frame; + if (tstate_is_freed(tstate)) { + PUTS(fd, " \n"); + return; + } + + _PyInterpreterFrame *frame = tstate->current_frame; if (frame == NULL) { PUTS(fd, " \n"); return; } - depth = 0; + unsigned int depth = 0; while (1) { if (MAX_FRAME_DEPTH <= depth) { PUTS(fd, " ...\n"); @@ -1295,9 +1317,6 @@ const char* _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate) { - PyThreadState *tstate; - unsigned int nthreads; - if (current_tstate == NULL) { /* _Py_DumpTracebackThreads() is called from signal handlers by faulthandler. @@ -1313,6 +1332,10 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, current_tstate = PyGILState_GetThisThreadState(); } + if (current_tstate != NULL && tstate_is_freed(current_tstate)) { + return "tstate is freed"; + } + if (interp == NULL) { if (current_tstate == NULL) { interp = _PyGILState_GetInterpreterStateUnsafe(); @@ -1327,14 +1350,18 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, } assert(interp != NULL); + if (interp_is_freed(interp)) { + return "interp is freed"; + } + /* Get the current interpreter from the current thread */ - tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); if (tstate == NULL) return "unable to get the thread head state"; /* Dump the traceback of each thread */ tstate = PyInterpreterState_ThreadHead(interp); - nthreads = 0; + unsigned int nthreads = 0; _Py_BEGIN_SUPPRESS_IPH do { From 7dc2c5093ef027aab57bca953ac2d6477a4a440b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 28 Sep 2023 19:08:04 -0700 Subject: [PATCH 063/124] gh-110045: Update symtable module for PEP 695 (#110066) --- Doc/library/symtable.rst | 12 ++++++- Lib/symtable.py | 15 ++++++--- Lib/test/test_symtable.py | 31 +++++++++++++++++++ ...-09-28-18-08-02.gh-issue-110045.0YIGKv.rst | 2 ++ Modules/symtablemodule.c | 8 +++++ 5 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-18-08-02.gh-issue-110045.0YIGKv.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 65ff5bfe7abd61..85eae5f3822575 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -38,7 +38,13 @@ Examining Symbol Tables .. method:: get_type() Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, and ``'function'``. + ``'module'``, ``'function'``, ``'annotation'``, ``'TypeVar bound'``, + ``'type alias'``, and ``'type parameter'``. The latter four refer to + different flavors of :ref:`annotation scopes `. + + .. versionchanged:: 3.12 + Added ``'annotation'``, ``'TypeVar bound'``, ``'type alias'``, + and ``'type parameter'`` as possible return values. .. method:: get_id() @@ -49,6 +55,10 @@ Examining Symbol Tables Return the table's name. This is the name of the class if the table is for a class, the name of the function if the table is for a function, or ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``). + For type parameter scopes (which are used for generic classes, functions, + and type aliases), it is the name of the underlying class, function, or + type alias. For type alias scopes, it is the name of the type alias. + For :class:`~typing.TypeVar` bound scopes, it is the name of the ``TypeVar``. .. method:: get_lineno() diff --git a/Lib/symtable.py b/Lib/symtable.py index 5dd71ffc6b4f19..4b0bc6f497a553 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -62,8 +62,8 @@ def __repr__(self): def get_type(self): """Return the type of the symbol table. - The values returned are 'class', 'module' and - 'function'. + The values returned are 'class', 'module', 'function', + 'annotation', 'TypeVar bound', 'type alias', and 'type parameter'. """ if self._table.type == _symtable.TYPE_MODULE: return "module" @@ -71,8 +71,15 @@ def get_type(self): return "function" if self._table.type == _symtable.TYPE_CLASS: return "class" - assert self._table.type in (1, 2, 3), \ - "unexpected type: {0}".format(self._table.type) + if self._table.type == _symtable.TYPE_ANNOTATION: + return "annotation" + if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: + return "TypeVar bound" + if self._table.type == _symtable.TYPE_TYPE_ALIAS: + return "type alias" + if self._table.type == _symtable.TYPE_TYPE_PARAM: + return "type parameter" + assert False, f"unexpected type: {self._table.type}" def get_id(self): """Return an identifier for the table. diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 36cb7b3f242e4c..82c1d7c856a1e5 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -40,6 +40,15 @@ def foo(): def namespace_test(): pass def namespace_test(): pass + +type Alias = int +type GenericAlias[T] = list[T] + +def generic_spam[T](a): + pass + +class GenericMine[T: int]: + pass """ @@ -59,6 +68,14 @@ class SymtableTest(unittest.TestCase): internal = find_block(spam, "internal") other_internal = find_block(spam, "other_internal") foo = find_block(top, "foo") + Alias = find_block(top, "Alias") + GenericAlias = find_block(top, "GenericAlias") + GenericAlias_inner = find_block(GenericAlias, "GenericAlias") + generic_spam = find_block(top, "generic_spam") + generic_spam_inner = find_block(generic_spam, "generic_spam") + GenericMine = find_block(top, "GenericMine") + GenericMine_inner = find_block(GenericMine, "GenericMine") + T = find_block(GenericMine, "T") def test_type(self): self.assertEqual(self.top.get_type(), "module") @@ -66,6 +83,15 @@ def test_type(self): self.assertEqual(self.a_method.get_type(), "function") self.assertEqual(self.spam.get_type(), "function") self.assertEqual(self.internal.get_type(), "function") + self.assertEqual(self.foo.get_type(), "function") + self.assertEqual(self.Alias.get_type(), "type alias") + self.assertEqual(self.GenericAlias.get_type(), "type parameter") + self.assertEqual(self.GenericAlias_inner.get_type(), "type alias") + self.assertEqual(self.generic_spam.get_type(), "type parameter") + self.assertEqual(self.generic_spam_inner.get_type(), "function") + self.assertEqual(self.GenericMine.get_type(), "type parameter") + self.assertEqual(self.GenericMine_inner.get_type(), "class") + self.assertEqual(self.T.get_type(), "TypeVar bound") def test_id(self): self.assertGreater(self.top.get_id(), 0) @@ -73,6 +99,11 @@ def test_id(self): self.assertGreater(self.a_method.get_id(), 0) self.assertGreater(self.spam.get_id(), 0) self.assertGreater(self.internal.get_id(), 0) + self.assertGreater(self.foo.get_id(), 0) + self.assertGreater(self.Alias.get_id(), 0) + self.assertGreater(self.GenericAlias.get_id(), 0) + self.assertGreater(self.generic_spam.get_id(), 0) + self.assertGreater(self.GenericMine.get_id(), 0) def test_optimized(self): self.assertFalse(self.top.is_optimized()) diff --git a/Misc/NEWS.d/next/Library/2023-09-28-18-08-02.gh-issue-110045.0YIGKv.rst b/Misc/NEWS.d/next/Library/2023-09-28-18-08-02.gh-issue-110045.0YIGKv.rst new file mode 100644 index 00000000000000..44a6df1083762f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-18-08-02.gh-issue-110045.0YIGKv.rst @@ -0,0 +1,2 @@ +Update the :mod:`symtable` module to support the new scopes introduced by +:pep:`695`. diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index dba80034d310af..ddc9ac3324356d 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -86,6 +86,14 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_ANNOTATION", AnnotationBlock) < 0) + return -1; + if (PyModule_AddIntConstant(m, "TYPE_TYPE_VAR_BOUND", TypeVarBoundBlock) < 0) + return -1; + if (PyModule_AddIntConstant(m, "TYPE_TYPE_ALIAS", TypeAliasBlock) < 0) + return -1; + if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAM", TypeParamBlock) < 0) + return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, GLOBAL_EXPLICIT) < 0) return -1; From 05079d93e410fca1e41ed32e67c54d63cbd9b35b Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 29 Sep 2023 09:28:01 +0200 Subject: [PATCH 064/124] gh-109868: Skip deepcopy memo check for empty memo (GH-109869) --- Lib/copy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/copy.py b/Lib/copy.py index 6d7bb9a111b5b4..a69bc4e78c20b3 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -121,13 +121,13 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + d = id(x) if memo is None: memo = {} - - d = id(x) - y = memo.get(d, _nil) - if y is not _nil: - return y + else: + y = memo.get(d, _nil) + if y is not _nil: + return y cls = type(x) From d102d39bbe175f179f28e4d4bea99dc122da5f8e Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 29 Sep 2023 11:03:59 +0300 Subject: [PATCH 065/124] gh-101100: Fix sphinx warnings in `library/difflib.rst` (#110074) --- Doc/library/difflib.rst | 18 +++++++++--------- Doc/tools/.nitignore | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 5ee1f4a02c6816..c553611401d018 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -570,8 +570,8 @@ The :class:`SequenceMatcher` class has this constructor: The three methods that return the ratio of matching to total characters can give different results due to differing levels of approximation, although -:meth:`quick_ratio` and :meth:`real_quick_ratio` are always at least as large as -:meth:`ratio`: +:meth:`~SequenceMatcher.quick_ratio` and :meth:`~SequenceMatcher.real_quick_ratio` +are always at least as large as :meth:`~SequenceMatcher.ratio`: >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() @@ -593,15 +593,15 @@ This example compares two strings, considering blanks to be "junk": ... "private Thread currentThread;", ... "private volatile Thread currentThread;") -:meth:`ratio` returns a float in [0, 1], measuring the similarity of the -sequences. As a rule of thumb, a :meth:`ratio` value over 0.6 means the +:meth:`~SequenceMatcher.ratio` returns a float in [0, 1], measuring the similarity of the +sequences. As a rule of thumb, a :meth:`~SequenceMatcher.ratio` value over 0.6 means the sequences are close matches: >>> print(round(s.ratio(), 3)) 0.866 If you're only interested in where the sequences match, -:meth:`get_matching_blocks` is handy: +:meth:`~SequenceMatcher.get_matching_blocks` is handy: >>> for block in s.get_matching_blocks(): ... print("a[%d] and b[%d] match for %d elements" % block) @@ -609,12 +609,12 @@ If you're only interested in where the sequences match, a[8] and b[17] match for 21 elements a[29] and b[38] match for 0 elements -Note that the last tuple returned by :meth:`get_matching_blocks` is always a -dummy, ``(len(a), len(b), 0)``, and this is the only case in which the last +Note that the last tuple returned by :meth:`~SequenceMatcher.get_matching_blocks` +is always a dummy, ``(len(a), len(b), 0)``, and this is the only case in which the last tuple element (number of elements matched) is ``0``. If you want to know how to change the first sequence into the second, use -:meth:`get_opcodes`: +:meth:`~SequenceMatcher.get_opcodes`: >>> for opcode in s.get_opcodes(): ... print("%6s a[%d:%d] b[%d:%d]" % opcode) @@ -689,7 +689,7 @@ Differ Example This example compares two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be -obtained from the :meth:`~io.BaseIO.readlines` method of file-like objects): +obtained from the :meth:`~io.IOBase.readlines` method of file-like objects): >>> text1 = ''' 1. Beautiful is better than ugly. ... 2. Explicit is better than implicit. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 2478f305884162..e6ababf0066c56 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -52,7 +52,6 @@ Doc/library/csv.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst -Doc/library/difflib.rst Doc/library/doctest.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst From f1b1680a72cc7eb650f785b9438a3900ca6f695e Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 29 Sep 2023 11:26:55 +0300 Subject: [PATCH 066/124] gh-109961: Use proper `module` for `copy` method docs (#110027) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/copy.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 811b2c51dd3d52..74333b2e934814 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -87,6 +87,8 @@ pickle functions from the :mod:`copyreg` module. single: __copy__() (copy protocol) single: __deepcopy__() (copy protocol) +.. currentmodule:: None + In order for a class to define its own copy implementation, it can define special methods :meth:`~object.__copy__` and :meth:`~object.__deepcopy__`. @@ -101,7 +103,7 @@ special methods :meth:`~object.__copy__` and :meth:`~object.__deepcopy__`. Called to implement the deep copy operation; it is passed one argument, the *memo* dictionary. If the ``__deepcopy__`` implementation needs - to make a deep copy of a component, it should call the :func:`deepcopy` function + to make a deep copy of a component, it should call the :func:`~copy.deepcopy` function with the component as first argument and the *memo* dictionary as second argument. The *memo* dictionary should be treated as an opaque object. @@ -109,7 +111,8 @@ special methods :meth:`~object.__copy__` and :meth:`~object.__deepcopy__`. .. index:: single: __replace__() (replace protocol) -Function :func:`replace` is more limited than :func:`copy` and :func:`deepcopy`, +Function :func:`!copy.replace` is more limited +than :func:`~copy.copy` and :func:`~copy.deepcopy`, and only supports named tuples created by :func:`~collections.namedtuple`, :mod:`dataclasses`, and other classes which define method :meth:`~object.__replace__`. From 8898a8683b5631c24d51a6a7babf55a255874950 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Fri, 29 Sep 2023 10:27:43 +0200 Subject: [PATCH 067/124] gh-101100: Fix Sphinx warnings in `tutorial/controlflow.rst` (#109424) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/tools/.nitignore | 1 - Doc/tutorial/controlflow.rst | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index e6ababf0066c56..f217da9052ca78 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -146,7 +146,6 @@ Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst -Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst Doc/using/cmdline.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 4bcc3768111ccd..aa9caa101da40a 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -534,7 +534,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`~list.append` shown in the example is defined for list objects; it + The method :meth:`!append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. @@ -1046,7 +1046,7 @@ Function Annotations information about the types used by user-defined functions (see :pep:`3107` and :pep:`484` for more information). -:term:`Annotations ` are stored in the :attr:`__annotations__` +:term:`Annotations ` are stored in the :attr:`!__annotations__` attribute of the function as a dictionary and have no effect on any other part of the function. Parameter annotations are defined by a colon after the parameter name, followed by an expression evaluating to the value of the annotation. Return annotations are From bfd94ab9e9f4055ecedaa500b46b0270da9ffe12 Mon Sep 17 00:00:00 2001 From: Yuki K Date: Fri, 29 Sep 2023 17:35:29 +0900 Subject: [PATCH 068/124] gh-101100: Fix references to ``URLError`` and ``HTTPError`` in ``howto/urllib2.rst`` (#107966) Co-authored-by: Hugo van Kemenade --- Doc/howto/urllib2.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 86137fb38c9b93..570435d48866d3 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -194,11 +194,11 @@ which comes after we have a look at what happens when things go wrong. Handling Exceptions =================== -*urlopen* raises :exc:`URLError` when it cannot handle a response (though as +*urlopen* raises :exc:`~urllib.error.URLError` when it cannot handle a response (though as usual with Python APIs, built-in exceptions such as :exc:`ValueError`, :exc:`TypeError` etc. may also be raised). -:exc:`HTTPError` is the subclass of :exc:`URLError` raised in the specific case of +:exc:`~urllib.error.HTTPError` is the subclass of :exc:`~urllib.error.URLError` raised in the specific case of HTTP URLs. The exception classes are exported from the :mod:`urllib.error` module. @@ -229,12 +229,12 @@ the status code indicates that the server is unable to fulfil the request. The default handlers will handle some of these responses for you (for example, if the response is a "redirection" that requests the client fetch the document from a different URL, urllib will handle that for you). For those it can't handle, -urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not +urlopen will raise an :exc:`~urllib.error.HTTPError`. Typical errors include '404' (page not found), '403' (request forbidden), and '401' (authentication required). See section 10 of :rfc:`2616` for a reference on all the HTTP error codes. -The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which +The :exc:`~urllib.error.HTTPError` instance raised will have an integer 'code' attribute, which corresponds to the error sent by the server. Error Codes @@ -317,7 +317,7 @@ dictionary is reproduced here for convenience :: } When an error is raised the server responds by returning an HTTP error code -*and* an error page. You can use the :exc:`HTTPError` instance as a response on the +*and* an error page. You can use the :exc:`~urllib.error.HTTPError` instance as a response on the page returned. This means that as well as the code attribute, it also has read, geturl, and info, methods as returned by the ``urllib.response`` module:: @@ -338,7 +338,7 @@ geturl, and info, methods as returned by the ``urllib.response`` module:: Wrapping it Up -------------- -So if you want to be prepared for :exc:`HTTPError` *or* :exc:`URLError` there are two +So if you want to be prepared for :exc:`~urllib.error.HTTPError` *or* :exc:`~urllib.error.URLError` there are two basic approaches. I prefer the second approach. Number 1 @@ -365,7 +365,7 @@ Number 1 .. note:: The ``except HTTPError`` *must* come first, otherwise ``except URLError`` - will *also* catch an :exc:`HTTPError`. + will *also* catch an :exc:`~urllib.error.HTTPError`. Number 2 ~~~~~~~~ @@ -391,7 +391,7 @@ Number 2 info and geturl =============== -The response returned by urlopen (or the :exc:`HTTPError` instance) has two +The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two useful methods :meth:`info` and :meth:`geturl` and is defined in the module :mod:`urllib.response`.. From 8b626a47bafdb2d1ebb1321e50ffa5d6c721bf3a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 10:56:49 +0200 Subject: [PATCH 069/124] gh-110079: Remove extern "C" { ...} in C code (#110080) --- Modules/_scproxy.c | 8 -------- Modules/_stat.c | 8 -------- Modules/main.c | 8 -------- Modules/posixmodule.c | 8 -------- Objects/fileobject.c | 9 --------- Objects/object.c | 8 -------- Objects/unicodeobject.c | 10 ---------- Python/dtoa.c | 7 ------- Python/errors.c | 9 --------- Python/getargs.c | 8 -------- Python/getopt.c | 9 --------- Python/import.c | 8 -------- Python/pathconfig.c | 9 --------- Python/pyhash.c | 8 -------- Python/pylifecycle.c | 9 --------- Python/pystate.c | 21 ++++++--------------- Python/pythonrun.c | 9 --------- Python/sysmodule.c | 18 ------------------ 18 files changed, 6 insertions(+), 168 deletions(-) diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 0df0324df55f7d..6cc09088bdc869 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -249,16 +249,8 @@ static struct PyModuleDef _scproxy_module = { .m_slots = _scproxy_slots, }; -#ifdef __cplusplus -extern "C" { -#endif - PyMODINIT_FUNC PyInit__scproxy(void) { return PyModuleDef_Init(&_scproxy_module); } - -#ifdef __cplusplus -} -#endif diff --git a/Modules/_stat.c b/Modules/_stat.c index 6cea26175dee5e..3fd951b6fc1022 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -13,10 +13,6 @@ #include "Python.h" -#ifdef __cplusplus -extern "C" { -#endif - #ifdef HAVE_SYS_TYPES_H #include #endif /* HAVE_SYS_TYPES_H */ @@ -631,7 +627,3 @@ PyInit__stat(void) { return PyModuleDef_Init(&statmodule); } - -#ifdef __cplusplus -} -#endif diff --git a/Modules/main.c b/Modules/main.c index 7f88c97207475b..05bedff050699f 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -26,10 +26,6 @@ "Type \"help\", \"copyright\", \"credits\" or \"license\" " \ "for more information." -#ifdef __cplusplus -extern "C" { -#endif - /* --- pymain_init() ---------------------------------------------- */ static PyStatus @@ -742,7 +738,3 @@ Py_BytesMain(int argc, char **argv) .wchar_argv = NULL}; return pymain_main(&args); } - -#ifdef __cplusplus -} -#endif diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index abf449e25493fa..d7d3e365d2c553 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -222,10 +222,6 @@ #endif -#ifdef __cplusplus -extern "C" { -#endif - PyDoc_STRVAR(posix__doc__, "This module provides access to operating system functionality that is\n\ standardized by the C Standard and the POSIX standard (a thinly\n\ @@ -17002,7 +16998,3 @@ INITFUNC(void) { return PyModuleDef_Init(&posixmodule); } - -#ifdef __cplusplus -} -#endif diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 0cf2b47c3b3ae7..066172baf9f027 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -21,10 +21,6 @@ #define NEWLINE_LF 2 /* \n newline seen */ #define NEWLINE_CRLF 4 /* \r\n newline seen */ -#ifdef __cplusplus -extern "C" { -#endif - /* External C interface */ PyObject * @@ -539,8 +535,3 @@ _PyFile_Flush(PyObject *file) Py_DECREF(tmp); return 0; } - - -#ifdef __cplusplus -} -#endif diff --git a/Objects/object.c b/Objects/object.c index 15c2bf65de6acf..3ed272afdced7c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -28,10 +28,6 @@ # error "Py_LIMITED_API macro must not be defined" #endif -#ifdef __cplusplus -extern "C" { -#endif - /* Defined in tracemalloc.c */ extern void _PyMem_DumpTraceback(int fd, const void *ptr); @@ -2808,7 +2804,3 @@ int Py_IsFalse(PyObject *x) { return Py_Is(x, Py_False); } - -#ifdef __cplusplus -} -#endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index aca28e4842d645..49981a1f881c21 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -100,11 +100,6 @@ NOTE: In the interpreter's initialization phase, some globals are currently */ - -#ifdef __cplusplus -extern "C" { -#endif - // Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). // The value must be the same in fileutils.c. #define MAX_UNICODE 0x10ffff @@ -15397,8 +15392,3 @@ PyInit__string(void) { return PyModuleDef_Init(&_string_module); } - - -#ifdef __cplusplus -} -#endif diff --git a/Python/dtoa.c b/Python/dtoa.c index c5e343b82f74c5..5dfc0e179cbc34 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -172,10 +172,6 @@ typedef uint64_t ULLong; #define Bug(x) {fprintf(stderr, "%s\n", x); exit(1);} #endif -#ifdef __cplusplus -extern "C" { -#endif - typedef union { double d; ULong L[2]; } U; #ifdef IEEE_8087 @@ -2813,8 +2809,5 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, _Py_dg_freedtoa(s0); return NULL; } -#ifdef __cplusplus -} -#endif #endif // _PY_SHORT_FLOAT_REPR == 1 diff --git a/Python/errors.c b/Python/errors.c index b05b3ef1dda8fe..15af39b10dc07e 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -16,11 +16,6 @@ # include // _sys_nerr #endif - -#ifdef __cplusplus -extern "C" { -#endif - /* Forward declarations */ static PyObject * _PyErr_FormatV(PyThreadState *tstate, PyObject *exception, @@ -1918,7 +1913,3 @@ PyErr_ProgramTextObject(PyObject *filename, int lineno) { return _PyErr_ProgramDecodedTextObject(filename, lineno, NULL); } - -#ifdef __cplusplus -} -#endif diff --git a/Python/getargs.c b/Python/getargs.c index cbfe561111176c..d590e2e153389e 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -7,10 +7,6 @@ #include "pycore_pylifecycle.h" // _PyArg_Fini #include "pycore_tuple.h" // _PyTuple_ITEMS() -#ifdef __cplusplus -extern "C" { -#endif - /* Export Stable ABIs (abi only) */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, const char *, ...); @@ -2867,7 +2863,3 @@ _PyArg_Fini(void) } _PyRuntime.getargs.static_parsers = NULL; } - -#ifdef __cplusplus -}; -#endif diff --git a/Python/getopt.c b/Python/getopt.c index 4135bf1446ecfc..f64c89fa22734a 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -29,10 +29,6 @@ #include #include "pycore_getopt.h" -#ifdef __cplusplus -extern "C" { -#endif - int _PyOS_opterr = 1; /* generate error messages */ Py_ssize_t _PyOS_optind = 1; /* index into argv array */ const wchar_t *_PyOS_optarg = NULL; /* optional argument */ @@ -172,8 +168,3 @@ int _PyOS_GetOpt(Py_ssize_t argc, wchar_t * const *argv, int *longindex) return option; } - -#ifdef __cplusplus -} -#endif - diff --git a/Python/import.c b/Python/import.c index 5a06cb367e828b..5636968ed9e63b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -24,9 +24,6 @@ #ifdef HAVE_FCNTL_H #include #endif -#ifdef __cplusplus -extern "C" { -#endif /*[clinic input] @@ -3887,8 +3884,3 @@ PyInit__imp(void) { return PyModuleDef_Init(&imp_module); } - - -#ifdef __cplusplus -} -#endif diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 0ac64ec8110259..50c60093cd4e32 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -16,10 +16,6 @@ # include #endif -#ifdef __cplusplus -extern "C" { -#endif - /* External interface */ @@ -500,8 +496,3 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) *path0_p = path0_obj; return 1; } - - -#ifdef __cplusplus -} -#endif diff --git a/Python/pyhash.c b/Python/pyhash.c index b2bdab5099d86a..f9060b8003a0a7 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -14,10 +14,6 @@ # include #endif -#ifdef __cplusplus -extern "C" { -#endif - _Py_HashSecret_t _Py_HashSecret = {{0}}; #if Py_HASH_ALGORITHM == Py_HASH_EXTERNAL @@ -503,7 +499,3 @@ pysiphash(const void *src, Py_ssize_t src_sz) { static PyHash_FuncDef PyHash_Func = {pysiphash, "siphash24", 64, 128}; #endif - -#ifdef __cplusplus -} -#endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index aec8da10249d21..23f66ec3601df6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -61,11 +61,6 @@ #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) -#ifdef __cplusplus -extern "C" { -#endif - - /* Forward declarations */ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); @@ -3139,7 +3134,3 @@ PyOS_setsig(int sig, PyOS_sighandler_t handler) return oldhandler; #endif } - -#ifdef __cplusplus -} -#endif diff --git a/Python/pystate.c b/Python/pystate.c index 570b5242600c0c..01aa2552e56f0d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -29,16 +29,12 @@ to avoid the expense of doing their own locking). -------------------------------------------------------------------------- */ #ifdef HAVE_DLOPEN -#ifdef HAVE_DLFCN_H -#include -#endif -#if !HAVE_DECL_RTLD_LAZY -#define RTLD_LAZY 1 -#endif -#endif - -#ifdef __cplusplus -extern "C" { +# ifdef HAVE_DLFCN_H +# include +# endif +# if !HAVE_DECL_RTLD_LAZY +# define RTLD_LAZY 1 +# endif #endif @@ -2985,8 +2981,3 @@ _PyThreadState_MustExit(PyThreadState *tstate) } return 1; } - - -#ifdef __cplusplus -} -#endif diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 81ab78e95ab68c..1b282aa870bfd2 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -37,11 +37,6 @@ # include "windows.h" #endif - -#ifdef __cplusplus -extern "C" { -#endif - /* Forward */ static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, @@ -2017,7 +2012,3 @@ PyRun_InteractiveLoop(FILE *f, const char *p) { return PyRun_InteractiveLoopFlags(f, p, NULL); } - -#ifdef __cplusplus -} -#endif diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 9c1ee0215d7cf6..7ba7be10aacb92 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2039,11 +2039,6 @@ sys_call_tracing_impl(PyObject *module, PyObject *func, PyObject *funcargs) return _PyEval_CallTracing(func, funcargs); } - -#ifdef __cplusplus -extern "C" { -#endif - /*[clinic input] sys._debugmallocstats @@ -2072,10 +2067,6 @@ sys__debugmallocstats_impl(PyObject *module) extern PyObject *_Py_GetObjects(PyObject *, PyObject *); #endif -#ifdef __cplusplus -} -#endif - /*[clinic input] sys._clear_type_cache @@ -2297,11 +2288,6 @@ sys__getframemodulename_impl(PyObject *module, int depth) return Py_NewRef(r); } - -#ifdef __cplusplus -extern "C" { -#endif - static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2370,10 +2356,6 @@ PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void) { #endif } -#ifdef __cplusplus -} -#endif - static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ From e260087a8e7f1a564f9b797af6291f99e225a73c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 12:17:49 +0200 Subject: [PATCH 070/124] gh-108716: make regen-global-objects no longer builds deepfreeze.c (#110078) Remove more references to now unused Python/deepfreeze/deepfreeze.c. --- .github/workflows/build.yml | 3 --- Makefile.pre.in | 10 ++-------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a60632dc565235..ffcfbac290b726 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,9 +140,6 @@ jobs: run: make regen-configure - name: Build CPython run: | - # Deepfreeze will usually cause global objects to be added or removed, - # so we run it before regen-global-objects gets rum (in regen-all). - make regen-deepfreeze make -j4 regen-all make regen-stdlib-module-names - name: Check for changes diff --git a/Makefile.pre.in b/Makefile.pre.in index f7b52bdab316f1..d62b4d24b3e183 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -492,9 +492,6 @@ OBJECT_OBJS= \ Objects/weakrefobject.o \ @PERF_TRAMPOLINE_OBJ@ -DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c -DEEPFREEZE_OBJS = Python/deepfreeze/deepfreeze.o - ########################################################################## # objects that get linked into the Python library LIBRARY_OBJS_OMIT_FROZEN= \ @@ -1252,9 +1249,7 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) ############################################################################ # Deepfreeze targets -.PHONY: regen-deepfreeze -regen-deepfreeze: $(DEEPFREEZE_C) - +DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) # BEGIN: deepfreeze modules @@ -1294,10 +1289,9 @@ regen-importlib: regen-frozen # Global objects # Dependencies which can add and/or remove _Py_ID() identifiers: -# - deepfreeze.c # - "make clinic" .PHONY: regen-global-objects -regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py $(DEEPFREEZE_C) clinic +regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py clinic $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py ############################################################################ From e27adc68ccee8345e05b7516e6b46f6c7ff53371 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Fri, 29 Sep 2023 06:21:34 -0500 Subject: [PATCH 071/124] gh-109634: Fix `:samp:` syntax (GH-110073) --- Doc/library/codecs.rst | 2 +- Doc/reference/lexical_analysis.rst | 2 +- Misc/NEWS.d/3.8.0a1.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 053bf64addb821..2db4a67d1973d5 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1350,7 +1350,7 @@ encodings. +--------------------+---------+---------------------------+ | raw_unicode_escape | | Latin-1 encoding with | | | | :samp:`\\u{XXXX}` and | -| | | :samp:`\\U{XXXXXXXX}`` | +| | | :samp:`\\U{XXXXXXXX}` | | | | for other code points. | | | | Existing | | | | backslashes are not | diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 9fd80b1cb7f84c..e54e0ebb7fae96 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -582,7 +582,7 @@ Standard C. The recognized escape sequences are: +-------------------------+---------------------------------+-------+ | ``\v`` | ASCII Vertical Tab (VT) | | +-------------------------+---------------------------------+-------+ -| :samp:`\\{ooo}` | Character with octal value | (2,4) | +| :samp:`\\\\{ooo}` | Character with octal value | (2,4) | | | *ooo* | | +-------------------------+---------------------------------+-------+ | :samp:`\\x{hh}` | Character with hex value *hh* | (3,4) | diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 57f72e95b029fc..3cbbbf7465032b 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -8253,7 +8253,7 @@ Explain how IDLE's Shell displays output. Improve the doc about IDLE running user code. The section is renamed from "IDLE -- console differences" is renamed "Running user code". It mostly -covers the implications of using custom :samp:sys.std{xxx}` objects. +covers the implications of using custom :samp:`sys.std{xxx}` objects. .. From db0a258e796703e12befea9d6dec04e349ca2f5b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 13:49:30 +0200 Subject: [PATCH 072/124] gh-110088, gh-109878: Fix test_asyncio timeouts (#110092) Fix test_asyncio timeouts: don't measure the maximum duration, a test should not measure a CI performance. Only measure the minimum duration when a task has a timeout or delay. Add CLOCK_RES to test_asyncio.utils. --- Lib/test/test_asyncio/test_base_events.py | 7 ++----- Lib/test/test_asyncio/test_events.py | 8 ++++---- Lib/test/test_asyncio/test_timeouts.py | 20 ------------------- Lib/test/test_asyncio/test_waitfor.py | 15 -------------- Lib/test/test_asyncio/test_windows_events.py | 13 +++--------- Lib/test/test_asyncio/utils.py | 6 ++++++ ...-09-29-12-48-42.gh-issue-110088.qUhRga.rst | 4 ++++ 7 files changed, 19 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-29-12-48-42.gh-issue-110088.qUhRga.rst diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 3b4026cb73869a..abcb6f55c4b04e 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -273,7 +273,7 @@ def cb(): self.loop.stop() self.loop._process_events = mock.Mock() - delay = 0.1 + delay = 0.100 when = self.loop.time() + delay self.loop.call_at(when, cb) @@ -282,10 +282,7 @@ def cb(): dt = self.loop.time() - t0 # 50 ms: maximum granularity of the event loop - self.assertGreaterEqual(dt, delay - 0.050, dt) - # tolerate a difference of +800 ms because some Python buildbots - # are really slow - self.assertLessEqual(dt, 0.9, dt) + self.assertGreaterEqual(dt, delay - test_utils.CLOCK_RES) with self.assertRaises(TypeError, msg="when cannot be None"): self.loop.call_at(None, cb) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index f22cb5e58bba62..3ee6565b2b65ad 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -293,10 +293,11 @@ async def coro2(): # 15.6 msec, we use fairly long sleep times here (~100 msec). def test_run_until_complete(self): + delay = 0.100 t0 = self.loop.time() - self.loop.run_until_complete(asyncio.sleep(0.1)) - t1 = self.loop.time() - self.assertTrue(0.08 <= t1-t0 <= 0.8, t1-t0) + self.loop.run_until_complete(asyncio.sleep(delay)) + dt = self.loop.time() - t0 + self.assertGreaterEqual(dt, delay - test_utils.CLOCK_RES) def test_run_until_complete_stopped(self): @@ -1717,7 +1718,6 @@ def _run_once(): self.loop._run_once = _run_once async def wait(): - loop = self.loop await asyncio.sleep(1e-2) await asyncio.sleep(1e-4) await asyncio.sleep(1e-6) diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 8b6b9a1fea0be8..e9b59b953518b3 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -46,7 +46,6 @@ async def test_nested_timeouts(self): self.assertTrue(cm2.expired()) async def test_waiter_cancelled(self): - loop = asyncio.get_running_loop() cancelled = False with self.assertRaises(TimeoutError): async with asyncio.timeout(0.01): @@ -59,39 +58,26 @@ async def test_waiter_cancelled(self): async def test_timeout_not_called(self): loop = asyncio.get_running_loop() - t0 = loop.time() async with asyncio.timeout(10) as cm: await asyncio.sleep(0.01) t1 = loop.time() self.assertFalse(cm.expired()) - # 2 sec for slow CI boxes - self.assertLess(t1-t0, 2) self.assertGreater(cm.when(), t1) async def test_timeout_disabled(self): - loop = asyncio.get_running_loop() - t0 = loop.time() async with asyncio.timeout(None) as cm: await asyncio.sleep(0.01) - t1 = loop.time() self.assertFalse(cm.expired()) self.assertIsNone(cm.when()) - # 2 sec for slow CI boxes - self.assertLess(t1-t0, 2) async def test_timeout_at_disabled(self): - loop = asyncio.get_running_loop() - t0 = loop.time() async with asyncio.timeout_at(None) as cm: await asyncio.sleep(0.01) - t1 = loop.time() self.assertFalse(cm.expired()) self.assertIsNone(cm.when()) - # 2 sec for slow CI boxes - self.assertLess(t1-t0, 2) async def test_timeout_zero(self): loop = asyncio.get_running_loop() @@ -101,8 +87,6 @@ async def test_timeout_zero(self): await asyncio.sleep(10) t1 = loop.time() self.assertTrue(cm.expired()) - # 2 sec for slow CI boxes - self.assertLess(t1-t0, 2) self.assertTrue(t0 <= cm.when() <= t1) async def test_timeout_zero_sleep_zero(self): @@ -113,8 +97,6 @@ async def test_timeout_zero_sleep_zero(self): await asyncio.sleep(0) t1 = loop.time() self.assertTrue(cm.expired()) - # 2 sec for slow CI boxes - self.assertLess(t1-t0, 2) self.assertTrue(t0 <= cm.when() <= t1) async def test_timeout_in_the_past_sleep_zero(self): @@ -125,8 +107,6 @@ async def test_timeout_in_the_past_sleep_zero(self): await asyncio.sleep(0) t1 = loop.time() self.assertTrue(cm.expired()) - # 2 sec for slow CI boxes - self.assertLess(t1-t0, 2) self.assertTrue(t0 >= cm.when() <= t1) async def test_foreign_exception_passed(self): diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index e714b154c5cadf..d52f32534a0cfe 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -66,17 +66,12 @@ async def test_wait_for_timeout_less_then_0_or_0_future_done(self): fut = loop.create_future() fut.set_result('done') - t0 = loop.time() ret = await asyncio.wait_for(fut, 0) - t1 = loop.time() self.assertEqual(ret, 'done') self.assertTrue(fut.done()) - self.assertLess(t1 - t0, 0.1) async def test_wait_for_timeout_less_then_0_or_0_coroutine_do_not_started(self): - loop = asyncio.get_running_loop() - foo_started = False async def foo(): @@ -84,12 +79,9 @@ async def foo(): foo_started = True with self.assertRaises(asyncio.TimeoutError): - t0 = loop.time() await asyncio.wait_for(foo(), 0) - t1 = loop.time() self.assertEqual(foo_started, False) - self.assertLess(t1 - t0, 0.1) async def test_wait_for_timeout_less_then_0_or_0(self): loop = asyncio.get_running_loop() @@ -113,18 +105,14 @@ async def foo(): await started with self.assertRaises(asyncio.TimeoutError): - t0 = loop.time() await asyncio.wait_for(fut, timeout) - t1 = loop.time() self.assertTrue(fut.done()) # it should have been cancelled due to the timeout self.assertTrue(fut.cancelled()) self.assertEqual(foo_running, False) - self.assertLess(t1 - t0, 0.1) async def test_wait_for(self): - loop = asyncio.get_running_loop() foo_running = None async def foo(): @@ -139,13 +127,10 @@ async def foo(): fut = asyncio.create_task(foo()) with self.assertRaises(asyncio.TimeoutError): - t0 = loop.time() await asyncio.wait_for(fut, 0.1) - t1 = loop.time() self.assertTrue(fut.done()) # it should have been cancelled due to the timeout self.assertTrue(fut.cancelled()) - self.assertLess(t1 - t0, support.SHORT_TIMEOUT) self.assertEqual(foo_running, False) async def test_wait_for_blocking(self): diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index a36119a8004f9d..6e6c90a247b291 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -163,29 +163,25 @@ def test_wait_for_handle(self): # Wait for unset event with 0.5s timeout; # result should be False at timeout - fut = self.loop._proactor.wait_for_handle(event, 0.5) + timeout = 0.5 + fut = self.loop._proactor.wait_for_handle(event, timeout) start = self.loop.time() done = self.loop.run_until_complete(fut) elapsed = self.loop.time() - start self.assertEqual(done, False) self.assertFalse(fut.result()) - # bpo-31008: Tolerate only 450 ms (at least 500 ms expected), - # because of bad clock resolution on Windows - self.assertTrue(0.45 <= elapsed <= 0.9, elapsed) + self.assertGreaterEqual(elapsed, timeout - test_utils.CLOCK_RES) _overlapped.SetEvent(event) # Wait for set event; # result should be True immediately fut = self.loop._proactor.wait_for_handle(event, 10) - start = self.loop.time() done = self.loop.run_until_complete(fut) - elapsed = self.loop.time() - start self.assertEqual(done, True) self.assertTrue(fut.result()) - self.assertTrue(0 <= elapsed < 0.3, elapsed) # asyncio issue #195: cancelling a done _WaitHandleFuture # must not crash @@ -199,11 +195,8 @@ def test_wait_for_handle_cancel(self): # CancelledError should be raised immediately fut = self.loop._proactor.wait_for_handle(event, 10) fut.cancel() - start = self.loop.time() with self.assertRaises(asyncio.CancelledError): self.loop.run_until_complete(fut) - elapsed = self.loop.time() - start - self.assertTrue(0 <= elapsed < 0.1, elapsed) # asyncio issue #195: cancelling a _WaitHandleFuture twice # must not crash diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 1e5ab6eb935ef1..e1101bf42eb24e 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -36,6 +36,12 @@ from test.support import threading_helper +# Use the maximum known clock resolution (gh-75191, gh-110088): Windows +# GetTickCount64() has a resolution of 15.6 ms. Use 20 ms to tolerate rounding +# issues. +CLOCK_RES = 0.020 + + def data_file(*filename): fullname = os.path.join(support.TEST_HOME_DIR, *filename) if os.path.isfile(fullname): diff --git a/Misc/NEWS.d/next/Tests/2023-09-29-12-48-42.gh-issue-110088.qUhRga.rst b/Misc/NEWS.d/next/Tests/2023-09-29-12-48-42.gh-issue-110088.qUhRga.rst new file mode 100644 index 00000000000000..cf44a123c2c925 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-29-12-48-42.gh-issue-110088.qUhRga.rst @@ -0,0 +1,4 @@ +Fix test_asyncio timeouts: don't measure the maximum duration, a test should +not measure a CI performance. Only measure the minimum duration when a task has +a timeout or delay. Add ``CLOCK_RES`` to ``test_asyncio.utils``. Patch by +Victor Stinner. From 501939c9c1433c5b95ca19ae752e5881149f61b9 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 29 Sep 2023 21:18:18 +0900 Subject: [PATCH 073/124] gh-105323: Update readline module to detect apple editline variant (gh-108665) --- Doc/using/configure.rst | 7 ++++--- Modules/readline.c | 4 ++++ configure | 17 +++++++++++++++++ configure.ac | 10 ++++++++++ pyconfig.h.in | 3 +++ 5 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 9403c19a695776..83b4c7aa0481e9 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -758,11 +758,12 @@ Libraries options .. versionadded:: 3.3 -.. cmdoption:: --with-readline=editline +.. cmdoption:: --with-readline=readline|editline - Use ``editline`` library for backend of the :mod:`readline` module. + Designate a backend library for the :mod:`readline` module. - Define the ``WITH_EDITLINE`` macro. + * readline: Use readline as the backend. + * editline: Use editline as the backend. .. versionadded:: 3.10 diff --git a/Modules/readline.c b/Modules/readline.c index aeae654162f13f..4b473023c6e524 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1018,6 +1018,8 @@ on_hook(PyObject *func) static int #if defined(_RL_FUNCTION_TYPEDEF) on_startup_hook(void) +#elif defined(WITH_APPLE_EDITLINE) +on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #else on_startup_hook(void) #endif @@ -1033,6 +1035,8 @@ on_startup_hook(void) static int #if defined(_RL_FUNCTION_TYPEDEF) on_pre_input_hook(void) +#elif defined(WITH_APPLE_EDITLINE) +on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #else on_pre_input_hook(void) #endif diff --git a/configure b/configure index 098def9aab08bf..0e5f3f64c680b2 100755 --- a/configure +++ b/configure @@ -23781,6 +23781,7 @@ fi + # Check whether --with-readline was given. if test ${with_readline+y} then : @@ -23803,6 +23804,22 @@ else $as_nop fi +# gh-105323: Need to handle the macOS editline as an alias of readline. +case $ac_sys_system/$ac_sys_release in #( + Darwin/*) : + ac_fn_c_check_type "$LINENO" "Function" "ac_cv_type_Function" "#include +" +if test "x$ac_cv_type_Function" = xyes +then : + printf "%s\n" "#define WITH_APPLE_EDITLINE 1" >>confdefs.h + +fi + ;; #( + *) : + + ;; +esac + if test "x$with_readline" = xreadline then : diff --git a/configure.ac b/configure.ac index 3e6cbc69c21009..493868130414ee 100644 --- a/configure.ac +++ b/configure.ac @@ -5832,6 +5832,7 @@ dnl library (tinfo ncursesw ncurses termcap). We now assume that libreadline dnl or readline.pc provide correct linker information. AH_TEMPLATE([WITH_EDITLINE], [Define to build the readline module against libedit.]) +AH_TEMPLATE([WITH_APPLE_EDITLINE], [Define to build the readline module against Apple BSD editline.]) AC_ARG_WITH( [readline], @@ -5848,6 +5849,15 @@ AC_ARG_WITH( [with_readline=readline] ) +# gh-105323: Need to handle the macOS editline as an alias of readline. +AS_CASE([$ac_sys_system/$ac_sys_release], + [Darwin/*], [AC_CHECK_TYPE([Function], + [AC_DEFINE([WITH_APPLE_EDITLINE])], + [], + [@%:@include ])], + [] +) + AS_VAR_IF([with_readline], [readline], [ PKG_CHECK_MODULES([LIBREADLINE], [readline], [ LIBREADLINE=readline diff --git a/pyconfig.h.in b/pyconfig.h.in index 86c72cc6b4e62a..c2c75c96dcaad1 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1788,6 +1788,9 @@ /* Define if WINDOW in curses.h offers a field _flags. */ #undef WINDOW_HAS_FLAGS +/* Define to build the readline module against Apple BSD editline. */ +#undef WITH_APPLE_EDITLINE + /* Define if you want build the _decimal module using a coroutine-local rather than a thread-local context */ #undef WITH_DECIMAL_CONTEXTVAR From 743e3572ee940a6cf88fd518e5f4a447905ba5eb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 14:21:18 +0200 Subject: [PATCH 074/124] gh-109974: Fix more threading lock_tests race conditions (#110089) * Add context manager on Bunch class. * Bunch now catchs exceptions on executed functions and re-raise them at __exit__() as an ExceptionGroup. * Rewrite BarrierProxy.test_default_timeout(). Use a single thread. Only check that barrier.wait() blocks for at least default timeout seconds. * test_with(): inline _with() function. --- Lib/test/lock_tests.py | 486 ++++++++++++++------------ Lib/test/test_importlib/test_locks.py | 3 +- 2 files changed, 257 insertions(+), 232 deletions(-) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index cbaae3afd6dde3..024c6debcd4a54 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -39,40 +39,54 @@ def __init__(self, func, nthread, wait_before_exit=False): self.nthread = nthread self.started = [] self.finished = [] + self.exceptions = [] self._can_exit = not wait_before_exit - self.wait_thread = threading_helper.wait_threads_exit() - self.wait_thread.__enter__() + self._wait_thread = None - def task(): - tid = threading.get_ident() - self.started.append(tid) - try: - func() - finally: - self.finished.append(tid) - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if self._can_exit: - break + def task(self): + tid = threading.get_ident() + self.started.append(tid) + try: + self.func() + except BaseException as exc: + self.exceptions.append(exc) + finally: + self.finished.append(tid) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self._can_exit: + break + + def __enter__(self): + self._wait_thread = threading_helper.wait_threads_exit(support.SHORT_TIMEOUT) + self._wait_thread.__enter__() try: - for i in range(nthread): - start_new_thread(task, ()) + for _ in range(self.nthread): + start_new_thread(self.task, ()) except: self._can_exit = True raise - def wait_for_started(self): for _ in support.sleeping_retry(support.SHORT_TIMEOUT): if len(self.started) >= self.nthread: break - def wait_for_finished(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): for _ in support.sleeping_retry(support.SHORT_TIMEOUT): if len(self.finished) >= self.nthread: break - # Wait for threads exit - self.wait_thread.__exit__(None, None, None) + # Wait until threads completely exit according to _thread._count() + self._wait_thread.__exit__(None, None, None) + + # Break reference cycle + exceptions = self.exceptions + self.exceptions = None + if exceptions: + raise ExceptionGroup(f"{self.func} threads raised exceptions", + exceptions) def do_finish(self): self._can_exit = True @@ -143,7 +157,8 @@ def test_try_acquire_contended(self): result = [] def f(): result.append(lock.acquire(False)) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(result[0]) lock.release() @@ -154,33 +169,45 @@ def f(): lock.acquire() lock.release() - # Threads block on lock.acquire() N = 5 - b = Bunch(f, N) - b.wait_for_started() - wait_threads_blocked(N) - self.assertEqual(len(b.finished), 0) + with Bunch(f, N) as bunch: + # Threads block on lock.acquire() + wait_threads_blocked(N) + self.assertEqual(len(bunch.finished), 0) - # Threads unblocked - lock.release() - b.wait_for_finished() - self.assertEqual(len(b.finished), N) + # Threads unblocked + lock.release() + + self.assertEqual(len(bunch.finished), N) def test_with(self): lock = self.locktype() def f(): lock.acquire() lock.release() - def _with(err=None): + + def with_lock(err=None): with lock: if err is not None: raise err - _with() - # Check the lock is unacquired - Bunch(f, 1).wait_for_finished() - self.assertRaises(TypeError, _with, TypeError) - # Check the lock is unacquired - Bunch(f, 1).wait_for_finished() + + # Acquire the lock, do nothing, with releases the lock + with lock: + pass + + # Check that the lock is unacquired + with Bunch(f, 1): + pass + + # Acquire the lock, raise an exception, with releases the lock + with self.assertRaises(TypeError): + with lock: + raise TypeError + + # Check that the lock is unacquired even if after an exception + # was raised in the previous "with lock:" block + with Bunch(f, 1): + pass def test_thread_leak(self): # The lock shouldn't leak a Thread instance when used from a foreign @@ -192,7 +219,8 @@ def f(): # We run many threads in the hope that existing threads ids won't # be recycled. - Bunch(f, 15).wait_for_finished() + with Bunch(f, 15): + pass def test_timeout(self): lock = self.locktype() @@ -216,7 +244,8 @@ def f(): results.append(lock.acquire(timeout=0.5)) t2 = time.monotonic() results.append(t2 - t1) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) @@ -264,8 +293,8 @@ def test_different_thread(self): lock.acquire() def f(): lock.release() - b = Bunch(f, 1) - b.wait_for_finished() + with Bunch(f, 1): + pass lock.acquire() lock.release() @@ -376,12 +405,12 @@ def test_different_thread(self): lock = self.locktype() def f(): lock.acquire() - b = Bunch(f, 1, True) - try: - self.assertRaises(RuntimeError, lock.release) - finally: - b.do_finish() - b.wait_for_finished() + + with Bunch(f, 1, True) as bunch: + try: + self.assertRaises(RuntimeError, lock.release) + finally: + bunch.do_finish() def test__is_owned(self): lock = self.locktype() @@ -393,7 +422,8 @@ def test__is_owned(self): result = [] def f(): result.append(lock._is_owned()) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(result[0]) lock.release() self.assertTrue(lock._is_owned()) @@ -427,15 +457,14 @@ def f(): results1.append(evt.wait()) results2.append(evt.wait()) - # Threads blocked on first evt.wait() - b = Bunch(f, N) - b.wait_for_started() - wait_threads_blocked(N) - self.assertEqual(len(results1), 0) + with Bunch(f, N): + # Threads blocked on first evt.wait() + wait_threads_blocked(N) + self.assertEqual(len(results1), 0) + + # Threads unblocked + evt.set() - # Threads unblocked - evt.set() - b.wait_for_finished() self.assertEqual(results1, [True] * N) self.assertEqual(results2, [True] * N) @@ -458,16 +487,22 @@ def f(): r = evt.wait(0.5) t2 = time.monotonic() results2.append((r, t2 - t1)) - Bunch(f, N).wait_for_finished() + + with Bunch(f, N): + pass + self.assertEqual(results1, [False] * N) for r, dt in results2: self.assertFalse(r) self.assertTimeout(dt, 0.5) + # The event is set results1 = [] results2 = [] evt.set() - Bunch(f, N).wait_for_finished() + with Bunch(f, N): + pass + self.assertEqual(results1, [True] * N) for r, dt in results2: self.assertTrue(r) @@ -480,16 +515,15 @@ def test_set_and_clear(self): def f(): results.append(event.wait(support.LONG_TIMEOUT)) - # Threads blocked on event.wait() N = 5 - b = Bunch(f, N) - b.wait_for_started() - wait_threads_blocked(N) - - # Threads unblocked - event.set() - event.clear() - b.wait_for_finished() + with Bunch(f, N): + # Threads blocked on event.wait() + wait_threads_blocked(N) + + # Threads unblocked + event.set() + event.clear() + self.assertEqual(results, [True] * N) @requires_fork @@ -573,73 +607,71 @@ def f(): results2.append((result, phase_num)) N = 5 - b = Bunch(f, N) - b.wait_for_started() - # first wait, to ensure all workers settle into cond.wait() before - # we continue. See issues #8799 and #30727. - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if len(ready) >= N: - break + with Bunch(f, N): + # first wait, to ensure all workers settle into cond.wait() before + # we continue. See issues #8799 and #30727. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break - ready.clear() - self.assertEqual(results1, []) + ready.clear() + self.assertEqual(results1, []) - # Notify 3 threads at first - count1 = 3 - cond.acquire() - cond.notify(count1) - wait_threads_blocked(count1) + # Notify 3 threads at first + count1 = 3 + cond.acquire() + cond.notify(count1) + wait_threads_blocked(count1) - # Phase 1 - phase_num = 1 - cond.release() - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if len(results1) >= count1: - break + # Phase 1 + phase_num = 1 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) >= count1: + break - self.assertEqual(results1, [(True, 1)] * count1) - self.assertEqual(results2, []) + self.assertEqual(results1, [(True, 1)] * count1) + self.assertEqual(results2, []) - # Wait until awaken workers are blocked on cond.wait() - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if len(ready) >= count1 : - break + # Wait until awaken workers are blocked on cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= count1 : + break - # Notify 5 threads: they might be in their first or second wait - cond.acquire() - cond.notify(5) - wait_threads_blocked(N) + # Notify 5 threads: they might be in their first or second wait + cond.acquire() + cond.notify(5) + wait_threads_blocked(N) - # Phase 2 - phase_num = 2 - cond.release() - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if len(results1) + len(results2) >= (N + count1): - break + # Phase 2 + phase_num = 2 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= (N + count1): + break - count2 = N - count1 - self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) - self.assertEqual(results2, [(True, 2)] * count1) + count2 = N - count1 + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1) - # Make sure all workers settle into cond.wait() - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if len(ready) >= N: - break + # Make sure all workers settle into cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break - # Notify all threads: they are all in their second wait - cond.acquire() - cond.notify_all() - wait_threads_blocked(N) + # Notify all threads: they are all in their second wait + cond.acquire() + cond.notify_all() + wait_threads_blocked(N) - # Phase 3 - phase_num = 3 - cond.release() - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if len(results2) >= N: - break - self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) - self.assertEqual(results2, [(True, 2)] * count1 + [(True, 3)] * count2) - b.wait_for_finished() + # Phase 3 + phase_num = 3 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results2) >= N: + break + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1 + [(True, 3)] * count2) def test_notify(self): cond = self.condtype() @@ -660,7 +692,8 @@ def f(): results.append((t2 - t1, result)) N = 5 - Bunch(f, N).wait_for_finished() + with Bunch(f, N): + pass self.assertEqual(len(results), N) for dt, result in results: @@ -680,14 +713,13 @@ def f(): result = cond.wait_for(lambda: state == 4) self.assertTrue(result) self.assertEqual(state, 4) - b = Bunch(f, 1) - b.wait_for_started() - for i in range(4): - time.sleep(0.010) - with cond: - state += 1 - cond.notify() - b.wait_for_finished() + + with Bunch(f, 1): + for i in range(4): + time.sleep(0.010) + with cond: + state += 1 + cond.notify() def test_waitfor_timeout(self): cond = self.condtype() @@ -702,16 +734,14 @@ def f(): self.assertTimeout(dt, 0.1) success.append(None) - b = Bunch(f, 1) - b.wait_for_started() - # Only increment 3 times, so state == 4 is never reached. - for i in range(3): - time.sleep(0.010) - with cond: - state += 1 - cond.notify() + with Bunch(f, 1): + # Only increment 3 times, so state == 4 is never reached. + for i in range(3): + time.sleep(0.010) + with cond: + state += 1 + cond.notify() - b.wait_for_finished() self.assertEqual(len(success), 1) @@ -761,38 +791,37 @@ def wait_count(count): if len(results1) + len(results2) >= count: break - # Phase 0 N = 10 - b = Bunch(func, N) - b.wait_for_started() - count1 = sem_value - 1 - wait_count(count1) - self.assertEqual(results1 + results2, [0] * count1) - - # Phase 1 - phase_num = 1 - for i in range(sem_value): - sem.release() - count2 = sem_value - wait_count(count1 + count2) - self.assertEqual(sorted(results1 + results2), - [0] * count1 + [1] * count2) - - # Phase 2 - phase_num = 2 - count3 = (sem_value - 1) - for i in range(count3): + with Bunch(func, N): + # Phase 0 + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 + phase_num = 1 + for i in range(sem_value): + sem.release() + count2 = sem_value + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 + phase_num = 2 + count3 = (sem_value - 1) + for i in range(count3): + sem.release() + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + + # Final release, to let the last thread finish + count4 = 1 sem.release() - wait_count(count1 + count2 + count3) - self.assertEqual(sorted(results1 + results2), - [0] * count1 + [1] * count2 + [2] * count3) - # The semaphore is still locked - self.assertFalse(sem.acquire(False)) - # Final release, to let the last thread finish - count4 = 1 - sem.release() - b.wait_for_finished() self.assertEqual(sem_results, [True] * (count1 + count2 + count3 + count4)) @@ -816,34 +845,32 @@ def wait_count(count): if len(results1) + len(results2) >= count: break - # Phase 0 - b = Bunch(func, 10) - b.wait_for_started() - count1 = sem_value - 1 - wait_count(count1) - self.assertEqual(results1 + results2, [0] * count1) - - # Phase 1 - phase_num = 1 - count2 = sem_value - sem.release(count2) - wait_count(count1 + count2) - self.assertEqual(sorted(results1 + results2), - [0] * count1 + [1] * count2) - - # Phase 2 - phase_num = 2 - count3 = sem_value - 1 - sem.release(count3) - wait_count(count1 + count2 + count3) - self.assertEqual(sorted(results1 + results2), - [0] * count1 + [1] * count2 + [2] * count3) - # The semaphore is still locked - self.assertFalse(sem.acquire(False)) - - # Final release, to let the last thread finish - sem.release() - b.wait_for_finished() + with Bunch(func, 10): + # Phase 0 + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 + phase_num = 1 + count2 = sem_value + sem.release(count2) + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 + phase_num = 2 + count3 = sem_value - 1 + sem.release(count3) + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + + # Final release, to let the last thread finish + sem.release() def test_try_acquire(self): sem = self.semtype(2) @@ -860,7 +887,8 @@ def test_try_acquire_contended(self): def f(): results.append(sem.acquire(False)) results.append(sem.acquire(False)) - Bunch(f, 5).wait_for_finished() + with Bunch(f, 5): + pass # There can be a thread switch between acquiring the semaphore and # appending the result, therefore results will not necessarily be # ordered. @@ -887,15 +915,13 @@ def f(): sem.acquire() sem.release() - # Thread blocked on sem.acquire() - b = Bunch(f, 1) - b.wait_for_started() - wait_threads_blocked(1) - self.assertFalse(b.finished) + with Bunch(f, 1) as bunch: + # Thread blocked on sem.acquire() + wait_threads_blocked(1) + self.assertFalse(bunch.finished) - # Thread unblocked - sem.release() - b.wait_for_finished() + # Thread unblocked + sem.release() def test_with(self): sem = self.semtype(2) @@ -971,9 +997,8 @@ def tearDown(self): self.barrier.abort() def run_threads(self, f): - b = Bunch(f, self.N-1) - f() - b.wait_for_finished() + with Bunch(f, self.N): + pass def multipass(self, results, n): m = self.barrier.parties @@ -1126,27 +1151,27 @@ def f(): i = self.barrier.wait() if i == self.N // 2: # One thread is late! - time.sleep(1.0) + time.sleep(self.defaultTimeout / 2) # Default timeout is 2.0, so this is shorter. self.assertRaises(threading.BrokenBarrierError, - self.barrier.wait, 0.5) + self.barrier.wait, self.defaultTimeout / 4) self.run_threads(f) def test_default_timeout(self): """ Test the barrier's default timeout """ - # gh-109401: Barrier timeout should be long enough - # to create 4 threads on a slow CI. - timeout = 1.0 - barrier = self.barriertype(self.N, timeout=timeout) + timeout = 0.100 + barrier = self.barriertype(2, timeout=timeout) def f(): - i = barrier.wait() - if i == self.N // 2: - # One thread is later than the default timeout. - time.sleep(timeout * 2) - self.assertRaises(threading.BrokenBarrierError, barrier.wait) - self.run_threads(f) + self.assertRaises(threading.BrokenBarrierError, + barrier.wait) + + start_time = time.monotonic() + with Bunch(f, 1): + pass + dt = time.monotonic() - start_time + self.assertGreaterEqual(dt, timeout) def test_single_thread(self): b = self.barriertype(1) @@ -1160,19 +1185,18 @@ def test_repr(self): def f(): barrier.wait(timeout) - # Threads blocked on barrier.wait() N = 2 - bunch = Bunch(f, N) - bunch.wait_for_started() - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if barrier.n_waiting >= N: - break - self.assertRegex(repr(barrier), - r"<\w+\.Barrier at .*: waiters=2/3>") + with Bunch(f, N): + # Threads blocked on barrier.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if barrier.n_waiting >= N: + break + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=2/3>") + + # Threads unblocked + barrier.wait(timeout) - # Threads unblocked - barrier.wait(timeout) - bunch.wait_for_finished() self.assertRegex(repr(barrier), r"<\w+\.Barrier at .*: waiters=0/3>") diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 7091c36aaaf761..befac5d62b0abf 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -93,7 +93,8 @@ def f(): b.release() if ra: a.release() - lock_tests.Bunch(f, NTHREADS).wait_for_finished() + with lock_tests.Bunch(f, NTHREADS): + pass self.assertEqual(len(results), NTHREADS) return results From 86e76ab8af9a5018acbcdcbb6285678175b1bd8a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 14:41:33 +0200 Subject: [PATCH 075/124] gh-110031: Skip test_threading fork tests if ASAN (#110100) Skip test_threading tests using thread+fork if Python is built with Address Sanitizer (ASAN). --- Lib/test/test_threading.py | 41 +++++++++++-------- ...-09-29-14-11-30.gh-issue-110031.fQnFnc.rst | 2 + 2 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-29-14-11-30.gh-issue-110031.fQnFnc.rst diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 71fcad268b8036..13bfacbac83f13 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -35,6 +35,23 @@ platforms_to_skip = ('netbsd5', 'hp-ux11') +# gh-89363: Skip fork() test if Python is built with Address Sanitizer (ASAN) +# to work around a libasan race condition, dead lock in pthread_create(). +skip_if_asan_fork = support.skip_if_sanitizer( + "libasan has a pthread_create() dead lock", + address=True) + + +def skip_unless_reliable_fork(test): + if not support.has_fork_support: + return unittest.skip("requires working os.fork()")(test) + if sys.platform in platforms_to_skip: + return unittest.skip("due to known OS bug related to thread+fork")(test) + if support.check_sanitizer(address=True): + return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + return test + + def restore_default_excepthook(testcase): testcase.addCleanup(setattr, threading, 'excepthook', threading.excepthook) threading.excepthook = threading.__excepthook__ @@ -539,7 +556,7 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) - @support.requires_fork() + @skip_unless_reliable_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. @@ -571,11 +588,7 @@ def background_thread(evt): self.assertEqual(out, b'') self.assertEqual(err, b'') - @support.requires_fork() - # gh-89363: Skip multiprocessing tests if Python is built with ASAN to - # work around a libasan race condition: dead lock in pthread_create(). - @support.skip_if_sanitizer("libasan has a pthread_create() dead lock", - address=True) + @skip_unless_reliable_fork def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on # threads that vanished after a fork. @@ -611,7 +624,7 @@ def f(): th.start() th.join() - @support.requires_fork() + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): code = """if 1: @@ -632,8 +645,7 @@ def test_main_thread_after_fork(self): self.assertEqual(err, b"") self.assertEqual(data, "MainThread\nTrue\nTrue\n") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @support.requires_fork() + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_nonmain_thread(self): code = """if 1: @@ -1080,8 +1092,7 @@ def test_1_join_on_shutdown(self): """ self._run_and_join(script) - @support.requires_fork() - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @skip_unless_reliable_fork def test_2_join_in_forked_process(self): # Like the test above, but from a forked interpreter script = """if 1: @@ -1101,8 +1112,7 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) - @support.requires_fork() - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @skip_unless_reliable_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. @@ -1172,8 +1182,7 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) - @support.requires_fork() - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with # the ad-hoc TLS implementation. @@ -1199,7 +1208,7 @@ def do_fork_and_wait(): for t in threads: t.join() - @support.requires_fork() + @skip_unless_reliable_fork def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() diff --git a/Misc/NEWS.d/next/Tests/2023-09-29-14-11-30.gh-issue-110031.fQnFnc.rst b/Misc/NEWS.d/next/Tests/2023-09-29-14-11-30.gh-issue-110031.fQnFnc.rst new file mode 100644 index 00000000000000..a8a163c567d2b3 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-29-14-11-30.gh-issue-110031.fQnFnc.rst @@ -0,0 +1,2 @@ +Skip test_threading tests using thread+fork if Python is built with Address +Sanitizer (ASAN). Patch by Victor Stinner. From 9c73a9acec095c05a178e7dff638f7d9769318f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 15:20:59 +0200 Subject: [PATCH 076/124] gh-109592: test_eintr tolerates 20 ms when comparing timings (#110102) --- Lib/test/_test_eintr.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 006581f7cc6a9a..15586f15dfab30 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -25,6 +25,12 @@ from test.support import os_helper from test.support import socket_helper + +# gh-109592: Tolerate a difference of 20 ms when comparing timings +# (clock resolution) +CLOCK_RES = 0.020 + + @contextlib.contextmanager def kill_on_error(proc): """Context manager killing the subprocess if a Python exception is raised.""" @@ -75,6 +81,9 @@ def subprocess(self, *args, **kw): cmd_args = (sys.executable, '-c') + args return subprocess.Popen(cmd_args, **kw) + def check_elapsed_time(self, elapsed): + self.assertGreaterEqual(elapsed, self.sleep_time - CLOCK_RES) + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class OSEINTRTest(EINTRBaseTest): @@ -373,7 +382,7 @@ def test_sleep(self): time.sleep(self.sleep_time) self.stop_alarm() dt = time.monotonic() - t0 - self.assertGreaterEqual(dt, self.sleep_time) + self.check_elapsed_time(dt) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") @@ -435,7 +444,7 @@ def test_select(self): select.select([], [], [], self.sleep_time) dt = time.monotonic() - t0 self.stop_alarm() - self.assertGreaterEqual(dt, self.sleep_time) + self.check_elapsed_time(dt) @unittest.skipIf(sys.platform == "darwin", "poll may fail on macOS; see issue #28087") @@ -447,7 +456,7 @@ def test_poll(self): poller.poll(self.sleep_time * 1e3) dt = time.monotonic() - t0 self.stop_alarm() - self.assertGreaterEqual(dt, self.sleep_time) + self.check_elapsed_time(dt) @unittest.skipUnless(hasattr(select, 'epoll'), 'need select.epoll') def test_epoll(self): @@ -458,7 +467,7 @@ def test_epoll(self): poller.poll(self.sleep_time) dt = time.monotonic() - t0 self.stop_alarm() - self.assertGreaterEqual(dt, self.sleep_time) + self.check_elapsed_time(dt) @unittest.skipUnless(hasattr(select, 'kqueue'), 'need select.kqueue') def test_kqueue(self): @@ -469,7 +478,7 @@ def test_kqueue(self): kqueue.control(None, 1, self.sleep_time) dt = time.monotonic() - t0 self.stop_alarm() - self.assertGreaterEqual(dt, self.sleep_time) + self.check_elapsed_time(dt) @unittest.skipUnless(hasattr(select, 'devpoll'), 'need select.devpoll') def test_devpoll(self): @@ -480,7 +489,7 @@ def test_devpoll(self): poller.poll(self.sleep_time * 1e3) dt = time.monotonic() - t0 self.stop_alarm() - self.assertGreaterEqual(dt, self.sleep_time) + self.check_elapsed_time(dt) class FNTLEINTRTest(EINTRBaseTest): @@ -512,8 +521,8 @@ def _lock(self, lock_func, lock_name): # potential context switch delay lock_func(f, fcntl.LOCK_EX) dt = time.monotonic() - start_time - self.assertGreaterEqual(dt, self.sleep_time) self.stop_alarm() + self.check_elapsed_time(dt) proc.wait() # Issue 35633: See https://bugs.python.org/issue35633#msg333662 From 6364873d2abe0973e21af7c8c7dddbb5f8dc1e85 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 29 Sep 2023 09:20:23 -0600 Subject: [PATCH 077/124] gh-110024: Fix Pointer Type Warnings (gh-110053) The warnings were introduced by gh-109794 (for gh-109793). --- Include/cpython/pyatomic.h | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index ce23e13bf3838c..7a783058c173aa 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -506,15 +506,23 @@ static inline void _Py_atomic_fence_release(void); // --- aliases --------------------------------------------------------------- #if SIZEOF_LONG == 8 -# define _Py_atomic_load_ulong _Py_atomic_load_uint64 -# define _Py_atomic_load_ulong_relaxed _Py_atomic_load_uint64_relaxed -# define _Py_atomic_store_ulong _Py_atomic_store_uint64 -# define _Py_atomic_store_ulong_relaxed _Py_atomic_store_uint64_relaxed +# define _Py_atomic_load_ulong(p) \ + _Py_atomic_load_uint64((uint64_t *)p) +# define _Py_atomic_load_ulong_relaxed(p) \ + _Py_atomic_load_uint64_relaxed((uint64_t *)p) +# define _Py_atomic_store_ulong(p, v) \ + _Py_atomic_store_uint64((uint64_t *)p, v) +# define _Py_atomic_store_ulong_relaxed(p, v) \ + _Py_atomic_store_uint64_relaxed((uint64_t *)p, v) #elif SIZEOF_LONG == 4 -# define _Py_atomic_load_ulong _Py_atomic_load_uint32 -# define _Py_atomic_load_ulong_relaxed _Py_atomic_load_uint32_relaxed -# define _Py_atomic_store_ulong _Py_atomic_store_uint32 -# define _Py_atomic_store_ulong_relaxed _Py_atomic_store_uint32_relaxed +# define _Py_atomic_load_ulong(p) \ + _Py_atomic_load_uint32((uint32_t *)p) +# define _Py_atomic_load_ulong_relaxed(p) \ + _Py_atomic_load_uint32_relaxed((uint32_t *)p) +# define _Py_atomic_store_ulong(p, v) \ + _Py_atomic_store_uint32((uint32_t *)p, v) +# define _Py_atomic_store_ulong_relaxed(p, v) \ + _Py_atomic_store_uint32_relaxed((uint32_t *)p, v) #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG From 20bc5f7c28a6f8a2e156c4a748ffabb5efc7c761 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 29 Sep 2023 16:24:38 +0100 Subject: [PATCH 078/124] gh-109615: Look for 'Modules' as landmark for test_copy_python_src_ignore (GH-110108) --- Lib/test/test_support.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e4a246ba3ddd4d..902bec78451307 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -812,7 +812,9 @@ def test_copy_python_src_ignore(self): if not os.path.exists(src_dir): self.skipTest(f"cannot access Python source code directory:" f" {src_dir!r}") - landmark = os.path.join(src_dir, 'Lib', 'os.py') + # Check that the landmark copy_python_src_ignore() expects is available + # (Previously we looked for 'Lib\os.py', which is always present on Windows.) + landmark = os.path.join(src_dir, 'Modules') if not os.path.exists(landmark): self.skipTest(f"cannot access Python source code directory:" f" {landmark!r} landmark is missing") From 7d57288f6d0e7fffb2002ceb460784d39277584a Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Fri, 29 Sep 2023 17:57:32 +0100 Subject: [PATCH 079/124] gh-109495: Remove unused slots from the Python implementation of datetime (GH-109494) --- Lib/_pydatetime.py | 2 +- .../next/Library/2023-09-16-15-44-16.gh-issue-109495.m2H5Bk.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-16-15-44-16.gh-issue-109495.m2H5Bk.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 88275481e7002b..bca2acf1fc88cf 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1684,7 +1684,7 @@ class datetime(date): The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. """ - __slots__ = date.__slots__ + time.__slots__ + __slots__ = time.__slots__ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): diff --git a/Misc/NEWS.d/next/Library/2023-09-16-15-44-16.gh-issue-109495.m2H5Bk.rst b/Misc/NEWS.d/next/Library/2023-09-16-15-44-16.gh-issue-109495.m2H5Bk.rst new file mode 100644 index 00000000000000..a7e1b3a64fa785 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-16-15-44-16.gh-issue-109495.m2H5Bk.rst @@ -0,0 +1 @@ +Remove unnecessary extra ``__slots__`` in :py:class:`datetime`\'s pure python implementation to reduce memory size, as they are defined in the superclass. Patch by James Hilton-Balfe From 3439cb004943788d5e4698bfceec891e7118254c Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 29 Sep 2023 20:07:09 +0300 Subject: [PATCH 080/124] gh-66143: Allow copying and pickling of CodecInfo object (GH-109235) Co-authored-by: Robert Lehmann --- Lib/codecs.py | 3 + Lib/test/test_codecs.py | 70 +++++++++++++++++++ ...3-09-10-20-23-20.gh-issue-66143.71xvgL.rst | 2 + 3 files changed, 75 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-10-20-23-20.gh-issue-66143.71xvgL.rst diff --git a/Lib/codecs.py b/Lib/codecs.py index 82f23983e719c2..9b35b6127dd01c 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -111,6 +111,9 @@ def __repr__(self): (self.__class__.__module__, self.__class__.__qualname__, self.name, id(self)) + def __getnewargs__(self): + return tuple(self) + class Codec: """ Defines the interface for stateless encoders/decoders. diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index b5e9271ac0c3cd..c899111e1e4c16 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1762,6 +1762,76 @@ def test_file_closes_if_lookup_error_raised(self): file().close.assert_called() + def test_copy(self): + orig = codecs.lookup('utf-8') + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_deepcopy(self): + orig = codecs.lookup('utf-8') + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_pickle(self): + codec_info = codecs.lookup('utf-8') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertTrue(unpickled_codec_info._is_text_encoding) + + # Test a CodecInfo with _is_text_encoding equal to false. + codec_info = codecs.lookup('base64') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertFalse(unpickled_codec_info._is_text_encoding) + class StreamReaderTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-09-10-20-23-20.gh-issue-66143.71xvgL.rst b/Misc/NEWS.d/next/Library/2023-09-10-20-23-20.gh-issue-66143.71xvgL.rst new file mode 100644 index 00000000000000..2769c05d937353 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-10-20-23-20.gh-issue-66143.71xvgL.rst @@ -0,0 +1,2 @@ +The :class:`codecs.CodecInfo` object has been made copyable and pickleable. +Patched by Robert Lehmann and Furkan Onder. From f3df8fa669158f89af69b5661e98314d98fb916f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 21:16:29 +0200 Subject: [PATCH 081/124] gh-109566: PCbuild/rt.bat now uses --fast-ci (#110120) Replace "--fail-env-changed --fail-rerun" with "--fast-ci". Tools/buildbot/test.bat pass --slow-ci which has the priority over --fast-ci. --- PCbuild/rt.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 7ae7141bfc4eaa..332ba5edcf4082 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -32,7 +32,7 @@ set pcbuild=%~dp0 set suffix= set qmode= set dashO= -set regrtestargs=--fail-env-changed --fail-rerun +set regrtestargs=--fast-ci set exe= :CheckOpts From 635184212179b0511768ea1cd57256e134ba2d75 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 21:31:19 +0200 Subject: [PATCH 082/124] gh-109047: concurrent.futures catches PythonFinalizationError (#109810) concurrent.futures: The *executor manager thread* now catches exceptions when adding an item to the *call queue*. During Python finalization, creating a new thread can now raise RuntimeError. Catch the exception and call terminate_broken() in this case. Add test_python_finalization_error() to test_concurrent_futures. concurrent.futures._ExecutorManagerThread changes: * terminate_broken() no longer calls shutdown_workers() since the call queue is no longer working anymore (read and write ends of the queue pipe are closed). * terminate_broken() now terminates child processes, not only wait until they complete. * _ExecutorManagerThread.terminate_broken() now holds shutdown_lock to prevent race conditons with ProcessPoolExecutor.submit(). multiprocessing.Queue changes: * Add _terminate_broken() method. * _start_thread() sets _thread to None on exception to prevent leaking "dangling threads" even if the thread was not started yet. --- Lib/concurrent/futures/process.py | 47 ++++++++++++------- Lib/multiprocessing/queues.py | 30 ++++++++++-- .../test_process_pool.py | 29 ++++++++++++ ...-09-25-02-11-14.gh-issue-109047.b1TrqG.rst | 4 ++ 4 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-109047.b1TrqG.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 73bdcbe8693991..3990e6b1833d78 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -341,7 +341,14 @@ def run(self): # Main loop for the executor manager thread. while True: - self.add_call_item_to_queue() + # gh-109047: During Python finalization, self.call_queue.put() + # creation of a thread can fail with RuntimeError. + try: + self.add_call_item_to_queue() + except BaseException as exc: + cause = format_exception(exc) + self.terminate_broken(cause) + return result_item, is_broken, cause = self.wait_result_broken_or_wakeup() @@ -425,8 +432,8 @@ def wait_result_broken_or_wakeup(self): try: result_item = result_reader.recv() is_broken = False - except BaseException as e: - cause = format_exception(type(e), e, e.__traceback__) + except BaseException as exc: + cause = format_exception(exc) elif wakeup_reader in ready: is_broken = False @@ -463,7 +470,7 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def terminate_broken(self, cause): + def _terminate_broken(self, cause): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that # lead the executor into becoming broken. @@ -490,7 +497,7 @@ def terminate_broken(self, cause): for work_id, work_item in self.pending_work_items.items(): try: work_item.future.set_exception(bpe) - except _base.InvalidStateError as exc: + except _base.InvalidStateError: # set_exception() fails if the future is cancelled: ignore it. # Trying to check if the future is cancelled before calling # set_exception() would leave a race condition if the future is @@ -505,17 +512,14 @@ def terminate_broken(self, cause): for p in self.processes.values(): p.terminate() - # Prevent queue writing to a pipe which is no longer read. - # https://github.com/python/cpython/issues/94777 - self.call_queue._reader.close() - - # gh-107219: Close the connection writer which can unblock - # Queue._feed() if it was stuck in send_bytes(). - if sys.platform == 'win32': - self.call_queue._writer.close() + self.call_queue._terminate_broken() # clean up resources - self.join_executor_internals() + self._join_executor_internals(broken=True) + + def terminate_broken(self, cause): + with self.shutdown_lock: + self._terminate_broken(cause) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -558,15 +562,24 @@ def shutdown_workers(self): break def join_executor_internals(self): - self.shutdown_workers() + with self.shutdown_lock: + self._join_executor_internals() + + def _join_executor_internals(self, broken=False): + # If broken, call_queue was closed and so can no longer be used. + if not broken: + self.shutdown_workers() + # Release the queue's resources as soon as possible. self.call_queue.close() self.call_queue.join_thread() - with self.shutdown_lock: - self.thread_wakeup.close() + self.thread_wakeup.close() + # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on Mac OS X. for p in self.processes.values(): + if broken: + p.terminate() p.join() def get_n_children_alive(self): diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index daf9ee94a19431..852ae87b276861 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -158,6 +158,20 @@ def cancel_join_thread(self): except AttributeError: pass + def _terminate_broken(self): + # Close a Queue on error. + + # gh-94777: Prevent queue writing to a pipe which is no longer read. + self._reader.close() + + # gh-107219: Close the connection writer which can unblock + # Queue._feed() if it was stuck in send_bytes(). + if sys.platform == 'win32': + self._writer.close() + + self.close() + self.join_thread() + def _start_thread(self): debug('Queue._start_thread()') @@ -169,13 +183,19 @@ def _start_thread(self): self._wlock, self._reader.close, self._writer.close, self._ignore_epipe, self._on_queue_feeder_error, self._sem), - name='QueueFeederThread' + name='QueueFeederThread', + daemon=True, ) - self._thread.daemon = True - debug('doing self._thread.start()') - self._thread.start() - debug('... done self._thread.start()') + try: + debug('doing self._thread.start()') + self._thread.start() + debug('... done self._thread.start()') + except: + # gh-109047: During Python finalization, creating a thread + # can fail with RuntimeError. + self._thread = None + raise if not self._joincancelled: self._jointhread = Finalize( diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 7763a4946f110c..c73c2da1a01088 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -1,5 +1,6 @@ import os import sys +import threading import time import unittest from concurrent import futures @@ -187,6 +188,34 @@ def test_max_tasks_early_shutdown(self): for i, future in enumerate(futures): self.assertEqual(future.result(), mul(i, i)) + def test_python_finalization_error(self): + # gh-109047: Catch RuntimeError on thread creation + # during Python finalization. + + context = self.get_context() + + # gh-109047: Mock the threading.start_new_thread() function to inject + # RuntimeError: simulate the error raised during Python finalization. + # Block the second creation: create _ExecutorManagerThread, but block + # QueueFeederThread. + orig_start_new_thread = threading._start_new_thread + nthread = 0 + def mock_start_new_thread(func, *args): + nonlocal nthread + if nthread >= 1: + raise RuntimeError("can't create new thread at " + "interpreter shutdown") + nthread += 1 + return orig_start_new_thread(func, *args) + + with support.swap_attr(threading, '_start_new_thread', + mock_start_new_thread): + executor = self.executor_type(max_workers=2, mp_context=context) + with executor: + with self.assertRaises(BrokenProcessPool): + list(executor.map(mul, [(2, 3)] * 10)) + executor.shutdown() + create_executor_tests(globals(), ProcessPoolExecutorTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-109047.b1TrqG.rst b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-109047.b1TrqG.rst new file mode 100644 index 00000000000000..71cb5a80847d0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-109047.b1TrqG.rst @@ -0,0 +1,4 @@ +:mod:`concurrent.futures`: The *executor manager thread* now catches exceptions +when adding an item to the *call queue*. During Python finalization, creating a +new thread can now raise :exc:`RuntimeError`. Catch the exception and call +``terminate_broken()`` in this case. Patch by Victor Stinner. From 9c91141ffff0275cff99f50cbf805a0e9d645da8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 21:36:03 +0200 Subject: [PATCH 083/124] gh-109566: Remove make testall (#110122) Remove "make testall" target: use "make buildbottest" instead. --- Makefile.pre.in | 22 ++++--------------- ...-09-29-21-01-48.gh-issue-109566._enldb.rst | 2 ++ README.rst | 2 +- 3 files changed, 7 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-09-29-21-01-48.gh-issue-109566._enldb.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index d62b4d24b3e183..fa5b9e6654c26c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1853,21 +1853,6 @@ cleantest: all test: all $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) -# Run the full test suite twice - once without .pyc files, and once with. -# In the past, we've had problems where bugs in the marshalling or -# elsewhere caused bytecode read from .pyc files to behave differently -# than bytecode generated directly from a .py source file. Sometimes -# the bytecode read from a .pyc file had the bug, sometimes the directly -# generated bytecode. This is sometimes a very shy bug needing a lot of -# sample data. -.PHONY: testall -testall: all - -find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true - $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) - $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) - # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. .PHONY: testuniversal @@ -1880,8 +1865,9 @@ testuniversal: all $(RUNSHARED) /usr/libexec/oah/translate \ ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) -# Like testall, but with only one pass and without multiple processes. -# Run an optional script to include information about the build environment. +# Like test, but using --slow-ci which enables all test resources and use +# longer timeout. Run an optional pybuildbot.identify script to include +# information about the build environment. .PHONY: buildbottest buildbottest: all -@if which pybuildbot.identify >/dev/null 2>&1; then \ @@ -1889,7 +1875,7 @@ buildbottest: all fi $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) -# Like testall, but run Python tests with HOSTRUNNER directly. +# Like buildbottest, but run Python tests with HOSTRUNNER directly. .PHONY: hostrunnertest hostrunnertest: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) diff --git a/Misc/NEWS.d/next/Build/2023-09-29-21-01-48.gh-issue-109566._enldb.rst b/Misc/NEWS.d/next/Build/2023-09-29-21-01-48.gh-issue-109566._enldb.rst new file mode 100644 index 00000000000000..1141a4738b3151 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-09-29-21-01-48.gh-issue-109566._enldb.rst @@ -0,0 +1,2 @@ +Remove ``make testall`` target: use ``make buildbottest`` instead. Patch by +Victor Stinner. diff --git a/README.rst b/README.rst index 208bf8cec444a3..921da30a920168 100644 --- a/README.rst +++ b/README.rst @@ -177,7 +177,7 @@ is printed about a failed test or a traceback or core dump is produced, something is wrong. By default, tests are prevented from overusing resources like disk space and -memory. To enable these tests, run ``make testall``. +memory. To enable these tests, run ``make buildbottest``. If any tests fail, you can re-run the failing test(s) in verbose mode. For example, if ``test_os`` and ``test_gdb`` failed, you can run:: From 5ae6c6d053311d411a077200f85698d51d5fe8b9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 21:49:39 +0200 Subject: [PATCH 084/124] gh-109566: regrtest --fast-ci no longer enables --nowindows (#110121) The --nowindows option is deprecated and does nothing but logs a warning. --- Lib/test/libregrtest/cmdline.py | 6 +----- Lib/test/test_regrtest.py | 2 -- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0a863561d5273d..8562a48446b4a7 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -4,8 +4,6 @@ import sys from test.support import os_helper -from .utils import MS_WINDOWS - USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] @@ -414,7 +412,7 @@ def _parse_args(args, **kwargs): # Similar to options: # # -j0 --randomize --fail-env-changed --fail-rerun --rerun - # --slowest --verbose3 --nowindows + # --slowest --verbose3 if ns.use_mp is None: ns.use_mp = 0 ns.randomize = True @@ -424,8 +422,6 @@ def _parse_args(args, **kwargs): ns.rerun = True ns.print_slow = True ns.verbose3 = True - if MS_WINDOWS: - ns.nowindows = True # Silence alerts under Windows else: ns._add_python_opts = False diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index da1406def55543..c98b05abcea98c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -376,8 +376,6 @@ def test_unknown_option(self): def check_ci_mode(self, args, use_resources, rerun=True): ns = cmdline._parse_args(args) - if utils.MS_WINDOWS: - self.assertTrue(ns.nowindows) # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API From 2973970af8fb3f117ab2e8ab2d82e8a541fcb1da Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 29 Sep 2023 20:50:51 +0000 Subject: [PATCH 085/124] gh-110119: Temporarily skip test_cppext on --disable-gil builds. (#110123) The current version of pip does not support "t" in the ABI flags. Skip the test in `--disable-gil` builds until we can update pip. --- Lib/test/test_cppext/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 74bf420900367e..25b6fc64a03a51 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -14,6 +14,10 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') +# gh-110119: pip does not currently support 't' in the ABI flag use by +# --disable-gil builds. Once it does, we can remove this skip. +@unittest.skipIf(sysconfig.get_config_var('Py_NOGIL') == 1, + 'test does not work with --disable-gil') @support.requires_subprocess() class TestCPPExt(unittest.TestCase): @support.requires_resource('cpu') From 14098b78f7453adbd40c53e32c29588611b7c87b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Sep 2023 23:56:19 +0200 Subject: [PATCH 086/124] gh-107888: Fix test_mmap PROT_EXEC comment (#110125) --- Lib/test/test_mmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 92c99d645b25cc..dfcf3039422af5 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -258,7 +258,7 @@ def test_access_parameter(self): try: m = mmap.mmap(f.fileno(), mapsize, prot=prot) except PermissionError: - # on macOS 14, PROT_READ | PROT_WRITE is not allowed + # on macOS 14, PROT_READ | PROT_EXEC is not allowed pass else: self.assertRaises(TypeError, m.write, b"abcdef") From 613c0d4e866341e15a66704643a6392ce49058ba Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 29 Sep 2023 23:18:12 -0500 Subject: [PATCH 087/124] Add example for linear_regression() with proportional=True. (gh-110133) --- Doc/library/statistics.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index a8a79012565321..f3c1bf20ae3ac8 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -14,6 +14,7 @@ .. testsetup:: * from statistics import * + import math __name__ = '' -------------- @@ -741,6 +742,24 @@ However, for reading convenience, most of the examples show sorted sequences. *y = slope \* x + noise* + Continuing the example from :func:`correlation`, we look to see + how well a model based on major planets can predict the orbital + distances for dwarf planets: + + .. doctest:: + + >>> model = linear_regression(period_squared, dist_cubed, proportional=True) + >>> slope = model.slope + + >>> # Dwarf planets: Pluto, Eris, Makemake, Haumea, Ceres + >>> orbital_periods = [90_560, 204_199, 111_845, 103_410, 1_680] # days + >>> predicted_dist = [math.cbrt(slope * (p * p)) for p in orbital_periods] + >>> list(map(round, predicted_dist)) + [5912, 10166, 6806, 6459, 414] + + >>> [5_906, 10_152, 6_796, 6_450, 414] # actual distance in million km + [5906, 10152, 6796, 6450, 414] + .. versionadded:: 3.10 .. versionchanged:: 3.11 From cbdacc738a52a876aae5b74b4665d30a5f204766 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 30 Sep 2023 07:32:35 +0100 Subject: [PATCH 088/124] GH-101100: Fix reference warnings for ``namedtuple`` (#110113) --- Doc/whatsnew/2.6.rst | 6 +++--- Misc/NEWS.d/3.8.0a1.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index f3912d42180bfd..2f749dc40f7ee7 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1850,8 +1850,8 @@ changes, or look through the Subversion logs for all the details. special values and floating-point exceptions in a manner consistent with Annex 'G' of the C99 standard. -* A new data type in the :mod:`collections` module: :class:`namedtuple(typename, - fieldnames)` is a factory function that creates subclasses of the standard tuple +* A new data type in the :mod:`collections` module: ``namedtuple(typename, fieldnames)`` + is a factory function that creates subclasses of the standard tuple whose fields are accessible by name as well as index. For example:: >>> var_type = collections.namedtuple('variable', @@ -1873,7 +1873,7 @@ changes, or look through the Subversion logs for all the details. variable(id=1, name='amplitude', type='int', size=4) Several places in the standard library that returned tuples have - been modified to return :class:`namedtuple` instances. For example, + been modified to return :func:`namedtuple` instances. For example, the :meth:`Decimal.as_tuple` method now returns a named tuple with :attr:`sign`, :attr:`digits`, and :attr:`exponent` fields. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 3cbbbf7465032b..0dc6e945719ec7 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -380,7 +380,7 @@ Implement :pep:`572` (assignment expressions). Patch by Emily Morehouse. .. nonce: voIdcp .. section: Core and Builtins -Speed up :class:`namedtuple` attribute access by 1.6x using a C fast-path +Speed up :func:`namedtuple` attribute access by 1.6x using a C fast-path for the name descriptors. Patch by Pablo Galindo. .. From 0449fe999d56ba795a852d83380fe06514139935 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 30 Sep 2023 12:10:07 +0100 Subject: [PATCH 089/124] GH-101100: Fix reference warnings for ``gettext`` (#110115) --- Doc/library/gettext.rst | 55 +++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 88a65b980d310f..7ebe91b372d35a 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -58,7 +58,7 @@ class-based API instead. Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as - :func:`_` in the local namespace (see examples below). + :func:`!_` in the local namespace (see examples below). .. function:: dgettext(domain, message) @@ -98,7 +98,7 @@ class-based API instead. .. versionadded:: 3.8 -Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but +Note that GNU :program:`gettext` also defines a :func:`!dcgettext` method, but this was deemed not useful and so it is currently unimplemented. Here's an example of typical usage for this API:: @@ -119,7 +119,7 @@ greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format files, and has methods for returning strings. Instances of this class can also -install themselves in the built-in namespace as the function :func:`_`. +install themselves in the built-in namespace as the function :func:`!_`. .. function:: find(domain, localedir=None, languages=None, all=False) @@ -150,15 +150,12 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False) - Return a :class:`*Translations` instance based on the *domain*, *localedir*, + Return a ``*Translations`` instance based on the *domain*, *localedir*, and *languages*, which are first passed to :func:`find` to get a list of the associated :file:`.mo` file paths. Instances with identical :file:`.mo` file names are cached. The actual class instantiated is *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must - take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the - :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` - methods. + take a single :term:`file object` argument. If multiple files are found, later files are used as fallbacks for earlier ones. To allow setting the fallback, :func:`copy.copy` is used to clone each @@ -177,19 +174,19 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: install(domain, localedir=None, *, names=None) - This installs the function :func:`_` in Python's builtins namespace, based on + This installs the function :func:`!_` in Python's builtins namespace, based on *domain* and *localedir* which are passed to the function :func:`translation`. For the *names* parameter, please see the description of the translation object's :meth:`~NullTranslations.install` method. As seen below, you usually mark the strings in your application that are - candidates for translation, by wrapping them in a call to the :func:`_` + candidates for translation, by wrapping them in a call to the :func:`!_` function, like this:: print(_('This string will be translated.')) - For convenience, you want the :func:`_` function to be installed in Python's + For convenience, you want the :func:`!_` function to be installed in Python's builtins namespace, so it is easily accessible in all modules of your application. @@ -276,20 +273,20 @@ are the methods of :class:`!NullTranslations`: If the *names* parameter is given, it must be a sequence containing the names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, - ``'pgettext'``, ``'npgettext'``, ``'lgettext'``, and ``'lngettext'``. + addition to :func:`!_`. Supported names are ``'gettext'``, ``'ngettext'``, + ``'pgettext'``, and ``'npgettext'``. Note that this is only one way, albeit the most convenient way, to make - the :func:`_` function available to your application. Because it affects + the :func:`!_` function available to your application. Because it affects the entire application globally, and specifically the built-in namespace, - localized modules should never install :func:`_`. Instead, they should use - this code to make :func:`_` available to their module:: + localized modules should never install :func:`!_`. Instead, they should use + this code to make :func:`!_` available to their module:: import gettext t = gettext.translation('mymodule', ...) _ = t.gettext - This puts :func:`_` only in the module's global namespace and so only + This puts :func:`!_` only in the module's global namespace and so only affects calls within this module. .. versionchanged:: 3.8 @@ -314,7 +311,7 @@ initialize the "protected" :attr:`_charset` instance variable, defaulting to ids and message strings read from the catalog are converted to Unicode using this encoding, else ASCII is assumed. -Since message ids are read as Unicode strings too, all :meth:`*gettext` methods +Since message ids are read as Unicode strings too, all ``*gettext()`` methods will assume message ids as Unicode strings, not byte strings. The entire set of key/value pairs are placed into a dictionary and set as the @@ -404,7 +401,7 @@ version has a slightly different API. Its documented usage was:: _ = cat.gettext print(_('hello world')) -For compatibility with this older module, the function :func:`Catalog` is an +For compatibility with this older module, the function :func:`!Catalog` is an alias for the :func:`translation` function described above. One difference between this module and Henstridge's: his catalog objects @@ -432,7 +429,7 @@ take the following steps: In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping -it in ``_('...')`` --- that is, a call to the function :func:`_`. For example:: +it in ``_('...')`` --- that is, a call to the function :func:`_ `. For example:: filename = 'mylog.txt' message = _('writing a log message') @@ -504,7 +501,7 @@ module:: Localizing your application ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you are localizing your application, you can install the :func:`_` function +If you are localizing your application, you can install the :func:`!_` function globally into the built-in namespace, usually in the main driver file of your application. This will let all your application-specific files just use ``_('...')`` without having to explicitly install it in each file. @@ -581,13 +578,13 @@ Here is one way you can handle this situation:: for a in animals: print(_(a)) -This works because the dummy definition of :func:`_` simply returns the string +This works because the dummy definition of :func:`!_` simply returns the string unchanged. And this dummy definition will temporarily override any definition -of :func:`_` in the built-in namespace (until the :keyword:`del` command). Take -care, though if you have a previous definition of :func:`_` in the local +of :func:`!_` in the built-in namespace (until the :keyword:`del` command). Take +care, though if you have a previous definition of :func:`!_` in the local namespace. -Note that the second use of :func:`_` will not identify "a" as being +Note that the second use of :func:`!_` will not identify "a" as being translatable to the :program:`gettext` program, because the parameter is not a string literal. @@ -606,13 +603,13 @@ Another way to handle this is with the following example:: print(_(a)) In this case, you are marking translatable strings with the function -:func:`N_`, which won't conflict with any definition of :func:`_`. +:func:`!N_`, which won't conflict with any definition of :func:`!_`. However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`N_`. :program:`xgettext`, +look for translatable strings marked with :func:`!N_`. :program:`xgettext`, :program:`pygettext`, ``pybabel extract``, and :program:`xpot` all support this through the use of the :option:`!-k` command-line switch. -The choice of :func:`N_` here is totally arbitrary; it could have just -as easily been :func:`MarkThisStringForTranslation`. +The choice of :func:`!N_` here is totally arbitrary; it could have just +as easily been :func:`!MarkThisStringForTranslation`. Acknowledgements From 89966a694b54f81510f06a35b1406d56a2f2c8c5 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 30 Sep 2023 15:45:01 +0100 Subject: [PATCH 090/124] GH-89812: Add `pathlib._PathBase` (#106337) Add private `pathlib._PathBase` class. This will be used by an experimental PyPI package to incubate a `tarfile.TarPath` class. Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/pathlib.py | 446 ++++++++++++++---- Lib/test/test_pathlib.py | 400 +++++++++++++--- ...3-07-03-20-23-56.gh-issue-89812.cFkDOE.rst | 2 + 3 files changed, 687 insertions(+), 161 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index bd5f61b0b7c878..e6be9061013a8a 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -5,6 +5,7 @@ operating systems. """ +import contextlib import fnmatch import functools import io @@ -15,10 +16,19 @@ import sys import warnings from _collections_abc import Sequence -from errno import ENOENT, ENOTDIR, EBADF, ELOOP +from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from urllib.parse import quote_from_bytes as urlquote_from_bytes +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + __all__ = [ "UnsupportedOperation", @@ -30,6 +40,9 @@ # Internals # +# Maximum number of symlinks to follow in _PathBase.resolve() +_MAX_SYMLINKS = 40 + # Reference for Windows paths can be found at # https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . _WIN_RESERVED_NAMES = frozenset( @@ -292,6 +305,11 @@ class PurePath: # The `_hash` slot stores the hash of the case-normalized string # path. It's set when `__hash__()` is called for the first time. '_hash', + + # The '_resolving' slot stores a boolean indicating whether the path + # is being processed by `_PathBase.resolve()`. This prevents duplicate + # work from occurring when `resolve()` calls `stat()` or `readlink()`. + '_resolving', ) pathmod = os.path @@ -331,6 +349,7 @@ def __init__(self, *args): f"not {type(path).__name__!r}") paths.append(path) self._raw_paths = paths + self._resolving = False def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -416,7 +435,7 @@ def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) def as_uri(self): - """Return the path as a 'file' URI.""" + """Return the path as a URI.""" if not self.is_absolute(): raise ValueError("relative path can't be expressed as a file URI") @@ -691,7 +710,9 @@ def parent(self): tail = self._tail if not tail: return self - return self._from_parsed_parts(drv, root, tail[:-1]) + path = self._from_parsed_parts(drv, root, tail[:-1]) + path._resolving = self._resolving + return path @property def parents(self): @@ -776,23 +797,35 @@ class PureWindowsPath(PurePath): # Filesystem-accessing classes -class Path(PurePath): - """PurePath subclass that can make system calls. +class _PathBase(PurePath): + """Base class for concrete path objects. - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. + This class provides dummy implementations for many methods that derived + classes can override selectively; the default implementations raise + UnsupportedOperation. The most basic methods, such as stat() and open(), + directly raise UnsupportedOperation; these basic methods are called by + other methods such as is_dir() and read_text(). + + The Path class derives this class to implement local filesystem paths. + Users may derive their own classes to implement virtual filesystem paths, + such as paths in archive files or on remote storage systems. """ __slots__ = () + __bytes__ = None + __fspath__ = None # virtual paths have no local file system representation + + def _unsupported(self, method_name): + msg = f"{type(self).__name__}.{method_name}() is unsupported" + if isinstance(self, Path): + msg += " on this system" + raise UnsupportedOperation(msg) def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - return os.stat(self, follow_symlinks=follow_symlinks) + self._unsupported("stat") def lstat(self): """ @@ -859,7 +892,21 @@ def is_mount(self): """ Check if this path is a mount point """ - return os.path.ismount(self) + # Need to exist and be a dir + if not self.exists() or not self.is_dir(): + return False + + try: + parent_dev = self.parent.stat().st_dev + except OSError: + return False + + dev = self.stat().st_dev + if dev != parent_dev: + return True + ino = self.stat().st_ino + parent_ino = self.parent.stat().st_ino + return ino == parent_ino def is_symlink(self): """ @@ -880,7 +927,10 @@ def is_junction(self): """ Whether this path is a junction. """ - return os.path.isjunction(self) + # Junctions are a Windows-only feature, not present in POSIX nor the + # majority of virtual filesystems. There is no cross-platform idiom + # to check for junctions (using stat().st_mode). + return False def is_block_device(self): """ @@ -964,9 +1014,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed by this path and return a file object, as the built-in open() function does. """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) + self._unsupported("open") def read_bytes(self): """ @@ -1009,13 +1057,12 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - return (self._make_child_relpath(name) for name in os.listdir(self)) + self._unsupported("iterdir") def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) + # Emulate os.scandir(), which returns an object that can be used as a + # context manager. This method is called by walk() and glob(). + return contextlib.nullcontext(self.iterdir()) def _make_child_relpath(self, name): sep = self.pathmod.sep @@ -1144,13 +1191,13 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): # blow up for a minor reason when (say) a thousand readable # directories are still left to visit. That logic is copied here. try: - scandir_it = path._scandir() + scandir_obj = path._scandir() except OSError as error: if on_error is not None: on_error(error) continue - with scandir_it: + with scandir_obj as scandir_it: dirnames = [] filenames = [] for entry in scandir_it: @@ -1172,17 +1219,13 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) + Use resolve() to resolve symlinks and remove '..' segments. + """ + self._unsupported("absolute") @classmethod def cwd(cls): @@ -1193,18 +1236,264 @@ def cwd(cls): # os.path.abspath('.') == os.getcwd(). return cls().absolute() + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + self._unsupported("expanduser") + @classmethod def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). + """Return a new path pointing to expanduser('~'). """ return cls("~").expanduser() + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + self._unsupported("readlink") + readlink._supported = False + + def _split_stack(self): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + return self._from_parsed_parts(self.drive, self.root, []), self._tail[::-1] + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + if self._resolving: + return self + try: + path = self.absolute() + except UnsupportedOperation: + path = self + + # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported + # and (in non-strict mode) we can improve performance by not calling `stat()`. + querying = strict or getattr(self.readlink, '_supported', True) + link_count = 0 + stat_cache = {} + target_cache = {} + path, parts = path._split_stack() + while parts: + part = parts.pop() + if part == '..': + if not path._tail: + if path.root: + # Delete '..' segment immediately following root + continue + elif path._tail[-1] != '..': + # Delete '..' segment and its predecessor + path = path.parent + continue + # Join the current part onto the path. + path_parent = path + path = path._make_child_relpath(part) + if querying and part != '..': + path._resolving = True + try: + st = stat_cache.get(path) + if st is None: + st = stat_cache[path] = path.stat(follow_symlinks=False) + if S_ISLNK(st.st_mode): + # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are + # encountered during resolution. + link_count += 1 + if link_count >= _MAX_SYMLINKS: + raise OSError(ELOOP, "Too many symbolic links in path", str(path)) + target = target_cache.get(path) + if target is None: + target = target_cache[path] = path.readlink() + target, target_parts = target._split_stack() + # If the symlink target is absolute (like '/etc/hosts'), set the current + # path to its uppermost parent (like '/'). If not, the symlink target is + # relative to the symlink parent, which we recorded earlier. + path = target if target.root else path_parent + # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to + # the stack of unresolved path parts. + parts.extend(target_parts) + elif parts and not S_ISDIR(st.st_mode): + raise NotADirectoryError(ENOTDIR, "Not a directory", str(path)) + except OSError: + if strict: + raise + else: + querying = False + path._resolving = False + return path + + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + self._unsupported("symlink_to") + + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + self._unsupported("hardlink_to") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + self._unsupported("touch") + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + self._unsupported("mkdir") + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + self._unsupported("rename") + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + self._unsupported("replace") + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + self._unsupported("chmod") + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + self._unsupported("unlink") + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + self._unsupported("rmdir") + + def owner(self): + """ + Return the login name of the file owner. + """ + self._unsupported("owner") + + def group(self): + """ + Return the group name of the file gid. + """ + self._unsupported("group") + + def as_uri(self): + """Return the path as a URI.""" + self._unsupported("as_uri") + + +class Path(_PathBase): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + __bytes__ = PurePath.__bytes__ + __fspath__ = PurePath.__fspath__ + as_uri = PurePath.as_uri + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return (self._make_child_relpath(name) for name in os.listdir(self)) + + def _scandir(self): + return os.scandir(self) + def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. + """Return an absolute version of this path + No normalization or symlink resolution is performed. - Use resolve() to get the canonical path to a file. + Use resolve() to resolve symlinks and remove '..' segments. """ if self.is_absolute(): return self @@ -1232,34 +1521,26 @@ def resolve(self, strict=False): return self.with_segments(os.path.realpath(self, strict=strict)) - def owner(self): - """ - Return the login name of the file owner. - """ - try: - import pwd + if pwd: + def owner(self): + """ + Return the login name of the file owner. + """ return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise UnsupportedOperation("Path.owner() is unsupported on this system") - - def group(self): - """ - Return the group name of the file gid. - """ - try: - import grp + if grp: + def group(self): + """ + Return the group name of the file gid. + """ return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise UnsupportedOperation("Path.group() is unsupported on this system") - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - if not hasattr(os, "readlink"): - raise UnsupportedOperation("os.readlink() not available on this system") - return self.with_segments(os.readlink(self)) + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) def touch(self, mode=0o666, exist_ok=True): """ @@ -1306,13 +1587,6 @@ def chmod(self, mode, *, follow_symlinks=True): """ os.chmod(self, mode, follow_symlinks=follow_symlinks) - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - def unlink(self, missing_ok=False): """ Remove this file or link. @@ -1356,24 +1630,22 @@ def replace(self, target): os.replace(self, target) return self.with_segments(target) - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - if not hasattr(os, "symlink"): - raise UnsupportedOperation("os.symlink() not available on this system") - os.symlink(target, self, target_is_directory) - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - if not hasattr(os, "link"): - raise UnsupportedOperation("os.link() not available on this system") - os.link(target, self) + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) def expanduser(self): """ Return a new path with expanded ~ and ~user constructs diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 484a5e6c3bd64d..319148e9065a65 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1582,14 +1582,172 @@ def test_group(self): # -# Tests for the concrete classes. +# Tests for the virtual classes. # -class PathTest(unittest.TestCase): - """Tests for the FS-accessing functionalities of the Path classes.""" +class PathBaseTest(PurePathTest): + cls = pathlib._PathBase - cls = pathlib.Path - can_symlink = os_helper.can_symlink() + def test_unsupported_operation(self): + P = self.cls + p = self.cls() + e = pathlib.UnsupportedOperation + self.assertRaises(e, p.stat) + self.assertRaises(e, p.lstat) + self.assertRaises(e, p.exists) + self.assertRaises(e, p.samefile, 'foo') + self.assertRaises(e, p.is_dir) + self.assertRaises(e, p.is_file) + self.assertRaises(e, p.is_mount) + self.assertRaises(e, p.is_symlink) + self.assertRaises(e, p.is_block_device) + self.assertRaises(e, p.is_char_device) + self.assertRaises(e, p.is_fifo) + self.assertRaises(e, p.is_socket) + self.assertRaises(e, p.open) + self.assertRaises(e, p.read_bytes) + self.assertRaises(e, p.read_text) + self.assertRaises(e, p.write_bytes, b'foo') + self.assertRaises(e, p.write_text, 'foo') + self.assertRaises(e, p.iterdir) + self.assertRaises(e, p.glob, '*') + self.assertRaises(e, p.rglob, '*') + self.assertRaises(e, lambda: list(p.walk())) + self.assertRaises(e, p.absolute) + self.assertRaises(e, P.cwd) + self.assertRaises(e, p.expanduser) + self.assertRaises(e, p.home) + self.assertRaises(e, p.readlink) + self.assertRaises(e, p.symlink_to, 'foo') + self.assertRaises(e, p.hardlink_to, 'foo') + self.assertRaises(e, p.mkdir) + self.assertRaises(e, p.touch) + self.assertRaises(e, p.rename, 'foo') + self.assertRaises(e, p.replace, 'foo') + self.assertRaises(e, p.chmod, 0o755) + self.assertRaises(e, p.lchmod, 0o755) + self.assertRaises(e, p.unlink) + self.assertRaises(e, p.rmdir) + self.assertRaises(e, p.owner) + self.assertRaises(e, p.group) + self.assertRaises(e, p.as_uri) + + def test_as_uri_common(self): + e = pathlib.UnsupportedOperation + self.assertRaises(e, self.cls().as_uri) + + def test_fspath_common(self): + self.assertRaises(TypeError, os.fspath, self.cls()) + + def test_as_bytes_common(self): + self.assertRaises(TypeError, bytes, self.cls()) + + def test_matches_path_api(self): + our_names = {name for name in dir(self.cls) if name[0] != '_'} + path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} + self.assertEqual(our_names, path_names) + for attr_name in our_names: + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib.Path, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + +class DummyPathIO(io.BytesIO): + """ + Used by DummyPath to implement `open('w')` + """ + + def __init__(self, files, path): + super().__init__() + self.files = files + self.path = path + + def close(self): + self.files[self.path] = self.getvalue() + super().close() + + +class DummyPath(pathlib._PathBase): + """ + Simple implementation of PathBase that keeps files and directories in + memory. + """ + _files = {} + _directories = {} + _symlinks = {} + + def stat(self, *, follow_symlinks=True): + if follow_symlinks: + path = str(self.resolve()) + else: + path = str(self.parent.resolve() / self.name) + if path in self._files: + st_mode = stat.S_IFREG + elif path in self._directories: + st_mode = stat.S_IFDIR + elif path in self._symlinks: + st_mode = stat.S_IFLNK + else: + raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) + return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + if buffering != -1: + raise NotImplementedError + path_obj = self.resolve() + path = str(path_obj) + name = path_obj.name + parent = str(path_obj.parent) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + + text = 'b' not in mode + mode = ''.join(c for c in mode if c not in 'btU') + if mode == 'r': + if path not in self._files: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + stream = io.BytesIO(self._files[path]) + elif mode == 'w': + if parent not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", parent) + stream = DummyPathIO(self._files, path) + self._files[path] = b'' + self._directories[parent].add(name) + else: + raise NotImplementedError + if text: + stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) + return stream + + def iterdir(self): + path = str(self.resolve()) + if path in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path in self._directories: + return (self / name for name in self._directories[path]) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + try: + self._directories[str(self.parent)].add(self.name) + self._directories[str(self)] = set() + except KeyError: + if not parents or self.parent == self: + raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except FileExistsError: + if not exist_ok: + raise + + +class DummyPathTest(unittest.TestCase): + """Tests for PathBase methods that use stat(), open() and iterdir().""" + + cls = DummyPath + can_symlink = False # (BASE) # | @@ -1612,37 +1770,38 @@ class PathTest(unittest.TestCase): # def setUp(self): - def cleanup(): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) - self.addCleanup(cleanup) - os.mkdir(BASE) - os.mkdir(join('dirA')) - os.mkdir(join('dirB')) - os.mkdir(join('dirC')) - os.mkdir(join('dirC', 'dirD')) - os.mkdir(join('dirE')) - with open(join('fileA'), 'wb') as f: - f.write(b"this is file A\n") - with open(join('dirB', 'fileB'), 'wb') as f: - f.write(b"this is file B\n") - with open(join('dirC', 'fileC'), 'wb') as f: - f.write(b"this is file C\n") - with open(join('dirC', 'novel.txt'), 'wb') as f: - f.write(b"this is a novel\n") - with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: - f.write(b"this is file D\n") - os.chmod(join('dirE'), 0) - if self.can_symlink: - # Relative symlinks. - os.symlink('fileA', join('linkA')) - os.symlink('non-existing', join('brokenLink')) - os.symlink('dirB', join('linkB'), target_is_directory=True) - os.symlink(os.path.join('..', 'dirB'), join('dirA', 'linkC'), target_is_directory=True) - # This one goes upwards, creating a loop. - os.symlink(os.path.join('..', 'dirB'), join('dirB', 'linkD'), target_is_directory=True) - # Broken symlink (pointing to itself). - os.symlink('brokenLinkLoop', join('brokenLinkLoop')) + # note: this must be kept in sync with `PathTest.setUp()` + cls = self.cls + cls._files.clear() + cls._directories.clear() + cls._symlinks.clear() + join = cls.pathmod.join + cls._files.update({ + join(BASE, 'fileA'): b'this is file A\n', + join(BASE, 'dirB', 'fileB'): b'this is file B\n', + join(BASE, 'dirC', 'fileC'): b'this is file C\n', + join(BASE, 'dirC', 'dirD', 'fileD'): b'this is file D\n', + join(BASE, 'dirC', 'novel.txt'): b'this is a novel\n', + }) + cls._directories.update({ + BASE: {'dirA', 'dirB', 'dirC', 'dirE', 'fileA'}, + join(BASE, 'dirA'): set(), + join(BASE, 'dirB'): {'fileB'}, + join(BASE, 'dirC'): {'dirD', 'fileC', 'novel.txt'}, + join(BASE, 'dirC', 'dirD'): {'fileD'}, + join(BASE, 'dirE'): {}, + }) + dirname = BASE + while True: + dirname, basename = cls.pathmod.split(dirname) + if not basename: + break + cls._directories[dirname] = {basename} + + def tempdir(self): + path = self.cls(BASE).with_name('tmp-dirD') + path.mkdir() + return path def assertFileNotFound(self, func, *args, **kwargs): with self.assertRaises(FileNotFoundError) as cm: @@ -1991,9 +2150,11 @@ def test_rglob_symlink_loop(self): def test_glob_many_open_files(self): depth = 30 P = self.cls - base = P(BASE) / 'deep' - p = P(base, *(['d']*depth)) - p.mkdir(parents=True) + p = base = P(BASE) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() pattern = '/'.join(['*'] * depth) iters = [base.glob(pattern) for j in range(100)] for it in iters: @@ -2080,6 +2241,7 @@ def test_readlink(self): self.assertEqual((P / 'brokenLink').readlink(), self.cls('non-existing')) self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) with self.assertRaises(OSError): (P / 'fileA').readlink() @@ -2128,7 +2290,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt': + if os.name == 'nt' and isinstance(p, pathlib.Path): # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', @@ -2138,9 +2300,7 @@ def test_resolve_common(self): # resolves to 'dirB/..' first before resolving to parent of dirB. self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) # Now create absolute symlinks. - d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', - dir=os.getcwd())) - self.addCleanup(os_helper.rmtree, d) + d = self.tempdir() P(BASE, 'dirA', 'linkX').symlink_to(d) P(BASE, str(d), 'linkY').symlink_to(join('dirB')) p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') @@ -2150,7 +2310,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), False) p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt': + if os.name == 'nt' and isinstance(p, pathlib.Path): # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) @@ -2174,6 +2334,38 @@ def test_resolve_dot(self): # Non-strict self.assertEqual(r.resolve(strict=False), p / '3' / '4') + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + def test_resolve_loop(self): + if not self.can_symlink: + self.skipTest("symlinks required") + if os.name == 'nt' and issubclass(self.cls, pathlib.Path): + self.skipTest("symlink loops work differently with concrete Windows paths") + # Loops with relative symlinks. + self.cls(BASE, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(BASE, 'linkX') + self.cls(BASE, 'linkY').symlink_to('linkY') + self._check_symlink_loop(BASE, 'linkY') + self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(BASE, 'linkZ') + # Non-strict + p = self.cls(BASE, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) + self._check_symlink_loop(BASE, 'linkU') + self.cls(BASE, 'linkV').symlink_to(join('linkV')) + self._check_symlink_loop(BASE, 'linkV') + self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) + self._check_symlink_loop(BASE, 'linkW') + # Non-strict + q = self.cls(BASE, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + def test_stat(self): statA = self.cls(BASE).joinpath('fileA').stat() statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() @@ -2382,6 +2574,10 @@ def _check_complex_symlinks(self, link0_target): self.assertEqualNormCase(str(p), BASE) # Resolve relative paths. + try: + self.cls().absolute() + except pathlib.UnsupportedOperation: + return old_path = os.getcwd() os.chdir(BASE) try: @@ -2409,6 +2605,92 @@ def test_complex_symlinks_relative(self): def test_complex_symlinks_relative_dot_dot(self): self._check_complex_symlinks(os.path.join('dirA', '..')) + +class DummyPathWithSymlinks(DummyPath): + def readlink(self): + path = str(self.parent.resolve() / self.name) + if path in self._symlinks: + return self.with_segments(self._symlinks[path]) + elif path in self._files or path in self._directories: + raise OSError(errno.EINVAL, "Not a symlink", path) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def symlink_to(self, target, target_is_directory=False): + self._directories[str(self.parent)].add(self.name) + self._symlinks[str(self)] = str(target) + + +class DummyPathWithSymlinksTest(DummyPathTest): + cls = DummyPathWithSymlinks + can_symlink = True + + def setUp(self): + super().setUp() + cls = self.cls + join = cls.pathmod.join + cls._symlinks.update({ + join(BASE, 'linkA'): 'fileA', + join(BASE, 'linkB'): 'dirB', + join(BASE, 'dirA', 'linkC'): join('..', 'dirB'), + join(BASE, 'dirB', 'linkD'): join('..', 'dirB'), + join(BASE, 'brokenLink'): 'non-existing', + join(BASE, 'brokenLinkLoop'): 'brokenLinkLoop', + }) + cls._directories[BASE].update({'linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'}) + cls._directories[join(BASE, 'dirA')].add('linkC') + cls._directories[join(BASE, 'dirB')].add('linkD') + + +# +# Tests for the concrete classes. +# + +class PathTest(DummyPathTest): + """Tests for the FS-accessing functionalities of the Path classes.""" + cls = pathlib.Path + can_symlink = os_helper.can_symlink() + + def setUp(self): + # note: this must be kept in sync with `DummyPathTest.setUp()` + def cleanup(): + os.chmod(join('dirE'), 0o777) + os_helper.rmtree(BASE) + self.addCleanup(cleanup) + os.mkdir(BASE) + os.mkdir(join('dirA')) + os.mkdir(join('dirB')) + os.mkdir(join('dirC')) + os.mkdir(join('dirC', 'dirD')) + os.mkdir(join('dirE')) + with open(join('fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(join('dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(join('dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(join('dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + os.chmod(join('dirE'), 0) + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', join('linkA')) + os.symlink('non-existing', join('brokenLink')) + os.symlink('dirB', join('linkB'), target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), join('dirA', 'linkC'), target_is_directory=True) + # This one goes upwards, creating a loop. + os.symlink(os.path.join('..', 'dirB'), join('dirB', 'linkD'), target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', join('brokenLinkLoop')) + + def tempdir(self): + d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', + dir=os.getcwd())) + self.addCleanup(os_helper.rmtree, d) + return d + def test_concrete_class(self): if self.cls is pathlib.Path: expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath @@ -3178,12 +3460,6 @@ def test_absolute(self): self.assertEqual(str(P('//a').absolute()), '//a') self.assertEqual(str(P('//a/b').absolute()), '//a/b') - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - @unittest.skipIf( is_emscripten or is_wasi, "umask is not implemented on Emscripten/WASI." @@ -3230,30 +3506,6 @@ def test_touch_mode(self): st = os.stat(join('masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - # Loops with relative symlinks. - os.symlink('linkX/inside', join('linkX')) - self._check_symlink_loop(BASE, 'linkX') - os.symlink('linkY', join('linkY')) - self._check_symlink_loop(BASE, 'linkY') - os.symlink('linkZ/../linkZ', join('linkZ')) - self._check_symlink_loop(BASE, 'linkZ') - # Non-strict - p = self.cls(BASE, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - os.symlink(join('linkU/inside'), join('linkU')) - self._check_symlink_loop(BASE, 'linkU') - os.symlink(join('linkV'), join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - os.symlink(join('linkW/../linkW'), join('linkW')) - self._check_symlink_loop(BASE, 'linkW') - # Non-strict - q = self.cls(BASE, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - def test_glob(self): P = self.cls p = P(BASE) diff --git a/Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst b/Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst new file mode 100644 index 00000000000000..a4221fc4ca900b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst @@ -0,0 +1,2 @@ +Add private ``pathlib._PathBase`` class, which provides experimental support +for virtual filesystems, and may be made public in a future version of Python. From f3bb00ea12db6525f07d62368a65efec47d192b9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 19:24:06 +0200 Subject: [PATCH 091/124] gh-107954: Refactor initconfig.c: add CONFIG_SPEC (#110146) Add a specification of the PyConfig structure to factorize the code. --- Lib/test/test_embed.py | 1 + Python/initconfig.c | 577 ++++++++++++--------------- Tools/c-analyzer/cpython/ignored.tsv | 4 + 3 files changed, 262 insertions(+), 320 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7f1a4e665f3b5d..852b3578989cd8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -455,6 +455,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'code_debug_ranges': 1, 'show_ref_count': 0, 'dump_refs': 0, + 'dump_refs_file': None, 'malloc_stats': 0, 'filesystem_encoding': GET_DEFAULT_CONFIG, diff --git a/Python/initconfig.c b/Python/initconfig.c index a0467f51d4834e..089ede4623e23d 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,104 @@ # endif #endif +/* --- PyConfig spec ---------------------------------------------- */ + +typedef enum { + PyConfig_MEMBER_INT = 0, + PyConfig_MEMBER_UINT = 1, + PyConfig_MEMBER_ULONG = 2, + + PyConfig_MEMBER_WSTR = 10, + PyConfig_MEMBER_WSTR_OPT = 11, + PyConfig_MEMBER_WSTR_LIST = 12, +} PyConfigMemberType; + +typedef struct { + const char *name; + size_t offset; + PyConfigMemberType type; +} PyConfigSpec; + +#define SPEC(MEMBER, TYPE) \ + {#MEMBER, offsetof(PyConfig, MEMBER), PyConfig_MEMBER_##TYPE} + +static const PyConfigSpec PYCONFIG_SPEC[] = { + SPEC(_config_init, UINT), + SPEC(isolated, UINT), + SPEC(use_environment, UINT), + SPEC(dev_mode, UINT), + SPEC(install_signal_handlers, UINT), + SPEC(use_hash_seed, UINT), + SPEC(hash_seed, ULONG), + SPEC(faulthandler, UINT), + SPEC(tracemalloc, UINT), + SPEC(perf_profiling, UINT), + SPEC(import_time, UINT), + SPEC(code_debug_ranges, UINT), + SPEC(show_ref_count, UINT), + SPEC(dump_refs, UINT), + SPEC(dump_refs_file, WSTR_OPT), + SPEC(malloc_stats, UINT), + SPEC(filesystem_encoding, WSTR), + SPEC(filesystem_errors, WSTR), + SPEC(pycache_prefix, WSTR_OPT), + SPEC(parse_argv, UINT), + SPEC(orig_argv, WSTR_LIST), + SPEC(argv, WSTR_LIST), + SPEC(xoptions, WSTR_LIST), + SPEC(warnoptions, WSTR_LIST), + SPEC(site_import, UINT), + SPEC(bytes_warning, UINT), + SPEC(warn_default_encoding, UINT), + SPEC(inspect, UINT), + SPEC(interactive, UINT), + SPEC(optimization_level, UINT), + SPEC(parser_debug, UINT), + SPEC(write_bytecode, UINT), + SPEC(verbose, UINT), + SPEC(quiet, UINT), + SPEC(user_site_directory, UINT), + SPEC(configure_c_stdio, UINT), + SPEC(buffered_stdio, UINT), + SPEC(stdio_encoding, WSTR), + SPEC(stdio_errors, WSTR), +#ifdef MS_WINDOWS + SPEC(legacy_windows_stdio, UINT), +#endif + SPEC(check_hash_pycs_mode, WSTR), + SPEC(use_frozen_modules, UINT), + SPEC(safe_path, UINT), + SPEC(int_max_str_digits, INT), + SPEC(pathconfig_warnings, UINT), + SPEC(program_name, WSTR), + SPEC(pythonpath_env, WSTR_OPT), + SPEC(home, WSTR_OPT), + SPEC(platlibdir, WSTR), + SPEC(module_search_paths_set, UINT), + SPEC(module_search_paths, WSTR_LIST), + SPEC(stdlib_dir, WSTR_OPT), + SPEC(executable, WSTR_OPT), + SPEC(base_executable, WSTR_OPT), + SPEC(prefix, WSTR_OPT), + SPEC(base_prefix, WSTR_OPT), + SPEC(exec_prefix, WSTR_OPT), + SPEC(base_exec_prefix, WSTR_OPT), + SPEC(skip_source_first_line, UINT), + SPEC(run_command, WSTR_OPT), + SPEC(run_module, WSTR_OPT), + SPEC(run_filename, WSTR_OPT), + SPEC(_install_importlib, UINT), + SPEC(_init_main, UINT), + SPEC(_is_python_build, UINT), +#ifdef Py_STATS + SPEC(_pystats, UINT), +#endif + {NULL, 0, 0}, +}; + +#undef SPEC + + /* --- Command line options --------------------------------------- */ /* Short usage message (with %s for argv0) */ @@ -869,103 +967,47 @@ PyConfig_SetBytesString(PyConfig *config, wchar_t **config_str, PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { - PyStatus status; - PyConfig_Clear(config); -#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR -#define COPY_WSTR_ATTR(ATTR) \ - do { \ - status = PyConfig_SetString(config, &config->ATTR, config2->ATTR); \ - if (_PyStatus_EXCEPTION(status)) { \ - return status; \ - } \ - } while (0) -#define COPY_WSTRLIST(LIST) \ - do { \ - if (_PyWideStringList_Copy(&config->LIST, &config2->LIST) < 0) { \ - return _PyStatus_NO_MEMORY(); \ - } \ - } while (0) - - COPY_ATTR(_config_init); - COPY_ATTR(isolated); - COPY_ATTR(use_environment); - COPY_ATTR(dev_mode); - COPY_ATTR(install_signal_handlers); - COPY_ATTR(use_hash_seed); - COPY_ATTR(hash_seed); - COPY_ATTR(_install_importlib); - COPY_ATTR(faulthandler); - COPY_ATTR(tracemalloc); - COPY_ATTR(perf_profiling); - COPY_ATTR(import_time); - COPY_ATTR(code_debug_ranges); - COPY_ATTR(show_ref_count); - COPY_ATTR(dump_refs); - COPY_ATTR(dump_refs_file); - COPY_ATTR(malloc_stats); - - COPY_WSTR_ATTR(pycache_prefix); - COPY_WSTR_ATTR(pythonpath_env); - COPY_WSTR_ATTR(home); - COPY_WSTR_ATTR(program_name); - - COPY_ATTR(parse_argv); - COPY_WSTRLIST(argv); - COPY_WSTRLIST(warnoptions); - COPY_WSTRLIST(xoptions); - COPY_WSTRLIST(module_search_paths); - COPY_ATTR(module_search_paths_set); - COPY_WSTR_ATTR(stdlib_dir); - - COPY_WSTR_ATTR(executable); - COPY_WSTR_ATTR(base_executable); - COPY_WSTR_ATTR(prefix); - COPY_WSTR_ATTR(base_prefix); - COPY_WSTR_ATTR(exec_prefix); - COPY_WSTR_ATTR(base_exec_prefix); - COPY_WSTR_ATTR(platlibdir); - - COPY_ATTR(site_import); - COPY_ATTR(bytes_warning); - COPY_ATTR(warn_default_encoding); - COPY_ATTR(inspect); - COPY_ATTR(interactive); - COPY_ATTR(optimization_level); - COPY_ATTR(parser_debug); - COPY_ATTR(write_bytecode); - COPY_ATTR(verbose); - COPY_ATTR(quiet); - COPY_ATTR(user_site_directory); - COPY_ATTR(configure_c_stdio); - COPY_ATTR(buffered_stdio); - COPY_WSTR_ATTR(filesystem_encoding); - COPY_WSTR_ATTR(filesystem_errors); - COPY_WSTR_ATTR(stdio_encoding); - COPY_WSTR_ATTR(stdio_errors); -#ifdef MS_WINDOWS - COPY_ATTR(legacy_windows_stdio); -#endif - COPY_ATTR(skip_source_first_line); - COPY_WSTR_ATTR(run_command); - COPY_WSTR_ATTR(run_module); - COPY_WSTR_ATTR(run_filename); - COPY_WSTR_ATTR(check_hash_pycs_mode); - COPY_ATTR(pathconfig_warnings); - COPY_ATTR(_init_main); - COPY_ATTR(use_frozen_modules); - COPY_ATTR(safe_path); - COPY_WSTRLIST(orig_argv); - COPY_ATTR(_is_python_build); - COPY_ATTR(int_max_str_digits); -#ifdef Py_STATS - COPY_ATTR(_pystats); -#endif - -#undef COPY_ATTR -#undef COPY_WSTR_ATTR -#undef COPY_WSTRLIST + PyStatus status; + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + char *member = (char *)config + spec->offset; + char *member2 = (char *)config2 + spec->offset; + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + { + *(int*)member = *(int*)member2; + break; + } + case PyConfig_MEMBER_ULONG: + { + *(unsigned long*)member = *(unsigned long*)member2; + break; + } + case PyConfig_MEMBER_WSTR: + case PyConfig_MEMBER_WSTR_OPT: + { + const wchar_t *str = *(const wchar_t**)member2; + status = PyConfig_SetString(config, (wchar_t**)member, str); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + break; + } + case PyConfig_MEMBER_WSTR_LIST: + { + if (_PyWideStringList_Copy((PyWideStringList*)member, + (const PyWideStringList*)member2) < 0) { + return _PyStatus_NO_MEMORY(); + } + break; + } + default: + Py_UNREACHABLE(); + } + } return _PyStatus_OK(); } @@ -978,113 +1020,58 @@ _PyConfig_AsDict(const PyConfig *config) return NULL; } -#define SET_ITEM(KEY, EXPR) \ - do { \ - PyObject *obj = (EXPR); \ - if (obj == NULL) { \ - goto fail; \ - } \ - int res = PyDict_SetItemString(dict, (KEY), obj); \ - Py_DECREF(obj); \ - if (res < 0) { \ - goto fail; \ - } \ - } while (0) -#define SET_ITEM_INT(ATTR) \ - SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR)) -#define SET_ITEM_UINT(ATTR) \ - SET_ITEM(#ATTR, PyLong_FromUnsignedLong(config->ATTR)) -#define FROM_WSTRING(STR) \ - ((STR != NULL) ? \ - PyUnicode_FromWideChar(STR, -1) \ - : Py_NewRef(Py_None)) -#define SET_ITEM_WSTR(ATTR) \ - SET_ITEM(#ATTR, FROM_WSTRING(config->ATTR)) -#define SET_ITEM_WSTRLIST(LIST) \ - SET_ITEM(#LIST, _PyWideStringList_AsList(&config->LIST)) - - SET_ITEM_INT(_config_init); - SET_ITEM_INT(isolated); - SET_ITEM_INT(use_environment); - SET_ITEM_INT(dev_mode); - SET_ITEM_INT(install_signal_handlers); - SET_ITEM_INT(use_hash_seed); - SET_ITEM_UINT(hash_seed); - SET_ITEM_INT(faulthandler); - SET_ITEM_INT(tracemalloc); - SET_ITEM_INT(perf_profiling); - SET_ITEM_INT(import_time); - SET_ITEM_INT(code_debug_ranges); - SET_ITEM_INT(show_ref_count); - SET_ITEM_INT(dump_refs); - SET_ITEM_INT(malloc_stats); - SET_ITEM_WSTR(filesystem_encoding); - SET_ITEM_WSTR(filesystem_errors); - SET_ITEM_WSTR(pycache_prefix); - SET_ITEM_WSTR(program_name); - SET_ITEM_INT(parse_argv); - SET_ITEM_WSTRLIST(argv); - SET_ITEM_WSTRLIST(xoptions); - SET_ITEM_WSTRLIST(warnoptions); - SET_ITEM_WSTR(pythonpath_env); - SET_ITEM_WSTR(home); - SET_ITEM_INT(module_search_paths_set); - SET_ITEM_WSTRLIST(module_search_paths); - SET_ITEM_WSTR(stdlib_dir); - SET_ITEM_WSTR(executable); - SET_ITEM_WSTR(base_executable); - SET_ITEM_WSTR(prefix); - SET_ITEM_WSTR(base_prefix); - SET_ITEM_WSTR(exec_prefix); - SET_ITEM_WSTR(base_exec_prefix); - SET_ITEM_WSTR(platlibdir); - SET_ITEM_INT(site_import); - SET_ITEM_INT(bytes_warning); - SET_ITEM_INT(warn_default_encoding); - SET_ITEM_INT(inspect); - SET_ITEM_INT(interactive); - SET_ITEM_INT(optimization_level); - SET_ITEM_INT(parser_debug); - SET_ITEM_INT(write_bytecode); - SET_ITEM_INT(verbose); - SET_ITEM_INT(quiet); - SET_ITEM_INT(user_site_directory); - SET_ITEM_INT(configure_c_stdio); - SET_ITEM_INT(buffered_stdio); - SET_ITEM_WSTR(stdio_encoding); - SET_ITEM_WSTR(stdio_errors); -#ifdef MS_WINDOWS - SET_ITEM_INT(legacy_windows_stdio); -#endif - SET_ITEM_INT(skip_source_first_line); - SET_ITEM_WSTR(run_command); - SET_ITEM_WSTR(run_module); - SET_ITEM_WSTR(run_filename); - SET_ITEM_INT(_install_importlib); - SET_ITEM_WSTR(check_hash_pycs_mode); - SET_ITEM_INT(pathconfig_warnings); - SET_ITEM_INT(_init_main); - SET_ITEM_WSTRLIST(orig_argv); - SET_ITEM_INT(use_frozen_modules); - SET_ITEM_INT(safe_path); - SET_ITEM_INT(_is_python_build); - SET_ITEM_INT(int_max_str_digits); -#ifdef Py_STATS - SET_ITEM_INT(_pystats); -#endif + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + char *member = (char *)config + spec->offset; + PyObject *obj; + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + { + int value = *(int*)member; + obj = PyLong_FromLong(value); + break; + } + case PyConfig_MEMBER_ULONG: + { + unsigned long value = *(unsigned long*)member; + obj = PyLong_FromUnsignedLong(value); + break; + } + case PyConfig_MEMBER_WSTR: + case PyConfig_MEMBER_WSTR_OPT: + { + const wchar_t *wstr = *(const wchar_t**)member; + if (wstr != NULL) { + obj = PyUnicode_FromWideChar(wstr, -1); + } + else { + obj = Py_NewRef(Py_None); + } + break; + } + case PyConfig_MEMBER_WSTR_LIST: + { + const PyWideStringList *list = (const PyWideStringList*)member; + obj = _PyWideStringList_AsList(list); + break; + } + default: + Py_UNREACHABLE(); + } + if (obj == NULL) { + Py_DECREF(dict); + return NULL; + } + int res = PyDict_SetItemString(dict, spec->name, obj); + Py_DECREF(obj); + if (res < 0) { + Py_DECREF(dict); + return NULL; + } + } return dict; - -fail: - Py_DECREF(dict); - return NULL; - -#undef FROM_WSTRING -#undef SET_ITEM -#undef SET_ITEM_INT -#undef SET_ITEM_UINT -#undef SET_ITEM_WSTR -#undef SET_ITEM_WSTRLIST } @@ -1263,131 +1250,81 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) return -1; } -#define CHECK_VALUE(NAME, TEST) \ - if (!(TEST)) { \ - config_dict_invalid_value(NAME); \ - return -1; \ + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + char *member = (char *)config + spec->offset; + switch (spec->type) { + case PyConfig_MEMBER_INT: + if (config_dict_get_int(dict, spec->name, (int*)member) < 0) { + return -1; + } + break; + case PyConfig_MEMBER_UINT: + { + int value; + if (config_dict_get_int(dict, spec->name, &value) < 0) { + return -1; + } + if (value < 0) { + config_dict_invalid_value(spec->name); + return -1; + } + *(int*)member = value; + break; + } + case PyConfig_MEMBER_ULONG: + { + if (config_dict_get_ulong(dict, spec->name, + (unsigned long*)member) < 0) { + return -1; + } + break; + } + case PyConfig_MEMBER_WSTR: + { + wchar_t **wstr = (wchar_t**)member; + if (config_dict_get_wstr(dict, spec->name, config, wstr) < 0) { + return -1; + } + if (*wstr == NULL) { + config_dict_invalid_value(spec->name); + return -1; + } + break; + } + case PyConfig_MEMBER_WSTR_OPT: + { + wchar_t **wstr = (wchar_t**)member; + if (config_dict_get_wstr(dict, spec->name, config, wstr) < 0) { + return -1; + } + break; + } + case PyConfig_MEMBER_WSTR_LIST: + { + if (config_dict_get_wstrlist(dict, spec->name, config, + (PyWideStringList*)member) < 0) { + return -1; + } + break; + } + default: + Py_UNREACHABLE(); + } } -#define GET_UINT(KEY) \ - do { \ - if (config_dict_get_int(dict, #KEY, &config->KEY) < 0) { \ - return -1; \ - } \ - CHECK_VALUE(#KEY, config->KEY >= 0); \ - } while (0) -#define GET_INT(KEY) \ - do { \ - if (config_dict_get_int(dict, #KEY, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) -#define GET_WSTR(KEY) \ - do { \ - if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ - return -1; \ - } \ - CHECK_VALUE(#KEY, config->KEY != NULL); \ - } while (0) -#define GET_WSTR_OPT(KEY) \ - do { \ - if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) -#define GET_WSTRLIST(KEY) \ - do { \ - if (config_dict_get_wstrlist(dict, #KEY, config, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) - GET_UINT(_config_init); - CHECK_VALUE("_config_init", - config->_config_init == _PyConfig_INIT_COMPAT - || config->_config_init == _PyConfig_INIT_PYTHON - || config->_config_init == _PyConfig_INIT_ISOLATED); - GET_UINT(isolated); - GET_UINT(use_environment); - GET_UINT(dev_mode); - GET_UINT(install_signal_handlers); - GET_UINT(use_hash_seed); - if (config_dict_get_ulong(dict, "hash_seed", &config->hash_seed) < 0) { + if (!(config->_config_init == _PyConfig_INIT_COMPAT + || config->_config_init == _PyConfig_INIT_PYTHON + || config->_config_init == _PyConfig_INIT_ISOLATED)) + { + config_dict_invalid_value("_config_init"); return -1; } - CHECK_VALUE("hash_seed", config->hash_seed <= MAX_HASH_SEED); - GET_UINT(faulthandler); - GET_UINT(tracemalloc); - GET_UINT(perf_profiling); - GET_UINT(import_time); - GET_UINT(code_debug_ranges); - GET_UINT(show_ref_count); - GET_UINT(dump_refs); - GET_UINT(malloc_stats); - GET_WSTR(filesystem_encoding); - GET_WSTR(filesystem_errors); - GET_WSTR_OPT(pycache_prefix); - GET_UINT(parse_argv); - GET_WSTRLIST(orig_argv); - GET_WSTRLIST(argv); - GET_WSTRLIST(xoptions); - GET_WSTRLIST(warnoptions); - GET_UINT(site_import); - GET_UINT(bytes_warning); - GET_UINT(warn_default_encoding); - GET_UINT(inspect); - GET_UINT(interactive); - GET_UINT(optimization_level); - GET_UINT(parser_debug); - GET_UINT(write_bytecode); - GET_UINT(verbose); - GET_UINT(quiet); - GET_UINT(user_site_directory); - GET_UINT(configure_c_stdio); - GET_UINT(buffered_stdio); - GET_WSTR(stdio_encoding); - GET_WSTR(stdio_errors); -#ifdef MS_WINDOWS - GET_UINT(legacy_windows_stdio); -#endif - GET_WSTR(check_hash_pycs_mode); - - GET_UINT(pathconfig_warnings); - GET_WSTR(program_name); - GET_WSTR_OPT(pythonpath_env); - GET_WSTR_OPT(home); - GET_WSTR(platlibdir); - - // Path configuration output - GET_UINT(module_search_paths_set); - GET_WSTRLIST(module_search_paths); - GET_WSTR_OPT(stdlib_dir); - GET_WSTR_OPT(executable); - GET_WSTR_OPT(base_executable); - GET_WSTR_OPT(prefix); - GET_WSTR_OPT(base_prefix); - GET_WSTR_OPT(exec_prefix); - GET_WSTR_OPT(base_exec_prefix); - - GET_UINT(skip_source_first_line); - GET_WSTR_OPT(run_command); - GET_WSTR_OPT(run_module); - GET_WSTR_OPT(run_filename); - - GET_UINT(_install_importlib); - GET_UINT(_init_main); - GET_UINT(use_frozen_modules); - GET_UINT(safe_path); - GET_UINT(_is_python_build); - GET_INT(int_max_str_digits); -#ifdef Py_STATS - GET_UINT(_pystats); -#endif -#undef CHECK_VALUE -#undef GET_UINT -#undef GET_INT -#undef GET_WSTR -#undef GET_WSTR_OPT + if (config->hash_seed > MAX_HASH_SEED) { + config_dict_invalid_value("hash_seed"); + return -1; + } return 0; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1f398701a7a5b5..c6c69a3e222f07 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -88,6 +88,10 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - +# Internal constant list +Python/initconfig.c - PYCONFIG_SPEC - + + ##----------------------- ## public C-API From 74e425ec186dde6bcfb172616fe8f35ccb5a09bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 19:25:54 +0200 Subject: [PATCH 092/124] gh-110014: Fix _POSIX_THREADS and _POSIX_SEMAPHORES usage (#110139) * pycore_pythread.h is now the central place to make sure that _POSIX_THREADS and _POSIX_SEMAPHORES macros are defined if available. * Make sure that pycore_pythread.h is included when _POSIX_THREADS and _POSIX_SEMAPHORES macros are tested. * PY_TIMEOUT_MAX is now defined as a constant, since its value depends on _POSIX_THREADS, instead of being defined as a macro. * Prevent integer overflow in the preprocessor when computing PY_TIMEOUT_MAX_VALUE on Windows: replace "0xFFFFFFFELL * 1000 < LLONG_MAX" with "0xFFFFFFFELL < LLONG_MAX / 1000". * Document the change and give hints how to fix affected code. * Add an exception for PY_TIMEOUT_MAX name to smelly.py * Add PY_TIMEOUT_MAX to the stable ABI --- Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.13.rst | 4 +++ Include/internal/pycore_condvar.h | 12 +------- Include/internal/pycore_pythread.h | 46 ++++++++++++++--------------- Include/internal/pycore_semaphore.h | 4 ++- Include/pythread.h | 17 +---------- Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 4 +++ PC/python3dll.c | 1 + Python/condvar.h | 3 +- Python/thread.c | 22 +++++++++++++- Python/thread_pthread.h | 9 +++--- Tools/build/smelly.py | 7 ++++- 13 files changed, 73 insertions(+), 58 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index c189c78238f40f..07c6d514d19549 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,4 +1,5 @@ role,name,added,ifdef_note,struct_abi_kind +var,PY_TIMEOUT_MAX,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, function,PyAIter_Check,3.10,, function,PyArg_Parse,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c9e6ca8bf88866..d6188e63dd23e6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -989,6 +989,10 @@ Porting to Python 3.13 * ``Python.h`` no longer includes the ```` standard header file. If needed, it should now be included explicitly. For example, it provides the functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. + As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no + longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` + macros defined by ``Python.h`` can be used to decide if ```` and + ```` header files can be included. (Contributed by Victor Stinner in :gh:`108765`.) * ``Python.h`` no longer includes these standard header files: ````, diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 489e67d4ec4f9f..34c21aaad43197 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,18 +5,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#ifndef MS_WINDOWS -# include // _POSIX_THREADS -#endif +#include "pycore_pythread.h" // _POSIX_THREADS -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include // _POSIX_THREADS -# endif -#endif #ifdef _POSIX_THREADS /* diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 5ec2abda91e86b..98019c586bc5e2 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -9,29 +9,29 @@ extern "C" { #endif -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include // _POSIX_THREADS -# endif -# ifndef _POSIX_THREADS -/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implemented to support python - threads. - - This is valid for HP-UX 11.23 running on an ia64 system. If needed, add - a check of __ia64 to verify that we're running on an ia64 system instead - of a pa-risc system. -*/ -# ifdef __hpux -# ifdef _SC_THREADS -# define _POSIX_THREADS -# endif -# endif -# endif /* _POSIX_THREADS */ -#endif /* _POSIX_THREADS */ +// Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available +#if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) +# include // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if (defined(HAVE_PTHREAD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) + // This means pthreads are not implemented in libc headers, hence the macro + // not present in . But they still can be implemented as an + // external library (e.g. gnu pth in pthread emulation) +# include // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if !defined(_POSIX_THREADS) && defined(__hpux) && defined(_SC_THREADS) + // Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + // enough of the POSIX threads package is implemented to support Python + // threads. + // + // This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + // a check of __ia64 to verify that we're running on an ia64 system instead + // of a pa-risc system. +# define _POSIX_THREADS +#endif + #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) # define _USE_PTHREADS diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 2a4ecb7147acee..4c37df7b39a48a 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -7,7 +7,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t +#include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -26,6 +27,7 @@ # include #endif + #ifdef __cplusplus extern "C" { #endif diff --git a/Include/pythread.h b/Include/pythread.h index 63714437c496b7..2c2fd63d724286 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -44,22 +44,7 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); */ #define PY_TIMEOUT_T long long -#if defined(_POSIX_THREADS) - /* PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - convert microseconds to nanoseconds. */ -# define PY_TIMEOUT_MAX (LLONG_MAX / 1000) -#elif defined (NT_THREADS) - // WaitForSingleObject() accepts timeout in milliseconds in the range - // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no - // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. -# if 0xFFFFFFFELL * 1000 < LLONG_MAX -# define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000) -# else -# define PY_TIMEOUT_MAX LLONG_MAX -# endif -#else -# define PY_TIMEOUT_MAX LLONG_MAX -#endif +PyAPI_DATA(const long long) PY_TIMEOUT_MAX; /* If microseconds == 0, the call is non-blocking: it returns immediately diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 94f817f8e1d159..6e9496d40da477 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -35,6 +35,7 @@ def test_windows_feature_macros(self): SYMBOL_NAMES = ( + "PY_TIMEOUT_MAX", "PyAIter_Check", "PyArg_Parse", "PyArg_ParseTuple", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8df3f85e61eec6..46e2307614e26d 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1843,6 +1843,10 @@ [function.PyThread_start_new_thread] added = '3.2' +# Not mentioned in PEP 384, was implemented as a macro in Python <= 3.12 +[data.PY_TIMEOUT_MAX] + added = '3.2' + # The following were added in PC/python3.def in Python 3.3: # 7800f75827b1be557be16f3b18f5170fbf9fae08 # 9c56409d3353b8cd4cfc19e0467bbe23fd34fc92 diff --git a/PC/python3dll.c b/PC/python3dll.c index 2c1cc8098ce856..75728c7d8057ed 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -768,6 +768,7 @@ EXPORT_DATA(Py_FileSystemDefaultEncodeErrors) EXPORT_DATA(Py_FileSystemDefaultEncoding) EXPORT_DATA(Py_GenericAliasType) EXPORT_DATA(Py_HasFileSystemDefaultEncoding) +EXPORT_DATA(PY_TIMEOUT_MAX) EXPORT_DATA(Py_UTF8Mode) EXPORT_DATA(Py_Version) EXPORT_DATA(PyBaseObject_Type) diff --git a/Python/condvar.h b/Python/condvar.h index 4ddc5311cf8fad..d54db94f2c871d 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -41,7 +41,8 @@ #define _CONDVAR_IMPL_H_ #include "Python.h" -#include "pycore_condvar.h" +#include "pycore_pythread.h" // _POSIX_THREADS + #ifdef _POSIX_THREADS /* diff --git a/Python/thread.c b/Python/thread.c index 1ac2db2937e373..bf207cecb90505 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -8,7 +8,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_pythread.h" +#include "pycore_pythread.h" // _POSIX_THREADS #ifndef DONT_HAVE_STDIO_H # include @@ -17,6 +17,26 @@ #include +// Define PY_TIMEOUT_MAX constant. +#ifdef _POSIX_THREADS + // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), + // convert microseconds to nanoseconds. +# define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000) +#elif defined (NT_THREADS) + // WaitForSingleObject() accepts timeout in milliseconds in the range + // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no + // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. +# if 0xFFFFFFFELL < LLONG_MAX / 1000 +# define PY_TIMEOUT_MAX_VALUE (0xFFFFFFFELL * 1000) +# else +# define PY_TIMEOUT_MAX_VALUE LLONG_MAX +# endif +#else +# define PY_TIMEOUT_MAX_VALUE LLONG_MAX +#endif +const long long PY_TIMEOUT_MAX = PY_TIMEOUT_MAX_VALUE; + + static void PyThread__init_thread(void); /* Forward */ #define initialized _PyRuntime.threads.initialized diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index f96c57da64636d..76a1f7763f23b9 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,4 +1,5 @@ -#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_pythread.h" // _POSIX_SEMAPHORES /* Posix threads interface */ @@ -84,10 +85,10 @@ /* On FreeBSD 4.x, _POSIX_SEMAPHORES is defined empty, so we need to add 0 to make it work there as well. */ #if (_POSIX_SEMAPHORES+0) == -1 -#define HAVE_BROKEN_POSIX_SEMAPHORES +# define HAVE_BROKEN_POSIX_SEMAPHORES #else -#include -#include +# include +# include #endif #endif diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 276a5ab2cc84c6..ab345307ff9b64 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -11,6 +11,11 @@ if sys.platform == 'darwin': ALLOWED_PREFIXES += ('__Py',) +# "Legacy": some old symbols are prefixed by "PY_". +EXCEPTIONS = frozenset({ + 'PY_TIMEOUT_MAX', +}) + IGNORED_EXTENSION = "_ctypes_test" # Ignore constructor and destructor functions IGNORED_SYMBOLS = {'_init', '_fini'} @@ -72,7 +77,7 @@ def get_smelly_symbols(stdout): symbol = parts[-1] result = '%s (type: %s)' % (symbol, symtype) - if symbol.startswith(ALLOWED_PREFIXES): + if symbol.startswith(ALLOWED_PREFIXES) or symbol in EXCEPTIONS: python_symbols.append(result) continue From 0def8c712bb6f66f1081cab71deb3681566b846d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 20:23:26 +0200 Subject: [PATCH 093/124] gh-109748: Fix again venv test_zippath_from_non_installed_posix() (#110149) Call also copy_python_src_ignore() on listdir() names. shutil.copytree(): replace set() with an empty tuple. An empty tuple becomes a constant in the compiler and checking if an item is in an empty tuple is cheap. --- Lib/shutil.py | 2 +- Lib/test/test_support.py | 2 +- Lib/test/test_venv.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index b37bd082eee0c6..b903f13d8b76a7 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -481,7 +481,7 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, if ignore is not None: ignored_names = ignore(os.fspath(src), [x.name for x in entries]) else: - ignored_names = set() + ignored_names = () os.makedirs(dst, exist_ok=dirs_exist_ok) errors = [] diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 902bec78451307..97de81677b10bc 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -832,7 +832,7 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored | {'build', 'venv'}) - # An other directory + # Another directory path = os.path.join(src_dir, 'Objects') self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 0ffe3e1d0cc498..890672c5d27eec 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -569,7 +569,11 @@ def test_zippath_from_non_installed_posix(self): eachpath, os.path.join(non_installed_dir, platlibdir)) elif os.path.isfile(os.path.join(eachpath, "os.py")): - for name in os.listdir(eachpath): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue if name == "site-packages": continue fn = os.path.join(eachpath, name) From 7513994c927857679544449392744be308d36586 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 22:06:45 +0200 Subject: [PATCH 094/124] gh-110014: Include explicitly header (#110155) * Remove unused includes. * Remove unused include in traceback.h. * Remove redundant and includes. They are already included by "Python.h". * Remove include in faulthandler.c. Python.h already includes it. * Add missing in pycore_pythread.h if HAVE_PTHREAD_STUBS is defined. * Fix also warnings in pthread_stubs.h: don't redefine macros if they are already defined, like the __NEED_pthread_t macro. --- Doc/whatsnew/3.13.rst | 3 ++- Include/cpython/pthread_stubs.h | 30 ++++++++++++++++++++------ Include/internal/pycore_pythread.h | 3 ++- Modules/_io/fileio.c | 34 +++++++++++++++++------------- Modules/_randommodule.c | 7 ++++-- Modules/faulthandler.c | 9 +++++--- Modules/posixmodule.c | 9 ++++---- Objects/fileobject.c | 18 ++++++++++------ Parser/myreadline.c | 6 +++++- Parser/tokenizer.c | 8 ++++--- Python/bltinmodule.c | 5 +++++ Python/bootstrap_hash.c | 14 ++++++++---- Python/fileutils.c | 13 +++++++----- Python/frozenmain.c | 6 +++++- Python/pylifecycle.c | 3 +++ Python/sysmodule.c | 4 +++- Python/traceback.c | 7 +++--- 17 files changed, 120 insertions(+), 59 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d6188e63dd23e6..56618b9af16b95 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -988,7 +988,8 @@ Porting to Python 3.13 * ``Python.h`` no longer includes the ```` standard header file. If needed, it should now be included explicitly. For example, it provides the - functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``. + functions: ``read()``, ``write()``, ``close()``, ``isatty()``, ``lseek()``, + ``getpid()``, ``getcwd()``, ``sysconf()`` and ``getpagesize()``. As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` macros defined by ``Python.h`` can be used to decide if ```` and diff --git a/Include/cpython/pthread_stubs.h b/Include/cpython/pthread_stubs.h index d95ee03d8308ce..5246968ea05476 100644 --- a/Include/cpython/pthread_stubs.h +++ b/Include/cpython/pthread_stubs.h @@ -21,13 +21,29 @@ #ifdef __wasi__ // WASI's bits/alltypes.h provides type definitions when __NEED_ is set. // The header file can be included multiple times. -# define __NEED_pthread_cond_t 1 -# define __NEED_pthread_condattr_t 1 -# define __NEED_pthread_mutex_t 1 -# define __NEED_pthread_mutexattr_t 1 -# define __NEED_pthread_key_t 1 -# define __NEED_pthread_t 1 -# define __NEED_pthread_attr_t 1 +// +// may also define these macros. +# ifndef __NEED_pthread_cond_t +# define __NEED_pthread_cond_t 1 +# endif +# ifndef __NEED_pthread_condattr_t +# define __NEED_pthread_condattr_t 1 +# endif +# ifndef __NEED_pthread_mutex_t +# define __NEED_pthread_mutex_t 1 +# endif +# ifndef __NEED_pthread_mutexattr_t +# define __NEED_pthread_mutexattr_t 1 +# endif +# ifndef __NEED_pthread_key_t +# define __NEED_pthread_key_t 1 +# endif +# ifndef __NEED_pthread_t +# define __NEED_pthread_t 1 +# endif +# ifndef __NEED_pthread_attr_t +# define __NEED_pthread_attr_t 1 +# endif # include #else typedef struct { void *__x; } pthread_cond_t; diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 98019c586bc5e2..f679c1bdb75499 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ && !defined(_POSIX_SEMAPHORES)) @@ -44,6 +43,8 @@ extern "C" { #if defined(HAVE_PTHREAD_STUBS) +#include // bool + // pthread_key struct py_stub_tls_entry { bool in_use; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index fb416700e22523..8a73ea0365b7a3 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -5,20 +5,23 @@ #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include +#include // bool +#ifdef HAVE_UNISTD_H +# include // lseek() +#endif #ifdef HAVE_SYS_TYPES_H -#include +# include #endif #ifdef HAVE_SYS_STAT_H -#include +# include #endif #ifdef HAVE_IO_H -#include +# include #endif #ifdef HAVE_FCNTL_H -#include +# include // open() #endif -#include /* For offsetof */ + #include "_iomodule.h" /* @@ -35,22 +38,23 @@ */ #ifdef MS_WINDOWS -/* can simulate truncate with Win32 API functions; see file_truncate */ -#define HAVE_FTRUNCATE -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include + // can simulate truncate with Win32 API functions; see file_truncate +# define HAVE_FTRUNCATE +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include #endif #if BUFSIZ < (8*1024) -#define SMALLCHUNK (8*1024) +# define SMALLCHUNK (8*1024) #elif (BUFSIZ >= (2 << 25)) -#error "unreasonable BUFSIZ > 64 MiB defined" +# error "unreasonable BUFSIZ > 64 MiB defined" #else -#define SMALLCHUNK BUFSIZ +# define SMALLCHUNK BUFSIZ #endif + /*[clinic input] module _io class _io.FileIO "fileio *" "clinic_state()->PyFileIO_Type" diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 18811d03adb451..d41093c8806476 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -74,12 +74,15 @@ #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() + +#ifdef HAVE_UNISTD_H +# include // getpid() +#endif #ifdef HAVE_PROCESS_H # include // getpid() #endif - #ifdef MS_WINDOWS -# include +# include // GetCurrentProcessId() #endif /* Period parameters -- These are all magic. Don't change. */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 4b6bf68be07202..a2e3c2300b3ce8 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -6,8 +6,10 @@ #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DumpTracebackThreads -#include -#include +#ifdef HAVE_UNISTD_H +# include // _exit() +#endif +#include // sigaction() #include // abort() #if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) && defined(HAVE_PTHREAD_H) # include @@ -16,7 +18,7 @@ # include #endif #ifdef HAVE_SYS_RESOURCE_H -# include +# include // setrlimit() #endif #if defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_LINUX_AUXVEC_H) && defined(HAVE_SYS_AUXV_H) @@ -24,6 +26,7 @@ # include // getauxval() #endif + /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d7d3e365d2c553..0b252092573e5c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -24,6 +24,10 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG +#ifdef HAVE_UNISTD_H +# include // symlink() +#endif + #ifdef MS_WINDOWS # include # if !defined(MS_WINDOWS_GAMES) || defined(MS_WINDOWS_DESKTOP) @@ -37,7 +41,6 @@ # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ #endif - #ifndef MS_WINDOWS # include "posixmodule.h" #else @@ -285,10 +288,6 @@ corresponding Unix manual entries for more information on calls."); # include #endif -#ifdef HAVE_COPY_FILE_RANGE -# include // copy_file_range() -#endif - #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) # undef HAVE_SCHED_SETAFFINITY #endif diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 066172baf9f027..5522eba34eace9 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -4,15 +4,19 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_runtime.h" // _PyRuntime +#ifdef HAVE_UNISTD_H +# include // isatty() +#endif + #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER) -/* clang MemorySanitizer doesn't yet understand getc_unlocked. */ -#define GETC(f) getc_unlocked(f) -#define FLOCKFILE(f) flockfile(f) -#define FUNLOCKFILE(f) funlockfile(f) + /* clang MemorySanitizer doesn't yet understand getc_unlocked. */ +# define GETC(f) getc_unlocked(f) +# define FLOCKFILE(f) flockfile(f) +# define FUNLOCKFILE(f) funlockfile(f) #else -#define GETC(f) getc(f) -#define FLOCKFILE(f) -#define FUNLOCKFILE(f) +# define GETC(f) getc(f) +# define FLOCKFILE(f) +# define FUNLOCKFILE(f) #endif /* Newline flags */ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 815387388218c6..719a178f244a28 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -14,11 +14,15 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS # ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN # endif # include "windows.h" #endif /* MS_WINDOWS */ +#ifdef HAVE_UNISTD_H +# include // isatty() +#endif + // Export the symbol since it's used by the readline shared extension PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 46b7159ff0516b..41d0d16a471dd6 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -4,10 +4,12 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() -#include +#include "tokenizer.h" // struct tok_state +#include "errcode.h" // E_OK -#include "tokenizer.h" -#include "errcode.h" +#ifdef HAVE_UNISTD_H +# include // read() +#endif /* Alternate tab spacing */ #define ALTTABSIZE 1 diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 69056bf23f4058..c373585c0986ce 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -17,6 +17,11 @@ #include "clinic/bltinmodule.c.h" +#ifdef HAVE_UNISTD_H +# include // isatty() +#endif + + static PyObject* update_bases(PyObject *bases, PyObject *const *args, Py_ssize_t nargs) { diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index ef693e5df1fcc4..86a16916304cab 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -4,22 +4,28 @@ #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _PyRuntime +#undef HAVE_GETRANDOM +#undef HAVE_GETENTROPY + +#ifdef HAVE_UNISTD_H +# include // close() +#endif #ifdef MS_WINDOWS # include # include #else -# include +# include // O_RDONLY # ifdef HAVE_SYS_STAT_H # include # endif # ifdef HAVE_LINUX_RANDOM_H -# include +# include // GRND_NONBLOCK # endif # if defined(HAVE_SYS_RANDOM_H) && (defined(HAVE_GETRANDOM) || defined(HAVE_GETENTROPY)) -# include +# include // getrandom() # endif # if !defined(HAVE_GETRANDOM) && defined(HAVE_GETRANDOM_SYSCALL) -# include +# include // SYS_getrandom # endif #endif diff --git a/Python/fileutils.c b/Python/fileutils.c index 9bc1de2db84006..17a4ae56ef0528 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2,8 +2,11 @@ #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "osdefs.h" // SEP -#include + #include // mbstowcs() +#ifdef HAVE_UNISTD_H +# include // getcwd() +#endif #ifdef MS_WINDOWS # include @@ -19,7 +22,7 @@ extern int winerror_to_errno(int); #endif #ifdef HAVE_LANGINFO_H -#include +# include // nl_langinfo(CODESET) #endif #ifdef HAVE_SYS_IOCTL_H @@ -27,12 +30,12 @@ extern int winerror_to_errno(int); #endif #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION -#include +# include // iconv_open() #endif #ifdef HAVE_FCNTL_H -#include -#endif /* HAVE_FCNTL_H */ +# include // fcntl(F_GETFD) +#endif #ifdef O_CLOEXEC /* Does open() support the O_CLOEXEC flag? Possible values: diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 767f9804903a9e..3ce9476c9ad46c 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -3,7 +3,11 @@ #include "Python.h" #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_runtime.h" // _PyRuntime_Initialize() -#include + +#ifdef HAVE_UNISTD_H +# include // isatty() +#endif + #ifdef MS_WINDOWS extern void PyWinFreeze_ExeInit(void); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 23f66ec3601df6..f3ed77e516237a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -37,6 +37,9 @@ #include // setlocale() #include // getenv() +#ifdef HAVE_UNISTD_H +# include // isatty() +#endif #if defined(__APPLE__) # include diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7ba7be10aacb92..b00301765e1890 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -40,7 +40,9 @@ Data members: #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names -#include +#ifdef HAVE_UNISTD_H +# include // getpid() +#endif #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN diff --git a/Python/traceback.c b/Python/traceback.c index 7e791d0a59bd82..5de1bff9943c6c 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -20,13 +20,14 @@ #include "frameobject.h" // PyFrame_New() #include "osdefs.h" // SEP -#ifdef HAVE_FCNTL_H -# include +#ifdef HAVE_UNISTD_H +# include // lseek() #endif -#define OFF(x) offsetof(PyTracebackObject, x) +#define OFF(x) offsetof(PyTracebackObject, x) #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) + #define MAX_STRING_LENGTH 500 #define MAX_FRAME_DEPTH 100 #define MAX_NTHREADS 100 From c62b49ecc8da13fa9522865ef6fe0aec194fd0d8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 22:40:10 +0200 Subject: [PATCH 095/124] gh-110088: Fix asyncio test_prompt_cancellation() (#110157) Don't measure the CI performance: don't test the maximum elapsed time. The check failed on a slow CI. --- Lib/test/test_asyncio/test_events.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 3ee6565b2b65ad..b25c0975736e20 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1693,12 +1693,9 @@ async def main(): self.loop.stop() return res - start = time.monotonic() t = self.loop.create_task(main()) self.loop.run_forever() - elapsed = time.monotonic() - start - self.assertLess(elapsed, 0.1) self.assertEqual(t.result(), 'cancelled') self.assertRaises(asyncio.CancelledError, f.result) if ov is not None: From 2c234196ea30b9da370780204ed9068f1fb134c6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 22:48:26 +0200 Subject: [PATCH 096/124] gh-109276: regrtest: add WORKER_FAILED state (#110148) Rename WORKER_ERROR to WORKER_BUG. Add WORKER_FAILED state: it does not stop the manager, whereas WORKER_BUG does. Change also TestResults.display_result() order: display failed tests at the end, the important important information. WorkerThread now tries to get the signal name for negative exit code. --- Lib/test/libregrtest/result.py | 18 ++++++++++------ Lib/test/libregrtest/results.py | 32 ++++++++++++++++++----------- Lib/test/libregrtest/run_workers.py | 29 +++++++++++++++++--------- Lib/test/libregrtest/utils.py | 22 ++++++++++++++++++++ Lib/test/test_regrtest.py | 10 +++++++++ 5 files changed, 83 insertions(+), 28 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index bf885264657d5c..d6b0d5ad383a5b 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -19,7 +19,8 @@ class State: ENV_CHANGED = "ENV_CHANGED" RESOURCE_DENIED = "RESOURCE_DENIED" INTERRUPTED = "INTERRUPTED" - MULTIPROCESSING_ERROR = "MULTIPROCESSING_ERROR" + WORKER_FAILED = "WORKER_FAILED" # non-zero worker process exit code + WORKER_BUG = "WORKER_BUG" # exception when running a worker DID_NOT_RUN = "DID_NOT_RUN" TIMEOUT = "TIMEOUT" @@ -29,7 +30,8 @@ def is_failed(state): State.FAILED, State.UNCAUGHT_EXC, State.REFLEAK, - State.MULTIPROCESSING_ERROR, + State.WORKER_FAILED, + State.WORKER_BUG, State.TIMEOUT} @staticmethod @@ -42,14 +44,16 @@ def has_meaningful_duration(state): State.SKIPPED, State.RESOURCE_DENIED, State.INTERRUPTED, - State.MULTIPROCESSING_ERROR, + State.WORKER_FAILED, + State.WORKER_BUG, State.DID_NOT_RUN} @staticmethod def must_stop(state): return state in { State.INTERRUPTED, - State.MULTIPROCESSING_ERROR} + State.WORKER_BUG, + } @dataclasses.dataclass(slots=True) @@ -108,8 +112,10 @@ def __str__(self) -> str: return f"{self.test_name} skipped (resource denied)" case State.INTERRUPTED: return f"{self.test_name} interrupted" - case State.MULTIPROCESSING_ERROR: - return f"{self.test_name} process crashed" + case State.WORKER_FAILED: + return f"{self.test_name} worker non-zero exit code" + case State.WORKER_BUG: + return f"{self.test_name} worker bug" case State.DID_NOT_RUN: return f"{self.test_name} ran no tests" case State.TIMEOUT: diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 35df50d581ff6a..3708078ff0bf3a 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -30,6 +30,7 @@ def __init__(self): self.rerun_results: list[TestResult] = [] self.interrupted: bool = False + self.worker_bug: bool = False self.test_times: list[tuple[float, TestName]] = [] self.stats = TestStats() # used by --junit-xml @@ -38,7 +39,8 @@ def __init__(self): def is_all_good(self): return (not self.bad and not self.skipped - and not self.interrupted) + and not self.interrupted + and not self.worker_bug) def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) @@ -60,6 +62,8 @@ def get_state(self, fail_env_changed): if self.interrupted: state.append("INTERRUPTED") + if self.worker_bug: + state.append("WORKER BUG") if not state: state.append("SUCCESS") @@ -77,6 +81,8 @@ def get_exitcode(self, fail_env_changed, fail_rerun): exitcode = EXITCODE_NO_TESTS_RAN elif fail_rerun and self.rerun: exitcode = EXITCODE_RERUN_FAIL + elif self.worker_bug: + exitcode = EXITCODE_BAD_TEST return exitcode def accumulate_result(self, result: TestResult, runtests: RunTests): @@ -105,6 +111,9 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): else: raise ValueError(f"invalid test state: {result.state!r}") + if result.state == State.WORKER_BUG: + self.worker_bug = True + if result.has_meaningful_duration() and not rerun: self.test_times.append((result.duration, test_name)) if result.stats is not None: @@ -173,12 +182,6 @@ def write_junit(self, filename: StrPath): f.write(s) def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): - omitted = set(tests) - self.get_executed() - if omitted: - print() - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if print_slowest: self.test_times.sort(reverse=True) print() @@ -186,16 +189,21 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): for test_time, test in self.test_times[:10]: print("- %s: %s" % (test, format_duration(test_time))) - all_tests = [ - (self.bad, "test", "{} failed:"), - (self.env_changed, "test", "{} altered the execution environment (env changed):"), - ] + all_tests = [] + omitted = set(tests) - self.get_executed() + + # less important + all_tests.append((omitted, "test", "{} omitted:")) if not quiet: all_tests.append((self.skipped, "test", "{} skipped:")) all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) - all_tests.append((self.rerun, "re-run test", "{}:")) all_tests.append((self.run_no_tests, "test", "{} run no tests:")) + # more important + all_tests.append((self.env_changed, "test", "{} altered the execution environment (env changed):")) + all_tests.append((self.rerun, "re-run test", "{}:")) + all_tests.append((self.bad, "test", "{} failed:")) + for tests_list, count_text, title_format in all_tests: if tests_list: print() diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 41ed7b0bac01ad..6eb32e59635865 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, MS_WINDOWS, - format_duration, print_warning, count, plural) + format_duration, print_warning, count, plural, get_signal_name) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -92,7 +92,7 @@ def __init__(self, test_name: TestName, err_msg: str | None, stdout: str | None, - state: str = State.MULTIPROCESSING_ERROR): + state: str): result = TestResult(test_name, state=state) self.mp_result = MultiprocessResult(result, stdout, err_msg) super().__init__() @@ -298,7 +298,9 @@ def read_stdout(self, stdout_file: TextIO) -> str: # gh-101634: Catch UnicodeDecodeError if stdout cannot be # decoded from encoding raise WorkerError(self.test_name, - f"Cannot read process stdout: {exc}", None) + f"Cannot read process stdout: {exc}", + stdout=None, + state=State.WORKER_BUG) def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, stdout: str) -> tuple[TestResult, str]: @@ -317,10 +319,11 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, # decoded from encoding err_msg = f"Failed to read worker process JSON: {exc}" raise WorkerError(self.test_name, err_msg, stdout, - state=State.MULTIPROCESSING_ERROR) + state=State.WORKER_BUG) if not worker_json: - raise WorkerError(self.test_name, "empty JSON", stdout) + raise WorkerError(self.test_name, "empty JSON", stdout, + state=State.WORKER_BUG) try: result = TestResult.from_json(worker_json) @@ -329,7 +332,7 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, # decoded from encoding err_msg = f"Failed to parse worker process JSON: {exc}" raise WorkerError(self.test_name, err_msg, stdout, - state=State.MULTIPROCESSING_ERROR) + state=State.WORKER_BUG) return (result, stdout) @@ -345,9 +348,15 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: stdout = self.read_stdout(stdout_file) if retcode is None: - raise WorkerError(self.test_name, None, stdout, state=State.TIMEOUT) + raise WorkerError(self.test_name, stdout=stdout, + err_msg=None, + state=State.TIMEOUT) if retcode != 0: - raise WorkerError(self.test_name, f"Exit code {retcode}", stdout) + name = get_signal_name(retcode) + if name: + retcode = f"{retcode} ({name})" + raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, + state=State.WORKER_FAILED) result, stdout = self.read_json(json_file, json_tmpfile, stdout) @@ -527,7 +536,7 @@ def display_result(self, mp_result: MultiprocessResult) -> None: text = str(result) if mp_result.err_msg: - # MULTIPROCESSING_ERROR + # WORKER_BUG text += ' (%s)' % mp_result.err_msg elif (result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) @@ -543,7 +552,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") - result = TestResult("", state=State.MULTIPROCESSING_ERROR) + result = TestResult("", state=State.WORKER_BUG) self.results.accumulate_result(result, self.runtests) return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 46451152b8859f..dc1fa51b80dea1 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,6 +5,7 @@ import os.path import platform import random +import signal import sys import sysconfig import tempfile @@ -581,3 +582,24 @@ def cleanup_temp_dir(tmp_dir: StrPath): else: print("Remove file: %s" % name) os_helper.unlink(name) + +WINDOWS_STATUS = { + 0xC0000005: "STATUS_ACCESS_VIOLATION", + 0xC00000FD: "STATUS_STACK_OVERFLOW", + 0xC000013A: "STATUS_CONTROL_C_EXIT", +} + +def get_signal_name(exitcode): + if exitcode < 0: + signum = -exitcode + try: + return signal.Signals(signum).name + except ValueError: + pass + + try: + return WINDOWS_STATUS[exitcode] + except KeyError: + pass + + return None diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c98b05abcea98c..e940cf04321d04 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -14,6 +14,7 @@ import random import re import shlex +import signal import subprocess import sys import sysconfig @@ -2066,6 +2067,15 @@ def test_normalize_test_name(self): self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(utils.get_signal_name(exitcode), expected, exitcode) + if __name__ == '__main__': unittest.main() From c81521020d643b4a5183098470ef7e6470facefb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 1 Oct 2023 00:12:51 +0200 Subject: [PATCH 097/124] gh-109649: Add os.process_cpu_count() function (#109907) * Refactor os_sched_getaffinity_impl(): move variable definitions to their first assignment. * Fix test_posix.test_sched_getaffinity(): restore the old CPU mask when the test completes! * Doc: Specify that os.cpu_count() counts *logicial* CPUs. * Doc: Specify that os.sched_getaffinity(0) is related to the calling thread. --- Doc/library/os.rst | 31 ++++++-- Doc/whatsnew/3.13.rst | 7 ++ Lib/os.py | 14 ++++ Lib/test/test_os.py | 36 ++++++++- Lib/test/test_posix.py | 1 + ...-09-21-16-21-19.gh-issue-109649.YYCjAF.rst | 2 + Modules/clinic/posixmodule.c.h | 8 +- Modules/posixmodule.c | 73 +++++++++++-------- 8 files changed, 125 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4ffd520f9ecd8b..141ab0bff5b4bf 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5141,8 +5141,12 @@ operating system. .. function:: sched_getaffinity(pid, /) - Return the set of CPUs the process with PID *pid* (or the current process - if zero) is restricted to. + Return the set of CPUs the process with PID *pid* is restricted to. + + If *pid* is zero, return the set of CPUs the calling thread of the current + process is restricted to. + + See also the :func:`process_cpu_count` function. .. _os-path: @@ -5183,12 +5187,11 @@ Miscellaneous System Information .. function:: cpu_count() - Return the number of CPUs in the system. Returns ``None`` if undetermined. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` + Return the number of logical CPUs in the **system**. Returns ``None`` if + undetermined. + The :func:`process_cpu_count` function can be used to get the number of + logical CPUs usable by the calling thread of the **current process**. .. versionadded:: 3.4 @@ -5202,6 +5205,20 @@ Miscellaneous System Information .. availability:: Unix. +.. function:: process_cpu_count() + + Get the number of logical CPUs usable by the calling thread of the **current + process**. Returns ``None`` if undetermined. It can be less than + :func:`cpu_count` depending on the CPU affinity. + + The :func:`cpu_count` function can be used to get the number of logical CPUs + in the **system**. + + See also the :func:`sched_getaffinity` functions. + + .. versionadded:: 3.13 + + .. function:: sysconf(name, /) Return integer-valued system configuration values. If the configuration value diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 56618b9af16b95..484443a086fdd6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -163,6 +163,13 @@ opcode documented or exposed through ``dis``, and were not intended to be used externally. +os +-- + +* Add :func:`os.process_cpu_count` function to get the number of logical CPUs + usable by the calling thread of the current process. + (Contributed by Victor Stinner in :gh:`109649`.) + pathlib ------- diff --git a/Lib/os.py b/Lib/os.py index d8c9ba4b15400a..35842cedf14fc7 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1136,3 +1136,17 @@ def add_dll_directory(path): cookie, nt._remove_dll_directory ) + + +if _exists('sched_getaffinity'): + def process_cpu_count(): + """ + Get the number of CPUs of the current process. + + Return the number of logical CPUs usable by the calling thread of the + current process. Return None if indeterminable. + """ + return len(sched_getaffinity(0)) +else: + # Just an alias to cpu_count() (same docstring) + process_cpu_count = cpu_count diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 66aece2c4b3eb9..c1a78a70c09441 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3996,14 +3996,42 @@ def test_oserror_filename(self): self.fail(f"No exception thrown by {func}") class CPUCountTests(unittest.TestCase): + def check_cpu_count(self, cpus): + if cpus is None: + self.skipTest("Could not determine the number of CPUs") + + self.assertIsInstance(cpus, int) + self.assertGreater(cpus, 0) + def test_cpu_count(self): cpus = os.cpu_count() - if cpus is not None: - self.assertIsInstance(cpus, int) - self.assertGreater(cpus, 0) - else: + self.check_cpu_count(cpus) + + def test_process_cpu_count(self): + cpus = os.process_cpu_count() + self.assertLessEqual(cpus, os.cpu_count()) + self.check_cpu_count(cpus) + + @unittest.skipUnless(hasattr(os, 'sched_setaffinity'), + "don't have sched affinity support") + def test_process_cpu_count_affinity(self): + ncpu = os.cpu_count() + if ncpu is None: self.skipTest("Could not determine the number of CPUs") + # Disable one CPU + mask = os.sched_getaffinity(0) + if len(mask) <= 1: + self.skipTest(f"sched_getaffinity() returns less than " + f"2 CPUs: {sorted(mask)}") + self.addCleanup(os.sched_setaffinity, 0, list(mask)) + mask.pop() + os.sched_setaffinity(0, mask) + + # test process_cpu_count() + affinity = os.process_cpu_count() + self.assertEqual(affinity, ncpu - 1) + # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 444f8abe4607b7..9d72dba159c6be 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1205,6 +1205,7 @@ def test_sched_getaffinity(self): @requires_sched_affinity def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) + self.addCleanup(posix.sched_setaffinity, 0, list(mask)) if len(mask) > 1: # Empty masks are forbidden mask.pop() diff --git a/Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst b/Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst new file mode 100644 index 00000000000000..ab708e6fb9a7d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-21-16-21-19.gh-issue-109649.YYCjAF.rst @@ -0,0 +1,2 @@ +Add :func:`os.process_cpu_count` function to get the number of logical CPUs +usable by the calling thread of the current process. Patch by Victor Stinner. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index e77a31b947f45e..fc39ab72bf2a51 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10425,11 +10425,9 @@ PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" "\n" -"Return the number of CPUs in the system; return None if indeterminable.\n" +"Return the number of logical CPUs in the system.\n" "\n" -"This number is not equivalent to the number of CPUs the current process can\n" -"use. The number of usable CPUs can be obtained with\n" -"``len(os.sched_getaffinity(0))``"); +"Return None if indeterminable."); #define OS_CPU_COUNT_METHODDEF \ {"cpu_count", (PyCFunction)os_cpu_count, METH_NOARGS, os_cpu_count__doc__}, @@ -11988,4 +11986,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=51aa26bc6a41e1da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8b60de6ddb925bc3 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0b252092573e5c..d3c0aa6f3c5382 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8133,39 +8133,45 @@ static PyObject * os_sched_getaffinity_impl(PyObject *module, pid_t pid) /*[clinic end generated code: output=f726f2c193c17a4f input=983ce7cb4a565980]*/ { - int cpu, ncpus, count; + int ncpus = NCPUS_START; size_t setsize; - cpu_set_t *mask = NULL; - PyObject *res = NULL; + cpu_set_t *mask; - ncpus = NCPUS_START; while (1) { setsize = CPU_ALLOC_SIZE(ncpus); mask = CPU_ALLOC(ncpus); - if (mask == NULL) + if (mask == NULL) { return PyErr_NoMemory(); - if (sched_getaffinity(pid, setsize, mask) == 0) + } + if (sched_getaffinity(pid, setsize, mask) == 0) { break; + } CPU_FREE(mask); - if (errno != EINVAL) + if (errno != EINVAL) { return posix_error(); + } if (ncpus > INT_MAX / 2) { - PyErr_SetString(PyExc_OverflowError, "could not allocate " - "a large enough CPU set"); + PyErr_SetString(PyExc_OverflowError, + "could not allocate a large enough CPU set"); return NULL; } - ncpus = ncpus * 2; + ncpus *= 2; } - res = PySet_New(NULL); - if (res == NULL) + PyObject *res = PySet_New(NULL); + if (res == NULL) { goto error; - for (cpu = 0, count = CPU_COUNT_S(setsize, mask); count; cpu++) { + } + + int cpu = 0; + int count = CPU_COUNT_S(setsize, mask); + for (; count; cpu++) { if (CPU_ISSET_S(cpu, setsize, mask)) { PyObject *cpu_num = PyLong_FromLong(cpu); --count; - if (cpu_num == NULL) + if (cpu_num == NULL) { goto error; + } if (PySet_Add(res, cpu_num)) { Py_DECREF(cpu_num); goto error; @@ -8177,12 +8183,12 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid) return res; error: - if (mask) + if (mask) { CPU_FREE(mask); + } Py_XDECREF(res); return NULL; } - #endif /* HAVE_SCHED_SETAFFINITY */ #endif /* HAVE_SCHED_H */ @@ -14333,44 +14339,49 @@ os_get_terminal_size_impl(PyObject *module, int fd) /*[clinic input] os.cpu_count -Return the number of CPUs in the system; return None if indeterminable. +Return the number of logical CPUs in the system. -This number is not equivalent to the number of CPUs the current process can -use. The number of usable CPUs can be obtained with -``len(os.sched_getaffinity(0))`` +Return None if indeterminable. [clinic start generated code]*/ static PyObject * os_cpu_count_impl(PyObject *module) -/*[clinic end generated code: output=5fc29463c3936a9c input=e7c8f4ba6dbbadd3]*/ +/*[clinic end generated code: output=5fc29463c3936a9c input=ba2f6f8980a0e2eb]*/ { - int ncpu = 0; + int ncpu; #ifdef MS_WINDOWS -#ifdef MS_WINDOWS_DESKTOP +# ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); -#endif +# else + ncpu = 0; +# endif + #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); + #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) ncpu = sysconf(_SC_NPROCESSORS_ONLN); + #elif defined(__VXWORKS__) ncpu = _Py_popcount32(vxCpuEnabledGet()); + #elif defined(__DragonFly__) || \ defined(__OpenBSD__) || \ defined(__FreeBSD__) || \ defined(__NetBSD__) || \ defined(__APPLE__) - int mib[2]; + ncpu = 0; size_t len = sizeof(ncpu); - mib[0] = CTL_HW; - mib[1] = HW_NCPU; - if (sysctl(mib, 2, &ncpu, &len, NULL, 0) != 0) + int mib[2] = {CTL_HW, HW_NCPU}; + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) != 0) { ncpu = 0; + } #endif - if (ncpu >= 1) - return PyLong_FromLong(ncpu); - else + + if (ncpu < 1) { Py_RETURN_NONE; + } + return PyLong_FromLong(ncpu); } From d3728ddc572fff7ffcc95301bf5265717dbaf476 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 1 Oct 2023 00:21:20 +0200 Subject: [PATCH 098/124] gh-110014: Fix bootstrap_hash.c: remove debug code (#110161) Oops, I commited debug code by mistake, sorry about that. --- Python/bootstrap_hash.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 86a16916304cab..92f2301a012c0a 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -4,9 +4,6 @@ #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _PyRuntime -#undef HAVE_GETRANDOM -#undef HAVE_GETENTROPY - #ifdef HAVE_UNISTD_H # include // close() #endif From 53eb9a676f8c59b206dfc536b7590f6563ad65e0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 1 Oct 2023 00:37:23 +0200 Subject: [PATCH 099/124] gh-110152: regrtest handles cross compilation and HOSTRUNNER (#110156) * _add_python_opts() now handles cross compilation and HOSTRUNNER. * display_header() now tells if Python is cross-compiled, display HOSTRUNNER, and get the host platform. * Remove Tools/scripts/run_tests.py script. * Remove "make hostrunnertest": use "make buildbottest" or "make test" instead. --- Lib/test/libregrtest/main.py | 98 ++++++++++++++++--- Lib/test/libregrtest/utils.py | 39 +++++++- Lib/test/test_regrtest.py | 8 -- Makefile.pre.in | 7 +- ...-09-30-20-18-38.gh-issue-110152.4Kxve1.rst | 5 + Tools/scripts/run_tests.py | 78 --------------- 6 files changed, 125 insertions(+), 110 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst delete mode 100644 Tools/scripts/run_tests.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dcb2c5870de176..19bf2358456036 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -3,6 +3,7 @@ import re import shlex import sys +import sysconfig import time from test import support @@ -22,6 +23,7 @@ strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, + is_cross_compiled, get_host_runner, MS_WINDOWS, EXIT_TIMEOUT) @@ -71,10 +73,9 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks - ci_mode = (ns.fast_ci or ns.slow_ci) + self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.want_add_python_opts: bool = (_add_python_opts - and ns._add_python_opts - and ci_mode) + and ns._add_python_opts) # Select tests if ns.match_tests: @@ -431,7 +432,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if (self.want_header or not(self.pgo or self.quiet or self.single_test_run or tests or self.cmdline_args)): - display_header(self.use_resources) + display_header(self.use_resources, self.python_cmd) if self.randomize: print("Using random seed", self.random_seed) @@ -489,8 +490,56 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: # processes. return self._run_tests(selected, tests) - def _add_python_opts(self): - python_opts = [] + def _add_cross_compile_opts(self, regrtest_opts): + # WASM/WASI buildbot builders pass multiple PYTHON environment + # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. + keep_environ = bool(self.python_cmd) + environ = None + + # Are we using cross-compilation? + cross_compile = is_cross_compiled() + + # Get HOSTRUNNER + hostrunner = get_host_runner() + + if cross_compile: + # emulate -E, but keep PYTHONPATH + cross compile env vars, + # so test executable can load correct sysconfigdata file. + keep = { + '_PYTHON_PROJECT_BASE', + '_PYTHON_HOST_PLATFORM', + '_PYTHON_SYSCONFIGDATA_NAME', + 'PYTHONPATH' + } + old_environ = os.environ + new_environ = { + name: value for name, value in os.environ.items() + if not name.startswith(('PYTHON', '_PYTHON')) or name in keep + } + # Only set environ if at least one variable was removed + if new_environ != old_environ: + environ = new_environ + keep_environ = True + + if cross_compile and hostrunner: + if self.num_workers == 0: + # For now use only two cores for cross-compiled builds; + # hostrunner can be expensive. + regrtest_opts.extend(['-j', '2']) + + # If HOSTRUNNER is set and -p/--python option is not given, then + # use hostrunner to execute python binary for tests. + if not self.python_cmd: + buildpython = sysconfig.get_config_var("BUILDPYTHON") + python_cmd = f"{hostrunner} {buildpython}" + regrtest_opts.extend(["--python", python_cmd]) + keep_environ = True + + return (environ, keep_environ) + + def _add_ci_python_opts(self, python_opts, keep_environ): + # --fast-ci and --slow-ci add options to Python: + # "-u -W default -bb -E" # Unbuffered stdout and stderr if not sys.stdout.write_through: @@ -504,32 +553,27 @@ def _add_python_opts(self): if sys.flags.bytes_warning < 2: python_opts.append('-bb') - # WASM/WASI buildbot builders pass multiple PYTHON environment - # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. - if not self.python_cmd: + if not keep_environ: # Ignore PYTHON* environment variables if not sys.flags.ignore_environment: python_opts.append('-E') - if not python_opts: - return - - cmd = [*sys.orig_argv, "--dont-add-python-opts"] - cmd[1:1] = python_opts - + def _execute_python(self, cmd, environ): # Make sure that messages before execv() are logged sys.stdout.flush() sys.stderr.flush() cmd_text = shlex.join(cmd) try: + print(f"+ {cmd_text}", flush=True) + if hasattr(os, 'execv') and not MS_WINDOWS: os.execv(cmd[0], cmd) # On success, execv() do no return. # On error, it raises an OSError. else: import subprocess - with subprocess.Popen(cmd) as proc: + with subprocess.Popen(cmd, env=environ) as proc: try: proc.wait() except KeyboardInterrupt: @@ -548,6 +592,28 @@ def _add_python_opts(self): f"Command: {cmd_text}") # continue executing main() + def _add_python_opts(self): + python_opts = [] + regrtest_opts = [] + + environ, keep_environ = self._add_cross_compile_opts(regrtest_opts) + if self.ci_mode: + self._add_ci_python_opts(python_opts, keep_environ) + + if (not python_opts) and (not regrtest_opts) and (environ is None): + # Nothing changed: nothing to do + return + + # Create new command line + cmd = list(sys.orig_argv) + if python_opts: + cmd[1:1] = python_opts + if regrtest_opts: + cmd.extend(regrtest_opts) + cmd.append("--dont-add-python-opts") + + self._execute_python(cmd, environ) + def _init(self): # Set sys.stdout encoder error handler to backslashreplace, # similar to sys.stderr error handler, to avoid UnicodeEncodeError diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index dc1fa51b80dea1..d2c274d9970738 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,7 +5,9 @@ import os.path import platform import random +import shlex import signal +import subprocess import sys import sysconfig import tempfile @@ -523,7 +525,18 @@ def adjust_rlimit_nofile(): f"{new_fd_limit}: {err}.") -def display_header(use_resources: tuple[str, ...]): +def get_host_runner(): + if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: + hostrunner = sysconfig.get_config_var("HOSTRUNNER") + return hostrunner + + +def is_cross_compiled(): + return ('_PYTHON_HOST_PLATFORM' in os.environ) + + +def display_header(use_resources: tuple[str, ...], + python_cmd: tuple[str, ...] | None): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), @@ -537,13 +550,35 @@ def display_header(use_resources: tuple[str, ...]): print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) - if use_resources: print(f"== resources ({len(use_resources)}): " f"{', '.join(sorted(use_resources))}") else: print("== resources: (all disabled, use -u option)") + cross_compile = is_cross_compiled() + if cross_compile: + print("== cross compiled: Yes") + if python_cmd: + cmd = shlex.join(python_cmd) + print(f"== host python: {cmd}") + + get_cmd = [*python_cmd, '-m', 'platform'] + proc = subprocess.run( + get_cmd, + stdout=subprocess.PIPE, + text=True, + cwd=os_helper.SAVEDCWD) + stdout = proc.stdout.replace('\n', ' ').strip() + if stdout: + print(f"== host platform: {stdout}") + elif proc.returncode: + print(f"== host platform: ") + else: + hostrunner = get_host_runner() + if hostrunner: + print(f"== host runner: {hostrunner}") + # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e940cf04321d04..0e052e28ec2609 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -788,14 +788,6 @@ def test_script_autotest(self): args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) - @unittest.skipUnless(sysconfig.is_python_build(), - 'run_tests.py script is not installed') - def test_tools_script_run_tests(self): - # Tools/scripts/run_tests.py - script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - args = [script, *self.regrtest_args, *self.tests] - self.run_tests(args) - def run_batch(self, *args): proc = self.run_command(args) self.check_output(proc.stdout) diff --git a/Makefile.pre.in b/Makefile.pre.in index fa5b9e6654c26c..cf03c86f18b3c3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1837,7 +1837,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) TESTOPTS= $(EXTRATESTOPTS) TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS) -TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py +TESTRUNNER= $(TESTPYTHON) -m test TESTTIMEOUT= # Remove "test_python_*" directories of previous failed test jobs. @@ -1875,11 +1875,6 @@ buildbottest: all fi $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) -# Like buildbottest, but run Python tests with HOSTRUNNER directly. -.PHONY: hostrunnertest -hostrunnertest: all - $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) - .PHONY: pythoninfo pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo diff --git a/Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst b/Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst new file mode 100644 index 00000000000000..2fb6cbbad0c449 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst @@ -0,0 +1,5 @@ +Remove ``Tools/scripts/run_tests.py`` and ``make hostrunnertest``. Just run +``./python -m test --slow-ci``, ``make buildbottest`` or ``make test`` instead. +Python test runner (regrtest) now handles cross-compilation and HOSTRUNNER. It +also adds options to Python such fast ``-u -E -W default -bb`` when +``--fast-ci`` or ``--slow-ci`` option is used. Patch by Victor Stinner. diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py deleted file mode 100644 index 3e3d15d3b0da5c..00000000000000 --- a/Tools/scripts/run_tests.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Run Python's test suite in a fast, rigorous way. - -The defaults are meant to be reasonably thorough, while skipping certain -tests that can be time-consuming or resource-intensive (e.g. largefile), -or distracting (e.g. audio and gui). These defaults can be overridden by -simply passing a -u option to this script. - -""" - -import os -import shlex -import sys -import sysconfig -import test.support - - -def is_multiprocess_flag(arg): - return arg.startswith('-j') or arg.startswith('--multiprocess') - - -def is_python_flag(arg): - return arg.startswith('-p') or arg.startswith('--python') - - -def main(regrtest_args): - args = [sys.executable] - - cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ - if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: - hostrunner = sysconfig.get_config_var("HOSTRUNNER") - if cross_compile: - # emulate -E, but keep PYTHONPATH + cross compile env vars, so - # test executable can load correct sysconfigdata file. - keep = { - '_PYTHON_PROJECT_BASE', - '_PYTHON_HOST_PLATFORM', - '_PYTHON_SYSCONFIGDATA_NAME', - 'PYTHONPATH' - } - environ = { - name: value for name, value in os.environ.items() - if not name.startswith(('PYTHON', '_PYTHON')) or name in keep - } - else: - environ = os.environ.copy() - - # Allow user-specified interpreter options to override our defaults. - args.extend(test.support.args_from_interpreter_flags()) - - args.extend(['-m', 'test', # Run the test suite - '--fast-ci', # Fast Continuous Integration mode - ]) - if not any(is_multiprocess_flag(arg) for arg in regrtest_args): - if cross_compile and hostrunner: - # For now use only two cores for cross-compiled builds; - # hostrunner can be expensive. - args.extend(['-j', '2']) - - if cross_compile and hostrunner: - # If HOSTRUNNER is set and -p/--python option is not given, then - # use hostrunner to execute python binary for tests. - if not any(is_python_flag(arg) for arg in regrtest_args): - buildpython = sysconfig.get_config_var("BUILDPYTHON") - args.extend(["--python", f"{hostrunner} {buildpython}"]) - - args.extend(regrtest_args) - - print(shlex.join(args), flush=True) - - if sys.platform == 'win32': - from subprocess import call - sys.exit(call(args)) - else: - os.execve(sys.executable, args, environ) - - -if __name__ == '__main__': - main(sys.argv[1:]) From a46e96076898d126c9f276aef1934195aac34b4e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 1 Oct 2023 03:14:57 +0200 Subject: [PATCH 100/124] gh-109649: Use os.process_cpu_count() (#110165) Replace os.cpu_count() with os.process_cpu_count() in modules: * compileall * concurrent.futures * multiprocessing Replace os.cpu_count() with os.process_cpu_count() in programs: * _decimal deccheck.py test * freeze.py * multissltests.py * python -m test (regrtest) * wasm_build.py Other changes: * test.pythoninfo logs os.process_cpu_count(). * regrtest gets os.process_cpu_count() / os.cpu_count() in headers. --- Doc/library/compileall.rst | 2 +- Doc/library/concurrent.futures.rst | 10 +++++++++- Doc/library/multiprocessing.rst | 12 ++++++++---- Doc/whatsnew/3.13.rst | 7 +++++++ Lib/concurrent/futures/process.py | 2 +- Lib/concurrent/futures/thread.py | 4 ++-- Lib/multiprocessing/pool.py | 2 +- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/utils.py | 3 +++ Lib/test/pythoninfo.py | 1 + Lib/test/test_concurrent_futures/test_thread_pool.py | 2 +- .../2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst | 4 ++++ Modules/_decimal/tests/deccheck.py | 2 +- Tools/freeze/test/freeze.py | 2 +- Tools/ssl/multissltests.py | 5 ++++- Tools/wasm/wasm_build.py | 6 +++++- 16 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index a7455aeb0ec1cd..b4723b98f67bb2 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -90,7 +90,7 @@ compile Python sources. .. cmdoption:: -j N Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.cpu_count()` + If ``0`` is used, then the result of :func:`os.process_cpu_count()` will be used. .. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 6503d1fcf70a32..dca51459a2df98 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -188,6 +188,10 @@ And:: ThreadPoolExecutor now reuses idle worker threads before starting *max_workers* worker threads too. + .. versionchanged:: 3.13 + Default value of *max_workers* is changed to + ``min(32, (os.process_cpu_count() or 1) + 4)``. + .. _threadpoolexecutor-example: @@ -243,7 +247,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not - given, it will default to the number of processors on the machine. + given, it will default to :func:`os.process_cpu_count`. If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` will be raised. On Windows, *max_workers* must be less than or equal to ``61``. If it is not @@ -301,6 +305,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. different start method. See the :func:`os.fork` documentation for further explanation. + .. versionchanged:: 3.13 + *max_workers* uses :func:`os.process_cpu_count` by default, instead of + :func:`os.cpu_count`. + .. _processpoolexecutor-example: ProcessPoolExecutor Example diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2f0f1f800fdc94..d19f911dd7016c 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -996,13 +996,13 @@ Miscellaneous This number is not equivalent to the number of CPUs the current process can use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` + :func:`os.process_cpu_count`. When the number of CPUs cannot be determined a :exc:`NotImplementedError` is raised. .. seealso:: - :func:`os.cpu_count` + :func:`os.cpu_count` and :func:`os.process_cpu_count` .. function:: current_process() @@ -2214,7 +2214,7 @@ with the :class:`Pool` class. callbacks and has a parallel map implementation. *processes* is the number of worker processes to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. + ``None`` then the number returned by :func:`os.process_cpu_count` is used. If *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. @@ -2249,6 +2249,10 @@ with the :class:`Pool` class. .. versionadded:: 3.4 *context* + .. versionchanged:: 3.13 + *processes* uses :func:`os.process_cpu_count` by default, instead of + :func:`os.cpu_count`. + .. note:: Worker processes within a :class:`Pool` typically live for the complete @@ -2775,7 +2779,7 @@ worker threads rather than worker processes. :meth:`~multiprocessing.pool.Pool.terminate` manually. *processes* is the number of worker threads to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. + ``None`` then the number returned by :func:`os.process_cpu_count` is used. If *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 484443a086fdd6..a789084a79c397 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -91,6 +91,13 @@ Other Language Changes of the ``optimize`` argument. (Contributed by Irit Katriel in :gh:`108113`). +* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: + Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the + default number of worker threads and processes. Get the CPU affinity + if supported. + (Contributed by Victor Stinner in :gh:`109649`.) + + New Modules =========== diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 3990e6b1833d78..ffaffdb8b3d0aa 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -666,7 +666,7 @@ def __init__(self, max_workers=None, mp_context=None, _check_system_limits() if max_workers is None: - self._max_workers = os.cpu_count() or 1 + self._max_workers = os.process_cpu_count() or 1 if sys.platform == 'win32': self._max_workers = min(_MAX_WINDOWS_WORKERS, self._max_workers) diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 3b3a36a5093336..a024033f35fb54 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -139,10 +139,10 @@ def __init__(self, max_workers=None, thread_name_prefix='', # * CPU bound task which releases GIL # * I/O bound task (which releases GIL, of course) # - # We use cpu_count + 4 for both types of tasks. + # We use process_cpu_count + 4 for both types of tasks. # But we limit it to 32 to avoid consuming surprisingly large resource # on many core machine. - max_workers = min(32, (os.cpu_count() or 1) + 4) + max_workers = min(32, (os.process_cpu_count() or 1) + 4) if max_workers <= 0: raise ValueError("max_workers must be greater than 0") diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 4f5d88cb975cb7..f979890170b1a1 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -200,7 +200,7 @@ def __init__(self, processes=None, initializer=None, initargs=(), self._initargs = initargs if processes is None: - processes = os.cpu_count() or 1 + processes = os.process_cpu_count() or 1 if processes < 1: raise ValueError("Number of processes must be at least 1") if maxtasksperchild is not None: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 19bf2358456036..5f2baac9cba9e0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -426,7 +426,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.num_workers < 0: # Use all CPUs + 2 extra worker processes for tests # that like to sleep - self.num_workers = (os.cpu_count() or 1) + 2 + self.num_workers = (os.process_cpu_count() or 1) + 2 # For a partial run, we do not need to clutter the output. if (self.want_header diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d2c274d9970738..86fb820a23f535 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -546,6 +546,9 @@ def display_header(use_resources: tuple[str, ...], cpu_count = os.cpu_count() if cpu_count: + process_cpu_count = os.process_cpu_count() + if process_cpu_count and process_cpu_count != cpu_count: + cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 0e7528ef97c5f6..58d906ffc62a53 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -239,6 +239,7 @@ def format_attr(attr, value): 'getresgid', 'getresuid', 'getuid', + 'process_cpu_count', 'uname', ): call_func(info_add, 'os.%s' % func, os, func) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 812f989d8f3ad2..5926a632aa4bec 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -25,7 +25,7 @@ def record_finished(n): def test_default_workers(self): executor = self.executor_type() - expected = min(32, (os.cpu_count() or 1) + 4) + expected = min(32, (os.process_cpu_count() or 1) + 4) self.assertEqual(executor._max_workers, expected) def test_saturation(self): diff --git a/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst new file mode 100644 index 00000000000000..888fd79962b412 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-01-01-47-21.gh-issue-109649.BizOaD.rst @@ -0,0 +1,4 @@ +:mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: +Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the +default number of worker threads and processes. Get the CPU affinity if +supported. Patch by Victor Stinner. diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py index edf753f3704a18..bf277dd6879ffe 100644 --- a/Modules/_decimal/tests/deccheck.py +++ b/Modules/_decimal/tests/deccheck.py @@ -1301,7 +1301,7 @@ def tfunc(): out, _ = p.communicate() write_output(out, p.returncode) - N = os.cpu_count() + N = os.process_cpu_count() t = N * [None] for i in range(N): diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index cdf77c57bbb6ae..9030ad4d4e5f93 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -130,7 +130,7 @@ def prepare(script=None, outdir=None): if not MAKE: raise UnsupportedError('make') - cores = os.cpu_count() + cores = os.process_cpu_count() if cores and cores >= 3: # this test is most often run as part of the whole suite with a lot # of other tests running in parallel, from 1-2 vCPU systems up to diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index f066fb52cfd496..120e3883adc795 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -151,7 +151,10 @@ class AbstractBuilder(object): build_template = None depend_target = None install_target = 'install' - jobs = os.cpu_count() + if hasattr(os, 'process_cpu_count'): + jobs = os.process_cpu_count() + else: + jobs = os.cpu_count() module_files = ( os.path.join(PYTHONROOT, "Modules/_ssl.c"), diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 3558ecd869dfc5..c0b9999a5dad03 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -516,7 +516,11 @@ def make_cmd(self) -> List[str]: def getenv(self) -> Dict[str, Any]: """Generate environ dict for platform""" env = os.environ.copy() - env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}") + if hasattr(os, 'process_cpu_count'): + cpu_count = os.process_cpu_count() + else: + cpu_count = os.cpu_count() + env.setdefault("MAKEFLAGS", f"-j{cpu_count}") platenv = self.host.platform.getenv(self) for key, value in platenv.items(): if value is None: From 62405c7867b03730f0d278ea845855692d262d44 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 30 Sep 2023 23:35:54 -0500 Subject: [PATCH 101/124] gh-110150: Fix base case handling in quantiles() (gh-110151) --- Doc/library/statistics.rst | 7 ++++++- Lib/statistics.py | 4 +++- Lib/test/test_statistics.py | 7 ++++++- .../Library/2023-09-30-12-50-47.gh-issue-110150.9j0Ij5.rst | 2 ++ 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-30-12-50-47.gh-issue-110150.9j0Ij5.rst diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index f3c1bf20ae3ac8..5c8ad3a7dd7380 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -585,7 +585,7 @@ However, for reading convenience, most of the examples show sorted sequences. The *data* can be any iterable containing sample data. For meaningful results, the number of data points in *data* should be larger than *n*. - Raises :exc:`StatisticsError` if there are not at least two data points. + Raises :exc:`StatisticsError` if there is not at least one data point. The cut points are linearly interpolated from the two nearest data points. For example, if a cut point falls one-third @@ -625,6 +625,11 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + No longer raises an exception for an input with only a single data point. + This allows quantile estimates to be built up one sample point + at a time becoming gradually more refined with each new data point. + .. function:: covariance(x, y, /) Return the sample covariance of two inputs *x* and *y*. Covariance diff --git a/Lib/statistics.py b/Lib/statistics.py index 96c803483057e7..4da06889c6db46 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -844,7 +844,9 @@ def quantiles(data, *, n=4, method='exclusive'): data = sorted(data) ld = len(data) if ld < 2: - raise StatisticsError('must have at least two data points') + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') if method == 'inclusive': m = ld - 1 result = [] diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index f9b0ac2ad7b116..b24fc3c3d077fe 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2454,6 +2454,11 @@ def f(x): data = random.choices(range(100), k=k) q1, q2, q3 = quantiles(data, method='inclusive') self.assertEqual(q2, statistics.median(data)) + # Base case with a single data point: When estimating quantiles from + # a sample, we want to be able to add one sample point at a time, + # getting increasingly better estimates. + self.assertEqual(quantiles([10], n=4), [10.0, 10.0, 10.0]) + self.assertEqual(quantiles([10], n=4, method='exclusive'), [10.0, 10.0, 10.0]) def test_equal_inputs(self): quantiles = statistics.quantiles @@ -2504,7 +2509,7 @@ def test_error_cases(self): with self.assertRaises(ValueError): quantiles([10, 20, 30], method='X') # method is unknown with self.assertRaises(StatisticsError): - quantiles([10], n=4) # not enough data points + quantiles([], n=4) # not enough data points with self.assertRaises(TypeError): quantiles([10, None, 30], n=4) # data is non-numeric diff --git a/Misc/NEWS.d/next/Library/2023-09-30-12-50-47.gh-issue-110150.9j0Ij5.rst b/Misc/NEWS.d/next/Library/2023-09-30-12-50-47.gh-issue-110150.9j0Ij5.rst new file mode 100644 index 00000000000000..3c4dde59f71a93 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-30-12-50-47.gh-issue-110150.9j0Ij5.rst @@ -0,0 +1,2 @@ +Fix base case handling in statistics.quantiles. Now allows a single data +point. From 038c3564fb4ac6439ff0484e6746b0790794f41b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 12:38:46 +0100 Subject: [PATCH 102/124] PEG generator: bump types-setuptools from 68.1.0.1 to 68.2.0.0 (#110175) build(deps-dev): bump types-setuptools in /Tools Bumps [types-setuptools](https://github.com/python/typeshed) from 68.1.0.1 to 68.2.0.0. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-setuptools dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 35bceb205e8a9b..add28b1bb38183 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.5.1 # needed for peg_generator: types-psutil==5.9.5.16 -types-setuptools==68.1.0.1 +types-setuptools==68.2.0.0 From 06faa9a39bd93c5e7999d52b52043ecdd0774dac Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 1 Oct 2023 17:20:01 +0300 Subject: [PATCH 103/124] gh-110160: Fix flaky `test_find_periodic_pattern` in `string_tests` (#110170) --- Lib/test/string_tests.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 8d210b198d248d..cecf309dca9194 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -327,11 +327,12 @@ def reference_find(p, s): for i in range(len(s)): if s.startswith(p, i): return i + if p == '' and s == '': + return 0 return -1 - rr = random.randrange - choices = random.choices - for _ in range(1000): + def check_pattern(rr): + choices = random.choices p0 = ''.join(choices('abcde', k=rr(10))) * rr(10, 20) p = p0[:len(p0) - rr(10)] # pop off some characters left = ''.join(choices('abcdef', k=rr(2000))) @@ -341,6 +342,13 @@ def reference_find(p, s): self.checkequal(reference_find(p, text), text, 'find', p) + rr = random.randrange + for _ in range(1000): + check_pattern(rr) + + # Test that empty string always work: + check_pattern(lambda *args: 0) + def test_find_many_lengths(self): haystack_repeats = [a * 10**e for e in range(6) for a in (1,2,5)] haystacks = [(n, self.fixtype("abcab"*n + "da")) for n in haystack_repeats] From 15de493395c3251b8b82063bbe22a379792b9404 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 1 Oct 2023 16:14:02 +0100 Subject: [PATCH 104/124] GH-107465: Add `pathlib.Path.from_uri()` classmethod. (#107640) This method supports file URIs (including variants) as described in RFC 8089, such as URIs generated by `pathlib.Path.as_uri()` and `urllib.request.pathname2url()`. The method is added to `Path` rather than `PurePath` because it uses `os.fsdecode()`, and so its results vary from system to system. I intend to deprecate `PurePath.as_uri()` and move it to `Path` for the same reason. Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/pathlib.rst | 36 +++++++++++++++ Doc/whatsnew/3.13.rst | 4 ++ Lib/pathlib.py | 40 ++++++++++++++--- Lib/test/test_pathlib.py | 44 +++++++++++++++++++ ...-08-04-19-00-53.gh-issue-107465.Vc1Il3.rst | 1 + 5 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-04-19-00-53.gh-issue-107465.Vc1Il3.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 48d6176d26bb8f..8ee89a003a339a 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -850,6 +850,42 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 +.. classmethod:: Path.from_uri(uri) + + Return a new path object from parsing a 'file' URI conforming to + :rfc:`8089`. For example:: + + >>> p = Path.from_uri('file:///etc/hosts') + PosixPath('/etc/hosts') + + On Windows, DOS device and UNC paths may be parsed from URIs:: + + >>> p = Path.from_uri('file:///c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file://server/share') + WindowsPath('//server/share') + + Several variant forms are supported:: + + >>> p = Path.from_uri('file:////server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file://///server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file:c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file:/c|/windows') + WindowsPath('c:/windows') + + :exc:`ValueError` is raised if the URI does not start with ``file:``, or + the parsed path isn't absolute. + + :func:`os.fsdecode` is used to decode percent-escaped byte sequences, and + so file URIs are not portable across machines with different + :ref:`filesystem encodings `. + + .. versionadded:: 3.13 + + .. method:: Path.stat(*, follow_symlinks=True) Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a789084a79c397..1de5479a924375 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -184,6 +184,10 @@ pathlib :exc:`NotImplementedError` when a path operation isn't supported. (Contributed by Barney Gale in :gh:`89812`.) +* Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` + object from a 'file' URI (``file:/``). + (Contributed by Barney Gale in :gh:`107465`.) + * Add support for recursive wildcards in :meth:`pathlib.PurePath.match`. (Contributed by Barney Gale in :gh:`73435`.) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index e6be9061013a8a..9e6d0754eccf3e 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -18,7 +18,6 @@ from _collections_abc import Sequence from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from urllib.parse import quote_from_bytes as urlquote_from_bytes try: import pwd @@ -452,7 +451,8 @@ def as_uri(self): # It's a posix path => 'file:///etc/hosts' prefix = 'file://' path = str(self) - return prefix + urlquote_from_bytes(os.fsencode(path)) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) @property def _str_normcase(self): @@ -814,9 +814,10 @@ class _PathBase(PurePath): __bytes__ = None __fspath__ = None # virtual paths have no local file system representation - def _unsupported(self, method_name): - msg = f"{type(self).__name__}.{method_name}() is unsupported" - if isinstance(self, Path): + @classmethod + def _unsupported(cls, method_name): + msg = f"{cls.__name__}.{method_name}() is unsupported" + if issubclass(cls, Path): msg += " on this system" raise UnsupportedOperation(msg) @@ -1418,6 +1419,11 @@ def group(self): """ self._unsupported("group") + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + cls._unsupported("from_uri") + def as_uri(self): """Return the path as a URI.""" self._unsupported("as_uri") @@ -1661,6 +1667,30 @@ def expanduser(self): return self + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + class PosixPath(Path, PurePosixPath): """Path subclass for non-Windows systems. diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 319148e9065a65..76918addf8b613 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,6 +11,7 @@ import tempfile import unittest from unittest import mock +from urllib.request import pathname2url from test.support import import_helper from test.support import set_recursion_limit @@ -3602,6 +3603,24 @@ def test_handling_bad_descriptor(self): self.fail("Bad file descriptor not handled.") raise + def test_from_uri(self): + P = self.cls + self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, '/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + def test_from_uri_pathname2url(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) + self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) + @only_nt class WindowsPathTest(PathTest): @@ -3721,6 +3740,31 @@ def check(): env['HOME'] = 'C:\\Users\\eve' check() + def test_from_uri(self): + P = self.cls + # DOS drive paths + self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) + # UNC paths + self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) + # Localhost paths + self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) + # Invalid paths + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + def test_from_uri_pathname2url(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) class PathSubclassTest(PathTest): diff --git a/Misc/NEWS.d/next/Library/2023-08-04-19-00-53.gh-issue-107465.Vc1Il3.rst b/Misc/NEWS.d/next/Library/2023-08-04-19-00-53.gh-issue-107465.Vc1Il3.rst new file mode 100644 index 00000000000000..e98092f546e393 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-04-19-00-53.gh-issue-107465.Vc1Il3.rst @@ -0,0 +1 @@ +Add :meth:`pathlib.Path.from_uri` classmethod. From d642c5bbf58b42c90053dc553885445d53f247fe Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 1 Oct 2023 16:28:02 +0100 Subject: [PATCH 105/124] gh-110180: Remove unused `_PickleUsingNameMixin` class from `typing` (#110181) --- Lib/typing.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 639be75747dae0..d1f371377b88f8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -937,13 +937,6 @@ def _is_typevar_like(x: Any) -> bool: return isinstance(x, (TypeVar, ParamSpec)) or _is_unpacked_typevartuple(x) -class _PickleUsingNameMixin: - """Mixin enabling pickling based on self.__name__.""" - - def __reduce__(self): - return self.__name__ - - def _typevar_subst(self, arg): msg = "Parameters to generic types must be types." arg = _type_check(arg, msg, is_argument=True) From a431a0f988f26dae387f2a107c7e695f5dfc0096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 17:25:29 +0100 Subject: [PATCH 106/124] build(deps): bump hypothesis from 6.84.0 to 6.87.1 in /Tools (#110174) Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 6.84.0 to 6.87.1. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-6.84.0...hypothesis-python-6.87.1) --- updated-dependencies: - dependency-name: hypothesis dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-hypothesis.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 9db2b74c87cfb0..b95300a07dd2b4 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.84.0 +hypothesis==6.87.1 From 31097df611bb5c8084190202e095ae47e8b81c0f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 1 Oct 2023 20:18:19 +0300 Subject: [PATCH 107/124] gh-101100: Fix sphinx warnings in `library/site.rst` (#110144) Co-authored-by: Alex Waygood --- Doc/library/exceptions.rst | 14 ++++++++++---- Doc/library/site.rst | 22 +++++++++++++++------- Doc/tools/.nitignore | 2 -- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index fae0cf621323c8..cd85df8723a76b 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -220,10 +220,16 @@ The following exceptions are the exceptions that are usually raised. load a module. Also raised when the "from list" in ``from ... import`` has a name that cannot be found. - The :attr:`name` and :attr:`path` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the module - that was attempted to be imported and the path to any file which triggered - the exception, respectively. + The optional *name* and *path* keyword-only arguments + set the corresponding attributes: + + .. attribute:: name + + The name of the module that was attempted to be imported. + + .. attribute:: path + + The path to any file which triggered the exception. .. versionchanged:: 3.3 Added the :attr:`name` and :attr:`path` attributes. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index ea3b2e996574ef..ebd78917a01373 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -19,7 +19,7 @@ Importing this module will append site-specific paths to the module search path and add a few builtins, unless :option:`-S` was used. In that case, this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific -additions, call the :func:`site.main` function. +additions, call the :func:`main` function. .. versionchanged:: 3.3 Importing the module used to trigger paths manipulation even when using @@ -109,32 +109,40 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -.. index:: pair: module; sitecustomize +:mod:`sitecustomize` +-------------------- + +.. module:: sitecustomize After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` or its subclass -exception, and the exception's :attr:`name` attribute equals to ``'sitecustomize'``, +exception, and the exception's :attr:`~ImportError.name` +attribute equals to ``'sitecustomize'``, it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), attempted output from :mod:`sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -.. index:: pair: module; usercustomize +:mod:`usercustomize` +-------------------- + +.. module:: usercustomize After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if -:data:`ENABLE_USER_SITE` is true. This file is intended to be created in the +:data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or -its subclass exception, and the exception's :attr:`name` attribute equals to -``'usercustomize'``, it is silently ignored. +its subclass exception, and the exception's :attr:`~ImportError.name` +attribute equals to ``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of :mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +.. currentmodule:: site .. _rlcompleter-config: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f217da9052ca78..fbc9fc33ecea1b 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -106,7 +106,6 @@ Doc/library/select.rst Doc/library/selectors.rst Doc/library/shelve.rst Doc/library/signal.rst -Doc/library/site.rst Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/socketserver.rst @@ -114,7 +113,6 @@ Doc/library/ssl.rst Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/sys_path_init.rst Doc/library/syslog.rst Doc/library/tarfile.rst Doc/library/tempfile.rst From adf0f15a06c6e8ddd1a6d59b28efcbb26289f080 Mon Sep 17 00:00:00 2001 From: Quentin Agren Date: Sun, 1 Oct 2023 14:32:43 -0400 Subject: [PATCH 108/124] gh-110138: Improve grammar in idiomatic usage of ``__main__.py`` (#110142) --- Doc/library/__main__.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index fd60d92d4eb0f9..d378e40b3906c6 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -238,9 +238,9 @@ package. For more details, see :ref:`intra-package-references` in the Idiomatic Usage ^^^^^^^^^^^^^^^ -The contents of ``__main__.py`` typically isn't fenced with -``if __name__ == '__main__'`` blocks. Instead, those files are kept short, -functions to execute from other modules. Those other modules can then be +The content of ``__main__.py`` typically isn't fenced with an +``if __name__ == '__main__'`` block. Instead, those files are kept +short and import functions to execute from other modules. Those other modules can then be easily unit-tested and are properly reusable. If used, an ``if __name__ == '__main__'`` block will still work as expected From 65c285062ce2769249610348636d3d73153e0144 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 1 Oct 2023 22:41:03 +0200 Subject: [PATCH 109/124] gh-110164: regrtest disables random if SOURCE_DATE_EPOCH (#110168) If the SOURCE_DATE_EPOCH environment variable is defined, regrtest now disables randomization of tests. --- Lib/test/libregrtest/main.py | 9 +++++++-- Lib/test/test_regrtest.py | 8 ++++++++ .../2023-10-01-02-58-00.gh-issue-110164.z7TMCq.rst | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-01-02-58-00.gh-issue-110164.z7TMCq.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5f2baac9cba9e0..af5fb0f464f5b5 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -106,8 +106,6 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.fail_env_changed: bool = ns.fail_env_changed self.fail_rerun: bool = ns.fail_rerun self.forever: bool = ns.forever - self.randomize: bool = ns.randomize - self.random_seed: int | None = ns.random_seed self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout if ns.huntrleaks: @@ -129,6 +127,13 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.coverage_dir: StrPath | None = ns.coverdir self.tmp_dir: StrPath | None = ns.tempdir + # Randomize + self.randomize: bool = ns.randomize + self.random_seed: int | None = ns.random_seed + if 'SOURCE_DATE_EPOCH' in os.environ: + self.randomize = False + self.random_seed = None + # tests self.first_runtests: RunTests | None = None diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0e052e28ec2609..38071341006092 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -148,6 +148,14 @@ def test_randomize(self): ns = self.parse_args([opt]) self.assertTrue(ns.randomize) + with os_helper.EnvironmentVarGuard() as env: + env['SOURCE_DATE_EPOCH'] = '1' + + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertFalse(regrtest.randomize) + self.assertIsNone(regrtest.random_seed) + def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-01-02-58-00.gh-issue-110164.z7TMCq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-01-02-58-00.gh-issue-110164.z7TMCq.rst new file mode 100644 index 00000000000000..086d70f30e204f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-01-02-58-00.gh-issue-110164.z7TMCq.rst @@ -0,0 +1,2 @@ +regrtest: If the ``SOURCE_DATE_EPOCH`` environment variable is defined, +regrtest now disables tests randomization. Patch by Victor Stinner. From 29c3a445d99f7e29086f6fdc4612e200cbbdc0ff Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 1 Oct 2023 23:34:09 -0400 Subject: [PATCH 110/124] fix misaligned versionchanged blocks in sqlite3 docs (GH-110191) --- Doc/library/sqlite3.rst | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 0abdab52340dfd..7b8c7810165164 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -992,9 +992,8 @@ Connection objects Added support for disabling the authorizer using ``None``. .. versionchanged:: 3.13 - - Passing *authorizer_callback* as a keyword argument to is deprecated. - The parameter will become positional-only in Python 3.15. + Passing *authorizer_callback* as a keyword argument is deprecated. + The parameter will become positional-only in Python 3.15. .. method:: set_progress_handler(progress_handler, n) @@ -1012,9 +1011,8 @@ Connection objects exception. .. versionchanged:: 3.13 - - Passing *progress_handler* as a keyword argument to is deprecated. - The parameter will become positional-only in Python 3.15. + Passing *progress_handler* as a keyword argument is deprecated. + The parameter will become positional-only in Python 3.15. .. method:: set_trace_callback(trace_callback) @@ -1041,9 +1039,8 @@ Connection objects .. versionadded:: 3.3 .. versionchanged:: 3.13 - - Passing *trace_callback* as a keyword argument to is deprecated. - The parameter will become positional-only in Python 3.15. + Passing *trace_callback* as a keyword argument is deprecated. + The parameter will become positional-only in Python 3.15. .. method:: enable_load_extension(enabled, /) From 29b875bb93099171aeb7a60cd18d4e1f4ea3c1db Mon Sep 17 00:00:00 2001 From: Charles Machalow Date: Mon, 2 Oct 2023 01:27:30 -0700 Subject: [PATCH 111/124] gh-109590: Update shutil.which on Windows to prefer a PATHEXT extension on executable files (GH-109995) The default arguments for shutil.which() request an executable file, but extensionless files are not executable on Windows and should be ignored. --- Doc/library/shutil.rst | 6 ++ Lib/shutil.py | 12 ++- Lib/test/test_shutil.py | 82 ++++++++++++++++--- ...-09-24-06-04-14.gh-issue-109590.9EMofC.rst | 3 + 4 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-24-06-04-14.gh-issue-109590.9EMofC.rst diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 4390a8e22306fa..d1949d698f5614 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -476,6 +476,12 @@ Directory and files operations or ends with an extension that is in ``PATHEXT``; and filenames that have no extension can now be found. + .. versionchanged:: 3.12.1 + On Windows, if *mode* includes ``os.X_OK``, executables with an + extension in ``PATHEXT`` will be preferred over executables without a + matching extension. + This brings behavior closer to that of Python 3.11. + .. exception:: Error This exception collects exceptions that are raised during a multi-file diff --git a/Lib/shutil.py b/Lib/shutil.py index b903f13d8b76a7..a278b74fab2ddb 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1554,8 +1554,16 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - # Always try checking the originally given cmd, if it doesn't match, try pathext - files = [cmd] + [cmd + ext for ext in pathext] + files = ([cmd] + [cmd + ext for ext in pathext]) + + # gh-109590. If we are looking for an executable, we need to look + # for a PATHEXT match. The first cmd is the direct match + # (e.g. python.exe instead of python) + # Check that direct match first if and only if the extension is in PATHEXT + # Otherwise check it last + suffix = os.path.splitext(files[0])[1].upper() + if mode & os.X_OK and not any(suffix == ext.upper() for ext in pathext): + files.append(files.pop(0)) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index a2ca4df135846f..d231e66b7b889f 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2067,6 +2067,14 @@ def setUp(self): self.curdir = os.curdir self.ext = ".EXE" + def to_text_type(self, s): + ''' + In this class we're testing with str, so convert s to a str + ''' + if isinstance(s, bytes): + return s.decode() + return s + def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) @@ -2254,9 +2262,9 @@ def test_empty_path_no_PATH(self): @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = ".xyz" + ext = self.to_text_type(".xyz") temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp2", suffix=ext) + prefix=self.to_text_type("Tmp2"), suffix=ext) os.chmod(temp_filexyz.name, stat.S_IXUSR) self.addCleanup(temp_filexyz.close) @@ -2265,16 +2273,16 @@ def test_pathext(self): program = os.path.splitext(program)[0] with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = ext + env['PATHEXT'] = ext if isinstance(ext, str) else ext.decode() rv = shutil.which(program, path=self.temp_dir) self.assertEqual(rv, temp_filexyz.name) # Issue 40592: See https://bugs.python.org/issue40592 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_with_empty_str(self): - ext = ".xyz" + ext = self.to_text_type(".xyz") temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp2", suffix=ext) + prefix=self.to_text_type("Tmp2"), suffix=ext) self.addCleanup(temp_filexyz.close) # strip path and extension @@ -2282,7 +2290,7 @@ def test_pathext_with_empty_str(self): program = os.path.splitext(program)[0] with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = f"{ext};" # note the ; + env['PATHEXT'] = f"{ext if isinstance(ext, str) else ext.decode()};" # note the ; rv = shutil.which(program, path=self.temp_dir) self.assertEqual(rv, temp_filexyz.name) @@ -2290,13 +2298,14 @@ def test_pathext_with_empty_str(self): @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_applied_on_files_in_path(self): with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir + env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() env["PATHEXT"] = ".test" - test_path = pathlib.Path(self.temp_dir) / "test_program.test" - test_path.touch(mode=0o755) + test_path = os.path.join(self.temp_dir, self.to_text_type("test_program.test")) + open(test_path, 'w').close() + os.chmod(test_path, 0o755) - self.assertEqual(shutil.which("test_program"), str(test_path)) + self.assertEqual(shutil.which(self.to_text_type("test_program")), test_path) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') @@ -2312,6 +2321,50 @@ def test_win_path_needs_curdir(self): self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) need_curdir_mock.assert_called_once_with('dontcare') + # See GH-109590 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_preferred_for_execute(self): + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() + env["PATHEXT"] = ".test" + + exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) + open(exe, 'w').close() + os.chmod(exe, 0o755) + + # default behavior allows a direct match if nothing in PATHEXT matches + self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) + + dot_test = os.path.join(self.temp_dir, self.to_text_type("test.exe.test")) + open(dot_test, 'w').close() + os.chmod(dot_test, 0o755) + + # now we have a PATHEXT match, so it take precedence + self.assertEqual(shutil.which(self.to_text_type("test.exe")), dot_test) + + # but if we don't use os.X_OK we don't change the order based off PATHEXT + # and therefore get the direct match. + self.assertEqual(shutil.which(self.to_text_type("test.exe"), mode=os.F_OK), exe) + + # See GH-109590 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_given_extension_preferred(self): + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() + env["PATHEXT"] = ".exe2;.exe" + + exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) + open(exe, 'w').close() + os.chmod(exe, 0o755) + + exe2 = os.path.join(self.temp_dir, self.to_text_type("test.exe2")) + open(exe2, 'w').close() + os.chmod(exe2, 0o755) + + # even though .exe2 is preferred in PATHEXT, we matched directly to test.exe + self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) + self.assertEqual(shutil.which(self.to_text_type("test")), exe2) + class TestWhichBytes(TestWhich): def setUp(self): @@ -2319,9 +2372,18 @@ def setUp(self): self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) self.temp_file.name = os.fsencode(self.temp_file.name) + self.temp_dir = os.fsencode(self.temp_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) + def to_text_type(self, s): + ''' + In this class we're testing with bytes, so convert s to a bytes + ''' + if isinstance(s, str): + return s.encode() + return s + class TestMove(BaseTest, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-09-24-06-04-14.gh-issue-109590.9EMofC.rst b/Misc/NEWS.d/next/Library/2023-09-24-06-04-14.gh-issue-109590.9EMofC.rst new file mode 100644 index 00000000000000..647e84e71b42d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-24-06-04-14.gh-issue-109590.9EMofC.rst @@ -0,0 +1,3 @@ +:func:`shutil.which` will prefer files with an extension in ``PATHEXT`` if the given mode includes ``os.X_OK`` on win32. +If no ``PATHEXT`` match is found, a file without an extension in ``PATHEXT`` can be returned. +This change will have :func:`shutil.which` act more similarly to previous behavior in Python 3.11. From 9cb8927bfc73b66e7c36ab02ca2a3077172ea0ac Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 2 Oct 2023 04:31:23 -0600 Subject: [PATCH 112/124] Docs: bump Pygments to fix contrast ratios to meet WCAG AA guidelines (#110208) --- Doc/constraints.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 54888eaab242ee..147de1271eb2b7 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -10,8 +10,7 @@ colorama<0.5 imagesize<1.5 Jinja2<3.2 packaging<24 -# Pygments==2.15.0 breaks CI -Pygments<2.16,!=2.15.0 +Pygments>=2.16.1,<3 requests<3 snowballstemmer<3 sphinxcontrib-applehelp<1.1 From f16e81f368d08891e28dc1f038c1826ea80d7801 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 2 Oct 2023 11:40:03 +0100 Subject: [PATCH 113/124] GH-92584: Move installation schemes overview to sysconfig docs (#108018) * Add new installation path functions subsection * Add content from install/index to sysconfig * Fix table * Update note about installers * Clean up the list of schemes, remove references to Distutils --- Doc/library/site.rst | 2 +- Doc/library/sysconfig.rst | 181 ++++++++++++++++++++++++++++++++++---- Doc/using/cmdline.rst | 2 +- 3 files changed, 167 insertions(+), 18 deletions(-) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index ebd78917a01373..02880c56416615 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -199,7 +199,7 @@ Module contents :file:`~/Library/Python/{X.Y}` for macOS framework builds, and :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, - etc. for the user installation scheme. + etc. for the :ref:`user installation scheme `. See also :envvar:`PYTHONUSERBASE`. diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index c625c1e1d72954..905abc3a7c9f9b 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -20,6 +20,7 @@ The :mod:`sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. + Configuration variables ----------------------- @@ -60,6 +61,7 @@ Example of usage:: >>> sysconfig.get_config_vars('AR', 'CXX') ['ar', 'g++'] + .. _installation_paths: Installation paths @@ -68,27 +70,24 @@ Installation paths Python uses an installation scheme that differs depending on the platform and on the installation options. These schemes are stored in :mod:`sysconfig` under unique identifiers based on the value returned by :const:`os.name`. - -Every new component that is installed using :mod:`!distutils` or a -Distutils-based system will follow the same scheme to copy its file in the right -places. +The schemes are used by package installers to determine where to copy files to. Python currently supports nine schemes: - *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is the default scheme used when Python or a component is installed. -- *posix_home*: scheme for POSIX platforms used when a *home* option is used - upon installation. This scheme is used when a component is installed through - Distutils with a specific home prefix. -- *posix_user*: scheme for POSIX platforms used when a component is installed - through Distutils and the *user* option is used. This scheme defines paths - located under the user home directory. +- *posix_home*: scheme for POSIX platforms, when the *home* option is used. + This scheme defines paths located under a specific home prefix. +- *posix_user*: scheme for POSIX platforms, when the *user* option is used. + This scheme defines paths located under the user's home directory + (:const:`site.USER_BASE`). - *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX platforms; by default it is the same as *posix_prefix*. -- *nt*: scheme for NT platforms like Windows. -- *nt_user*: scheme for NT platforms, when the *user* option is used. -- *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt*. +- *nt*: scheme for Windows. + This is the default scheme used when Python or a component is installed. +- *nt_user*: scheme for Windows, when the *user* option is used. +- *nt_venv*: scheme for :mod:`Python virtual environments ` on Windows; + by default it is the same as *nt*. - *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending on the platform Python runs on. - *osx_framework_user*: scheme for macOS, when the *user* option is used. @@ -101,7 +100,7 @@ identifier. Python currently uses eight paths: - *platstdlib*: directory containing the standard Python library files that are platform-specific. - *platlib*: directory for site-specific, platform-specific files. -- *purelib*: directory for site-specific, non-platform-specific files. +- *purelib*: directory for site-specific, non-platform-specific files ('pure' Python). - *include*: directory for non-platform-specific header files for the Python C-API. - *platinclude*: directory for platform-specific header files for @@ -109,7 +108,157 @@ identifier. Python currently uses eight paths: - *scripts*: directory for script files. - *data*: directory for data files. -:mod:`sysconfig` provides some functions to determine these paths. + +.. _sysconfig-user-scheme: + +User scheme +--------------- + +This scheme is designed to be the most convenient solution for users that don't +have write permission to the global site-packages directory or don't want to +install into it. + +Files will be installed into subdirectories of :const:`site.USER_BASE` (written +as :file:`{userbase}` hereafter). This scheme installs pure Python modules and +extension modules in the same location (also known as :const:`site.USER_SITE`). + +``posix_user`` +^^^^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{userbase}/lib/python{X.Y}` +*platstdlib* :file:`{userbase}/lib/python{X.Y}` +*platlib* :file:`{userbase}/lib/python{X.Y}/site-packages` +*purelib* :file:`{userbase}/lib/python{X.Y}/site-packages` +*include* :file:`{userbase}/include/python{X.Y}` +*scripts* :file:`{userbase}/bin` +*data* :file:`{userbase}` +============== =========================================================== + +``nt_user`` +^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{userbase}\\Python{XY}` +*platstdlib* :file:`{userbase}\\Python{XY}` +*platlib* :file:`{userbase}\\Python{XY}\\site-packages` +*purelib* :file:`{userbase}\\Python{XY}\\site-packages` +*include* :file:`{userbase}\\Python{XY}\\Include` +*scripts* :file:`{userbase}\\Python{XY}\\Scripts` +*data* :file:`{userbase}` +============== =========================================================== + +``osx_framework_user`` +^^^^^^^^^^^^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{userbase}/lib/python` +*platstdlib* :file:`{userbase}/lib/python` +*platlib* :file:`{userbase}/lib/python/site-packages` +*purelib* :file:`{userbase}/lib/python/site-packages` +*include* :file:`{userbase}/include/python{X.Y}` +*scripts* :file:`{userbase}/bin` +*data* :file:`{userbase}` +============== =========================================================== + + +.. _sysconfig-home-scheme: + +Home scheme +----------- + +The idea behind the "home scheme" is that you build and maintain a personal +stash of Python modules. This scheme's name is derived from the idea of a +"home" directory on Unix, since it's not unusual for a Unix user to make their +home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. +This scheme can be used by anyone, regardless of the operating system they +are installing for. + +``posix_home`` +^^^^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{home}/lib/python` +*platstdlib* :file:`{home}/lib/python` +*platlib* :file:`{home}/lib/python` +*purelib* :file:`{home}/lib/python` +*include* :file:`{home}/include/python` +*platinclude* :file:`{home}/include/python` +*scripts* :file:`{home}/bin` +*data* :file:`{home}` +============== =========================================================== + + +.. _sysconfig-prefix-scheme: + +Prefix scheme +------------- + +The "prefix scheme" is useful when you wish to use one Python installation to +perform the build/install (i.e., to run the setup script), but install modules +into the third-party module directory of a different Python installation (or +something that looks like a different Python installation). If this sounds a +trifle unusual, it is---that's why the user and home schemes come before. However, +there are at least two known cases where the prefix scheme will be useful. + +First, consider that many Linux distributions put Python in :file:`/usr`, rather +than the more traditional :file:`/usr/local`. This is entirely appropriate, +since in those cases Python is part of "the system" rather than a local add-on. +However, if you are installing Python modules from source, you probably want +them to go in :file:`/usr/local/lib/python2.{X}` rather than +:file:`/usr/lib/python2.{X}`. + +Another possibility is a network filesystem where the name used to write to a +remote directory is different from the name used to read it: for example, the +Python interpreter accessed as :file:`/usr/local/bin/python` might search for +modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to +be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. + +``posix_prefix`` +^^^^^^^^^^^^^^^^ + +============== ========================================================== +Path Installation directory +============== ========================================================== +*stdlib* :file:`{prefix}/lib/python{X.Y}` +*platstdlib* :file:`{prefix}/lib/python{X.Y}` +*platlib* :file:`{prefix}/lib/python{X.Y}/site-packages` +*purelib* :file:`{prefix}/lib/python{X.Y}/site-packages` +*include* :file:`{prefix}/include/python{X.Y}` +*platinclude* :file:`{prefix}/include/python{X.Y}` +*scripts* :file:`{prefix}/bin` +*data* :file:`{prefix}` +============== ========================================================== + +``nt`` +^^^^^^ + +============== ========================================================== +Path Installation directory +============== ========================================================== +*stdlib* :file:`{prefix}\\Lib` +*platstdlib* :file:`{prefix}\\Lib` +*platlib* :file:`{prefix}\\Lib\\site-packages` +*purelib* :file:`{prefix}\\Lib\\site-packages` +*include* :file:`{prefix}\\Include` +*platinclude* :file:`{prefix}\\Include` +*scripts* :file:`{prefix}\\Scripts` +*data* :file:`{prefix}` +============== ========================================================== + + +Installation path functions +--------------------------- + +:mod:`sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 921b6a6961c7b2..bade3ca6650e62 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,7 +811,7 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and installation paths for + and :ref:`installation paths ` for ``python -m pip install --user``. .. seealso:: From 44b1e4ea4842c6cdc1bedba7aaeb93f236b3ec08 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 2 Oct 2023 19:07:56 +0800 Subject: [PATCH 114/124] gh-108963: using random to generate unique string in sys.intern test (#109491) --- Lib/test/test_sys.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 16050171ad139d..ae241d7a502749 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -5,6 +5,7 @@ import locale import operator import os +import random import struct import subprocess import sys @@ -30,10 +31,6 @@ def requires_subinterpreters(meth): 'subinterpreters required')(meth) -# count the number of test runs, used to create unique -# strings to intern in test_intern() -INTERN_NUMRUNS = 0 - DICT_KEY_STRUCT_FORMAT = 'n2BI2n' class DisplayHookTest(unittest.TestCase): @@ -696,10 +693,8 @@ def test_43581(self): self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) def test_intern(self): - global INTERN_NUMRUNS - INTERN_NUMRUNS += 1 self.assertRaises(TypeError, sys.intern) - s = "never interned before" + str(INTERN_NUMRUNS) + s = "never interned before" + str(random.randrange(0, 10**9)) self.assertTrue(sys.intern(s) is s) s2 = s.swapcase().swapcase() self.assertTrue(sys.intern(s2) is s) @@ -717,9 +712,7 @@ def __hash__(self): @requires_subinterpreters def test_subinterp_intern_dynamically_allocated(self): - global INTERN_NUMRUNS - INTERN_NUMRUNS += 1 - s = "never interned before" + str(INTERN_NUMRUNS) + s = "never interned before" + str(random.randrange(0, 10**9)) t = sys.intern(s) self.assertIs(t, s) From 6139bf5e0c755ed22bdfb027a5299493f0c71be9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:15:58 +0100 Subject: [PATCH 115/124] GH-109190: Announce final release in What's New in Python 3.12 (#110117) Prepare What's New in Python 3.12 for final release --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ec39616d7c9d2b..3aa495d0537a69 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -46,7 +46,7 @@ researching a change. This article explains the new features in Python 3.12, compared to 3.11. -Python 3.12 will be released on October 2, 2023. +Python 3.12 was released on October 2, 2023. For full details, see the :ref:`changelog `. .. seealso:: From 1b3bc610fd40e7c26ecb98e92f37c4ed17625c41 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 2 Oct 2023 13:22:55 +0100 Subject: [PATCH 116/124] gh-83180: Made launcher treat shebang 'python' tags as low priority so that active virtual environments are preferred (GH-108101) --- Doc/using/windows.rst | 20 +++++++--- Lib/test/test_launcher.py | 22 +++++++++++ ...3-08-18-00-01-21.gh-issue-83180.DdLffv.rst | 3 ++ PC/launcher2.c | 38 ++++++++++++++++--- 4 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-08-18-00-01-21.gh-issue-83180.DdLffv.rst diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2476e60a26d485..51afba9265d055 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -867,17 +867,18 @@ For example, if the first line of your script starts with #! /usr/bin/python -The default Python will be located and used. As many Python scripts written -to work on Unix will already have this line, you should find these scripts can -be used by the launcher without modification. If you are writing a new script -on Windows which you hope will be useful on Unix, you should use one of the -shebang lines starting with ``/usr``. +The default Python or an active virtual environment will be located and used. +As many Python scripts written to work on Unix will already have this line, +you should find these scripts can be used by the launcher without modification. +If you are writing a new script on Windows which you hope will be useful on +Unix, you should use one of the shebang lines starting with ``/usr``. Any of the above virtual commands can be suffixed with an explicit version (either just the major version, or the major and minor version). Furthermore the 32-bit version can be requested by adding "-32" after the minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the -32-bit python 3.7. +32-bit Python 3.7. If a virtual environment is active, the version will be +ignored and the environment will be used. .. versionadded:: 3.7 @@ -891,6 +892,13 @@ minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the not provably i386/32-bit". To request a specific environment, use the new :samp:`-V:{TAG}` argument with the complete tag. +.. versionchanged:: 3.13 + + Virtual commands referencing ``python`` now prefer an active virtual + environment rather than searching :envvar:`PATH`. This handles cases where + the shebang specifies ``/usr/bin/env python3`` but :file:`python3.exe` is + not present in the active environment. + The ``/usr/bin/env`` form of shebang line has one further special property. Before looking for installed Python interpreters, this form will search the executable :envvar:`PATH` for a Python executable matching the name provided diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 362b507d158288..bcd4ed63bf25a0 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -717,3 +717,25 @@ def test_literal_shebang_invalid_template(self): f"{expect} arg1 {script}", data["stdout"].strip(), ) + + def test_shebang_command_in_venv(self): + stem = "python-that-is-not-on-path" + + # First ensure that our test name doesn't exist, and the launcher does + # not match any installed env + with self.script(f'#! /usr/bin/env {stem} arg1') as script: + data = self.run_py([script], expect_returncode=103) + + with self.fake_venv() as (venv_exe, env): + # Put a real Python (ourselves) on PATH as a distraction. + # The active VIRTUAL_ENV should be preferred when the name isn't an + # exact match. + env["PATH"] = f"{Path(sys.executable).parent};{os.environ['PATH']}" + + with self.script(f'#! /usr/bin/env {stem} arg1') as script: + data = self.run_py([script], env=env) + self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}") + + with self.script(f'#! /usr/bin/env {Path(sys.executable).stem} arg1') as script: + data = self.run_py([script], env=env) + self.assertEqual(data["stdout"].strip(), f"{sys.executable} arg1 {script}") diff --git a/Misc/NEWS.d/next/Windows/2023-08-18-00-01-21.gh-issue-83180.DdLffv.rst b/Misc/NEWS.d/next/Windows/2023-08-18-00-01-21.gh-issue-83180.DdLffv.rst new file mode 100644 index 00000000000000..1e59765a7674b1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-08-18-00-01-21.gh-issue-83180.DdLffv.rst @@ -0,0 +1,3 @@ +Changes the :ref:`launcher` to prefer an active virtual environment when the +launched script has a shebang line using a Unix-like virtual command, even +if the command requests a specific version of Python. diff --git a/PC/launcher2.c b/PC/launcher2.c index bb500d4b6bfb07..116091f01227b8 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -195,6 +195,13 @@ join(wchar_t *buffer, size_t bufferLength, const wchar_t *fragment) } +bool +split_parent(wchar_t *buffer, size_t bufferLength) +{ + return SUCCEEDED(PathCchRemoveFileSpec(buffer, bufferLength)); +} + + int _compare(const wchar_t *x, int xLen, const wchar_t *y, int yLen) { @@ -414,8 +421,8 @@ typedef struct { // if true, treats 'tag' as a non-PEP 514 filter bool oldStyleTag; // if true, ignores 'tag' when a high priority environment is found - // gh-92817: This is currently set when a tag is read from configuration or - // the environment, rather than the command line or a shebang line, and the + // gh-92817: This is currently set when a tag is read from configuration, + // the environment, or a shebang, rather than the command line, and the // only currently possible high priority environment is an active virtual // environment bool lowPriorityTag; @@ -794,6 +801,8 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) } } + debug(L"# Search PATH for %s\n", filename); + wchar_t pathVariable[MAXLEN]; int n = GetEnvironmentVariableW(L"PATH", pathVariable, MAXLEN); if (!n) { @@ -1031,8 +1040,11 @@ checkShebang(SearchInfo *search) debug(L"Shebang: %s\n", shebang); // Handle shebangs that we should search PATH for + int executablePathWasSetByUsrBinEnv = 0; exitCode = searchPath(search, shebang, shebangLength); - if (exitCode != RC_NO_SHEBANG) { + if (exitCode == 0) { + executablePathWasSetByUsrBinEnv = 1; + } else if (exitCode != RC_NO_SHEBANG) { return exitCode; } @@ -1067,7 +1079,7 @@ checkShebang(SearchInfo *search) search->tagLength = commandLength; // If we had 'python3.12.exe' then we want to strip the suffix // off of the tag - if (search->tagLength > 4) { + if (search->tagLength >= 4) { const wchar_t *suffix = &search->tag[search->tagLength - 4]; if (0 == _comparePath(suffix, 4, L".exe", -1)) { search->tagLength -= 4; @@ -1075,13 +1087,14 @@ checkShebang(SearchInfo *search) } // If we had 'python3_d' then we want to strip the '_d' (any // '.exe' is already gone) - if (search->tagLength > 2) { + if (search->tagLength >= 2) { const wchar_t *suffix = &search->tag[search->tagLength - 2]; if (0 == _comparePath(suffix, 2, L"_d", -1)) { search->tagLength -= 2; } } search->oldStyleTag = true; + search->lowPriorityTag = true; search->executableArgs = &command[commandLength]; search->executableArgsLength = shebangLength - commandLength; if (search->tag && search->tagLength) { @@ -1095,6 +1108,11 @@ checkShebang(SearchInfo *search) } } + // Didn't match a template, but we found it on PATH + if (executablePathWasSetByUsrBinEnv) { + return 0; + } + // Unrecognised executables are first tried as command aliases commandLength = 0; while (commandLength < shebangLength && !isspace(shebang[commandLength])) { @@ -1765,7 +1783,15 @@ virtualenvSearch(const SearchInfo *search, EnvironmentInfo **result) return 0; } - if (INVALID_FILE_ATTRIBUTES == GetFileAttributesW(buffer)) { + DWORD attr = GetFileAttributesW(buffer); + if (INVALID_FILE_ATTRIBUTES == attr && search->lowPriorityTag) { + if (!split_parent(buffer, MAXLEN) || !join(buffer, MAXLEN, L"python.exe")) { + return 0; + } + attr = GetFileAttributesW(buffer); + } + + if (INVALID_FILE_ATTRIBUTES == attr) { debug(L"Python executable %s missing from virtual env\n", buffer); return 0; } From 8d92b6eff3bac45e7d4871c46c4511218b9b685a Mon Sep 17 00:00:00 2001 From: numbermaniac <5206120+numbermaniac@users.noreply.github.com> Date: Tue, 3 Oct 2023 00:13:44 +1100 Subject: [PATCH 117/124] 3.12 What's New: Remove duplicate "up to" (#110219) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3aa495d0537a69..3a2adc469f48c9 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -121,7 +121,7 @@ Significant improvements in the standard library: * A :ref:`command-line interface ` has been added to the :mod:`uuid` module * Due to the changes in :ref:`PEP 701 `, - producing tokens via the :mod:`tokenize` module is up to up to 64% faster. + producing tokens via the :mod:`tokenize` module is up to 64% faster. Security improvements: From 732ad44cec971be5255b1accbac6555d3615c2bf Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 2 Oct 2023 08:03:53 -0700 Subject: [PATCH 118/124] gh-110178: Use fewer weakrefs in test_typing.py (#110194) Confirmed that without the C changes from #108517, this test still segfaults with only 10 weakrefs. --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9e891f113840be..9559e35e9f02c1 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -550,7 +550,7 @@ def test_many_weakrefs(self): with self.subTest(cls=cls): vals = weakref.WeakValueDictionary() - for x in range(100000): + for x in range(10): vals[x] = cls(str(x)) del vals From 4d0d1c3866cc408ff3382a9a0220ac0e4f2b3b34 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Oct 2023 18:07:56 +0200 Subject: [PATCH 119/124] gh-110014: Remove PY_TIMEOUT_MAX from limited C API (#110217) If the timeout is greater than PY_TIMEOUT_MAX, PyThread_acquire_lock_timed() uses a timeout of PY_TIMEOUT_MAX microseconds, which is around 280.6 years. This case is unlikely and limiting a timeout to 280.6 years sounds like a reasonable trade-off. The constant PY_TIMEOUT_MAX is not used in PyPI top 5,000 projects. --- Doc/data/stable_abi.dat | 1 - Doc/whatsnew/3.13.rst | 3 +++ Include/cpython/pythread.h | 8 ++++++++ Include/pythread.h | 17 ++++------------- Lib/test/test_stable_abi_ctypes.py | 1 - ...23-10-02-13-39-57.gh-issue-110014.gfQ4jU.rst | 2 ++ Misc/stable_abi.toml | 4 ---- PC/python3dll.c | 1 - 8 files changed, 17 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-10-02-13-39-57.gh-issue-110014.gfQ4jU.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 07c6d514d19549..c189c78238f40f 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,5 +1,4 @@ role,name,added,ifdef_note,struct_abi_kind -var,PY_TIMEOUT_MAX,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, function,PyAIter_Check,3.10,, function,PyArg_Parse,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 1de5479a924375..3a1b283a75bf2e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1290,3 +1290,6 @@ removed, although there is currently no date scheduled for their removal. * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get`. * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete`. * :c:func:`PyThread_ReInitTLS`: no longer needed. + +* Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. + (Contributed by Victor Stinner in :gh:`110014`.) diff --git a/Include/cpython/pythread.h b/Include/cpython/pythread.h index cd2aab72d52df3..03f710a9f7ef2e 100644 --- a/Include/cpython/pythread.h +++ b/Include/cpython/pythread.h @@ -2,6 +2,14 @@ # error "this header file must not be included directly" #endif +// PY_TIMEOUT_MAX is the highest usable value (in microseconds) of PY_TIMEOUT_T +// type, and depends on the system threading API. +// +// NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread module +// exposes a higher-level API, with timeouts expressed in seconds and +// floating-point numbers allowed. +PyAPI_DATA(const long long) PY_TIMEOUT_MAX; + #define PYTHREAD_INVALID_THREAD_ID ((unsigned long)-1) #ifdef HAVE_PTHREAD_H diff --git a/Include/pythread.h b/Include/pythread.h index 2c2fd63d724286..0784f6b2e5391f 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -33,27 +33,18 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); #define WAIT_LOCK 1 #define NOWAIT_LOCK 0 -/* PY_TIMEOUT_T is the integral type used to specify timeouts when waiting - on a lock (see PyThread_acquire_lock_timed() below). - PY_TIMEOUT_MAX is the highest usable value (in microseconds) of that - type, and depends on the system threading API. - - NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread - module exposes a higher-level API, with timeouts expressed in seconds - and floating-point numbers allowed. -*/ +// PY_TIMEOUT_T is the integral type used to specify timeouts when waiting +// on a lock (see PyThread_acquire_lock_timed() below). #define PY_TIMEOUT_T long long -PyAPI_DATA(const long long) PY_TIMEOUT_MAX; - /* If microseconds == 0, the call is non-blocking: it returns immediately even when the lock can't be acquired. If microseconds > 0, the call waits up to the specified duration. If microseconds < 0, the call waits until success (or abnormal failure) - microseconds must be less than PY_TIMEOUT_MAX. Behaviour otherwise is - undefined. + If *microseconds* is greater than PY_TIMEOUT_MAX, clamp the timeout to + PY_TIMEOUT_MAX microseconds. If intr_flag is true and the acquire is interrupted by a signal, then the call will return PY_LOCK_INTR. The caller may reattempt to acquire the diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 6e9496d40da477..94f817f8e1d159 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -35,7 +35,6 @@ def test_windows_feature_macros(self): SYMBOL_NAMES = ( - "PY_TIMEOUT_MAX", "PyAIter_Check", "PyArg_Parse", "PyArg_ParseTuple", diff --git a/Misc/NEWS.d/next/C API/2023-10-02-13-39-57.gh-issue-110014.gfQ4jU.rst b/Misc/NEWS.d/next/C API/2023-10-02-13-39-57.gh-issue-110014.gfQ4jU.rst new file mode 100644 index 00000000000000..3a5ff7d43bbc01 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-10-02-13-39-57.gh-issue-110014.gfQ4jU.rst @@ -0,0 +1,2 @@ +Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. +Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 46e2307614e26d..8df3f85e61eec6 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1843,10 +1843,6 @@ [function.PyThread_start_new_thread] added = '3.2' -# Not mentioned in PEP 384, was implemented as a macro in Python <= 3.12 -[data.PY_TIMEOUT_MAX] - added = '3.2' - # The following were added in PC/python3.def in Python 3.3: # 7800f75827b1be557be16f3b18f5170fbf9fae08 # 9c56409d3353b8cd4cfc19e0467bbe23fd34fc92 diff --git a/PC/python3dll.c b/PC/python3dll.c index 75728c7d8057ed..2c1cc8098ce856 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -768,7 +768,6 @@ EXPORT_DATA(Py_FileSystemDefaultEncodeErrors) EXPORT_DATA(Py_FileSystemDefaultEncoding) EXPORT_DATA(Py_GenericAliasType) EXPORT_DATA(Py_HasFileSystemDefaultEncoding) -EXPORT_DATA(PY_TIMEOUT_MAX) EXPORT_DATA(Py_UTF8Mode) EXPORT_DATA(Py_Version) EXPORT_DATA(PyBaseObject_Type) From 6387b5313c60c1403785b2245db33372476ac304 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Oct 2023 18:53:38 +0200 Subject: [PATCH 120/124] gh-108494: Document how to add a project in PCbuild/readme.txt (#110077) Add _testclinic_limited to Tools/msi/test/test_files.wxs. --- PCbuild/readme.txt | 28 ++++++++++++++++++++++++++++ Tools/msi/test/test_files.wxs | 18 +++++++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 199aacdf7687ed..98b37014907604 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -293,3 +293,31 @@ project, with some projects overriding certain specific values. The GUI doesn't always reflect the correct settings and may confuse the user with false information, especially for settings that automatically adapt for different configurations. + +Add a new project +----------------- + +For example, add a new _testclinic_limited project to build a new +_testclinic_limited extension, the file Modules/_testclinic_limited.c: + +* In PCbuild/, copy _testclinic.vcxproj to _testclinic_limited.vcxproj, + replace RootNamespace value with `_testclinic_limited`, replace + `_asyncio.c` with `_testclinic_limited.c`. +* Open Visual Studio, open PCbuild\pcbuild.sln solution, add the + PCbuild\_testclinic_limited.vcxproj project to the solution ("add existing + project). +* Add a dependency on the python project to the new _testclinic_limited + project. +* Save and exit Visual Studio. +* Add `;_testclinic_limited` to `` in + PCbuild\pcbuild.proj. +* Update "exts" in Tools\msi\lib\lib_files.wxs file or in + Tools\msi\test\test_files.wxs file (for tests). +* PC\layout\main.py needs updating if you add a test-only extension whose name + doesn't start with "_test". +* Add the extension to PCbuild\readme.txt (this file). +* Build Python from scratch (clean the solution) to check that the new project + is built successfully. +* Ensure the new .vcxproj and .vcxproj.filters files are added to your commit, + as well as the changes to pcbuild.sln, pcbuild.proj and any other modified + files. diff --git a/Tools/msi/test/test_files.wxs b/Tools/msi/test/test_files.wxs index 87e164cb6759f6..bb9b258692a62f 100644 --- a/Tools/msi/test/test_files.wxs +++ b/Tools/msi/test/test_files.wxs @@ -1,41 +1,41 @@ - + - + - + - + - + - + - + - + - + From fc2cb86d210555d509debaeefd370d5331cd9d93 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Oct 2023 19:24:08 +0200 Subject: [PATCH 121/124] gh-107073: Make PyObject_VisitManagedDict() public (#108763) Make PyObject_VisitManagedDict() and PyObject_ClearManagedDict() functions public in Python 3.13 C API. * Rename _PyObject_VisitManagedDict() to PyObject_VisitManagedDict(). * Rename _PyObject_ClearManagedDict() to PyObject_ClearManagedDict(). * Document these functions. --- Doc/c-api/object.rst | 18 ++++++++++++ Doc/c-api/typeobj.rst | 28 ++++++++++++++++++- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.13.rst | 6 ++++ Include/cpython/object.h | 4 +-- ...-09-01-15-35-05.gh-issue-107073.zCz0iN.rst | 3 ++ Modules/_asynciomodule.c | 6 ++-- Modules/_testcapi/heaptype.c | 6 ++-- Modules/_testcapimodule.c | 2 +- Objects/dictobject.c | 4 +-- Objects/typeobject.c | 4 +-- Objects/typevarobject.c | 18 ++++++------ 12 files changed, 77 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-09-01-15-35-05.gh-issue-107073.zCz0iN.rst diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index bf55b5788efa47..a4e3e74861a315 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -489,3 +489,21 @@ Object Protocol :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set. .. versionadded:: 3.12 + +.. c:function:: int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) + + Visit the managed dictionary of *obj*. + + This function must only be called in a traverse function of the type which + has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. + + .. versionadded:: 3.13 + +.. c:function:: void PyObject_ClearManagedDict(PyObject *obj) + + Clear the managed dictionary of *obj*. + + This function must only be called in a traverse function of the type which + has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 1fa3f2a6f53735..10c05beda7c66f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1131,6 +1131,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + The type traverse function must call :c:func:`PyObject_VisitManagedDict` + and its clear function must call :c:func:`PyObject_ClearManagedDict`. + .. versionadded:: 3.12 **Inheritance:** @@ -1368,6 +1371,23 @@ and :c:data:`PyType_Type` effectively act as defaults.) debugging aid you may want to visit it anyway just so the :mod:`gc` module's :func:`~gc.get_referents` function will include it. + Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with:: + + Py_VISIT(Py_TYPE(self)); + + It is only needed since Python 3.9. To support Python 3.8 and older, this + line must be conditionnal:: + + #if PY_VERSION_HEX >= 0x03090000 + Py_VISIT(Py_TYPE(self)); + #endif + + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_VisitManagedDict` like this:: + + PyObject_VisitManagedDict((PyObject*)self, visit, arg); + .. warning:: When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members that the instance *owns* (by having :term:`strong references @@ -1451,6 +1471,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_ClearManagedDict` like this:: + + PyObject_ClearManagedDict((PyObject*)self); + Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called before an instance is deallocated. For example, when reference counting is enough to determine that an object is no longer used, the cyclic garbage @@ -1801,7 +1827,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) field is ``NULL`` then no :attr:`~object.__dict__` gets created for instances. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_dictoffset` will be set to ``-1``, to indicate that it is unsafe to use this field. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3a2adc469f48c9..6fe00bb9eb5df9 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -2123,7 +2123,7 @@ Porting to Python 3.12 The use of ``tp_dictoffset`` and ``tp_weaklistoffset`` is still supported, but does not fully support multiple inheritance (:gh:`95589`), and performance may be worse. - Classes declaring :c:macro:`Py_TPFLAGS_MANAGED_DICT` should call + Classes declaring :c:macro:`Py_TPFLAGS_MANAGED_DICT` must call :c:func:`!_PyObject_VisitManagedDict` and :c:func:`!_PyObject_ClearManagedDict` to traverse and clear their instance's dictionaries. To clear weakrefs, call :c:func:`PyObject_ClearWeakRefs`, as before. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3a1b283a75bf2e..1ef04fa7ae6adc 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -995,6 +995,12 @@ New Features references) now supports the :ref:`Limited API `. (Contributed by Victor Stinner in :gh:`108634`.) +* Add :c:func:`PyObject_VisitManagedDict` and + :c:func:`PyObject_ClearManagedDict` functions which must be called by the + traverse and clear functions of a type using + :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. + (Contributed by Victor Stinner in :gh:`107073`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index e5987191cfe08c..3838f19c75a230 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -444,8 +444,8 @@ PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); -PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); -PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); +PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 diff --git a/Misc/NEWS.d/next/C API/2023-09-01-15-35-05.gh-issue-107073.zCz0iN.rst b/Misc/NEWS.d/next/C API/2023-09-01-15-35-05.gh-issue-107073.zCz0iN.rst new file mode 100644 index 00000000000000..866809091aa5da --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-09-01-15-35-05.gh-issue-107073.zCz0iN.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyObject_VisitManagedDict` and :c:func:`PyObject_ClearManagedDict` +functions which must be called by the traverse and clear functions of a type +using :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. Patch by Victor Stinner. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c66a8623413f4b..e911286660b56e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -816,7 +816,7 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); - _PyObject_ClearManagedDict((PyObject *)fut); + PyObject_ClearManagedDict((PyObject *)fut); return 0; } @@ -834,7 +834,7 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); - _PyObject_VisitManagedDict((PyObject *)fut, visit, arg); + PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } @@ -2181,7 +2181,7 @@ TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); - _PyObject_VisitManagedDict((PyObject *)fut, visit, arg); + PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index d14a1763184207..4526583a8059d9 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -805,13 +805,13 @@ static int heapmanaged_traverse(HeapCTypeObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); - return _PyObject_VisitManagedDict((PyObject *)self, visit, arg); + return PyObject_VisitManagedDict((PyObject *)self, visit, arg); } static int heapmanaged_clear(HeapCTypeObject *self) { - _PyObject_ClearManagedDict((PyObject *)self); + PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -819,7 +819,7 @@ static void heapmanaged_dealloc(HeapCTypeObject *self) { PyTypeObject *tp = Py_TYPE(self); - _PyObject_ClearManagedDict((PyObject *)self); + PyObject_ClearManagedDict((PyObject *)self); PyObject_GC_UnTrack(self); PyObject_GC_Del(self); Py_DECREF(tp); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e09fd8806d2f64..64bcb49d365774 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2923,7 +2923,7 @@ settrace_to_error(PyObject *self, PyObject *list) static PyObject * clear_managed_dict(PyObject *self, PyObject *obj) { - _PyObject_ClearManagedDict(obj); + PyObject_ClearManagedDict(obj); Py_RETURN_NONE; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 1fb795f5097897..361f8e93064b25 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5649,7 +5649,7 @@ _PyObject_FreeInstanceAttributes(PyObject *self) } int -_PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { PyTypeObject *tp = Py_TYPE(obj); if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { @@ -5672,7 +5672,7 @@ _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) } void -_PyObject_ClearManagedDict(PyObject *obj) +PyObject_ClearManagedDict(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 893d8420bba4c4..3261a14a053dc8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1835,7 +1835,7 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg) assert(base->tp_dictoffset == 0); if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { assert(type->tp_dictoffset == -1); - int err = _PyObject_VisitManagedDict(self, visit, arg); + int err = PyObject_VisitManagedDict(self, visit, arg); if (err) { return err; } @@ -1905,7 +1905,7 @@ subtype_clear(PyObject *self) __dict__ slots (as in the case 'self.__dict__ is self'). */ if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - _PyObject_ClearManagedDict(self); + PyObject_ClearManagedDict(self); } } else if (type->tp_dictoffset != base->tp_dictoffset) { diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 0f04523b0032ed..73cdf48788efe1 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -200,7 +200,7 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); - _PyObject_ClearManagedDict(self); + PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); Py_TYPE(self)->tp_free(self); @@ -216,7 +216,7 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); - _PyObject_VisitManagedDict(self, visit, arg); + PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -227,7 +227,7 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); Py_CLEAR(self->evaluate_constraints); - _PyObject_ClearManagedDict((PyObject *)self); + PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -744,7 +744,7 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); - _PyObject_ClearManagedDict(self); + PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); Py_TYPE(self)->tp_free(self); @@ -757,7 +757,7 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); paramspecobject *ps = (paramspecobject *)self; Py_VISIT(ps->bound); - _PyObject_VisitManagedDict(self, visit, arg); + PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -765,7 +765,7 @@ static int paramspec_clear(paramspecobject *self) { Py_CLEAR(self->bound); - _PyObject_ClearManagedDict((PyObject *)self); + PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -1026,7 +1026,7 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = (typevartupleobject *)self; Py_DECREF(tvt->name); - _PyObject_ClearManagedDict(self); + PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); Py_TYPE(self)->tp_free(self); @@ -1165,14 +1165,14 @@ static int typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); - _PyObject_VisitManagedDict(self, visit, arg); + PyObject_VisitManagedDict(self, visit, arg); return 0; } static int typevartuple_clear(PyObject *self) { - _PyObject_ClearManagedDict(self); + PyObject_ClearManagedDict(self); return 0; } From a040a32ea2f13f16172394d3e3e3f80f47f25a68 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 13:59:05 -0600 Subject: [PATCH 122/124] gh-109853: Fix sys.path[0] For Subinterpreters (gh-109994) This change makes sure sys.path[0] is set properly for subinterpreters. Before, it wasn't getting set at all. This PR does not address the broader concerns from gh-109853. --- Include/cpython/initconfig.h | 3 + Lib/test/test_embed.py | 3 + Lib/test/test_interpreters.py | 151 ++++++++++++++++++ ...-09-27-18-01-06.gh-issue-109853.coQQiL.rst | 1 + Modules/main.c | 38 +++-- Python/initconfig.c | 3 + Python/pylifecycle.c | 25 +++ 7 files changed, 214 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index ee130467824daa..5d7b4e2d929e5b 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -204,6 +204,9 @@ typedef struct PyConfig { wchar_t *run_module; wchar_t *run_filename; + /* --- Set by Py_Main() -------------------------- */ + wchar_t *sys_path_0; + /* --- Private fields ---------------------------- */ // Install importlib? If equals to 0, importlib is not initialized at all. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 852b3578989cd8..dc476ef83c2519 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -505,6 +505,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'run_command': None, 'run_module': None, 'run_filename': None, + 'sys_path_0': None, '_install_importlib': 1, 'check_hash_pycs_mode': 'default', @@ -1132,6 +1133,7 @@ def test_init_run_main(self): 'program_name': './python3', 'run_command': code + '\n', 'parse_argv': 2, + 'sys_path_0': '', } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1147,6 +1149,7 @@ def test_init_main(self): 'run_command': code + '\n', 'parse_argv': 2, '_init_main': 0, + 'sys_path_0': '', } self.check_all_configs("test_init_main", config, api=API_PYTHON, diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 9c0dac7d6c61fb..9cd71e519036c3 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,5 +1,7 @@ import contextlib +import json import os +import os.path import sys import threading from textwrap import dedent @@ -9,6 +11,7 @@ from test import support from test.support import import_helper from test.support import threading_helper +from test.support import os_helper _interpreters = import_helper.import_module('_xxsubinterpreters') _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters @@ -488,6 +491,154 @@ def task(): pass +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.run(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + class FinalizationTests(TestBase): def test_gh_109793(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst new file mode 100644 index 00000000000000..45de3ba8877b01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst @@ -0,0 +1 @@ +``sys.path[0]`` is now set correctly for subinterpreters. diff --git a/Modules/main.c b/Modules/main.c index 05bedff050699f..8184bedca027a3 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -556,6 +556,11 @@ pymain_run_python(int *exitcode) goto error; } + // XXX Calculate config->sys_path_0 in getpath.py. + // The tricky part is that we can't check the path importers yet + // at that point. + assert(config->sys_path_0 == NULL); + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be @@ -571,24 +576,37 @@ pymain_run_python(int *exitcode) // import readline and rlcompleter before script dir is added to sys.path pymain_import_readline(config); + PyObject *path0 = NULL; if (main_importer_path != NULL) { - if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - goto error; - } + path0 = Py_NewRef(main_importer_path); } else if (!config->safe_path) { - PyObject *path0 = NULL; int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0); if (res < 0) { goto error; } - - if (res > 0) { - if (pymain_sys_path_add_path0(interp, path0) < 0) { - Py_DECREF(path0); - goto error; - } + else if (res == 0) { + Py_CLEAR(path0); + } + } + // XXX Apply config->sys_path_0 in init_interp_main(). We have + // to be sure to get readline/rlcompleter imported at the correct time. + if (path0 != NULL) { + wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL); + if (wstr == NULL) { Py_DECREF(path0); + goto error; + } + config->sys_path_0 = _PyMem_RawWcsdup(wstr); + PyMem_Free(wstr); + if (config->sys_path_0 == NULL) { + Py_DECREF(path0); + goto error; + } + int res = pymain_sys_path_add_path0(interp, path0); + Py_DECREF(path0); + if (res < 0) { + goto error; } } diff --git a/Python/initconfig.c b/Python/initconfig.c index 089ede4623e23d..6b76b4dc681b74 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -97,6 +97,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(pythonpath_env, WSTR_OPT), SPEC(home, WSTR_OPT), SPEC(platlibdir, WSTR), + SPEC(sys_path_0, WSTR_OPT), SPEC(module_search_paths_set, UINT), SPEC(module_search_paths, WSTR_LIST), SPEC(stdlib_dir, WSTR_OPT), @@ -770,6 +771,7 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->exec_prefix); CLEAR(config->base_exec_prefix); CLEAR(config->platlibdir); + CLEAR(config->sys_path_0); CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); @@ -3051,6 +3053,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" import site = %i\n", config->site_import); PySys_WriteStderr(" is in build tree = %i\n", config->_is_python_build); DUMP_CONFIG("stdlib dir", stdlib_dir); + DUMP_CONFIG("sys.path[0]", sys_path_0); #undef DUMP_CONFIG #define DUMP_SYS(NAME) \ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f3ed77e516237a..c0323763f44890 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1209,6 +1209,31 @@ init_interp_main(PyThreadState *tstate) } } + if (!is_main_interp) { + // The main interpreter is handled in Py_Main(), for now. + if (config->sys_path_0 != NULL) { + PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); + if (path0 == NULL) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sysdict = interp->sysdict; + if (sysdict == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); + if (sys_path == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + int res = PyList_Insert(sys_path, 0, path0); + Py_DECREF(path0); + if (res) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + } + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); From 1dd9dee45d2591b4e701039d1673282380696849 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 14:12:12 -0600 Subject: [PATCH 123/124] gh-105716: Support Background Threads in Subinterpreters Consistently (gh-109921) The existence of background threads running on a subinterpreter was preventing interpreters from getting properly destroyed, as well as impacting the ability to run the interpreter again. It also affected how we wait for non-daemon threads to finish. We add PyInterpreterState.threads.main, with some internal C-API functions. --- Include/cpython/pystate.h | 1 + Include/internal/pycore_interp.h | 2 + Include/internal/pycore_pystate.h | 5 + Lib/test/test_interpreters.py | 97 +++++++++++++++++++ Lib/test/test_threading.py | 49 ++++++++++ Lib/threading.py | 4 +- ...-09-26-14-00-25.gh-issue-105716.SUJkW1.rst | 3 + Modules/_threadmodule.c | 16 ++- Modules/_xxsubinterpretersmodule.c | 84 ++++++++-------- Modules/main.c | 4 + Python/pystate.c | 37 +++++++ 11 files changed, 257 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 5e184d0ca0944b..7e4c57efc7c00c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -211,6 +211,7 @@ struct _ts { * if it is NULL. */ PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); + // Disable tracing and profiling. PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0912bd175fe4f7..ebf02281a7a2a6 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -73,6 +73,8 @@ struct _is { uint64_t next_unique_id; /* The linked list of threads, newest first. */ PyThreadState *head; + /* The thread currently executing in the __main__ module, if any. */ + PyThreadState *main; /* Used in Modules/_threadmodule.c. */ long count; /* Support for runtime thread stack size tuning. diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 2e568f8aeeb152..6a36dba3708e38 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -44,6 +44,11 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) interp == &_PyRuntime._main_interpreter); } +// Export for _xxsubinterpreters module. +PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); + static inline const PyConfig * _Py_GetMainConfig(void) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 9cd71e519036c3..e62859a9c2b08e 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -261,6 +261,16 @@ def test_subinterpreter(self): self.assertTrue(interp.is_running()) self.assertFalse(interp.is_running()) + def test_finished(self): + r, w = os.pipe() + interp = interpreters.create() + interp.run(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" @@ -288,6 +298,31 @@ def test_bad_id(self): with self.assertRaises(ValueError): interp.is_running() + def test_with_only_background_threads(self): + r_interp, w_interp = os.pipe() + r_thread, w_thread = os.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.run('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + class TestInterpreterClose(TestBase): @@ -389,6 +424,37 @@ def test_still_running(self): interp.close() self.assertTrue(interp.is_running()) + def test_subthreads_still_running(self): + r_interp, w_interp = os.pipe() + r_thread, w_thread = os.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + class TestInterpreterRun(TestBase): @@ -465,6 +531,37 @@ def test_bytes_for_script(self): with self.assertRaises(TypeError): interp.run(b'print("spam")') + def test_with_background_threads_still_running(self): + r_interp, w_interp = os.pipe() + r_thread, w_thread = os.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.run(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.run('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 13bfacbac83f13..f8b81942cf1732 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -26,6 +26,11 @@ from test import lock_tests from test import support +try: + from test.support import interpreters +except ModuleNotFoundError: + interpreters = None + threading_helper.requires_working_threading(module=True) # Between fork() and exec(), only async-safe functions are allowed (issues @@ -52,6 +57,12 @@ def skip_unless_reliable_fork(test): return test +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, + 'subinterpreters required')(meth) + + def restore_default_excepthook(testcase): testcase.addCleanup(setattr, threading, 'excepthook', threading.excepthook) threading.excepthook = threading.__excepthook__ @@ -1311,6 +1322,44 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + @requires_subinterpreters + def test_threads_join_with_no_main(self): + r_interp, w_interp = self.pipe() + + INTERP = b'I' + FINI = b'F' + DONE = b'D' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + os.write({w_interp}, {FINI!r}) + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {DONE!r}) + t = threading.Thread(target=task) + t.start() + + os.write({w_interp}, {INTERP!r}) + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), INTERP) + self.assertEqual(os.read(r_interp, 1), FINI) + self.assertEqual(os.read(r_interp, 1), DONE) + @cpython_only def test_daemon_threads_fatal_error(self): subinterp_code = f"""if 1: diff --git a/Lib/threading.py b/Lib/threading.py index 31cefd2143a8c4..0edfaf880f711a 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -38,6 +38,7 @@ _allocate_lock = _thread.allocate_lock _set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident +_is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id _HAVE_THREAD_NATIVE_ID = True @@ -1574,7 +1575,7 @@ def _shutdown(): # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped: + if _main_thread._is_stopped and _is_main_interpreter(): # _shutdown() was already called return @@ -1627,6 +1628,7 @@ def main_thread(): In normal conditions, the main thread is the thread from which the Python interpreter was started. """ + # XXX Figure this out for subinterpreters. (See gh-75698.) return _main_thread # get thread-local implementation, either from the thread diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst new file mode 100644 index 00000000000000..b35550fa650dcc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst @@ -0,0 +1,3 @@ +Subinterpreters now correctly handle the case where they have threads +running in the background. Before, such threads would interfere with +cleaning up and destroying them, as well as prevent running another script. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index e77e30dfe5e821..ee46b37d92e128 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1605,6 +1605,18 @@ PyDoc_STRVAR(excepthook_doc, \n\ Handle uncaught Thread.run() exception."); +static PyObject * +thread__is_main_interpreter(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyBool_FromLong(_Py_IsMainInterpreter(interp)); +} + +PyDoc_STRVAR(thread__is_main_interpreter_doc, +"_is_main_interpreter()\n\ +\n\ +Return True if the current interpreter is the main Python interpreter."); + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, @@ -1634,8 +1646,10 @@ static PyMethodDef thread_methods[] = { METH_VARARGS, stack_size_doc}, {"_set_sentinel", thread__set_sentinel, METH_NOARGS, _set_sentinel_doc}, - {"_excepthook", thread_excepthook, + {"_excepthook", thread_excepthook, METH_O, excepthook_doc}, + {"_is_main_interpreter", thread__is_main_interpreter, + METH_NOARGS, thread__is_main_interpreter_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 1ddf64909bf18a..e1c7d4ab2fd78f 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" @@ -358,41 +359,14 @@ exceptions_init(PyObject *mod) } static int -_is_running(PyInterpreterState *interp) -{ - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - if (PyThreadState_Next(tstate) != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter has more than one thread"); - return -1; - } - - assert(!PyErr_Occurred()); - struct _PyInterpreterFrame *frame = tstate->current_frame; - if (frame == NULL) { - return 0; - } - return 1; -} - -static int -_ensure_not_running(PyInterpreterState *interp) +_run_script(PyInterpreterState *interp, const char *codestr, + _sharedns *shared, _sharedexception *sharedexc) { - int is_running = _is_running(interp); - if (is_running < 0) { + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + // We skip going through the shared exception. return -1; } - if (is_running) { - PyErr_Format(PyExc_RuntimeError, "interpreter already running"); - return -1; - } - return 0; -} -static int -_run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception *sharedexc) -{ PyObject *excval = NULL; PyObject *main_mod = PyUnstable_InterpreterState_GetMainModule(interp); if (main_mod == NULL) { @@ -422,6 +396,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, else { Py_DECREF(result); // We throw away the result. } + _PyInterpreterState_SetNotRunningMain(interp); *sharedexc = no_exception; return 0; @@ -437,6 +412,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, } Py_XDECREF(excval); assert(!PyErr_Occurred()); + _PyInterpreterState_SetNotRunningMain(interp); return -1; } @@ -444,9 +420,6 @@ static int _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, const char *codestr, PyObject *shareables) { - if (_ensure_not_running(interp) < 0) { - return -1; - } module_state *state = get_module_state(mod); _sharedns *shared = _get_shared_ns(shareables); @@ -457,8 +430,26 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Switch to interpreter. PyThreadState *save_tstate = NULL; if (interp != PyInterpreterState_Get()) { - // XXX Using the "head" thread isn't strictly correct. + // XXX gh-109860: Using the "head" thread isn't strictly correct. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + assert(tstate != NULL); + // Hack (until gh-109860): The interpreter's initial thread state + // is least likely to break. + while(tstate->next != NULL) { + tstate = tstate->next; + } + // We must do this check before switching interpreters, so any + // exception gets raised in the right one. + // XXX gh-109860: Drop this redundant check once we stop + // re-using tstates that might already be in use. + if (_PyInterpreterState_IsRunningMain(interp)) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter already running"); + if (shared != NULL) { + _sharedns_free(shared); + } + return -1; + } // XXX Possible GILState issues? save_tstate = PyThreadState_Swap(tstate); } @@ -478,8 +469,10 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, _sharedexception_apply(&exc, state->RunFailedError); } else if (result != 0) { - // We were unable to allocate a shared exception. - PyErr_NoMemory(); + if (!PyErr_Occurred()) { + // We were unable to allocate a shared exception. + PyErr_NoMemory(); + } } if (shared != NULL) { @@ -574,12 +567,20 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) // Ensure the interpreter isn't running. /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ - if (_ensure_not_running(interp) < 0) { + if (_PyInterpreterState_IsRunningMain(interp)) { + PyErr_Format(PyExc_RuntimeError, "interpreter running"); return NULL; } // Destroy the interpreter. + // XXX gh-109860: Using the "head" thread isn't strictly correct. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + assert(tstate != NULL); + // Hack (until gh-109860): The interpreter's initial thread state + // is least likely to break. + while(tstate->next != NULL) { + tstate = tstate->next; + } // XXX Possible GILState issues? PyThreadState *save_tstate = PyThreadState_Swap(tstate); Py_EndInterpreter(tstate); @@ -748,11 +749,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - int is_running = _is_running(interp); - if (is_running < 0) { - return NULL; - } - if (is_running) { + if (_PyInterpreterState_IsRunningMain(interp)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -763,6 +760,7 @@ PyDoc_STRVAR(is_running_doc, \n\ Return whether or not the identified interpreter is running."); + static PyMethodDef module_functions[] = { {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, diff --git a/Modules/main.c b/Modules/main.c index 8184bedca027a3..b5ee34d0141daf 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -612,6 +612,9 @@ pymain_run_python(int *exitcode) pymain_header(config); + _PyInterpreterState_SetRunningMain(interp); + assert(!PyErr_Occurred()); + if (config->run_command) { *exitcode = pymain_run_command(config->run_command); } @@ -635,6 +638,7 @@ pymain_run_python(int *exitcode) *exitcode = pymain_exit_err_print(); done: + _PyInterpreterState_SetNotRunningMain(interp); Py_XDECREF(main_importer_path); } diff --git a/Python/pystate.c b/Python/pystate.c index 01aa2552e56f0d..fe056bf4687026 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1091,6 +1091,39 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) #endif +int +_PyInterpreterState_SetRunningMain(PyInterpreterState *interp) +{ + if (interp->threads.main != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter already running"); + return -1; + } + PyThreadState *tstate = current_fast_get(&_PyRuntime); + _Py_EnsureTstateNotNULL(tstate); + if (tstate->interp != interp) { + PyErr_SetString(PyExc_RuntimeError, + "current tstate has wrong interpreter"); + return -1; + } + interp->threads.main = tstate; + return 0; +} + +void +_PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) +{ + assert(interp->threads.main == current_fast_get(&_PyRuntime)); + interp->threads.main = NULL; +} + +int +_PyInterpreterState_IsRunningMain(PyInterpreterState *interp) +{ + return (interp->threads.main != NULL); +} + + //---------- // accessors //---------- @@ -2801,6 +2834,10 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) } +/*************/ +/* Other API */ +/*************/ + _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) { From 014aacda6239f0e33b3ad5ece343df66701804b2 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 2 Oct 2023 21:13:48 +0100 Subject: [PATCH 124/124] Enable ruff on `Lib/test/test_typing.py` (#110179) --- .pre-commit-config.yaml | 2 +- Lib/test/.ruff.toml | 1 - Lib/test/test_typing.py | 104 ++++++++++++++++++++-------------------- 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c1fd20ea921b8..a5d32a04fc2d7d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.288 + rev: v0.0.292 hooks: - id: ruff name: Run Ruff on Lib/test/ diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index e202766b147e6d..b1e6424e785408 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -26,7 +26,6 @@ extend-exclude = [ "test_keywordonlyarg.py", "test_pkg.py", "test_subclassinit.py", - "test_typing.py", "test_unittest/testmock/testpatch.py", "test_yield_from.py", "time_hashlib.py", diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9559e35e9f02c1..c24cf3bc776fc1 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -185,7 +185,7 @@ def test_cannot_subclass(self): class A(self.bottom_type): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class A(type(self.bottom_type)): + class B(type(self.bottom_type)): pass def test_cannot_instantiate(self): @@ -282,7 +282,7 @@ class C(type(Self)): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Self'): - class C(Self): + class D(Self): pass def test_cannot_init(self): @@ -339,7 +339,7 @@ class C(type(LiteralString)): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.LiteralString'): - class C(LiteralString): + class D(LiteralString): pass def test_cannot_init(self): @@ -483,7 +483,7 @@ class V(TypeVar): pass T = TypeVar("T") with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_INSTANCE % 'TypeVar'): - class V(T): pass + class W(T): pass def test_cannot_instantiate_vars(self): with self.assertRaises(TypeError): @@ -1244,20 +1244,20 @@ class C(TypeVarTuple): pass Ts = TypeVarTuple('Ts') with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_INSTANCE % 'TypeVarTuple'): - class C(Ts): pass + class D(Ts): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(Unpack)): pass + class E(type(Unpack)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(*Ts)): pass + class F(type(*Ts)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(Unpack[Ts])): pass + class G(type(Unpack[Ts])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Unpack'): - class C(Unpack): pass + class H(Unpack): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): - class C(*Ts): pass + class I(*Ts): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): - class C(Unpack[Ts]): pass + class J(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): T = TypeVar('T') @@ -1431,12 +1431,12 @@ def test_variadic_class_with_duplicate_typevartuples_fails(self): with self.assertRaises(TypeError): class C(Generic[*Ts1, *Ts1]): pass with self.assertRaises(TypeError): - class C(Generic[Unpack[Ts1], Unpack[Ts1]]): pass + class D(Generic[Unpack[Ts1], Unpack[Ts1]]): pass with self.assertRaises(TypeError): - class C(Generic[*Ts1, *Ts2, *Ts1]): pass + class E(Generic[*Ts1, *Ts2, *Ts1]): pass with self.assertRaises(TypeError): - class C(Generic[Unpack[Ts1], Unpack[Ts2], Unpack[Ts1]]): pass + class F(Generic[Unpack[Ts1], Unpack[Ts2], Unpack[Ts1]]): pass def test_type_concatenation_in_variadic_class_argument_list_succeeds(self): Ts = TypeVarTuple('Ts') @@ -1804,11 +1804,11 @@ def test_cannot_subclass(self): class C(Union): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(Union)): + class D(type(Union)): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Union\[int, str\]'): - class C(Union[int, str]): + class E(Union[int, str]): pass def test_cannot_instantiate(self): @@ -2557,10 +2557,10 @@ class BP(Protocol): pass class P(C, Protocol): pass with self.assertRaises(TypeError): - class P(Protocol, C): + class Q(Protocol, C): pass with self.assertRaises(TypeError): - class P(BP, C, Protocol): + class R(BP, C, Protocol): pass class D(BP, C): pass @@ -2836,7 +2836,7 @@ class NotAProtocolButAnImplicitSubclass3: meth: Callable[[], None] meth2: Callable[[int, str], bool] def meth(self): pass - def meth(self, x, y): return True + def meth2(self, x, y): return True self.assertNotIsSubclass(AnnotatedButNotAProtocol, CallableMembersProto) self.assertIsSubclass(NotAProtocolButAnImplicitSubclass, CallableMembersProto) @@ -3658,11 +3658,11 @@ def test_protocols_bad_subscripts(self): with self.assertRaises(TypeError): class P(Protocol[T, T]): pass with self.assertRaises(TypeError): - class P(Protocol[int]): pass + class Q(Protocol[int]): pass with self.assertRaises(TypeError): - class P(Protocol[T], Protocol[S]): pass + class R(Protocol[T], Protocol[S]): pass with self.assertRaises(TypeError): - class P(typing.Mapping[T, S], Protocol[T]): pass + class S(typing.Mapping[T, S], Protocol[T]): pass def test_generic_protocols_repr(self): T = TypeVar('T') @@ -4094,12 +4094,12 @@ class NewGeneric(Generic): ... with self.assertRaises(TypeError): class MyGeneric(Generic[T], Generic[S]): ... with self.assertRaises(TypeError): - class MyGeneric(List[T], Generic[S]): ... + class MyGeneric2(List[T], Generic[S]): ... with self.assertRaises(TypeError): Generic[()] - class C(Generic[T]): pass + class D(Generic[T]): pass with self.assertRaises(TypeError): - C[()] + D[()] def test_generic_subclass_checks(self): for typ in [list[int], List[int], @@ -4836,7 +4836,7 @@ class Test(Generic[T], Final): class Subclass(Test): pass with self.assertRaises(FinalException): - class Subclass(Test[int]): + class Subclass2(Test[int]): pass def test_nested(self): @@ -5074,15 +5074,15 @@ def test_cannot_subclass(self): class C(type(ClassVar)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(ClassVar[int])): + class D(type(ClassVar[int])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.ClassVar'): - class C(ClassVar): + class E(ClassVar): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.ClassVar\[int\]'): - class C(ClassVar[int]): + class F(ClassVar[int]): pass def test_cannot_init(self): @@ -5124,15 +5124,15 @@ def test_cannot_subclass(self): class C(type(Final)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(Final[int])): + class D(type(Final[int])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Final'): - class C(Final): + class E(Final): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Final\[int\]'): - class C(Final[int]): + class F(Final[int]): pass def test_cannot_init(self): @@ -7265,15 +7265,15 @@ class A: class X(NamedTuple, A): x: int with self.assertRaises(TypeError): - class X(NamedTuple, tuple): + class Y(NamedTuple, tuple): x: int with self.assertRaises(TypeError): - class X(NamedTuple, NamedTuple): + class Z(NamedTuple, NamedTuple): x: int - class A(NamedTuple): + class B(NamedTuple): x: int with self.assertRaises(TypeError): - class X(NamedTuple, A): + class C(NamedTuple, B): y: str def test_generic(self): @@ -8037,15 +8037,15 @@ def test_cannot_subclass(self): class C(type(Required)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(Required[int])): + class D(type(Required[int])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Required'): - class C(Required): + class E(Required): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Required\[int\]'): - class C(Required[int]): + class F(Required[int]): pass def test_cannot_init(self): @@ -8085,15 +8085,15 @@ def test_cannot_subclass(self): class C(type(NotRequired)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(NotRequired[int])): + class D(type(NotRequired[int])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.NotRequired'): - class C(NotRequired): + class E(NotRequired): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.NotRequired\[int\]'): - class C(NotRequired[int]): + class F(NotRequired[int]): pass def test_cannot_init(self): @@ -8192,7 +8192,7 @@ class A(typing.Match): TypeError, r"type 're\.Pattern' is not an acceptable base type", ): - class A(typing.Pattern): + class B(typing.Pattern): pass @@ -8539,7 +8539,7 @@ class C(TypeAlias): pass with self.assertRaises(TypeError): - class C(type(TypeAlias)): + class D(type(TypeAlias)): pass def test_repr(self): @@ -8929,19 +8929,19 @@ def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpec'): class C(ParamSpec): pass with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpecArgs'): - class C(ParamSpecArgs): pass + class D(ParamSpecArgs): pass with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpecKwargs'): - class C(ParamSpecKwargs): pass + class E(ParamSpecKwargs): pass P = ParamSpec('P') with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_INSTANCE % 'ParamSpec'): - class C(P): pass + class F(P): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_INSTANCE % 'ParamSpecArgs'): - class C(P.args): pass + class G(P.args): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_INSTANCE % 'ParamSpecKwargs'): - class C(P.kwargs): pass + class H(P.kwargs): pass class ConcatenateTests(BaseTestCase): @@ -9022,15 +9022,15 @@ def test_cannot_subclass(self): class C(type(TypeGuard)): pass with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(TypeGuard[int])): + class D(type(TypeGuard[int])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.TypeGuard'): - class C(TypeGuard): + class E(TypeGuard): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.TypeGuard\[int\]'): - class C(TypeGuard[int]): + class F(TypeGuard[int]): pass def test_cannot_init(self):