Merge branch 'main' into pickle_copy

.github/workflows/build.yml

@@ -140,9 +140,6 @@ jobs:
         run: make regen-configure
       - name: Build CPython
         run: |
-          # Deepfreeze will usually cause global objects to be added or removed,
-          # so we run it before regen-global-objects gets rum (in regen-all).
-          make regen-deepfreeze
           make -j4 regen-all
           make regen-stdlib-module-names
       - name: Check for changes
@@ -182,7 +179,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
 
   build_win_amd64:
     name: 'Windows (x64)'
@@ -201,7 +198,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
 
   build_win_arm64:
     name: 'Windows (arm64)'
@@ -252,7 +249,7 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: make test
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -261,7 +258,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -319,7 +316,7 @@ jobs:
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
     - name: Tests
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make test
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
@@ -330,7 +327,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1v, 3.0.10, 3.1.2]
+        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -382,7 +379,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -491,7 +488,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
@@ -535,7 +532,7 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make test
 
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.288
+    rev: v0.0.292
     hooks:
       - id: ruff
         name: Run Ruff on Lib/test/

Doc/c-api/object.rst

@@ -489,3 +489,21 @@ Object Protocol
    :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set.
 
    .. versionadded:: 3.12
+
+.. c:function:: int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
+
+   Visit the managed dictionary of *obj*.
+
+   This function must only be called in a traverse function of the type which
+   has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
+
+   .. versionadded:: 3.13
+
+.. c:function:: void PyObject_ClearManagedDict(PyObject *obj)
+
+   Clear the managed dictionary of *obj*.
+
+   This function must only be called in a traverse function of the type which
+   has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
+
+   .. versionadded:: 3.13

Doc/c-api/typeobj.rst

@@ -1131,6 +1131,9 @@ and :c:data:`PyType_Type` effectively act as defaults.)
 
       If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set.
 
+      The type traverse function must call :c:func:`PyObject_VisitManagedDict`
+      and its clear function must call :c:func:`PyObject_ClearManagedDict`.
+
       .. versionadded:: 3.12
 
       **Inheritance:**
@@ -1368,6 +1371,23 @@ and :c:data:`PyType_Type` effectively act as defaults.)
    debugging aid you may want to visit it anyway just so the :mod:`gc` module's
    :func:`~gc.get_referents` function will include it.
 
+   Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with::
+
+       Py_VISIT(Py_TYPE(self));
+
+   It is only needed since Python 3.9. To support Python 3.8 and older, this
+   line must be conditionnal::
+
+       #if PY_VERSION_HEX >= 0x03090000
+           Py_VISIT(Py_TYPE(self));
+       #endif
+
+   If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
+   :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call
+   :c:func:`PyObject_VisitManagedDict` like this::
+
+       PyObject_VisitManagedDict((PyObject*)self, visit, arg);
+
    .. warning::
        When implementing :c:member:`~PyTypeObject.tp_traverse`, only the
        members that the instance *owns* (by having :term:`strong references
@@ -1451,6 +1471,12 @@ and :c:data:`PyType_Type` effectively act as defaults.)
    so that *self* knows the contained object can no longer be used.  The
    :c:func:`Py_CLEAR` macro performs the operations in a safe order.
 
+   If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
+   :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call
+   :c:func:`PyObject_ClearManagedDict` like this::
+
+       PyObject_ClearManagedDict((PyObject*)self);
+
    Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called
    before an instance is deallocated. For example, when reference counting
    is enough to determine that an object is no longer used, the cyclic garbage
@@ -1801,7 +1827,7 @@ and :c:data:`PyType_Type` effectively act as defaults.)
    field is ``NULL`` then no :attr:`~object.__dict__` gets created for instances.
 
    If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
-   :c:member:`~PyTypeObject.tp_dict` field, then
+   :c:member:`~PyTypeObject.tp_flags` field, then
    :c:member:`~PyTypeObject.tp_dictoffset` will be set to ``-1``, to indicate
    that it is unsafe to use this field.
 

Doc/constraints.txt

@@ -10,8 +10,7 @@ colorama<0.5
 imagesize<1.5
 Jinja2<3.2
 packaging<24
-# Pygments==2.15.0 breaks CI
-Pygments<2.16,!=2.15.0
+Pygments>=2.16.1,<3
 requests<3
 snowballstemmer<3
 sphinxcontrib-applehelp<1.1

Doc/howto/urllib2.rst

@@ -194,11 +194,11 @@ which comes after we have a look at what happens when things go wrong.
 Handling Exceptions
 ===================
 
-*urlopen* raises :exc:`URLError` when it cannot handle a response (though as
+*urlopen* raises :exc:`~urllib.error.URLError` when it cannot handle a response (though as
 usual with Python APIs, built-in exceptions such as :exc:`ValueError`,
 :exc:`TypeError` etc. may also be raised).
 
-:exc:`HTTPError` is the subclass of :exc:`URLError` raised in the specific case of
+:exc:`~urllib.error.HTTPError` is the subclass of :exc:`~urllib.error.URLError` raised in the specific case of
 HTTP URLs.
 
 The exception classes are exported from the :mod:`urllib.error` module.
@@ -229,12 +229,12 @@ the status code indicates that the server is unable to fulfil the request. The
 default handlers will handle some of these responses for you (for example, if
 the response is a "redirection" that requests the client fetch the document from
 a different URL, urllib will handle that for you). For those it can't handle,
-urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not
+urlopen will raise an :exc:`~urllib.error.HTTPError`. Typical errors include '404' (page not
 found), '403' (request forbidden), and '401' (authentication required).
 
 See section 10 of :rfc:`2616` for a reference on all the HTTP error codes.
 
-The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which
+The :exc:`~urllib.error.HTTPError` instance raised will have an integer 'code' attribute, which
 corresponds to the error sent by the server.
 
 Error Codes
@@ -317,7 +317,7 @@ dictionary is reproduced here for convenience ::
         }
 
 When an error is raised the server responds by returning an HTTP error code
-*and* an error page. You can use the :exc:`HTTPError` instance as a response on the
+*and* an error page. You can use the :exc:`~urllib.error.HTTPError` instance as a response on the
 page returned. This means that as well as the code attribute, it also has read,
 geturl, and info, methods as returned by the ``urllib.response`` module::
 
@@ -338,7 +338,7 @@ geturl, and info, methods as returned by the ``urllib.response`` module::
 Wrapping it Up
 --------------
 
-So if you want to be prepared for :exc:`HTTPError` *or* :exc:`URLError` there are two
+So if you want to be prepared for :exc:`~urllib.error.HTTPError` *or* :exc:`~urllib.error.URLError` there are two
 basic approaches. I prefer the second approach.
 
 Number 1
@@ -365,7 +365,7 @@ Number 1
 .. note::
 
     The ``except HTTPError`` *must* come first, otherwise ``except URLError``
-    will *also* catch an :exc:`HTTPError`.
+    will *also* catch an :exc:`~urllib.error.HTTPError`.
 
 Number 2
 ~~~~~~~~
@@ -391,7 +391,7 @@ Number 2
 info and geturl
 ===============
 
-The response returned by urlopen (or the :exc:`HTTPError` instance) has two
+The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two
 useful methods :meth:`info` and :meth:`geturl` and is defined in the module
 :mod:`urllib.response`..
 

Doc/library/__main__.rst

@@ -238,9 +238,9 @@ package.  For more details, see :ref:`intra-package-references` in the
 Idiomatic Usage
 ^^^^^^^^^^^^^^^
 
-The contents of ``__main__.py`` typically isn't fenced with
-``if __name__ == '__main__'`` blocks.  Instead, those files are kept short,
-functions to execute from other modules.  Those other modules can then be
+The content of ``__main__.py`` typically isn't fenced with an
+``if __name__ == '__main__'`` block.  Instead, those files are kept
+short and import functions to execute from other modules.  Those other modules can then be
 easily unit-tested and are properly reusable.
 
 If used, an ``if __name__ == '__main__'`` block will still work as expected

Doc/library/asyncio-stream.rst

@@ -157,8 +157,8 @@ and work with streams:
    .. versionchanged:: 3.10
       Removed the *loop* parameter.
 
-  .. versionchanged:: 3.11
-     Added the *ssl_shutdown_timeout* parameter.
+   .. versionchanged:: 3.11
+      Added the *ssl_shutdown_timeout* parameter.
 
 
 .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \

Doc/library/codecs.rst

@@ -1350,7 +1350,7 @@ encodings.
 +--------------------+---------+---------------------------+
 | raw_unicode_escape |         | Latin-1 encoding with     |
 |                    |         | :samp:`\\u{XXXX}` and     |
-|                    |         | :samp:`\\U{XXXXXXXX}``    |
+|                    |         | :samp:`\\U{XXXXXXXX}`     |
 |                    |         | for other code points.    |
 |                    |         | Existing                  |
 |                    |         | backslashes are not       |

Doc/library/collections.rst

@@ -358,7 +358,7 @@ Common patterns for working with :class:`Counter` objects::
     list(c)                         # list unique elements
     set(c)                          # convert to a set
     dict(c)                         # convert to a regular dictionary
-    c.items()                       # convert to a list of (elem, cnt) pairs
+    c.items()                       # access the (elem, cnt) pairs
     Counter(dict(list_of_pairs))    # convert from a list of (elem, cnt) pairs
     c.most_common()[:-n-1:-1]       # n least common elements
     +c                              # remove zero and negative counts

Doc/library/compileall.rst

@@ -90,7 +90,7 @@ compile Python sources.
 .. cmdoption:: -j N
 
    Use *N* workers to compile the files within the given directory.
-   If ``0`` is used, then the result of :func:`os.cpu_count()`
+   If ``0`` is used, then the result of :func:`os.process_cpu_count()`
    will be used.
 
 .. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash]

Doc/library/concurrent.futures.rst

@@ -188,6 +188,10 @@ And::
       ThreadPoolExecutor now reuses idle worker threads before starting
       *max_workers* worker threads too.
 
+   .. versionchanged:: 3.13
+      Default value of *max_workers* is changed to
+      ``min(32, (os.process_cpu_count() or 1) + 4)``.
+
 
 .. _threadpoolexecutor-example:
 
@@ -243,7 +247,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
 
    An :class:`Executor` subclass that executes calls asynchronously using a pool
    of at most *max_workers* processes.  If *max_workers* is ``None`` or not
-   given, it will default to the number of processors on the machine.
+   given, it will default to :func:`os.process_cpu_count`.
    If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError`
    will be raised.
    On Windows, *max_workers* must be less than or equal to ``61``. If it is not
@@ -301,6 +305,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
       different start method. See the :func:`os.fork` documentation for
       further explanation.
 
+   .. versionchanged:: 3.13
+      *max_workers* uses :func:`os.process_cpu_count` by default, instead of
+      :func:`os.cpu_count`.
+
 .. _processpoolexecutor-example:
 
 ProcessPoolExecutor Example

Doc/library/copy.rst

@@ -87,28 +87,40 @@ pickle functions from the :mod:`copyreg` module.
    single: __copy__() (copy protocol)
    single: __deepcopy__() (copy protocol)
 
+.. currentmodule:: None
+
 In order for a class to define its own copy implementation, it can define
-special methods :meth:`__copy__` and :meth:`__deepcopy__`.  The former is called
-to implement the shallow copy operation; no additional arguments are passed.
-The latter is called to implement the deep copy operation; it is passed one
-argument, the ``memo`` dictionary.  If the :meth:`__deepcopy__` implementation needs
-to make a deep copy of a component, it should call the :func:`deepcopy` function
-with the component as first argument and the memo dictionary as second argument.
-The memo dictionary should be treated as an opaque object.
+special methods :meth:`~object.__copy__` and :meth:`~object.__deepcopy__`.
+
+.. method:: object.__copy__(self)
+   :noindexentry:
+
+   Called to implement the shallow copy operation;
+   no additional arguments are passed.
+
+.. method:: object.__deepcopy__(self, memo)
+   :noindexentry:
+
+   Called to implement the deep copy operation; it is passed one
+   argument, the *memo* dictionary.  If the ``__deepcopy__`` implementation needs
+   to make a deep copy of a component, it should call the :func:`~copy.deepcopy` function
+   with the component as first argument and the *memo* dictionary as second argument.
+   The *memo* dictionary should be treated as an opaque object.
 
 
 .. index::
    single: __replace__() (replace protocol)
 
-Function :func:`replace` is more limited than :func:`copy` and :func:`deepcopy`,
+Function :func:`!copy.replace` is more limited
+than :func:`~copy.copy` and :func:`~copy.deepcopy`,
 and only supports named tuples created by :func:`~collections.namedtuple`,
-:mod:`dataclasses`, and other classes which define method :meth:`!__replace__`.
+:mod:`dataclasses`, and other classes which define method :meth:`~object.__replace__`.
 
-   .. method:: __replace__(self, /, **changes)
-      :noindex:
+.. method:: object.__replace__(self, /, **changes)
+   :noindexentry:
 
-:meth:`!__replace__` should create a new object of the same type,
-replacing fields with values from *changes*.
+   This method should create a new object of the same type,
+   replacing fields with values from *changes*.
 
 
 .. seealso::

Doc/library/devmode.rst

@@ -59,8 +59,9 @@ Effects of the Python Development Mode:
   ``default``.
 
 * Call :func:`faulthandler.enable` at Python startup to install handlers for
-  the :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and
-  :const:`SIGILL` signals to dump the Python traceback on a crash.
+  the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`,
+  :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` and
+  :const:`~signal.SIGILL` signals to dump the Python traceback on a crash.
 
   It behaves as if the :option:`-X faulthandler <-X>` command line option is
   used or if the :envvar:`PYTHONFAULTHANDLER` environment variable is set to