From 92397d5ead38dde4154e70d00f24973bcf2a925a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 27 Mar 2024 09:04:32 -0500 Subject: [PATCH 1/4] Add statistics recipe for sampling from an estimated probability density distribution (#117221) --- Doc/library/statistics.rst | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index fc7e0c1ccad286..197c123f8356d8 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1148,6 +1148,64 @@ The final prediction goes to the largest posterior. This is known as the 'female' +Sampling from kernel density estimation +*************************************** + +The :func:`kde()` function creates a continuous probability density +function from discrete samples. Some applications need a way to make +random selections from that distribution. + +The technique is to pick a sample from a bandwidth scaled kernel +function and recenter the result around a randomly chosen point from +the input data. This can be done with any kernel that has a known or +accurately approximated inverse cumulative distribution function. + +.. testcode:: + + from random import choice, random, seed + from math import sqrt, log, pi, tan, asin + from statistics import NormalDist + + kernel_invcdfs = { + 'normal': NormalDist().inv_cdf, + 'logistic': lambda p: log(p / (1 - p)), + 'sigmoid': lambda p: log(tan(p * pi/2)), + 'rectangular': lambda p: 2*p - 1, + 'triangular': lambda p: sqrt(2*p) - 1 if p < 0.5 else 1 - sqrt(2 - 2*p), + 'cosine': lambda p: 2*asin(2*p - 1)/pi, + } + + def kde_random(data, h, kernel='normal'): + 'Return a function that samples from kde() smoothed data.' + kernel_invcdf = kernel_invcdfs[kernel] + def rand(): + return h * kernel_invcdf(random()) + choice(data) + return rand + +For example: + +.. doctest:: + + >>> discrete_samples = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(discrete_samples, h=1.5) + >>> seed(8675309) + >>> selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in selections] + [4.7, 7.4, 1.2, 7.8, 6.9, -1.3, 5.8, 0.2, -1.4, 5.7] + +.. testcode:: + :hide: + + from statistics import kde + from math import isclose + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + for kernel, invcdf in kernel_invcdfs.items(): + cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) + for x in xarr: + assert isclose(invcdf(cdf(x)), x, abs_tol=1E-9) + .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; From ce00de4c8cd39816f992e749c1074487d93abe9d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:46:35 +0200 Subject: [PATCH 2/4] gh-117225: doctest: only print "and X failed" when non-zero, don't pluralise "1 items" (#117228) --- Doc/library/doctest.rst | 10 ++-- Lib/doctest.py | 59 ++++++++++++------- Lib/test/test_doctest/test_doctest.py | 46 +++++++-------- ...-03-25-21-15-56.gh-issue-117225.oOaZXb.rst | 2 + 4 files changed, 69 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 835a3a76806148..135758187894ec 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -123,10 +123,10 @@ And so on, eventually ending with: OverflowError: n too large ok 2 items passed all tests: - 1 tests in __main__ - 8 tests in __main__.factorial - 9 tests in 2 items. - 9 passed and 0 failed. + 1 test in __main__ + 6 tests in __main__.factorial + 7 tests in 2 items. + 7 passed. Test passed. $ @@ -1933,7 +1933,7 @@ such a test runner:: optionflags=flags) else: fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) + print(f"{fail} failures out of {total} tests") .. rubric:: Footnotes diff --git a/Lib/doctest.py b/Lib/doctest.py index 6049423b5147a5..7a9f4e40d814d6 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1191,9 +1191,9 @@ class DocTestRunner: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) @@ -1568,49 +1568,59 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] + + notests, passed, failed = [], [], [] total_tries = total_failures = total_skips = 0 - for item in self._stats.items(): - name, (failures, tries, skips) = item + + for name, (failures, tries, skips) in self._stats.items(): assert failures <= tries total_tries += tries total_failures += failures total_skips += skips + if tries == 0: notests.append(name) elif failures == 0: passed.append((name, tries)) else: - failed.append(item) + failed.append((name, (failures, tries, skips))) + if verbose: if notests: - print(f"{len(notests)} items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() for name in notests: print(f" {name}") + if passed: - print(f"{len(passed)} items passed all tests:") - passed.sort() - for name, count in passed: - print(f" {count:3d} tests in {name}") + print(f"{_n_items(passed)} passed all tests:") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {count:3d} test{s} in {name}") + if failed: print(self.DIVIDER) - print(f"{len(failed)} items had failures:") - failed.sort() - for name, (failures, tries, skips) in failed: + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(f"{total_tries} tests in {len(self._stats)} items.") - print(f"{total_tries - total_failures} passed and {total_failures} failed.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = f" and {total_failures} failed" if total_failures else "" + print(f"{total_tries - total_failures} passed{and_f}.") + if total_failures: - msg = f"***Test Failed*** {total_failures} failures" + s = "" if total_failures == 1 else "s" + msg = f"***Test Failed*** {total_failures} failure{s}" if total_skips: - msg = f"{msg} and {total_skips} skipped tests" + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {total_skips} skipped test{s}" print(f"{msg}.") elif verbose: print("Test passed.") + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// @@ -1627,6 +1637,15 @@ def merge(self, other): d[name] = (failures, tries, skips) +def _n_items(items: list) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + + class OutputChecker: """ A class used to check the whether the actual output from a doctest diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 43be200b983227..3e883c56f6c766 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2628,9 +2628,9 @@ def test_testfile(): r""" ... NameError: name 'favorite_color' is not defined ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2657,9 +2657,9 @@ def test_testfile(): r""" Got: 'red' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2689,10 +2689,10 @@ def test_testfile(): r""" b ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. @@ -2749,7 +2749,7 @@ def test_testfile(): r""" ********************************************************************** ... ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 2 in test_doctest4.txt ***Test Failed*** 2 failures. TestResults(failed=2, attempted=2) @@ -2772,10 +2772,10 @@ def test_testfile(): r""" Expecting: 'b\u0105r' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest4.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. @@ -2997,10 +2997,10 @@ def test_CLI(): r""" Expecting: 'a' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile.doc - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. Now we'll write a couple files, one with three tests, the other a python module @@ -3074,7 +3074,7 @@ def test_CLI(): r""" Got: 'ajkml' ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 3 in myfile.doc ***Test Failed*** 2 failures. @@ -3101,9 +3101,9 @@ def test_CLI(): r""" Got: 'abcdef' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in myfile.doc - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. The fifth test uses verbose with the two options, so we should get verbose success output for the tests in both files: @@ -3126,10 +3126,10 @@ def test_CLI(): r""" Expecting: 'a...l' ok - 1 items passed all tests: + 1 item passed all tests: 3 tests in myfile.doc - 3 tests in 1 items. - 3 passed and 0 failed. + 3 tests in 1 item. + 3 passed. Test passed. Trying: 1 + 1 @@ -3141,12 +3141,12 @@ def test_CLI(): r""" Expecting: 'abc def' ok - 1 items had no tests: + 1 item had no tests: myfile2 - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile2.test_func 2 tests in 2 items. - 2 passed and 0 failed. + 2 passed. Test passed. We should also check some typical error cases. diff --git a/Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst b/Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst new file mode 100644 index 00000000000000..b6c4850f608c2a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst @@ -0,0 +1,2 @@ +doctest: only print "and X failed" when non-zero, don't pluralise "1 items". +Patch by Hugo van Kemenade. From 74c8568d07719529b874897598d8b3bc25ff0434 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 27 Mar 2024 16:53:27 +0000 Subject: [PATCH 3/4] gh-71042: Add `platform.android_ver` (#116674) --- Doc/library/platform.rst | 36 +++++++++++++ Doc/library/sys.rst | 4 +- Lib/platform.py | 46 +++++++++++++++++ Lib/test/pythoninfo.py | 3 ++ Lib/test/support/__init__.py | 16 +++--- Lib/test/test_asyncio/test_base_events.py | 5 ++ Lib/test/test_platform.py | 50 +++++++++++++++++++ Lib/test/test_socket.py | 18 ++++--- ...4-03-12-19-32-17.gh-issue-71042.oI0Ron.rst | 2 + 9 files changed, 164 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 4bc3956449b930..6af9168d15749f 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -301,3 +301,39 @@ Linux Platforms return ids .. versionadded:: 3.10 + + +Android Platform +---------------- + +.. function:: android_ver(release="", api_level=0, manufacturer="", \ + model="", device="", is_emulator=False) + + Get Android device information. Returns a :func:`~collections.namedtuple` + with the following attributes. Values which cannot be determined are set to + the defaults given as parameters. + + * ``release`` - Android version, as a string (e.g. ``"14"``). + + * ``api_level`` - API level of the running device, as an integer (e.g. ``34`` + for Android 14). To get the API level which Python was built against, see + :func:`sys.getandroidapilevel`. + + * ``manufacturer`` - `Manufacturer name + `__. + + * ``model`` - `Model name + `__ – + typically the marketing name or model number. + + * ``device`` - `Device name + `__ – + typically the model number or a codename. + + * ``is_emulator`` - ``True`` if the device is an emulator; ``False`` if it's + a physical device. + + Google maintains a `list of known model and device names + `__. + + .. versionadded:: 3.13 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 087a3454c33272..19d6856efe5d09 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -753,7 +753,9 @@ always available. .. function:: getandroidapilevel() - Return the build time API version of Android as an integer. + Return the build-time API level of Android as an integer. This represents the + minimum version of Android this build of Python can run on. For runtime + version information, see :func:`platform.android_ver`. .. availability:: Android. diff --git a/Lib/platform.py b/Lib/platform.py index 2756f298f9676f..df1d987036455f 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -542,6 +542,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -972,6 +1013,11 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 814358746d6d8a..5612c55746a516 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -179,6 +179,9 @@ def collect_platform(info_add): info_add(f'platform.freedesktop_os_release[{key}]', os_release[key]) + if sys.platform == 'android': + call_func(info_add, 'platform.android_ver', platform, 'android_ver') + def collect_locale(info_add): import locale diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a1c7987fa0db47..3d7868768231f5 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1801,18 +1801,18 @@ def missing_compiler_executable(cmd_names=[]): return cmd[0] -_is_android_emulator = None +_old_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python # to hang (issue #26939). - minimum_interval = 1e-5 + minimum_interval = 1e-4 # 100 us if is_android and interval < minimum_interval: - global _is_android_emulator - if _is_android_emulator is None: - import subprocess - _is_android_emulator = (subprocess.check_output( - ['getprop', 'ro.kernel.qemu']).strip() == b'1') - if _is_android_emulator: + global _old_android_emulator + if _old_android_emulator is None: + import platform + av = platform.android_ver() + _old_android_emulator = av.is_emulator and av.api_level < 24 + if _old_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 4cd872d3a5b2d8..c14a0bb180d79b 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -3,6 +3,7 @@ import concurrent.futures import errno import math +import platform import socket import sys import threading @@ -1430,6 +1431,10 @@ def test_create_connection_no_inet_pton(self, m_socket): self._test_create_connection_ip_addr(m_socket, False) @patch_socket + @unittest.skipIf( + support.is_android and platform.android_ver().api_level < 23, + "Issue gh-71123: this fails on Android before API level 23" + ) def test_create_connection_service_name(self, m_socket): m_socket.getaddrinfo = socket.getaddrinfo sock = m_socket.socket.return_value diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 9f8aeeea257311..57f27b247d9d15 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -219,6 +219,19 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + @unittest.skipUnless(sys.platform.startswith('win'), "windows only test") def test_uname_win32_without_wmi(self): def raises_oserror(*a): @@ -458,6 +471,43 @@ def test_libc_ver(self): self.assertEqual(platform.libc_ver(filename, chunksize=chunksize), ('glibc', '1.23.4')) + def test_android_ver(self): + res = platform.android_ver() + self.assertIsInstance(res, tuple) + self.assertEqual(res, (res.release, res.api_level, res.manufacturer, + res.model, res.device, res.is_emulator)) + + if sys.platform == "android": + for name in ["release", "manufacturer", "model", "device"]: + with self.subTest(name): + value = getattr(res, name) + self.assertIsInstance(value, str) + self.assertNotEqual(value, "") + + self.assertIsInstance(res.api_level, int) + self.assertGreaterEqual(res.api_level, sys.getandroidapilevel()) + + self.assertIsInstance(res.is_emulator, bool) + + # When not running on Android, it should return the default values. + else: + self.assertEqual(res.release, "") + self.assertEqual(res.api_level, 0) + self.assertEqual(res.manufacturer, "") + self.assertEqual(res.model, "") + self.assertEqual(res.device, "") + self.assertEqual(res.is_emulator, False) + + # Default values may also be overridden using parameters. + res = platform.android_ver( + "alpha", 1, "bravo", "charlie", "delta", True) + self.assertEqual(res.release, "alpha") + self.assertEqual(res.api_level, 1) + self.assertEqual(res.manufacturer, "bravo") + self.assertEqual(res.model, "charlie") + self.assertEqual(res.device, "delta") + self.assertEqual(res.is_emulator, True) + @support.cpython_only def test__comparable_version(self): from platform import _comparable_version as V diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index a7e657f5718524..661a859b0d0601 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -209,7 +209,10 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_VSOCK = _have_socket_vsock() -HAVE_SOCKET_UDPLITE = hasattr(socket, "IPPROTO_UDPLITE") +# Older Android versions block UDPLITE with SELinux. +HAVE_SOCKET_UDPLITE = ( + hasattr(socket, "IPPROTO_UDPLITE") + and not (support.is_android and platform.android_ver().api_level < 29)) HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() @@ -1217,8 +1220,8 @@ def testGetServBy(self): else: raise OSError # Try same call with optional protocol omitted - # Issue #26936: Android getservbyname() was broken before API 23. - if (not support.is_android) or sys.getandroidapilevel() >= 23: + # Issue gh-71123: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1229,8 +1232,9 @@ def testGetServBy(self): else: eq(udpport, port) # Now make sure the lookup by port returns the same service name - # Issue #26936: Android getservbyport() is broken. - if not support.is_android: + # Issue #26936: when the protocol is omitted, this fails on Android + # before API level 28. + if not (support.is_android and platform.android_ver().api_level < 28): eq(socket.getservbyport(port2), service) eq(socket.getservbyport(port, 'tcp'), service) if udpport is not None: @@ -1575,8 +1579,8 @@ def testGetaddrinfo(self): socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None - # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not support.is_android) or sys.getandroidapilevel() >= 23: + # Issue #26936: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) diff --git a/Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst b/Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst new file mode 100644 index 00000000000000..3641cbb9b2fc1a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst @@ -0,0 +1,2 @@ +Add :func:`platform.android_ver`, which provides device and OS information +on Android. From 262fb911ab7df8e890ebd0efb0773c3e0b5a757f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:38:19 +0000 Subject: [PATCH 4/4] gh-117288: Allocate fewer label IDs in _PyCfg_ToInstructionSequence (#117290) --- Include/internal/pycore_compile.h | 1 + Python/assemble.c | 3 +++ Python/compile.c | 34 +++++++++++++++++++++++++++---- Python/flowgraph.c | 5 +++-- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 0f446a00b4df22..f54f4f7f37acee 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -66,6 +66,7 @@ int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, int opcode, int oparg, _PyCompilerSrcLocation loc); +int _PyCompile_InstructionSequence_ApplyLabelMap(_PyCompile_InstructionSequence *seq); typedef struct { PyObject *u_name; diff --git a/Python/assemble.c b/Python/assemble.c index 569454ebf3b9cb..09db2fab48d95c 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -736,6 +736,9 @@ _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *umd, PyObject *const_cac int nlocalsplus, int code_flags, PyObject *filename) { + if (_PyCompile_InstructionSequence_ApplyLabelMap(instrs) < 0) { + return NULL; + } if (resolve_unconditional_jumps(instrs) < 0) { return NULL; } diff --git a/Python/compile.c b/Python/compile.c index e9507e47dac8fe..43b3cbd4e1894c 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -235,6 +235,28 @@ _PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl) return SUCCESS; } +int +_PyCompile_InstructionSequence_ApplyLabelMap(instr_sequence *instrs) +{ + /* Replace labels by offsets in the code */ + for (int i=0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg < instrs->s_labelmap_size); + instr->i_oparg = instrs->s_labelmap[instr->i_oparg]; + } + _PyCompile_ExceptHandlerInfo *hi = &instr->i_except_handler_info; + if (hi->h_label >= 0) { + assert(hi->h_label < instrs->s_labelmap_size); + hi->h_label = instrs->s_labelmap[hi->h_label]; + } + } + /* Clear label map so it's never used again */ + PyMem_Free(instrs->s_labelmap); + instrs->s_labelmap = NULL; + instrs->s_labelmap_size = 0; + return SUCCESS; +} #define MAX_OPCODE 511 @@ -7824,11 +7846,8 @@ instr_sequence_to_instructions(instr_sequence *seq) for (int i = 0; i < seq->s_used; i++) { instruction *instr = &seq->s_instrs[i]; location loc = instr->i_loc; - int arg = HAS_TARGET(instr->i_opcode) ? - seq->s_labelmap[instr->i_oparg] : instr->i_oparg; - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, arg, + "(iiiiii)", instr->i_opcode, instr->i_oparg, loc.lineno, loc.end_lineno, loc.col_offset, loc.end_col_offset); if (inst_tuple == NULL) { @@ -7855,6 +7874,9 @@ cfg_to_instructions(cfg_builder *g) if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { return NULL; } + if (_PyCompile_InstructionSequence_ApplyLabelMap(&seq) < 0) { + return NULL; + } PyObject *res = instr_sequence_to_instructions(&seq); instr_sequence_fini(&seq); return res; @@ -8026,6 +8048,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } + if (_PyCompile_InstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { + return NULL; + } + PyObject *insts = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); if (insts == NULL) { goto finally; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 2f47e47bf9d29d..5437c3875ff7b0 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2717,13 +2717,14 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; + lbl += 1; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (OPCODE_HAS_JUMP(instr->i_opcode) || is_block_push(instr)) { + if (HAS_TARGET(instr->i_opcode)) { + /* Set oparg to the label id (it will later be mapped to an offset) */ instr->i_oparg = instr->i_target->b_label.id; } RETURN_IF_ERROR(